From 9e32c64834ea1d5d21f68b51455ac5921a3fdf24 Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 6 Nov 2026 11:47:19 +0800 Subject: [PATCH 01/14] Add AgiLink OmniGlove OpenXR hand tracking plugin Signed-off-by: zhouyingchun --- CMakeLists.txt | 1 + docs/source/_data/devices.yaml | 20 + docs/source/device/hand_tracking/index.rst | 1 + .../source/device/hand_tracking/omniglove.rst | 197 +++++++++ src/plugins/CMakeLists.txt | 3 + src/plugins/omniglove/.gitignore | 4 + src/plugins/omniglove/60-omniglove-usb.rules | 6 + src/plugins/omniglove/CMakeLists.txt | 110 +++++ src/plugins/omniglove/README.md | 108 +++++ src/plugins/omniglove/VALIDATION.md | 145 +++++++ src/plugins/omniglove/app/CMakeLists.txt | 15 + src/plugins/omniglove/app/main.cpp | 66 +++ .../omniglove/compat_probe/CMakeLists.txt | 13 + src/plugins/omniglove/compat_probe/main.cpp | 43 ++ src/plugins/omniglove/core/CMakeLists.txt | 36 ++ .../omniglove/omniglove_hand_conversion.hpp | 29 ++ .../core/inc/omniglove/omniglove_plugin.hpp | 107 +++++ .../core/omniglove_hand_conversion.cpp | 167 +++++++ .../omniglove/core/omniglove_plugin.cpp | 408 ++++++++++++++++++ src/plugins/omniglove/install.sh | 126 ++++++ src/plugins/omniglove/install_udev_rules.sh | 35 ++ src/plugins/omniglove/package_cts.py | 79 ++++ src/plugins/omniglove/plugin.yaml | 14 + src/plugins/omniglove/record_cts.sh | 89 ++++ src/plugins/omniglove/tests/CMakeLists.txt | 7 + .../tests/hand_conversion_contract_test.cpp | 181 ++++++++ 26 files changed, 2010 insertions(+) create mode 100644 docs/source/device/hand_tracking/omniglove.rst create mode 100644 src/plugins/omniglove/.gitignore create mode 100644 src/plugins/omniglove/60-omniglove-usb.rules create mode 100644 src/plugins/omniglove/CMakeLists.txt create mode 100644 src/plugins/omniglove/README.md create mode 100644 src/plugins/omniglove/VALIDATION.md create mode 100644 src/plugins/omniglove/app/CMakeLists.txt create mode 100644 src/plugins/omniglove/app/main.cpp create mode 100644 src/plugins/omniglove/compat_probe/CMakeLists.txt create mode 100644 src/plugins/omniglove/compat_probe/main.cpp create mode 100644 src/plugins/omniglove/core/CMakeLists.txt create mode 100644 src/plugins/omniglove/core/inc/omniglove/omniglove_hand_conversion.hpp create mode 100644 src/plugins/omniglove/core/inc/omniglove/omniglove_plugin.hpp create mode 100644 src/plugins/omniglove/core/omniglove_hand_conversion.cpp create mode 100644 src/plugins/omniglove/core/omniglove_plugin.cpp create mode 100755 src/plugins/omniglove/install.sh create mode 100755 src/plugins/omniglove/install_udev_rules.sh create mode 100755 src/plugins/omniglove/package_cts.py create mode 100644 src/plugins/omniglove/plugin.yaml create mode 100755 src/plugins/omniglove/record_cts.sh create mode 100644 src/plugins/omniglove/tests/CMakeLists.txt create mode 100644 src/plugins/omniglove/tests/hand_conversion_contract_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 521ec03c30..1dd21f022f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,7 @@ option(BUILD_PLUGIN_OAK_CAMERA "Build OAK camera plugin (requires vcpkg for Dept option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi SDK)" OFF) option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF) option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF) +option(BUILD_PLUGIN_OMNIGLOVE "Build AgiLink OmniGlove plugin (requires the Omniglove SDK)" OFF) option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF) option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) diff --git a/docs/source/_data/devices.yaml b/docs/source/_data/devices.yaml index 92facf6c24..a77dd5478d 100644 --- a/docs/source/_data/devices.yaml +++ b/docs/source/_data/devices.yaml @@ -273,6 +273,26 @@ devices: url: https://wuji.tech/en/ - email: sales@wuji.tech + - id: agilink-omniglove + name: AgiLink OmniGlove + url: https://github.com/AgibotTech/agilink_omniglove_sdk + group: peripheral + modes: High-fidelity hand tracking + since: main + details: + setup: + - label: OmniGlove docs + doc: /device/hand_tracking/omniglove + - label: OmniGlove Plugin + url: https://github.com/NVIDIA/IsaacCapture/tree/main/src/plugins/omniglove + requirements: + - Linux, x86_64 or aarch64 + - AgiLink OmniGlove SDK + - USB glove connection and host udev rule + acquire: + - label: AgiLink OmniGlove SDK + url: https://github.com/AgibotTech/agilink_omniglove_sdk + - id: noitom name: Noitom Robotics Motion Capture url: https://www.noitomrobotics.com diff --git a/docs/source/device/hand_tracking/index.rst b/docs/source/device/hand_tracking/index.rst index e792568c1c..dae40b38d2 100644 --- a/docs/source/device/hand_tracking/index.rst +++ b/docs/source/device/hand_tracking/index.rst @@ -12,6 +12,7 @@ and hand exoskeletons. See :doc:`/device/trackers/index` for the common tracking :titlesonly: manus + omniglove wuji_glove sharpa_avatar haptikos diff --git a/docs/source/device/hand_tracking/omniglove.rst b/docs/source/device/hand_tracking/omniglove.rst new file mode 100644 index 0000000000..175d4618eb --- /dev/null +++ b/docs/source/device/hand_tracking/omniglove.rst @@ -0,0 +1,197 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +.. SPDX-License-Identifier: Apache-2.0 + +AgiLink OmniGlove +================= + +OmniGlove is a USB glove that measures the operator's finger skeleton. Its +Linux C++ plugin wraps the AgiLink OmniGlove SDK and publishes standard OpenXR +hand tracking for Isaac Capture. Robot-specific joint mapping runs in the +host's hand retargeter. + +.. contents:: On this page + :local: + :depth: 2 + +Hardware and prerequisites +-------------------------- + +- Linux x86_64 or aarch64; the current hardware validation used x86_64. +- The Isaac Capture source-build dependencies and the + `AgiLink OmniGlove SDK `_. +- One left and one right OmniGlove for two-hand CTS, connected over USB. + USB identifiers are ``cafe:4001`` (left) and ``cafe:4002`` (right). +- Host USB permission, CloudXR and a supported XR headset. +- Optical wrist tracking, or controllers rigidly mounted to the gloves with + separately measured left/right controller-aim-to-wrist transforms. + +Data flow +--------- + +The SDK supplies ``OMNIGLOVE_STREAM_POST_DATA`` in DIRECT mode at a nominal +125 Hz: a wrist-relative skeleton with 21 landmarks. The plugin converts SDK +joint axes, combines the skeleton with a world-space wrist pose from +``plugin_utils::WristPoseSource``, and pushes each hand through its own +``plugin_utils::HandInjector``. The host reads the injected hands using the +standard ``HandsSource``. + +.. code-block:: text + + OmniGlove SDK (finger skeleton) ─┐ + ├─► OmniGlove plugin ─► OpenXR ─► HandsSource + Optical hand / controller pose ─┘ + +Installation +------------ + +Install the USB rule on the host, then build in the Isaac Capture checkout: + +.. code-block:: bash + + ./src/plugins/omniglove/install_udev_rules.sh + ./src/plugins/omniglove/install.sh + +The build script uses a sibling ``agilink_omniglove_sdk`` checkout when one is +available. Set ``OMNIGLOVE_SDK_ROOT`` or pass ``--sdk-root`` for a different +checkout. Otherwise it creates and updates a managed checkout. It validates +that SDK against its ``SHA256SUMS``, runs the API probe and conversion tests, +and installs the executable, manifest, runtime libraries and license notices +under ``install/plugins/omniglove``. + +For an existing Python 3.12 build used by CTS: + +.. code-block:: bash + + ./src/plugins/omniglove/install.sh --build-dir build-py312 + +Running +------- + +Start CloudXR in one terminal and connect the headset: + +.. code-block:: bash + + python -m isaaccapture.cloudxr.service run + +In another terminal, source the runtime environment, select the wrist source +as described below, and launch the installed plugin: + +.. code-block:: bash + + source ~/.cloudxr/run/cloudxr.env + ./install/plugins/omniglove/omniglove_plugin + +Wrist source and calibration +---------------------------- + +``OMNIGLOVE_WRIST_SOURCE`` selects ``hand_tracking`` (default), ``controller`` +or ``auto``. Optical mode needs the headset to track the wrist while the glove +is worn. Controller mode uses controller aim poses and per-hand installation +offsets. Auto mode allows both sources; validate each source and the transition +before using it. + +The successful hardware CTS on 2026-11-06 used **controller** mode. Optical and +Auto modes have not been qualified by that take. + +Rigidly attach each controller using its final mount. From simultaneous +controller aim and trusted wrist poses, calculate: + +.. code-block:: text + + T_aim_to_wrist = inverse(T_world_aim) * T_world_wrist + T_world_joint = T_world_aim * T_aim_to_wrist * T_wrist_joint + +Use multiple poses to estimate the fixed transform, then verify it with new +positions and wrist rotations. Keep the controller mounted rigidly; changes +to its installation require recalibration. Each transform is a translation +in metres followed by a unit quaternion in ``(x, y, z, w)`` order: + +.. code-block:: bash + + export OMNIGLOVE_WRIST_SOURCE=controller + export OMNIGLOVE_AIM_TO_WRIST_LEFT="px,py,pz,qx,qy,qz,qw" + export OMNIGLOVE_AIM_TO_WRIST_RIGHT="px,py,pz,qx,qy,qz,qw" + +These are format placeholders; replace them with measured values. Controller +and Auto mode reject missing transforms. The tested mount's offsets and +calibration evidence are in the :code-file:`validation record +`; those values apply only to that mount. + +Published collections and joint contract +---------------------------------------- + +``plugin.yaml`` declares ``/hand/left`` and ``/hand/right``, both with +``type: hand_tracker``. The plugin publishes ``XR_EXT_hand_tracking`` joints; +the host records them as ``core.HandPoseRecord`` from ``hand.fbs``. + +- Each frame has ``XrHandJointLocationEXT[26]`` slots in ``XrHandJointEXT`` + order. The wrist and four joints per finger are valid: 21 required joints. +- Palm and the four non-thumb metacarpals remain invalid because the SDK + does not supply them. No neighbouring joint is copied into an absent slot. +- Valid joints have both ``POSITION_VALID`` and ``ORIENTATION_VALID`` set. + Tracking bits additionally follow the validity of the world-space wrist. +- Positions are metres and orientations are unit ``(x, y, z, w)`` + quaternions, in the base space passed to ``HandInjector``. +- OpenXR joint ``+Z`` points away from the fingertip and ``+Y`` toward the + back of the hand. SDK finger orientations receive a local Y half-turn; + the right wrist basis is converted separately. Wrist and finger poses are + composed with the same world-space wrist transform. +- SDK monotonic sample times are converted into runtime ``XrTime`` through + ``XrTimeConverter``; the wrist is queried at that same time. +- Unmapped, unconverged IK, invalid bone data, non-finite poses, zero + quaternions and stale frames are withheld. A sample is pushed only once. + After 200 ms without fresh data the injector is destroyed, reporting the + hand inactive. A fresh valid sample and wrist pose recreate it. + +The plugin publishes the operator's skeleton. It emits no robot joint +commands or hand ``JointStateOutput`` data and defines no vendor hand schema. + +Validation and CTS +------------------ + +The 2026-11-06 controller-wrist take passed the hand CTS with 39 checks passing +and no failures. The optional-joint geometry check was unanswered because +those joints are not supplied. USB/SDK discovery, two-hand 125 Hz streaming, +five-minute streaming stability and SDK reconnect also have hardware evidence. +OpenXR hot-unplug/recovery and a recording from the pushed final plugin commit +remain separate acceptance steps. See the :code-file:`validation record +` for evidence and pending items. + +Follow :doc:`/conformance/setup` to prepare CTS. From a clean plugin commit +already pushed to the integration PR, load the mount configuration and record: + +.. code-block:: bash + + ./src/plugins/omniglove/record_cts.sh --wrist-config /path/to/mount.env + +Use ``--no-launch-cloudxr-runtime`` if CloudXR is already running. For +development takes, ``--allow-dirty`` retains the recorder's dirty provenance. +Include headset, controller, firmware and mount details in ``--note``. + +Review the take in the official panel, then package it using the panel's +Submission button or the wrapper that uses the same bundle builder: + +.. code-block:: bash + + src/python/isaaccapture/cts/hand/checker/.venv/bin/python -m hand_cts.panel /path/to/take-hand.mcap + src/python/isaaccapture/cts/hand/checker/.venv/bin/python src/plugins/omniglove/package_cts.py /path/to/take-hand.mcap + +Follow :doc:`/conformance/submit` to upload the archive and associate it with +the integration PR. Keep the archive outside the Git repository. + +Troubleshooting +--------------- + +- **No USB glove:** install the udev rule on the host, reconnect the device + and check user access to ``cafe:4001`` / ``cafe:4002``. +- **No world-space hand:** connect the headset, load ``cloudxr.env`` and + check that the selected wrist source is available. +- **Misplaced or rotated hand:** verify the final controller mount and its + aim-to-wrist transform, multiplication order, units and quaternion order. +- **Inactive hand:** the plugin logs ``hand inactive`` when SDK data or the + wrist becomes invalid. On recovery, ``hand active`` includes the SDK + connection generation and source sequence. Do not expect a frozen pose. +- **Persistent IK rejection:** check glove fit and SDK/firmware state; + invalid skeleton frames are intentionally withheld. +- **Header/runtime ABI mismatch:** rebuild against the selected SDK and + verify that the installed executable loads its accompanying runtime library. diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index c26d9384c2..737b8b4899 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -40,6 +40,9 @@ endif() if(BUILD_PLUGIN_WUJI_GLOVE) add_subdirectory(wuji_glove) endif() +if(BUILD_PLUGIN_OMNIGLOVE) + add_subdirectory(omniglove) +endif() # Gated: Linux + Avatar SDK under /opt/avatar-sdk. Missing SDK skips the plugin. if(BUILD_PLUGIN_SHARPA_AVATAR) add_subdirectory(sharpa_avatar) diff --git a/src/plugins/omniglove/.gitignore b/src/plugins/omniglove/.gitignore new file mode 100644 index 0000000000..043f278876 --- /dev/null +++ b/src/plugins/omniglove/.gitignore @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +/.sdk/ diff --git a/src/plugins/omniglove/60-omniglove-usb.rules b/src/plugins/omniglove/60-omniglove-usb.rules new file mode 100644 index 0000000000..47cbb1675d --- /dev/null +++ b/src/plugins/omniglove/60-omniglove-usb.rules @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +# Runtime interfaces only; bootloader interfaces intentionally stay restricted. +ACTION=="add|change", SUBSYSTEM=="usb", ENV{DEVTYPE}=="usb_device", ATTR{idVendor}=="cafe", ATTR{idProduct}=="4001", GROUP="plugdev", MODE="0660", TAG+="uaccess" +ACTION=="add|change", SUBSYSTEM=="usb", ENV{DEVTYPE}=="usb_device", ATTR{idVendor}=="cafe", ATTR{idProduct}=="4002", GROUP="plugdev", MODE="0660", TAG+="uaccess" diff --git a/src/plugins/omniglove/CMakeLists.txt b/src/plugins/omniglove/CMakeLists.txt new file mode 100644 index 0000000000..d3c178bc2c --- /dev/null +++ b/src/plugins/omniglove/CMakeLists.txt @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +include(GNUInstallDirs) + +find_package(OmnigloveSDK CONFIG REQUIRED) + +if(NOT TARGET OmnigloveSDK::api) + message(FATAL_ERROR "OmnigloveSDK package did not define OmnigloveSDK::api") +endif() + +get_target_property(OMNIGLOVE_SDK_LIBRARY OmnigloveSDK::api IMPORTED_LOCATION) +if(NOT OMNIGLOVE_SDK_LIBRARY OR NOT EXISTS "${OMNIGLOVE_SDK_LIBRARY}") + message(FATAL_ERROR "OmnigloveSDK::api has no usable IMPORTED_LOCATION") +endif() + +if(NOT OMNIGLOVE_SDK_RELEASE_ROOT) + get_filename_component(_omniglove_platform_root "${OmnigloveSDK_DIR}/.." ABSOLUTE) + get_filename_component(_omniglove_release_candidate "${_omniglove_platform_root}/.." ABSOLUTE) + if(EXISTS "${_omniglove_release_candidate}/LICENSE") + set(OMNIGLOVE_SDK_RELEASE_ROOT "${_omniglove_release_candidate}") + endif() +endif() + +if(OMNIGLOVE_SDK_RELEASE_ROOT AND + (NOT OMNIGLOVE_SDK_GIT_COMMIT OR OMNIGLOVE_SDK_GIT_COMMIT STREQUAL "unknown")) + find_package(Git QUIET) + if(GIT_FOUND) + # A release archive inside another repository must not inherit its commit. + execute_process( + COMMAND "${GIT_EXECUTABLE}" -C "${OMNIGLOVE_SDK_RELEASE_ROOT}" rev-parse --show-toplevel + OUTPUT_VARIABLE _omniglove_git_root + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + file(REAL_PATH "${OMNIGLOVE_SDK_RELEASE_ROOT}" _omniglove_release_root) + if(_omniglove_git_root STREQUAL _omniglove_release_root) + execute_process( + COMMAND "${GIT_EXECUTABLE}" -C "${OMNIGLOVE_SDK_RELEASE_ROOT}" rev-parse HEAD + RESULT_VARIABLE _omniglove_git_result + OUTPUT_VARIABLE _omniglove_git_commit + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + if(_omniglove_git_result EQUAL 0) + set(OMNIGLOVE_SDK_GIT_COMMIT "${_omniglove_git_commit}") + endif() + endif() + endif() +endif() + +add_subdirectory(core) +add_subdirectory(app) +add_subdirectory(compat_probe) + +if(BUILD_TESTING) + add_subdirectory(tests) +endif() + +get_filename_component(_omniglove_sdk_library_dir "${OMNIGLOVE_SDK_LIBRARY}" DIRECTORY) +file(GLOB _omniglove_sdk_libraries "${_omniglove_sdk_library_dir}/libomniglove.so*") +file(GLOB _omniglove_libusb_libraries "${_omniglove_sdk_library_dir}/omniglove/libusb-1.0.so*") + +if(NOT _omniglove_sdk_libraries) + message(FATAL_ERROR "No libomniglove.so files found beside ${OMNIGLOVE_SDK_LIBRARY}") +endif() +if(NOT _omniglove_libusb_libraries) + message(FATAL_ERROR "No bundled libusb runtime found under ${_omniglove_sdk_library_dir}/omniglove") +endif() + +install(FILES ${_omniglove_sdk_libraries} + DESTINATION plugins/omniglove + COMPONENT omniglove +) +install(FILES ${_omniglove_libusb_libraries} + DESTINATION plugins/omniglove/omniglove + COMPONENT omniglove +) + +if(OMNIGLOVE_SDK_RELEASE_ROOT) + install(FILES + "${OMNIGLOVE_SDK_RELEASE_ROOT}/LICENSE" + DESTINATION plugins/omniglove + RENAME LICENSE.omniglove-sdk + COMPONENT omniglove + ) + install(FILES + "${OMNIGLOVE_SDK_RELEASE_ROOT}/THIRD_PARTY_NOTICES.md" + DESTINATION plugins/omniglove + RENAME THIRD_PARTY_NOTICES.omniglove-sdk.md + COMPONENT omniglove + ) + install(FILES + "${OMNIGLOVE_SDK_RELEASE_ROOT}/licenses/libusb-COPYING" + DESTINATION plugins/omniglove + RENAME LICENSE.libusb + COMPONENT omniglove + ) +else() + message(WARNING "OMNIGLOVE_SDK_RELEASE_ROOT is unset; SDK license files will not be installed") +endif() + +install(FILES plugin.yaml README.md VALIDATION.md 60-omniglove-usb.rules + DESTINATION plugins/omniglove + COMPONENT omniglove +) +install(PROGRAMS install_udev_rules.sh + DESTINATION plugins/omniglove + COMPONENT omniglove +) diff --git a/src/plugins/omniglove/README.md b/src/plugins/omniglove/README.md new file mode 100644 index 0000000000..713415c6bd --- /dev/null +++ b/src/plugins/omniglove/README.md @@ -0,0 +1,108 @@ + + +# AgiLink OmniGlove plugin + +This plugin converts the OmniGlove SDK's wrist-relative 21-landmark +`OMNIGLOVE_STREAM_POST_DATA` stream to OpenXR hand joints and injects it into +the Isaac Capture runtime. It supports one left and one right USB glove. + +## Install + +Install the host USB rule once, outside a container: + +```bash +./src/plugins/omniglove/install_udev_rules.sh +``` + +Then build the plugin. The installer uses a sibling +`../agilink_omniglove_sdk` checkout when present. Otherwise it clones the +current SDK from GitHub into `.sdk/`, or fast-forwards that managed checkout on +later runs. + +```bash +./src/plugins/omniglove/install.sh +``` + +Set `OMNIGLOVE_SDK_ROOT` or pass `--sdk-root` to use another SDK checkout. The +installer verifies that checkout with its own `SHA256SUMS`; it does not reset +the SDK to a historical tag. The SDK package configuration selects its shared +library, so a changed ABI version number alone is not a build blocker. Missing +or changed required C APIs fail the compile/link compatibility probe. At +runtime, a header/library ABI mismatch is rejected before device discovery. + +## Run + +Start CloudXR, load its environment, connect a calibrated glove, and run: + +```bash +python -m isaaccapture.cloudxr.service run +source ~/.cloudxr/run/cloudxr.env +./install/plugins/omniglove/omniglove_plugin +``` + +The glove provides finger poses relative to its wrist but no world-space wrist +pose. By default the plugin obtains wrist pose from OpenXR optical hand +tracking. Set `OMNIGLOVE_WRIST_SOURCE` to `controller` or `auto` only after +measuring controller-aim-to-wrist transforms and exporting them as +`px,py,pz,qx,qy,qz,qw`: + +```bash +export OMNIGLOVE_WRIST_SOURCE=controller +export OMNIGLOVE_AIM_TO_WRIST_LEFT="px,py,pz,qx,qy,qz,qw" +export OMNIGLOVE_AIM_TO_WRIST_RIGHT="px,py,pz,qx,qy,qz,qw" +``` + +Replace these format placeholders with measured offsets. Controller and Auto +mode reject missing transforms. For the tested mount's actual configuration +and calibration evidence, see [VALIDATION.md](VALIDATION.md). + +## Data contract + +- SDK positions are metres and quaternions are `(x, y, z, w)`. +- Finger orientations are converted from SDK distal +Z to OpenXR distal + -Z by a local Y half-turn, preserving dorsal +Y. The wrist retains its + proximal +Z; the right-hand wrist basis is converted separately. +- The 21 supplied wrist/finger landmarks map to their OpenXR equivalents. +- OpenXR palm and the index, middle, ring, and little metacarpals remain + invalid because the SDK stream does not supply them. +- Unmapped, unconverged, invalid-bone-model, non-finite, zero-quaternion, and + stale frames are not injected. + +## Validation status + +The API probe and hand-conversion contract test run during installation. +The controller-wrist hardware take `omniglove_2026-11-06_110404` passed hand CTS +(39 pass, zero fail). It was recorded before the plugin was committed, so it +is development evidence; formal submission needs a take from the pushed final +commit. OpenXR hot-unplug/recovery remains a separate hardware acceptance item. +See [VALIDATION.md](VALIDATION.md) for the evidence and remaining steps. + +## Formal recording and packaging + +After committing and pushing the plugin, record from a clean checkout: + +```bash +./src/plugins/omniglove/record_cts.sh --wrist-config /path/to/mount.env +``` + +The wrapper rejects uncommitted/unpushed code and changes to the official CTS. +It records the installed binary's SHA-256 and the wrist configuration in +provenance notes. `--allow-dirty` enables development takes with their original +dirty provenance. `--note` adds hardware/firmware details; use +`--no-launch-cloudxr-runtime` when CloudXR is already running. + +Review the take with the official viewer and package it with the same builder: + +```bash +src/python/isaaccapture/cts/hand/checker/.venv/bin/python -m hand_cts.panel /path/to/take-hand.mcap +src/python/isaaccapture/cts/hand/checker/.venv/bin/python src/plugins/omniglove/package_cts.py /path/to/take-hand.mcap +``` + +Packaging reruns all official checks, preserves the original MCAP, labels, +provenance and log, checks their hashes, and prints the archive hash. Existing +archives are not overwritten. Upload the archive through the +[CTS form](https://forms.gle/hLqAhNkae4bhAUX79) with the integration PR URL; +keep recordings and archives outside Git. diff --git a/src/plugins/omniglove/VALIDATION.md b/src/plugins/omniglove/VALIDATION.md new file mode 100644 index 0000000000..3c5a4beee7 --- /dev/null +++ b/src/plugins/omniglove/VALIDATION.md @@ -0,0 +1,145 @@ + + +# OmniGlove integration validation + +Updated: 2026-11-06 (Asia/Shanghai). Hardware qualification and formal upstream +submission have separate evidence. A pending item below is not a pass. + +## Recorded hardware baseline + +| Item | Evidence | +| --- | --- | +| Isaac Capture baseline | `71f22fd93c0bdff871e731d66b1d10589b2e3aeb` | +| Plugin | `omniglove_plugin`, manifest version `1.0.0` | +| SDK release / commit | `1.0.0` / `a2f899cd45c51ae01c9bbca9b893e4afba41c0d7` | +| SDK header / runtime ABI | Both `6.0.0` | +| Platform tested | Linux x86_64; aarch64 supported by the build, not hardware-tested here | +| USB devices | Left `cafe:4001`, right `cafe:4002` | +| SDK stream | POST_DATA, DIRECT, nominal 125 Hz | +| Wrist source qualified | `controller` | +| Headset / controller model | Not recorded in the take; operator confirmation pending | +| Glove hardware / firmware revision | Not recorded in the take; operator confirmation pending | +| Physical controller mount | Final mount description/photo pending operator confirmation | + +## Hardware CTS evidence + +- Take: `omniglove_2026-11-06_110404`, recorded at 11:04:04 UTC+08:00. +- Verdict: **PASS**; 18,678 hand records, 39 passing checks, zero failures. +- `optional_joints.geometry_consistent`: insufficient data, expected because + the five optional joints are invalid. The overall verdict is pass. +- All 11 action windows were backed by frames. Left/right each had 9,339 + records, including 9,337 active records. +- Captured commit: the baseline above, with `repo_dirty=true`. The plugin was + staged but not committed; this is development evidence. Do not edit its + provenance to claim a later commit. +- Raw MCAP, label sidecar, provenance and recorder log stay together outside + Git. The submission archive uses the official CTS bundle builder and adds + `report.json`, `report.txt`, per-input hashes and checker provenance. +- Manual replay review and final-commit recording remain pending. The report + establishes automated checks; it does not establish the operator's replay review. + +## Tested wrist installation offsets + +The independent successful take used these exact fixed transforms. Values +are metres and quaternion `qx,qy,qz,qw` in controller aim coordinates. +They apply to the tested mount, not every OmniGlove installation. + +```bash +export OMNIGLOVE_WRIST_SOURCE=controller +export OMNIGLOVE_AIM_TO_WRIST_LEFT="0.044351775,-0.022869715,0.071703646,-0.122489204,-0.165170245,0.646336332,0.734822789" +export OMNIGLOVE_AIM_TO_WRIST_RIGHT="-0.044435690,-0.021819216,0.082681299,-0.130124469,0.161913518,-0.661683797,0.720434721" +``` + +The final left offset was estimated from the physical CTS action constraints +in `omniglove_2026-11-06_104642`: table level and fingertip contact/roll, using +held-out time blocks and excluding the final flat-hand window from fitting. +The original take's offline reprojection passed 36 checks with zero failures; +the new independent take above then passed 39 checks. Only a fixed wrist +installation transform was adjusted. Original recordings, labels, finger +geometry and checker thresholds were not changed. This was geometric mount +calibration; it was not a simultaneous optical-wrist measurement. + +For a new mount, obtain a trusted wrist reference and compute +`T_aim_to_wrist = inverse(T_world_aim) * T_world_wrist` over multiple poses. +Verify the result using independent positions, wrist rotations and physical +contact, then record a fresh CTS take. + +## Other existing evidence + +The 2026-11-01 handover records: + +- Left and right USB identification and individually valid 21-joint conversion. +- Five-minute dual-hand SDK streaming: 37,501 frames per hand, about 125 Hz; + zero sequence gaps, source errors, malformed frames and callback drops. +- SDK hot-unplug/reconnect, tested with the right glove representing the same + hardware/firmware pair: automatic recovery and connection generation 1→3. + +These historical results establish the USB/SDK layer. They do not establish +OpenXR injector deactivation/recreation or long-term headset tracking. + +## OpenXR hot-unplug/recovery procedure — pending + +Keep the final wrist configuration and both gloves connected, connect the +headset, and start a session consuming the two standard hand collections. +Capture the plugin log and consumer recording as a separate hardware test. +Do not use unplug segments as CTS action labels. + +1. Confirm both hands are active and their fingers are changing. +2. Unplug the left glove and wait at least two seconds. Record the physical + unplug time and the last SDK sample time. +3. Check the left injector deactivates after 200 ms without a fresh sample + (allow one worker scheduling interval), that the consumer reports the left + hand inactive, and that the right hand continues changing. +4. Reconnect the left glove without restarting any process. Check that the + left hand becomes active, its connection generation increases, and fingers + update from new samples. +5. Repeat for the right glove and check the left remains active. +6. Save timestamps, the consumer's active flags and the plugin log. A log + transition alone does not establish what the downstream consumer saw. + +The plugin now logs `left/right hand inactive: SDK frame stale or invalid`, +`hand inactive: wrist pose unavailable`, and `hand active` with connection +generation/source sequence. Logs occur on injector transitions, not every +worker iteration. + +## Software verification on 2026-11-06 + +- Installer: selected local SDK checksum verification, build and installation passed. +- CTest: `omniglove_api_compat_probe` and `omniglove_hand_conversion_contract_test` passed. +- All plugin C++ sources/headers passed clang-format 14. +- Strict Sphinx `make current-docs` passed with no warnings. +- Full pre-commit passed with `SKIP=check-copyright-year`, matching CI. +- Controller/Auto with missing mount offsets failed before opening USB/XR. +- Recorder smoke checks verified clean/upstream guards, unchanged CTS, + exported offsets, binary hash and preservation of operator notes. +- Existing take packaging passed archive integrity and per-input hash checks. + +## Final submission checklist + +- [ ] Confirm headset, controller, glove hardware/firmware and final mount. +- [ ] OpenXR hot-unplug/recovery evidence for both hands. +- [x] Software build, conversion/ABI tests, C++ format, strict docs and full + pre-commit verified against the final source. +- [ ] Commit the plugin with the operator's Git identity and DCO sign-off. +- [ ] Push to the user-specified repository and open the integration PR. +- [ ] Record from that pushed commit with a clean checkout and unchanged CTS: + +```bash +./src/plugins/omniglove/record_cts.sh --wrist-config /path/to/mount.env \ + --no-launch-cloudxr-runtime --note "headset/model, controller/model, firmware, mount" +``` + +- [ ] Review the new take in `hand_cts.panel`; verify handedness, axes, + continuity and the physical wrist/finger alignment. +- [ ] Package with the viewer or `package_cts.py`; quote take, verdict, archive + size and SHA-256 prefix in the PR. +- [ ] Upload via https://forms.gle/hLqAhNkae4bhAUX79, associating the archive + with the PR. Keep the archive outside Git. +- [ ] Resolve upstream CI and reviewer feedback. + +Optical and Auto wrist modes remain unqualified by the controller take. +Repeat hardware tests after changing the SDK, firmware, skeleton conversion +or physical mount. diff --git a/src/plugins/omniglove/app/CMakeLists.txt b/src/plugins/omniglove/app/CMakeLists.txt new file mode 100644 index 0000000000..409aeea089 --- /dev/null +++ b/src/plugins/omniglove/app/CMakeLists.txt @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(omniglove_plugin main.cpp) + +target_link_libraries(omniglove_plugin PRIVATE isaac::omniglove_plugin) + +set_target_properties(omniglove_plugin PROPERTIES + INSTALL_RPATH "$ORIGIN;$ORIGIN/omniglove;$ORIGIN/../../${CMAKE_INSTALL_LIBDIR}" +) + +install(TARGETS omniglove_plugin + RUNTIME DESTINATION plugins/omniglove + COMPONENT omniglove +) diff --git a/src/plugins/omniglove/app/main.cpp b/src/plugins/omniglove/app/main.cpp new file mode 100644 index 0000000000..fc34826b4b --- /dev/null +++ b/src/plugins/omniglove/app/main.cpp @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + +static_assert(ATOMIC_BOOL_LOCK_FREE == 2, "lock-free atomic bool is required for signal safety"); + +std::atomic stop_requested{ false }; + +void signal_handler(int signal) +{ + if (signal == SIGINT || signal == SIGTERM) + { + stop_requested.store(true, std::memory_order_relaxed); + } +} + +} // namespace + +int main(int argc, char** argv) +try +{ + std::string plugin_root_id = "omniglove"; + for (int index = 1; index < argc; ++index) + { + const std::string argument = argv[index]; + constexpr const char* prefix = "--plugin-root-id="; + if (argument.rfind(prefix, 0) == 0) + { + plugin_root_id = argument.substr(std::char_traits::length(prefix)); + } + } + + std::signal(SIGINT, signal_handler); + std::signal(SIGTERM, signal_handler); + + auto logger = isaaccapture::Logger::get("isaaccapture.plugins.omniglove.main"); + auto plugin = std::make_unique(plugin_root_id); + while (!stop_requested.load(std::memory_order_relaxed) && plugin->is_running()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return plugin->has_failed() ? 1 : 0; +} +catch (const std::exception& error) +{ + isaaccapture::Logger::get("isaaccapture.plugins.omniglove.main")->error("{}: {}", argv[0], error.what()); + return 1; +} +catch (...) +{ + isaaccapture::Logger::get("isaaccapture.plugins.omniglove.main")->error("{}: unknown error", argv[0]); + return 1; +} diff --git a/src/plugins/omniglove/compat_probe/CMakeLists.txt b/src/plugins/omniglove/compat_probe/CMakeLists.txt new file mode 100644 index 0000000000..152e1d9731 --- /dev/null +++ b/src/plugins/omniglove/compat_probe/CMakeLists.txt @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(omniglove_api_compat_probe main.cpp) +target_link_libraries(omniglove_api_compat_probe PRIVATE OmnigloveSDK::api) + +set_target_properties(omniglove_api_compat_probe PROPERTIES + BUILD_RPATH "$;$/omniglove" +) + +if(BUILD_TESTING) + add_test(NAME omniglove_api_compat_probe COMMAND omniglove_api_compat_probe) +endif() diff --git a/src/plugins/omniglove/compat_probe/main.cpp b/src/plugins/omniglove/compat_probe/main.cpp new file mode 100644 index 0000000000..25d3480a05 --- /dev/null +++ b/src/plugins/omniglove/compat_probe/main.cpp @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include + +namespace +{ + +void callback(const omniglove_stream_sample_t*, void*) +{ +} + +} // namespace + +int main() +{ + omniglove_stream_config_t config{}; + config.struct_size = sizeof(config); + config.kind = OMNIGLOVE_STREAM_POST_DATA; + config.stream_mode = OMNIGLOVE_STREAM_MODE_DEFAULT; + config.output_mode = OMNIGLOVE_STREAM_OUTPUT_DIRECT; + config.source_interval_ms = 8; + config.source_timeout_ms = 200; + config.callback = callback; + config.callback_queue_capacity = 64; + + omniglove_post_data_t post{}; + post.thumb.landmarks.cmc.position_m[0] = 0.0F; + post.index.landmarks.mcp.quaternion_xyzw[3] = 1.0F; + post.wrist.landmark.quaternion_xyzw[3] = 1.0F; + + static_cast(config); + static_cast(post); + + const uint32_t runtime = omniglove_abi_version(); + std::cout << "Omniglove header ABI " << OMNIGLOVE_ABI_VERSION_MAJOR << '.' << OMNIGLOVE_ABI_VERSION_MINOR << '.' + << OMNIGLOVE_ABI_VERSION_PATCH << ", runtime ABI " << ((runtime >> 16U) & 0xffU) << '.' + << ((runtime >> 8U) & 0xffU) << '.' << (runtime & 0xffU) << '\n'; + return runtime == OMNIGLOVE_ABI_VERSION ? 0 : 1; +} diff --git a/src/plugins/omniglove/core/CMakeLists.txt b/src/plugins/omniglove/core/CMakeLists.txt new file mode 100644 index 0000000000..abb6e327d6 --- /dev/null +++ b/src/plugins/omniglove/core/CMakeLists.txt @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +add_library(omniglove_plugin_core STATIC + omniglove_hand_conversion.cpp + omniglove_plugin.cpp +) + +target_include_directories(omniglove_plugin_core + INTERFACE + $ +) + +target_link_libraries(omniglove_plugin_core + PUBLIC + OmnigloveSDK::api + Teleop::plugin_utils + deviceio::deviceio_session + deviceio::deviceio_trackers + oxr::oxr_core + oxr::oxr_utils + OpenXR::openxr_loader + Teleop::openxr_extensions + log_bridge::log_bridge_core +) + +if(NOT OMNIGLOVE_SDK_GIT_COMMIT) + set(OMNIGLOVE_SDK_GIT_COMMIT "unknown") +endif() +target_compile_definitions(omniglove_plugin_core + PRIVATE + OMNIGLOVE_SDK_RELEASE_VERSION="${OmnigloveSDK_VERSION}" + OMNIGLOVE_SDK_GIT_COMMIT="${OMNIGLOVE_SDK_GIT_COMMIT}" +) + +add_library(isaac::omniglove_plugin ALIAS omniglove_plugin_core) diff --git a/src/plugins/omniglove/core/inc/omniglove/omniglove_hand_conversion.hpp b/src/plugins/omniglove/core/inc/omniglove/omniglove_hand_conversion.hpp new file mode 100644 index 0000000000..2594e60789 --- /dev/null +++ b/src/plugins/omniglove/core/inc/omniglove/omniglove_hand_conversion.hpp @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#include + +namespace plugins::omniglove +{ + +enum class ConversionStatus +{ + Success, + NotMapped, + IkNotConverged, + BoneModelInvalid, + InvalidPose, +}; + +ConversionStatus convert_post_data(const omniglove_post_data_t& post, + bool is_left, + std::array& joints); + +const char* conversion_status_name(ConversionStatus status) noexcept; + +} // namespace plugins::omniglove diff --git a/src/plugins/omniglove/core/inc/omniglove/omniglove_plugin.hpp b/src/plugins/omniglove/core/inc/omniglove/omniglove_plugin.hpp new file mode 100644 index 0000000000..fdacedf67f --- /dev/null +++ b/src/plugins/omniglove/core/inc/omniglove/omniglove_plugin.hpp @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "omniglove_hand_conversion.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins::omniglove +{ + +class OmniglovePlugin +{ +public: + explicit OmniglovePlugin(const std::string& plugin_root_id); + ~OmniglovePlugin(); + + bool is_running() const noexcept; + bool has_failed() const noexcept; + + OmniglovePlugin(const OmniglovePlugin&) = delete; + OmniglovePlugin& operator=(const OmniglovePlugin&) = delete; + OmniglovePlugin(OmniglovePlugin&&) = delete; + OmniglovePlugin& operator=(OmniglovePlugin&&) = delete; + +private: + struct HandFrame + { + std::array joints{}; + uint64_t sample_monotonic_ns = 0; + uint64_t connection_generation = 0; + uint64_t source_sequence = 0; + bool valid = false; + }; + + struct StreamContext + { + OmniglovePlugin* plugin = nullptr; + bool is_left = false; + }; + + struct DeviceStream + { + omniglove_device_t* device = nullptr; + StreamContext context; + bool started = false; + }; + + struct SessionDeleter + { + void operator()(omniglove_session_t* session) const noexcept; + }; + + static void stream_callback(const omniglove_stream_sample_t* sample, void* user_data); + + void initialize_sdk(); + void start_device_streams(); + void on_sample(const omniglove_stream_sample_t& sample, bool is_left); + void invalidate_hand(bool is_left); + void worker_thread(); + void pump_hand(std::unique_ptr& injector, + XrHandEXT hand, + const HandFrame& frame, + uint64_t& last_pushed_ns); + + std::shared_ptr m_logger = + isaaccapture::Logger::get("isaaccapture.plugins.omniglove.OmniglovePlugin"); + + std::shared_ptr m_session; + std::unique_ptr m_deviceio_session; + std::unique_ptr m_wrist_source; + std::unique_ptr m_left_injector; + std::unique_ptr m_right_injector; + std::unique_ptr m_time_converter; + + // The SDK session is declared after callback contexts so it stops and + // joins callback workers before those contexts are destroyed. + std::vector> m_streams; + std::unique_ptr m_sdk_session; + + std::mutex m_frame_mutex; + HandFrame m_left; + HandFrame m_right; + uint64_t m_left_last_pushed_ns = 0; + uint64_t m_right_last_pushed_ns = 0; + + std::thread m_worker; + std::atomic m_running{ false }; + std::atomic m_failed{ false }; +}; + +} // namespace plugins::omniglove diff --git a/src/plugins/omniglove/core/omniglove_hand_conversion.cpp b/src/plugins/omniglove/core/omniglove_hand_conversion.cpp new file mode 100644 index 0000000000..0906aaf8fc --- /dev/null +++ b/src/plugins/omniglove/core/omniglove_hand_conversion.cpp @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/omniglove/omniglove_hand_conversion.hpp" + +#include +#include +#include + +namespace plugins::omniglove +{ + +namespace +{ + +constexpr uint32_t kPostIkConverged = 1U << 0U; +constexpr uint32_t kPostBoneModelValid = 1U << 2U; +constexpr float kJointRadiusM = 0.01F; +constexpr XrSpaceLocationFlags kValidFlags = + XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; + +struct JointSource +{ + XrHandJointEXT joint; + const omniglove_pose6_t* pose; +}; + +std::array joint_sources(const omniglove_post_data_t& post) +{ + return { JointSource{ XR_HAND_JOINT_WRIST_EXT, &post.wrist.landmark }, + JointSource{ XR_HAND_JOINT_THUMB_METACARPAL_EXT, &post.thumb.landmarks.cmc }, + JointSource{ XR_HAND_JOINT_THUMB_PROXIMAL_EXT, &post.thumb.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_THUMB_DISTAL_EXT, &post.thumb.landmarks.ip }, + JointSource{ XR_HAND_JOINT_THUMB_TIP_EXT, &post.thumb.landmarks.tip }, + JointSource{ XR_HAND_JOINT_INDEX_PROXIMAL_EXT, &post.index.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_INDEX_INTERMEDIATE_EXT, &post.index.landmarks.pip }, + JointSource{ XR_HAND_JOINT_INDEX_DISTAL_EXT, &post.index.landmarks.dip }, + JointSource{ XR_HAND_JOINT_INDEX_TIP_EXT, &post.index.landmarks.tip }, + JointSource{ XR_HAND_JOINT_MIDDLE_PROXIMAL_EXT, &post.middle.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_MIDDLE_INTERMEDIATE_EXT, &post.middle.landmarks.pip }, + JointSource{ XR_HAND_JOINT_MIDDLE_DISTAL_EXT, &post.middle.landmarks.dip }, + JointSource{ XR_HAND_JOINT_MIDDLE_TIP_EXT, &post.middle.landmarks.tip }, + JointSource{ XR_HAND_JOINT_RING_PROXIMAL_EXT, &post.ring.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_RING_INTERMEDIATE_EXT, &post.ring.landmarks.pip }, + JointSource{ XR_HAND_JOINT_RING_DISTAL_EXT, &post.ring.landmarks.dip }, + JointSource{ XR_HAND_JOINT_RING_TIP_EXT, &post.ring.landmarks.tip }, + JointSource{ XR_HAND_JOINT_LITTLE_PROXIMAL_EXT, &post.pinky.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_LITTLE_INTERMEDIATE_EXT, &post.pinky.landmarks.pip }, + JointSource{ XR_HAND_JOINT_LITTLE_DISTAL_EXT, &post.pinky.landmarks.dip }, + JointSource{ XR_HAND_JOINT_LITTLE_TIP_EXT, &post.pinky.landmarks.tip } }; +} + +bool finite(float value) +{ + return std::isfinite(value); +} + +bool convert_pose(const omniglove_pose6_t& source, bool is_left, XrPosef& destination) +{ + for (float value : source.position_m) + { + if (!finite(value)) + { + return false; + } + } + for (float value : source.quaternion_xyzw) + { + if (!finite(value)) + { + return false; + } + } + + const float norm = std::sqrt( + source.quaternion_xyzw[0] * source.quaternion_xyzw[0] + source.quaternion_xyzw[1] * source.quaternion_xyzw[1] + + source.quaternion_xyzw[2] * source.quaternion_xyzw[2] + source.quaternion_xyzw[3] * source.quaternion_xyzw[3]); + if (norm < 1.0e-6F) + { + return false; + } + + destination.position = { source.position_m[0], source.position_m[1], source.position_m[2] }; + destination.orientation = { source.quaternion_xyzw[0] / norm, source.quaternion_xyzw[1] / norm, + source.quaternion_xyzw[2] / norm, source.quaternion_xyzw[3] / norm }; + + // OmniGlove's right wrist frame has +Y palmward. OpenXR requires +Y + // dorsal on both hands, so conjugate right-hand poses by a Z half-turn. + if (!is_left) + { + destination.position.x = -destination.position.x; + destination.position.y = -destination.position.y; + destination.orientation.x = -destination.orientation.x; + destination.orientation.y = -destination.orientation.y; + } + return true; +} + +} // namespace + +ConversionStatus convert_post_data(const omniglove_post_data_t& post, + bool is_left, + std::array& joints) +{ + for (auto& joint : joints) + { + joint = {}; + joint.pose.orientation.w = 1.0F; + joint.radius = kJointRadiusM; + } + + if (post.mapped == 0U) + { + return ConversionStatus::NotMapped; + } + if ((post.flags & kPostIkConverged) == 0U) + { + return ConversionStatus::IkNotConverged; + } + if ((post.flags & kPostBoneModelValid) == 0U) + { + return ConversionStatus::BoneModelInvalid; + } + + for (const JointSource& source : joint_sources(post)) + { + XrHandJointLocationEXT& destination = joints[static_cast(source.joint)]; + if (!convert_pose(*source.pose, is_left, destination.pose)) + { + for (auto& joint : joints) + { + joint.locationFlags = 0; + } + return ConversionStatus::InvalidPose; + } + if (source.joint != XR_HAND_JOINT_WRIST_EXT) + { + // SDK finger frames point +Z toward the tip; OpenXR points -Z. + // Post-multiply by a local Y half-turn to preserve dorsal +Y. + // The SDK wrist already has proximal +Z and must not be flipped. + const XrQuaternionf q = destination.pose.orientation; + destination.pose.orientation = { -q.z, q.w, q.x, -q.y }; + } + destination.locationFlags = kValidFlags; + } + return ConversionStatus::Success; +} + +const char* conversion_status_name(ConversionStatus status) noexcept +{ + switch (status) + { + case ConversionStatus::Success: + return "success"; + case ConversionStatus::NotMapped: + return "not mapped"; + case ConversionStatus::IkNotConverged: + return "IK not converged"; + case ConversionStatus::BoneModelInvalid: + return "bone model invalid"; + case ConversionStatus::InvalidPose: + return "invalid pose"; + } + return "unknown"; +} + +} // namespace plugins::omniglove diff --git a/src/plugins/omniglove/core/omniglove_plugin.cpp b/src/plugins/omniglove/core/omniglove_plugin.cpp new file mode 100644 index 0000000000..62cb04ce9c --- /dev/null +++ b/src/plugins/omniglove/core/omniglove_plugin.cpp @@ -0,0 +1,408 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/omniglove/omniglove_plugin.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins::omniglove +{ + +namespace +{ + +constexpr auto kWorkerPeriod = std::chrono::milliseconds(2); +constexpr int64_t kStaleThresholdNs = 200'000'000; +constexpr XrSpaceLocationFlags kTrackedFlags = + XR_SPACE_LOCATION_POSITION_TRACKED_BIT | XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT; + +plugin_utils::WristSourceMode wrist_source_mode_from_env() +{ + const char* value = std::getenv("OMNIGLOVE_WRIST_SOURCE"); + if (value == nullptr || *value == '\0' || std::strcmp(value, "hand_tracking") == 0) + { + return plugin_utils::WristSourceMode::HandTracking; + } + if (std::strcmp(value, "auto") == 0) + { + return plugin_utils::WristSourceMode::Auto; + } + if (std::strcmp(value, "controller") == 0) + { + return plugin_utils::WristSourceMode::Controller; + } + isaaccapture::Logger::get("isaaccapture.plugins.omniglove.main") + ->warn("unknown OMNIGLOVE_WRIST_SOURCE '{}', using 'hand_tracking'", value); + return plugin_utils::WristSourceMode::HandTracking; +} + +XrPosef pose_from_env(const char* name) +{ + XrPosef pose{ { 0.0F, 0.0F, 0.0F, 1.0F }, { 0.0F, 0.0F, 0.0F } }; + const char* value = std::getenv(name); + if (value == nullptr || *value == '\0') + { + return pose; + } + if (std::sscanf(value, "%f,%f,%f,%f,%f,%f,%f", &pose.position.x, &pose.position.y, &pose.position.z, + &pose.orientation.x, &pose.orientation.y, &pose.orientation.z, &pose.orientation.w) != 7) + { + throw std::runtime_error(std::string(name) + " must be px,py,pz,qx,qy,qz,qw"); + } + const float values[] = { pose.position.x, pose.position.y, pose.position.z, pose.orientation.x, + pose.orientation.y, pose.orientation.z, pose.orientation.w }; + if (!std::all_of(std::begin(values), std::end(values), [](float component) { return std::isfinite(component); })) + { + throw std::runtime_error(std::string(name) + " contains a non-finite value"); + } + const float norm = std::sqrt(pose.orientation.x * pose.orientation.x + pose.orientation.y * pose.orientation.y + + pose.orientation.z * pose.orientation.z + pose.orientation.w * pose.orientation.w); + if (norm < 1.0e-6F) + { + throw std::runtime_error(std::string(name) + " contains a zero quaternion"); + } + pose.orientation.x /= norm; + pose.orientation.y /= norm; + pose.orientation.z /= norm; + pose.orientation.w /= norm; + return pose; +} + +const char* hand_name(bool is_left) +{ + return is_left ? "left" : "right"; +} + +} // namespace + +void OmniglovePlugin::SessionDeleter::operator()(omniglove_session_t* session) const noexcept +{ + omniglove_session_destroy(session); +} + +OmniglovePlugin::OmniglovePlugin(const std::string& plugin_root_id) +{ + m_logger->info("initializing with root {}", plugin_root_id); + + plugin_utils::WristSourceConfig wrist_config; + wrist_config.mode = wrist_source_mode_from_env(); + if (wrist_config.mode != plugin_utils::WristSourceMode::HandTracking) + { + // Controller aim is not a wrist pose without an explicit mount calibration. + for (const char* name : { "OMNIGLOVE_AIM_TO_WRIST_LEFT", "OMNIGLOVE_AIM_TO_WRIST_RIGHT" }) + { + const char* value = std::getenv(name); + if (value == nullptr || *value == '\0') + { + throw std::runtime_error(std::string(name) + " is required for controller and auto wrist sources"); + } + } + } + wrist_config.left_aim_to_wrist = pose_from_env("OMNIGLOVE_AIM_TO_WRIST_LEFT"); + wrist_config.right_aim_to_wrist = pose_from_env("OMNIGLOVE_AIM_TO_WRIST_RIGHT"); + const char* wrist_mode = wrist_config.mode == plugin_utils::WristSourceMode::HandTracking ? "hand_tracking" : + wrist_config.mode == plugin_utils::WristSourceMode::Controller ? "controller" : + "auto"; + m_logger->info("wrist source {}", wrist_mode); + initialize_sdk(); + const auto wrist_requirements = plugin_utils::WristPoseSource::collect_requirements(wrist_config.mode); + + auto extensions = core::DeviceIOSession::get_required_extensions(wrist_requirements.trackers); + extensions.push_back(XR_NVX1_DEVICE_INTERFACE_BASE_EXTENSION_NAME); + extensions.insert(extensions.end(), wrist_requirements.extensions.begin(), wrist_requirements.extensions.end()); + + m_session = std::make_shared("Omniglove", extensions); + const auto handles = m_session->get_handles(); + m_deviceio_session = core::DeviceIOSession::run(wrist_requirements.trackers, handles); + m_wrist_source = std::make_unique( + wrist_config, handles, m_deviceio_session.get(), wrist_requirements.controller_tracker); + m_time_converter = std::make_unique(handles); + + start_device_streams(); + m_running.store(true, std::memory_order_release); + try + { + m_worker = std::thread(&OmniglovePlugin::worker_thread, this); + } + catch (...) + { + m_running.store(false, std::memory_order_release); + throw; + } + m_logger->info("initialized with {} glove stream(s)", m_streams.size()); +} + +OmniglovePlugin::~OmniglovePlugin() +{ + m_running.store(false, std::memory_order_release); + if (m_worker.joinable()) + { + m_worker.join(); + } + m_left_injector.reset(); + m_right_injector.reset(); + m_sdk_session.reset(); + m_streams.clear(); +} + +bool OmniglovePlugin::is_running() const noexcept +{ + return m_running.load(std::memory_order_acquire); +} + +bool OmniglovePlugin::has_failed() const noexcept +{ + return m_failed.load(std::memory_order_acquire); +} + +void OmniglovePlugin::initialize_sdk() +{ + // Accept any SDK ABI version when its header and runtime agree. A new ABI + // number is not itself an incompatibility; required API changes fail build. + const uint32_t runtime_abi = omniglove_abi_version(); + if (runtime_abi != OMNIGLOVE_ABI_VERSION) + { + throw std::runtime_error("Omniglove header/runtime ABI mismatch: header=" + + std::to_string(OMNIGLOVE_ABI_VERSION) + " runtime=" + std::to_string(runtime_abi)); + } + m_logger->info( + "Omniglove ABI {}.{}.{}", (runtime_abi >> 16U) & 0xffU, (runtime_abi >> 8U) & 0xffU, runtime_abi & 0xffU); + m_logger->info("Omniglove SDK release {} ({})", OMNIGLOVE_SDK_RELEASE_VERSION, OMNIGLOVE_SDK_GIT_COMMIT); + + omniglove_session_t* session = nullptr; + omniglove_init_report_t* report = nullptr; + const omniglove_error_t error = omniglove_initialize(nullptr, &session, &report); + + if (report != nullptr) + { + const size_t size = omniglove_init_report_message(report, nullptr, 0); + if (size > 1) + { + std::vector message(size); + omniglove_init_report_message(report, message.data(), message.size()); + m_logger->info("SDK discovery: {}", message.data()); + } + omniglove_init_report_destroy(report); + } + + if (error != OMNIGLOVE_ERROR_NONE || session == nullptr) + { + throw std::runtime_error(std::string("omniglove_initialize failed: ") + omniglove_error_name(error)); + } + m_sdk_session.reset(session); +} + +void OmniglovePlugin::start_device_streams() +{ + bool have_left = false; + bool have_right = false; + const size_t count = omniglove_session_device_count(m_sdk_session.get()); + + for (size_t index = 0; index < count; ++index) + { + omniglove_device_t* device = omniglove_session_device_at(m_sdk_session.get(), index); + if (device == nullptr) + { + m_logger->warn("ignoring null glove entry {}", index); + continue; + } + const omniglove_hand_t hand = omniglove_device_hand(device); + if (hand == OMNIGLOVE_HAND_UNKNOWN) + { + m_logger->warn("ignoring glove {} because its hand side is unknown", index); + continue; + } + const bool is_left = hand == OMNIGLOVE_HAND_LEFT; + bool& occupied = is_left ? have_left : have_right; + if (occupied) + { + m_logger->warn("ignoring a second {} glove", hand_name(is_left)); + continue; + } + + auto stream = std::make_unique(); + stream->device = device; + stream->context = { this, is_left }; + + omniglove_stream_config_t config{}; + config.struct_size = sizeof(config); + config.kind = OMNIGLOVE_STREAM_POST_DATA; + config.stream_mode = OMNIGLOVE_STREAM_MODE_DEFAULT; + config.output_mode = OMNIGLOVE_STREAM_OUTPUT_DIRECT; + config.source_interval_ms = 8; + config.source_timeout_ms = 200; + config.callback = &OmniglovePlugin::stream_callback; + config.callback_user_data = &stream->context; + config.callback_queue_capacity = 64; + + const auto configure_error = omniglove_device_stream_configure(device, &config); + if (configure_error != OMNIGLOVE_STREAM_ERROR_NONE) + { + m_logger->error("could not configure {} POST_DATA stream: {}", hand_name(is_left), + static_cast(configure_error)); + continue; + } + const auto start_error = omniglove_device_stream_start(device, OMNIGLOVE_STREAM_POST_DATA); + if (start_error != OMNIGLOVE_STREAM_ERROR_NONE) + { + m_logger->error( + "could not start {} POST_DATA stream: {}", hand_name(is_left), static_cast(start_error)); + continue; + } + stream->started = true; + occupied = true; + m_logger->info("started {} glove at {} Hz", hand_name(is_left), omniglove_device_nominal_source_rate_hz(device)); + m_streams.push_back(std::move(stream)); + } + + if (m_streams.empty()) + { + throw std::runtime_error("no left or right Omniglove POST_DATA stream could be started"); + } +} + +void OmniglovePlugin::stream_callback(const omniglove_stream_sample_t* sample, void* user_data) +{ + if (sample == nullptr || user_data == nullptr) + { + return; + } + auto* context = static_cast(user_data); + context->plugin->on_sample(*sample, context->is_left); +} + +void OmniglovePlugin::on_sample(const omniglove_stream_sample_t& sample, bool is_left) +{ + if (sample.kind != OMNIGLOVE_STREAM_POST_DATA || sample.layout_version != OMNIGLOVE_STREAM_POST_LAYOUT_VERSION || + sample.data.postdata == nullptr || sample.sample_monotonic_ns > std::numeric_limits::max()) + { + invalidate_hand(is_left); + return; + } + + HandFrame converted; + const ConversionStatus status = convert_post_data(*sample.data.postdata, is_left, converted.joints); + if (status != ConversionStatus::Success) + { + invalidate_hand(is_left); + m_logger->debug("{} POST_DATA rejected: {}", hand_name(is_left), conversion_status_name(status)); + return; + } + converted.sample_monotonic_ns = sample.sample_monotonic_ns; + converted.connection_generation = sample.connection_generation; + converted.source_sequence = sample.source_sequence; + converted.valid = true; + + std::lock_guard lock(m_frame_mutex); + (is_left ? m_left : m_right) = converted; +} + +void OmniglovePlugin::invalidate_hand(bool is_left) +{ + std::lock_guard lock(m_frame_mutex); + (is_left ? m_left : m_right).valid = false; +} + +void OmniglovePlugin::pump_hand(std::unique_ptr& injector, + XrHandEXT hand, + const HandFrame& frame, + uint64_t& last_pushed_ns) +{ + const bool is_left = hand == XR_HAND_LEFT_EXT; + const int64_t now_ns = core::os_monotonic_now_ns(); + const bool fresh = frame.valid && frame.sample_monotonic_ns <= static_cast(now_ns) && + now_ns - static_cast(frame.sample_monotonic_ns) < kStaleThresholdNs; + if (!fresh) + { + if (injector) + { + m_logger->info("{} hand inactive: SDK frame stale or invalid", hand_name(is_left)); + } + injector.reset(); + last_pushed_ns = 0; + return; + } + if (frame.sample_monotonic_ns == last_pushed_ns) + { + return; + } + + const XrTime time = m_time_converter->convert_monotonic_ns_to_xrtime(static_cast(frame.sample_monotonic_ns)); + const plugin_utils::WristSample wrist = m_wrist_source->query(is_left, time); + if (!wrist.valid) + { + if (injector) + { + m_logger->info("{} hand inactive: wrist pose unavailable", hand_name(is_left)); + } + injector.reset(); + return; + } + + if (!injector) + { + const auto handles = m_session->get_handles(); + injector = std::make_unique(handles.instance, handles.session, hand, handles.space); + m_logger->info("{} hand active: connection generation {}, source sequence {}", hand_name(is_left), + frame.connection_generation, frame.source_sequence); + } + + auto joints = frame.joints; + for (auto& joint : joints) + { + if ((joint.locationFlags & XR_SPACE_LOCATION_POSITION_VALID_BIT) == 0) + { + continue; + } + joint.pose = oxr_utils::multiply_poses(wrist.pose, joint.pose); + if (wrist.tracked) + { + joint.locationFlags |= kTrackedFlags; + } + } + injector->push(joints.data(), time); + last_pushed_ns = frame.sample_monotonic_ns; +} + +void OmniglovePlugin::worker_thread() +{ + while (m_running.load(std::memory_order_acquire)) + { + try + { + m_deviceio_session->update(); + HandFrame left; + HandFrame right; + { + std::lock_guard lock(m_frame_mutex); + left = m_left; + right = m_right; + } + pump_hand(m_left_injector, XR_HAND_LEFT_EXT, left, m_left_last_pushed_ns); + pump_hand(m_right_injector, XR_HAND_RIGHT_EXT, right, m_right_last_pushed_ns); + } + catch (const std::exception& error) + { + m_logger->error("update failed: {}", error.what()); + m_left_injector.reset(); + m_right_injector.reset(); + m_failed.store(true, std::memory_order_release); + m_running.store(false, std::memory_order_release); + return; + } + std::this_thread::sleep_for(kWorkerPeriod); + } +} + +} // namespace plugins::omniglove diff --git a/src/plugins/omniglove/install.sh b/src/plugins/omniglove/install.sh new file mode 100755 index 0000000000..7aad81e791 --- /dev/null +++ b/src/plugins/omniglove/install.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +build_dir="" +sdk_root="${OMNIGLOVE_SDK_ROOT:-}" +isaac_root="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --build-dir) + build_dir="${2:?--build-dir requires a path}" + shift 2 + ;; + --sdk-root) + sdk_root="${2:?--sdk-root requires a path}" + shift 2 + ;; + -h | --help) + cat <&2 + exit 1 + ;; + *) + if [[ -n "$isaac_root" ]]; then + echo "error: only one Isaac Capture root may be supplied" >&2 + exit 1 + fi + isaac_root="$1" + shift + ;; + esac +done + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +isaac_root="${isaac_root:-$(cd "$script_dir/../../.." && pwd)}" +build_dir="${build_dir:-$isaac_root/build}" +if [[ "$build_dir" != /* ]]; then + build_dir="$PWD/$build_dir" +fi + +workspace_sdk="$isaac_root/../agilink_omniglove_sdk" +if [[ -z "$sdk_root" ]] && [[ -d "$workspace_sdk/.git" ]]; then + sdk_root="$workspace_sdk" +fi + +if [[ -z "$sdk_root" ]]; then + sdk_root="$script_dir/.sdk/agilink_omniglove_sdk" + if [[ -d "$sdk_root/.git" ]]; then + echo "==> Updating managed OmniGlove SDK checkout" + git -C "$sdk_root" pull --ff-only + else + echo "==> Cloning latest OmniGlove SDK release repository" + mkdir -p "$(dirname "$sdk_root")" + git clone --depth 1 https://github.com/AgibotTech/agilink_omniglove_sdk.git "$sdk_root" + fi +fi + +sdk_root="$(cd "$sdk_root" && pwd)" +if [[ ! -f "$sdk_root/VERSION" ]] || [[ ! -f "$sdk_root/SHA256SUMS" ]]; then + echo "error: $sdk_root is not a complete OmniGlove SDK release checkout" >&2 + exit 1 +fi + +echo "==> Verifying OmniGlove SDK $(tr -d '[:space:]' < "$sdk_root/VERSION")" +(cd "$sdk_root" && sha256sum --check --quiet SHA256SUMS) + +case "$(uname -m)" in + x86_64) platform="linux-x86_64" ;; + aarch64 | arm64) platform="linux-aarch64" ;; + *) + echo "error: unsupported architecture $(uname -m)" >&2 + exit 1 + ;; +esac + +platform_root="$sdk_root/$platform" +for required in \ + "$platform_root/cmake/OmnigloveSDKConfig.cmake" \ + "$platform_root/include/omniglove/omniglove.h"; do + if [[ ! -f "$required" ]]; then + echo "error: SDK artifact is missing: $required" >&2 + exit 1 + fi +done + +# Follow the selected checkout's package config instead of pinning an ABI-named +# library here. CMake and the compatibility probe reject missing required APIs. + +sdk_commit="unknown" +if [[ -d "$sdk_root/.git" ]]; then + sdk_commit="$(git -C "$sdk_root" rev-parse HEAD)" +fi + +echo "==> Configuring OmniGlove plugin against $platform ($sdk_commit)" +cmake -S "$isaac_root" -B "$build_dir" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=ON \ + -DBUILD_PLUGINS=ON \ + -DBUILD_PLUGIN_OMNIGLOVE=ON \ + -DOmnigloveSDK_DIR="$platform_root/cmake" \ + -DOMNIGLOVE_SDK_RELEASE_ROOT="$sdk_root" \ + -DOMNIGLOVE_SDK_GIT_COMMIT="$sdk_commit" + +cmake --build "$build_dir" --target \ + omniglove_api_compat_probe \ + omniglove_hand_conversion_contract_test \ + omniglove_plugin --parallel + +ctest --test-dir "$build_dir" --output-on-failure -R '^omniglove_' +cmake --install "$build_dir" --component omniglove + +install_prefix="$(awk '/^CMAKE_INSTALL_PREFIX:PATH=/{sub(/^[^=]*=/, ""); print; exit}' "$build_dir/CMakeCache.txt")" +echo "==> Installed $install_prefix/plugins/omniglove/omniglove_plugin" +echo "Install USB permissions once on the host with:" +echo " $script_dir/install_udev_rules.sh" diff --git a/src/plugins/omniglove/install_udev_rules.sh b/src/plugins/omniglove/install_udev_rules.sh new file mode 100755 index 0000000000..85ba60b598 --- /dev/null +++ b/src/plugins/omniglove/install_udev_rules.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +rule_source="$script_dir/60-omniglove-usb.rules" +rule_destination="/etc/udev/rules.d/60-omniglove-usb.rules" + +if [[ -f /.dockerenv ]] || grep -Eq '(docker|containerd|kubepods)' /proc/1/cgroup 2>/dev/null; then + echo "error: install udev rules on the host, not inside a container" >&2 + exit 1 +fi +if [[ ! -f "$rule_source" ]]; then + echo "error: missing $rule_source" >&2 + exit 1 +fi +if ! command -v udevadm >/dev/null 2>&1; then + echo "error: udevadm is required" >&2 + exit 1 +fi + +if [[ -f "$rule_destination" ]] && sudo cmp -s "$rule_source" "$rule_destination"; then + echo "OmniGlove udev rule is already installed" + exit 0 +fi + +sudo install -m 0644 "$rule_source" "$rule_destination" +sudo udevadm control --reload-rules +sudo udevadm trigger --subsystem-match=usb --action=change +sudo udevadm settle + +echo "Installed $rule_destination" +echo "Unplug and reconnect the gloves, then run applications as a normal user." diff --git a/src/plugins/omniglove/package_cts.py b/src/plugins/omniglove/package_cts.py new file mode 100755 index 0000000000..424defde17 --- /dev/null +++ b/src/plugins/omniglove/package_cts.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Check and package a hand take with the official CTS viewer's bundle builder.""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +from pathlib import Path +import zipfile + +from hand_cts.labels import StepTimeline +from hand_cts.mcap_source import McapFrameSource +from hand_cts.panel import bundle +from hand_cts.panel.track import HandTee +from hand_cts.report import run + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("recording", type=Path) + parser.add_argument("--output-dir", type=Path) + args = parser.parse_args() + recording = args.recording.resolve() + required = [ + recording, + *(recording.with_suffix(s) for s in (".labels.json", ".json", ".log")), + ] + missing = [str(path) for path in required if not path.is_file()] + if missing: + parser.error("missing take files: " + ", ".join(missing)) + + before = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in required} + timeline = StepTimeline.beside(recording) + source = HandTee(McapFrameSource(recording), timeline) + report = run(source, timeline=timeline) + name, data = bundle.build(report, source.tracks(), recording, timeline) + with zipfile.ZipFile(io.BytesIO(data)) as archive: + if archive.testzip() is not None: + raise RuntimeError("CTS archive failed its integrity check") + root = name.removesuffix(".zip") + manifest = json.loads(archive.read(f"{root}/report.json")) + for item in manifest["inputs"]["files"]: + path = recording.parent / item["name"] + if item["sha256"] != before[path]: + raise RuntimeError(f"take changed during packaging: {path}") + if any( + hashlib.sha256(path.read_bytes()).hexdigest() != digest + for path, digest in before.items() + ): + raise RuntimeError( + "take changed during packaging; finish recording before packaging" + ) + + output_dir = args.output_dir or recording.parent + output_dir.mkdir(parents=True, exist_ok=True) + output = output_dir / name + with output.open("xb") as destination: + destination.write(data) + print( + json.dumps( + { + "archive": str(output), + "verdict": report.verdict, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "checker": manifest["tool"], + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/src/plugins/omniglove/plugin.yaml b/src/plugins/omniglove/plugin.yaml new file mode 100644 index 0000000000..7b0cc581e5 --- /dev/null +++ b/src/plugins/omniglove/plugin.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +name: omniglove_plugin +description: "AgiLink OmniGlove hand tracking" +command: "./omniglove_plugin" +version: "1.0.0" +devices: + - path: "/hand/left" + type: "hand_tracker" + description: "Left OmniGlove" + - path: "/hand/right" + type: "hand_tracker" + description: "Right OmniGlove" diff --git a/src/plugins/omniglove/record_cts.sh b/src/plugins/omniglove/record_cts.sh new file mode 100755 index 0000000000..7a72262769 --- /dev/null +++ b/src/plugins/omniglove/record_cts.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_dir="$(cd -- "$script_dir/../../.." && pwd)" +allow_dirty=false +capture_args=() +operator_note="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --wrist-config) + source "${2:?--wrist-config requires an environment file}" + shift 2 + ;; + --allow-dirty) + allow_dirty=true + shift + ;; + --note) + operator_note="${2:?--note requires text}" + shift 2 + ;; + --note=*) operator_note="${1#--note=}"; shift ;; + -h | --help) + cat <<'EOF' +Usage: record_cts.sh [--wrist-config PATH] [--allow-dirty] [capture options] + +Formal recordings require a clean, pushed plugin commit and an unchanged CTS. +--allow-dirty permits development takes; their provenance remains unchanged. +Set OMNIGLOVE_WRIST_SOURCE and measured per-hand controller transforms, or +load them from --wrist-config. Other options go to the official recorder. +EOF + exit 0 + ;; + --plugin | --plugin=* | --wrist-source | --wrist-source=*) + echo "error: plugin and wrist source are selected by this script" >&2 + exit 1 + ;; + *) capture_args+=("$1"); shift ;; + esac +done + +cd -- "$repo_dir" +if [[ -n "$(git status --porcelain -- src/python/isaaccapture/cts)" ]]; then + echo "error: CTS has local changes; restore the official tools before recording" >&2 + exit 1 +fi +if [[ "$allow_dirty" == false ]]; then + if [[ -n "$(git status --porcelain)" ]] || + ! git cat-file -e HEAD:src/plugins/omniglove/plugin.yaml 2>/dev/null; then + echo "error: commit the final plugin and keep the repository clean before a formal take" >&2 + exit 1 + fi + if ! git rev-parse --verify '@{upstream}' >/dev/null 2>&1 || + ! git merge-base --is-ancestor HEAD '@{upstream}'; then + echo "error: push this commit and set its upstream before a formal take" >&2 + exit 1 + fi +fi + +wrist_source="${OMNIGLOVE_WRIST_SOURCE:-hand_tracking}" +case "$wrist_source" in + hand_tracking) ;; + controller | auto) + : "${OMNIGLOVE_AIM_TO_WRIST_LEFT:?set the measured left aim-to-wrist transform}" + : "${OMNIGLOVE_AIM_TO_WRIST_RIGHT:?set the measured right aim-to-wrist transform}" + ;; + *) echo "error: unsupported OMNIGLOVE_WRIST_SOURCE=$wrist_source" >&2; exit 1 ;; +esac +export OMNIGLOVE_WRIST_SOURCE="$wrist_source" +if [[ "$wrist_source" != hand_tracking ]]; then + export OMNIGLOVE_AIM_TO_WRIST_LEFT OMNIGLOVE_AIM_TO_WRIST_RIGHT +fi +plugin_binary="$repo_dir/install/plugins/omniglove/omniglove_plugin" +if [[ ! -x "$plugin_binary" ]]; then + echo "error: build and install the OmniGlove plugin first" >&2 + exit 1 +fi +plugin_sha="$(sha256sum "$plugin_binary")" +plugin_sha="${plugin_sha%% *}" +exec src/python/isaaccapture/cts/hand/capture/record.sh omniglove \ + --plugin omniglove_plugin \ + --wrist-source "$wrist_source" \ + --note "OmniGlove; binary_sha256=$plugin_sha; wrist=$wrist_source; LEFT=${OMNIGLOVE_AIM_TO_WRIST_LEFT:-unused}; RIGHT=${OMNIGLOVE_AIM_TO_WRIST_RIGHT:-unused}; $operator_note" \ + "${capture_args[@]}" diff --git a/src/plugins/omniglove/tests/CMakeLists.txt b/src/plugins/omniglove/tests/CMakeLists.txt new file mode 100644 index 0000000000..e656036b5e --- /dev/null +++ b/src/plugins/omniglove/tests/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(omniglove_hand_conversion_contract_test hand_conversion_contract_test.cpp) +target_link_libraries(omniglove_hand_conversion_contract_test PRIVATE isaac::omniglove_plugin) + +add_test(NAME omniglove_hand_conversion_contract_test COMMAND omniglove_hand_conversion_contract_test) diff --git a/src/plugins/omniglove/tests/hand_conversion_contract_test.cpp b/src/plugins/omniglove/tests/hand_conversion_contract_test.cpp new file mode 100644 index 0000000000..3de041e20e --- /dev/null +++ b/src/plugins/omniglove/tests/hand_conversion_contract_test.cpp @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include + +namespace +{ + +using plugins::omniglove::ConversionStatus; + +void fill_pose(omniglove_pose6_t& pose, float x, float y, float z, const XrQuaternionf& orientation) +{ + pose.position_m[0] = x; + pose.position_m[1] = y; + pose.position_m[2] = z; + pose.quaternion_xyzw[0] = orientation.x; + pose.quaternion_xyzw[1] = orientation.y; + pose.quaternion_xyzw[2] = orientation.z; + pose.quaternion_xyzw[3] = orientation.w; +} + +void fill_hand(omniglove_post_data_t& post, const XrQuaternionf& orientation = { 0.0F, 0.0F, 0.0F, 2.0F }) +{ + post.mapped = 1; + post.flags = (1U << 0U) | (1U << 2U); + float x = 0.01F; + auto fill = [&x, &orientation](omniglove_pose6_t& pose) + { + fill_pose(pose, x, x + 0.01F, x + 0.02F, orientation); + x += 0.01F; + }; + fill(post.wrist.landmark); + fill(post.thumb.landmarks.cmc); + fill(post.thumb.landmarks.mcp); + fill(post.thumb.landmarks.ip); + fill(post.thumb.landmarks.tip); + const std::array fingers = { &post.index, &post.middle, &post.ring, &post.pinky }; + for (auto* finger : fingers) + { + fill(finger->landmarks.mcp); + fill(finger->landmarks.pip); + fill(finger->landmarks.dip); + fill(finger->landmarks.tip); + } +} + +bool near(float left, float right) +{ + return std::abs(left - right) < 1.0e-6F; +} + +std::array rotate(const XrQuaternionf& q, const std::array& v) +{ + const std::array t = { 2.0F * (q.y * v[2] - q.z * v[1]), 2.0F * (q.z * v[0] - q.x * v[2]), + 2.0F * (q.x * v[1] - q.y * v[0]) }; + return { v[0] + q.w * t[0] + q.y * t[2] - q.z * t[1], v[1] + q.w * t[1] + q.z * t[0] - q.x * t[2], + v[2] + q.w * t[2] + q.x * t[1] - q.y * t[0] }; +} + +bool openxr_axes(bool is_left, const XrQuaternionf& orientation) +{ + omniglove_post_data_t post{}; + fill_hand(post, orientation); + std::array joints{}; + if (plugins::omniglove::convert_post_data(post, is_left, joints) != ConversionStatus::Success) + { + return false; + } + const float norm = std::sqrt(orientation.x * orientation.x + orientation.y * orientation.y + + orientation.z * orientation.z + orientation.w * orientation.w); + const XrQuaternionf unit = { orientation.x / norm, orientation.y / norm, orientation.z / norm, orientation.w / norm }; + for (std::size_t index = 0; index < joints.size(); ++index) + { + const auto& joint = joints[index]; + if (joint.locationFlags == 0) + { + continue; + } + for (const std::array& axis : + { std::array{ 0.0F, 1.0F, 0.0F }, std::array{ 0.0F, 0.0F, 1.0F } }) + { + auto sdk_axis = axis; + if (!is_left) + { + sdk_axis[0] = -sdk_axis[0]; + sdk_axis[1] = -sdk_axis[1]; + } + auto expected = rotate(unit, sdk_axis); + if (!is_left) + { + expected[0] = -expected[0]; + expected[1] = -expected[1]; + } + auto xr_axis = axis; + if (index != XR_HAND_JOINT_WRIST_EXT && axis[2] != 0.0F) + { + xr_axis[2] = -xr_axis[2]; + } + const auto actual = rotate(joint.pose.orientation, xr_axis); + for (std::size_t component = 0; component < actual.size(); ++component) + { + if (!near(actual[component], expected[component])) + { + return false; + } + } + } + } + return true; +} + +int fail(const char* message) +{ + std::cerr << message << '\n'; + return 1; +} + +} // namespace + +int main() +{ + omniglove_post_data_t post{}; + fill_hand(post); + std::array joints{}; + + if (plugins::omniglove::convert_post_data(post, true, joints) != ConversionStatus::Success) + { + return fail("left hand conversion failed"); + } + constexpr XrSpaceLocationFlags valid = XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; + int valid_count = 0; + for (const auto& joint : joints) + { + valid_count += (joint.locationFlags & valid) == valid; + } + if (valid_count != 21 || joints[XR_HAND_JOINT_PALM_EXT].locationFlags != 0 || + joints[XR_HAND_JOINT_INDEX_METACARPAL_EXT].locationFlags != 0) + { + return fail("the required/optional joint set is wrong"); + } + if (!near(joints[XR_HAND_JOINT_WRIST_EXT].pose.orientation.w, 1.0F)) + { + return fail("quaternion was not normalized"); + } + + for (bool is_left : { true, false }) + { + if (!openxr_axes(is_left, { 0.0F, 0.0F, 0.0F, 2.0F }) || !openxr_axes(is_left, { 1.0F, 2.0F, 3.0F, 4.0F })) + { + return fail("finger -Z/dorsal +Y or wrist axes do not follow OpenXR"); + } + } + + const XrPosef left_wrist = joints[XR_HAND_JOINT_WRIST_EXT].pose; + if (plugins::omniglove::convert_post_data(post, false, joints) != ConversionStatus::Success || + !near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.x, -left_wrist.position.x) || + !near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.y, -left_wrist.position.y) || + !near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.z, left_wrist.position.z)) + { + return fail("right-hand wrist-basis conversion is wrong"); + } + + post.flags = 1U << 2U; + if (plugins::omniglove::convert_post_data(post, true, joints) != ConversionStatus::IkNotConverged) + { + return fail("unconverged IK was accepted"); + } + fill_hand(post); + post.index.landmarks.mcp.position_m[0] = std::numeric_limits::quiet_NaN(); + if (plugins::omniglove::convert_post_data(post, true, joints) != ConversionStatus::InvalidPose) + { + return fail("non-finite pose was accepted"); + } + return 0; +} From f3b939dab27531e0d06f4e4d5976e6c1b831af20 Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 6 Nov 2026 14:37:07 +0800 Subject: [PATCH 02/14] Keep OmniGlove take history in the validation record Signed-off-by: zhouyingchun --- docs/source/device/hand_tracking/omniglove.rst | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/source/device/hand_tracking/omniglove.rst b/docs/source/device/hand_tracking/omniglove.rst index 175d4618eb..fb0488fa3c 100644 --- a/docs/source/device/hand_tracking/omniglove.rst +++ b/docs/source/device/hand_tracking/omniglove.rst @@ -90,9 +90,6 @@ is worn. Controller mode uses controller aim poses and per-hand installation offsets. Auto mode allows both sources; validate each source and the transition before using it. -The successful hardware CTS on 2026-11-06 used **controller** mode. Optical and -Auto modes have not been qualified by that take. - Rigidly attach each controller using its final mount. From simultaneous controller aim and trusted wrist poses, calculate: @@ -149,13 +146,10 @@ commands or hand ``JointStateOutput`` data and defines no vendor hand schema. Validation and CTS ------------------ -The 2026-11-06 controller-wrist take passed the hand CTS with 39 checks passing -and no failures. The optional-joint geometry check was unanswered because -those joints are not supplied. USB/SDK discovery, two-hand 125 Hz streaming, -five-minute streaming stability and SDK reconnect also have hardware evidence. -OpenXR hot-unplug/recovery and a recording from the pushed final plugin commit -remain separate acceptance steps. See the :code-file:`validation record -` for evidence and pending items. +.. Keep dated take results and acceptance status in the plugin validation record. + +See the :code-file:`validation record ` +for hardware test evidence and acceptance status. Follow :doc:`/conformance/setup` to prepare CTS. From a clean plugin commit already pushed to the integration PR, load the mount configuration and record: From 97c62090a97c5376cf8264ff8b90acd8f72a237d Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 6 Nov 2026 15:02:43 +0800 Subject: [PATCH 03/14] Record confirmed OmniGlove headset and controller models Signed-off-by: zhouyingchun --- docs/source/device/hand_tracking/omniglove.rst | 2 +- src/plugins/omniglove/README.md | 2 +- src/plugins/omniglove/VALIDATION.md | 12 +++++------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/source/device/hand_tracking/omniglove.rst b/docs/source/device/hand_tracking/omniglove.rst index fb0488fa3c..80ffa9a4f4 100644 --- a/docs/source/device/hand_tracking/omniglove.rst +++ b/docs/source/device/hand_tracking/omniglove.rst @@ -160,7 +160,7 @@ already pushed to the integration PR, load the mount configuration and record: Use ``--no-launch-cloudxr-runtime`` if CloudXR is already running. For development takes, ``--allow-dirty`` retains the recorder's dirty provenance. -Include headset, controller, firmware and mount details in ``--note``. +Use ``--note`` to record the headset and controller models. Review the take in the official panel, then package it using the panel's Submission button or the wrapper that uses the same bundle builder: diff --git a/src/plugins/omniglove/README.md b/src/plugins/omniglove/README.md index 713415c6bd..ef8cc26677 100644 --- a/src/plugins/omniglove/README.md +++ b/src/plugins/omniglove/README.md @@ -91,7 +91,7 @@ After committing and pushing the plugin, record from a clean checkout: The wrapper rejects uncommitted/unpushed code and changes to the official CTS. It records the installed binary's SHA-256 and the wrist configuration in provenance notes. `--allow-dirty` enables development takes with their original -dirty provenance. `--note` adds hardware/firmware details; use +dirty provenance. `--note` records the headset and controller models; use `--no-launch-cloudxr-runtime` when CloudXR is already running. Review the take with the official viewer and package it with the same builder: diff --git a/src/plugins/omniglove/VALIDATION.md b/src/plugins/omniglove/VALIDATION.md index 3c5a4beee7..e5ac147581 100644 --- a/src/plugins/omniglove/VALIDATION.md +++ b/src/plugins/omniglove/VALIDATION.md @@ -20,9 +20,7 @@ submission have separate evidence. A pending item below is not a pass. | USB devices | Left `cafe:4001`, right `cafe:4002` | | SDK stream | POST_DATA, DIRECT, nominal 125 Hz | | Wrist source qualified | `controller` | -| Headset / controller model | Not recorded in the take; operator confirmation pending | -| Glove hardware / firmware revision | Not recorded in the take; operator confirmation pending | -| Physical controller mount | Final mount description/photo pending operator confirmation | +| Headset / controller model | PICO 4 Ultra / PICO 4 controllers, operator-confirmed; original take has no model fields | ## Hardware CTS evidence @@ -119,17 +117,17 @@ worker iteration. ## Final submission checklist -- [ ] Confirm headset, controller, glove hardware/firmware and final mount. +- [x] Confirm headset and controller models: PICO 4 Ultra / PICO 4 controllers. - [ ] OpenXR hot-unplug/recovery evidence for both hands. - [x] Software build, conversion/ABI tests, C++ format, strict docs and full pre-commit verified against the final source. -- [ ] Commit the plugin with the operator's Git identity and DCO sign-off. -- [ ] Push to the user-specified repository and open the integration PR. +- [x] Commit the plugin with the operator's Git identity and DCO sign-off. +- [x] Push to AgibotTech/IsaacCapture and open integration PR #1445. - [ ] Record from that pushed commit with a clean checkout and unchanged CTS: ```bash ./src/plugins/omniglove/record_cts.sh --wrist-config /path/to/mount.env \ - --no-launch-cloudxr-runtime --note "headset/model, controller/model, firmware, mount" + --no-launch-cloudxr-runtime --note "headset=PICO 4 Ultra; controller=PICO 4 controllers" ``` - [ ] Review the new take in `hand_cts.panel`; verify handedness, axes, From 445cd3e1fc59aa327e120c2fb2514ea042189c57 Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 6 Nov 2026 15:15:56 +0800 Subject: [PATCH 04/14] Document dual-hand OpenXR hotplug recovery evidence Signed-off-by: zhouyingchun --- src/plugins/omniglove/README.md | 3 ++- src/plugins/omniglove/VALIDATION.md | 35 +++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/plugins/omniglove/README.md b/src/plugins/omniglove/README.md index ef8cc26677..57e0a78d3c 100644 --- a/src/plugins/omniglove/README.md +++ b/src/plugins/omniglove/README.md @@ -77,7 +77,8 @@ The API probe and hand-conversion contract test run during installation. The controller-wrist hardware take `omniglove_2026-11-06_110404` passed hand CTS (39 pass, zero fail). It was recorded before the plugin was committed, so it is development evidence; formal submission needs a take from the pushed final -commit. OpenXR hot-unplug/recovery remains a separate hardware acceptance item. +commit. A separate take, `omniglove_2026-11-06_150705`, verified both hands' +functional OpenXR hot-unplug/recovery while the other hand remained active. See [VALIDATION.md](VALIDATION.md) for the evidence and remaining steps. ## Formal recording and packaging diff --git a/src/plugins/omniglove/VALIDATION.md b/src/plugins/omniglove/VALIDATION.md index e5ac147581..b98a245732 100644 --- a/src/plugins/omniglove/VALIDATION.md +++ b/src/plugins/omniglove/VALIDATION.md @@ -78,7 +78,37 @@ The 2026-11-01 handover records: These historical results establish the USB/SDK layer. They do not establish OpenXR injector deactivation/recreation or long-term headset tracking. -## OpenXR hot-unplug/recovery procedure — pending +## OpenXR hot-unplug/recovery evidence + +Take `omniglove_2026-11-06_150705` was recorded at pushed commit +`97c62090a97c5376cf8264ff8b90acd8f72a237d`, with `repo_dirty=false`, the same +controller offsets, PICO 4 Ultra and PICO 4 controllers. The complete MCAP +contains 6,747 hand records per side over 81.94 seconds. + +Both hands became inactive downstream and recovered in the same process run: + +| Inactive hand | Downstream inactive interval | Other hand during that interval | Recovery generation | +| --- | --- | --- | --- | +| Left | 22.53 s | Right: 1,875/1,875 records with 21 valid joints; max record gap 14.55 ms | 1→3 | +| Right, main interval | 25.50 s | Left: 2,096/2,096 records with 21 valid joints; max record gap 15.38 ms | 3→5 | + +The other hand's wrist-relative fingertip displacement reached 100.79 mm +during the left interval and 125.54 mm during the right interval, establishing +changing finger data. Both recovered hands also continued updating. +The operator waited before reconnecting; inactive interval lengths include +that wait and do not measure latency after reinsertion. + +An additional right-hand inactive interval lasted 0.18 s before the main +interval, with generation 1→3 on recovery. Its physical cause is unknown; +the left hand remained active throughout. This event is retained in the evidence. + +Functional hot-unplug/recovery is verified. Exact 200 ms cutoff and reinsertion +latency were not measured because SDK sample and physical unplug/reinsert +timestamps are absent. Downstream inactivity is encoded by an absent +`HandPoseRecord.data` payload, not a separate `is_active` field. +The take has no action labels and is separate from formal CTS evidence. + +## OpenXR hot-unplug/recovery procedure Keep the final wrist configuration and both gloves connected, connect the headset, and start a session consuming the two standard hand collections. @@ -118,7 +148,8 @@ worker iteration. ## Final submission checklist - [x] Confirm headset and controller models: PICO 4 Ultra / PICO 4 controllers. -- [ ] OpenXR hot-unplug/recovery evidence for both hands. +- [x] Functional OpenXR hot-unplug/recovery evidence for both hands; timing + measurement limits and the extra right-hand transition recorded above. - [x] Software build, conversion/ABI tests, C++ format, strict docs and full pre-commit verified against the final source. - [x] Commit the plugin with the operator's Git identity and DCO sign-off. From 00fcf99b7c56693a484b1f91e22d06c9439ecc23 Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 6 Nov 2026 18:03:31 +0800 Subject: [PATCH 05/14] docs(omniglove): record clean hardware CTS qualification Signed-off-by: zhouyingchun --- src/plugins/omniglove/README.md | 8 ++-- src/plugins/omniglove/VALIDATION.md | 60 ++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/plugins/omniglove/README.md b/src/plugins/omniglove/README.md index 57e0a78d3c..6619b418bd 100644 --- a/src/plugins/omniglove/README.md +++ b/src/plugins/omniglove/README.md @@ -74,11 +74,9 @@ and calibration evidence, see [VALIDATION.md](VALIDATION.md). ## Validation status The API probe and hand-conversion contract test run during installation. -The controller-wrist hardware take `omniglove_2026-11-06_110404` passed hand CTS -(39 pass, zero fail). It was recorded before the plugin was committed, so it -is development evidence; formal submission needs a take from the pushed final -commit. A separate take, `omniglove_2026-11-06_150705`, verified both hands' -functional OpenXR hot-unplug/recovery while the other hand remained active. +Controller-wrist hardware qualification passed hand CTS (39 pass, zero fail) +from a clean, pushed commit. Both hands' functional OpenXR hot-unplug/recovery +was verified while the other hand remained active. See [VALIDATION.md](VALIDATION.md) for the evidence and remaining steps. ## Formal recording and packaging diff --git a/src/plugins/omniglove/VALIDATION.md b/src/plugins/omniglove/VALIDATION.md index b98a245732..e9a0ce9076 100644 --- a/src/plugins/omniglove/VALIDATION.md +++ b/src/plugins/omniglove/VALIDATION.md @@ -22,7 +22,7 @@ submission have separate evidence. A pending item below is not a pass. | Wrist source qualified | `controller` | | Headset / controller model | PICO 4 Ultra / PICO 4 controllers, operator-confirmed; original take has no model fields | -## Hardware CTS evidence +## Development hardware CTS evidence - Take: `omniglove_2026-11-06_110404`, recorded at 11:04:04 UTC+08:00. - Verdict: **PASS**; 18,678 hand records, 39 passing checks, zero failures. @@ -36,30 +36,62 @@ submission have separate evidence. A pending item below is not a pass. - Raw MCAP, label sidecar, provenance and recorder log stay together outside Git. The submission archive uses the official CTS bundle builder and adds `report.json`, `report.txt`, per-input hashes and checker provenance. -- Manual replay review and final-commit recording remain pending. The report - establishes automated checks; it does not establish the operator's replay review. +- This take remains development evidence. Formal qualification uses the clean + recording below; the original provenance is retained. + +## Formal hardware CTS evidence + +- Take: `omniglove_2026-11-06_173720`, using PICO 4 Ultra / PICO 4 controllers. +- Captured and checked commit: `445cd3e1fc59aa327e120c2fb2514ea042189c57`, + pushed before recording, with capture and checker `dirty=false`. +- Verdict: **PASS**, 39 passing checks, zero failures, and one expected optional + joint geometry check with insufficient data. All 11 action windows have data. +- Left/right each have 9,307 records and 9,306 active records. The only absent + poses are at startup; no action window loses hand data. +- Fist straightness: left 0.581668, right 0.589687 (limit 0.600000). +- Table tilt, opening/closing windows: left 1.561° / 1.453°, + right 5.078° / 5.392° (limit 10°). +- Roll fingertip excursion: right 10.784 mm, left 14.634 mm (limit 30 mm). +- Installed plugin SHA-256: + `04e27c2300aca915b188925bb40050e5f082ccbfb9ec768e31a9d6ab2ebfe073`. +- The official viewer was visually sampled across all 11 action windows, + including multiple samples of each roll. Samples show the expected poses; + full-record continuity checks pass. Physical alignment was accepted by the + operator after the on-site review instructions; this is operator confirmation, + not an independent optical measurement. +- Official archive: `omniglove_2026-11-06_173720-hand.pass.zip`, 8,752,908 bytes; + SHA-256 `b28807100069aaf5d9daf09b777115b1e7b0962dc77764d5bb6058e9b4609d02`. + Archive integrity and all four original input hashes were verified. The + archive and original inputs remain outside Git. Form submission is pending. ## Tested wrist installation offsets -The independent successful take used these exact fixed transforms. Values +The formal successful take used these exact fixed transforms. Values are metres and quaternion `qx,qy,qz,qw` in controller aim coordinates. They apply to the tested mount, not every OmniGlove installation. ```bash export OMNIGLOVE_WRIST_SOURCE=controller export OMNIGLOVE_AIM_TO_WRIST_LEFT="0.044351775,-0.022869715,0.071703646,-0.122489204,-0.165170245,0.646336332,0.734822789" -export OMNIGLOVE_AIM_TO_WRIST_RIGHT="-0.044435690,-0.021819216,0.082681299,-0.130124469,0.161913518,-0.661683797,0.720434721" +export OMNIGLOVE_AIM_TO_WRIST_RIGHT="-0.044435690,-0.021819216,0.082681299,-0.158978409,0.193265088,-0.616438139,0.746577854" ``` The final left offset was estimated from the physical CTS action constraints in `omniglove_2026-11-06_104642`: table level and fingertip contact/roll, using held-out time blocks and excluding the final flat-hand window from fitting. The original take's offline reprojection passed 36 checks with zero failures; -the new independent take above then passed 39 checks. Only a fixed wrist +the independent development take `110404` then passed 39 checks. Only a fixed wrist installation transform was adjusted. Original recordings, labels, finger geometry and checker thresholds were not changed. This was geometric mount calibration; it was not a simultaneous optical-wrist measurement. +The right mount rotation was subsequently corrected by 7.727777° using table +normal and fingertip-contact constraints from takes `164904`, `165355` and +`170455`. Final flat windows and alternating 0.5-second blocks were excluded +from fitting; take `163816` was an independent holdout. Left offsets and right +translation stayed unchanged. Offline checks introduced no new failures, and +the independent formal take above passed with the applied right rotation. + For a new mount, obtain a trusted wrist reference and compute `T_aim_to_wrist = inverse(T_world_aim) * T_world_wrist` over multiple poses. Verify the result using independent positions, wrist rotations and physical @@ -81,8 +113,9 @@ OpenXR injector deactivation/recreation or long-term headset tracking. ## OpenXR hot-unplug/recovery evidence Take `omniglove_2026-11-06_150705` was recorded at pushed commit -`97c62090a97c5376cf8264ff8b90acd8f72a237d`, with `repo_dirty=false`, the same -controller offsets, PICO 4 Ultra and PICO 4 controllers. The complete MCAP +`97c62090a97c5376cf8264ff8b90acd8f72a237d`, with `repo_dirty=false`, the baseline +controller offsets before the right rotation correction, PICO 4 Ultra and PICO 4 +controllers. The complete MCAP contains 6,747 hand records per side over 81.94 seconds. Both hands became inactive downstream and recovered in the same process run: @@ -154,17 +187,18 @@ worker iteration. pre-commit verified against the final source. - [x] Commit the plugin with the operator's Git identity and DCO sign-off. - [x] Push to AgibotTech/IsaacCapture and open integration PR #1445. -- [ ] Record from that pushed commit with a clean checkout and unchanged CTS: +- [x] Record from the pushed commit with a clean checkout and unchanged CTS; + formal PASS take `173720` above. The recording command is: ```bash ./src/plugins/omniglove/record_cts.sh --wrist-config /path/to/mount.env \ --no-launch-cloudxr-runtime --note "headset=PICO 4 Ultra; controller=PICO 4 controllers" ``` -- [ ] Review the new take in `hand_cts.panel`; verify handedness, axes, - continuity and the physical wrist/finger alignment. -- [ ] Package with the viewer or `package_cts.py`; quote take, verdict, archive - size and SHA-256 prefix in the PR. +- [x] Review action-window samples in `hand_cts.panel`, with full-record + continuity/axis checks and operator-confirmed physical alignment. +- [x] Package with `package_cts.py`; take, verdict, archive size and SHA-256 + are recorded above and in the PR. - [ ] Upload via https://forms.gle/hLqAhNkae4bhAUX79, associating the archive with the PR. Keep the archive outside Git. - [ ] Resolve upstream CI and reviewer feedback. From d30eec82962a1587e5a14920cb487f64de44fe44 Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 6 Nov 2026 18:12:33 +0800 Subject: [PATCH 06/14] docs(omniglove): link device to AgiLink website Signed-off-by: zhouyingchun --- docs/source/_data/devices.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/_data/devices.yaml b/docs/source/_data/devices.yaml index a77dd5478d..0070c93979 100644 --- a/docs/source/_data/devices.yaml +++ b/docs/source/_data/devices.yaml @@ -275,7 +275,7 @@ devices: - id: agilink-omniglove name: AgiLink OmniGlove - url: https://github.com/AgibotTech/agilink_omniglove_sdk + url: https://www.agilink-ai.com group: peripheral modes: High-fidelity hand tracking since: main From c5c89bab012f658fcb2af521739d15c58727baba Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 6 Nov 2026 18:34:38 +0800 Subject: [PATCH 07/14] docs(omniglove): confirm CTS form submission Signed-off-by: zhouyingchun --- src/plugins/omniglove/VALIDATION.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/plugins/omniglove/VALIDATION.md b/src/plugins/omniglove/VALIDATION.md index e9a0ce9076..a6104f2353 100644 --- a/src/plugins/omniglove/VALIDATION.md +++ b/src/plugins/omniglove/VALIDATION.md @@ -62,7 +62,8 @@ submission have separate evidence. A pending item below is not a pass. - Official archive: `omniglove_2026-11-06_173720-hand.pass.zip`, 8,752,908 bytes; SHA-256 `b28807100069aaf5d9daf09b777115b1e7b0962dc77764d5bb6058e9b4609d02`. Archive integrity and all four original input hashes were verified. The - archive and original inputs remain outside Git. Form submission is pending. + archive and original inputs remain outside Git. The operator submitted the + CTS form for PR #1445 on 2026-11-06 and provided its response confirmation. ## Tested wrist installation offsets @@ -199,8 +200,8 @@ worker iteration. continuity/axis checks and operator-confirmed physical alignment. - [x] Package with `package_cts.py`; take, verdict, archive size and SHA-256 are recorded above and in the PR. -- [ ] Upload via https://forms.gle/hLqAhNkae4bhAUX79, associating the archive - with the PR. Keep the archive outside Git. +- [x] Submit via https://forms.gle/hLqAhNkae4bhAUX79 for PR #1445; + the operator provided the successful response confirmation. - [ ] Resolve upstream CI and reviewer feedback. Optical and Auto wrist modes remain unqualified by the controller take. From f604b964209ac93644689788dbe3b0236a54834f Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 6 Nov 2026 19:16:29 +0800 Subject: [PATCH 08/14] fix(omniglove): pin managed SDK checkout to validated commit Signed-off-by: zhouyingchun --- .../source/device/hand_tracking/omniglove.rst | 16 ++++-- src/plugins/omniglove/README.md | 23 ++++++--- src/plugins/omniglove/install.sh | 49 +++++++++++++++---- 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/docs/source/device/hand_tracking/omniglove.rst b/docs/source/device/hand_tracking/omniglove.rst index 80ffa9a4f4..77b89b33df 100644 --- a/docs/source/device/hand_tracking/omniglove.rst +++ b/docs/source/device/hand_tracking/omniglove.rst @@ -53,10 +53,18 @@ Install the USB rule on the host, then build in the Isaac Capture checkout: The build script uses a sibling ``agilink_omniglove_sdk`` checkout when one is available. Set ``OMNIGLOVE_SDK_ROOT`` or pass ``--sdk-root`` for a different -checkout. Otherwise it creates and updates a managed checkout. It validates -that SDK against its ``SHA256SUMS``, runs the API probe and conversion tests, -and installs the executable, manifest, runtime libraries and license notices -under ``install/plugins/omniglove``. +local checkout, which is not updated or reset. Otherwise it creates a managed +checkout pinned to SDK commit +``a2f899cd45c51ae01c9bbca9b893e4afba41c0d7``. Set ``OMNIGLOVE_SDK_REF`` or pass +``--sdk-ref REF`` to select another managed commit, tag, or branch and bypass +sibling discovery. Root and ref overrides cannot be combined. Tags and branches +are fetched on each install; use a full commit SHA for reproducibility. Managed +checkouts with local changes are rejected before fetching or switching commits. + +The script validates SDK files against that checkout's ``SHA256SUMS`` to detect +corruption; this does not authenticate a user-selected SDK. It runs the API +probe and conversion tests, and installs the executable, manifest, runtime +libraries and license notices under ``install/plugins/omniglove``. For an existing Python 3.12 build used by CTS: diff --git a/src/plugins/omniglove/README.md b/src/plugins/omniglove/README.md index 6619b418bd..4492666bd3 100644 --- a/src/plugins/omniglove/README.md +++ b/src/plugins/omniglove/README.md @@ -18,18 +18,27 @@ Install the host USB rule once, outside a container: ``` Then build the plugin. The installer uses a sibling -`../agilink_omniglove_sdk` checkout when present. Otherwise it clones the -current SDK from GitHub into `.sdk/`, or fast-forwards that managed checkout on -later runs. +`../agilink_omniglove_sdk` checkout when present. Otherwise it creates a managed +checkout in `.sdk/`, pinned to SDK commit +`a2f899cd45c51ae01c9bbca9b893e4afba41c0d7`. Repeated installs reuse that commit +without following upstream branch updates. A managed checkout with local +changes is rejected before fetching or switching commits. ```bash ./src/plugins/omniglove/install.sh ``` -Set `OMNIGLOVE_SDK_ROOT` or pass `--sdk-root` to use another SDK checkout. The -installer verifies that checkout with its own `SHA256SUMS`; it does not reset -the SDK to a historical tag. The SDK package configuration selects its shared -library, so a changed ABI version number alone is not a build blocker. Missing +Set `OMNIGLOVE_SDK_ROOT` or pass `--sdk-root` to use another local SDK checkout; +local checkouts are not updated or reset. Set `OMNIGLOVE_SDK_REF` or pass +`--sdk-ref REF` to select a different managed SDK commit, tag, or branch, +bypassing sibling checkout discovery. Root and ref overrides cannot be combined. +Tags and branches are fetched on each install; use a full commit SHA for a +reproducible override. + +The installer verifies SDK files against that checkout's `SHA256SUMS` to detect +corruption; this does not authenticate a user-selected SDK. The SDK package +configuration selects its shared library, so a changed ABI version number +alone is not a build blocker. Missing or changed required C APIs fail the compile/link compatibility probe. At runtime, a header/library ABI mismatch is rejected before device discovery. diff --git a/src/plugins/omniglove/install.sh b/src/plugins/omniglove/install.sh index 7aad81e791..2b5fbdc5e5 100755 --- a/src/plugins/omniglove/install.sh +++ b/src/plugins/omniglove/install.sh @@ -6,6 +6,7 @@ set -euo pipefail build_dir="" sdk_root="${OMNIGLOVE_SDK_ROOT:-}" +sdk_ref="${OMNIGLOVE_SDK_REF:-}" isaac_root="" while [[ $# -gt 0 ]]; do @@ -18,12 +19,19 @@ while [[ $# -gt 0 ]]; do sdk_root="${2:?--sdk-root requires a path}" shift 2 ;; + --sdk-ref) + sdk_ref="${2:?--sdk-ref requires a commit, tag, or branch}" + shift 2 + ;; -h | --help) cat <&2 + exit 1 +fi + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" isaac_root="${isaac_root:-$(cd "$script_dir/../../.." && pwd)}" build_dir="${build_dir:-$isaac_root/build}" @@ -50,20 +63,36 @@ if [[ "$build_dir" != /* ]]; then fi workspace_sdk="$isaac_root/../agilink_omniglove_sdk" -if [[ -z "$sdk_root" ]] && [[ -d "$workspace_sdk/.git" ]]; then +if [[ -z "$sdk_root" && -z "$sdk_ref" ]] && [[ -e "$workspace_sdk/.git" ]]; then sdk_root="$workspace_sdk" fi if [[ -z "$sdk_root" ]]; then sdk_root="$script_dir/.sdk/agilink_omniglove_sdk" - if [[ -d "$sdk_root/.git" ]]; then - echo "==> Updating managed OmniGlove SDK checkout" - git -C "$sdk_root" pull --ff-only - else - echo "==> Cloning latest OmniGlove SDK release repository" + # Pin managed dependencies: their own SHA256SUMS detects corruption, not + # authenticity. An upstream branch update must not silently change the SDK. + sdk_ref="${sdk_ref:-a2f899cd45c51ae01c9bbca9b893e4afba41c0d7}" + if [[ ! -e "$sdk_root/.git" ]]; then + if [[ -e "$sdk_root" ]]; then + echo "error: managed SDK path exists but is not a Git checkout: $sdk_root" >&2 + exit 1 + fi mkdir -p "$(dirname "$sdk_root")" - git clone --depth 1 https://github.com/AgibotTech/agilink_omniglove_sdk.git "$sdk_root" + git init --quiet "$sdk_root" + git -C "$sdk_root" remote add origin https://github.com/AgibotTech/agilink_omniglove_sdk.git + fi + if [[ -n "$(git -C "$sdk_root" status --porcelain)" ]]; then + echo "error: managed SDK checkout has local changes; preserve them before installing: $sdk_root" >&2 + exit 1 + fi + if [[ "$sdk_ref" =~ ^[0-9a-fA-F]{40}$ ]] && git -C "$sdk_root" cat-file -e "$sdk_ref^{commit}" 2>/dev/null; then + sdk_commit="$(git -C "$sdk_root" rev-parse "$sdk_ref^{commit}")" + else + git -C "$sdk_root" fetch --depth 1 -- origin "$sdk_ref" + sdk_commit="$(git -C "$sdk_root" rev-parse 'FETCH_HEAD^{commit}')" fi + echo "==> Using managed OmniGlove SDK $sdk_ref ($sdk_commit)" + git -C "$sdk_root" checkout --detach "$sdk_commit" -- fi sdk_root="$(cd "$sdk_root" && pwd)" @@ -98,7 +127,7 @@ done # library here. CMake and the compatibility probe reject missing required APIs. sdk_commit="unknown" -if [[ -d "$sdk_root/.git" ]]; then +if [[ -e "$sdk_root/.git" ]]; then sdk_commit="$(git -C "$sdk_root" rev-parse HEAD)" fi From 75da62fac7a1b0a6823a93dfc9936360d8f6afbe Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Wed, 11 Nov 2026 11:34:33 +0800 Subject: [PATCH 09/14] Refactor OmniGlove integration to address upstream review Signed-off-by: zhouyingchun --- CMakeLists.txt | 2 +- cmake/ClangFormat.cmake | 1 + .../source/device/hand_tracking/omniglove.rst | 84 +++---- src/plugins/omniglove/CMakeLists.txt | 54 +--- src/plugins/omniglove/README.md | 80 ++---- src/plugins/omniglove/VALIDATION.md | 209 ---------------- src/plugins/omniglove/app/main.cpp | 13 +- .../omniglove/compat_probe/CMakeLists.txt | 13 - src/plugins/omniglove/compat_probe/main.cpp | 43 ---- src/plugins/omniglove/core/CMakeLists.txt | 3 +- .../core/inc/omniglove/omniglove_plugin.hpp | 107 -------- .../omniglove_hand_conversion.hpp | 10 +- .../inc/omniglove_plugin/omniglove_plugin.hpp | 92 +++++++ .../core/omniglove_hand_conversion.cpp | 6 +- .../omniglove/core/omniglove_plugin.cpp | 233 +++++------------- src/plugins/omniglove/install.sh | 155 ------------ src/plugins/omniglove/record_cts.sh | 89 ------- src/plugins/omniglove/sdk.cmake | 80 ++++++ src/plugins/omniglove/tests/CMakeLists.txt | 7 - src/plugins/plugin_utils/CMakeLists.txt | 1 + .../inc/plugin_utils/wrist_pose_source.hpp | 9 + .../plugin_utils/wrist_source_config.cpp | 79 ++++++ src/plugins/wuji_glove/wuji_glove_plugin.cpp | 56 +---- .../isaaccapture/cts/hand/checker/README.md | 10 + .../cts/hand/checker/src/hand_cts/package.py} | 5 +- .../cts/hand/checker/tests/test_package.py | 75 ++++++ tests/cpp/plugins/CMakeLists.txt | 4 + tests/cpp/plugins/omniglove/CMakeLists.txt | 8 + .../omniglove/test_hand_conversion.cpp | 72 ++---- tests/cpp/plugins/plugin_utils/CMakeLists.txt | 1 + .../plugin_utils/test_wrist_source_config.cpp | 100 ++++++++ 31 files changed, 642 insertions(+), 1059 deletions(-) delete mode 100644 src/plugins/omniglove/VALIDATION.md delete mode 100644 src/plugins/omniglove/compat_probe/CMakeLists.txt delete mode 100644 src/plugins/omniglove/compat_probe/main.cpp delete mode 100644 src/plugins/omniglove/core/inc/omniglove/omniglove_plugin.hpp rename src/plugins/omniglove/core/inc/{omniglove => omniglove_plugin}/omniglove_hand_conversion.hpp (64%) create mode 100644 src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_plugin.hpp delete mode 100755 src/plugins/omniglove/install.sh delete mode 100755 src/plugins/omniglove/record_cts.sh create mode 100644 src/plugins/omniglove/sdk.cmake delete mode 100644 src/plugins/omniglove/tests/CMakeLists.txt create mode 100644 src/plugins/plugin_utils/wrist_source_config.cpp rename src/{plugins/omniglove/package_cts.py => python/isaaccapture/cts/hand/checker/src/hand_cts/package.py} (93%) create mode 100644 src/python/isaaccapture/cts/hand/checker/tests/test_package.py create mode 100644 tests/cpp/plugins/omniglove/CMakeLists.txt rename src/plugins/omniglove/tests/hand_conversion_contract_test.cpp => tests/cpp/plugins/omniglove/test_hand_conversion.cpp (71%) create mode 100644 tests/cpp/plugins/plugin_utils/test_wrist_source_config.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1dd21f022f..2b0d8c196b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,7 +70,7 @@ option(BUILD_PLUGIN_OAK_CAMERA "Build OAK camera plugin (requires vcpkg for Dept option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi SDK)" OFF) option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF) option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF) -option(BUILD_PLUGIN_OMNIGLOVE "Build AgiLink OmniGlove plugin (requires the Omniglove SDK)" OFF) +option(BUILD_PLUGIN_OMNIGLOVE "Build AgiLink OmniGlove plugin (fetches a pinned SDK)" OFF) option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF) option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) diff --git a/cmake/ClangFormat.cmake b/cmake/ClangFormat.cmake index b0500093e1..26bdb91ecd 100644 --- a/cmake/ClangFormat.cmake +++ b/cmake/ClangFormat.cmake @@ -38,6 +38,7 @@ set(_cf_patterns set(_cf_exclude_patterns ".*/build/.*" ".*/deps/.*" + ".*/\\.sdk/.*" ".*/manus/ManusSDK/.*" ".*/wuji_glove/wuji-sdk-c-.*" ".*/node_modules/.*" diff --git a/docs/source/device/hand_tracking/omniglove.rst b/docs/source/device/hand_tracking/omniglove.rst index 77b89b33df..050db9faa7 100644 --- a/docs/source/device/hand_tracking/omniglove.rst +++ b/docs/source/device/hand_tracking/omniglove.rst @@ -44,44 +44,38 @@ standard ``HandsSource``. Installation ------------ -Install the USB rule on the host, then build in the Isaac Capture checkout: +Install the USB rule on the host: .. code-block:: bash ./src/plugins/omniglove/install_udev_rules.sh - ./src/plugins/omniglove/install.sh -The build script uses a sibling ``agilink_omniglove_sdk`` checkout when one is -available. Set ``OMNIGLOVE_SDK_ROOT`` or pass ``--sdk-root`` for a different -local checkout, which is not updated or reset. Otherwise it creates a managed -checkout pinned to SDK commit -``a2f899cd45c51ae01c9bbca9b893e4afba41c0d7``. Set ``OMNIGLOVE_SDK_REF`` or pass -``--sdk-ref REF`` to select another managed commit, tag, or branch and bypass -sibling discovery. Root and ref overrides cannot be combined. Tags and branches -are fetched on each install; use a full commit SHA for reproducibility. Managed -checkouts with local changes are rejected before fetching or switching commits. - -The script validates SDK files against that checkout's ``SHA256SUMS`` to detect -corruption; this does not authenticate a user-selected SDK. It runs the API -probe and conversion tests, and installs the executable, manifest, runtime -libraries and license notices under ``install/plugins/omniglove``. - -For an existing Python 3.12 build used by CTS: +CMake retrieves the pinned SDK into the build directory and validates its +release checksums. The target architecture selects the SDK package. Runtime +libraries and license notices are installed with the plugin. .. code-block:: bash - ./src/plugins/omniglove/install.sh --build-dir build-py312 + cmake -S . -B out -DBUILD_PLUGIN_OMNIGLOVE=ON + cmake --build out + cmake --install out --component omniglove + +For an explicit local release, set +``-DOMNIGLOVE_SDK_ROOT=/path/to/agilink_omniglove_sdk``; CMake does not modify it. +``-DOMNIGLOVE_SDK_REF=`` selects another managed release. Use a +full commit SHA for a reproducible override. Source build options remain under +the user's control. Running ------- -Start CloudXR in one terminal and connect the headset: +Start CloudXR and connect the headset: .. code-block:: bash - python -m isaaccapture.cloudxr.service run + python -m isaaccapture.cloudxr.service start -In another terminal, source the runtime environment, select the wrist source +Source the runtime environment, select the wrist source as described below, and launch the installed plugin: .. code-block:: bash @@ -117,10 +111,9 @@ in metres followed by a unit quaternion in ``(x, y, z, w)`` order: export OMNIGLOVE_AIM_TO_WRIST_LEFT="px,py,pz,qx,qy,qz,qw" export OMNIGLOVE_AIM_TO_WRIST_RIGHT="px,py,pz,qx,qy,qz,qw" -These are format placeholders; replace them with measured values. Controller -and Auto mode reject missing transforms. The tested mount's offsets and -calibration evidence are in the :code-file:`validation record -`; those values apply only to that mount. +These are format placeholders; replace them with measured values for the +current mount. Controller and Auto mode reject missing transforms. Unknown +modes and invalid offsets are errors. Optical mode does not read the offsets. Published collections and joint contract ---------------------------------------- @@ -134,7 +127,8 @@ the host records them as ``core.HandPoseRecord`` from ``hand.fbs``. - Palm and the four non-thumb metacarpals remain invalid because the SDK does not supply them. No neighbouring joint is copied into an absent slot. - Valid joints have both ``POSITION_VALID`` and ``ORIENTATION_VALID`` set. - Tracking bits additionally follow the validity of the world-space wrist. + Tracking bits additionally follow ``wrist.tracked``. A valid but untracked + wrist is still published with the tracking bits cleared. - Positions are metres and orientations are unit ``(x, y, z, w)`` quaternions, in the base space passed to ``HandInjector``. - OpenXR joint ``+Z`` points away from the fingertip and ``+Y`` toward the @@ -151,35 +145,17 @@ the host records them as ``core.HandPoseRecord`` from ``hand.fbs``. The plugin publishes the operator's skeleton. It emits no robot joint commands or hand ``JointStateOutput`` data and defines no vendor hand schema. -Validation and CTS ------------------- - -.. Keep dated take results and acceptance status in the plugin validation record. - -See the :code-file:`validation record ` -for hardware test evidence and acceptance status. - -Follow :doc:`/conformance/setup` to prepare CTS. From a clean plugin commit -already pushed to the integration PR, load the mount configuration and record: - -.. code-block:: bash - - ./src/plugins/omniglove/record_cts.sh --wrist-config /path/to/mount.env - -Use ``--no-launch-cloudxr-runtime`` if CloudXR is already running. For -development takes, ``--allow-dirty`` retains the recorder's dirty provenance. -Use ``--note`` to record the headset and controller models. - -Review the take in the official panel, then package it using the panel's -Submission button or the wrapper that uses the same bundle builder: - -.. code-block:: bash +Device status and CTS +--------------------- - src/python/isaaccapture/cts/hand/checker/.venv/bin/python -m hand_cts.panel /path/to/take-hand.mcap - src/python/isaaccapture/cts/hand/checker/.venv/bin/python src/plugins/omniglove/package_cts.py /path/to/take-hand.mcap +The shared device status publisher reports left and right hand availability +under the configured plugin root. Stale SDK frames report a disconnected hand; +a fresh frame without a usable wrist reports a degraded hand. A discovered +glove that fails stream configuration or startup stops the plugin with an error. -Follow :doc:`/conformance/submit` to upload the archive and associate it with -the integration PR. Keep the archive outside the Git repository. +Follow :doc:`/conformance/setup`, :doc:`/conformance/hand` and +:doc:`/conformance/submit` for recording, review and submission. Keep the take +summary and hardware evidence in the integration PR description. Troubleshooting --------------- diff --git a/src/plugins/omniglove/CMakeLists.txt b/src/plugins/omniglove/CMakeLists.txt index d3c178bc2c..1944ca1d6d 100644 --- a/src/plugins/omniglove/CMakeLists.txt +++ b/src/plugins/omniglove/CMakeLists.txt @@ -2,60 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 include(GNUInstallDirs) - -find_package(OmnigloveSDK CONFIG REQUIRED) - -if(NOT TARGET OmnigloveSDK::api) - message(FATAL_ERROR "OmnigloveSDK package did not define OmnigloveSDK::api") -endif() - -get_target_property(OMNIGLOVE_SDK_LIBRARY OmnigloveSDK::api IMPORTED_LOCATION) -if(NOT OMNIGLOVE_SDK_LIBRARY OR NOT EXISTS "${OMNIGLOVE_SDK_LIBRARY}") - message(FATAL_ERROR "OmnigloveSDK::api has no usable IMPORTED_LOCATION") -endif() - -if(NOT OMNIGLOVE_SDK_RELEASE_ROOT) - get_filename_component(_omniglove_platform_root "${OmnigloveSDK_DIR}/.." ABSOLUTE) - get_filename_component(_omniglove_release_candidate "${_omniglove_platform_root}/.." ABSOLUTE) - if(EXISTS "${_omniglove_release_candidate}/LICENSE") - set(OMNIGLOVE_SDK_RELEASE_ROOT "${_omniglove_release_candidate}") - endif() -endif() - -if(OMNIGLOVE_SDK_RELEASE_ROOT AND - (NOT OMNIGLOVE_SDK_GIT_COMMIT OR OMNIGLOVE_SDK_GIT_COMMIT STREQUAL "unknown")) - find_package(Git QUIET) - if(GIT_FOUND) - # A release archive inside another repository must not inherit its commit. - execute_process( - COMMAND "${GIT_EXECUTABLE}" -C "${OMNIGLOVE_SDK_RELEASE_ROOT}" rev-parse --show-toplevel - OUTPUT_VARIABLE _omniglove_git_root - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) - file(REAL_PATH "${OMNIGLOVE_SDK_RELEASE_ROOT}" _omniglove_release_root) - if(_omniglove_git_root STREQUAL _omniglove_release_root) - execute_process( - COMMAND "${GIT_EXECUTABLE}" -C "${OMNIGLOVE_SDK_RELEASE_ROOT}" rev-parse HEAD - RESULT_VARIABLE _omniglove_git_result - OUTPUT_VARIABLE _omniglove_git_commit - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) - if(_omniglove_git_result EQUAL 0) - set(OMNIGLOVE_SDK_GIT_COMMIT "${_omniglove_git_commit}") - endif() - endif() - endif() -endif() +include("${CMAKE_CURRENT_LIST_DIR}/sdk.cmake") add_subdirectory(core) add_subdirectory(app) -add_subdirectory(compat_probe) - -if(BUILD_TESTING) - add_subdirectory(tests) -endif() get_filename_component(_omniglove_sdk_library_dir "${OMNIGLOVE_SDK_LIBRARY}" DIRECTORY) file(GLOB _omniglove_sdk_libraries "${_omniglove_sdk_library_dir}/libomniglove.so*") @@ -100,7 +50,7 @@ else() message(WARNING "OMNIGLOVE_SDK_RELEASE_ROOT is unset; SDK license files will not be installed") endif() -install(FILES plugin.yaml README.md VALIDATION.md 60-omniglove-usb.rules +install(FILES plugin.yaml README.md 60-omniglove-usb.rules DESTINATION plugins/omniglove COMPONENT omniglove ) diff --git a/src/plugins/omniglove/README.md b/src/plugins/omniglove/README.md index 4492666bd3..6967809ea2 100644 --- a/src/plugins/omniglove/README.md +++ b/src/plugins/omniglove/README.md @@ -17,37 +17,30 @@ Install the host USB rule once, outside a container: ./src/plugins/omniglove/install_udev_rules.sh ``` -Then build the plugin. The installer uses a sibling -`../agilink_omniglove_sdk` checkout when present. Otherwise it creates a managed -checkout in `.sdk/`, pinned to SDK commit -`a2f899cd45c51ae01c9bbca9b893e4afba41c0d7`. Repeated installs reuse that commit -without following upstream branch updates. A managed checkout with local -changes is rejected before fetching or switching commits. +Configure and build from the Isaac Capture repository root: ```bash -./src/plugins/omniglove/install.sh +cmake -S . -B out -DBUILD_PLUGIN_OMNIGLOVE=ON +cmake --build out +cmake --install out --component omniglove ``` -Set `OMNIGLOVE_SDK_ROOT` or pass `--sdk-root` to use another local SDK checkout; -local checkouts are not updated or reset. Set `OMNIGLOVE_SDK_REF` or pass -`--sdk-ref REF` to select a different managed SDK commit, tag, or branch, -bypassing sibling checkout discovery. Root and ref overrides cannot be combined. -Tags and branches are fetched on each install; use a full commit SHA for a -reproducible override. +CMake retrieves a pinned SDK release into the build directory, verifies its +checksums, and selects the package for the target architecture. It installs +runtime libraries and license notices with the plugin. Set +`-DOMNIGLOVE_SDK_ROOT=/path/to/agilink_omniglove_sdk` to explicitly use a local +release without modifying it, or `-DOMNIGLOVE_SDK_REF=` to select +a managed release. A full commit SHA makes an override reproducible. -The installer verifies SDK files against that checkout's `SHA256SUMS` to detect -corruption; this does not authenticate a user-selected SDK. The SDK package -configuration selects its shared library, so a changed ABI version number -alone is not a build blocker. Missing -or changed required C APIs fail the compile/link compatibility probe. At -runtime, a header/library ABI mismatch is rejected before device discovery. +Missing or changed SDK APIs fail compilation or linking. At runtime, a +header/library ABI mismatch is rejected before device discovery. ## Run Start CloudXR, load its environment, connect a calibrated glove, and run: ```bash -python -m isaaccapture.cloudxr.service run +python -m isaaccapture.cloudxr.service start source ~/.cloudxr/run/cloudxr.env ./install/plugins/omniglove/omniglove_plugin ``` @@ -64,9 +57,9 @@ export OMNIGLOVE_AIM_TO_WRIST_LEFT="px,py,pz,qx,qy,qz,qw" export OMNIGLOVE_AIM_TO_WRIST_RIGHT="px,py,pz,qx,qy,qz,qw" ``` -Replace these format placeholders with measured offsets. Controller and Auto -mode reject missing transforms. For the tested mount's actual configuration -and calibration evidence, see [VALIDATION.md](VALIDATION.md). +Replace these placeholders with measured offsets for the current mount. +Controller and Auto modes reject missing transforms. Unknown wrist modes and +malformed offsets are errors; optical mode does not read the offsets. ## Data contract @@ -80,37 +73,14 @@ and calibration evidence, see [VALIDATION.md](VALIDATION.md). - Unmapped, unconverged, invalid-bone-model, non-finite, zero-quaternion, and stale frames are not injected. -## Validation status +## Device status and CTS -The API probe and hand-conversion contract test run during installation. -Controller-wrist hardware qualification passed hand CTS (39 pass, zero fail) -from a clean, pushed commit. Both hands' functional OpenXR hot-unplug/recovery -was verified while the other hand remained active. -See [VALIDATION.md](VALIDATION.md) for the evidence and remaining steps. +The plugin publishes both hand states through the shared device status +publisher under the supplied `--plugin-root-id`. A stale SDK frame makes a +hand disconnected; a fresh frame without a usable wrist pose makes it degraded. +A glove that is discovered but fails stream configuration or startup stops the +plugin with an error. -## Formal recording and packaging - -After committing and pushing the plugin, record from a clean checkout: - -```bash -./src/plugins/omniglove/record_cts.sh --wrist-config /path/to/mount.env -``` - -The wrapper rejects uncommitted/unpushed code and changes to the official CTS. -It records the installed binary's SHA-256 and the wrist configuration in -provenance notes. `--allow-dirty` enables development takes with their original -dirty provenance. `--note` records the headset and controller models; use -`--no-launch-cloudxr-runtime` when CloudXR is already running. - -Review the take with the official viewer and package it with the same builder: - -```bash -src/python/isaaccapture/cts/hand/checker/.venv/bin/python -m hand_cts.panel /path/to/take-hand.mcap -src/python/isaaccapture/cts/hand/checker/.venv/bin/python src/plugins/omniglove/package_cts.py /path/to/take-hand.mcap -``` - -Packaging reruns all official checks, preserves the original MCAP, labels, -provenance and log, checks their hashes, and prints the archive hash. Existing -archives are not overwritten. Upload the archive through the -[CTS form](https://forms.gle/hLqAhNkae4bhAUX79) with the integration PR URL; -keep recordings and archives outside Git. +Follow the official [hand CTS procedure](https://nvidia.github.io/IsaacCapture/main/conformance/hand.html) +and [submission procedure](https://nvidia.github.io/IsaacCapture/main/conformance/submit.html). +Keep take results and hardware evidence in the integration PR description. diff --git a/src/plugins/omniglove/VALIDATION.md b/src/plugins/omniglove/VALIDATION.md deleted file mode 100644 index a6104f2353..0000000000 --- a/src/plugins/omniglove/VALIDATION.md +++ /dev/null @@ -1,209 +0,0 @@ - - -# OmniGlove integration validation - -Updated: 2026-11-06 (Asia/Shanghai). Hardware qualification and formal upstream -submission have separate evidence. A pending item below is not a pass. - -## Recorded hardware baseline - -| Item | Evidence | -| --- | --- | -| Isaac Capture baseline | `71f22fd93c0bdff871e731d66b1d10589b2e3aeb` | -| Plugin | `omniglove_plugin`, manifest version `1.0.0` | -| SDK release / commit | `1.0.0` / `a2f899cd45c51ae01c9bbca9b893e4afba41c0d7` | -| SDK header / runtime ABI | Both `6.0.0` | -| Platform tested | Linux x86_64; aarch64 supported by the build, not hardware-tested here | -| USB devices | Left `cafe:4001`, right `cafe:4002` | -| SDK stream | POST_DATA, DIRECT, nominal 125 Hz | -| Wrist source qualified | `controller` | -| Headset / controller model | PICO 4 Ultra / PICO 4 controllers, operator-confirmed; original take has no model fields | - -## Development hardware CTS evidence - -- Take: `omniglove_2026-11-06_110404`, recorded at 11:04:04 UTC+08:00. -- Verdict: **PASS**; 18,678 hand records, 39 passing checks, zero failures. -- `optional_joints.geometry_consistent`: insufficient data, expected because - the five optional joints are invalid. The overall verdict is pass. -- All 11 action windows were backed by frames. Left/right each had 9,339 - records, including 9,337 active records. -- Captured commit: the baseline above, with `repo_dirty=true`. The plugin was - staged but not committed; this is development evidence. Do not edit its - provenance to claim a later commit. -- Raw MCAP, label sidecar, provenance and recorder log stay together outside - Git. The submission archive uses the official CTS bundle builder and adds - `report.json`, `report.txt`, per-input hashes and checker provenance. -- This take remains development evidence. Formal qualification uses the clean - recording below; the original provenance is retained. - -## Formal hardware CTS evidence - -- Take: `omniglove_2026-11-06_173720`, using PICO 4 Ultra / PICO 4 controllers. -- Captured and checked commit: `445cd3e1fc59aa327e120c2fb2514ea042189c57`, - pushed before recording, with capture and checker `dirty=false`. -- Verdict: **PASS**, 39 passing checks, zero failures, and one expected optional - joint geometry check with insufficient data. All 11 action windows have data. -- Left/right each have 9,307 records and 9,306 active records. The only absent - poses are at startup; no action window loses hand data. -- Fist straightness: left 0.581668, right 0.589687 (limit 0.600000). -- Table tilt, opening/closing windows: left 1.561° / 1.453°, - right 5.078° / 5.392° (limit 10°). -- Roll fingertip excursion: right 10.784 mm, left 14.634 mm (limit 30 mm). -- Installed plugin SHA-256: - `04e27c2300aca915b188925bb40050e5f082ccbfb9ec768e31a9d6ab2ebfe073`. -- The official viewer was visually sampled across all 11 action windows, - including multiple samples of each roll. Samples show the expected poses; - full-record continuity checks pass. Physical alignment was accepted by the - operator after the on-site review instructions; this is operator confirmation, - not an independent optical measurement. -- Official archive: `omniglove_2026-11-06_173720-hand.pass.zip`, 8,752,908 bytes; - SHA-256 `b28807100069aaf5d9daf09b777115b1e7b0962dc77764d5bb6058e9b4609d02`. - Archive integrity and all four original input hashes were verified. The - archive and original inputs remain outside Git. The operator submitted the - CTS form for PR #1445 on 2026-11-06 and provided its response confirmation. - -## Tested wrist installation offsets - -The formal successful take used these exact fixed transforms. Values -are metres and quaternion `qx,qy,qz,qw` in controller aim coordinates. -They apply to the tested mount, not every OmniGlove installation. - -```bash -export OMNIGLOVE_WRIST_SOURCE=controller -export OMNIGLOVE_AIM_TO_WRIST_LEFT="0.044351775,-0.022869715,0.071703646,-0.122489204,-0.165170245,0.646336332,0.734822789" -export OMNIGLOVE_AIM_TO_WRIST_RIGHT="-0.044435690,-0.021819216,0.082681299,-0.158978409,0.193265088,-0.616438139,0.746577854" -``` - -The final left offset was estimated from the physical CTS action constraints -in `omniglove_2026-11-06_104642`: table level and fingertip contact/roll, using -held-out time blocks and excluding the final flat-hand window from fitting. -The original take's offline reprojection passed 36 checks with zero failures; -the independent development take `110404` then passed 39 checks. Only a fixed wrist -installation transform was adjusted. Original recordings, labels, finger -geometry and checker thresholds were not changed. This was geometric mount -calibration; it was not a simultaneous optical-wrist measurement. - -The right mount rotation was subsequently corrected by 7.727777° using table -normal and fingertip-contact constraints from takes `164904`, `165355` and -`170455`. Final flat windows and alternating 0.5-second blocks were excluded -from fitting; take `163816` was an independent holdout. Left offsets and right -translation stayed unchanged. Offline checks introduced no new failures, and -the independent formal take above passed with the applied right rotation. - -For a new mount, obtain a trusted wrist reference and compute -`T_aim_to_wrist = inverse(T_world_aim) * T_world_wrist` over multiple poses. -Verify the result using independent positions, wrist rotations and physical -contact, then record a fresh CTS take. - -## Other existing evidence - -The 2026-11-01 handover records: - -- Left and right USB identification and individually valid 21-joint conversion. -- Five-minute dual-hand SDK streaming: 37,501 frames per hand, about 125 Hz; - zero sequence gaps, source errors, malformed frames and callback drops. -- SDK hot-unplug/reconnect, tested with the right glove representing the same - hardware/firmware pair: automatic recovery and connection generation 1→3. - -These historical results establish the USB/SDK layer. They do not establish -OpenXR injector deactivation/recreation or long-term headset tracking. - -## OpenXR hot-unplug/recovery evidence - -Take `omniglove_2026-11-06_150705` was recorded at pushed commit -`97c62090a97c5376cf8264ff8b90acd8f72a237d`, with `repo_dirty=false`, the baseline -controller offsets before the right rotation correction, PICO 4 Ultra and PICO 4 -controllers. The complete MCAP -contains 6,747 hand records per side over 81.94 seconds. - -Both hands became inactive downstream and recovered in the same process run: - -| Inactive hand | Downstream inactive interval | Other hand during that interval | Recovery generation | -| --- | --- | --- | --- | -| Left | 22.53 s | Right: 1,875/1,875 records with 21 valid joints; max record gap 14.55 ms | 1→3 | -| Right, main interval | 25.50 s | Left: 2,096/2,096 records with 21 valid joints; max record gap 15.38 ms | 3→5 | - -The other hand's wrist-relative fingertip displacement reached 100.79 mm -during the left interval and 125.54 mm during the right interval, establishing -changing finger data. Both recovered hands also continued updating. -The operator waited before reconnecting; inactive interval lengths include -that wait and do not measure latency after reinsertion. - -An additional right-hand inactive interval lasted 0.18 s before the main -interval, with generation 1→3 on recovery. Its physical cause is unknown; -the left hand remained active throughout. This event is retained in the evidence. - -Functional hot-unplug/recovery is verified. Exact 200 ms cutoff and reinsertion -latency were not measured because SDK sample and physical unplug/reinsert -timestamps are absent. Downstream inactivity is encoded by an absent -`HandPoseRecord.data` payload, not a separate `is_active` field. -The take has no action labels and is separate from formal CTS evidence. - -## OpenXR hot-unplug/recovery procedure - -Keep the final wrist configuration and both gloves connected, connect the -headset, and start a session consuming the two standard hand collections. -Capture the plugin log and consumer recording as a separate hardware test. -Do not use unplug segments as CTS action labels. - -1. Confirm both hands are active and their fingers are changing. -2. Unplug the left glove and wait at least two seconds. Record the physical - unplug time and the last SDK sample time. -3. Check the left injector deactivates after 200 ms without a fresh sample - (allow one worker scheduling interval), that the consumer reports the left - hand inactive, and that the right hand continues changing. -4. Reconnect the left glove without restarting any process. Check that the - left hand becomes active, its connection generation increases, and fingers - update from new samples. -5. Repeat for the right glove and check the left remains active. -6. Save timestamps, the consumer's active flags and the plugin log. A log - transition alone does not establish what the downstream consumer saw. - -The plugin now logs `left/right hand inactive: SDK frame stale or invalid`, -`hand inactive: wrist pose unavailable`, and `hand active` with connection -generation/source sequence. Logs occur on injector transitions, not every -worker iteration. - -## Software verification on 2026-11-06 - -- Installer: selected local SDK checksum verification, build and installation passed. -- CTest: `omniglove_api_compat_probe` and `omniglove_hand_conversion_contract_test` passed. -- All plugin C++ sources/headers passed clang-format 14. -- Strict Sphinx `make current-docs` passed with no warnings. -- Full pre-commit passed with `SKIP=check-copyright-year`, matching CI. -- Controller/Auto with missing mount offsets failed before opening USB/XR. -- Recorder smoke checks verified clean/upstream guards, unchanged CTS, - exported offsets, binary hash and preservation of operator notes. -- Existing take packaging passed archive integrity and per-input hash checks. - -## Final submission checklist - -- [x] Confirm headset and controller models: PICO 4 Ultra / PICO 4 controllers. -- [x] Functional OpenXR hot-unplug/recovery evidence for both hands; timing - measurement limits and the extra right-hand transition recorded above. -- [x] Software build, conversion/ABI tests, C++ format, strict docs and full - pre-commit verified against the final source. -- [x] Commit the plugin with the operator's Git identity and DCO sign-off. -- [x] Push to AgibotTech/IsaacCapture and open integration PR #1445. -- [x] Record from the pushed commit with a clean checkout and unchanged CTS; - formal PASS take `173720` above. The recording command is: - -```bash -./src/plugins/omniglove/record_cts.sh --wrist-config /path/to/mount.env \ - --no-launch-cloudxr-runtime --note "headset=PICO 4 Ultra; controller=PICO 4 controllers" -``` - -- [x] Review action-window samples in `hand_cts.panel`, with full-record - continuity/axis checks and operator-confirmed physical alignment. -- [x] Package with `package_cts.py`; take, verdict, archive size and SHA-256 - are recorded above and in the PR. -- [x] Submit via https://forms.gle/hLqAhNkae4bhAUX79 for PR #1445; - the operator provided the successful response confirmation. -- [ ] Resolve upstream CI and reviewer feedback. - -Optical and Auto wrist modes remain unqualified by the controller take. -Repeat hardware tests after changing the SDK, firmware, skeleton conversion -or physical mount. diff --git a/src/plugins/omniglove/app/main.cpp b/src/plugins/omniglove/app/main.cpp index fc34826b4b..3e2b930628 100644 --- a/src/plugins/omniglove/app/main.cpp +++ b/src/plugins/omniglove/app/main.cpp @@ -2,13 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 #include -#include +#include #include #include #include #include -#include #include #include @@ -46,13 +45,13 @@ try std::signal(SIGINT, signal_handler); std::signal(SIGTERM, signal_handler); - auto logger = isaaccapture::Logger::get("isaaccapture.plugins.omniglove.main"); - auto plugin = std::make_unique(plugin_root_id); - while (!stop_requested.load(std::memory_order_relaxed) && plugin->is_running()) + plugins::omniglove::OmniglovePlugin plugin(plugin_root_id); + while (!stop_requested.load(std::memory_order_relaxed)) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + plugin.update(); + std::this_thread::sleep_for(std::chrono::milliseconds(2)); } - return plugin->has_failed() ? 1 : 0; + return 0; } catch (const std::exception& error) { diff --git a/src/plugins/omniglove/compat_probe/CMakeLists.txt b/src/plugins/omniglove/compat_probe/CMakeLists.txt deleted file mode 100644 index 152e1d9731..0000000000 --- a/src/plugins/omniglove/compat_probe/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. -# SPDX-License-Identifier: Apache-2.0 - -add_executable(omniglove_api_compat_probe main.cpp) -target_link_libraries(omniglove_api_compat_probe PRIVATE OmnigloveSDK::api) - -set_target_properties(omniglove_api_compat_probe PROPERTIES - BUILD_RPATH "$;$/omniglove" -) - -if(BUILD_TESTING) - add_test(NAME omniglove_api_compat_probe COMMAND omniglove_api_compat_probe) -endif() diff --git a/src/plugins/omniglove/compat_probe/main.cpp b/src/plugins/omniglove/compat_probe/main.cpp deleted file mode 100644 index 25d3480a05..0000000000 --- a/src/plugins/omniglove/compat_probe/main.cpp +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include -#include - -namespace -{ - -void callback(const omniglove_stream_sample_t*, void*) -{ -} - -} // namespace - -int main() -{ - omniglove_stream_config_t config{}; - config.struct_size = sizeof(config); - config.kind = OMNIGLOVE_STREAM_POST_DATA; - config.stream_mode = OMNIGLOVE_STREAM_MODE_DEFAULT; - config.output_mode = OMNIGLOVE_STREAM_OUTPUT_DIRECT; - config.source_interval_ms = 8; - config.source_timeout_ms = 200; - config.callback = callback; - config.callback_queue_capacity = 64; - - omniglove_post_data_t post{}; - post.thumb.landmarks.cmc.position_m[0] = 0.0F; - post.index.landmarks.mcp.quaternion_xyzw[3] = 1.0F; - post.wrist.landmark.quaternion_xyzw[3] = 1.0F; - - static_cast(config); - static_cast(post); - - const uint32_t runtime = omniglove_abi_version(); - std::cout << "Omniglove header ABI " << OMNIGLOVE_ABI_VERSION_MAJOR << '.' << OMNIGLOVE_ABI_VERSION_MINOR << '.' - << OMNIGLOVE_ABI_VERSION_PATCH << ", runtime ABI " << ((runtime >> 16U) & 0xffU) << '.' - << ((runtime >> 8U) & 0xffU) << '.' << (runtime & 0xffU) << '\n'; - return runtime == OMNIGLOVE_ABI_VERSION ? 0 : 1; -} diff --git a/src/plugins/omniglove/core/CMakeLists.txt b/src/plugins/omniglove/core/CMakeLists.txt index abb6e327d6..39f8ad4433 100644 --- a/src/plugins/omniglove/core/CMakeLists.txt +++ b/src/plugins/omniglove/core/CMakeLists.txt @@ -12,8 +12,9 @@ target_include_directories(omniglove_plugin_core ) target_link_libraries(omniglove_plugin_core - PUBLIC + PRIVATE OmnigloveSDK::api + PUBLIC Teleop::plugin_utils deviceio::deviceio_session deviceio::deviceio_trackers diff --git a/src/plugins/omniglove/core/inc/omniglove/omniglove_plugin.hpp b/src/plugins/omniglove/core/inc/omniglove/omniglove_plugin.hpp deleted file mode 100644 index fdacedf67f..0000000000 --- a/src/plugins/omniglove/core/inc/omniglove/omniglove_plugin.hpp +++ /dev/null @@ -1,107 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. -// SPDX-License-Identifier: Apache-2.0 - -#pragma once - -#include "omniglove_hand_conversion.hpp" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace plugins::omniglove -{ - -class OmniglovePlugin -{ -public: - explicit OmniglovePlugin(const std::string& plugin_root_id); - ~OmniglovePlugin(); - - bool is_running() const noexcept; - bool has_failed() const noexcept; - - OmniglovePlugin(const OmniglovePlugin&) = delete; - OmniglovePlugin& operator=(const OmniglovePlugin&) = delete; - OmniglovePlugin(OmniglovePlugin&&) = delete; - OmniglovePlugin& operator=(OmniglovePlugin&&) = delete; - -private: - struct HandFrame - { - std::array joints{}; - uint64_t sample_monotonic_ns = 0; - uint64_t connection_generation = 0; - uint64_t source_sequence = 0; - bool valid = false; - }; - - struct StreamContext - { - OmniglovePlugin* plugin = nullptr; - bool is_left = false; - }; - - struct DeviceStream - { - omniglove_device_t* device = nullptr; - StreamContext context; - bool started = false; - }; - - struct SessionDeleter - { - void operator()(omniglove_session_t* session) const noexcept; - }; - - static void stream_callback(const omniglove_stream_sample_t* sample, void* user_data); - - void initialize_sdk(); - void start_device_streams(); - void on_sample(const omniglove_stream_sample_t& sample, bool is_left); - void invalidate_hand(bool is_left); - void worker_thread(); - void pump_hand(std::unique_ptr& injector, - XrHandEXT hand, - const HandFrame& frame, - uint64_t& last_pushed_ns); - - std::shared_ptr m_logger = - isaaccapture::Logger::get("isaaccapture.plugins.omniglove.OmniglovePlugin"); - - std::shared_ptr m_session; - std::unique_ptr m_deviceio_session; - std::unique_ptr m_wrist_source; - std::unique_ptr m_left_injector; - std::unique_ptr m_right_injector; - std::unique_ptr m_time_converter; - - // The SDK session is declared after callback contexts so it stops and - // joins callback workers before those contexts are destroyed. - std::vector> m_streams; - std::unique_ptr m_sdk_session; - - std::mutex m_frame_mutex; - HandFrame m_left; - HandFrame m_right; - uint64_t m_left_last_pushed_ns = 0; - uint64_t m_right_last_pushed_ns = 0; - - std::thread m_worker; - std::atomic m_running{ false }; - std::atomic m_failed{ false }; -}; - -} // namespace plugins::omniglove diff --git a/src/plugins/omniglove/core/inc/omniglove/omniglove_hand_conversion.hpp b/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_hand_conversion.hpp similarity index 64% rename from src/plugins/omniglove/core/inc/omniglove/omniglove_hand_conversion.hpp rename to src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_hand_conversion.hpp index 2594e60789..1ba1ad4114 100644 --- a/src/plugins/omniglove/core/inc/omniglove/omniglove_hand_conversion.hpp +++ b/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_hand_conversion.hpp @@ -3,14 +3,20 @@ #pragma once -#include #include #include +#include + +struct omniglove_post_data; namespace plugins::omniglove { +// SDK STREAM_DATA_V1.md defines these POST_DATA bits; its C header has no names. +inline constexpr uint32_t kPostIkConverged = 1U << 0U; +inline constexpr uint32_t kPostBoneModelValid = 1U << 2U; + enum class ConversionStatus { Success, @@ -20,7 +26,7 @@ enum class ConversionStatus InvalidPose, }; -ConversionStatus convert_post_data(const omniglove_post_data_t& post, +ConversionStatus convert_post_data(const omniglove_post_data& post, bool is_left, std::array& joints); diff --git a/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_plugin.hpp b/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_plugin.hpp new file mode 100644 index 0000000000..07b7b256e0 --- /dev/null +++ b/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_plugin.hpp @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "omniglove_hand_conversion.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +struct omniglove_session; +struct omniglove_stream_sample; + +namespace plugins::omniglove +{ + +class OmniglovePlugin +{ +public: + explicit OmniglovePlugin(const std::string& plugin_root_id); + ~OmniglovePlugin(); + + void update(); + +private: + struct HandFrame + { + std::array joints{}; + int64_t sample_monotonic_ns = 0; + uint64_t connection_generation = 0; + uint64_t source_sequence = 0; + }; + + struct HandState + { + std::optional frame; + std::unique_ptr injector; + int64_t last_pushed_ns = 0; + core::PluginDeviceState state = core::PluginDeviceState_DISCONNECTED; + core::PluginDeviceReason reason = core::PluginDeviceReason_NO_HARDWARE_SIGNAL; + bool sample_error_logged = false; + }; + + struct StreamContext + { + OmniglovePlugin* plugin; + bool is_left; + }; + + struct SessionDeleter + { + void operator()(omniglove_session* session) const noexcept; + }; + + static void stream_callback(const omniglove_stream_sample* sample, void* user_data); + + void initialize_sdk(); + void start_device_streams(); + void on_sample(const omniglove_stream_sample& sample, bool is_left); + void invalidate_hand(bool is_left); + void pump_hand(HandState& hand, bool is_left, const std::optional& frame); + + std::shared_ptr m_logger = + isaaccapture::Logger::get("isaaccapture.plugins.omniglove.OmniglovePlugin"); + + std::unique_ptr m_session; + std::unique_ptr m_deviceio_session; + std::unique_ptr m_wrist_source; + std::optional m_time_converter; + std::optional m_status_publisher; + + std::mutex m_frame_mutex; + std::array m_hands; + std::vector> m_streams; + // Destroy the SDK first: its callbacks borrow the contexts, frames and mutex above. + std::unique_ptr m_sdk_session; +}; + +} // namespace plugins::omniglove diff --git a/src/plugins/omniglove/core/omniglove_hand_conversion.cpp b/src/plugins/omniglove/core/omniglove_hand_conversion.cpp index 0906aaf8fc..98527e9a1c 100644 --- a/src/plugins/omniglove/core/omniglove_hand_conversion.cpp +++ b/src/plugins/omniglove/core/omniglove_hand_conversion.cpp @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. // SPDX-License-Identifier: Apache-2.0 -#include "inc/omniglove/omniglove_hand_conversion.hpp" +#include "inc/omniglove_plugin/omniglove_hand_conversion.hpp" + +#include #include #include @@ -13,8 +15,6 @@ namespace plugins::omniglove namespace { -constexpr uint32_t kPostIkConverged = 1U << 0U; -constexpr uint32_t kPostBoneModelValid = 1U << 2U; constexpr float kJointRadiusM = 0.01F; constexpr XrSpaceLocationFlags kValidFlags = XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; diff --git a/src/plugins/omniglove/core/omniglove_plugin.cpp b/src/plugins/omniglove/core/omniglove_plugin.cpp index 62cb04ce9c..56734c6c7f 100644 --- a/src/plugins/omniglove/core/omniglove_plugin.cpp +++ b/src/plugins/omniglove/core/omniglove_plugin.cpp @@ -1,18 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. // SPDX-License-Identifier: Apache-2.0 -#include "inc/omniglove/omniglove_plugin.hpp" +#include "inc/omniglove_plugin/omniglove_plugin.hpp" +#include #include #include #include #include -#include -#include -#include -#include -#include #include #include @@ -22,63 +18,13 @@ namespace plugins::omniglove namespace { -constexpr auto kWorkerPeriod = std::chrono::milliseconds(2); -constexpr int64_t kStaleThresholdNs = 200'000'000; +constexpr auto kSourceTimeout = std::chrono::milliseconds(200); +constexpr int64_t kStaleThresholdNs = std::chrono::duration_cast(kSourceTimeout).count(); +constexpr uint32_t kSourceIntervalMs = 8; +constexpr uint32_t kCallbackQueueCapacity = 64; constexpr XrSpaceLocationFlags kTrackedFlags = XR_SPACE_LOCATION_POSITION_TRACKED_BIT | XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT; -plugin_utils::WristSourceMode wrist_source_mode_from_env() -{ - const char* value = std::getenv("OMNIGLOVE_WRIST_SOURCE"); - if (value == nullptr || *value == '\0' || std::strcmp(value, "hand_tracking") == 0) - { - return plugin_utils::WristSourceMode::HandTracking; - } - if (std::strcmp(value, "auto") == 0) - { - return plugin_utils::WristSourceMode::Auto; - } - if (std::strcmp(value, "controller") == 0) - { - return plugin_utils::WristSourceMode::Controller; - } - isaaccapture::Logger::get("isaaccapture.plugins.omniglove.main") - ->warn("unknown OMNIGLOVE_WRIST_SOURCE '{}', using 'hand_tracking'", value); - return plugin_utils::WristSourceMode::HandTracking; -} - -XrPosef pose_from_env(const char* name) -{ - XrPosef pose{ { 0.0F, 0.0F, 0.0F, 1.0F }, { 0.0F, 0.0F, 0.0F } }; - const char* value = std::getenv(name); - if (value == nullptr || *value == '\0') - { - return pose; - } - if (std::sscanf(value, "%f,%f,%f,%f,%f,%f,%f", &pose.position.x, &pose.position.y, &pose.position.z, - &pose.orientation.x, &pose.orientation.y, &pose.orientation.z, &pose.orientation.w) != 7) - { - throw std::runtime_error(std::string(name) + " must be px,py,pz,qx,qy,qz,qw"); - } - const float values[] = { pose.position.x, pose.position.y, pose.position.z, pose.orientation.x, - pose.orientation.y, pose.orientation.z, pose.orientation.w }; - if (!std::all_of(std::begin(values), std::end(values), [](float component) { return std::isfinite(component); })) - { - throw std::runtime_error(std::string(name) + " contains a non-finite value"); - } - const float norm = std::sqrt(pose.orientation.x * pose.orientation.x + pose.orientation.y * pose.orientation.y + - pose.orientation.z * pose.orientation.z + pose.orientation.w * pose.orientation.w); - if (norm < 1.0e-6F) - { - throw std::runtime_error(std::string(name) + " contains a zero quaternion"); - } - pose.orientation.x /= norm; - pose.orientation.y /= norm; - pose.orientation.z /= norm; - pose.orientation.w /= norm; - return pose; -} - const char* hand_name(bool is_left) { return is_left ? "left" : "right"; @@ -96,21 +42,13 @@ OmniglovePlugin::OmniglovePlugin(const std::string& plugin_root_id) m_logger->info("initializing with root {}", plugin_root_id); plugin_utils::WristSourceConfig wrist_config; - wrist_config.mode = wrist_source_mode_from_env(); + wrist_config.mode = + plugin_utils::wrist_source_mode_from_env("OMNIGLOVE_WRIST_SOURCE", plugin_utils::WristSourceMode::HandTracking); if (wrist_config.mode != plugin_utils::WristSourceMode::HandTracking) { - // Controller aim is not a wrist pose without an explicit mount calibration. - for (const char* name : { "OMNIGLOVE_AIM_TO_WRIST_LEFT", "OMNIGLOVE_AIM_TO_WRIST_RIGHT" }) - { - const char* value = std::getenv(name); - if (value == nullptr || *value == '\0') - { - throw std::runtime_error(std::string(name) + " is required for controller and auto wrist sources"); - } - } + wrist_config.left_aim_to_wrist = plugin_utils::pose_from_env("OMNIGLOVE_AIM_TO_WRIST_LEFT"); + wrist_config.right_aim_to_wrist = plugin_utils::pose_from_env("OMNIGLOVE_AIM_TO_WRIST_RIGHT"); } - wrist_config.left_aim_to_wrist = pose_from_env("OMNIGLOVE_AIM_TO_WRIST_LEFT"); - wrist_config.right_aim_to_wrist = pose_from_env("OMNIGLOVE_AIM_TO_WRIST_RIGHT"); const char* wrist_mode = wrist_config.mode == plugin_utils::WristSourceMode::HandTracking ? "hand_tracking" : wrist_config.mode == plugin_utils::WristSourceMode::Controller ? "controller" : "auto"; @@ -122,49 +60,21 @@ OmniglovePlugin::OmniglovePlugin(const std::string& plugin_root_id) extensions.push_back(XR_NVX1_DEVICE_INTERFACE_BASE_EXTENSION_NAME); extensions.insert(extensions.end(), wrist_requirements.extensions.begin(), wrist_requirements.extensions.end()); - m_session = std::make_shared("Omniglove", extensions); + const auto status_extensions = plugin_utils::PluginDeviceStatusPublisher::get_required_extensions(); + extensions.insert(extensions.end(), status_extensions.begin(), status_extensions.end()); + m_session = std::make_unique("Omniglove", extensions); const auto handles = m_session->get_handles(); m_deviceio_session = core::DeviceIOSession::run(wrist_requirements.trackers, handles); m_wrist_source = std::make_unique( wrist_config, handles, m_deviceio_session.get(), wrist_requirements.controller_tracker); - m_time_converter = std::make_unique(handles); + m_time_converter.emplace(handles); + m_status_publisher.emplace(handles, plugin_root_id); start_device_streams(); - m_running.store(true, std::memory_order_release); - try - { - m_worker = std::thread(&OmniglovePlugin::worker_thread, this); - } - catch (...) - { - m_running.store(false, std::memory_order_release); - throw; - } m_logger->info("initialized with {} glove stream(s)", m_streams.size()); } -OmniglovePlugin::~OmniglovePlugin() -{ - m_running.store(false, std::memory_order_release); - if (m_worker.joinable()) - { - m_worker.join(); - } - m_left_injector.reset(); - m_right_injector.reset(); - m_sdk_session.reset(); - m_streams.clear(); -} - -bool OmniglovePlugin::is_running() const noexcept -{ - return m_running.load(std::memory_order_acquire); -} - -bool OmniglovePlugin::has_failed() const noexcept -{ - return m_failed.load(std::memory_order_acquire); -} +OmniglovePlugin::~OmniglovePlugin() = default; void OmniglovePlugin::initialize_sdk() { @@ -231,39 +141,34 @@ void OmniglovePlugin::start_device_streams() continue; } - auto stream = std::make_unique(); - stream->device = device; - stream->context = { this, is_left }; + // Own the callback context before starting the stream, including failure cleanup. + m_streams.push_back(std::make_unique(StreamContext{ this, is_left })); omniglove_stream_config_t config{}; config.struct_size = sizeof(config); config.kind = OMNIGLOVE_STREAM_POST_DATA; config.stream_mode = OMNIGLOVE_STREAM_MODE_DEFAULT; config.output_mode = OMNIGLOVE_STREAM_OUTPUT_DIRECT; - config.source_interval_ms = 8; - config.source_timeout_ms = 200; + config.source_interval_ms = kSourceIntervalMs; + config.source_timeout_ms = static_cast(kSourceTimeout.count()); config.callback = &OmniglovePlugin::stream_callback; - config.callback_user_data = &stream->context; - config.callback_queue_capacity = 64; + config.callback_user_data = m_streams.back().get(); + config.callback_queue_capacity = kCallbackQueueCapacity; const auto configure_error = omniglove_device_stream_configure(device, &config); if (configure_error != OMNIGLOVE_STREAM_ERROR_NONE) { - m_logger->error("could not configure {} POST_DATA stream: {}", hand_name(is_left), - static_cast(configure_error)); - continue; + throw std::runtime_error(std::string("could not configure ") + hand_name(is_left) + + " POST_DATA stream: " + std::to_string(static_cast(configure_error))); } const auto start_error = omniglove_device_stream_start(device, OMNIGLOVE_STREAM_POST_DATA); if (start_error != OMNIGLOVE_STREAM_ERROR_NONE) { - m_logger->error( - "could not start {} POST_DATA stream: {}", hand_name(is_left), static_cast(start_error)); - continue; + throw std::runtime_error(std::string("could not start ") + hand_name(is_left) + + " POST_DATA stream: " + std::to_string(static_cast(start_error))); } - stream->started = true; occupied = true; m_logger->info("started {} glove at {} Hz", hand_name(is_left), omniglove_device_nominal_source_rate_hz(device)); - m_streams.push_back(std::move(stream)); } if (m_streams.empty()) @@ -274,7 +179,7 @@ void OmniglovePlugin::start_device_streams() void OmniglovePlugin::stream_callback(const omniglove_stream_sample_t* sample, void* user_data) { - if (sample == nullptr || user_data == nullptr) + if (sample == nullptr) { return; } @@ -287,7 +192,16 @@ void OmniglovePlugin::on_sample(const omniglove_stream_sample_t& sample, bool is if (sample.kind != OMNIGLOVE_STREAM_POST_DATA || sample.layout_version != OMNIGLOVE_STREAM_POST_LAYOUT_VERSION || sample.data.postdata == nullptr || sample.sample_monotonic_ns > std::numeric_limits::max()) { - invalidate_hand(is_left); + std::lock_guard lock(m_frame_mutex); + auto& hand = m_hands[is_left ? 0 : 1]; + hand.frame.reset(); + if (!hand.sample_error_logged) + { + m_logger->error("{} SDK sample contract mismatch: kind {}, layout {}, null data {}, timestamp {}", + hand_name(is_left), static_cast(sample.kind), sample.layout_version, + sample.data.postdata == nullptr, sample.sample_monotonic_ns); + hand.sample_error_logged = true; + } return; } @@ -299,30 +213,26 @@ void OmniglovePlugin::on_sample(const omniglove_stream_sample_t& sample, bool is m_logger->debug("{} POST_DATA rejected: {}", hand_name(is_left), conversion_status_name(status)); return; } - converted.sample_monotonic_ns = sample.sample_monotonic_ns; + converted.sample_monotonic_ns = static_cast(sample.sample_monotonic_ns); converted.connection_generation = sample.connection_generation; converted.source_sequence = sample.source_sequence; - converted.valid = true; std::lock_guard lock(m_frame_mutex); - (is_left ? m_left : m_right) = converted; + m_hands[is_left ? 0 : 1].frame = converted; } void OmniglovePlugin::invalidate_hand(bool is_left) { std::lock_guard lock(m_frame_mutex); - (is_left ? m_left : m_right).valid = false; + m_hands[is_left ? 0 : 1].frame.reset(); } -void OmniglovePlugin::pump_hand(std::unique_ptr& injector, - XrHandEXT hand, - const HandFrame& frame, - uint64_t& last_pushed_ns) +void OmniglovePlugin::pump_hand(HandState& hand, bool is_left, const std::optional& frame) { - const bool is_left = hand == XR_HAND_LEFT_EXT; + auto& injector = hand.injector; const int64_t now_ns = core::os_monotonic_now_ns(); - const bool fresh = frame.valid && frame.sample_monotonic_ns <= static_cast(now_ns) && - now_ns - static_cast(frame.sample_monotonic_ns) < kStaleThresholdNs; + const bool fresh = + frame && frame->sample_monotonic_ns <= now_ns && now_ns - frame->sample_monotonic_ns < kStaleThresholdNs; if (!fresh) { if (injector) @@ -330,15 +240,16 @@ void OmniglovePlugin::pump_hand(std::unique_ptr& inj m_logger->info("{} hand inactive: SDK frame stale or invalid", hand_name(is_left)); } injector.reset(); - last_pushed_ns = 0; + hand.state = core::PluginDeviceState_DISCONNECTED; + hand.reason = core::PluginDeviceReason_NO_HARDWARE_SIGNAL; return; } - if (frame.sample_monotonic_ns == last_pushed_ns) + if (frame->sample_monotonic_ns == hand.last_pushed_ns) { return; } - const XrTime time = m_time_converter->convert_monotonic_ns_to_xrtime(static_cast(frame.sample_monotonic_ns)); + const XrTime time = m_time_converter->convert_monotonic_ns_to_xrtime(frame->sample_monotonic_ns); const plugin_utils::WristSample wrist = m_wrist_source->query(is_left, time); if (!wrist.valid) { @@ -347,18 +258,21 @@ void OmniglovePlugin::pump_hand(std::unique_ptr& inj m_logger->info("{} hand inactive: wrist pose unavailable", hand_name(is_left)); } injector.reset(); + hand.state = core::PluginDeviceState_DEGRADED; + hand.reason = core::PluginDeviceReason_NO_CURRENT_DATA; return; } if (!injector) { const auto handles = m_session->get_handles(); - injector = std::make_unique(handles.instance, handles.session, hand, handles.space); + injector = std::make_unique( + handles.instance, handles.session, is_left ? XR_HAND_LEFT_EXT : XR_HAND_RIGHT_EXT, handles.space); m_logger->info("{} hand active: connection generation {}, source sequence {}", hand_name(is_left), - frame.connection_generation, frame.source_sequence); + frame->connection_generation, frame->source_sequence); } - auto joints = frame.joints; + auto joints = frame->joints; for (auto& joint : joints) { if ((joint.locationFlags & XR_SPACE_LOCATION_POSITION_VALID_BIT) == 0) @@ -372,37 +286,24 @@ void OmniglovePlugin::pump_hand(std::unique_ptr& inj } } injector->push(joints.data(), time); - last_pushed_ns = frame.sample_monotonic_ns; + hand.last_pushed_ns = frame->sample_monotonic_ns; + hand.state = core::PluginDeviceState_CONNECTED; + hand.reason = core::PluginDeviceReason_NONE; } -void OmniglovePlugin::worker_thread() +void OmniglovePlugin::update() { - while (m_running.load(std::memory_order_acquire)) + m_deviceio_session->update(); + std::array, 2> frames; { - try - { - m_deviceio_session->update(); - HandFrame left; - HandFrame right; - { - std::lock_guard lock(m_frame_mutex); - left = m_left; - right = m_right; - } - pump_hand(m_left_injector, XR_HAND_LEFT_EXT, left, m_left_last_pushed_ns); - pump_hand(m_right_injector, XR_HAND_RIGHT_EXT, right, m_right_last_pushed_ns); - } - catch (const std::exception& error) - { - m_logger->error("update failed: {}", error.what()); - m_left_injector.reset(); - m_right_injector.reset(); - m_failed.store(true, std::memory_order_release); - m_running.store(false, std::memory_order_release); - return; - } - std::this_thread::sleep_for(kWorkerPeriod); + std::lock_guard lock(m_frame_mutex); + frames = { m_hands[0].frame, m_hands[1].frame }; } + pump_hand(m_hands[0], true, frames[0]); + pump_hand(m_hands[1], false, frames[1]); + m_status_publisher->publish_if_changed({ { "/hand/left", m_hands[0].state, m_hands[0].reason, {} }, + { "/hand/right", m_hands[1].state, m_hands[1].reason, {} } }, + core::os_monotonic_now_ns()); } } // namespace plugins::omniglove diff --git a/src/plugins/omniglove/install.sh b/src/plugins/omniglove/install.sh deleted file mode 100755 index 2b5fbdc5e5..0000000000 --- a/src/plugins/omniglove/install.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -build_dir="" -sdk_root="${OMNIGLOVE_SDK_ROOT:-}" -sdk_ref="${OMNIGLOVE_SDK_REF:-}" -isaac_root="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --build-dir) - build_dir="${2:?--build-dir requires a path}" - shift 2 - ;; - --sdk-root) - sdk_root="${2:?--sdk-root requires a path}" - shift 2 - ;; - --sdk-ref) - sdk_ref="${2:?--sdk-ref requires a commit, tag, or branch}" - shift 2 - ;; - -h | --help) - cat <&2 - exit 1 - ;; - *) - if [[ -n "$isaac_root" ]]; then - echo "error: only one Isaac Capture root may be supplied" >&2 - exit 1 - fi - isaac_root="$1" - shift - ;; - esac -done - -if [[ -n "$sdk_root" && -n "$sdk_ref" ]]; then - echo "error: SDK root and managed SDK ref cannot be supplied together" >&2 - exit 1 -fi - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -isaac_root="${isaac_root:-$(cd "$script_dir/../../.." && pwd)}" -build_dir="${build_dir:-$isaac_root/build}" -if [[ "$build_dir" != /* ]]; then - build_dir="$PWD/$build_dir" -fi - -workspace_sdk="$isaac_root/../agilink_omniglove_sdk" -if [[ -z "$sdk_root" && -z "$sdk_ref" ]] && [[ -e "$workspace_sdk/.git" ]]; then - sdk_root="$workspace_sdk" -fi - -if [[ -z "$sdk_root" ]]; then - sdk_root="$script_dir/.sdk/agilink_omniglove_sdk" - # Pin managed dependencies: their own SHA256SUMS detects corruption, not - # authenticity. An upstream branch update must not silently change the SDK. - sdk_ref="${sdk_ref:-a2f899cd45c51ae01c9bbca9b893e4afba41c0d7}" - if [[ ! -e "$sdk_root/.git" ]]; then - if [[ -e "$sdk_root" ]]; then - echo "error: managed SDK path exists but is not a Git checkout: $sdk_root" >&2 - exit 1 - fi - mkdir -p "$(dirname "$sdk_root")" - git init --quiet "$sdk_root" - git -C "$sdk_root" remote add origin https://github.com/AgibotTech/agilink_omniglove_sdk.git - fi - if [[ -n "$(git -C "$sdk_root" status --porcelain)" ]]; then - echo "error: managed SDK checkout has local changes; preserve them before installing: $sdk_root" >&2 - exit 1 - fi - if [[ "$sdk_ref" =~ ^[0-9a-fA-F]{40}$ ]] && git -C "$sdk_root" cat-file -e "$sdk_ref^{commit}" 2>/dev/null; then - sdk_commit="$(git -C "$sdk_root" rev-parse "$sdk_ref^{commit}")" - else - git -C "$sdk_root" fetch --depth 1 -- origin "$sdk_ref" - sdk_commit="$(git -C "$sdk_root" rev-parse 'FETCH_HEAD^{commit}')" - fi - echo "==> Using managed OmniGlove SDK $sdk_ref ($sdk_commit)" - git -C "$sdk_root" checkout --detach "$sdk_commit" -- -fi - -sdk_root="$(cd "$sdk_root" && pwd)" -if [[ ! -f "$sdk_root/VERSION" ]] || [[ ! -f "$sdk_root/SHA256SUMS" ]]; then - echo "error: $sdk_root is not a complete OmniGlove SDK release checkout" >&2 - exit 1 -fi - -echo "==> Verifying OmniGlove SDK $(tr -d '[:space:]' < "$sdk_root/VERSION")" -(cd "$sdk_root" && sha256sum --check --quiet SHA256SUMS) - -case "$(uname -m)" in - x86_64) platform="linux-x86_64" ;; - aarch64 | arm64) platform="linux-aarch64" ;; - *) - echo "error: unsupported architecture $(uname -m)" >&2 - exit 1 - ;; -esac - -platform_root="$sdk_root/$platform" -for required in \ - "$platform_root/cmake/OmnigloveSDKConfig.cmake" \ - "$platform_root/include/omniglove/omniglove.h"; do - if [[ ! -f "$required" ]]; then - echo "error: SDK artifact is missing: $required" >&2 - exit 1 - fi -done - -# Follow the selected checkout's package config instead of pinning an ABI-named -# library here. CMake and the compatibility probe reject missing required APIs. - -sdk_commit="unknown" -if [[ -e "$sdk_root/.git" ]]; then - sdk_commit="$(git -C "$sdk_root" rev-parse HEAD)" -fi - -echo "==> Configuring OmniGlove plugin against $platform ($sdk_commit)" -cmake -S "$isaac_root" -B "$build_dir" \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TESTING=ON \ - -DBUILD_PLUGINS=ON \ - -DBUILD_PLUGIN_OMNIGLOVE=ON \ - -DOmnigloveSDK_DIR="$platform_root/cmake" \ - -DOMNIGLOVE_SDK_RELEASE_ROOT="$sdk_root" \ - -DOMNIGLOVE_SDK_GIT_COMMIT="$sdk_commit" - -cmake --build "$build_dir" --target \ - omniglove_api_compat_probe \ - omniglove_hand_conversion_contract_test \ - omniglove_plugin --parallel - -ctest --test-dir "$build_dir" --output-on-failure -R '^omniglove_' -cmake --install "$build_dir" --component omniglove - -install_prefix="$(awk '/^CMAKE_INSTALL_PREFIX:PATH=/{sub(/^[^=]*=/, ""); print; exit}' "$build_dir/CMakeCache.txt")" -echo "==> Installed $install_prefix/plugins/omniglove/omniglove_plugin" -echo "Install USB permissions once on the host with:" -echo " $script_dir/install_udev_rules.sh" diff --git a/src/plugins/omniglove/record_cts.sh b/src/plugins/omniglove/record_cts.sh deleted file mode 100755 index 7a72262769..0000000000 --- a/src/plugins/omniglove/record_cts.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_dir="$(cd -- "$script_dir/../../.." && pwd)" -allow_dirty=false -capture_args=() -operator_note="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --wrist-config) - source "${2:?--wrist-config requires an environment file}" - shift 2 - ;; - --allow-dirty) - allow_dirty=true - shift - ;; - --note) - operator_note="${2:?--note requires text}" - shift 2 - ;; - --note=*) operator_note="${1#--note=}"; shift ;; - -h | --help) - cat <<'EOF' -Usage: record_cts.sh [--wrist-config PATH] [--allow-dirty] [capture options] - -Formal recordings require a clean, pushed plugin commit and an unchanged CTS. ---allow-dirty permits development takes; their provenance remains unchanged. -Set OMNIGLOVE_WRIST_SOURCE and measured per-hand controller transforms, or -load them from --wrist-config. Other options go to the official recorder. -EOF - exit 0 - ;; - --plugin | --plugin=* | --wrist-source | --wrist-source=*) - echo "error: plugin and wrist source are selected by this script" >&2 - exit 1 - ;; - *) capture_args+=("$1"); shift ;; - esac -done - -cd -- "$repo_dir" -if [[ -n "$(git status --porcelain -- src/python/isaaccapture/cts)" ]]; then - echo "error: CTS has local changes; restore the official tools before recording" >&2 - exit 1 -fi -if [[ "$allow_dirty" == false ]]; then - if [[ -n "$(git status --porcelain)" ]] || - ! git cat-file -e HEAD:src/plugins/omniglove/plugin.yaml 2>/dev/null; then - echo "error: commit the final plugin and keep the repository clean before a formal take" >&2 - exit 1 - fi - if ! git rev-parse --verify '@{upstream}' >/dev/null 2>&1 || - ! git merge-base --is-ancestor HEAD '@{upstream}'; then - echo "error: push this commit and set its upstream before a formal take" >&2 - exit 1 - fi -fi - -wrist_source="${OMNIGLOVE_WRIST_SOURCE:-hand_tracking}" -case "$wrist_source" in - hand_tracking) ;; - controller | auto) - : "${OMNIGLOVE_AIM_TO_WRIST_LEFT:?set the measured left aim-to-wrist transform}" - : "${OMNIGLOVE_AIM_TO_WRIST_RIGHT:?set the measured right aim-to-wrist transform}" - ;; - *) echo "error: unsupported OMNIGLOVE_WRIST_SOURCE=$wrist_source" >&2; exit 1 ;; -esac -export OMNIGLOVE_WRIST_SOURCE="$wrist_source" -if [[ "$wrist_source" != hand_tracking ]]; then - export OMNIGLOVE_AIM_TO_WRIST_LEFT OMNIGLOVE_AIM_TO_WRIST_RIGHT -fi -plugin_binary="$repo_dir/install/plugins/omniglove/omniglove_plugin" -if [[ ! -x "$plugin_binary" ]]; then - echo "error: build and install the OmniGlove plugin first" >&2 - exit 1 -fi -plugin_sha="$(sha256sum "$plugin_binary")" -plugin_sha="${plugin_sha%% *}" -exec src/python/isaaccapture/cts/hand/capture/record.sh omniglove \ - --plugin omniglove_plugin \ - --wrist-source "$wrist_source" \ - --note "OmniGlove; binary_sha256=$plugin_sha; wrist=$wrist_source; LEFT=${OMNIGLOVE_AIM_TO_WRIST_LEFT:-unused}; RIGHT=${OMNIGLOVE_AIM_TO_WRIST_RIGHT:-unused}; $operator_note" \ - "${capture_args[@]}" diff --git a/src/plugins/omniglove/sdk.cmake b/src/plugins/omniglove/sdk.cmake new file mode 100644 index 0000000000..67802056d1 --- /dev/null +++ b/src/plugins/omniglove/sdk.cmake @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +set(OMNIGLOVE_SDK_ROOT "" CACHE PATH "Explicit OmniGlove SDK release root; empty fetches the pinned SDK") +set(OMNIGLOVE_SDK_REF "a2f899cd45c51ae01c9bbca9b893e4afba41c0d7" CACHE STRING "Managed OmniGlove SDK commit or ref") + +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "The OmniGlove SDK supports Linux only") +endif() +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$") + set(_omniglove_platform linux-x86_64) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + set(_omniglove_platform linux-aarch64) +else() + message(FATAL_ERROR "Unsupported OmniGlove SDK architecture: ${CMAKE_SYSTEM_PROCESSOR}") +endif() + +if(OMNIGLOVE_SDK_ROOT) + file(REAL_PATH "${OMNIGLOVE_SDK_ROOT}" _omniglove_sdk_root BASE_DIRECTORY "${CMAKE_SOURCE_DIR}") +else() + include(FetchContent) + FetchContent_Declare(omniglove_sdk + GIT_REPOSITORY https://github.com/AgibotTech/agilink_omniglove_sdk.git + GIT_TAG "${OMNIGLOVE_SDK_REF}" + # Consume the prebuilt package, not the SDK's standalone examples/build. + SOURCE_SUBDIR isaac-prebuilt-sdk + ) + FetchContent_MakeAvailable(omniglove_sdk) + set(_omniglove_sdk_root "${omniglove_sdk_SOURCE_DIR}") +endif() + +if(NOT EXISTS "${_omniglove_sdk_root}/SHA256SUMS" OR NOT EXISTS "${_omniglove_sdk_root}/VERSION") + message(FATAL_ERROR "Not a complete OmniGlove SDK release: ${_omniglove_sdk_root}") +endif() +# The pinned commit selects the managed release; its checksums detect corruption. +file(STRINGS "${_omniglove_sdk_root}/SHA256SUMS" _omniglove_checksums) +foreach(_omniglove_entry IN LISTS _omniglove_checksums) + if(NOT _omniglove_entry MATCHES "^([0-9a-fA-F]+) (.+)$") + message(FATAL_ERROR "Malformed OmniGlove SDK checksum entry: ${_omniglove_entry}") + endif() + set(_omniglove_expected "${CMAKE_MATCH_1}") + set(_omniglove_file "${_omniglove_sdk_root}/${CMAKE_MATCH_2}") + if(NOT EXISTS "${_omniglove_file}") + message(FATAL_ERROR "Missing OmniGlove SDK release file: ${_omniglove_file}") + endif() + file(SHA256 "${_omniglove_file}" _omniglove_actual) + if(NOT _omniglove_actual STREQUAL _omniglove_expected) + message(FATAL_ERROR "OmniGlove SDK checksum mismatch: ${_omniglove_file}") + endif() +endforeach() + +set(OmnigloveSDK_DIR "${_omniglove_sdk_root}/${_omniglove_platform}/cmake") + +# Tests outside this plugin directory also consume the imported SDK target. +find_package(OmnigloveSDK CONFIG REQUIRED GLOBAL + PATHS "${_omniglove_sdk_root}/${_omniglove_platform}/cmake" + NO_DEFAULT_PATH +) +get_target_property(OMNIGLOVE_SDK_LIBRARY OmnigloveSDK::api IMPORTED_LOCATION) +set(OMNIGLOVE_SDK_RELEASE_ROOT "${_omniglove_sdk_root}") +set(OMNIGLOVE_SDK_GIT_COMMIT "unknown") +find_package(Git QUIET) +if(GIT_FOUND) + execute_process( + COMMAND "${GIT_EXECUTABLE}" -C "${_omniglove_sdk_root}" rev-parse --show-toplevel + OUTPUT_VARIABLE _omniglove_git_root + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET + ) + # An unpacked SDK inside a project must not inherit the enclosing repo's commit. + if(_omniglove_git_root STREQUAL _omniglove_sdk_root) + execute_process( + COMMAND "${GIT_EXECUTABLE}" -C "${_omniglove_sdk_root}" rev-parse HEAD + OUTPUT_VARIABLE OMNIGLOVE_SDK_GIT_COMMIT + OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND_ERROR_IS_FATAL ANY + ) + endif() +endif() +message(STATUS "OmniGlove SDK ${OmnigloveSDK_VERSION}: ${OMNIGLOVE_SDK_GIT_COMMIT}") diff --git a/src/plugins/omniglove/tests/CMakeLists.txt b/src/plugins/omniglove/tests/CMakeLists.txt deleted file mode 100644 index e656036b5e..0000000000 --- a/src/plugins/omniglove/tests/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. -# SPDX-License-Identifier: Apache-2.0 - -add_executable(omniglove_hand_conversion_contract_test hand_conversion_contract_test.cpp) -target_link_libraries(omniglove_hand_conversion_contract_test PRIVATE isaac::omniglove_plugin) - -add_test(NAME omniglove_hand_conversion_contract_test COMMAND omniglove_hand_conversion_contract_test) diff --git a/src/plugins/plugin_utils/CMakeLists.txt b/src/plugins/plugin_utils/CMakeLists.txt index a1d0f4c452..1fd8966863 100644 --- a/src/plugins/plugin_utils/CMakeLists.txt +++ b/src/plugins/plugin_utils/CMakeLists.txt @@ -8,6 +8,7 @@ add_library(teleop_plugin_utils STATIC plugin_device_status_publisher.cpp inc/plugin_utils/plugin_device_status_publisher.hpp wrist_pose_source.cpp + wrist_source_config.cpp inc/plugin_utils/wrist_pose_source.hpp ) diff --git a/src/plugins/plugin_utils/inc/plugin_utils/wrist_pose_source.hpp b/src/plugins/plugin_utils/inc/plugin_utils/wrist_pose_source.hpp index 24645e13b9..40117aee4f 100644 --- a/src/plugins/plugin_utils/inc/plugin_utils/wrist_pose_source.hpp +++ b/src/plugins/plugin_utils/inc/plugin_utils/wrist_pose_source.hpp @@ -39,6 +39,15 @@ struct WristSourceConfig XrPosef right_aim_to_wrist{ { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; }; +/** @brief Read a wrist mode; an unset variable uses fallback, an unknown value throws. */ +WristSourceMode wrist_source_mode_from_env(const char* name, WristSourceMode fallback); + +/** @brief Read and normalize px,py,pz,qx,qy,qz,qw; missing or invalid values throw. */ +XrPosef pose_from_env(const char* name); + +/** @brief Use fallback only for an unset variable; invalid values throw. */ +XrPosef pose_from_env(const char* name, const XrPosef& fallback); + /** @brief One wrist query result, in the session base space. */ struct WristSample { diff --git a/src/plugins/plugin_utils/wrist_source_config.cpp b/src/plugins/plugin_utils/wrist_source_config.cpp new file mode 100644 index 0000000000..ab99ad5597 --- /dev/null +++ b/src/plugins/plugin_utils/wrist_source_config.cpp @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/plugin_utils/wrist_pose_source.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugin_utils +{ + +WristSourceMode wrist_source_mode_from_env(const char* name, WristSourceMode fallback) +{ + const char* value = std::getenv(name); + if (value == nullptr || *value == '\0') + { + return fallback; + } + if (std::strcmp(value, "hand_tracking") == 0) + { + return WristSourceMode::HandTracking; + } + if (std::strcmp(value, "controller") == 0) + { + return WristSourceMode::Controller; + } + if (std::strcmp(value, "auto") == 0) + { + return WristSourceMode::Auto; + } + throw std::runtime_error(std::string(name) + " must be hand_tracking, controller or auto; got '" + value + "'"); +} + +XrPosef pose_from_env(const char* name) +{ + const char* value = std::getenv(name); + if (value == nullptr || *value == '\0') + { + throw std::runtime_error(std::string(name) + " is required"); + } + XrPosef pose{}; + char trailing; + if (std::sscanf(value, "%f,%f,%f,%f,%f,%f,%f %c", &pose.position.x, &pose.position.y, &pose.position.z, + &pose.orientation.x, &pose.orientation.y, &pose.orientation.z, &pose.orientation.w, &trailing) != 7) + { + throw std::runtime_error(std::string(name) + " must be px,py,pz,qx,qy,qz,qw"); + } + const float values[] = { pose.position.x, pose.position.y, pose.position.z, pose.orientation.x, + pose.orientation.y, pose.orientation.z, pose.orientation.w }; + if (!std::all_of(std::begin(values), std::end(values), [](float component) { return std::isfinite(component); })) + { + throw std::runtime_error(std::string(name) + " contains a non-finite value"); + } + const double norm = std::hypot(std::hypot(static_cast(pose.orientation.x), pose.orientation.y), + std::hypot(static_cast(pose.orientation.z), pose.orientation.w)); + if (norm < 1.0e-6) + { + throw std::runtime_error(std::string(name) + " contains a zero quaternion"); + } + pose.orientation.x = static_cast(pose.orientation.x / norm); + pose.orientation.y = static_cast(pose.orientation.y / norm); + pose.orientation.z = static_cast(pose.orientation.z / norm); + pose.orientation.w = static_cast(pose.orientation.w / norm); + return pose; +} + +XrPosef pose_from_env(const char* name, const XrPosef& fallback) +{ + const char* value = std::getenv(name); + return value == nullptr || *value == '\0' ? fallback : pose_from_env(name); +} + +} // namespace plugin_utils diff --git a/src/plugins/wuji_glove/wuji_glove_plugin.cpp b/src/plugins/wuji_glove/wuji_glove_plugin.cpp index 4154af8f28..671ad4139c 100644 --- a/src/plugins/wuji_glove/wuji_glove_plugin.cpp +++ b/src/plugins/wuji_glove/wuji_glove_plugin.cpp @@ -153,67 +153,31 @@ bool convert_skeleton(const WujiHandSkeleton* frame, return true; } -// Optional aim-to-wrist offset override. -// WUJI_GLOVE_AIM_TO_WRIST_{LEFT,RIGHT}="px,py,pz,qx,qy,qz,qw" (meters + quat). XrPosef pose_from_env(const char* name, const XrPosef& fallback) { - const char* value = std::getenv(name); - if (value == nullptr || *value == '\0') + try { - return fallback; + return plugin_utils::pose_from_env(name, fallback); } - auto logger = isaaccapture::Logger::get("isaaccapture.plugins.wuji_glove.main"); - XrPosef pose{}; - if (std::sscanf(value, "%f,%f,%f,%f,%f,%f,%f", &pose.position.x, &pose.position.y, &pose.position.z, - &pose.orientation.x, &pose.orientation.y, &pose.orientation.z, &pose.orientation.w) != 7) + catch (const std::runtime_error& error) { - logger->warn("could not parse {} ('{}', want px,py,pz,qx,qy,qz,qw); using the built-in offset", name, value); + isaaccapture::Logger::get("isaaccapture.plugins.wuji_glove.main") + ->warn("{}; using the built-in offset", error.what()); return fallback; } - // sscanf("%f") happily accepts "nan" and "inf"; a non-finite offset would - // poison every injected joint pose, and NaN also slips past the norm check - // below (every comparison against NaN is false). - if (!std::isfinite(pose.position.x) || !std::isfinite(pose.position.y) || !std::isfinite(pose.position.z) || - !std::isfinite(pose.orientation.x) || !std::isfinite(pose.orientation.y) || - !std::isfinite(pose.orientation.z) || !std::isfinite(pose.orientation.w)) - { - logger->warn("{} ('{}') has non-finite values; using the built-in offset", name, value); - return fallback; - } - const float norm = std::sqrt(pose.orientation.x * pose.orientation.x + pose.orientation.y * pose.orientation.y + - pose.orientation.z * pose.orientation.z + pose.orientation.w * pose.orientation.w); - if (norm < 1e-6f) - { - logger->warn("{} has a zero quaternion; using the built-in offset", name); - return fallback; - } - pose.orientation.x /= norm; - pose.orientation.y /= norm; - pose.orientation.z /= norm; - pose.orientation.w /= norm; - return pose; } -// Wrist-source selection: WUJI_GLOVE_WRIST_SOURCE = auto (default) | -// hand_tracking | controller. plugin_utils::WristSourceMode wrist_source_mode_from_env() { - const char* value = std::getenv("WUJI_GLOVE_WRIST_SOURCE"); - if (value == nullptr || *value == '\0' || std::strcmp(value, "auto") == 0) - { - return plugin_utils::WristSourceMode::Auto; - } - if (std::strcmp(value, "hand_tracking") == 0) + try { - return plugin_utils::WristSourceMode::HandTracking; + return plugin_utils::wrist_source_mode_from_env("WUJI_GLOVE_WRIST_SOURCE", plugin_utils::WristSourceMode::Auto); } - if (std::strcmp(value, "controller") == 0) + catch (const std::runtime_error& error) { - return plugin_utils::WristSourceMode::Controller; + isaaccapture::Logger::get("isaaccapture.plugins.wuji_glove.main")->warn("{}; using 'auto'", error.what()); + return plugin_utils::WristSourceMode::Auto; } - isaaccapture::Logger::get("isaaccapture.plugins.wuji_glove.main") - ->warn("unknown WUJI_GLOVE_WRIST_SOURCE '{}', using 'auto'", value); - return plugin_utils::WristSourceMode::Auto; } // Resolve the glove's hand side explicitly via the device's "hand_side" GET diff --git a/src/python/isaaccapture/cts/hand/checker/README.md b/src/python/isaaccapture/cts/hand/checker/README.md index 3d6cc25ea8..67dc49fcf0 100644 --- a/src/python/isaaccapture/cts/hand/checker/README.md +++ b/src/python/isaaccapture/cts/hand/checker/README.md @@ -60,6 +60,16 @@ Labels are optional. Without them the report carries a note, the window checks a `consistency.position_orientation_same_frame` (which reads the roll windows) report that they could not be answered, and the rest are judged as usual. +## Package without the viewer + +```bash +.venv/bin/python -m hand_cts.package /path/to/take-hand.mcap --output-dir /path/to/output +``` + +This uses the viewer's bundle builder, reruns all checks, verifies the original +recording and companion files, and prints the archive's verdict and SHA-256. +It requires the panel dependencies. Existing archives are never overwritten. + ## Read the report ```text diff --git a/src/plugins/omniglove/package_cts.py b/src/python/isaaccapture/cts/hand/checker/src/hand_cts/package.py similarity index 93% rename from src/plugins/omniglove/package_cts.py rename to src/python/isaaccapture/cts/hand/checker/src/hand_cts/package.py index 424defde17..481959bb37 100755 --- a/src/plugins/omniglove/package_cts.py +++ b/src/python/isaaccapture/cts/hand/checker/src/hand_cts/package.py @@ -20,11 +20,12 @@ from hand_cts.report import run -def main() -> None: +def main(argv: list[str] | None = None) -> None: + """Build and verify one complete take without overwriting an existing archive.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("recording", type=Path) parser.add_argument("--output-dir", type=Path) - args = parser.parse_args() + args = parser.parse_args(argv) recording = args.recording.resolve() required = [ recording, diff --git a/src/python/isaaccapture/cts/hand/checker/tests/test_package.py b/src/python/isaaccapture/cts/hand/checker/tests/test_package.py new file mode 100644 index 0000000000..fba7725354 --- /dev/null +++ b/src/python/isaaccapture/cts/hand/checker/tests/test_package.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Headless packaging preserves the take and rejects incomplete or changing inputs.""" + +from __future__ import annotations + +import hashlib +import json +import zipfile +from pathlib import Path + +import pytest +import synth + +from hand_cts import package + + +@pytest.fixture +def take(tmp_path: Path) -> Path: + recording = synth.write_recording(tmp_path / "take-hand.mcap", synth.both_hands(60)) + recording.with_suffix(".labels.json").write_text('{"steps": []}') + recording.with_suffix(".json").write_text('{"device": "example"}') + recording.with_suffix(".log").write_text("capture log\n") + return recording + + +def test_headless_archive_preserves_all_inputs(take, tmp_path, capsys): + paths = [take, *(take.with_suffix(s) for s in (".labels.json", ".json", ".log"))] + before = {path.name: path.read_bytes() for path in paths} + package.main([str(take), "--output-dir", str(tmp_path / "output")]) + result = json.loads(capsys.readouterr().out) + archive = Path(result["archive"]) + assert hashlib.sha256(archive.read_bytes()).hexdigest() == result["sha256"] + with zipfile.ZipFile(archive) as bundle: + assert bundle.testzip() is None + root = archive.stem + report = json.loads(bundle.read(f"{root}/report.json")) + assert report["verdict"] == result["verdict"] + for item in report["inputs"]["files"]: + original = before[item["name"]] + assert bundle.read(f"{root}/{item['name']}") == original + assert item["sha256"] == hashlib.sha256(original).hexdigest() + assert {path.name: path.read_bytes() for path in paths} == before + + +def test_headless_archive_never_overwrites(take, capsys): + package.main([str(take)]) + output = Path(json.loads(capsys.readouterr().out)["archive"]) + original = output.read_bytes() + with pytest.raises(FileExistsError): + package.main([str(take)]) + assert output.read_bytes() == original + + +def test_headless_packaging_rejects_incomplete_takes(take): + take.with_suffix(".labels.json").unlink() + with pytest.raises(SystemExit) as exited: + package.main([str(take)]) + assert exited.value.code == 2 + assert not list(take.parent.glob("*.zip")) + + +def test_headless_packaging_rejects_a_concurrent_input_change(take, monkeypatch): + original = package.bundle.build + + def changing_input(*args, **kwargs): + archive = original(*args, **kwargs) + take.with_suffix(".log").write_text("modified during packaging\n") + return archive + + monkeypatch.setattr(package.bundle, "build", changing_input) + with pytest.raises(RuntimeError, match="take changed during packaging"): + package.main([str(take)]) + assert not list(take.parent.glob("*.zip")) diff --git a/tests/cpp/plugins/CMakeLists.txt b/tests/cpp/plugins/CMakeLists.txt index fd7e2d2e68..f5ef077786 100644 --- a/tests/cpp/plugins/CMakeLists.txt +++ b/tests/cpp/plugins/CMakeLists.txt @@ -21,3 +21,7 @@ endif() if(TARGET manus_calibration_core) add_subdirectory(manus_calibration) endif() + +if(BUILD_PLUGIN_OMNIGLOVE) + add_subdirectory(omniglove) +endif() diff --git a/tests/cpp/plugins/omniglove/CMakeLists.txt b/tests/cpp/plugins/omniglove/CMakeLists.txt new file mode 100644 index 0000000000..08dc2c2f23 --- /dev/null +++ b/tests/cpp/plugins/omniglove/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(omniglove_plugin_tests test_hand_conversion.cpp) +target_link_libraries(omniglove_plugin_tests + PRIVATE Catch2::Catch2WithMain isaac::omniglove_plugin OmnigloveSDK::api +) +catch_discover_tests(omniglove_plugin_tests TEST_PREFIX "omniglove_") diff --git a/src/plugins/omniglove/tests/hand_conversion_contract_test.cpp b/tests/cpp/plugins/omniglove/test_hand_conversion.cpp similarity index 71% rename from src/plugins/omniglove/tests/hand_conversion_contract_test.cpp rename to tests/cpp/plugins/omniglove/test_hand_conversion.cpp index 3de041e20e..ca2b0da8a5 100644 --- a/src/plugins/omniglove/tests/hand_conversion_contract_test.cpp +++ b/tests/cpp/plugins/omniglove/test_hand_conversion.cpp @@ -1,12 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. // SPDX-License-Identifier: Apache-2.0 -#include +#include +#include +#include #include #include #include -#include #include namespace @@ -28,7 +29,7 @@ void fill_pose(omniglove_pose6_t& pose, float x, float y, float z, const XrQuate void fill_hand(omniglove_post_data_t& post, const XrQuaternionf& orientation = { 0.0F, 0.0F, 0.0F, 2.0F }) { post.mapped = 1; - post.flags = (1U << 0U) | (1U << 2U); + post.flags = plugins::omniglove::kPostIkConverged | plugins::omniglove::kPostBoneModelValid; float x = 0.01F; auto fill = [&x, &orientation](omniglove_pose6_t& pose) { @@ -115,67 +116,44 @@ bool openxr_axes(bool is_left, const XrQuaternionf& orientation) return true; } -int fail(const char* message) -{ - std::cerr << message << '\n'; - return 1; -} - } // namespace -int main() +TEST_CASE("OmniGlove conversion preserves its OpenXR hand contract", "[omniglove][conversion]") { omniglove_post_data_t post{}; fill_hand(post); std::array joints{}; - - if (plugins::omniglove::convert_post_data(post, true, joints) != ConversionStatus::Success) - { - return fail("left hand conversion failed"); - } + REQUIRE(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::Success); constexpr XrSpaceLocationFlags valid = XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; int valid_count = 0; for (const auto& joint : joints) { valid_count += (joint.locationFlags & valid) == valid; } - if (valid_count != 21 || joints[XR_HAND_JOINT_PALM_EXT].locationFlags != 0 || - joints[XR_HAND_JOINT_INDEX_METACARPAL_EXT].locationFlags != 0) - { - return fail("the required/optional joint set is wrong"); - } - if (!near(joints[XR_HAND_JOINT_WRIST_EXT].pose.orientation.w, 1.0F)) - { - return fail("quaternion was not normalized"); - } + CHECK(valid_count == 21); + CHECK(joints[XR_HAND_JOINT_PALM_EXT].locationFlags == 0); + CHECK(joints[XR_HAND_JOINT_INDEX_METACARPAL_EXT].locationFlags == 0); + CHECK(near(joints[XR_HAND_JOINT_WRIST_EXT].pose.orientation.w, 1.0F)); for (bool is_left : { true, false }) { - if (!openxr_axes(is_left, { 0.0F, 0.0F, 0.0F, 2.0F }) || !openxr_axes(is_left, { 1.0F, 2.0F, 3.0F, 4.0F })) - { - return fail("finger -Z/dorsal +Y or wrist axes do not follow OpenXR"); - } + CHECK(openxr_axes(is_left, { 0.0F, 0.0F, 0.0F, 2.0F })); + CHECK(openxr_axes(is_left, { 1.0F, 2.0F, 3.0F, 4.0F })); } - const XrPosef left_wrist = joints[XR_HAND_JOINT_WRIST_EXT].pose; - if (plugins::omniglove::convert_post_data(post, false, joints) != ConversionStatus::Success || - !near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.x, -left_wrist.position.x) || - !near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.y, -left_wrist.position.y) || - !near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.z, left_wrist.position.z)) - { - return fail("right-hand wrist-basis conversion is wrong"); - } - - post.flags = 1U << 2U; - if (plugins::omniglove::convert_post_data(post, true, joints) != ConversionStatus::IkNotConverged) - { - return fail("unconverged IK was accepted"); - } + REQUIRE(plugins::omniglove::convert_post_data(post, false, joints) == ConversionStatus::Success); + CHECK(near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.x, -left_wrist.position.x)); + CHECK(near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.y, -left_wrist.position.y)); + CHECK(near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.z, left_wrist.position.z)); + + post.flags = plugins::omniglove::kPostBoneModelValid; + CHECK(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::IkNotConverged); + post.flags = plugins::omniglove::kPostIkConverged; + CHECK(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::BoneModelInvalid); + fill_hand(post); + post.mapped = 0; + CHECK(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::NotMapped); fill_hand(post); post.index.landmarks.mcp.position_m[0] = std::numeric_limits::quiet_NaN(); - if (plugins::omniglove::convert_post_data(post, true, joints) != ConversionStatus::InvalidPose) - { - return fail("non-finite pose was accepted"); - } - return 0; + CHECK(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::InvalidPose); } diff --git a/tests/cpp/plugins/plugin_utils/CMakeLists.txt b/tests/cpp/plugins/plugin_utils/CMakeLists.txt index 9534f3849b..09301b972e 100644 --- a/tests/cpp/plugins/plugin_utils/CMakeLists.txt +++ b/tests/cpp/plugins/plugin_utils/CMakeLists.txt @@ -3,6 +3,7 @@ add_executable(plugin_device_status_publisher_tests test_plugin_device_status_publisher.cpp + test_wrist_source_config.cpp ) target_link_libraries(plugin_device_status_publisher_tests diff --git a/tests/cpp/plugins/plugin_utils/test_wrist_source_config.cpp b/tests/cpp/plugins/plugin_utils/test_wrist_source_config.cpp new file mode 100644 index 0000000000..a41af35775 --- /dev/null +++ b/tests/cpp/plugins/plugin_utils/test_wrist_source_config.cpp @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include +#include + +namespace +{ + +class Environment +{ +public: + explicit Environment(const char* name) : m_name(name) + { + if (const char* value = std::getenv(name)) + { + m_previous = value; + } + unsetenv(name); + } + ~Environment() + { + if (m_previous) + { + setenv(m_name, m_previous->c_str(), 1); + } + else + { + unsetenv(m_name); + } + } + void set(const char* value) + { + setenv(m_name, value, 1); + } + +private: + const char* m_name; + std::optional m_previous; +}; + +constexpr const char* kVariable = "ISAAC_TEST_WRIST_CONFIG"; + +} // namespace + +TEST_CASE("Wrist source mode accepts only named modes", "[plugin_utils][wrist_config]") +{ + Environment env(kVariable); + using plugin_utils::WristSourceMode; + CHECK(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::Auto) == WristSourceMode::Auto); + env.set("hand_tracking"); + CHECK(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::Auto) == WristSourceMode::HandTracking); + env.set("controller"); + CHECK(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::Auto) == WristSourceMode::Controller); + env.set("auto"); + CHECK(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::HandTracking) == WristSourceMode::Auto); + env.set("controler"); + CHECK_THROWS_AS(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::Auto), std::runtime_error); +} + +TEST_CASE("Required wrist offsets reject missing and malformed input", "[plugin_utils][wrist_config]") +{ + Environment env(kVariable); + CHECK_THROWS_AS(plugin_utils::pose_from_env(kVariable), std::runtime_error); + for (const char* value : + { "", "0,0,0,0,0,0", "0,0,0,0,0,0,0", "nan,0,0,0,0,0,1", "0,0,0,0,0,inf,1", "0,0,0,0,0,0,1,junk" }) + { + env.set(value); + CHECK_THROWS_AS(plugin_utils::pose_from_env(kVariable), std::runtime_error); + } +} + +TEST_CASE("Wrist offsets normalize finite quaternions and preserve metres", "[plugin_utils][wrist_config]") +{ + Environment env(kVariable); + for (const char* value : { "0.1,-0.2,0.3,0,0,0,2", "0.1,-0.2,0.3,0,0,0,3e38" }) + { + env.set(value); + const auto pose = plugin_utils::pose_from_env(kVariable); + CHECK(std::abs(pose.position.x - 0.1F) < 1.0e-6F); + CHECK(std::abs(pose.position.y + 0.2F) < 1.0e-6F); + CHECK(std::abs(pose.position.z - 0.3F) < 1.0e-6F); + CHECK(std::abs(pose.orientation.w - 1.0F) < 1.0e-6F); + } +} + +TEST_CASE("Optional wrist offsets fall back only when unset", "[plugin_utils][wrist_config]") +{ + Environment env(kVariable); + const XrPosef fallback{ { 0, 0, 0, 1 }, { 1, 2, 3 } }; + CHECK(plugin_utils::pose_from_env(kVariable, fallback).position.y == 2); + env.set("invalid"); + CHECK_THROWS_AS(plugin_utils::pose_from_env(kVariable, fallback), std::runtime_error); +} From 2461f89a618f09b9269827edb5c359b4eb88a0cc Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Wed, 11 Nov 2026 11:43:21 +0800 Subject: [PATCH 10/14] Guard diagnostic reads of typed OmniGlove SDK samples Signed-off-by: zhouyingchun --- src/plugins/omniglove/core/omniglove_plugin.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/plugins/omniglove/core/omniglove_plugin.cpp b/src/plugins/omniglove/core/omniglove_plugin.cpp index 56734c6c7f..dbb4ac926e 100644 --- a/src/plugins/omniglove/core/omniglove_plugin.cpp +++ b/src/plugins/omniglove/core/omniglove_plugin.cpp @@ -189,17 +189,19 @@ void OmniglovePlugin::stream_callback(const omniglove_stream_sample_t* sample, v void OmniglovePlugin::on_sample(const omniglove_stream_sample_t& sample, bool is_left) { - if (sample.kind != OMNIGLOVE_STREAM_POST_DATA || sample.layout_version != OMNIGLOVE_STREAM_POST_LAYOUT_VERSION || - sample.data.postdata == nullptr || sample.sample_monotonic_ns > std::numeric_limits::max()) + // The SDK data union may only be read through the member selected by kind. + const bool post_data_present = sample.kind == OMNIGLOVE_STREAM_POST_DATA && sample.data.postdata != nullptr; + if (!post_data_present || sample.layout_version != OMNIGLOVE_STREAM_POST_LAYOUT_VERSION || + sample.sample_monotonic_ns > std::numeric_limits::max()) { std::lock_guard lock(m_frame_mutex); auto& hand = m_hands[is_left ? 0 : 1]; hand.frame.reset(); if (!hand.sample_error_logged) { - m_logger->error("{} SDK sample contract mismatch: kind {}, layout {}, null data {}, timestamp {}", + m_logger->error("{} SDK sample contract mismatch: kind {}, layout {}, post data present {}, timestamp {}", hand_name(is_left), static_cast(sample.kind), sample.layout_version, - sample.data.postdata == nullptr, sample.sample_monotonic_ns); + post_data_present, sample.sample_monotonic_ns); hand.sample_error_logged = true; } return; From 9c003ca70a75fb4523b763ece3f51800348b6e77 Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Thu, 12 Nov 2026 10:43:53 +0800 Subject: [PATCH 11/14] fix(omniglove): expose Catch2 tags as CTest labels Signed-off-by: zhouyingchun --- tests/cpp/plugins/omniglove/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cpp/plugins/omniglove/CMakeLists.txt b/tests/cpp/plugins/omniglove/CMakeLists.txt index 08dc2c2f23..f9fbf6a12e 100644 --- a/tests/cpp/plugins/omniglove/CMakeLists.txt +++ b/tests/cpp/plugins/omniglove/CMakeLists.txt @@ -5,4 +5,4 @@ add_executable(omniglove_plugin_tests test_hand_conversion.cpp) target_link_libraries(omniglove_plugin_tests PRIVATE Catch2::Catch2WithMain isaac::omniglove_plugin OmnigloveSDK::api ) -catch_discover_tests(omniglove_plugin_tests TEST_PREFIX "omniglove_") +catch_discover_tests(omniglove_plugin_tests TEST_PREFIX "omniglove_" ADD_TAGS_AS_LABELS) From 4d4ed74b8023dac15f58a455e77ac544bd2d3ea2 Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 13 Nov 2026 18:03:32 +0800 Subject: [PATCH 12/14] docs(omniglove): describe observable hand inactivity and recovery Signed-off-by: zhouyingchun --- docs/source/device/hand_tracking/omniglove.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/device/hand_tracking/omniglove.rst b/docs/source/device/hand_tracking/omniglove.rst index 050db9faa7..2981277771 100644 --- a/docs/source/device/hand_tracking/omniglove.rst +++ b/docs/source/device/hand_tracking/omniglove.rst @@ -139,8 +139,8 @@ the host records them as ``core.HandPoseRecord`` from ``hand.fbs``. ``XrTimeConverter``; the wrist is queried at that same time. - Unmapped, unconverged IK, invalid bone data, non-finite poses, zero quaternions and stale frames are withheld. A sample is pushed only once. - After 200 ms without fresh data the injector is destroyed, reporting the - hand inactive. A fresh valid sample and wrist pose recreate it. + Each hand is reported inactive when fresh, usable glove data or a valid + wrist pose is unavailable. Tracking resumes when both are available. The plugin publishes the operator's skeleton. It emits no robot joint commands or hand ``JointStateOutput`` data and defines no vendor hand schema. From d02ebabf04a957b01f267748aaedfbccaa02434f Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Fri, 13 Nov 2026 18:52:38 +0800 Subject: [PATCH 13/14] fix(omniglove): address remaining upstream review feedback Signed-off-by: zhouyingchun --- cmake/ClangFormat.cmake | 1 - .../source/device/hand_tracking/omniglove.rst | 45 ++-- .../device/hand_tracking/wuji_glove.rst | 4 +- src/plugins/omniglove/.gitignore | 4 - src/plugins/omniglove/60-omniglove-usb.rules | 2 +- src/plugins/omniglove/CMakeLists.txt | 48 ++--- src/plugins/omniglove/README.md | 84 +------- src/plugins/omniglove/app/CMakeLists.txt | 2 +- src/plugins/omniglove/app/main.cpp | 3 +- src/plugins/omniglove/core/CMakeLists.txt | 11 +- .../omniglove_hand_conversion.hpp | 9 +- .../inc/omniglove_plugin/omniglove_plugin.hpp | 45 ++-- .../core/omniglove_hand_conversion.cpp | 82 +++----- .../omniglove/core/omniglove_plugin.cpp | 196 ++++++++++-------- src/plugins/omniglove/sdk.cmake | 82 ++++---- src/plugins/plugin_utils/CMakeLists.txt | 2 + .../inc/plugin_utils/hand_pose_basis.hpp | 25 +++ .../inc/plugin_utils/wrist_pose_source.hpp | 32 +-- .../inc/plugin_utils/wrist_source_config.hpp | 45 ++++ .../plugin_utils/wrist_source_config.cpp | 31 ++- src/plugins/wuji_glove/wuji_glove_plugin.cpp | 65 +----- .../isaaccapture/cts/hand/checker/README.md | 3 +- .../cts/hand/checker/src/hand_cts/package.py | 13 +- .../omniglove/test_hand_conversion.cpp | 45 +++- tests/cpp/plugins/plugin_utils/CMakeLists.txt | 6 +- .../plugin_utils/test_wrist_source_config.cpp | 30 ++- 26 files changed, 452 insertions(+), 463 deletions(-) delete mode 100644 src/plugins/omniglove/.gitignore create mode 100644 src/plugins/plugin_utils/inc/plugin_utils/hand_pose_basis.hpp create mode 100644 src/plugins/plugin_utils/inc/plugin_utils/wrist_source_config.hpp diff --git a/cmake/ClangFormat.cmake b/cmake/ClangFormat.cmake index 26bdb91ecd..b0500093e1 100644 --- a/cmake/ClangFormat.cmake +++ b/cmake/ClangFormat.cmake @@ -38,7 +38,6 @@ set(_cf_patterns set(_cf_exclude_patterns ".*/build/.*" ".*/deps/.*" - ".*/\\.sdk/.*" ".*/manus/ManusSDK/.*" ".*/wuji_glove/wuji-sdk-c-.*" ".*/node_modules/.*" diff --git a/docs/source/device/hand_tracking/omniglove.rst b/docs/source/device/hand_tracking/omniglove.rst index 2981277771..5558301f53 100644 --- a/docs/source/device/hand_tracking/omniglove.rst +++ b/docs/source/device/hand_tracking/omniglove.rst @@ -16,7 +16,8 @@ host's hand retargeter. Hardware and prerequisites -------------------------- -- Linux x86_64 or aarch64; the current hardware validation used x86_64. +- Linux x86_64 or aarch64 with the system libusb runtime (``libusb-1.0-0`` + on Ubuntu); the current hardware validation used x86_64. - The Isaac Capture source-build dependencies and the `AgiLink OmniGlove SDK `_. - One left and one right OmniGlove for two-hand CTS, connected over USB. @@ -51,20 +52,19 @@ Install the USB rule on the host: ./src/plugins/omniglove/install_udev_rules.sh CMake retrieves the pinned SDK into the build directory and validates its -release checksums. The target architecture selects the SDK package. Runtime -libraries and license notices are installed with the plugin. +release checksums. The target architecture selects the SDK package. The SDK +runtime library and its license notices are installed with the plugin. libusb +is supplied by the operating system. .. code-block:: bash - cmake -S . -B out -DBUILD_PLUGIN_OMNIGLOVE=ON - cmake --build out - cmake --install out --component omniglove + cmake -S . -B build -DBUILD_PLUGIN_OMNIGLOVE=ON + cmake --build build + cmake --install build --component omniglove -For an explicit local release, set -``-DOMNIGLOVE_SDK_ROOT=/path/to/agilink_omniglove_sdk``; CMake does not modify it. -``-DOMNIGLOVE_SDK_REF=`` selects another managed release. Use a -full commit SHA for a reproducible override. Source build options remain under -the user's control. +Existing build directories using ``OMNIGLOVE_SDK_ROOT`` or ``OMNIGLOVE_SDK_REF`` +should remove those cache entries. Local SDK checkouts use CMake's standard +``-DFETCHCONTENT_SOURCE_DIR_OMNIGLOVE_SDK=/path/to/agilink_omniglove_sdk`` override. Running ------- @@ -149,13 +149,14 @@ Device status and CTS --------------------- The shared device status publisher reports left and right hand availability -under the configured plugin root. Stale SDK frames report a disconnected hand; -a fresh frame without a usable wrist reports a degraded hand. A discovered -glove that fails stream configuration or startup stops the plugin with an error. +under the configured plugin root. A glove that stops streaming reports a +disconnected hand. A glove still streaming without a usable skeleton or wrist +reports a degraded hand. A transient rejected sample retains the last usable +pose until it becomes stale. Stream configuration or startup failures and +incompatible SDK sample contracts stop the plugin with an error. Follow :doc:`/conformance/setup`, :doc:`/conformance/hand` and -:doc:`/conformance/submit` for recording, review and submission. Keep the take -summary and hardware evidence in the integration PR description. +:doc:`/conformance/submit` for recording, review and submission. Troubleshooting --------------- @@ -173,3 +174,15 @@ Troubleshooting invalid skeleton frames are intentionally withheld. - **Header/runtime ABI mismatch:** rebuild against the selected SDK and verify that the installed executable loads its accompanying runtime library. + +License +------- + +The plugin source is Apache-2.0. The AgiLink OmniGlove SDK is licensed under +MulanPSL-2.0; its shared library is redistributed with the plugin, together with +``LICENSE.omniglove-sdk`` and ``THIRD_PARTY_NOTICES.omniglove-sdk.md``. + +The SDK depends on libusb, licensed under LGPL-2.1-or-later. This plugin uses +the operating system's libusb and does not redistribute the SDK's bundled +libusb binary. The SDK notices describe the vendor's complete release; +only the SDK runtime library is copied into this plugin's install tree. diff --git a/docs/source/device/hand_tracking/wuji_glove.rst b/docs/source/device/hand_tracking/wuji_glove.rst index b293283d99..889ea36b00 100644 --- a/docs/source/device/hand_tracking/wuji_glove.rst +++ b/docs/source/device/hand_tracking/wuji_glove.rst @@ -108,7 +108,9 @@ controller offsets when needed: export WUJI_GLOVE_AIM_TO_WRIST_RIGHT="px,py,pz,qx,qy,qz,qw" Position values are in meters and quaternion values use ``(x, y, z, w)``. -The built-in offsets assume mirrored left/right mounts. +The built-in offsets assume mirrored left/right mounts. Unknown wrist-source +modes and malformed explicit offsets stop the plugin with an error; unset +offsets use the built-in values. Troubleshooting --------------- diff --git a/src/plugins/omniglove/.gitignore b/src/plugins/omniglove/.gitignore deleted file mode 100644 index 043f278876..0000000000 --- a/src/plugins/omniglove/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. -# SPDX-License-Identifier: Apache-2.0 - -/.sdk/ diff --git a/src/plugins/omniglove/60-omniglove-usb.rules b/src/plugins/omniglove/60-omniglove-usb.rules index 47cbb1675d..331a24eb9c 100644 --- a/src/plugins/omniglove/60-omniglove-usb.rules +++ b/src/plugins/omniglove/60-omniglove-usb.rules @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. # SPDX-License-Identifier: Apache-2.0 -# Runtime interfaces only; bootloader interfaces intentionally stay restricted. +# Grant access to USB devices with the listed runtime product IDs. ACTION=="add|change", SUBSYSTEM=="usb", ENV{DEVTYPE}=="usb_device", ATTR{idVendor}=="cafe", ATTR{idProduct}=="4001", GROUP="plugdev", MODE="0660", TAG+="uaccess" ACTION=="add|change", SUBSYSTEM=="usb", ENV{DEVTYPE}=="usb_device", ATTR{idVendor}=="cafe", ATTR{idProduct}=="4002", GROUP="plugdev", MODE="0660", TAG+="uaccess" diff --git a/src/plugins/omniglove/CMakeLists.txt b/src/plugins/omniglove/CMakeLists.txt index 1944ca1d6d..9f9dd1cef6 100644 --- a/src/plugins/omniglove/CMakeLists.txt +++ b/src/plugins/omniglove/CMakeLists.txt @@ -7,49 +7,25 @@ include("${CMAKE_CURRENT_LIST_DIR}/sdk.cmake") add_subdirectory(core) add_subdirectory(app) -get_filename_component(_omniglove_sdk_library_dir "${OMNIGLOVE_SDK_LIBRARY}" DIRECTORY) -file(GLOB _omniglove_sdk_libraries "${_omniglove_sdk_library_dir}/libomniglove.so*") -file(GLOB _omniglove_libusb_libraries "${_omniglove_sdk_library_dir}/omniglove/libusb-1.0.so*") - -if(NOT _omniglove_sdk_libraries) - message(FATAL_ERROR "No libomniglove.so files found beside ${OMNIGLOVE_SDK_LIBRARY}") -endif() -if(NOT _omniglove_libusb_libraries) - message(FATAL_ERROR "No bundled libusb runtime found under ${_omniglove_sdk_library_dir}/omniglove") -endif() +# Install the verified target artifact; SDK linker aliases may not be in its manifest. +get_target_property(_omniglove_sdk_soname OmnigloveSDK::api IMPORTED_SONAME) +install(FILES "${OMNIGLOVE_SDK_LIBRARY}" + DESTINATION plugins/omniglove + RENAME "${_omniglove_sdk_soname}" + COMPONENT omniglove +) -install(FILES ${_omniglove_sdk_libraries} +install(FILES "${OMNIGLOVE_SDK_RELEASE_ROOT}/LICENSE" DESTINATION plugins/omniglove + RENAME LICENSE.omniglove-sdk COMPONENT omniglove ) -install(FILES ${_omniglove_libusb_libraries} - DESTINATION plugins/omniglove/omniglove +install(FILES "${OMNIGLOVE_SDK_RELEASE_ROOT}/THIRD_PARTY_NOTICES.md" + DESTINATION plugins/omniglove + RENAME THIRD_PARTY_NOTICES.omniglove-sdk.md COMPONENT omniglove ) -if(OMNIGLOVE_SDK_RELEASE_ROOT) - install(FILES - "${OMNIGLOVE_SDK_RELEASE_ROOT}/LICENSE" - DESTINATION plugins/omniglove - RENAME LICENSE.omniglove-sdk - COMPONENT omniglove - ) - install(FILES - "${OMNIGLOVE_SDK_RELEASE_ROOT}/THIRD_PARTY_NOTICES.md" - DESTINATION plugins/omniglove - RENAME THIRD_PARTY_NOTICES.omniglove-sdk.md - COMPONENT omniglove - ) - install(FILES - "${OMNIGLOVE_SDK_RELEASE_ROOT}/licenses/libusb-COPYING" - DESTINATION plugins/omniglove - RENAME LICENSE.libusb - COMPONENT omniglove - ) -else() - message(WARNING "OMNIGLOVE_SDK_RELEASE_ROOT is unset; SDK license files will not be installed") -endif() - install(FILES plugin.yaml README.md 60-omniglove-usb.rules DESTINATION plugins/omniglove COMPONENT omniglove diff --git a/src/plugins/omniglove/README.md b/src/plugins/omniglove/README.md index 6967809ea2..b23b26a674 100644 --- a/src/plugins/omniglove/README.md +++ b/src/plugins/omniglove/README.md @@ -3,84 +3,10 @@ SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. SPDX-License-Identifier: Apache-2.0 --> -# AgiLink OmniGlove plugin +# AgiLink OmniGlove -This plugin converts the OmniGlove SDK's wrist-relative 21-landmark -`OMNIGLOVE_STREAM_POST_DATA` stream to OpenXR hand joints and injects it into -the Isaac Capture runtime. It supports one left and one right USB glove. +The Linux plugin publishes OmniGlove finger tracking as OpenXR hands for +Isaac Capture. -## Install - -Install the host USB rule once, outside a container: - -```bash -./src/plugins/omniglove/install_udev_rules.sh -``` - -Configure and build from the Isaac Capture repository root: - -```bash -cmake -S . -B out -DBUILD_PLUGIN_OMNIGLOVE=ON -cmake --build out -cmake --install out --component omniglove -``` - -CMake retrieves a pinned SDK release into the build directory, verifies its -checksums, and selects the package for the target architecture. It installs -runtime libraries and license notices with the plugin. Set -`-DOMNIGLOVE_SDK_ROOT=/path/to/agilink_omniglove_sdk` to explicitly use a local -release without modifying it, or `-DOMNIGLOVE_SDK_REF=` to select -a managed release. A full commit SHA makes an override reproducible. - -Missing or changed SDK APIs fail compilation or linking. At runtime, a -header/library ABI mismatch is rejected before device discovery. - -## Run - -Start CloudXR, load its environment, connect a calibrated glove, and run: - -```bash -python -m isaaccapture.cloudxr.service start -source ~/.cloudxr/run/cloudxr.env -./install/plugins/omniglove/omniglove_plugin -``` - -The glove provides finger poses relative to its wrist but no world-space wrist -pose. By default the plugin obtains wrist pose from OpenXR optical hand -tracking. Set `OMNIGLOVE_WRIST_SOURCE` to `controller` or `auto` only after -measuring controller-aim-to-wrist transforms and exporting them as -`px,py,pz,qx,qy,qz,qw`: - -```bash -export OMNIGLOVE_WRIST_SOURCE=controller -export OMNIGLOVE_AIM_TO_WRIST_LEFT="px,py,pz,qx,qy,qz,qw" -export OMNIGLOVE_AIM_TO_WRIST_RIGHT="px,py,pz,qx,qy,qz,qw" -``` - -Replace these placeholders with measured offsets for the current mount. -Controller and Auto modes reject missing transforms. Unknown wrist modes and -malformed offsets are errors; optical mode does not read the offsets. - -## Data contract - -- SDK positions are metres and quaternions are `(x, y, z, w)`. -- Finger orientations are converted from SDK distal +Z to OpenXR distal - -Z by a local Y half-turn, preserving dorsal +Y. The wrist retains its - proximal +Z; the right-hand wrist basis is converted separately. -- The 21 supplied wrist/finger landmarks map to their OpenXR equivalents. -- OpenXR palm and the index, middle, ring, and little metacarpals remain - invalid because the SDK stream does not supply them. -- Unmapped, unconverged, invalid-bone-model, non-finite, zero-quaternion, and - stale frames are not injected. - -## Device status and CTS - -The plugin publishes both hand states through the shared device status -publisher under the supplied `--plugin-root-id`. A stale SDK frame makes a -hand disconnected; a fresh frame without a usable wrist pose makes it degraded. -A glove that is discovered but fails stream configuration or startup stops the -plugin with an error. - -Follow the official [hand CTS procedure](https://nvidia.github.io/IsaacCapture/main/conformance/hand.html) -and [submission procedure](https://nvidia.github.io/IsaacCapture/main/conformance/submit.html). -Keep take results and hardware evidence in the integration PR description. +Installation, wrist calibration, running, licensing and troubleshooting: +[AgiLink OmniGlove documentation](https://nvidia.github.io/IsaacCapture/main/device/hand_tracking/omniglove.html). diff --git a/src/plugins/omniglove/app/CMakeLists.txt b/src/plugins/omniglove/app/CMakeLists.txt index 409aeea089..43a740fb1f 100644 --- a/src/plugins/omniglove/app/CMakeLists.txt +++ b/src/plugins/omniglove/app/CMakeLists.txt @@ -6,7 +6,7 @@ add_executable(omniglove_plugin main.cpp) target_link_libraries(omniglove_plugin PRIVATE isaac::omniglove_plugin) set_target_properties(omniglove_plugin PROPERTIES - INSTALL_RPATH "$ORIGIN;$ORIGIN/omniglove;$ORIGIN/../../${CMAKE_INSTALL_LIBDIR}" + INSTALL_RPATH "$ORIGIN" ) install(TARGETS omniglove_plugin diff --git a/src/plugins/omniglove/app/main.cpp b/src/plugins/omniglove/app/main.cpp index 3e2b930628..f6ff0e25f4 100644 --- a/src/plugins/omniglove/app/main.cpp +++ b/src/plugins/omniglove/app/main.cpp @@ -49,7 +49,8 @@ try while (!stop_requested.load(std::memory_order_relaxed)) { plugin.update(); - std::this_thread::sleep_for(std::chrono::milliseconds(2)); + constexpr auto kFramePeriod = plugins::omniglove::OmniglovePlugin::kSourceInterval; + std::this_thread::sleep_for(kFramePeriod); } return 0; } diff --git a/src/plugins/omniglove/core/CMakeLists.txt b/src/plugins/omniglove/core/CMakeLists.txt index 39f8ad4433..b9bef1b202 100644 --- a/src/plugins/omniglove/core/CMakeLists.txt +++ b/src/plugins/omniglove/core/CMakeLists.txt @@ -4,6 +4,8 @@ add_library(omniglove_plugin_core STATIC omniglove_hand_conversion.cpp omniglove_plugin.cpp + inc/omniglove_plugin/omniglove_hand_conversion.hpp + inc/omniglove_plugin/omniglove_plugin.hpp ) target_include_directories(omniglove_plugin_core @@ -25,13 +27,4 @@ target_link_libraries(omniglove_plugin_core log_bridge::log_bridge_core ) -if(NOT OMNIGLOVE_SDK_GIT_COMMIT) - set(OMNIGLOVE_SDK_GIT_COMMIT "unknown") -endif() -target_compile_definitions(omniglove_plugin_core - PRIVATE - OMNIGLOVE_SDK_RELEASE_VERSION="${OmnigloveSDK_VERSION}" - OMNIGLOVE_SDK_GIT_COMMIT="${OMNIGLOVE_SDK_GIT_COMMIT}" -) - add_library(isaac::omniglove_plugin ALIAS omniglove_plugin_core) diff --git a/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_hand_conversion.hpp b/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_hand_conversion.hpp index 1ba1ad4114..5c3d867af0 100644 --- a/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_hand_conversion.hpp +++ b/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_hand_conversion.hpp @@ -7,6 +7,7 @@ #include #include +#include struct omniglove_post_data; @@ -19,16 +20,16 @@ inline constexpr uint32_t kPostBoneModelValid = 1U << 2U; enum class ConversionStatus { - Success, NotMapped, IkNotConverged, BoneModelInvalid, InvalidPose, }; -ConversionStatus convert_post_data(const omniglove_post_data& post, - bool is_left, - std::array& joints); +using HandJoints = std::array; +using ConversionResult = std::variant; + +ConversionResult convert_post_data(const omniglove_post_data& post, bool is_left); const char* conversion_status_name(ConversionStatus status) noexcept; diff --git a/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_plugin.hpp b/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_plugin.hpp index 07b7b256e0..a50c0f9cff 100644 --- a/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_plugin.hpp +++ b/src/plugins/omniglove/core/inc/omniglove_plugin/omniglove_plugin.hpp @@ -14,12 +14,14 @@ #include #include +#include +#include #include +#include #include #include #include #include -#include struct omniglove_session; struct omniglove_stream_sample; @@ -27,9 +29,17 @@ struct omniglove_stream_sample; namespace plugins::omniglove { +struct SdkSessionDeleter +{ + void operator()(omniglove_session* session) const noexcept; +}; +using SdkSession = std::unique_ptr; + class OmniglovePlugin { public: + static constexpr auto kSourceInterval = std::chrono::milliseconds(8); + explicit OmniglovePlugin(const std::string& plugin_root_id); ~OmniglovePlugin(); @@ -44,14 +54,10 @@ class OmniglovePlugin uint64_t source_sequence = 0; }; - struct HandState + struct CallbackState { std::optional frame; - std::unique_ptr injector; - int64_t last_pushed_ns = 0; - core::PluginDeviceState state = core::PluginDeviceState_DISCONNECTED; - core::PluginDeviceReason reason = core::PluginDeviceReason_NO_HARDWARE_SIGNAL; - bool sample_error_logged = false; + int64_t last_received_ns = 0; }; struct StreamContext @@ -60,18 +66,26 @@ class OmniglovePlugin bool is_left; }; - struct SessionDeleter + struct HandState { - void operator()(omniglove_session* session) const noexcept; + std::optional stream; + std::unique_ptr injector; + int64_t last_pushed_ns = 0; + core::PluginDeviceState state = core::PluginDeviceState_DISCONNECTED; + core::PluginDeviceReason reason = core::PluginDeviceReason_NO_HARDWARE_SIGNAL; }; - static void stream_callback(const omniglove_stream_sample* sample, void* user_data); + static constexpr std::size_t hand_index(bool is_left) + { + return is_left ? 0 : 1; + } + + HandState& hand_state(bool is_left); + static void stream_callback(const omniglove_stream_sample* sample, void* user_data) noexcept; - void initialize_sdk(); void start_device_streams(); void on_sample(const omniglove_stream_sample& sample, bool is_left); - void invalidate_hand(bool is_left); - void pump_hand(HandState& hand, bool is_left, const std::optional& frame); + void pump_hand(HandState& hand, bool is_left, const CallbackState& input); std::shared_ptr m_logger = isaaccapture::Logger::get("isaaccapture.plugins.omniglove.OmniglovePlugin"); @@ -84,9 +98,10 @@ class OmniglovePlugin std::mutex m_frame_mutex; std::array m_hands; - std::vector> m_streams; + std::array m_inputs; + std::exception_ptr m_callback_error; // Destroy the SDK first: its callbacks borrow the contexts, frames and mutex above. - std::unique_ptr m_sdk_session; + SdkSession m_sdk_session; }; } // namespace plugins::omniglove diff --git a/src/plugins/omniglove/core/omniglove_hand_conversion.cpp b/src/plugins/omniglove/core/omniglove_hand_conversion.cpp index 98527e9a1c..a7402eec13 100644 --- a/src/plugins/omniglove/core/omniglove_hand_conversion.cpp +++ b/src/plugins/omniglove/core/omniglove_hand_conversion.cpp @@ -4,6 +4,8 @@ #include "inc/omniglove_plugin/omniglove_hand_conversion.hpp" #include +#include +#include #include #include @@ -25,48 +27,43 @@ struct JointSource const omniglove_pose6_t* pose; }; -std::array joint_sources(const omniglove_post_data_t& post) +auto joint_sources(const omniglove_post_data_t& post) { - return { JointSource{ XR_HAND_JOINT_WRIST_EXT, &post.wrist.landmark }, - JointSource{ XR_HAND_JOINT_THUMB_METACARPAL_EXT, &post.thumb.landmarks.cmc }, - JointSource{ XR_HAND_JOINT_THUMB_PROXIMAL_EXT, &post.thumb.landmarks.mcp }, - JointSource{ XR_HAND_JOINT_THUMB_DISTAL_EXT, &post.thumb.landmarks.ip }, - JointSource{ XR_HAND_JOINT_THUMB_TIP_EXT, &post.thumb.landmarks.tip }, - JointSource{ XR_HAND_JOINT_INDEX_PROXIMAL_EXT, &post.index.landmarks.mcp }, - JointSource{ XR_HAND_JOINT_INDEX_INTERMEDIATE_EXT, &post.index.landmarks.pip }, - JointSource{ XR_HAND_JOINT_INDEX_DISTAL_EXT, &post.index.landmarks.dip }, - JointSource{ XR_HAND_JOINT_INDEX_TIP_EXT, &post.index.landmarks.tip }, - JointSource{ XR_HAND_JOINT_MIDDLE_PROXIMAL_EXT, &post.middle.landmarks.mcp }, - JointSource{ XR_HAND_JOINT_MIDDLE_INTERMEDIATE_EXT, &post.middle.landmarks.pip }, - JointSource{ XR_HAND_JOINT_MIDDLE_DISTAL_EXT, &post.middle.landmarks.dip }, - JointSource{ XR_HAND_JOINT_MIDDLE_TIP_EXT, &post.middle.landmarks.tip }, - JointSource{ XR_HAND_JOINT_RING_PROXIMAL_EXT, &post.ring.landmarks.mcp }, - JointSource{ XR_HAND_JOINT_RING_INTERMEDIATE_EXT, &post.ring.landmarks.pip }, - JointSource{ XR_HAND_JOINT_RING_DISTAL_EXT, &post.ring.landmarks.dip }, - JointSource{ XR_HAND_JOINT_RING_TIP_EXT, &post.ring.landmarks.tip }, - JointSource{ XR_HAND_JOINT_LITTLE_PROXIMAL_EXT, &post.pinky.landmarks.mcp }, - JointSource{ XR_HAND_JOINT_LITTLE_INTERMEDIATE_EXT, &post.pinky.landmarks.pip }, - JointSource{ XR_HAND_JOINT_LITTLE_DISTAL_EXT, &post.pinky.landmarks.dip }, - JointSource{ XR_HAND_JOINT_LITTLE_TIP_EXT, &post.pinky.landmarks.tip } }; -} - -bool finite(float value) -{ - return std::isfinite(value); + return std::array{ JointSource{ XR_HAND_JOINT_WRIST_EXT, &post.wrist.landmark }, + JointSource{ XR_HAND_JOINT_THUMB_METACARPAL_EXT, &post.thumb.landmarks.cmc }, + JointSource{ XR_HAND_JOINT_THUMB_PROXIMAL_EXT, &post.thumb.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_THUMB_DISTAL_EXT, &post.thumb.landmarks.ip }, + JointSource{ XR_HAND_JOINT_THUMB_TIP_EXT, &post.thumb.landmarks.tip }, + JointSource{ XR_HAND_JOINT_INDEX_PROXIMAL_EXT, &post.index.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_INDEX_INTERMEDIATE_EXT, &post.index.landmarks.pip }, + JointSource{ XR_HAND_JOINT_INDEX_DISTAL_EXT, &post.index.landmarks.dip }, + JointSource{ XR_HAND_JOINT_INDEX_TIP_EXT, &post.index.landmarks.tip }, + JointSource{ XR_HAND_JOINT_MIDDLE_PROXIMAL_EXT, &post.middle.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_MIDDLE_INTERMEDIATE_EXT, &post.middle.landmarks.pip }, + JointSource{ XR_HAND_JOINT_MIDDLE_DISTAL_EXT, &post.middle.landmarks.dip }, + JointSource{ XR_HAND_JOINT_MIDDLE_TIP_EXT, &post.middle.landmarks.tip }, + JointSource{ XR_HAND_JOINT_RING_PROXIMAL_EXT, &post.ring.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_RING_INTERMEDIATE_EXT, &post.ring.landmarks.pip }, + JointSource{ XR_HAND_JOINT_RING_DISTAL_EXT, &post.ring.landmarks.dip }, + JointSource{ XR_HAND_JOINT_RING_TIP_EXT, &post.ring.landmarks.tip }, + JointSource{ XR_HAND_JOINT_LITTLE_PROXIMAL_EXT, &post.pinky.landmarks.mcp }, + JointSource{ XR_HAND_JOINT_LITTLE_INTERMEDIATE_EXT, &post.pinky.landmarks.pip }, + JointSource{ XR_HAND_JOINT_LITTLE_DISTAL_EXT, &post.pinky.landmarks.dip }, + JointSource{ XR_HAND_JOINT_LITTLE_TIP_EXT, &post.pinky.landmarks.tip } }; } bool convert_pose(const omniglove_pose6_t& source, bool is_left, XrPosef& destination) { for (float value : source.position_m) { - if (!finite(value)) + if (!std::isfinite(value)) { return false; } } for (float value : source.quaternion_xyzw) { - if (!finite(value)) + if (!std::isfinite(value)) { return false; } @@ -84,24 +81,15 @@ bool convert_pose(const omniglove_pose6_t& source, bool is_left, XrPosef& destin destination.orientation = { source.quaternion_xyzw[0] / norm, source.quaternion_xyzw[1] / norm, source.quaternion_xyzw[2] / norm, source.quaternion_xyzw[3] / norm }; - // OmniGlove's right wrist frame has +Y palmward. OpenXR requires +Y - // dorsal on both hands, so conjugate right-hand poses by a Z half-turn. - if (!is_left) - { - destination.position.x = -destination.position.x; - destination.position.y = -destination.position.y; - destination.orientation.x = -destination.orientation.x; - destination.orientation.y = -destination.orientation.y; - } + destination = plugin_utils::remap_to_openxr_basis(destination, is_left); return true; } } // namespace -ConversionStatus convert_post_data(const omniglove_post_data_t& post, - bool is_left, - std::array& joints) +ConversionResult convert_post_data(const omniglove_post_data_t& post, bool is_left) { + HandJoints joints{}; for (auto& joint : joints) { joint = {}; @@ -127,10 +115,6 @@ ConversionStatus convert_post_data(const omniglove_post_data_t& post, XrHandJointLocationEXT& destination = joints[static_cast(source.joint)]; if (!convert_pose(*source.pose, is_left, destination.pose)) { - for (auto& joint : joints) - { - joint.locationFlags = 0; - } return ConversionStatus::InvalidPose; } if (source.joint != XR_HAND_JOINT_WRIST_EXT) @@ -138,20 +122,18 @@ ConversionStatus convert_post_data(const omniglove_post_data_t& post, // SDK finger frames point +Z toward the tip; OpenXR points -Z. // Post-multiply by a local Y half-turn to preserve dorsal +Y. // The SDK wrist already has proximal +Z and must not be flipped. - const XrQuaternionf q = destination.pose.orientation; - destination.pose.orientation = { -q.z, q.w, q.x, -q.y }; + constexpr XrPosef kYHalfTurn{ { 0.0F, 1.0F, 0.0F, 0.0F }, { 0.0F, 0.0F, 0.0F } }; + destination.pose = oxr_utils::multiply_poses(destination.pose, kYHalfTurn); } destination.locationFlags = kValidFlags; } - return ConversionStatus::Success; + return joints; } const char* conversion_status_name(ConversionStatus status) noexcept { switch (status) { - case ConversionStatus::Success: - return "success"; case ConversionStatus::NotMapped: return "not mapped"; case ConversionStatus::IkNotConverged: diff --git a/src/plugins/omniglove/core/omniglove_plugin.cpp b/src/plugins/omniglove/core/omniglove_plugin.cpp index dbb4ac926e..1807559f81 100644 --- a/src/plugins/omniglove/core/omniglove_plugin.cpp +++ b/src/plugins/omniglove/core/omniglove_plugin.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace plugins::omniglove { @@ -20,7 +21,6 @@ namespace constexpr auto kSourceTimeout = std::chrono::milliseconds(200); constexpr int64_t kStaleThresholdNs = std::chrono::duration_cast(kSourceTimeout).count(); -constexpr uint32_t kSourceIntervalMs = 8; constexpr uint32_t kCallbackQueueCapacity = 64; constexpr XrSpaceLocationFlags kTrackedFlags = XR_SPACE_LOCATION_POSITION_TRACKED_BIT | XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT; @@ -30,9 +30,47 @@ const char* hand_name(bool is_left) return is_left ? "left" : "right"; } +SdkSession create_sdk_session(const std::shared_ptr& logger) +{ + // Accept any SDK ABI version when its header and runtime agree. A new ABI + // number is not itself an incompatibility; required API changes fail build. + const uint32_t runtime_abi = omniglove_abi_version(); + if (runtime_abi != OMNIGLOVE_ABI_VERSION) + { + throw std::runtime_error("Omniglove header/runtime ABI mismatch: header=" + + std::to_string(OMNIGLOVE_ABI_VERSION) + " runtime=" + std::to_string(runtime_abi)); + } + logger->info( + "Omniglove ABI {}.{}.{}", (runtime_abi >> 16U) & 0xffU, (runtime_abi >> 8U) & 0xffU, runtime_abi & 0xffU); + + omniglove_session_t* session = nullptr; + omniglove_init_report_t* report = nullptr; + const omniglove_error_t error = omniglove_initialize(nullptr, &session, &report); + SdkSession owned_session(session); + const std::unique_ptr owned_report( + report, &omniglove_init_report_destroy); + + if (report != nullptr) + { + const size_t size = omniglove_init_report_message(report, nullptr, 0); + if (size > 1) + { + std::vector message(size); + omniglove_init_report_message(report, message.data(), message.size()); + logger->info("SDK discovery: {}", message.data()); + } + } + + if (error != OMNIGLOVE_ERROR_NONE || session == nullptr) + { + throw std::runtime_error(std::string("omniglove_initialize failed: ") + omniglove_error_name(error)); + } + return owned_session; +} + } // namespace -void OmniglovePlugin::SessionDeleter::operator()(omniglove_session_t* session) const noexcept +void SdkSessionDeleter::operator()(omniglove_session_t* session) const noexcept { omniglove_session_destroy(session); } @@ -49,11 +87,8 @@ OmniglovePlugin::OmniglovePlugin(const std::string& plugin_root_id) wrist_config.left_aim_to_wrist = plugin_utils::pose_from_env("OMNIGLOVE_AIM_TO_WRIST_LEFT"); wrist_config.right_aim_to_wrist = plugin_utils::pose_from_env("OMNIGLOVE_AIM_TO_WRIST_RIGHT"); } - const char* wrist_mode = wrist_config.mode == plugin_utils::WristSourceMode::HandTracking ? "hand_tracking" : - wrist_config.mode == plugin_utils::WristSourceMode::Controller ? "controller" : - "auto"; - m_logger->info("wrist source {}", wrist_mode); - initialize_sdk(); + m_logger->info("wrist source {}", plugin_utils::to_string(wrist_config.mode)); + m_sdk_session = create_sdk_session(m_logger); const auto wrist_requirements = plugin_utils::WristPoseSource::collect_requirements(wrist_config.mode); auto extensions = core::DeviceIOSession::get_required_extensions(wrist_requirements.trackers); @@ -71,52 +106,15 @@ OmniglovePlugin::OmniglovePlugin(const std::string& plugin_root_id) m_status_publisher.emplace(handles, plugin_root_id); start_device_streams(); - m_logger->info("initialized with {} glove stream(s)", m_streams.size()); + m_logger->info( + "initialized with {} glove stream(s)", + std::count_if(m_hands.begin(), m_hands.end(), [](const auto& hand) { return hand.stream.has_value(); })); } OmniglovePlugin::~OmniglovePlugin() = default; -void OmniglovePlugin::initialize_sdk() -{ - // Accept any SDK ABI version when its header and runtime agree. A new ABI - // number is not itself an incompatibility; required API changes fail build. - const uint32_t runtime_abi = omniglove_abi_version(); - if (runtime_abi != OMNIGLOVE_ABI_VERSION) - { - throw std::runtime_error("Omniglove header/runtime ABI mismatch: header=" + - std::to_string(OMNIGLOVE_ABI_VERSION) + " runtime=" + std::to_string(runtime_abi)); - } - m_logger->info( - "Omniglove ABI {}.{}.{}", (runtime_abi >> 16U) & 0xffU, (runtime_abi >> 8U) & 0xffU, runtime_abi & 0xffU); - m_logger->info("Omniglove SDK release {} ({})", OMNIGLOVE_SDK_RELEASE_VERSION, OMNIGLOVE_SDK_GIT_COMMIT); - - omniglove_session_t* session = nullptr; - omniglove_init_report_t* report = nullptr; - const omniglove_error_t error = omniglove_initialize(nullptr, &session, &report); - - if (report != nullptr) - { - const size_t size = omniglove_init_report_message(report, nullptr, 0); - if (size > 1) - { - std::vector message(size); - omniglove_init_report_message(report, message.data(), message.size()); - m_logger->info("SDK discovery: {}", message.data()); - } - omniglove_init_report_destroy(report); - } - - if (error != OMNIGLOVE_ERROR_NONE || session == nullptr) - { - throw std::runtime_error(std::string("omniglove_initialize failed: ") + omniglove_error_name(error)); - } - m_sdk_session.reset(session); -} - void OmniglovePlugin::start_device_streams() { - bool have_left = false; - bool have_right = false; const size_t count = omniglove_session_device_count(m_sdk_session.get()); for (size_t index = 0; index < count; ++index) @@ -124,8 +122,7 @@ void OmniglovePlugin::start_device_streams() omniglove_device_t* device = omniglove_session_device_at(m_sdk_session.get(), index); if (device == nullptr) { - m_logger->warn("ignoring null glove entry {}", index); - continue; + throw std::runtime_error("SDK returned a null glove for valid index " + std::to_string(index)); } const omniglove_hand_t hand = omniglove_device_hand(device); if (hand == OMNIGLOVE_HAND_UNKNOWN) @@ -134,25 +131,25 @@ void OmniglovePlugin::start_device_streams() continue; } const bool is_left = hand == OMNIGLOVE_HAND_LEFT; - bool& occupied = is_left ? have_left : have_right; - if (occupied) + auto& hand_state = this->hand_state(is_left); + if (hand_state.stream) { m_logger->warn("ignoring a second {} glove", hand_name(is_left)); continue; } // Own the callback context before starting the stream, including failure cleanup. - m_streams.push_back(std::make_unique(StreamContext{ this, is_left })); + hand_state.stream.emplace(StreamContext{ this, is_left }); omniglove_stream_config_t config{}; config.struct_size = sizeof(config); config.kind = OMNIGLOVE_STREAM_POST_DATA; config.stream_mode = OMNIGLOVE_STREAM_MODE_DEFAULT; config.output_mode = OMNIGLOVE_STREAM_OUTPUT_DIRECT; - config.source_interval_ms = kSourceIntervalMs; + config.source_interval_ms = static_cast(kSourceInterval.count()); config.source_timeout_ms = static_cast(kSourceTimeout.count()); config.callback = &OmniglovePlugin::stream_callback; - config.callback_user_data = m_streams.back().get(); + config.callback_user_data = &*hand_state.stream; config.callback_queue_capacity = kCallbackQueueCapacity; const auto configure_error = omniglove_device_stream_configure(device, &config); @@ -167,24 +164,35 @@ void OmniglovePlugin::start_device_streams() throw std::runtime_error(std::string("could not start ") + hand_name(is_left) + " POST_DATA stream: " + std::to_string(static_cast(start_error))); } - occupied = true; m_logger->info("started {} glove at {} Hz", hand_name(is_left), omniglove_device_nominal_source_rate_hz(device)); } - if (m_streams.empty()) + if (std::none_of(m_hands.begin(), m_hands.end(), [](const auto& hand) { return hand.stream.has_value(); })) { throw std::runtime_error("no left or right Omniglove POST_DATA stream could be started"); } } -void OmniglovePlugin::stream_callback(const omniglove_stream_sample_t* sample, void* user_data) +void OmniglovePlugin::stream_callback(const omniglove_stream_sample_t* sample, void* user_data) noexcept { if (sample == nullptr) { return; } auto* context = static_cast(user_data); - context->plugin->on_sample(*sample, context->is_left); + try + { + context->plugin->on_sample(*sample, context->is_left); + } + catch (...) + { + // Exceptions must reach update(), never cross the SDK's C callback boundary. + std::lock_guard lock(context->plugin->m_frame_mutex); + if (!context->plugin->m_callback_error) + { + context->plugin->m_callback_error = std::current_exception(); + } + } } void OmniglovePlugin::on_sample(const omniglove_stream_sample_t& sample, bool is_left) @@ -194,43 +202,42 @@ void OmniglovePlugin::on_sample(const omniglove_stream_sample_t& sample, bool is if (!post_data_present || sample.layout_version != OMNIGLOVE_STREAM_POST_LAYOUT_VERSION || sample.sample_monotonic_ns > std::numeric_limits::max()) { - std::lock_guard lock(m_frame_mutex); - auto& hand = m_hands[is_left ? 0 : 1]; - hand.frame.reset(); - if (!hand.sample_error_logged) - { - m_logger->error("{} SDK sample contract mismatch: kind {}, layout {}, post data present {}, timestamp {}", - hand_name(is_left), static_cast(sample.kind), sample.layout_version, - post_data_present, sample.sample_monotonic_ns); - hand.sample_error_logged = true; - } - return; + throw std::runtime_error(std::string(hand_name(is_left)) + " SDK sample contract mismatch: kind=" + + std::to_string(sample.kind) + " layout=" + std::to_string(sample.layout_version) + + " post_data_present=" + std::to_string(post_data_present) + + " timestamp=" + std::to_string(sample.sample_monotonic_ns)); } - HandFrame converted; - const ConversionStatus status = convert_post_data(*sample.data.postdata, is_left, converted.joints); - if (status != ConversionStatus::Success) { - invalidate_hand(is_left); - m_logger->debug("{} POST_DATA rejected: {}", hand_name(is_left), conversion_status_name(status)); + std::lock_guard lock(m_frame_mutex); + m_inputs[hand_index(is_left)].last_received_ns = core::os_monotonic_now_ns(); + } + auto result = convert_post_data(*sample.data.postdata, is_left); + const auto* joints = std::get_if(&result); + if (joints == nullptr) + { + m_logger->debug("{} POST_DATA rejected: {}", hand_name(is_left), + conversion_status_name(std::get(result))); return; } + HandFrame converted; + converted.joints = *joints; converted.sample_monotonic_ns = static_cast(sample.sample_monotonic_ns); converted.connection_generation = sample.connection_generation; converted.source_sequence = sample.source_sequence; std::lock_guard lock(m_frame_mutex); - m_hands[is_left ? 0 : 1].frame = converted; + m_inputs[hand_index(is_left)].frame = converted; } -void OmniglovePlugin::invalidate_hand(bool is_left) +OmniglovePlugin::HandState& OmniglovePlugin::hand_state(bool is_left) { - std::lock_guard lock(m_frame_mutex); - m_hands[is_left ? 0 : 1].frame.reset(); + return m_hands[hand_index(is_left)]; } -void OmniglovePlugin::pump_hand(HandState& hand, bool is_left, const std::optional& frame) +void OmniglovePlugin::pump_hand(HandState& hand, bool is_left, const CallbackState& input) { + const auto& frame = input.frame; auto& injector = hand.injector; const int64_t now_ns = core::os_monotonic_now_ns(); const bool fresh = @@ -242,8 +249,9 @@ void OmniglovePlugin::pump_hand(HandState& hand, bool is_left, const std::option m_logger->info("{} hand inactive: SDK frame stale or invalid", hand_name(is_left)); } injector.reset(); - hand.state = core::PluginDeviceState_DISCONNECTED; - hand.reason = core::PluginDeviceReason_NO_HARDWARE_SIGNAL; + const bool receiving = input.last_received_ns > 0 && now_ns - input.last_received_ns < kStaleThresholdNs; + hand.state = receiving ? core::PluginDeviceState_DEGRADED : core::PluginDeviceState_DISCONNECTED; + hand.reason = receiving ? core::PluginDeviceReason_NO_CURRENT_DATA : core::PluginDeviceReason_NO_HARDWARE_SIGNAL; return; } if (frame->sample_monotonic_ns == hand.last_pushed_ns) @@ -295,17 +303,27 @@ void OmniglovePlugin::pump_hand(HandState& hand, bool is_left, const std::option void OmniglovePlugin::update() { - m_deviceio_session->update(); - std::array, 2> frames; + std::array inputs; + std::exception_ptr callback_error; { std::lock_guard lock(m_frame_mutex); - frames = { m_hands[0].frame, m_hands[1].frame }; + inputs = m_inputs; + callback_error = m_callback_error; + } + if (callback_error) + { + std::rethrow_exception(callback_error); + } + m_deviceio_session->update(); + for (bool is_left : { true, false }) + { + pump_hand(hand_state(is_left), is_left, inputs[hand_index(is_left)]); } - pump_hand(m_hands[0], true, frames[0]); - pump_hand(m_hands[1], false, frames[1]); - m_status_publisher->publish_if_changed({ { "/hand/left", m_hands[0].state, m_hands[0].reason, {} }, - { "/hand/right", m_hands[1].state, m_hands[1].reason, {} } }, - core::os_monotonic_now_ns()); + const auto& left = hand_state(true); + const auto& right = hand_state(false); + m_status_publisher->publish_if_changed( + { { "/hand/left", left.state, left.reason, {} }, { "/hand/right", right.state, right.reason, {} } }, + core::os_monotonic_now_ns()); } } // namespace plugins::omniglove diff --git a/src/plugins/omniglove/sdk.cmake b/src/plugins/omniglove/sdk.cmake index 67802056d1..47fc2352f1 100644 --- a/src/plugins/omniglove/sdk.cmake +++ b/src/plugins/omniglove/sdk.cmake @@ -1,9 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. # SPDX-License-Identifier: Apache-2.0 -set(OMNIGLOVE_SDK_ROOT "" CACHE PATH "Explicit OmniGlove SDK release root; empty fetches the pinned SDK") -set(OMNIGLOVE_SDK_REF "a2f899cd45c51ae01c9bbca9b893e4afba41c0d7" CACHE STRING "Managed OmniGlove SDK commit or ref") - if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") message(FATAL_ERROR "The OmniGlove SDK supports Linux only") endif() @@ -15,31 +12,39 @@ else() message(FATAL_ERROR "Unsupported OmniGlove SDK architecture: ${CMAKE_SYSTEM_PROCESSOR}") endif() -if(OMNIGLOVE_SDK_ROOT) - file(REAL_PATH "${OMNIGLOVE_SDK_ROOT}" _omniglove_sdk_root BASE_DIRECTORY "${CMAKE_SOURCE_DIR}") -else() - include(FetchContent) - FetchContent_Declare(omniglove_sdk - GIT_REPOSITORY https://github.com/AgibotTech/agilink_omniglove_sdk.git - GIT_TAG "${OMNIGLOVE_SDK_REF}" - # Consume the prebuilt package, not the SDK's standalone examples/build. - SOURCE_SUBDIR isaac-prebuilt-sdk - ) - FetchContent_MakeAvailable(omniglove_sdk) - set(_omniglove_sdk_root "${omniglove_sdk_SOURCE_DIR}") -endif() +include(FetchContent) +FetchContent_Declare(omniglove_sdk + GIT_REPOSITORY https://github.com/AgibotTech/agilink_omniglove_sdk.git + GIT_TAG a2f899cd45c51ae01c9bbca9b893e4afba41c0d7 # v1.0.0 + # This deliberately absent subdirectory prevents running the vendor's CMake. + SOURCE_SUBDIR isaac-prebuilt-sdk +) +FetchContent_MakeAvailable(omniglove_sdk) +file(REAL_PATH "${omniglove_sdk_SOURCE_DIR}" _omniglove_sdk_root) if(NOT EXISTS "${_omniglove_sdk_root}/SHA256SUMS" OR NOT EXISTS "${_omniglove_sdk_root}/VERSION") message(FATAL_ERROR "Not a complete OmniGlove SDK release: ${_omniglove_sdk_root}") endif() # The pinned commit selects the managed release; its checksums detect corruption. file(STRINGS "${_omniglove_sdk_root}/SHA256SUMS" _omniglove_checksums) +if(NOT _omniglove_checksums) + message(FATAL_ERROR "Empty OmniGlove SDK checksum manifest") +endif() +set(_omniglove_verified_files) foreach(_omniglove_entry IN LISTS _omniglove_checksums) if(NOT _omniglove_entry MATCHES "^([0-9a-fA-F]+) (.+)$") message(FATAL_ERROR "Malformed OmniGlove SDK checksum entry: ${_omniglove_entry}") endif() set(_omniglove_expected "${CMAKE_MATCH_1}") - set(_omniglove_file "${_omniglove_sdk_root}/${CMAKE_MATCH_2}") + string(LENGTH "${_omniglove_expected}" _omniglove_digest_length) + if(NOT _omniglove_digest_length EQUAL 64) + message(FATAL_ERROR "Invalid OmniGlove SDK SHA256 digest: ${_omniglove_entry}") + endif() + file(REAL_PATH "${_omniglove_sdk_root}/${CMAKE_MATCH_2}" _omniglove_file) + cmake_path(IS_PREFIX _omniglove_sdk_root "${_omniglove_file}" NORMALIZE _omniglove_inside_root) + if(NOT _omniglove_inside_root) + message(FATAL_ERROR "OmniGlove SDK checksum path escapes release root: ${_omniglove_file}") + endif() if(NOT EXISTS "${_omniglove_file}") message(FATAL_ERROR "Missing OmniGlove SDK release file: ${_omniglove_file}") endif() @@ -47,6 +52,26 @@ foreach(_omniglove_entry IN LISTS _omniglove_checksums) if(NOT _omniglove_actual STREQUAL _omniglove_expected) message(FATAL_ERROR "OmniGlove SDK checksum mismatch: ${_omniglove_file}") endif() + list(APPEND _omniglove_verified_files "${_omniglove_file}") +endforeach() + +function(omniglove_require_verified_file path) + if(NOT EXISTS "${path}") + message(FATAL_ERROR "Missing OmniGlove SDK release file: ${path}") + endif() + file(REAL_PATH "${path}" _omniglove_required_file) + if(NOT _omniglove_required_file IN_LIST _omniglove_verified_files) + message(FATAL_ERROR "OmniGlove SDK file is not covered by SHA256SUMS: ${path}") + endif() +endfunction() + +foreach(_omniglove_required IN ITEMS + VERSION LICENSE THIRD_PARTY_NOTICES.md licenses/libusb-COPYING + "${_omniglove_platform}/cmake/OmnigloveSDKConfig.cmake" + "${_omniglove_platform}/cmake/OmnigloveSDKConfigVersion.cmake" + "${_omniglove_platform}/include/omniglove/omniglove.h" +) + omniglove_require_verified_file("${_omniglove_sdk_root}/${_omniglove_required}") endforeach() set(OmnigloveSDK_DIR "${_omniglove_sdk_root}/${_omniglove_platform}/cmake") @@ -58,23 +83,6 @@ find_package(OmnigloveSDK CONFIG REQUIRED GLOBAL ) get_target_property(OMNIGLOVE_SDK_LIBRARY OmnigloveSDK::api IMPORTED_LOCATION) set(OMNIGLOVE_SDK_RELEASE_ROOT "${_omniglove_sdk_root}") -set(OMNIGLOVE_SDK_GIT_COMMIT "unknown") -find_package(Git QUIET) -if(GIT_FOUND) - execute_process( - COMMAND "${GIT_EXECUTABLE}" -C "${_omniglove_sdk_root}" rev-parse --show-toplevel - OUTPUT_VARIABLE _omniglove_git_root - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET - ) - # An unpacked SDK inside a project must not inherit the enclosing repo's commit. - if(_omniglove_git_root STREQUAL _omniglove_sdk_root) - execute_process( - COMMAND "${GIT_EXECUTABLE}" -C "${_omniglove_sdk_root}" rev-parse HEAD - OUTPUT_VARIABLE OMNIGLOVE_SDK_GIT_COMMIT - OUTPUT_STRIP_TRAILING_WHITESPACE - COMMAND_ERROR_IS_FATAL ANY - ) - endif() -endif() -message(STATUS "OmniGlove SDK ${OmnigloveSDK_VERSION}: ${OMNIGLOVE_SDK_GIT_COMMIT}") +omniglove_require_verified_file("${OMNIGLOVE_SDK_LIBRARY}") +find_library(_omniglove_system_libusb NAMES usb-1.0 libusb-1.0.so.0 REQUIRED) +message(STATUS "OmniGlove SDK ${OmnigloveSDK_VERSION}; system libusb: ${_omniglove_system_libusb}") diff --git a/src/plugins/plugin_utils/CMakeLists.txt b/src/plugins/plugin_utils/CMakeLists.txt index 1fd8966863..d45e25bde4 100644 --- a/src/plugins/plugin_utils/CMakeLists.txt +++ b/src/plugins/plugin_utils/CMakeLists.txt @@ -9,6 +9,8 @@ add_library(teleop_plugin_utils STATIC inc/plugin_utils/plugin_device_status_publisher.hpp wrist_pose_source.cpp wrist_source_config.cpp + inc/plugin_utils/wrist_source_config.hpp + inc/plugin_utils/hand_pose_basis.hpp inc/plugin_utils/wrist_pose_source.hpp ) diff --git a/src/plugins/plugin_utils/inc/plugin_utils/hand_pose_basis.hpp b/src/plugins/plugin_utils/inc/plugin_utils/hand_pose_basis.hpp new file mode 100644 index 0000000000..6a1811fba2 --- /dev/null +++ b/src/plugins/plugin_utils/inc/plugin_utils/hand_pose_basis.hpp @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Wuji Technology. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace plugin_utils +{ + +// SDK right-hand +Y is palmar; conjugate by a Z half-turn for OpenXR dorsal +Y. +constexpr XrPosef remap_to_openxr_basis(XrPosef pose, bool is_left) +{ + if (!is_left) + { + pose.position.x = -pose.position.x; + pose.position.y = -pose.position.y; + pose.orientation.x = -pose.orientation.x; + pose.orientation.y = -pose.orientation.y; + } + return pose; +} + +} // namespace plugin_utils diff --git a/src/plugins/plugin_utils/inc/plugin_utils/wrist_pose_source.hpp b/src/plugins/plugin_utils/inc/plugin_utils/wrist_pose_source.hpp index 40117aee4f..48bb3546a5 100644 --- a/src/plugins/plugin_utils/inc/plugin_utils/wrist_pose_source.hpp +++ b/src/plugins/plugin_utils/inc/plugin_utils/wrist_pose_source.hpp @@ -5,6 +5,8 @@ // XR_MNDX_xdev_space preferred, controller aim pose as the fallback. #pragma once +#include "wrist_source_config.hpp" + #include #include #include @@ -18,36 +20,6 @@ namespace plugin_utils { -/** @brief Which device provides the wrist pose. */ -enum class WristSourceMode -{ - Auto, //!< Optical hand tracking preferred, controller aim pose as fallback. - HandTracking, //!< XR_MNDX_xdev_space optical hand tracking only. - Controller, //!< Controller aim pose (plus the configured rigid offset) only. -}; - -/** @brief Source selection plus per-device tuning. */ -struct WristSourceConfig -{ - WristSourceMode mode = WristSourceMode::Auto; - // Rigid transform from the controller aim pose to the wrist joint, per - // hand. Depends on the controller model and how it is mounted on the - // glove, so each plugin supplies its own calibrated pair (see - // kLeft/kRightAimToWrist in wuji_glove_plugin.cpp and - // kLeft/kRightHandOffset in manus_hand_tracking_plugin.cpp). - XrPosef left_aim_to_wrist{ { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; - XrPosef right_aim_to_wrist{ { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; -}; - -/** @brief Read a wrist mode; an unset variable uses fallback, an unknown value throws. */ -WristSourceMode wrist_source_mode_from_env(const char* name, WristSourceMode fallback); - -/** @brief Read and normalize px,py,pz,qx,qy,qz,qw; missing or invalid values throw. */ -XrPosef pose_from_env(const char* name); - -/** @brief Use fallback only for an unset variable; invalid values throw. */ -XrPosef pose_from_env(const char* name, const XrPosef& fallback); - /** @brief One wrist query result, in the session base space. */ struct WristSample { diff --git a/src/plugins/plugin_utils/inc/plugin_utils/wrist_source_config.hpp b/src/plugins/plugin_utils/inc/plugin_utils/wrist_source_config.hpp new file mode 100644 index 0000000000..eadcd8324a --- /dev/null +++ b/src/plugins/plugin_utils/inc/plugin_utils/wrist_source_config.hpp @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 Wuji Technology. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace plugin_utils +{ + +/** @brief Which device provides the wrist pose. */ +enum class WristSourceMode +{ + Auto, //!< Optical hand tracking preferred, controller aim pose as fallback. + HandTracking, //!< XR_MNDX_xdev_space optical hand tracking only. + Controller, //!< Controller aim pose (plus the configured rigid offset) only. +}; + +/** @brief Source selection plus per-device tuning. */ +struct WristSourceConfig +{ + WristSourceMode mode = WristSourceMode::Auto; + // Rigid transform from the controller aim pose to the wrist joint, per + // hand. Depends on the controller model and how it is mounted on the + // glove, so each plugin supplies its own calibrated pair (see + // kLeft/kRightAimToWrist in wuji_glove_plugin.cpp and + // kLeft/kRightHandOffset in manus_hand_tracking_plugin.cpp). + XrPosef left_aim_to_wrist{ { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; + XrPosef right_aim_to_wrist{ { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; +}; + +/** @brief Read a wrist mode; an unset variable uses fallback, an unknown value throws. */ +WristSourceMode wrist_source_mode_from_env(const char* name, WristSourceMode fallback); + +/** @brief Read and normalize px,py,pz,qx,qy,qz,qw; missing or invalid values throw. */ +XrPosef pose_from_env(const char* name); + +/** @brief Use fallback only for an unset variable; invalid values throw. */ +XrPosef pose_from_env(const char* name, const XrPosef& fallback); + +/** @brief The canonical environment-variable spelling of a wrist mode. */ +const char* to_string(WristSourceMode mode); + +} // namespace plugin_utils diff --git a/src/plugins/plugin_utils/wrist_source_config.cpp b/src/plugins/plugin_utils/wrist_source_config.cpp index ab99ad5597..793dc58fa7 100644 --- a/src/plugins/plugin_utils/wrist_source_config.cpp +++ b/src/plugins/plugin_utils/wrist_source_config.cpp @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 Agibot Co., Ltd. // SPDX-License-Identifier: Apache-2.0 -#include "inc/plugin_utils/wrist_pose_source.hpp" +#include "inc/plugin_utils/wrist_source_config.hpp" #include #include @@ -15,6 +15,20 @@ namespace plugin_utils { +const char* to_string(WristSourceMode mode) +{ + switch (mode) + { + case WristSourceMode::Auto: + return "auto"; + case WristSourceMode::HandTracking: + return "hand_tracking"; + case WristSourceMode::Controller: + return "controller"; + } + throw std::invalid_argument("invalid wrist source mode"); +} + WristSourceMode wrist_source_mode_from_env(const char* name, WristSourceMode fallback) { const char* value = std::getenv(name); @@ -22,17 +36,12 @@ WristSourceMode wrist_source_mode_from_env(const char* name, WristSourceMode fal { return fallback; } - if (std::strcmp(value, "hand_tracking") == 0) - { - return WristSourceMode::HandTracking; - } - if (std::strcmp(value, "controller") == 0) - { - return WristSourceMode::Controller; - } - if (std::strcmp(value, "auto") == 0) + for (const auto mode : { WristSourceMode::HandTracking, WristSourceMode::Controller, WristSourceMode::Auto }) { - return WristSourceMode::Auto; + if (std::strcmp(value, to_string(mode)) == 0) + { + return mode; + } } throw std::runtime_error(std::string(name) + " must be hand_tracking, controller or auto; got '" + value + "'"); } diff --git a/src/plugins/wuji_glove/wuji_glove_plugin.cpp b/src/plugins/wuji_glove/wuji_glove_plugin.cpp index 671ad4139c..da87050e09 100644 --- a/src/plugins/wuji_glove/wuji_glove_plugin.cpp +++ b/src/plugins/wuji_glove/wuji_glove_plugin.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -77,35 +78,11 @@ constexpr auto kStaleThreshold = std::chrono::milliseconds(200); constexpr auto kDiscoveryInterval = std::chrono::seconds(1); constexpr auto kShutdownPollInterval = std::chrono::milliseconds(100); -// Re-express a wrist-relative joint pose from the Wuji skeleton basis in the -// OpenXR hand-joint basis. -// -// The SDK skeleton poses are FK link frames of wuji-sdk's default hand URDFs: -// the bone runs along -Z (matching XR_EXT_hand_tracking's "+Z points away from -// the fingertip"), and the palmar side is +Y on the right hand / -Y on the -// left (the left URDF is Y-mirrored). OpenXR puts the dorsal side at +Y for -// both hands, so the left hand already matches and the right hand differs by -// a 180° rotation about Z — whose conjugation simply negates the x and y -// components of both the position and the quaternion. -constexpr XrPosef remap_to_openxr_basis(const XrPosef& pose, bool is_left) -{ - if (is_left) - { - return pose; - } - XrPosef out = pose; - out.position.x = -pose.position.x; - out.position.y = -pose.position.y; - out.orientation.x = -pose.orientation.x; - out.orientation.y = -pose.orientation.y; - return out; -} - constexpr bool remap_to_openxr_basis_is_correct() { constexpr XrPosef input{ { 1.0f, 2.0f, 3.0f, 4.0f }, { 5.0f, 6.0f, 7.0f } }; - constexpr XrPosef left = remap_to_openxr_basis(input, true); - constexpr XrPosef right = remap_to_openxr_basis(input, false); + constexpr XrPosef left = plugin_utils::remap_to_openxr_basis(input, true); + constexpr XrPosef right = plugin_utils::remap_to_openxr_basis(input, false); return left.orientation.x == 1.0f && left.orientation.y == 2.0f && left.orientation.z == 3.0f && left.orientation.w == 4.0f && left.position.x == 5.0f && left.position.y == 6.0f && left.position.z == 7.0f && right.orientation.x == -1.0f && right.orientation.y == -2.0f && @@ -146,40 +123,13 @@ bool convert_skeleton(const WujiHandSkeleton* frame, const XrPosef sdk_pose{ XrQuaternionf{ src.pose.orientation.x, src.pose.orientation.y, src.pose.orientation.z, src.pose.orientation.w }, XrVector3f{ src.pose.position[0], src.pose.position[1], src.pose.position[2] } }; - dst.pose = remap_to_openxr_basis(sdk_pose, is_left); + dst.pose = plugin_utils::remap_to_openxr_basis(sdk_pose, is_left); dst.radius = kDefaultJointRadius; dst.locationFlags = kPoseValidFlags; } return true; } -XrPosef pose_from_env(const char* name, const XrPosef& fallback) -{ - try - { - return plugin_utils::pose_from_env(name, fallback); - } - catch (const std::runtime_error& error) - { - isaaccapture::Logger::get("isaaccapture.plugins.wuji_glove.main") - ->warn("{}; using the built-in offset", error.what()); - return fallback; - } -} - -plugin_utils::WristSourceMode wrist_source_mode_from_env() -{ - try - { - return plugin_utils::wrist_source_mode_from_env("WUJI_GLOVE_WRIST_SOURCE", plugin_utils::WristSourceMode::Auto); - } - catch (const std::runtime_error& error) - { - isaaccapture::Logger::get("isaaccapture.plugins.wuji_glove.main")->warn("{}; using 'auto'", error.what()); - return plugin_utils::WristSourceMode::Auto; - } -} - // Resolve the glove's hand side explicitly via the device's "hand_side" GET // (a NUL-terminated "left"/"right" string) — never inferred from frame contents. std::optional query_is_left(WujiDevice* dev) @@ -219,9 +169,10 @@ WujiGlovePlugin::WujiGlovePlugin(const std::string& plugin_root_id) noexcept(fal // OpenXR session exists for the push-device (injection) extension, the // wrist-source queries, and the XrTime base. plugin_utils::WristSourceConfig wrist_config; - wrist_config.mode = wrist_source_mode_from_env(); - wrist_config.left_aim_to_wrist = pose_from_env("WUJI_GLOVE_AIM_TO_WRIST_LEFT", kLeftAimToWrist); - wrist_config.right_aim_to_wrist = pose_from_env("WUJI_GLOVE_AIM_TO_WRIST_RIGHT", kRightAimToWrist); + wrist_config.mode = + plugin_utils::wrist_source_mode_from_env("WUJI_GLOVE_WRIST_SOURCE", plugin_utils::WristSourceMode::Auto); + wrist_config.left_aim_to_wrist = plugin_utils::pose_from_env("WUJI_GLOVE_AIM_TO_WRIST_LEFT", kLeftAimToWrist); + wrist_config.right_aim_to_wrist = plugin_utils::pose_from_env("WUJI_GLOVE_AIM_TO_WRIST_RIGHT", kRightAimToWrist); auto wrist_requirements = plugin_utils::WristPoseSource::collect_requirements(wrist_config.mode); std::vector> trackers = wrist_requirements.trackers; diff --git a/src/python/isaaccapture/cts/hand/checker/README.md b/src/python/isaaccapture/cts/hand/checker/README.md index 67dc49fcf0..926cbf87a0 100644 --- a/src/python/isaaccapture/cts/hand/checker/README.md +++ b/src/python/isaaccapture/cts/hand/checker/README.md @@ -68,7 +68,8 @@ they could not be answered, and the rest are judged as usual. This uses the viewer's bundle builder, reruns all checks, verifies the original recording and companion files, and prints the archive's verdict and SHA-256. -It requires the panel dependencies. Existing archives are never overwritten. +It does not require the optional panel dependencies. Existing archives are never +overwritten. ## Read the report diff --git a/src/python/isaaccapture/cts/hand/checker/src/hand_cts/package.py b/src/python/isaaccapture/cts/hand/checker/src/hand_cts/package.py index 481959bb37..c0ff88596c 100755 --- a/src/python/isaaccapture/cts/hand/checker/src/hand_cts/package.py +++ b/src/python/isaaccapture/cts/hand/checker/src/hand_cts/package.py @@ -13,6 +13,8 @@ from pathlib import Path import zipfile +from cts_common.panel.bundle import companions + from hand_cts.labels import StepTimeline from hand_cts.mcap_source import McapFrameSource from hand_cts.panel import bundle @@ -27,11 +29,10 @@ def main(argv: list[str] | None = None) -> None: parser.add_argument("--output-dir", type=Path) args = parser.parse_args(argv) recording = args.recording.resolve() - required = [ - recording, - *(recording.with_suffix(s) for s in (".labels.json", ".json", ".log")), - ] - missing = [str(path) for path in required if not path.is_file()] + sidecars, missing = companions(recording) + required = [recording, *sidecars] + if not recording.is_file(): + missing.insert(0, str(recording)) if missing: parser.error("missing take files: " + ", ".join(missing)) @@ -41,8 +42,6 @@ def main(argv: list[str] | None = None) -> None: report = run(source, timeline=timeline) name, data = bundle.build(report, source.tracks(), recording, timeline) with zipfile.ZipFile(io.BytesIO(data)) as archive: - if archive.testzip() is not None: - raise RuntimeError("CTS archive failed its integrity check") root = name.removesuffix(".zip") manifest = json.loads(archive.read(f"{root}/report.json")) for item in manifest["inputs"]["files"]: diff --git a/tests/cpp/plugins/omniglove/test_hand_conversion.cpp b/tests/cpp/plugins/omniglove/test_hand_conversion.cpp index ca2b0da8a5..3b002de3da 100644 --- a/tests/cpp/plugins/omniglove/test_hand_conversion.cpp +++ b/tests/cpp/plugins/omniglove/test_hand_conversion.cpp @@ -14,6 +14,7 @@ namespace { using plugins::omniglove::ConversionStatus; +using plugins::omniglove::HandJoints; void fill_pose(omniglove_pose6_t& pose, float x, float y, float z, const XrQuaternionf& orientation) { @@ -68,11 +69,13 @@ bool openxr_axes(bool is_left, const XrQuaternionf& orientation) { omniglove_post_data_t post{}; fill_hand(post, orientation); - std::array joints{}; - if (plugins::omniglove::convert_post_data(post, is_left, joints) != ConversionStatus::Success) + const auto result = plugins::omniglove::convert_post_data(post, is_left); + const auto* converted = std::get_if(&result); + if (converted == nullptr) { return false; } + const auto& joints = *converted; const float norm = std::sqrt(orientation.x * orientation.x + orientation.y * orientation.y + orientation.z * orientation.z + orientation.w * orientation.w); const XrQuaternionf unit = { orientation.x / norm, orientation.y / norm, orientation.z / norm, orientation.w / norm }; @@ -122,8 +125,9 @@ TEST_CASE("OmniGlove conversion preserves its OpenXR hand contract", "[omniglove { omniglove_post_data_t post{}; fill_hand(post); - std::array joints{}; - REQUIRE(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::Success); + auto result = plugins::omniglove::convert_post_data(post, true); + REQUIRE(std::holds_alternative(result)); + auto joints = std::get(result); constexpr XrSpaceLocationFlags valid = XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; int valid_count = 0; for (const auto& joint : joints) @@ -141,19 +145,42 @@ TEST_CASE("OmniGlove conversion preserves its OpenXR hand contract", "[omniglove CHECK(openxr_axes(is_left, { 1.0F, 2.0F, 3.0F, 4.0F })); } const XrPosef left_wrist = joints[XR_HAND_JOINT_WRIST_EXT].pose; - REQUIRE(plugins::omniglove::convert_post_data(post, false, joints) == ConversionStatus::Success); + result = plugins::omniglove::convert_post_data(post, false); + REQUIRE(std::holds_alternative(result)); + joints = std::get(result); CHECK(near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.x, -left_wrist.position.x)); CHECK(near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.y, -left_wrist.position.y)); CHECK(near(joints[XR_HAND_JOINT_WRIST_EXT].pose.position.z, left_wrist.position.z)); post.flags = plugins::omniglove::kPostBoneModelValid; - CHECK(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::IkNotConverged); + CHECK(std::get(plugins::omniglove::convert_post_data(post, true)) == + ConversionStatus::IkNotConverged); post.flags = plugins::omniglove::kPostIkConverged; - CHECK(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::BoneModelInvalid); + CHECK(std::get(plugins::omniglove::convert_post_data(post, true)) == + ConversionStatus::BoneModelInvalid); fill_hand(post); post.mapped = 0; - CHECK(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::NotMapped); + CHECK(std::get(plugins::omniglove::convert_post_data(post, true)) == ConversionStatus::NotMapped); fill_hand(post); post.index.landmarks.mcp.position_m[0] = std::numeric_limits::quiet_NaN(); - CHECK(plugins::omniglove::convert_post_data(post, true, joints) == ConversionStatus::InvalidPose); + CHECK(std::get(plugins::omniglove::convert_post_data(post, true)) == ConversionStatus::InvalidPose); +} + +TEST_CASE("OmniGlove returns no joints when the last landmark is invalid", "[omniglove][conversion]") +{ + omniglove_post_data_t post{}; + fill_hand(post); + post.pinky.landmarks.tip.quaternion_xyzw[3] = std::numeric_limits::infinity(); + auto result = plugins::omniglove::convert_post_data(post, true); + REQUIRE(std::holds_alternative(result)); + CHECK(std::get(result) == ConversionStatus::InvalidPose); + + fill_hand(post); + for (auto& component : post.pinky.landmarks.tip.quaternion_xyzw) + { + component = 0.0F; + } + result = plugins::omniglove::convert_post_data(post, false); + REQUIRE(std::holds_alternative(result)); + CHECK(std::get(result) == ConversionStatus::InvalidPose); } diff --git a/tests/cpp/plugins/plugin_utils/CMakeLists.txt b/tests/cpp/plugins/plugin_utils/CMakeLists.txt index 09301b972e..1edff4676e 100644 --- a/tests/cpp/plugins/plugin_utils/CMakeLists.txt +++ b/tests/cpp/plugins/plugin_utils/CMakeLists.txt @@ -1,15 +1,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -add_executable(plugin_device_status_publisher_tests +add_executable(plugin_utils_tests test_plugin_device_status_publisher.cpp test_wrist_source_config.cpp ) -target_link_libraries(plugin_device_status_publisher_tests +target_link_libraries(plugin_utils_tests PRIVATE Catch2::Catch2WithMain Teleop::plugin_utils ) -catch_discover_tests(plugin_device_status_publisher_tests) +catch_discover_tests(plugin_utils_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/plugins/plugin_utils/test_wrist_source_config.cpp b/tests/cpp/plugins/plugin_utils/test_wrist_source_config.cpp index a41af35775..7dc0ebb275 100644 --- a/tests/cpp/plugins/plugin_utils/test_wrist_source_config.cpp +++ b/tests/cpp/plugins/plugin_utils/test_wrist_source_config.cpp @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 #include -#include +#include +#include #include #include @@ -60,6 +61,11 @@ TEST_CASE("Wrist source mode accepts only named modes", "[plugin_utils][wrist_co CHECK(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::Auto) == WristSourceMode::Controller); env.set("auto"); CHECK(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::HandTracking) == WristSourceMode::Auto); + for (const auto mode : { WristSourceMode::HandTracking, WristSourceMode::Controller, WristSourceMode::Auto }) + { + env.set(plugin_utils::to_string(mode)); + CHECK(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::Auto) == mode); + } env.set("controler"); CHECK_THROWS_AS(plugin_utils::wrist_source_mode_from_env(kVariable, WristSourceMode::Auto), std::runtime_error); } @@ -98,3 +104,25 @@ TEST_CASE("Optional wrist offsets fall back only when unset", "[plugin_utils][wr env.set("invalid"); CHECK_THROWS_AS(plugin_utils::pose_from_env(kVariable, fallback), std::runtime_error); } + +TEST_CASE("Glove basis conversion preserves left poses and is reversible on the right", "[plugin_utils][hand_basis]") +{ + constexpr XrPosef pose{ { 0.1F, 0.2F, 0.3F, 0.9F }, { 1.0F, 2.0F, 3.0F } }; + for (bool is_left : { true, false }) + { + const auto converted = plugin_utils::remap_to_openxr_basis(pose, is_left); + const float sign = is_left ? 1.0F : -1.0F; + CHECK(converted.position.x == sign * pose.position.x); + CHECK(converted.position.y == sign * pose.position.y); + CHECK(converted.position.z == pose.position.z); + CHECK(converted.orientation.x == sign * pose.orientation.x); + CHECK(converted.orientation.y == sign * pose.orientation.y); + CHECK(converted.orientation.z == pose.orientation.z); + CHECK(converted.orientation.w == pose.orientation.w); + const auto restored = plugin_utils::remap_to_openxr_basis(converted, is_left); + CHECK(restored.position.x == pose.position.x); + CHECK(restored.position.y == pose.position.y); + CHECK(restored.orientation.x == pose.orientation.x); + CHECK(restored.orientation.y == pose.orientation.y); + } +} From 62e4c102567abd0949e6fe3ad3b0819633038bb9 Mon Sep 17 00:00:00 2001 From: zhouyingchun Date: Tue, 17 Nov 2026 19:39:35 +0800 Subject: [PATCH 14/14] Fix OmniGlove thumb joint axes and SDK license installation Apply the handed local Z quarter-turn to thumb orientations so bent thumb hinges align with OpenXR +X. Add a geometry-based regression covering both hands and transformed poses. Verify and install the statically included fmt and serial MIT texts, remove the obsolete required libusb notice entry, and drop the PR-only SDK cache migration sentence from the device guide. Signed-off-by: zhouyingchun --- .../source/device/hand_tracking/omniglove.rst | 11 ++- src/plugins/omniglove/CMakeLists.txt | 6 ++ .../core/omniglove_hand_conversion.cpp | 9 +- src/plugins/omniglove/sdk.cmake | 3 +- .../omniglove/test_hand_conversion.cpp | 99 +++++++++++++++++++ 5 files changed, 121 insertions(+), 7 deletions(-) diff --git a/docs/source/device/hand_tracking/omniglove.rst b/docs/source/device/hand_tracking/omniglove.rst index 5558301f53..54e14cbd1b 100644 --- a/docs/source/device/hand_tracking/omniglove.rst +++ b/docs/source/device/hand_tracking/omniglove.rst @@ -62,8 +62,7 @@ is supplied by the operating system. cmake --build build cmake --install build --component omniglove -Existing build directories using ``OMNIGLOVE_SDK_ROOT`` or ``OMNIGLOVE_SDK_REF`` -should remove those cache entries. Local SDK checkouts use CMake's standard +Local SDK checkouts use CMake's standard ``-DFETCHCONTENT_SOURCE_DIR_OMNIGLOVE_SDK=/path/to/agilink_omniglove_sdk`` override. Running @@ -132,9 +131,11 @@ the host records them as ``core.HandPoseRecord`` from ``hand.fbs``. - Positions are metres and orientations are unit ``(x, y, z, w)`` quaternions, in the base space passed to ``HandInjector``. - OpenXR joint ``+Z`` points away from the fingertip and ``+Y`` toward the - back of the hand. SDK finger orientations receive a local Y half-turn; - the right wrist basis is converted separately. Wrist and finger poses are - composed with the same world-space wrist transform. + back of each digit. SDK digit orientations receive a local Y half-turn, + followed for the thumb by a local Z rotation of -90 degrees on the left + and +90 degrees on the right to align the flexion hinge with local +X. + The right wrist basis is converted separately. Wrist and finger poses + are composed with the same world-space wrist transform. - SDK monotonic sample times are converted into runtime ``XrTime`` through ``XrTimeConverter``; the wrist is queried at that same time. - Unmapped, unconverged IK, invalid bone data, non-finite poses, zero diff --git a/src/plugins/omniglove/CMakeLists.txt b/src/plugins/omniglove/CMakeLists.txt index 9f9dd1cef6..90d154f334 100644 --- a/src/plugins/omniglove/CMakeLists.txt +++ b/src/plugins/omniglove/CMakeLists.txt @@ -25,6 +25,12 @@ install(FILES "${OMNIGLOVE_SDK_RELEASE_ROOT}/THIRD_PARTY_NOTICES.md" RENAME THIRD_PARTY_NOTICES.omniglove-sdk.md COMPONENT omniglove ) +install(FILES + "${OMNIGLOVE_SDK_RELEASE_ROOT}/licenses/fmt-LICENSE" + "${OMNIGLOVE_SDK_RELEASE_ROOT}/licenses/wjwwood-serial-LICENSE" + DESTINATION plugins/omniglove/licenses + COMPONENT omniglove +) install(FILES plugin.yaml README.md 60-omniglove-usb.rules DESTINATION plugins/omniglove diff --git a/src/plugins/omniglove/core/omniglove_hand_conversion.cpp b/src/plugins/omniglove/core/omniglove_hand_conversion.cpp index a7402eec13..6d1e403ee4 100644 --- a/src/plugins/omniglove/core/omniglove_hand_conversion.cpp +++ b/src/plugins/omniglove/core/omniglove_hand_conversion.cpp @@ -18,6 +18,9 @@ namespace { constexpr float kJointRadiusM = 0.01F; +constexpr float kSqrtHalf = 0.7071067811865476F; +constexpr XrPosef kLeftThumbRoll{ { 0.0F, 0.0F, -kSqrtHalf, kSqrtHalf }, {} }; +constexpr XrPosef kRightThumbRoll{ { 0.0F, 0.0F, kSqrtHalf, kSqrtHalf }, {} }; constexpr XrSpaceLocationFlags kValidFlags = XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; @@ -120,11 +123,15 @@ ConversionResult convert_post_data(const omniglove_post_data_t& post, bool is_le if (source.joint != XR_HAND_JOINT_WRIST_EXT) { // SDK finger frames point +Z toward the tip; OpenXR points -Z. - // Post-multiply by a local Y half-turn to preserve dorsal +Y. // The SDK wrist already has proximal +Z and must not be flipped. constexpr XrPosef kYHalfTurn{ { 0.0F, 1.0F, 0.0F, 0.0F }, { 0.0F, 0.0F, 0.0F } }; destination.pose = oxr_utils::multiply_poses(destination.pose, kYHalfTurn); } + if (source.joint >= XR_HAND_JOINT_THUMB_METACARPAL_EXT && source.joint <= XR_HAND_JOINT_THUMB_TIP_EXT) + { + // SDK thumb hinges use local Y; this local Z roll aligns them with OpenXR +X. + destination.pose = oxr_utils::multiply_poses(destination.pose, is_left ? kLeftThumbRoll : kRightThumbRoll); + } destination.locationFlags = kValidFlags; } return joints; diff --git a/src/plugins/omniglove/sdk.cmake b/src/plugins/omniglove/sdk.cmake index 47fc2352f1..e15a5c1588 100644 --- a/src/plugins/omniglove/sdk.cmake +++ b/src/plugins/omniglove/sdk.cmake @@ -66,7 +66,8 @@ function(omniglove_require_verified_file path) endfunction() foreach(_omniglove_required IN ITEMS - VERSION LICENSE THIRD_PARTY_NOTICES.md licenses/libusb-COPYING + VERSION LICENSE THIRD_PARTY_NOTICES.md + licenses/fmt-LICENSE licenses/wjwwood-serial-LICENSE "${_omniglove_platform}/cmake/OmnigloveSDKConfig.cmake" "${_omniglove_platform}/cmake/OmnigloveSDKConfigVersion.cmake" "${_omniglove_platform}/include/omniglove/omniglove.h" diff --git a/tests/cpp/plugins/omniglove/test_hand_conversion.cpp b/tests/cpp/plugins/omniglove/test_hand_conversion.cpp index 3b002de3da..bf8a9f87c8 100644 --- a/tests/cpp/plugins/omniglove/test_hand_conversion.cpp +++ b/tests/cpp/plugins/omniglove/test_hand_conversion.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -65,6 +66,27 @@ std::array rotate(const XrQuaternionf& q, const std::array& v[2] + q.w * t[2] + q.x * t[1] - q.y * t[0] }; } +std::array subtract(const XrVector3f& a, const XrVector3f& b) +{ + return { a.x - b.x, a.y - b.y, a.z - b.z }; +} + +float dot(const std::array& a, const std::array& b) +{ + return a[0] * b[0] + a[1] * b[1] + a[2] * b[2]; +} + +std::array normalized(const std::array& v) +{ + const float length = std::sqrt(dot(v, v)); + return { v[0] / length, v[1] / length, v[2] / length }; +} + +std::array cross(const std::array& a, const std::array& b) +{ + return { a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0] }; +} + bool openxr_axes(bool is_left, const XrQuaternionf& orientation) { omniglove_post_data_t post{}; @@ -102,6 +124,10 @@ bool openxr_axes(bool is_left, const XrQuaternionf& orientation) expected[1] = -expected[1]; } auto xr_axis = axis; + if (index >= XR_HAND_JOINT_THUMB_METACARPAL_EXT && index <= XR_HAND_JOINT_THUMB_TIP_EXT && axis[1] != 0.0F) + { + xr_axis = { is_left ? -1.0F : 1.0F, 0.0F, 0.0F }; + } if (index != XR_HAND_JOINT_WRIST_EXT && axis[2] != 0.0F) { xr_axis[2] = -xr_axis[2]; @@ -166,6 +192,79 @@ TEST_CASE("OmniGlove conversion preserves its OpenXR hand contract", "[omniglove CHECK(std::get(plugins::omniglove::convert_post_data(post, true)) == ConversionStatus::InvalidPose); } +TEST_CASE("OmniGlove bent thumb hinges align with local X", "[omniglove][conversion]") +{ + const float inverse_norm = 1.0F / std::sqrt(30.0F); + const std::array source_transforms = { XrPosef{ { 0.0F, 0.0F, 0.0F, 1.0F }, {} }, + XrPosef{ { inverse_norm, 2.0F * inverse_norm, + 3.0F * inverse_norm, 4.0F * inverse_norm }, + { 0.11F, -0.07F, 0.09F } } }; + constexpr std::array thumb_joints = { XR_HAND_JOINT_THUMB_METACARPAL_EXT, XR_HAND_JOINT_THUMB_PROXIMAL_EXT, + XR_HAND_JOINT_THUMB_DISTAL_EXT, XR_HAND_JOINT_THUMB_TIP_EXT }; + for (bool is_left : { true, false }) + { + for (const XrPosef& transform : source_transforms) + { + for (float ip_bend : { 0.4F, 1.1F }) + { + CAPTURE(is_left, ip_bend, transform.orientation.x); + constexpr float mcp_bend = 0.45F; + const float distal_angle = mcp_bend + ip_bend; + const XrQuaternionf mcp_orientation{ 0.0F, std::sin(mcp_bend / 2.0F), 0.0F, std::cos(mcp_bend / 2.0F) }; + const XrQuaternionf ip_orientation{ 0.0F, std::sin(distal_angle / 2.0F), 0.0F, + std::cos(distal_angle / 2.0F) }; + const XrVector3f mcp{ 0.02F, 0.01F, 0.025F }; + const XrVector3f ip{ mcp.x + 0.03F * std::sin(mcp_bend), mcp.y, mcp.z + 0.03F * std::cos(mcp_bend) }; + const XrVector3f tip{ ip.x + 0.02F * std::sin(distal_angle), ip.y, ip.z + 0.02F * std::cos(distal_angle) }; + // Thumb flexion is in the SDK XZ plane, with its hinge along Y. + const std::array local_poses = { + XrPosef{ { 0.0F, 0.0F, 0.0F, 1.0F }, { 0.02F, 0.01F, 0.0F } }, XrPosef{ mcp_orientation, mcp }, + XrPosef{ ip_orientation, ip }, XrPosef{ ip_orientation, tip } + }; + omniglove_post_data_t post{}; + fill_hand(post); + const std::array landmarks = { &post.thumb.landmarks.cmc, &post.thumb.landmarks.mcp, + &post.thumb.landmarks.ip, &post.thumb.landmarks.tip }; + std::array expected_positions; + for (std::size_t i = 0; i < local_poses.size(); ++i) + { + const XrPosef pose = oxr_utils::multiply_poses(transform, local_poses[i]); + fill_pose(*landmarks[i], pose.position.x, pose.position.y, pose.position.z, pose.orientation); + expected_positions[i] = pose.position; + if (!is_left) + { + expected_positions[i].x = -expected_positions[i].x; + expected_positions[i].y = -expected_positions[i].y; + } + } + std::array, 4> bone_directions; + for (std::size_t i = 0; i < bone_directions.size() - 1; ++i) + { + bone_directions[i] = normalized(subtract(expected_positions[i + 1], expected_positions[i])); + } + bone_directions.back() = bone_directions[2]; + const auto ip_hinge = normalized(cross(bone_directions[2], bone_directions[1])); + const auto mcp_hinge = normalized(cross(bone_directions[1], bone_directions[0])); + const auto result = plugins::omniglove::convert_post_data(post, is_left); + REQUIRE(std::holds_alternative(result)); + const auto& joints = std::get(result); + for (std::size_t i = 0; i < thumb_joints.size(); ++i) + { + CAPTURE(i); + const auto& pose = joints[thumb_joints[i]].pose; + CHECK(near(pose.position.x, expected_positions[i].x)); + CHECK(near(pose.position.y, expected_positions[i].y)); + CHECK(near(pose.position.z, expected_positions[i].z)); + const auto x_axis = rotate(pose.orientation, { 1.0F, 0.0F, 0.0F }); + CHECK(dot(x_axis, ip_hinge) > 0.99999F); + CHECK(dot(x_axis, mcp_hinge) > 0.99999F); + CHECK(dot(rotate(pose.orientation, { 0.0F, 0.0F, -1.0F }), bone_directions[i]) > 0.99999F); + } + } + } + } +} + TEST_CASE("OmniGlove returns no joints when the last landmark is invalid", "[omniglove][conversion]") { omniglove_post_data_t post{};