Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi
option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF)
option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF)
option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF)
option(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE "Build the remote device bridge (remote_device_bridge app and server) and the RemotePluginSession library (POSIX sockets, skipped on Windows)" ON)
option(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE "Build the remote device bridge (app and server), the RemotePluginSession library and the synthetic sender; enables manus --remote (POSIX sockets, skipped on Windows)" ON)
option(BUILD_EXAMPLES "Build examples" ON)
option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF)
option(BUILD_TESTING "Build unit tests" ON)
Expand Down
9 changes: 9 additions & 0 deletions docs/source/device/hand_tracking/manus.rst
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,15 @@ haptic read. Restrict datasets with ``--datasets=`` (comma-separated):
./install/plugins/manus/manus_hand_plugin --datasets=human # skeleton only
./install/plugins/manus/manus_hand_plugin --datasets=sensors # flex sensors only

To run the plugin on a machine without an OpenXR runtime, point it at a
``remote_device_bridge`` running next to the runtime with ``--remote=``
(``haptic`` is not available over the remote session):

.. code-block:: bash

./install/plugins/manus/manus_hand_plugin --remote=<host:port> # human,sensors only
./install/plugins/manus/manus_hand_plugin --remote=ws://<host:port> # bridge started with --ws-listen

4. Load glove calibration files (optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Expand Down
8 changes: 4 additions & 4 deletions src/plugins/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@ add_subdirectory(gamepad)
add_subdirectory(xsens_full_body)
add_subdirectory(so101_leader)
add_subdirectory(rebot_devarm_leader)
add_subdirectory(manus)
add_subdirectory(manus_calibration)
add_subdirectory(haptikos)

# Before manus: its app links Teleop::remote_plugin_session when the target exists.
if(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE)
add_subdirectory(remote_device_bridge)
endif()
add_subdirectory(manus)
add_subdirectory(manus_calibration)
add_subdirectory(haptikos)

if(BUILD_PLUGIN_NOITOM_MOCAP)
add_subdirectory(noitom_mocap)
Expand Down
7 changes: 7 additions & 0 deletions src/plugins/manus/app/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@ target_link_libraries(manus_hand_plugin
manus_plugin_core
)

# --remote=<host:port> needs the RemotePluginSession library (BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE, non-Windows);
# without it the flag is rejected at startup.
if(TARGET Teleop::remote_plugin_session)
target_link_libraries(manus_hand_plugin PRIVATE Teleop::remote_plugin_session)
target_compile_definitions(manus_hand_plugin PRIVATE MANUS_HAVE_REMOTE_PLUGIN_SESSION)
endif()

set_target_properties(manus_hand_plugin PROPERTIES
CXX_VISIBILITY_PRESET hidden
VISIBILITY_INLINES_HIDDEN YES
Expand Down
60 changes: 54 additions & 6 deletions src/plugins/manus/app/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@

#include <log_bridge/logger.hpp>
#include <manus/manus_hand_tracking_plugin.hpp>
#if defined(MANUS_HAVE_REMOTE_PLUGIN_SESSION)
# include <remote_plugin_session/remote_address.hpp>
# include <remote_plugin_session/remote_plugin_session.hpp>
#endif

#include <algorithm>
#include <array>
Expand All @@ -11,6 +15,8 @@
#include <cmath>
#include <csignal>
#include <cstdio>
#include <memory>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <string>
Expand Down Expand Up @@ -108,10 +114,18 @@ XrPosef parse_aim_to_wrist(const std::string& arg, spdlog::logger& logger)
return pose;
}

ManusPluginConfig parse_args(int argc, char** argv)
struct AppArgs
{
ManusPluginConfig config;
std::string datasets_arg = "human,sensors,haptic";
//! host[:port] or ws://host[:port][/path] of a remote_device_bridge; empty selects the local OpenXR session.
std::string remote_endpoint;
};

AppArgs parse_args(int argc, char** argv)
{
AppArgs args;
ManusPluginConfig& config = args.config;
std::optional<std::string> datasets_arg;
auto logger = isaaccapture::Logger::get("isaaccapture.plugins.manus.main");

for (int i = 1; i < argc; ++i)
Expand All @@ -121,6 +135,10 @@ ManusPluginConfig parse_args(int argc, char** argv)
{
datasets_arg = arg.substr(std::string("--datasets=").size());
}
else if (starts_with(arg, "--remote="))
{
args.remote_endpoint = arg.substr(std::string("--remote=").size());
}
else if (starts_with(arg, "--left-calibration="))
{
config.left_calibration_file = arg.substr(std::string("--left-calibration=").size());
Expand Down Expand Up @@ -150,7 +168,7 @@ ManusPluginConfig parse_args(int argc, char** argv)
config.human = false;
config.sensors = false;
config.haptic = false;
for (const auto& ds : split_csv(datasets_arg))
for (const auto& ds : split_csv(datasets_arg.value_or("human,sensors,haptic")))
{
if (ds == "human")
{
Expand All @@ -175,7 +193,20 @@ ManusPluginConfig parse_args(int argc, char** argv)
throw std::runtime_error("ManusHandPlugin: --datasets must enable at least one of human,sensors,haptic");
}

return config;
// RemotePluginSession's pull channel exposes no trackers, so the haptic reader cannot run over --remote.
if (!args.remote_endpoint.empty() && config.haptic)
{
if (datasets_arg)
{
throw std::invalid_argument(
"ManusHandPlugin: --remote cannot be combined with the 'haptic' data set; "
"use --datasets=human,sensors");
}
config.haptic = false;
logger->info("ManusHandPlugin: --remote given; defaulting to --datasets=human,sensors");
}

return args;
}

} // namespace
Expand All @@ -186,12 +217,29 @@ try
auto logger = isaaccapture::Logger::get("isaaccapture.plugins.manus.main");
logger->info("Manus Hand Plugin starting...");

const ManusPluginConfig config = parse_args(argc, argv);
const AppArgs args = parse_args(argc, argv);
const ManusPluginConfig& config = args.config;

// Empty for the local path: ManusTracker then opens its own OpenXR session, so this executable links no loader.
core::PluginSessionFactory remote_session_factory;
if (!args.remote_endpoint.empty())
{
#if defined(MANUS_HAVE_REMOTE_PLUGIN_SESSION)
// Parsed here so a malformed --remote fails at startup, not when ManusTracker opens its session.
remote_session_factory = remote_plugin_session::make_remote_plugin_session_factory(
"ManusHandPlugin", remote_plugin_session::parse_bridge_remote(args.remote_endpoint));
logger->info("Using remote plugin session at {}", args.remote_endpoint);
#else
throw std::invalid_argument(
"ManusHandPlugin: --remote requires a build with BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE=ON");
#endif
}

std::signal(SIGINT, signal_handler);
std::signal(SIGTERM, signal_handler);

auto& tracker = ManusTracker::instance(config);
auto& tracker = remote_session_factory ? ManusTracker::instance(config, remote_session_factory) :
ManusTracker::instance(config);

logger->info("Plugin running. Press Ctrl+C to stop.");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ class __attribute__((visibility("default"))) ManusTracker
// Hand-publishing methods
void inject_hand_data();

void log_status();

// -- Member Variables --

ManusPluginConfig m_config;
Expand All @@ -144,6 +146,11 @@ class __attribute__((visibility("default"))) ManusTracker
std::vector<unsigned char> m_right_calibration_file;
std::array<bool, 2> m_calibration_failed{ { false, false } };
bool is_connected = false;
// Last landscape as the SDK reported it (before the one-left/one-right filter); for log_status().
uint32_t m_sdk_glove_count = 0;
std::optional<bool> m_license_ok;
static constexpr std::chrono::seconds kStatusInterval{ 5 };
std::chrono::steady_clock::time_point m_next_status = std::chrono::steady_clock::now() + kStatusInterval;

// Haptic state — the per-side log-once flags use std::atomic to stay
// quiet when many frames in a row fail (e.g. the glove was disconnected
Expand Down
20 changes: 20 additions & 0 deletions src/plugins/manus/core/manus_hand_tracking_plugin.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,8 @@ ManusTracker& ManusTracker::instance(const ConstructionArgs* args)

void ManusTracker::update()
{
log_status();

if (!m_plugin_session)
{
// The configured plugin session is unavailable, so no session-backed
Expand Down Expand Up @@ -616,6 +618,22 @@ void ManusTracker::OnSkeletonStream(const SkeletonStreamInfo* skeleton_stream_in
}
}

//! Operator liveness line for the gloves and the Manus license, once per kStatusInterval.
void ManusTracker::log_status()
{
const auto now = std::chrono::steady_clock::now();
if (now < m_next_status)
{
return;
}
m_next_status = now + kStatusInterval;
std::lock_guard<std::mutex> lock(landscape_mutex);
m_logger->info("{}s: gloves={} license={}", kStatusInterval.count(), m_sdk_glove_count,
!m_license_ok ? "unknown" :
*m_license_ok ? "ok" :
"missing");
}

void ManusTracker::OnLandscapeStream(const Landscape* landscape)
{
auto& tracker = instance(nullptr);
Expand All @@ -628,6 +646,8 @@ void ManusTracker::OnLandscapeStream(const Landscape* landscape)
const auto& gloves = landscape->gloveDevices;

std::lock_guard<std::mutex> landscape_lock(tracker.landscape_mutex);
tracker.m_sdk_glove_count = gloves.gloveCount;
tracker.m_license_ok = landscape->settings.license.sdk || landscape->settings.license.integrated;

// We only support one left and one right glove
if (gloves.gloveCount > 2)
Expand Down
3 changes: 2 additions & 1 deletion src/plugins/remote_device_bridge/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@ SPDX-License-Identifier: Apache-2.0
- `session/protocol_frame.*`, `session/protocol_payloads.*` and the `Reader`/`Writer` in `session/protocol_io.hpp` are the only serialization points. Every frame is encoded and decoded there and nowhere else; changing the wire format means changing them and their codec tests together. Sessions hold an `ITransport` (`session/inc/remote_plugin_session/transport.hpp`) and only call `send/receive/shutdown` on complete frames; TCP framing lives in `TcpTransport`.
- Wire format v1 (`kProtocolVersion` = 1, the only version number): 24-byte header `RPS0` magic, type u8, version u8, channel u16, seq u32, payload size u32, `t_send` i64; frame types 1..13 with no gaps; the hand byte is 0 = left, 1 = right in every payload; PING has no payload and PONG carries the PING's header `t_send` and the client's receive time. Any change to a header, payload or type number bumps `kProtocolVersion`; `decode_header` rejects other versions before reading the type, so old senders and bridges fail with a readable reason instead of mis-decoding.
- A frame's `t_send` is its send time for clock exchange: call `stamp_header` where the frame is written to the transport, never when it is queued.
- WebSocket is a second transport, not a second protocol: `WebSocketTransport` carries exactly one wire frame per binary message (no TLS; terminate `wss://` in a proxy). Clients pick it with a `ws://host:port[/path]` address via `parse_remote`/`connect_transport`. The bridge serves it on a separate `--ws-listen` port sharing the fork-per-client loop and `--max-clients`; never sniff the protocol on the TCP port. The server-side HTTP upgrade runs in the first `receive()` so an accept loop never blocks on a client.
- WebSocket is a second transport, not a second protocol: `WebSocketTransport` carries exactly one wire frame per binary message (no TLS; terminate `wss://` in a proxy). Clients pick it with `--remote=ws://host:port[/path]` via `parse_remote`/`connect_transport`. The bridge serves it on a separate `--ws-listen` port sharing the fork-per-client loop and `--max-clients`; never sniff the protocol on the TCP port. The server-side HTTP upgrade runs in the first `receive()` so an accept loop never blocks on a client.
- Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to `Teleop::remote_plugin_session`: a plugin that talks to a bridge through this library must run without an OpenXR runtime.
- `Teleop::remote_device_bridge_server` (`server/`) gets each client's session from a `SessionFactory`; only `app/main.cpp` builds the OpenXR session, with `plugin_utils::make_openxr_plugin_session_factory`. Keep the OpenXR loader and `plugin_utils` out of `server/` so `tests/cpp/plugins/remote_device_bridge_server` can run the real fork-per-client bridge against a fake session.
- ERROR on channel 0 ends the session; ERROR on a channel id refuses only that OPEN (schema above `--max-schema-bytes`, a hand someone else owns) and the session and its other channels continue. Refuse per channel whenever the session itself is still sound.
- `RemotePluginSession` never throws on the sample path for transport failures (`push`, `push_buffer`, `query`, `update`): after one they become no-ops; only channel creation and the constructor throw. Caller bugs (null joints or buffer) still throw `std::invalid_argument`.
- The bridge (`server/`) talks to OpenXR from one thread per client (the thread running that `ClientSession`). Socket reading happens on a helper thread that only parses frames into a queue. Keep it that way; `OpenXRPluginSession` and its channels are not thread-safe.
- Plugins are not modified for remoting: only the composition root (`main.cpp`) chooses the `core::PluginSessionFactory` (`remote_plugin_session::make_remote_plugin_session_factory` or `plugin_utils::make_openxr_plugin_session_factory`). Do not add transport branches inside plugin classes.
- Timestamps cross the wire in the sender's `CLOCK_MONOTONIC`; the receiver maps them with `ClockOffsetEstimator`. The bridge's offset comes from PING/PONG (the minimum-RTT recent exchange, `--clock-source=two-way`); the one-way minimum of `local_recv - remote_send` hides the minimum network delay and is only the fallback (no PONG in time) or the `--clock-source=one-way` A/B baseline. Do not push raw remote timestamps into OpenXR.
- One bridge process serves N clients. The accept loop never creates threads or OpenXR objects so it may `fork()` a child per client (the isolation of N local plugin processes). Do not serve clients on threads: the OpenXR loader allows one `XrInstance` per process (a second one fails with -10). State the children must share (hand ownership) lives in a `MAP_SHARED` page mapped before the first fork; the parent clears a dead child's entries when it reaps it. Sample paths are lossy by design: per-channel latest-wins queues on the client, coalescing and a staleness guard on the bridge; control frames are never dropped or reordered.
1 change: 1 addition & 0 deletions src/plugins/remote_device_bridge/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ endif()
add_subdirectory(session)
add_subdirectory(server)
add_subdirectory(app)
add_subdirectory(tools)

install(FILES plugin.yaml
DESTINATION plugins/remote_device_bridge
Expand Down
22 changes: 22 additions & 0 deletions src/plugins/remote_device_bridge/tools/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

include(GNUInstallDirs)

add_executable(remote_hand_sender_synthetic
remote_hand_sender_synthetic.cpp
)

target_link_libraries(remote_hand_sender_synthetic
PRIVATE
Teleop::remote_plugin_session
log_bridge::log_bridge_core
)

set_target_properties(remote_hand_sender_synthetic PROPERTIES
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin"
)

install(TARGETS remote_hand_sender_synthetic
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)
Loading
Loading