Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi
option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF)
option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF)
option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF)
option(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE "Build the remote device bridge wire codec library (Teleop::remote_plugin_session)" ON)
option(BUILD_EXAMPLES "Build examples" ON)
option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF)
option(BUILD_TESTING "Build unit tests" ON)
Expand Down
4 changes: 4 additions & 0 deletions src/plugins/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ add_subdirectory(manus)
add_subdirectory(manus_calibration)
add_subdirectory(haptikos)

if(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE)
add_subdirectory(remote_device_bridge)
endif()

if(BUILD_PLUGIN_NOITOM_MOCAP)
add_subdirectory(noitom_mocap)
endif()
Expand Down
13 changes: 13 additions & 0 deletions src/plugins/remote_device_bridge/AGENTS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<!--
SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
SPDX-License-Identifier: Apache-2.0
-->

# Agent notes — `src/plugins/remote_device_bridge`

**CRITICAL:** Complete the mandatory `AGENTS.md` preflight in [`../../../AGENTS.md`](../../../AGENTS.md) before editing here.

- `session/protocol_frame.*`, `session/protocol_payloads.*` and the `Reader`/`Writer` in `session/protocol_io.hpp` are the only serialization points. Every frame is encoded and decoded there and nowhere else; changing the wire format means changing them and their codec tests together.
- Wire format v1 (`kProtocolVersion` = 1, the only version number): 24-byte header `RPS0` magic, type u8, version u8, channel u16, seq u32, payload size u32, `t_send` i64; frame types 1..13 with no gaps; the hand byte is 0 = left, 1 = right in every payload; PING has no payload and PONG carries the PING's header `t_send` and the client's receive time. Any change to a header, payload or type number bumps `kProtocolVersion`; `decode_header` rejects other versions before reading the type, so old senders and bridges fail with a readable reason instead of mis-decoding.
- A frame's `t_send` is its send time for clock exchange: call `stamp_header` where the frame is written to the transport, never when it is queued.
- Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to `Teleop::remote_plugin_session`: a plugin that talks to a bridge through this library must run without an OpenXR runtime.
4 changes: 4 additions & 0 deletions src/plugins/remote_device_bridge/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

add_subdirectory(session)
22 changes: 22 additions & 0 deletions src/plugins/remote_device_bridge/session/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

add_library(remote_plugin_session STATIC
protocol_frame.cpp
protocol_payloads.cpp
protocol_io.hpp
inc/remote_plugin_session/protocol_frame.hpp
inc/remote_plugin_session/protocol_payloads.hpp
)

target_include_directories(remote_plugin_session
INTERFACE
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/inc>
)

target_link_libraries(remote_plugin_session
PUBLIC
pusherio::pusherio
)

add_library(Teleop::remote_plugin_session ALIAS remote_plugin_session)
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

#pragma once

#include <cstddef>
#include <cstdint>
#include <span>
#include <stdexcept>
#include <vector>

namespace remote_plugin_protocol
{

/*!
* Remote plugin protocol v1: fixed little-endian frames over one connection per
* plugin session. Frame = header + payload; payload layouts live in
* protocol_payloads.hpp. kProtocolVersion versions the whole format: any change
* to a header, payload or frame type number bumps it, and decode_header rejects
* every other version before it reads the type or anything after it.
*/

constexpr uint32_t kMagic = 0x30535052u; // "RPS0"
constexpr uint8_t kProtocolVersion = 1;
//! Header byte offsets: magic u32, type u8, version u8, channel u16, seq u32, payload size u32, t_send i64.
constexpr size_t kTypeOffset = sizeof(kMagic);
constexpr size_t kVersionOffset = kTypeOffset + sizeof(uint8_t);
constexpr size_t kChannelOffset = kVersionOffset + sizeof(uint8_t);
constexpr size_t kSeqOffset = kChannelOffset + sizeof(uint16_t);
constexpr size_t kPayloadSizeOffset = kSeqOffset + sizeof(uint32_t);
constexpr size_t kSendTimeOffset = kPayloadSizeOffset + sizeof(uint32_t);
constexpr size_t kHeaderSize = kSendTimeOffset + sizeof(int64_t);
constexpr uint16_t kSessionChannelId = 0;
constexpr uint32_t kMaxPayloadSize = 16u * 1024u * 1024u; // large tactile/sensor payloads; still bounded

//! Direction is c->s (plugin to bridge) or s->c. OPEN_PULL, CLOSE, HEARTBEAT and PING have empty payloads.
enum class FrameType : uint8_t
{
Hello = 1, // c->s
HelloAck = 2, // s->c
OpenHand = 3, // c->s
OpenSchema = 4, // c->s
OpenPull = 5, // c->s
Close = 6, // c->s
HandSample = 7, // c->s
SchemaSample = 8, // c->s
WristSample = 9, // s->c
Heartbeat = 10, // both
Error = 11, // s->c
Ping = 12, // s->c
Pong = 13, // c->s
};

struct FrameHeader
{
FrameType type{}; // 0 is not a frame type
uint16_t channel_id = kSessionChannelId;
uint32_t seq = 0;
uint32_t payload_size = 0;
int64_t t_send_monotonic_ns = 0;
};

struct Frame
{
FrameHeader header;
std::vector<uint8_t> payload;
};

class DecodeError : public std::runtime_error
{
public:
using std::runtime_error::runtime_error;
};

//! Reads the header from the first kHeaderSize bytes of `in`. Throws DecodeError, checking in this order: `in`
//! shorter than a header, bad magic, oversized payload, another protocol version, unknown type. The version is
//! checked before the type byte is interpreted, since another version may number its frame types differently.
FrameHeader decode_header(std::span<const uint8_t> in);

//! What framing needs: the payload size from the first kHeaderSize bytes of `in`, checking only the magic and the
//! payload limit, which sit at the same offsets in every protocol version. Version and type are left to
//! decode_header, so a frame from another version still reaches the decoder and fails there with a readable reason.
//! Throws DecodeError if `in` is shorter than a header, on bad magic or an oversized payload.
uint32_t frame_payload_size(std::span<const uint8_t> in);

//! Writes header + payload into one contiguous buffer. seq and t_send stay zero until stamp_header(), which the
//! sender calls at the moment it writes the frame: t_send is the send time PING/PONG clock exchanges use.
std::vector<uint8_t> encode_frame(FrameType type, uint16_t channel_id, std::span<const uint8_t> payload);
//! Patches seq and t_send into an encoded frame's header in place.
void stamp_header(std::vector<uint8_t>& frame, uint32_t seq, int64_t t_send_monotonic_ns);
//! Splits one complete message into header and payload. The payload keeps `message`'s buffer (its bytes are
//! shifted down by the header size). Throws DecodeError.
Frame split_frame(std::vector<uint8_t> message);

} // namespace remote_plugin_protocol
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

#pragma once

#include "protocol_frame.hpp"

#include <openxr/openxr.h>
#include <pusherio/plugin_session.hpp>

#include <array>
#include <cstddef>
#include <cstdint>
#include <span>
#include <string>
#include <vector>

namespace remote_plugin_protocol
{

/*!
* Payload builders and parsers. `str` = u32 length + bytes, `pose` = 7 x f32 in
* XrPosef order (orientation x,y,z,w then position x,y,z).
*/

struct Hello
{
std::string app_name;
core::PluginSessionRequirements requirements;
};

struct HelloAck
{
bool accepted = false;
std::string reason;
};

struct HandSample
{
int64_t sample_time_local_common_clock_ns = 0;
std::array<XrHandJointLocationEXT, XR_HAND_JOINT_COUNT_EXT> joints{};
};

struct SchemaSample
{
int64_t sample_time_local_common_clock_ns = 0;
int64_t sample_time_raw_device_clock_ns = 0;
std::vector<uint8_t> bytes;
};

//! On the wire the hand is one byte, 0 = left and 1 = right, as in OPEN_HAND.
struct WristSample
{
int64_t sample_time_local_common_clock_ns = 0;
XrHandEXT hand = XR_HAND_LEFT_EXT;
core::WristTrackingSample sample;
};

//! Two-way clock exchange; times are CLOCK_MONOTONIC ns of the side that took them. The PING carries no payload:
//! its send time is its header t_send. The PONG's own send time is the PONG header t_send.
struct Pong
{
int64_t ping_sent_ns = 0; //!< bridge clock: the PING header t_send, echoed
int64_t ping_received_ns = 0; //!< client clock
};

constexpr size_t kPoseSize = 7 * sizeof(float);
constexpr size_t kJointSize = sizeof(uint64_t) + kPoseSize + sizeof(float);
constexpr size_t kHandSamplePayloadSize = sizeof(int64_t) + XR_HAND_JOINT_COUNT_EXT * kJointSize;
//! WRIST_SAMPLE payload: sample time i64, hand u8, valid u8, tracked u8, pose.
constexpr size_t kWristSamplePayloadSize = sizeof(int64_t) + 3 * sizeof(uint8_t) + kPoseSize;
//! SCHEMA_SAMPLE payload before the schema bytes: two sample times (i64) and the byte count (u32).
constexpr size_t kSchemaSampleFixedSize = 2 * sizeof(int64_t) + sizeof(uint32_t);

static_assert(kJointSize == 40);
static_assert(kHandSamplePayloadSize == 1048);
static_assert(kWristSamplePayloadSize == 39);
static_assert(kSchemaSampleFixedSize == 20);

//! One hand's joints, as IHandTrackingPushChannel::push receives them.
using HandJoints = std::span<const XrHandJointLocationEXT, XR_HAND_JOINT_COUNT_EXT>;

std::vector<uint8_t> encode_hello(const Hello& value);
std::vector<uint8_t> encode_hello_ack(const HelloAck& value);
//! One byte: 0 = left, 1 = right.
std::vector<uint8_t> encode_open_hand(XrHandEXT hand);
std::vector<uint8_t> encode_open_schema(const core::SchemaPusherConfig& value);
std::vector<uint8_t> encode_wrist_sample(const WristSample& value);
std::vector<uint8_t> encode_error(const std::string& message);
std::vector<uint8_t> encode_pong(const Pong& value);

// HAND_SAMPLE and SCHEMA_SAMPLE frames are only built this way: the builders read the plugin's buffers directly so a
// push copies once.
std::vector<uint8_t> encode_hand_sample_frame(uint16_t channel_id,
HandJoints joints,
int64_t sample_time_local_common_clock_ns);
std::vector<uint8_t> encode_schema_sample_frame(uint16_t channel_id,
std::span<const uint8_t> bytes,
int64_t sample_time_local_common_clock_ns,
int64_t sample_time_raw_device_clock_ns);

// Parsers throw DecodeError, naming the frame, on truncated, over-long or out-of-range payloads.
Hello decode_hello(std::span<const uint8_t> payload);
HelloAck decode_hello_ack(std::span<const uint8_t> payload);
XrHandEXT decode_open_hand(std::span<const uint8_t> payload);
core::SchemaPusherConfig decode_open_schema(std::span<const uint8_t> payload);
HandSample decode_hand_sample(std::span<const uint8_t> payload);
SchemaSample decode_schema_sample(std::span<const uint8_t> payload);
WristSample decode_wrist_sample(std::span<const uint8_t> payload);
std::string decode_error(std::span<const uint8_t> payload);
Pong decode_pong(std::span<const uint8_t> payload);

} // namespace remote_plugin_protocol
Loading
Loading