From 7fa5b910793fe1b49f4ccad2404f19be9139680a Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Wed, 21 Oct 2026 10:42:30 -0700 Subject: [PATCH 01/10] feat(remote_device_bridge): add RemotePluginSession over ITransport RemotePluginSession implements core::IPluginSession for a plugin that runs without an OpenXR runtime: HELLO/HELLO_ACK handshake, hand, schema and pull/wrist channels as wire frames, per-channel latest-wins sample queues (control frames are never dropped), heartbeats, PONG answers to bridge PINGs, and per-channel refusal that keeps the session alive. ClockOffsetEstimator maps remote CLOCK_MONOTONIC to local time from two-way PING/PONG exchanges (min-RTT of the last 32), with the one-way minimum as fallback; PingScheduler drives the exchange. Signed-off-by: Jonghyun Kim --- src/plugins/remote_device_bridge/AGENTS.md | 4 + .../session/CMakeLists.txt | 4 + .../session/clock_offset_estimator.cpp | 119 +++ .../clock_offset_estimator.hpp | 108 +++ .../remote_plugin_session.hpp | 79 ++ .../session/remote_plugin_session.cpp | 780 ++++++++++++++++++ .../remote_device_bridge/CMakeLists.txt | 2 + .../test_clock_offset_estimator.cpp | 198 +++++ .../test_remote_plugin_session.cpp | 768 +++++++++++++++++ 9 files changed, 2062 insertions(+) create mode 100644 src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp create mode 100644 src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp create mode 100644 src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp create mode 100644 src/plugins/remote_device_bridge/session/remote_plugin_session.cpp create mode 100644 tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp create mode 100644 tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp diff --git a/src/plugins/remote_device_bridge/AGENTS.md b/src/plugins/remote_device_bridge/AGENTS.md index 70b309a38d..104a9998aa 100644 --- a/src/plugins/remote_device_bridge/AGENTS.md +++ b/src/plugins/remote_device_bridge/AGENTS.md @@ -12,3 +12,7 @@ SPDX-License-Identifier: Apache-2.0 - A frame's `t_send` is its send time for clock exchange: call `stamp_header` where the frame is written to the transport, never when it is queued. - WebSocket is a second transport, not a second protocol: `WebSocketTransport` carries exactly one wire frame per binary message (no TLS; terminate `wss://` in a proxy). Clients pick it with a `ws://host:port[/path]` address via `parse_remote`/`connect_transport`. A server serves it on its own port; never sniff the protocol on the TCP port. The server-side HTTP upgrade runs in the first `receive()` so an accept loop never blocks on a client. - Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to `Teleop::remote_plugin_session`: a plugin that talks to a bridge through this library must run without an OpenXR runtime. +- ERROR on channel 0 ends the session; ERROR on a channel id refuses only that OPEN (schema above `--max-schema-bytes`, a hand someone else owns) and the session and its other channels continue. Refuse per channel whenever the session itself is still sound. +- `RemotePluginSession` never throws on the sample path (`push`, `push_buffer`, `query`, `update`). After a transport failure those become no-ops; only channel creation and the constructor throw. +- Timestamps cross the wire in the sender's `CLOCK_MONOTONIC`; the receiver maps them with `ClockOffsetEstimator`. The bridge's offset comes from PING/PONG (min-RTT of the last 32 exchanges, `--clock-source=two-way`); the one-way minimum of `local_recv - remote_send` hides the minimum network delay and is only the fallback (HELLO v0 client, no PONG within 2 s) or the `--clock-source=one-way` A/B baseline. Never PING a HELLO v0 client: it drops the connection on unknown frame types. Do not push raw remote timestamps into OpenXR. +- Tests that need a stalled link use `BlockingTransport` in `test_remote_plugin_session.cpp`, never a TCP peer that stops reading: loopback socket buffers differ per host (416 KiB in one sandbox, 2×8 MiB on the teleop workstation), so "the sender is blocked by now" is not portable. diff --git a/src/plugins/remote_device_bridge/session/CMakeLists.txt b/src/plugins/remote_device_bridge/session/CMakeLists.txt index 4a6542aab2..c37b1e00e3 100644 --- a/src/plugins/remote_device_bridge/session/CMakeLists.txt +++ b/src/plugins/remote_device_bridge/session/CMakeLists.txt @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 add_library(remote_plugin_session STATIC + clock_offset_estimator.cpp + inc/remote_plugin_session/clock_offset_estimator.hpp protocol_frame.cpp protocol_payloads.cpp protocol_io.hpp @@ -9,6 +11,8 @@ add_library(remote_plugin_session STATIC inc/remote_plugin_session/protocol_payloads.hpp remote_address.cpp inc/remote_plugin_session/remote_address.hpp + remote_plugin_session.cpp + inc/remote_plugin_session/remote_plugin_session.hpp tcp_socket.cpp inc/remote_plugin_session/tcp_socket.hpp tcp_transport.cpp diff --git a/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp new file mode 100644 index 0000000000..88954c3324 --- /dev/null +++ b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/remote_plugin_session/clock_offset_estimator.hpp" + +#include +#include + +namespace remote_plugin_session +{ + +ClockOffsetEstimator::ClockOffsetEstimator(size_t max_samples, int64_t window_ns) + : m_max_samples(max_samples), m_window_ns(window_ns) +{ + if (max_samples == 0 || window_ns <= 0) + { + throw std::invalid_argument("ClockOffsetEstimator requires a non-empty window"); + } +} + +void ClockOffsetEstimator::add_sample(int64_t remote_t_send_ns, int64_t local_t_recv_ns) +{ + m_samples.push_back(Sample{ local_t_recv_ns, local_t_recv_ns - remote_t_send_ns }); + + while (m_samples.size() > m_max_samples) + { + m_samples.pop_front(); + } + while (!m_samples.empty() && m_samples.front().local_t_recv_ns < local_t_recv_ns - m_window_ns) + { + m_samples.pop_front(); + } + + m_offset_ns = std::min_element(m_samples.begin(), m_samples.end(), + [](const Sample& a, const Sample& b) { return a.offset_ns < b.offset_ns; }) + ->offset_ns; +} + +void ClockOffsetEstimator::add_two_way_sample(int64_t t1_ns, int64_t t2_ns, int64_t t3_ns, int64_t t4_ns) +{ + const int64_t rtt_ns = (t4_ns - t1_ns) - (t3_ns - t2_ns); + if (rtt_ns < 0) + { + return; + } + // local - remote, the same sign as the one-way offset. + const int64_t offset_ns = ((t1_ns - t2_ns) + (t4_ns - t3_ns)) / 2; + m_exchanges.push_back(Exchange{ rtt_ns, offset_ns }); + if (m_exchanges.size() > kTwoWayWindow) + { + m_exchanges.pop_front(); + } + const Exchange& best = *std::min_element(m_exchanges.begin(), m_exchanges.end(), + [](const Exchange& a, const Exchange& b) { return a.rtt_ns < b.rtt_ns; }); + m_two_way_offset_ns = best.offset_ns; + m_rtt_min_ns = best.rtt_ns; + m_rtt_last_ns = rtt_ns; +} + +int64_t ClockOffsetEstimator::offset_ns() const +{ + return m_exchanges.empty() ? m_offset_ns : m_two_way_offset_ns; +} + +int64_t ClockOffsetEstimator::remote_to_local(int64_t remote_ns) const +{ + return remote_ns + offset_ns(); +} + +int64_t ClockOffsetEstimator::rtt_min_ns() const +{ + return m_rtt_min_ns; +} + +int64_t ClockOffsetEstimator::rtt_last_ns() const +{ + return m_rtt_last_ns; +} + +const char* ClockOffsetEstimator::offset_source() const +{ + return m_exchanges.empty() ? "one-way" : "two-way"; +} + +PingScheduler::PingScheduler(int64_t start_ns) : m_start_ns(start_ns), m_next_ns(start_ns) +{ +} + +PingScheduler::Action PingScheduler::poll(int64_t now_ns) +{ + if (m_gave_up) + { + return Action::None; + } + if (!m_got_pong && now_ns - m_start_ns >= kPongTimeoutNs) + { + m_gave_up = true; + return Action::GiveUp; + } + if (now_ns < m_next_ns) + { + return Action::None; + } + ++m_sent; + m_next_ns = now_ns + (m_sent < kBurstCount ? kBurstPeriodNs : kSteadyPeriodNs); + return Action::Ping; +} + +void PingScheduler::on_pong() +{ + m_got_pong = true; +} + +bool PingScheduler::active() const +{ + return !m_gave_up; +} + +} // namespace remote_plugin_session diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp new file mode 100644 index 0000000000..280b716bbb --- /dev/null +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace remote_plugin_session +{ + +/*! + * @brief Maps a remote monotonic clock onto the local one (local ~= remote + offset). + * + * One-way: fed with (remote send time, local receive time) of every received + * frame; the offset is the running minimum of (local - remote) over a sliding + * window, which hides one minimum one-way delay. Two-way: once a PING/PONG + * exchange arrives the offset comes from the minimum-RTT exchange of the last + * kTwoWayWindow instead (NTP filter; error = path asymmetry / 2) and one-way + * samples no longer move it. Not thread-safe; the owner serializes access. + */ +class ClockOffsetEstimator +{ +public: + static constexpr size_t kTwoWayWindow = 32; + + explicit ClockOffsetEstimator(size_t max_samples = 200, int64_t window_ns = 5'000'000'000); + + void add_sample(int64_t remote_t_send_ns, int64_t local_t_recv_ns); + //! t1/t4: local PING send / PONG receive; t2/t3: remote PING receive / PONG send. Negative RTTs are ignored. + void add_two_way_sample(int64_t t1_ns, int64_t t2_ns, int64_t t3_ns, int64_t t4_ns); + + //! Zero until the first sample arrives. + int64_t offset_ns() const; + int64_t remote_to_local(int64_t remote_ns) const; + //! Zero until the first two-way exchange. + int64_t rtt_min_ns() const; + int64_t rtt_last_ns() const; + //! "two-way" once an exchange was added, else "one-way". + const char* offset_source() const; + +private: + struct Sample + { + int64_t local_t_recv_ns; + int64_t offset_ns; + }; + struct Exchange + { + int64_t rtt_ns; + int64_t offset_ns; + }; + + size_t m_max_samples; + int64_t m_window_ns; + std::deque m_samples; + int64_t m_offset_ns = 0; + std::deque m_exchanges; + int64_t m_two_way_offset_ns = 0; + int64_t m_rtt_min_ns = 0; + int64_t m_rtt_last_ns = 0; +}; + +/*! + * @brief When the bridge sends PINGs: a burst right after HELLO_ACK, then 1 Hz. + * + * Without any PONG within kPongTimeout of the start, poll() returns GiveUp once + * and nothing after that: the peer stays on the one-way estimate. + */ +class PingScheduler +{ +public: + enum class Action + { + None, + Ping, + GiveUp, + }; + + static constexpr int kBurstCount = 10; + static constexpr int64_t kBurstPeriodNs = 50'000'000; + static constexpr int64_t kSteadyPeriodNs = 1'000'000'000; + static constexpr int64_t kPongTimeoutNs = 2'000'000'000; + + explicit PingScheduler(int64_t start_ns); + + Action poll(int64_t now_ns); + void on_pong(); + //! False after GiveUp; late PONGs are then ignored. + bool active() const; + +private: + int64_t m_start_ns; + int64_t m_next_ns; + int m_sent = 0; + bool m_got_pong = false; + bool m_gave_up = false; +}; + +//! True when a mapped sample time is older than now - max_age; max_age <= 0 disables the guard. After a link +//! stall the backlog arrives in a burst; pushing it into OpenXR would replay old motion instead of skipping it. +inline bool should_drop_stale(int64_t mapped_sample_ns, int64_t now_ns, int64_t max_age_ns) +{ + return max_age_ns > 0 && now_ns - mapped_sample_ns > max_age_ns; +} + +} // namespace remote_plugin_session diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp new file mode 100644 index 0000000000..4aa48c95b9 --- /dev/null +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "transport.hpp" + +#include + +#include +#include +#include +#include + +namespace remote_plugin_session +{ + +namespace detail +{ +class RemotePluginSessionImpl; +} + +/*! + * @brief Plugin session that forwards every channel over TCP to a remote_device_bridge. + * + * Same contract as OpenXRPluginSession: the owner destroys all channels before + * the session and may call create_pull_channel() once. After the transport + * drops, pushes become silent no-ops and channels stay safe to destroy. + * Each push channel has a latest-wins queue (4 samples / 8 MiB); on a stalled + * link the oldest samples of that channel are dropped, other channels are + * unaffected. A channel the bridge refuses (ERROR on its channel id) turns + * into a no-op; the session and its other channels continue. P0: the pull + * channel exposes no trackers; wrist poses are the latest bridge samples (no + * interpolation). + */ +class RemotePluginSession final : public core::IPluginSession +{ +public: + struct Stats + { + uint64_t sent_hand = 0; + uint64_t sent_schema = 0; + uint64_t wrist_rx = 0; + uint64_t heartbeats = 0; + uint64_t dropped_queue = 0; //!< samples evicted by the per-channel latest-wins queues + uint64_t tx_bytes = 0; + std::map dropped_by_channel; //!< open push channels only + std::map refused_channels; //!< channel id -> the bridge's reason + }; + + //! Connects over TCP, sends HELLO and waits for HELLO_ACK; throws std::runtime_error on any failure. + RemotePluginSession(std::string host, uint16_t port, std::string app_name, core::PluginSessionRequirements requirements); + //! Same handshake over an already connected transport. + RemotePluginSession(std::unique_ptr transport, + std::string app_name, + core::PluginSessionRequirements requirements); + ~RemotePluginSession() override; + + RemotePluginSession(const RemotePluginSession&) = delete; + RemotePluginSession& operator=(const RemotePluginSession&) = delete; + RemotePluginSession(RemotePluginSession&&) = delete; + RemotePluginSession& operator=(RemotePluginSession&&) = delete; + + std::unique_ptr create_pull_channel() override; + std::unique_ptr create_schema_push_channel(const core::SchemaPusherConfig& config) override; + std::unique_ptr create_hand_tracking_push_channel(XrHandEXT hand) override; + + //! False once the transport failed; channel creation then throws and pushes are dropped. + bool connected() const; + //! Estimated bridge-to-local clock offset in ns (local ~= bridge + offset); 0 before the first frame. + int64_t clock_offset_ns() const; + //! Cumulative counters; also logged every 5 s as one "[remote] 5s:" line. + Stats stats() const; + +private: + std::shared_ptr m_impl; +}; + +} // namespace remote_plugin_session diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp new file mode 100644 index 0000000000..9f082053c9 --- /dev/null +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -0,0 +1,780 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/remote_plugin_session/remote_plugin_session.hpp" + +#include "inc/remote_plugin_session/clock_offset_estimator.hpp" +#include "inc/remote_plugin_session/tcp_socket.hpp" +#include "inc/remote_plugin_session/wire_codec.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace remote_plugin_session +{ + +namespace +{ + +std::shared_ptr logger() +{ + return isaaccapture::Logger::get("isaaccapture.plugins.remote_device_bridge.RemotePluginSession"); +} + +constexpr std::chrono::milliseconds kConnectTimeout{ 3000 }; +// The bridge answers HELLO only after its OpenXR session is up, which waits for a headset; on CloudXR that +// routinely takes longer than a few seconds. +constexpr std::chrono::milliseconds kHandshakeTimeout{ 30000 }; +constexpr std::chrono::milliseconds kHeartbeatPeriod{ 500 }; +constexpr std::chrono::milliseconds kPeerTimeout{ 3000 }; +constexpr std::chrono::milliseconds kReceivePoll{ 100 }; +constexpr std::chrono::seconds kStatsPeriod{ 5 }; +// Destruction lets the sender flush its CLOSE frames this long before the transport is cut under it. +constexpr std::chrono::milliseconds kShutdownDrain{ 500 }; +// Latest-wins per channel: on a stalled link each channel keeps at most this many (or this much) of its own +// newest samples; a fast channel can never evict a slow one. +constexpr size_t kMaxQueuedSamplesPerChannel = 4; +constexpr size_t kMaxQueuedBytesPerChannel = 8u * 1024u * 1024u; + +using FrameBytes = std::vector; // one encoded wire message, header included + +struct ChannelQueue +{ + wire::FrameType type = wire::FrameType::HandSample; + std::deque frames; + size_t bytes = 0; + uint64_t dropped = 0; +}; + +} // namespace + +class detail::RemotePluginSessionImpl +{ +public: + RemotePluginSessionImpl(std::unique_ptr transport, + const std::string& app_name, + core::PluginSessionRequirements requirements) + : m_requirements(requirements), m_transport(std::move(transport)) + { + if (!m_transport) + { + throw std::invalid_argument("RemotePluginSession: transport is null"); + } + + wire::Hello hello; + hello.app_name = app_name; + hello.requirements = requirements; + FrameBytes hello_frame = wire::encode_frame(wire::FrameType::Hello, wire::kSessionChannelId, wire::encode(hello)); + if (!send_now(hello_frame)) + { + throw std::runtime_error("RemotePluginSession: failed to send HELLO"); + } + + FrameBytes bytes; + const ReceiveResult result = m_transport->receive(bytes, kHandshakeTimeout); + if (result != ReceiveResult::Ok) + { + throw std::runtime_error(result == ReceiveResult::Timeout ? + "RemotePluginSession: timed out waiting for HELLO_ACK" : + "RemotePluginSession: connection lost while waiting for HELLO_ACK"); + } + const wire::Frame frame = wire::split_frame(std::move(bytes)); + if (frame.header.type != wire::FrameType::HelloAck) + { + throw std::runtime_error(std::string("RemotePluginSession: expected HELLO_ACK, received ") + + wire::to_string(frame.header.type)); + } + const wire::HelloAck ack = wire::decode_hello_ack(frame.payload); + if (!ack.accepted) + { + throw std::runtime_error("RemotePluginSession: bridge rejected the session: " + ack.reason); + } + m_clock.add_sample(frame.header.t_send_monotonic_ns, core::os_monotonic_now_ns()); + + m_connected.store(true); + try + { + m_sender = std::thread( + [this] + { + sender_loop(); + { + std::lock_guard lock(m_queue_mutex); + m_sender_done = true; + } + m_queue_cv.notify_all(); + }); + m_receiver = std::thread(&RemotePluginSessionImpl::receiver_loop, this); + } + catch (...) + { + shutdown(); + throw; + } + } + + ~RemotePluginSessionImpl() + { + shutdown(); + } + + const core::PluginSessionRequirements& requirements() const + { + return m_requirements; + } + + bool connected() const + { + return m_connected.load(); + } + + int64_t clock_offset_ns() const + { + std::lock_guard lock(m_clock_mutex); + return m_clock.offset_ns(); + } + + RemotePluginSession::Stats stats() const + { + RemotePluginSession::Stats stats; + stats.sent_hand = m_sent_hand.load(); + stats.sent_schema = m_sent_schema.load(); + stats.wrist_rx = m_wrist_rx.load(); + stats.heartbeats = m_heartbeats.load(); + stats.tx_bytes = m_tx_bytes.load(); + std::lock_guard lock(m_queue_mutex); + stats.dropped_queue = m_dropped_samples; + for (const auto& [channel_id, queue] : m_sample_queues) + { + stats.dropped_by_channel[channel_id] = queue.dropped; + } + stats.refused_channels = m_refused_channels; + return stats; + } + + void require_connected() const + { + if (!m_connected.load()) + { + throw std::runtime_error("RemotePluginSession: transport is disconnected"); + } + } + + //! Allocates a channel id and queues its OPEN_* frame; ids are never reused within a session. + uint16_t open_channel(wire::FrameType type, std::vector payload) + { + require_connected(); + uint16_t channel_id = 0; + { + std::lock_guard lock(m_state_mutex); + if (m_next_channel_id == 0) + { + throw std::runtime_error("RemotePluginSession: channel ids exhausted"); + } + channel_id = m_next_channel_id++; + m_open_channels.insert(channel_id); + } + enqueue_control(type, channel_id, std::move(payload)); + return channel_id; + } + + void send_control(wire::FrameType type, uint16_t channel_id, std::vector payload) + { + require_connected(); + enqueue_control(type, channel_id, std::move(payload)); + } + + void close_channel(uint16_t channel_id) + { + { + std::lock_guard lock(m_state_mutex); + if (m_open_channels.erase(channel_id) == 0) + { + return; + } + } + { + // Pending samples must not follow CLOSE onto the wire (the bridge would see an unknown channel). + std::lock_guard lock(m_queue_mutex); + if (const auto it = m_sample_queues.find(channel_id); it != m_sample_queues.end()) + { + m_queued_samples -= it->second.frames.size(); + m_sample_queues.erase(it); + } + } + if (m_connected.load()) + { + enqueue_control(wire::FrameType::Close, channel_id, {}); + } + } + + //! Per-channel latest-wins queue; control frames are never dropped. + void push_sample(wire::FrameType type, uint16_t channel_id, FrameBytes frame) + { + if (!m_connected.load()) + { + return; + } + { + std::lock_guard lock(m_queue_mutex); + if (m_refused_channels.count(channel_id) != 0) + { + return; + } + ChannelQueue& queue = m_sample_queues[channel_id]; + queue.type = type; + while (!queue.frames.empty() && (queue.frames.size() >= kMaxQueuedSamplesPerChannel || + queue.bytes + frame.size() > kMaxQueuedBytesPerChannel)) + { + queue.bytes -= queue.frames.front().size(); + queue.frames.pop_front(); + ++queue.dropped; + ++m_dropped_samples; + --m_queued_samples; + } + queue.bytes += frame.size(); + queue.frames.push_back(std::move(frame)); + ++m_queued_samples; + } + m_queue_cv.notify_one(); + } + + bool claim_pull_channel() + { + std::lock_guard lock(m_state_mutex); + return !std::exchange(m_pull_channel_claimed, true); + } + + core::WristTrackingSample latest_wrist(bool is_left) const + { + std::lock_guard lock(m_wrist_mutex); + return m_wrist[is_left ? 0 : 1]; + } + + void shutdown() + { + if (m_shutdown.exchange(true)) + { + return; + } + + std::set remaining; + { + std::lock_guard lock(m_state_mutex); + remaining.swap(m_open_channels); + } + if (m_connected.load()) + { + for (auto it = remaining.rbegin(); it != remaining.rend(); ++it) + { + enqueue_control(wire::FrameType::Close, *it, {}); + } + } + + { + std::unique_lock lock(m_queue_mutex); + m_stop = true; + m_queue_cv.notify_all(); + if (m_sender.joinable()) + { + // A sender stalled inside send() cannot finish on its own; the transport shutdown below cuts it off. + m_queue_cv.wait_for(lock, kShutdownDrain, [this] { return m_sender_done; }); + } + } + m_connected.store(false); + m_transport->shutdown(); + if (m_sender.joinable()) + { + m_sender.join(); + } + if (m_receiver.joinable()) + { + m_receiver.join(); + } + } + +private: + void enqueue_control(wire::FrameType type, uint16_t channel_id, const std::vector& payload) + { + FrameBytes frame = wire::encode_frame(type, channel_id, payload); + { + std::lock_guard lock(m_queue_mutex); + m_control_queue.push_back(std::move(frame)); + } + m_queue_cv.notify_one(); + } + + //! Stamps seq/t_send and sends one frame. Constructor thread during the handshake, sender thread after. + bool send_now(FrameBytes& frame) + { + wire::stamp_header(frame.data(), m_next_seq++, core::os_monotonic_now_ns()); + if (!m_transport->send(frame.data(), frame.size())) + { + return false; + } + m_tx_bytes.fetch_add(frame.size()); + return true; + } + + //! Waits until `deadline` for work. Control frames first and in order, then one sample per channel + //! (round-robin) so a 180 Hz channel cannot starve a 60 Hz one. Empty when the deadline passed. + std::vector next_batch(std::chrono::steady_clock::time_point deadline) + { + std::vector batch; + std::unique_lock lock(m_queue_mutex); + m_queue_cv.wait_until( + lock, deadline, + [this] { return m_stop || !m_connected.load() || !m_control_queue.empty() || m_queued_samples > 0; }); + for (FrameBytes& frame : m_control_queue) + { + batch.push_back(std::move(frame)); + } + m_control_queue.clear(); + if (m_stop) + { + return batch; + } + for (auto& [channel_id, queue] : m_sample_queues) + { + if (queue.frames.empty()) + { + continue; + } + queue.bytes -= queue.frames.front().size(); + batch.push_back(std::move(queue.frames.front())); + queue.frames.pop_front(); + --m_queued_samples; + (queue.type == wire::FrameType::HandSample ? m_sent_hand : m_sent_schema).fetch_add(1); + } + return batch; + } + + void log_stats() + { + const RemotePluginSession::Stats now = stats(); + const RemotePluginSession::Stats& last = m_logged_stats; + logger()->info( + "[remote] 5s: sent hand={} schema={} wrist_rx={} hb={} | dropped queue={} | tx_bytes={} | offset_ns={} | " + "connected={:d}", + now.sent_hand - last.sent_hand, now.sent_schema - last.sent_schema, now.wrist_rx - last.wrist_rx, + now.heartbeats - last.heartbeats, now.dropped_queue - last.dropped_queue, now.tx_bytes - last.tx_bytes, + clock_offset_ns(), m_connected.load()); + m_logged_stats = now; + } + + void on_disconnect(const char* reason) + { + if (!m_connected.exchange(false)) + { + return; + } + if (!m_stop) + { + logger()->warn("[RemotePluginSession] disconnected: {}", reason); + } + m_transport->shutdown(); + m_queue_cv.notify_all(); + } + + //! The bridge refused one channel: its pushes become no-ops, the session and other channels continue. + void refuse_channel(uint16_t channel_id, const std::string& reason) + { + logger()->warn("[RemotePluginSession] bridge refused channel {}: {}", channel_id, reason); + std::lock_guard lock(m_queue_mutex); + m_refused_channels[channel_id] = reason; + if (const auto it = m_sample_queues.find(channel_id); it != m_sample_queues.end()) + { + m_queued_samples -= it->second.frames.size(); + m_sample_queues.erase(it); + } + } + + //! Sends are done outside the queue lock: a stalled send must not block the plugin's push(). + void sender_loop() + { + auto last_send = std::chrono::steady_clock::now(); + auto next_stats = last_send + kStatsPeriod; + while (true) + { + std::vector batch = next_batch(std::min(last_send + kHeartbeatPeriod, next_stats)); + if (!m_connected.load()) + { + return; + } + if (batch.empty() && m_stop) + { + return; + } + const auto now = std::chrono::steady_clock::now(); + if (batch.empty() && now - last_send >= kHeartbeatPeriod) + { + batch.push_back(wire::encode_frame(wire::FrameType::Heartbeat, wire::kSessionChannelId, {})); + m_heartbeats.fetch_add(1); + } + for (FrameBytes& frame : batch) + { + if (!send_now(frame)) + { + on_disconnect("send failed"); + return; + } + last_send = std::chrono::steady_clock::now(); + } + if (now >= next_stats) + { + log_stats(); + next_stats += kStatsPeriod; + } + } + } + + void receiver_loop() + { + auto last_receive = std::chrono::steady_clock::now(); + while (m_connected.load() && !m_stop) + { + FrameBytes bytes; + switch (m_transport->receive(bytes, kReceivePoll)) + { + case ReceiveResult::Ok: + last_receive = std::chrono::steady_clock::now(); + handle_frame(std::move(bytes), core::os_monotonic_now_ns()); + break; + case ReceiveResult::Timeout: + if (std::chrono::steady_clock::now() - last_receive > kPeerTimeout) + { + on_disconnect("no frames from the bridge within the peer timeout"); + return; + } + break; + case ReceiveResult::Closed: + on_disconnect("connection closed by the bridge"); + return; + case ReceiveResult::Malformed: + on_disconnect("malformed frame from the bridge"); + return; + } + } + } + + void handle_frame(FrameBytes bytes, int64_t t_recv_ns) + { + try + { + const wire::Frame frame = wire::split_frame(std::move(bytes)); + { + std::lock_guard lock(m_clock_mutex); + m_clock.add_sample(frame.header.t_send_monotonic_ns, t_recv_ns); + } + switch (frame.header.type) + { + case wire::FrameType::WristSample: + { + // sample_time_ns is dropped: core::WristTrackingSample has no timestamp field and P0 serves the + // latest sample. + const wire::WristSample sample = wire::decode_wrist_sample(frame.payload); + m_wrist_rx.fetch_add(1); + std::lock_guard lock(m_wrist_mutex); + m_wrist[sample.is_left ? 0 : 1] = sample.sample; + break; + } + case wire::FrameType::Ping: + { + // Queued as control (never dropped); stamp_header() sets t3 when the sender thread sends it. + const wire::Ping ping = wire::decode_ping(frame.payload); + enqueue_control(wire::FrameType::Pong, wire::kSessionChannelId, + wire::encode(wire::Pong{ ping.t1_ns, t_recv_ns, 0 })); + if (!std::exchange(m_answered_ping, true)) + { + logger()->info("[remote] clock: answering bridge PINGs (two-way offset)"); + } + break; + } + case wire::FrameType::Error: + { + const wire::Error error = wire::decode_error(frame.payload); + if (frame.header.channel_id != wire::kSessionChannelId) + { + refuse_channel(frame.header.channel_id, error.message); + break; + } + logger()->error("[RemotePluginSession] bridge error: {}", error.message); + on_disconnect("bridge reported an error"); + break; + } + case wire::FrameType::Heartbeat: + case wire::FrameType::HelloAck: + break; + default: + logger()->warn("[RemotePluginSession] ignoring unexpected frame {}", wire::to_string(frame.header.type)); + break; + } + } + catch (const wire::DecodeError& e) + { + logger()->error("[RemotePluginSession] {}", e.what()); + on_disconnect("undecodable frame from the bridge"); + } + } + + core::PluginSessionRequirements m_requirements; + std::unique_ptr m_transport; + + std::atomic m_connected{ false }; + std::atomic m_stop{ false }; + std::atomic m_shutdown{ false }; + + std::mutex m_state_mutex; + uint16_t m_next_channel_id = 1; + std::set m_open_channels; + bool m_pull_channel_claimed = false; + + mutable std::mutex m_queue_mutex; + std::condition_variable m_queue_cv; + std::deque m_control_queue; + std::map m_sample_queues; + std::map m_refused_channels; + bool m_sender_done = false; + size_t m_queued_samples = 0; + uint64_t m_dropped_samples = 0; + uint32_t m_next_seq = 0; // handshake on the constructor thread, then the sender thread only + + std::atomic m_sent_hand{ 0 }; + std::atomic m_sent_schema{ 0 }; + std::atomic m_wrist_rx{ 0 }; + std::atomic m_heartbeats{ 0 }; + std::atomic m_tx_bytes{ 0 }; + bool m_answered_ping = false; // receiver thread only + RemotePluginSession::Stats m_logged_stats; // sender thread only + + mutable std::mutex m_wrist_mutex; + core::WristTrackingSample m_wrist[2]; + + mutable std::mutex m_clock_mutex; + ClockOffsetEstimator m_clock; + + std::thread m_sender; + std::thread m_receiver; +}; + +namespace +{ + +class RemoteHandTrackingPushChannel final : public core::IHandTrackingPushChannel +{ +public: + RemoteHandTrackingPushChannel(std::shared_ptr impl, uint16_t channel_id) + : m_impl(std::move(impl)), m_channel_id(channel_id) + { + } + + ~RemoteHandTrackingPushChannel() override + { + m_impl->close_channel(m_channel_id); + } + + void push(const XrHandJointLocationEXT* joint_locations, int64_t sample_time_local_common_clock_ns) override + { + if (joint_locations == nullptr) + { + throw std::invalid_argument("Hand tracking push requires joint locations"); + } + if (!m_impl->connected()) + { + return; + } + m_impl->push_sample( + wire::FrameType::HandSample, m_channel_id, + wire::encode_hand_sample_frame(m_channel_id, joint_locations, sample_time_local_common_clock_ns)); + } + +private: + std::shared_ptr m_impl; + uint16_t m_channel_id; +}; + +class RemoteSchemaPushChannel final : public core::ISchemaPushChannel +{ +public: + RemoteSchemaPushChannel(std::shared_ptr impl, + uint16_t channel_id, + core::SchemaPusherConfig config) + : m_impl(std::move(impl)), m_channel_id(channel_id), m_config(std::move(config)) + { + } + + ~RemoteSchemaPushChannel() override + { + m_impl->close_channel(m_channel_id); + } + + const core::SchemaPusherConfig& config() const override + { + return m_config; + } + + void push_buffer(const uint8_t* buffer, + size_t size, + int64_t sample_time_local_common_clock_ns, + int64_t sample_time_raw_device_clock_ns) override + { + if (buffer == nullptr && size != 0) + { + throw std::invalid_argument("push_buffer: buffer is null for a non-empty sample"); + } + if (!m_impl->connected()) + { + return; + } + m_impl->push_sample(wire::FrameType::SchemaSample, m_channel_id, + wire::encode_schema_sample_frame(m_channel_id, buffer, size, sample_time_local_common_clock_ns, + sample_time_raw_device_clock_ns)); + } + +private: + std::shared_ptr m_impl; + uint16_t m_channel_id; + core::SchemaPusherConfig m_config; +}; + +class RemoteWristTrackingSource final : public core::IWristTrackingSource +{ +public: + explicit RemoteWristTrackingSource(std::shared_ptr impl) : m_impl(std::move(impl)) + { + } + + // P0 returns the latest bridge sample; the requested time is not interpolated. + core::WristTrackingSample query(bool is_left, int64_t /*sample_time_local_common_clock_ns*/) override + { + return m_impl->latest_wrist(is_left); + } + +private: + std::shared_ptr m_impl; +}; + +class RemotePluginPullChannel final : public core::IPluginPullChannel +{ +public: + RemotePluginPullChannel(std::shared_ptr impl, uint16_t channel_id) + : m_impl(std::move(impl)), m_channel_id(channel_id) + { + } + + ~RemotePluginPullChannel() override + { + m_impl->close_channel(m_channel_id); + } + + // The receiver thread keeps the wrist cache current; nothing to poll here. + void update() override + { + } + + const core::ITrackerImpl& get_tracker_impl(const core::ITracker&) const override + { + throw std::runtime_error("RemotePluginSession pull channel exposes no trackers (P0)"); + } + + std::unique_ptr create_wrist_tracking_source( + const core::WristTrackingSourceConfig& config) override + { + if (!m_impl->requirements().wrist_tracking_pull) + { + throw std::logic_error( + "RemotePluginSession: wrist tracking pull was not declared in PluginSessionRequirements"); + } + m_impl->send_control(wire::FrameType::OpenWrist, m_channel_id, wire::encode(config)); + return std::make_unique(m_impl); + } + +private: + std::shared_ptr m_impl; + uint16_t m_channel_id; +}; + +} // namespace + +RemotePluginSession::RemotePluginSession(std::string host, + uint16_t port, + std::string app_name, + core::PluginSessionRequirements requirements) + : RemotePluginSession( + std::make_unique(TcpSocket::connect(Endpoint{ std::move(host), port }, kConnectTimeout)), + std::move(app_name), + requirements) +{ +} + +RemotePluginSession::RemotePluginSession(std::unique_ptr transport, + std::string app_name, + core::PluginSessionRequirements requirements) + : m_impl(std::make_shared(std::move(transport), app_name, requirements)) +{ +} + +RemotePluginSession::~RemotePluginSession() +{ + m_impl->shutdown(); +} + +std::unique_ptr RemotePluginSession::create_pull_channel() +{ + if (!m_impl->claim_pull_channel()) + { + throw std::logic_error("RemotePluginSession supports only one pull channel per session"); + } + const uint16_t channel_id = m_impl->open_channel(wire::FrameType::OpenPull, {}); + return std::make_unique(m_impl, channel_id); +} + +std::unique_ptr RemotePluginSession::create_schema_push_channel( + const core::SchemaPusherConfig& config) +{ + if (!m_impl->requirements().schema_push) + { + throw std::logic_error("RemotePluginSession: schema push was not declared in PluginSessionRequirements"); + } + const uint16_t channel_id = m_impl->open_channel(wire::FrameType::OpenSchema, wire::encode(config)); + return std::make_unique(m_impl, channel_id, config); +} + +std::unique_ptr RemotePluginSession::create_hand_tracking_push_channel(XrHandEXT hand) +{ + if (!m_impl->requirements().hand_tracking_push) + { + throw std::logic_error("RemotePluginSession: hand-tracking push was not declared in PluginSessionRequirements"); + } + const uint16_t channel_id = m_impl->open_channel(wire::FrameType::OpenHand, wire::encode(wire::OpenHand{ hand })); + return std::make_unique(m_impl, channel_id); +} + +bool RemotePluginSession::connected() const +{ + return m_impl->connected(); +} + +int64_t RemotePluginSession::clock_offset_ns() const +{ + return m_impl->clock_offset_ns(); +} + +RemotePluginSession::Stats RemotePluginSession::stats() const +{ + return m_impl->stats(); +} + +} // namespace remote_plugin_session diff --git a/tests/cpp/plugins/remote_device_bridge/CMakeLists.txt b/tests/cpp/plugins/remote_device_bridge/CMakeLists.txt index a513b47b05..a7f9589058 100644 --- a/tests/cpp/plugins/remote_device_bridge/CMakeLists.txt +++ b/tests/cpp/plugins/remote_device_bridge/CMakeLists.txt @@ -4,7 +4,9 @@ find_package(Threads REQUIRED) add_executable(remote_plugin_session_tests + test_clock_offset_estimator.cpp test_remote_address.cpp + test_remote_plugin_session.cpp test_tcp_transport.cpp test_websocket_transport.cpp test_wire_codec.cpp diff --git a/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp b/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp new file mode 100644 index 0000000000..c7cfe06fe4 --- /dev/null +++ b/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include + +using remote_plugin_session::ClockOffsetEstimator; +using remote_plugin_session::PingScheduler; +using remote_plugin_session::should_drop_stale; + +TEST_CASE("ClockOffsetEstimator starts empty and follows the sign convention", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator; + REQUIRE(estimator.offset_ns() == 0); + REQUIRE(estimator.remote_to_local(42) == 42); + + // Remote clock reads 1'000 when the local clock reads 5'000 (+ 100 of delay). + estimator.add_sample(1'000, 5'100); + REQUIRE(estimator.offset_ns() == 4'100); + REQUIRE(estimator.remote_to_local(1'000) == 5'100); +} + +TEST_CASE("ClockOffsetEstimator keeps the minimum delay sample", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator; + estimator.add_sample(1'000, 5'300); // delay 300 + estimator.add_sample(2'000, 6'050); // delay 50 + estimator.add_sample(3'000, 7'400); // delay 400 + REQUIRE(estimator.offset_ns() == 4'050); +} + +TEST_CASE("ClockOffsetEstimator evicts by sample count", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator(/*max_samples=*/2, /*window_ns=*/1'000'000); + estimator.add_sample(0, 10); // offset 10 (the eventual minimum, evicted first) + estimator.add_sample(100, 150); // offset 50 + estimator.add_sample(200, 260); // offset 60 + REQUIRE(estimator.offset_ns() == 50); +} + +TEST_CASE("ClockOffsetEstimator evicts by time window", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator(/*max_samples=*/100, /*window_ns=*/1'000); + estimator.add_sample(0, 10); // local 10, offset 10 + estimator.add_sample(500, 800); // local 800, offset 300 + REQUIRE(estimator.offset_ns() == 10); + estimator.add_sample(1'500, 1'700); // local 1'700, offset 200 -> drops samples older than 700 + REQUIRE(estimator.offset_ns() == 200); +} + +TEST_CASE("ClockOffsetEstimator handles negative offsets", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator; + estimator.add_sample(10'000, 2'000); + REQUIRE(estimator.offset_ns() == -8'000); + REQUIRE(estimator.remote_to_local(10'500) == 2'500); +} + +TEST_CASE("ClockOffsetEstimator rejects an empty window", "[remote_device_bridge][unit]") +{ + REQUIRE_THROWS_AS(ClockOffsetEstimator(0, 1), std::invalid_argument); + REQUIRE_THROWS_AS(ClockOffsetEstimator(1, 0), std::invalid_argument); +} + +TEST_CASE("should_drop_stale keeps fresh samples and drops the backlog after a stall", "[remote_device_bridge][unit]") +{ + constexpr int64_t kMaxAge = 250'000'000; // 250 ms + const int64_t now = 10'000'000'000; + REQUIRE_FALSE(should_drop_stale(now, now, kMaxAge)); + REQUIRE_FALSE(should_drop_stale(now - kMaxAge, now, kMaxAge)); // exactly max_age is still fresh + REQUIRE_FALSE(should_drop_stale(now + 5'000'000, now, kMaxAge)); // clock-offset bias may map into the future + REQUIRE(should_drop_stale(now - kMaxAge - 1, now, kMaxAge)); + REQUIRE(should_drop_stale(now - 3'000'000'000, now, kMaxAge)); + REQUIRE_FALSE(should_drop_stale(now - 3'000'000'000, now, 0)); // 0 disables the guard + REQUIRE_FALSE(should_drop_stale(now - 3'000'000'000, now, -1)); +} + +namespace +{ + +constexpr int64_t kMs = 1'000'000; + +//! One exchange with true offset `offset` (local - remote), forward delay d_fwd, return delay d_back. +void exchange(ClockOffsetEstimator& estimator, int64_t t1, int64_t offset, int64_t d_fwd, int64_t d_back) +{ + const int64_t t2 = t1 + d_fwd - offset; + const int64_t t3 = t2 + 1 * kMs; // client turnaround, excluded from the RTT + const int64_t t4 = t3 + offset + d_back; + estimator.add_two_way_sample(t1, t2, t3, t4); +} + +} // namespace + +TEST_CASE("Two-way exchanges recover the offset the one-way minimum biases", "[remote_device_bridge][unit]") +{ + constexpr int64_t kOffset = 5'000 * kMs; + ClockOffsetEstimator estimator; + // One-way: the fastest frame took 70 ms, so the one-way estimate is 70 ms too large. + estimator.add_sample(1'000 * kMs, 1'000 * kMs + kOffset + 70 * kMs); + REQUIRE(estimator.offset_ns() == kOffset + 70 * kMs); + REQUIRE(std::string(estimator.offset_source()) == "one-way"); + REQUIRE(estimator.rtt_min_ns() == 0); + + exchange(estimator, 2'000 * kMs, kOffset, 70 * kMs, 70 * kMs); + REQUIRE(std::string(estimator.offset_source()) == "two-way"); + REQUIRE(estimator.offset_ns() == kOffset); + REQUIRE(estimator.rtt_last_ns() == 140 * kMs); + REQUIRE(estimator.rtt_min_ns() == 140 * kMs); + REQUIRE(estimator.remote_to_local(0) == kOffset); + + // One-way samples no longer move a two-way estimate. + estimator.add_sample(3'000 * kMs, 3'000 * kMs + kOffset + 1 * kMs); + REQUIRE(estimator.offset_ns() == kOffset); +} + +TEST_CASE("Two-way offset error is half the path asymmetry", "[remote_device_bridge][unit]") +{ + constexpr int64_t kOffset = -123'456 * kMs; + ClockOffsetEstimator estimator; + // 10 ms out, 50 ms back: RTT 60 ms, estimate off by (50 - 10) / 2 = +20 ms. + exchange(estimator, 1'000 * kMs, kOffset, 10 * kMs, 50 * kMs); + REQUIRE(estimator.rtt_last_ns() == 60 * kMs); + REQUIRE(estimator.offset_ns() - kOffset == 20 * kMs); +} + +TEST_CASE("Two-way estimate follows the minimum-RTT exchange of the last 32", "[remote_device_bridge][unit]") +{ + constexpr int64_t kOffset = 7 * kMs; + ClockOffsetEstimator estimator; + // Exchange 0 has the lowest RTT and an asymmetry of +4 ms; the rest are slower and symmetric-ish. + exchange(estimator, 0, kOffset, 1 * kMs, 9 * kMs); // rtt 10, error +4 + for (int i = 1; i < 32; ++i) + { + exchange(estimator, i * 1'000 * kMs, kOffset, 30 * kMs, 50 * kMs); // rtt 80, error +10 + } + REQUIRE(estimator.rtt_min_ns() == 10 * kMs); + REQUIRE(estimator.offset_ns() == kOffset + 4 * kMs); + REQUIRE(estimator.rtt_last_ns() == 80 * kMs); + + exchange(estimator, 32'000 * kMs, kOffset, 30 * kMs, 50 * kMs); // evicts exchange 0 + REQUIRE(estimator.rtt_min_ns() == 80 * kMs); + REQUIRE(estimator.offset_ns() == kOffset + 10 * kMs); + + estimator.add_two_way_sample(0, 100, 200, 50); // negative RTT: ignored + REQUIRE(estimator.rtt_last_ns() == 80 * kMs); +} + +TEST_CASE("PingScheduler bursts 10 PINGs at 50 ms, then 1 Hz", "[remote_device_bridge][unit]") +{ + constexpr int64_t kStart = 1'000 * kMs; + PingScheduler pings(kStart); + std::vector sent; + for (int64_t t = kStart; t <= kStart + 3'500 * kMs; t += kMs) + { + const PingScheduler::Action action = pings.poll(t); + REQUIRE(action != PingScheduler::Action::GiveUp); + if (action == PingScheduler::Action::Ping) + { + sent.push_back(t - kStart); + pings.on_pong(); + } + } + const std::vector expected = { 0, 50 * kMs, 100 * kMs, 150 * kMs, 200 * kMs, + 250 * kMs, 300 * kMs, 350 * kMs, 400 * kMs, 450 * kMs, + 1450 * kMs, 2450 * kMs, 3450 * kMs }; + REQUIRE(sent == expected); + REQUIRE(pings.active()); +} + +TEST_CASE("PingScheduler gives up once when no PONG arrives within 2 s", "[remote_device_bridge][unit]") +{ + PingScheduler pings(0); + int given_up = 0; + int sent_after = 0; + for (int64_t t = 0; t <= 5'000 * kMs; t += kMs) + { + switch (pings.poll(t)) + { + case PingScheduler::Action::GiveUp: + REQUIRE(t == PingScheduler::kPongTimeoutNs); + ++given_up; + break; + case PingScheduler::Action::Ping: + sent_after += given_up; + break; + case PingScheduler::Action::None: + break; + } + } + REQUIRE(given_up == 1); + REQUIRE(sent_after == 0); + REQUIRE_FALSE(pings.active()); +} diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp new file mode 100644 index 0000000000..760c5f936e --- /dev/null +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -0,0 +1,768 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace remote_plugin_session; + +namespace +{ + +constexpr std::chrono::milliseconds kWait{ 2000 }; + +bool send_frame( + ITransport& transport, wire::FrameType type, uint16_t channel_id, uint32_t seq, const std::vector& payload) +{ + std::vector frame = wire::encode_frame(type, channel_id, payload); + wire::stamp_header(frame.data(), seq, core::os_monotonic_now_ns()); + return transport.send(frame.data(), frame.size()); +} + +//! Removes and returns the oldest frame of `type`. +std::optional take_frame(std::deque& frames, wire::FrameType type) +{ + const auto it = std::find_if( + frames.begin(), frames.end(), [type](const wire::Frame& frame) { return frame.header.type == type; }); + if (it == frames.end()) + { + return std::nullopt; + } + wire::Frame frame = std::move(*it); + frames.erase(it); + return frame; +} + +//! Minimal in-process bridge: answers HELLO, records every frame, can inject frames and drop the socket. +class FakeBridge +{ +public: + explicit FakeBridge(bool accept = true, std::string reason = "") + : m_listener(TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 1)), m_accept(accept), m_reason(std::move(reason)) + { + m_thread = std::thread(&FakeBridge::serve, this); + } + + ~FakeBridge() + { + m_stop = true; + { + std::lock_guard lock(m_mutex); + if (m_client) + { + m_client->shutdown(); + } + } + m_thread.join(); + } + + uint16_t port() const + { + return m_listener.local_port(); + } + + std::optional wait_for(wire::FrameType type, std::chrono::milliseconds timeout = kWait) + { + std::unique_lock lock(m_mutex); + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (true) + { + if (auto frame = take_frame(m_frames, type)) + { + return frame; + } + if (m_cv.wait_until(lock, deadline) == std::cv_status::timeout) + { + return std::nullopt; + } + } + } + + bool send(wire::FrameType type, uint16_t channel_id, const std::vector& payload) + { + std::lock_guard lock(m_mutex); + return m_client && send_frame(*m_client, type, channel_id, m_seq++, payload); + } + + void drop_connection() + { + std::lock_guard lock(m_mutex); + if (m_client) + { + m_client->shutdown(); + } + } + + bool client_closed(std::chrono::milliseconds timeout = kWait) + { + std::unique_lock lock(m_mutex); + return m_cv.wait_for(lock, timeout, [this] { return m_closed; }); + } + +private: + void serve() + { + std::unique_ptr client; + while (!m_stop && !client) + { + client = TcpTransport::accept(m_listener, std::chrono::milliseconds(50)); + } + if (!client) + { + return; + } + + std::vector bytes; + if (client->receive(bytes, kWait) != ReceiveResult::Ok) + { + return; + } + wire::Frame hello = wire::split_frame(std::move(bytes)); + if (hello.header.type != wire::FrameType::Hello) + { + return; + } + ITransport* transport = client.get(); + { + std::lock_guard lock(m_mutex); + m_client = std::move(client); + m_frames.push_back(std::move(hello)); + send_frame(*m_client, wire::FrameType::HelloAck, wire::kSessionChannelId, m_seq++, + wire::encode(wire::HelloAck{ m_accept, m_reason })); + } + m_cv.notify_all(); + + while (!m_stop) + { + const ReceiveResult result = transport->receive(bytes, std::chrono::milliseconds(50)); + if (result == ReceiveResult::Timeout) + { + continue; + } + std::lock_guard lock(m_mutex); + if (result == ReceiveResult::Ok) + { + m_frames.push_back(wire::split_frame(std::move(bytes))); + } + else + { + m_closed = true; + m_cv.notify_all(); + return; + } + m_cv.notify_all(); + } + } + + TcpSocket m_listener; + bool m_accept; + std::string m_reason; + std::atomic m_stop{ false }; + std::thread m_thread; + + std::mutex m_mutex; + std::condition_variable m_cv; + std::unique_ptr m_client; + std::deque m_frames; + uint32_t m_seq = 0; + bool m_closed = false; +}; + +//! In-memory ITransport standing in for a stalled TCP link: while held, send() blocks like a write into a full +//! socket; released frames are recorded. receive() serves the accepting HELLO_ACK and times out afterwards. +class BlockingTransport final : public ITransport +{ +public: + BlockingTransport() + { + m_inbox.push_back(wire::encode_frame( + wire::FrameType::HelloAck, wire::kSessionChannelId, wire::encode(wire::HelloAck{ true, "" }))); + wire::stamp_header(m_inbox.back().data(), 0, core::os_monotonic_now_ns()); + } + + bool send(const uint8_t* frame, size_t size) override + { + std::unique_lock lock(m_mutex); + m_blocked = m_held; + m_cv.wait(lock, [this] { return !m_held || m_closed; }); + m_blocked = false; + if (m_closed) + { + return false; + } + m_sent.push_back(wire::split_frame(std::vector(frame, frame + size))); + m_cv.notify_all(); + return true; + } + + ReceiveResult receive(std::vector& frame, std::chrono::milliseconds timeout) override + { + std::unique_lock lock(m_mutex); + if (!m_cv.wait_for(lock, timeout, [this] { return !m_inbox.empty() || m_closed; })) + { + return ReceiveResult::Timeout; + } + if (m_closed) + { + return ReceiveResult::Closed; + } + frame = std::move(m_inbox.front()); + m_inbox.pop_front(); + return ReceiveResult::Ok; + } + + void shutdown() override + { + std::lock_guard lock(m_mutex); + m_closed = true; + m_cv.notify_all(); + } + + void hold(bool held) + { + std::lock_guard lock(m_mutex); + m_held = held; + m_cv.notify_all(); + } + + //! True while a send() is waiting for hold(false). + bool blocked() const + { + std::lock_guard lock(m_mutex); + return m_blocked; + } + + std::optional wait_for(wire::FrameType type, std::chrono::milliseconds timeout = kWait) + { + std::unique_lock lock(m_mutex); + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (true) + { + if (auto frame = take_frame(m_sent, type)) + { + return frame; + } + if (m_cv.wait_until(lock, deadline) == std::cv_status::timeout) + { + return std::nullopt; + } + } + } + +private: + mutable std::mutex m_mutex; + std::condition_variable m_cv; + bool m_held = false; + bool m_blocked = false; + bool m_closed = false; + std::deque m_sent; + std::deque> m_inbox; +}; + +std::array make_joints(float seed) +{ + std::array joints{}; + for (size_t i = 0; i < joints.size(); ++i) + { + const float f = seed + 0.001f * static_cast(i); + joints[i].locationFlags = XR_SPACE_LOCATION_POSITION_VALID_BIT | static_cast(i) << 32; + joints[i].pose = { { f, -f, 2.0f * f, 1.0f - f }, { 3.0f * f, 4.0f * f, -5.0f * f } }; + joints[i].radius = 0.01f + f; + } + return joints; +} + +class DummyTracker final : public core::ITracker +{ +public: + std::string_view get_name() const override + { + return "dummy"; + } +}; + +bool wait_until(const std::function& predicate, std::chrono::milliseconds timeout = kWait) +{ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (!predicate()) + { + if (std::chrono::steady_clock::now() > deadline) + { + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return true; +} + +} // namespace + +TEST_CASE("RemotePluginSession performs the HELLO handshake", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", + { .schema_push = true, .hand_tracking_push = true, .wrist_tracking_pull = false }); + REQUIRE(session.connected()); + + const auto hello = bridge.wait_for(wire::FrameType::Hello); + REQUIRE(hello.has_value()); + REQUIRE(hello->header.channel_id == wire::kSessionChannelId); + REQUIRE(hello->header.seq == 0); + const wire::Hello decoded = wire::decode_hello(hello->payload); + REQUIRE(decoded.proto_version == wire::kHelloVersion); + REQUIRE(decoded.app_name == "test_app"); + REQUIRE(decoded.requirements.schema_push); + REQUIRE(decoded.requirements.hand_tracking_push); + REQUIRE_FALSE(decoded.requirements.wrist_tracking_pull); +} + +TEST_CASE("RemotePluginSession fails fast when the bridge refuses or is absent", "[remote_device_bridge][unit]") +{ + SECTION("HELLO_ACK with accepted=0 throws with the reason") + { + FakeBridge bridge(false, "no OpenXR runtime"); + try + { + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", {}); + FAIL("expected std::runtime_error"); + } + catch (const std::runtime_error& e) + { + REQUIRE(std::string(e.what()).find("no OpenXR runtime") != std::string::npos); + } + } + SECTION("nothing listening throws") + { + TcpSocket probe = TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 1); + const uint16_t free_port = probe.local_port(); + probe.close(); + REQUIRE_THROWS_AS(RemotePluginSession("127.0.0.1", free_port, "test_app", {}), std::runtime_error); + } + SECTION("a null transport is rejected") + { + REQUIRE_THROWS_AS(RemotePluginSession(std::unique_ptr(), "test_app", {}), std::invalid_argument); + } +} + +TEST_CASE("RemotePluginSession accepts an already connected transport", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session( + std::make_unique(TcpSocket::connect(Endpoint{ "127.0.0.1", bridge.port() }, kWait)), + "via_transport", { .hand_tracking_push = true }); + REQUIRE(session.connected()); + const auto hello = bridge.wait_for(wire::FrameType::Hello); + REQUIRE(hello.has_value()); + REQUIRE(wire::decode_hello(hello->payload).app_name == "via_transport"); +} + +TEST_CASE("RemotePluginSession enforces declared requirements", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", + { .schema_push = false, .hand_tracking_push = true, .wrist_tracking_pull = false }); + + REQUIRE_THROWS_AS(session.create_schema_push_channel(core::SchemaPusherConfig{}), std::logic_error); + auto pull = session.create_pull_channel(); + REQUIRE_THROWS_AS(pull->create_wrist_tracking_source({}), std::logic_error); + REQUIRE_THROWS_AS(session.create_pull_channel(), std::logic_error); + const DummyTracker tracker; + REQUIRE_THROWS_AS(pull->get_tracker_impl(tracker), std::runtime_error); + REQUIRE(bridge.wait_for(wire::FrameType::OpenPull).has_value()); +} + +TEST_CASE("RemotePluginSession rejects hand-tracking push when it was not declared", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", + { .schema_push = true, .hand_tracking_push = false, .wrist_tracking_pull = false }); + + REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT), std::logic_error); + REQUIRE_FALSE(bridge.wait_for(wire::FrameType::OpenHand, std::chrono::milliseconds(200)).has_value()); +} + +TEST_CASE("RemotePluginSession forwards OPEN_HAND, HAND_SAMPLE and CLOSE bit-exact", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true }); + + uint16_t channel_id = 0; + { + core::HandTrackingPusher pusher(session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT)); + const auto open = bridge.wait_for(wire::FrameType::OpenHand); + REQUIRE(open.has_value()); + REQUIRE(open->header.channel_id != wire::kSessionChannelId); + REQUIRE(wire::decode_open_hand(open->payload).hand == XR_HAND_RIGHT_EXT); + channel_id = open->header.channel_id; + + const auto joints = make_joints(0.25f); + pusher.push(joints.data(), 123456789LL); + + const auto sample = bridge.wait_for(wire::FrameType::HandSample); + REQUIRE(sample.has_value()); + REQUIRE(sample->header.channel_id == channel_id); + const wire::HandSample decoded = wire::decode_hand_sample(sample->payload); + REQUIRE(decoded.sample_time_ns == 123456789LL); + for (size_t i = 0; i < joints.size(); ++i) + { + REQUIRE(std::memcmp(&decoded.joints[i], &joints[i], sizeof(XrHandJointLocationEXT)) == 0); + } + } + + const auto close = bridge.wait_for(wire::FrameType::Close); + REQUIRE(close.has_value()); + REQUIRE(close->header.channel_id == channel_id); + REQUIRE(close->payload.empty()); +} + +TEST_CASE("RemotePluginSession forwards schema channels", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .schema_push = true }); + + const core::SchemaPusherConfig config{ .collection_id = "collection", + .max_flatbuffer_size = 64, + .tensor_identifier = "tensor", + .localized_name = "Localized", + .app_name = "test_app" }; + auto channel = session.create_schema_push_channel(config); + REQUIRE(channel->config().collection_id == "collection"); + REQUIRE(channel->config().max_flatbuffer_size == 64); + + const auto open = bridge.wait_for(wire::FrameType::OpenSchema); + REQUIRE(open.has_value()); + REQUIRE(wire::decode_open_schema(open->payload).localized_name == "Localized"); + + const std::vector bytes = { 1, 2, 3, 4, 5 }; + channel->push_buffer(bytes.data(), bytes.size(), 77, 88); + const auto sample = bridge.wait_for(wire::FrameType::SchemaSample); + REQUIRE(sample.has_value()); + REQUIRE(sample->header.channel_id == open->header.channel_id); + const wire::SchemaSample decoded = wire::decode_schema_sample(sample->payload); + REQUIRE(decoded.bytes == bytes); + REQUIRE(decoded.sample_time_local_common_clock_ns == 77); + REQUIRE(decoded.sample_time_raw_device_clock_ns == 88); +} + +TEST_CASE("RemotePluginSession exposes bridge wrist samples through the pull channel", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .wrist_tracking_pull = true }); + + auto pull = session.create_pull_channel(); + const auto open_pull = bridge.wait_for(wire::FrameType::OpenPull); + REQUIRE(open_pull.has_value()); + + core::WristTrackingSourceConfig config; + config.mode = core::WristTrackingSourceMode::HandTracking; + config.left_aim_to_wrist.position.x = 0.5f; + auto source = pull->create_wrist_tracking_source(config); + REQUIRE(source != nullptr); + const auto open_wrist = bridge.wait_for(wire::FrameType::OpenWrist); + REQUIRE(open_wrist.has_value()); + REQUIRE(open_wrist->header.channel_id == open_pull->header.channel_id); + const core::WristTrackingSourceConfig decoded = wire::decode_open_wrist(open_wrist->payload); + REQUIRE(decoded.mode == core::WristTrackingSourceMode::HandTracking); + REQUIRE(decoded.left_aim_to_wrist.position.x == 0.5f); + + pull->update(); + core::WristTrackingSample before = source->query(true, 0); + REQUIRE_FALSE(before.valid); + REQUIRE_FALSE(before.tracked); + + wire::WristSample wrist; + wrist.sample_time_ns = 1000; + wrist.is_left = true; + wrist.sample.valid = true; + wrist.sample.tracked = true; + wrist.sample.pose.position = { 0.1f, 1.2f, -0.4f }; + REQUIRE(bridge.send(wire::FrameType::WristSample, open_pull->header.channel_id, wire::encode(wrist))); + + REQUIRE(wait_until([&] { return source->query(true, 0).valid; })); + const core::WristTrackingSample left = source->query(true, 0); + REQUIRE(left.tracked); + REQUIRE(left.pose.position.y == 1.2f); + REQUIRE_FALSE(source->query(false, 0).valid); +} + +TEST_CASE("RemotePluginSession answers every PING with a PONG", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true }); + REQUIRE(bridge.wait_for(wire::FrameType::Hello).has_value()); + + for (int64_t t1 : { 111LL, 222LL, 333LL }) + { + const int64_t before = core::os_monotonic_now_ns(); + REQUIRE(bridge.send(wire::FrameType::Ping, wire::kSessionChannelId, wire::encode(wire::Ping{ t1 }))); + const auto frame = bridge.wait_for(wire::FrameType::Pong); + const int64_t after = core::os_monotonic_now_ns(); + REQUIRE(frame.has_value()); + REQUIRE(frame->header.channel_id == wire::kSessionChannelId); + const wire::Pong pong = wire::decode_pong(frame->payload); + REQUIRE(pong.t1_ns == t1); + // Same host, same clock: before <= t2 <= t3 == header t_send <= after. + REQUIRE(before <= pong.t2_ns); + REQUIRE(pong.t2_ns <= pong.t3_ns); + REQUIRE(pong.t3_ns == frame->header.t_send_monotonic_ns); + REQUIRE(pong.t3_ns <= after); + + ClockOffsetEstimator clock; + clock.add_two_way_sample(before, pong.t2_ns, pong.t3_ns, after); + REQUIRE(clock.rtt_last_ns() >= 0); + REQUIRE(clock.rtt_last_ns() <= after - before); + } +} + +TEST_CASE("RemotePluginSession keeps the session when the bridge refuses one channel", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session( + "127.0.0.1", bridge.port(), "test_app", { .schema_push = true, .hand_tracking_push = true }); + core::SchemaPusherConfig config; + config.collection_id = "tactile"; + config.max_flatbuffer_size = 40960; + auto refused = session.create_schema_push_channel(config); + auto hand = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + const auto open_schema = bridge.wait_for(wire::FrameType::OpenSchema); + const auto open_hand = bridge.wait_for(wire::FrameType::OpenHand); + REQUIRE(open_schema.has_value()); + REQUIRE(open_hand.has_value()); + + SECTION("ERROR on a channel id refuses only that channel") + { + const std::string reason = + "schema 'tactile' sample size 40960 bytes exceeds bridge cap 32768 bytes (runtime " + "limit); channel refused"; + REQUIRE(bridge.send(wire::FrameType::Error, open_schema->header.channel_id, wire::encode(wire::Error{ reason }))); + REQUIRE(wait_until([&] { return session.stats().refused_channels.count(open_schema->header.channel_id) == 1; })); + REQUIRE(session.stats().refused_channels.at(open_schema->header.channel_id) == reason); + REQUIRE(session.connected()); + + const std::vector bytes(16, 0x5A); + refused->push_buffer(bytes.data(), bytes.size(), 1, 1); + const auto joints = make_joints(0.3f); + hand->push(joints.data(), 42); + const auto sample = bridge.wait_for(wire::FrameType::HandSample); + REQUIRE(sample.has_value()); + REQUIRE(sample->header.channel_id == open_hand->header.channel_id); + REQUIRE_FALSE(bridge.wait_for(wire::FrameType::SchemaSample, std::chrono::milliseconds(200)).has_value()); + REQUIRE(session.connected()); + + refused.reset(); // CLOSE still goes out so the bridge forgets the refused id + const auto close = bridge.wait_for(wire::FrameType::Close); + REQUIRE(close.has_value()); + REQUIRE(close->header.channel_id == open_schema->header.channel_id); + } + SECTION("ERROR on channel 0 ends the session") + { + REQUIRE(bridge.send(wire::FrameType::Error, wire::kSessionChannelId, wire::encode(wire::Error{ "fatal" }))); + REQUIRE(wait_until([&] { return !session.connected(); })); + REQUIRE(session.stats().refused_channels.empty()); + } +} + +TEST_CASE("RemotePluginSession survives the bridge dropping the socket", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + std::unique_ptr channel; + std::unique_ptr pull; + { + RemotePluginSession session( + "127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true, .wrist_tracking_pull = true }); + channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + pull = session.create_pull_channel(); + REQUIRE(bridge.wait_for(wire::FrameType::OpenHand).has_value()); + + bridge.drop_connection(); + REQUIRE(wait_until([&] { return !session.connected(); })); + + const auto joints = make_joints(0.5f); + REQUIRE_NOTHROW(channel->push(joints.data(), 1)); + REQUIRE_NOTHROW(pull->update()); + REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT), std::runtime_error); + } + // Channels outliving a dead session must still be destroyable. + REQUIRE_NOTHROW(channel->push(make_joints(0.5f).data(), 2)); + REQUIRE_NOTHROW(channel.reset()); + REQUIRE_NOTHROW(pull.reset()); +} + +TEST_CASE("RemotePluginSession never reconnects after the link drops", "[remote_device_bridge][unit]") +{ + TcpSocket listener = TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 4); + std::unique_ptr bridge_side; + std::thread accept_once( + [&] + { + bridge_side = TcpTransport::accept(listener, kWait); + std::vector hello; + if (bridge_side && bridge_side->receive(hello, kWait) == ReceiveResult::Ok) + { + send_frame(*bridge_side, wire::FrameType::HelloAck, wire::kSessionChannelId, 0, + wire::encode(wire::HelloAck{ true, "" })); + } + }); + RemotePluginSession session("127.0.0.1", listener.local_port(), "test_app", { .hand_tracking_push = true }); + accept_once.join(); + REQUIRE(bridge_side); + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + + bridge_side->shutdown(); + REQUIRE(wait_until([&] { return !session.connected(); })); + const auto joints = make_joints(0.4f); + for (int i = 0; i < 20; ++i) + { + channel->push(joints.data(), i); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + // A new HELLO needs a new session (a restarted plugin); nothing connects back on its own. + REQUIRE(TcpTransport::accept(listener, std::chrono::milliseconds(500)) == nullptr); + REQUIRE_FALSE(session.connected()); + REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT), std::runtime_error); +} + +TEST_CASE("RemotePluginSession closes remaining channels and the socket on destruction", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + std::unique_ptr channel; + { + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true }); + channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + REQUIRE(bridge.wait_for(wire::FrameType::OpenHand).has_value()); + } + REQUIRE(bridge.wait_for(wire::FrameType::Close).has_value()); + REQUIRE(bridge.client_closed()); + REQUIRE_NOTHROW(channel.reset()); +} + +TEST_CASE("RemotePluginSession keeps the newest samples per channel while the link is stalled", + "[remote_device_bridge][unit]") +{ + auto transport = std::make_unique(); + BlockingTransport& link = *transport; + RemotePluginSession session(std::move(transport), "test_app", { .schema_push = true, .hand_tracking_push = true }); + core::SchemaPusherConfig filler_config; + filler_config.collection_id = "filler"; + auto filler = session.create_schema_push_channel(filler_config); + auto fast = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); // "A": 180 Hz + auto slow = session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT); // "B": 60 Hz + REQUIRE(link.wait_for(wire::FrameType::OpenSchema).has_value()); + const auto open_fast = link.wait_for(wire::FrameType::OpenHand); + const auto open_slow = link.wait_for(wire::FrameType::OpenHand); + REQUIRE(open_fast.has_value()); + REQUIRE(open_slow.has_value()); + const uint16_t fast_id = open_fast->header.channel_id; + const uint16_t slow_id = open_slow->header.channel_id; + + // Stall the link: the sender thread blocks inside send() on the filler sample, so every hand sample pushed + // from here on can only wait in its channel's queue. + link.hold(true); + const std::vector filler_bytes = { 0xAB }; + filler->push_buffer(filler_bytes.data(), filler_bytes.size(), 0, 0); + REQUIRE(wait_until([&] { return link.blocked(); })); + + const auto joints = make_joints(0.1f); + constexpr int kFastPushes = 180; + constexpr int kSlowPushes = kFastPushes / 3; + for (int i = 0; i < kFastPushes; ++i) + { + fast->push(joints.data(), 1000 + i); + if (i % 3 == 0) + { + slow->push(joints.data(), 2000 + i / 3); + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + const RemotePluginSession::Stats stalled = session.stats(); + REQUIRE(stalled.dropped_by_channel.at(fast_id) > 0); + REQUIRE(stalled.dropped_by_channel.at(slow_id) > 0); + REQUIRE(stalled.dropped_by_channel.at(fast_id) > stalled.dropped_by_channel.at(slow_id)); + REQUIRE(stalled.dropped_queue >= stalled.dropped_by_channel.at(fast_id) + stalled.dropped_by_channel.at(slow_id)); + REQUIRE(session.connected()); + + link.hold(false); + std::vector fast_times; + std::vector slow_times; + const auto deadline = std::chrono::steady_clock::now() + kWait; + while (std::chrono::steady_clock::now() < deadline && + (fast_times.empty() || fast_times.back() != 1000 + kFastPushes - 1 || slow_times.empty() || + slow_times.back() != 2000 + kSlowPushes - 1)) + { + const auto frame = link.wait_for(wire::FrameType::HandSample, std::chrono::milliseconds(200)); + if (!frame.has_value()) + { + continue; + } + (frame->header.channel_id == fast_id ? fast_times : slow_times) + .push_back(wire::decode_hand_sample(frame->payload).sample_time_ns); + } + + // Each channel delivered its newest sample, in order, from a queue that never held more than 4. + REQUIRE_FALSE(fast_times.empty()); + REQUIRE_FALSE(slow_times.empty()); + REQUIRE(fast_times.back() == 1000 + kFastPushes - 1); + REQUIRE(slow_times.back() == 2000 + kSlowPushes - 1); + REQUIRE(fast_times.size() <= 4); + REQUIRE(slow_times.size() <= 4); + REQUIRE(std::is_sorted(fast_times.begin(), fast_times.end())); + REQUIRE(std::is_sorted(slow_times.begin(), slow_times.end())); + REQUIRE(session.stats().sent_hand == fast_times.size() + slow_times.size()); +} + +TEST_CASE("RemotePluginSession drops a channel's pending samples when the channel closes", "[remote_device_bridge][unit]") +{ + auto transport = std::make_unique(); + BlockingTransport& link = *transport; + RemotePluginSession session(std::move(transport), "test_app", { .schema_push = true, .hand_tracking_push = true }); + core::SchemaPusherConfig filler_config; + filler_config.collection_id = "filler"; + auto filler = session.create_schema_push_channel(filler_config); + REQUIRE(link.wait_for(wire::FrameType::OpenSchema).has_value()); + + // Stall the sender on a filler sample. A paused TCP peer is not enough: with 8 MiB socket buffers the kernel + // absorbs the filler and the hand samples leave before the CLOSE. + link.hold(true); + const std::vector filler_bytes = { 0xCD }; + filler->push_buffer(filler_bytes.data(), filler_bytes.size(), 0, 0); + REQUIRE(wait_until([&] { return link.blocked(); })); + { + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + const auto joints = make_joints(0.2f); + for (int i = 0; i < 3; ++i) + { + channel->push(joints.data(), i); + } + } + link.hold(false); + + // OPEN_HAND and CLOSE go out in order; the samples queued behind the stalled filler never follow the CLOSE. + const auto open = link.wait_for(wire::FrameType::OpenHand); + REQUIRE(open.has_value()); + const auto close = link.wait_for(wire::FrameType::Close); + REQUIRE(close.has_value()); + REQUIRE(close->header.channel_id == open->header.channel_id); + REQUIRE_FALSE(link.wait_for(wire::FrameType::HandSample, std::chrono::milliseconds(200)).has_value()); +} From 3e81f2732cabe0d1d2ba5cab2c0d0feef78e6de2 Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Fri, 30 Oct 2026 10:33:16 -0700 Subject: [PATCH 02/10] rebase onto pr00 review Adapt RemotePluginSession to pr00's session API and pr01's split codec: - The session opens its pull channel (OPEN_PULL) in the constructor and hands it out through pull_channel(); the one-shot create_pull_channel() claim is gone. The pull channel owns the wrist source when HELLO declared wrist_tracking_pull and returns it from wrist_tracking_source(), throwing std::logic_error otherwise. OPEN_WRIST is no longer sent: the wrist config travels in HELLO. - RemoteWristTrackingSource::query takes XrHandEXT; WRIST_SAMPLE carries it. - RemoteSchemaPushChannel drops config() with ISchemaPushChannel's. - protocol builders/parsers: encode_hello, encode_open_schema, encode_open_hand(XrHandEXT), decode_ping -> int64_t, decode_error -> std::string; unexpected frame types are logged by number. stamp_header takes the frame vector and send_now() stamps a PONG's t3 with set_pong_t3 at the same instant as the header's t_send. - Tests use the reference-returning pull channel and plain hand channels (HandTrackingPusher is gone); AGENTS.md drops the HELLO v0 clock fallback since HELLO < 2 is rejected. - make_remote_plugin_session_factory(app_name, RemoteAddress) builds the core::PluginSessionFactory a --remote composition root needs; it throws std::logic_error when handed trackers, which the remote pull channel cannot expose. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- src/plugins/remote_device_bridge/AGENTS.md | 4 +- .../session/CMakeLists.txt | 5 + .../remote_plugin_session.hpp | 13 +- .../session/remote_plugin_session.cpp | 180 +++++++------ .../test_remote_plugin_session.cpp | 237 +++++++++--------- 5 files changed, 234 insertions(+), 205 deletions(-) diff --git a/src/plugins/remote_device_bridge/AGENTS.md b/src/plugins/remote_device_bridge/AGENTS.md index 104a9998aa..27d69760b7 100644 --- a/src/plugins/remote_device_bridge/AGENTS.md +++ b/src/plugins/remote_device_bridge/AGENTS.md @@ -11,8 +11,8 @@ SPDX-License-Identifier: Apache-2.0 - Wire format v1 (`kProtocolVersion` = 1, the only version number): 24-byte header `RPS0` magic, type u8, version u8, channel u16, seq u32, payload size u32, `t_send` i64; frame types 1..13 with no gaps; the hand byte is 0 = left, 1 = right in every payload; PING has no payload and PONG carries the PING's header `t_send` and the client's receive time. Any change to a header, payload or type number bumps `kProtocolVersion`; `decode_header` rejects other versions before reading the type, so old senders and bridges fail with a readable reason instead of mis-decoding. - A frame's `t_send` is its send time for clock exchange: call `stamp_header` where the frame is written to the transport, never when it is queued. - WebSocket is a second transport, not a second protocol: `WebSocketTransport` carries exactly one wire frame per binary message (no TLS; terminate `wss://` in a proxy). Clients pick it with a `ws://host:port[/path]` address via `parse_remote`/`connect_transport`. A server serves it on its own port; never sniff the protocol on the TCP port. The server-side HTTP upgrade runs in the first `receive()` so an accept loop never blocks on a client. -- Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to `Teleop::remote_plugin_session`: a plugin that talks to a bridge through this library must run without an OpenXR runtime. +- `Teleop::remote_plugin_session` links `pusherio::pusherio` (OpenXR value types via headers) plus `log_bridge` and `Threads` privately. Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to it: a plugin that talks to a bridge through this library must run without an OpenXR runtime. - ERROR on channel 0 ends the session; ERROR on a channel id refuses only that OPEN (schema above `--max-schema-bytes`, a hand someone else owns) and the session and its other channels continue. Refuse per channel whenever the session itself is still sound. - `RemotePluginSession` never throws on the sample path (`push`, `push_buffer`, `query`, `update`). After a transport failure those become no-ops; only channel creation and the constructor throw. -- Timestamps cross the wire in the sender's `CLOCK_MONOTONIC`; the receiver maps them with `ClockOffsetEstimator`. The bridge's offset comes from PING/PONG (min-RTT of the last 32 exchanges, `--clock-source=two-way`); the one-way minimum of `local_recv - remote_send` hides the minimum network delay and is only the fallback (HELLO v0 client, no PONG within 2 s) or the `--clock-source=one-way` A/B baseline. Never PING a HELLO v0 client: it drops the connection on unknown frame types. Do not push raw remote timestamps into OpenXR. +- Timestamps cross the wire in the sender's `CLOCK_MONOTONIC`; the receiver maps them with `ClockOffsetEstimator`. The bridge's offset comes from PING/PONG (min-RTT of the last 32 exchanges, `--clock-source=two-way`); the one-way minimum of `local_recv - remote_send` hides the minimum network delay and is only the fallback (no PONG within 2 s) or the `--clock-source=one-way` A/B baseline. Do not push raw remote timestamps into OpenXR. - Tests that need a stalled link use `BlockingTransport` in `test_remote_plugin_session.cpp`, never a TCP peer that stops reading: loopback socket buffers differ per host (416 KiB in one sandbox, 2×8 MiB on the teleop workstation), so "the sender is blocked by now" is not portable. diff --git a/src/plugins/remote_device_bridge/session/CMakeLists.txt b/src/plugins/remote_device_bridge/session/CMakeLists.txt index c37b1e00e3..3cbf20ac0c 100644 --- a/src/plugins/remote_device_bridge/session/CMakeLists.txt +++ b/src/plugins/remote_device_bridge/session/CMakeLists.txt @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +find_package(Threads REQUIRED) + add_library(remote_plugin_session STATIC clock_offset_estimator.cpp inc/remote_plugin_session/clock_offset_estimator.hpp @@ -30,6 +32,9 @@ target_include_directories(remote_plugin_session target_link_libraries(remote_plugin_session PUBLIC pusherio::pusherio + PRIVATE + log_bridge::log_bridge_core + Threads::Threads ) add_library(Teleop::remote_plugin_session ALIAS remote_plugin_session) diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp index 4aa48c95b9..45640c18b8 100644 --- a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp @@ -3,6 +3,7 @@ #pragma once +#include "remote_address.hpp" #include "transport.hpp" #include @@ -23,8 +24,9 @@ class RemotePluginSessionImpl; /*! * @brief Plugin session that forwards every channel over TCP to a remote_device_bridge. * - * Same contract as OpenXRPluginSession: the owner destroys all channels before - * the session and may call create_pull_channel() once. After the transport + * Same contract as the OpenXR plugin session: the session owns its pull channel + * (opened with the session, with the wrist source HELLO declared) and the owner + * destroys all push channels before the session. After the transport * drops, pushes become silent no-ops and channels stay safe to destroy. * Each push channel has a latest-wins queue (4 samples / 8 MiB); on a stalled * link the oldest samples of that channel are dropped, other channels are @@ -61,7 +63,7 @@ class RemotePluginSession final : public core::IPluginSession RemotePluginSession(RemotePluginSession&&) = delete; RemotePluginSession& operator=(RemotePluginSession&&) = delete; - std::unique_ptr create_pull_channel() override; + core::IPluginPullChannel& pull_channel() override; std::unique_ptr create_schema_push_channel(const core::SchemaPusherConfig& config) override; std::unique_ptr create_hand_tracking_push_channel(XrHandEXT hand) override; @@ -74,6 +76,11 @@ class RemotePluginSession final : public core::IPluginSession private: std::shared_ptr m_impl; + std::unique_ptr m_pull_channel; }; +//! Connects each session it builds to `remote`. The remote session exposes no trackers, so the factory throws +//! std::logic_error when handed any. +core::PluginSessionFactory make_remote_plugin_session_factory(std::string app_name, RemoteAddress remote); + } // namespace remote_plugin_session diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp index 9f082053c9..150b4e5906 100644 --- a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -4,8 +4,11 @@ #include "inc/remote_plugin_session/remote_plugin_session.hpp" #include "inc/remote_plugin_session/clock_offset_estimator.hpp" +#include "inc/remote_plugin_session/protocol_frame.hpp" +#include "inc/remote_plugin_session/protocol_payloads.hpp" +#include "inc/remote_plugin_session/remote_address.hpp" #include "inc/remote_plugin_session/tcp_socket.hpp" -#include "inc/remote_plugin_session/wire_codec.hpp" +#include "inc/remote_plugin_session/tcp_transport.hpp" #include #include @@ -17,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +29,8 @@ namespace remote_plugin_session { +namespace protocol = remote_plugin_protocol; + namespace { @@ -33,7 +39,6 @@ std::shared_ptr logger() return isaaccapture::Logger::get("isaaccapture.plugins.remote_device_bridge.RemotePluginSession"); } -constexpr std::chrono::milliseconds kConnectTimeout{ 3000 }; // The bridge answers HELLO only after its OpenXR session is up, which waits for a headset; on CloudXR that // routinely takes longer than a few seconds. constexpr std::chrono::milliseconds kHandshakeTimeout{ 30000 }; @@ -52,7 +57,7 @@ using FrameBytes = std::vector; // one encoded wire message, header inc struct ChannelQueue { - wire::FrameType type = wire::FrameType::HandSample; + protocol::FrameType type = protocol::FrameType::HandSample; std::deque frames; size_t bytes = 0; uint64_t dropped = 0; @@ -73,10 +78,11 @@ class detail::RemotePluginSessionImpl throw std::invalid_argument("RemotePluginSession: transport is null"); } - wire::Hello hello; + protocol::Hello hello; hello.app_name = app_name; hello.requirements = requirements; - FrameBytes hello_frame = wire::encode_frame(wire::FrameType::Hello, wire::kSessionChannelId, wire::encode(hello)); + FrameBytes hello_frame = protocol::encode_frame( + protocol::FrameType::Hello, protocol::kSessionChannelId, protocol::encode_hello(hello)); if (!send_now(hello_frame)) { throw std::runtime_error("RemotePluginSession: failed to send HELLO"); @@ -90,13 +96,13 @@ class detail::RemotePluginSessionImpl "RemotePluginSession: timed out waiting for HELLO_ACK" : "RemotePluginSession: connection lost while waiting for HELLO_ACK"); } - const wire::Frame frame = wire::split_frame(std::move(bytes)); - if (frame.header.type != wire::FrameType::HelloAck) + const protocol::Frame frame = protocol::split_frame(std::move(bytes)); + if (frame.header.type != protocol::FrameType::HelloAck) { - throw std::runtime_error(std::string("RemotePluginSession: expected HELLO_ACK, received ") + - wire::to_string(frame.header.type)); + throw std::runtime_error("RemotePluginSession: expected HELLO_ACK, received frame type " + + std::to_string(static_cast(frame.header.type))); } - const wire::HelloAck ack = wire::decode_hello_ack(frame.payload); + const protocol::HelloAck ack = protocol::decode_hello_ack(frame.payload); if (!ack.accepted) { throw std::runtime_error("RemotePluginSession: bridge rejected the session: " + ack.reason); @@ -173,7 +179,7 @@ class detail::RemotePluginSessionImpl } //! Allocates a channel id and queues its OPEN_* frame; ids are never reused within a session. - uint16_t open_channel(wire::FrameType type, std::vector payload) + uint16_t open_channel(protocol::FrameType type, std::vector payload) { require_connected(); uint16_t channel_id = 0; @@ -190,7 +196,7 @@ class detail::RemotePluginSessionImpl return channel_id; } - void send_control(wire::FrameType type, uint16_t channel_id, std::vector payload) + void send_control(protocol::FrameType type, uint16_t channel_id, std::vector payload) { require_connected(); enqueue_control(type, channel_id, std::move(payload)); @@ -216,12 +222,12 @@ class detail::RemotePluginSessionImpl } if (m_connected.load()) { - enqueue_control(wire::FrameType::Close, channel_id, {}); + enqueue_control(protocol::FrameType::Close, channel_id, {}); } } //! Per-channel latest-wins queue; control frames are never dropped. - void push_sample(wire::FrameType type, uint16_t channel_id, FrameBytes frame) + void push_sample(protocol::FrameType type, uint16_t channel_id, FrameBytes frame) { if (!m_connected.load()) { @@ -251,16 +257,10 @@ class detail::RemotePluginSessionImpl m_queue_cv.notify_one(); } - bool claim_pull_channel() - { - std::lock_guard lock(m_state_mutex); - return !std::exchange(m_pull_channel_claimed, true); - } - - core::WristTrackingSample latest_wrist(bool is_left) const + core::WristTrackingSample latest_wrist(XrHandEXT hand) const { std::lock_guard lock(m_wrist_mutex); - return m_wrist[is_left ? 0 : 1]; + return m_wrist[hand == XR_HAND_LEFT_EXT ? 0 : 1]; } void shutdown() @@ -279,7 +279,7 @@ class detail::RemotePluginSessionImpl { for (auto it = remaining.rbegin(); it != remaining.rend(); ++it) { - enqueue_control(wire::FrameType::Close, *it, {}); + enqueue_control(protocol::FrameType::Close, *it, {}); } } @@ -306,9 +306,9 @@ class detail::RemotePluginSessionImpl } private: - void enqueue_control(wire::FrameType type, uint16_t channel_id, const std::vector& payload) + void enqueue_control(protocol::FrameType type, uint16_t channel_id, const std::vector& payload) { - FrameBytes frame = wire::encode_frame(type, channel_id, payload); + FrameBytes frame = protocol::encode_frame(type, channel_id, payload); { std::lock_guard lock(m_queue_mutex); m_control_queue.push_back(std::move(frame)); @@ -316,10 +316,16 @@ class detail::RemotePluginSessionImpl m_queue_cv.notify_one(); } - //! Stamps seq/t_send and sends one frame. Constructor thread during the handshake, sender thread after. + //! Stamps seq/t_send (and a PONG's pong_sent_ns) and sends one frame. Constructor thread during the handshake, + //! sender thread after. bool send_now(FrameBytes& frame) { - wire::stamp_header(frame.data(), m_next_seq++, core::os_monotonic_now_ns()); + const int64_t t_send_ns = core::os_monotonic_now_ns(); + protocol::stamp_header(frame, m_next_seq++, t_send_ns); + if (protocol::decode_header(frame).type == protocol::FrameType::Pong) + { + protocol::set_pong_sent_time(frame, t_send_ns); + } if (!m_transport->send(frame.data(), frame.size())) { return false; @@ -356,7 +362,7 @@ class detail::RemotePluginSessionImpl batch.push_back(std::move(queue.frames.front())); queue.frames.pop_front(); --m_queued_samples; - (queue.type == wire::FrameType::HandSample ? m_sent_hand : m_sent_schema).fetch_add(1); + (queue.type == protocol::FrameType::HandSample ? m_sent_hand : m_sent_schema).fetch_add(1); } return batch; } @@ -420,7 +426,7 @@ class detail::RemotePluginSessionImpl const auto now = std::chrono::steady_clock::now(); if (batch.empty() && now - last_send >= kHeartbeatPeriod) { - batch.push_back(wire::encode_frame(wire::FrameType::Heartbeat, wire::kSessionChannelId, {})); + batch.push_back(protocol::encode_frame(protocol::FrameType::Heartbeat, protocol::kSessionChannelId, {})); m_heartbeats.fetch_add(1); } for (FrameBytes& frame : batch) @@ -473,56 +479,57 @@ class detail::RemotePluginSessionImpl { try { - const wire::Frame frame = wire::split_frame(std::move(bytes)); + const protocol::Frame frame = protocol::split_frame(std::move(bytes)); { std::lock_guard lock(m_clock_mutex); m_clock.add_sample(frame.header.t_send_monotonic_ns, t_recv_ns); } switch (frame.header.type) { - case wire::FrameType::WristSample: + case protocol::FrameType::WristSample: { // sample_time_ns is dropped: core::WristTrackingSample has no timestamp field and P0 serves the // latest sample. - const wire::WristSample sample = wire::decode_wrist_sample(frame.payload); + const protocol::WristSample sample = protocol::decode_wrist_sample(frame.payload); m_wrist_rx.fetch_add(1); std::lock_guard lock(m_wrist_mutex); - m_wrist[sample.is_left ? 0 : 1] = sample.sample; + m_wrist[sample.hand == XR_HAND_LEFT_EXT ? 0 : 1] = sample.sample; break; } - case wire::FrameType::Ping: + case protocol::FrameType::Ping: { - // Queued as control (never dropped); stamp_header() sets t3 when the sender thread sends it. - const wire::Ping ping = wire::decode_ping(frame.payload); - enqueue_control(wire::FrameType::Pong, wire::kSessionChannelId, - wire::encode(wire::Pong{ ping.t1_ns, t_recv_ns, 0 })); + // Queued as control (never dropped); send_now() sets pong_sent_ns when the sender thread sends it. + const int64_t t1_ns = protocol::decode_ping(frame.payload); + enqueue_control(protocol::FrameType::Pong, protocol::kSessionChannelId, + protocol::encode_pong(protocol::Pong{ t1_ns, t_recv_ns, 0 })); if (!std::exchange(m_answered_ping, true)) { logger()->info("[remote] clock: answering bridge PINGs (two-way offset)"); } break; } - case wire::FrameType::Error: + case protocol::FrameType::Error: { - const wire::Error error = wire::decode_error(frame.payload); - if (frame.header.channel_id != wire::kSessionChannelId) + const std::string message = protocol::decode_error(frame.payload); + if (frame.header.channel_id != protocol::kSessionChannelId) { - refuse_channel(frame.header.channel_id, error.message); + refuse_channel(frame.header.channel_id, message); break; } - logger()->error("[RemotePluginSession] bridge error: {}", error.message); + logger()->error("[RemotePluginSession] bridge error: {}", message); on_disconnect("bridge reported an error"); break; } - case wire::FrameType::Heartbeat: - case wire::FrameType::HelloAck: + case protocol::FrameType::Heartbeat: + case protocol::FrameType::HelloAck: break; default: - logger()->warn("[RemotePluginSession] ignoring unexpected frame {}", wire::to_string(frame.header.type)); + logger()->warn( + "[RemotePluginSession] ignoring unexpected frame type {}", static_cast(frame.header.type)); break; } } - catch (const wire::DecodeError& e) + catch (const protocol::DecodeError& e) { logger()->error("[RemotePluginSession] {}", e.what()); on_disconnect("undecodable frame from the bridge"); @@ -539,7 +546,6 @@ class detail::RemotePluginSessionImpl std::mutex m_state_mutex; uint16_t m_next_channel_id = 1; std::set m_open_channels; - bool m_pull_channel_claimed = false; mutable std::mutex m_queue_mutex; std::condition_variable m_queue_cv; @@ -596,8 +602,8 @@ class RemoteHandTrackingPushChannel final : public core::IHandTrackingPushChanne return; } m_impl->push_sample( - wire::FrameType::HandSample, m_channel_id, - wire::encode_hand_sample_frame(m_channel_id, joint_locations, sample_time_local_common_clock_ns)); + protocol::FrameType::HandSample, m_channel_id, + protocol::encode_hand_sample_frame(m_channel_id, joint_locations, sample_time_local_common_clock_ns)); } private: @@ -608,10 +614,8 @@ class RemoteHandTrackingPushChannel final : public core::IHandTrackingPushChanne class RemoteSchemaPushChannel final : public core::ISchemaPushChannel { public: - RemoteSchemaPushChannel(std::shared_ptr impl, - uint16_t channel_id, - core::SchemaPusherConfig config) - : m_impl(std::move(impl)), m_channel_id(channel_id), m_config(std::move(config)) + RemoteSchemaPushChannel(std::shared_ptr impl, uint16_t channel_id) + : m_impl(std::move(impl)), m_channel_id(channel_id) { } @@ -620,11 +624,6 @@ class RemoteSchemaPushChannel final : public core::ISchemaPushChannel m_impl->close_channel(m_channel_id); } - const core::SchemaPusherConfig& config() const override - { - return m_config; - } - void push_buffer(const uint8_t* buffer, size_t size, int64_t sample_time_local_common_clock_ns, @@ -638,15 +637,15 @@ class RemoteSchemaPushChannel final : public core::ISchemaPushChannel { return; } - m_impl->push_sample(wire::FrameType::SchemaSample, m_channel_id, - wire::encode_schema_sample_frame(m_channel_id, buffer, size, sample_time_local_common_clock_ns, - sample_time_raw_device_clock_ns)); + m_impl->push_sample( + protocol::FrameType::SchemaSample, m_channel_id, + protocol::encode_schema_sample_frame( + m_channel_id, buffer, size, sample_time_local_common_clock_ns, sample_time_raw_device_clock_ns)); } private: std::shared_ptr m_impl; uint16_t m_channel_id; - core::SchemaPusherConfig m_config; }; class RemoteWristTrackingSource final : public core::IWristTrackingSource @@ -657,9 +656,9 @@ class RemoteWristTrackingSource final : public core::IWristTrackingSource } // P0 returns the latest bridge sample; the requested time is not interpolated. - core::WristTrackingSample query(bool is_left, int64_t /*sample_time_local_common_clock_ns*/) override + core::WristTrackingSample query(XrHandEXT hand, int64_t /*sample_time_local_common_clock_ns*/) override { - return m_impl->latest_wrist(is_left); + return m_impl->latest_wrist(hand); } private: @@ -669,9 +668,14 @@ class RemoteWristTrackingSource final : public core::IWristTrackingSource class RemotePluginPullChannel final : public core::IPluginPullChannel { public: - RemotePluginPullChannel(std::shared_ptr impl, uint16_t channel_id) - : m_impl(std::move(impl)), m_channel_id(channel_id) + explicit RemotePluginPullChannel(std::shared_ptr impl) + : m_impl(std::move(impl)), m_channel_id(m_impl->open_channel(protocol::FrameType::OpenPull, {})) { + // The bridge streams WRIST_SAMPLEs on this channel when HELLO declared a wrist source. + if (m_impl->requirements().wrist_tracking_pull) + { + m_wrist_source.emplace(m_impl); + } } ~RemotePluginPullChannel() override @@ -689,21 +693,20 @@ class RemotePluginPullChannel final : public core::IPluginPullChannel throw std::runtime_error("RemotePluginSession pull channel exposes no trackers (P0)"); } - std::unique_ptr create_wrist_tracking_source( - const core::WristTrackingSourceConfig& config) override + core::IWristTrackingSource& wrist_tracking_source() override { - if (!m_impl->requirements().wrist_tracking_pull) + if (!m_wrist_source) { throw std::logic_error( "RemotePluginSession: wrist tracking pull was not declared in PluginSessionRequirements"); } - m_impl->send_control(wire::FrameType::OpenWrist, m_channel_id, wire::encode(config)); - return std::make_unique(m_impl); + return *m_wrist_source; } private: std::shared_ptr m_impl; uint16_t m_channel_id; + std::optional m_wrist_source; }; } // namespace @@ -722,7 +725,8 @@ RemotePluginSession::RemotePluginSession(std::string host, RemotePluginSession::RemotePluginSession(std::unique_ptr transport, std::string app_name, core::PluginSessionRequirements requirements) - : m_impl(std::make_shared(std::move(transport), app_name, requirements)) + : m_impl(std::make_shared(std::move(transport), app_name, requirements)), + m_pull_channel(std::make_unique(m_impl)) { } @@ -731,14 +735,9 @@ RemotePluginSession::~RemotePluginSession() m_impl->shutdown(); } -std::unique_ptr RemotePluginSession::create_pull_channel() +core::IPluginPullChannel& RemotePluginSession::pull_channel() { - if (!m_impl->claim_pull_channel()) - { - throw std::logic_error("RemotePluginSession supports only one pull channel per session"); - } - const uint16_t channel_id = m_impl->open_channel(wire::FrameType::OpenPull, {}); - return std::make_unique(m_impl, channel_id); + return *m_pull_channel; } std::unique_ptr RemotePluginSession::create_schema_push_channel( @@ -748,8 +747,9 @@ std::unique_ptr RemotePluginSession::create_schema_pus { throw std::logic_error("RemotePluginSession: schema push was not declared in PluginSessionRequirements"); } - const uint16_t channel_id = m_impl->open_channel(wire::FrameType::OpenSchema, wire::encode(config)); - return std::make_unique(m_impl, channel_id, config); + const uint16_t channel_id = + m_impl->open_channel(protocol::FrameType::OpenSchema, protocol::encode_open_schema(config)); + return std::make_unique(m_impl, channel_id); } std::unique_ptr RemotePluginSession::create_hand_tracking_push_channel(XrHandEXT hand) @@ -758,7 +758,7 @@ std::unique_ptr RemotePluginSession::create_hand { throw std::logic_error("RemotePluginSession: hand-tracking push was not declared in PluginSessionRequirements"); } - const uint16_t channel_id = m_impl->open_channel(wire::FrameType::OpenHand, wire::encode(wire::OpenHand{ hand })); + const uint16_t channel_id = m_impl->open_channel(protocol::FrameType::OpenHand, protocol::encode_open_hand(hand)); return std::make_unique(m_impl, channel_id); } @@ -777,4 +777,18 @@ RemotePluginSession::Stats RemotePluginSession::stats() const return m_impl->stats(); } +core::PluginSessionFactory make_remote_plugin_session_factory(std::string app_name, RemoteAddress remote) +{ + return [app_name = std::move(app_name), remote = std::move(remote)]( + const core::PluginSessionRequirements& requirements, + std::vector> trackers) -> std::unique_ptr + { + if (!trackers.empty()) + { + throw std::logic_error("RemotePluginSession exposes no trackers; " + app_name + " passed some"); + } + return std::make_unique(connect_transport(remote), app_name, requirements); + }; +} + } // namespace remote_plugin_session diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp index 760c5f936e..6392c2dfb2 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -3,11 +3,12 @@ #include #include -#include #include +#include +#include #include #include -#include +#include #include #include @@ -27,30 +28,39 @@ #include using namespace remote_plugin_session; +namespace protocol = remote_plugin_protocol; namespace { constexpr std::chrono::milliseconds kWait{ 2000 }; +const core::WristTrackingSourceConfig kWristConfig{ + .mode = core::WristTrackingSourceMode::Auto, + .left_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }, + .right_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } } +}; -bool send_frame( - ITransport& transport, wire::FrameType type, uint16_t channel_id, uint32_t seq, const std::vector& payload) +bool send_frame(ITransport& transport, + protocol::FrameType type, + uint16_t channel_id, + uint32_t seq, + const std::vector& payload) { - std::vector frame = wire::encode_frame(type, channel_id, payload); - wire::stamp_header(frame.data(), seq, core::os_monotonic_now_ns()); + std::vector frame = protocol::encode_frame(type, channel_id, payload); + protocol::stamp_header(frame, seq, core::os_monotonic_now_ns()); return transport.send(frame.data(), frame.size()); } //! Removes and returns the oldest frame of `type`. -std::optional take_frame(std::deque& frames, wire::FrameType type) +std::optional take_frame(std::deque& frames, protocol::FrameType type) { const auto it = std::find_if( - frames.begin(), frames.end(), [type](const wire::Frame& frame) { return frame.header.type == type; }); + frames.begin(), frames.end(), [type](const protocol::Frame& frame) { return frame.header.type == type; }); if (it == frames.end()) { return std::nullopt; } - wire::Frame frame = std::move(*it); + protocol::Frame frame = std::move(*it); frames.erase(it); return frame; } @@ -83,7 +93,7 @@ class FakeBridge return m_listener.local_port(); } - std::optional wait_for(wire::FrameType type, std::chrono::milliseconds timeout = kWait) + std::optional wait_for(protocol::FrameType type, std::chrono::milliseconds timeout = kWait) { std::unique_lock lock(m_mutex); const auto deadline = std::chrono::steady_clock::now() + timeout; @@ -100,7 +110,7 @@ class FakeBridge } } - bool send(wire::FrameType type, uint16_t channel_id, const std::vector& payload) + bool send(protocol::FrameType type, uint16_t channel_id, const std::vector& payload) { std::lock_guard lock(m_mutex); return m_client && send_frame(*m_client, type, channel_id, m_seq++, payload); @@ -139,8 +149,8 @@ class FakeBridge { return; } - wire::Frame hello = wire::split_frame(std::move(bytes)); - if (hello.header.type != wire::FrameType::Hello) + protocol::Frame hello = protocol::split_frame(std::move(bytes)); + if (hello.header.type != protocol::FrameType::Hello) { return; } @@ -149,8 +159,8 @@ class FakeBridge std::lock_guard lock(m_mutex); m_client = std::move(client); m_frames.push_back(std::move(hello)); - send_frame(*m_client, wire::FrameType::HelloAck, wire::kSessionChannelId, m_seq++, - wire::encode(wire::HelloAck{ m_accept, m_reason })); + send_frame(*m_client, protocol::FrameType::HelloAck, protocol::kSessionChannelId, m_seq++, + protocol::encode_hello_ack(protocol::HelloAck{ m_accept, m_reason })); } m_cv.notify_all(); @@ -164,7 +174,7 @@ class FakeBridge std::lock_guard lock(m_mutex); if (result == ReceiveResult::Ok) { - m_frames.push_back(wire::split_frame(std::move(bytes))); + m_frames.push_back(protocol::split_frame(std::move(bytes))); } else { @@ -185,7 +195,7 @@ class FakeBridge std::mutex m_mutex; std::condition_variable m_cv; std::unique_ptr m_client; - std::deque m_frames; + std::deque m_frames; uint32_t m_seq = 0; bool m_closed = false; }; @@ -197,9 +207,9 @@ class BlockingTransport final : public ITransport public: BlockingTransport() { - m_inbox.push_back(wire::encode_frame( - wire::FrameType::HelloAck, wire::kSessionChannelId, wire::encode(wire::HelloAck{ true, "" }))); - wire::stamp_header(m_inbox.back().data(), 0, core::os_monotonic_now_ns()); + m_inbox.push_back(protocol::encode_frame(protocol::FrameType::HelloAck, protocol::kSessionChannelId, + protocol::encode_hello_ack(protocol::HelloAck{ true, "" }))); + protocol::stamp_header(m_inbox.back(), 0, core::os_monotonic_now_ns()); } bool send(const uint8_t* frame, size_t size) override @@ -212,7 +222,7 @@ class BlockingTransport final : public ITransport { return false; } - m_sent.push_back(wire::split_frame(std::vector(frame, frame + size))); + m_sent.push_back(protocol::split_frame(std::vector(frame, frame + size))); m_cv.notify_all(); return true; } @@ -254,7 +264,7 @@ class BlockingTransport final : public ITransport return m_blocked; } - std::optional wait_for(wire::FrameType type, std::chrono::milliseconds timeout = kWait) + std::optional wait_for(protocol::FrameType type, std::chrono::milliseconds timeout = kWait) { std::unique_lock lock(m_mutex); const auto deadline = std::chrono::steady_clock::now() + timeout; @@ -277,7 +287,7 @@ class BlockingTransport final : public ITransport bool m_held = false; bool m_blocked = false; bool m_closed = false; - std::deque m_sent; + std::deque m_sent; std::deque> m_inbox; }; @@ -322,16 +332,15 @@ bool wait_until(const std::function& predicate, std::chrono::millisecond TEST_CASE("RemotePluginSession performs the HELLO handshake", "[remote_device_bridge][unit]") { FakeBridge bridge; - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", - { .schema_push = true, .hand_tracking_push = true, .wrist_tracking_pull = false }); + RemotePluginSession session( + "127.0.0.1", bridge.port(), "test_app", { .schema_push = true, .hand_tracking_push = true }); REQUIRE(session.connected()); - const auto hello = bridge.wait_for(wire::FrameType::Hello); + const auto hello = bridge.wait_for(protocol::FrameType::Hello); REQUIRE(hello.has_value()); - REQUIRE(hello->header.channel_id == wire::kSessionChannelId); + REQUIRE(hello->header.channel_id == protocol::kSessionChannelId); REQUIRE(hello->header.seq == 0); - const wire::Hello decoded = wire::decode_hello(hello->payload); - REQUIRE(decoded.proto_version == wire::kHelloVersion); + const protocol::Hello decoded = protocol::decode_hello(hello->payload); REQUIRE(decoded.app_name == "test_app"); REQUIRE(decoded.requirements.schema_push); REQUIRE(decoded.requirements.hand_tracking_push); @@ -373,34 +382,33 @@ TEST_CASE("RemotePluginSession accepts an already connected transport", "[remote std::make_unique(TcpSocket::connect(Endpoint{ "127.0.0.1", bridge.port() }, kWait)), "via_transport", { .hand_tracking_push = true }); REQUIRE(session.connected()); - const auto hello = bridge.wait_for(wire::FrameType::Hello); + const auto hello = bridge.wait_for(protocol::FrameType::Hello); REQUIRE(hello.has_value()); - REQUIRE(wire::decode_hello(hello->payload).app_name == "via_transport"); + REQUIRE(protocol::decode_hello(hello->payload).app_name == "via_transport"); } TEST_CASE("RemotePluginSession enforces declared requirements", "[remote_device_bridge][unit]") { FakeBridge bridge; - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", - { .schema_push = false, .hand_tracking_push = true, .wrist_tracking_pull = false }); + RemotePluginSession session( + "127.0.0.1", bridge.port(), "test_app", { .schema_push = false, .hand_tracking_push = true }); REQUIRE_THROWS_AS(session.create_schema_push_channel(core::SchemaPusherConfig{}), std::logic_error); - auto pull = session.create_pull_channel(); - REQUIRE_THROWS_AS(pull->create_wrist_tracking_source({}), std::logic_error); - REQUIRE_THROWS_AS(session.create_pull_channel(), std::logic_error); + core::IPluginPullChannel& pull = session.pull_channel(); + REQUIRE_THROWS_AS(pull.wrist_tracking_source(), std::logic_error); const DummyTracker tracker; - REQUIRE_THROWS_AS(pull->get_tracker_impl(tracker), std::runtime_error); - REQUIRE(bridge.wait_for(wire::FrameType::OpenPull).has_value()); + REQUIRE_THROWS_AS(pull.get_tracker_impl(tracker), std::runtime_error); + REQUIRE(bridge.wait_for(protocol::FrameType::OpenPull).has_value()); } TEST_CASE("RemotePluginSession rejects hand-tracking push when it was not declared", "[remote_device_bridge][unit]") { FakeBridge bridge; - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", - { .schema_push = true, .hand_tracking_push = false, .wrist_tracking_pull = false }); + RemotePluginSession session( + "127.0.0.1", bridge.port(), "test_app", { .schema_push = true, .hand_tracking_push = false }); REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT), std::logic_error); - REQUIRE_FALSE(bridge.wait_for(wire::FrameType::OpenHand, std::chrono::milliseconds(200)).has_value()); + REQUIRE_FALSE(bridge.wait_for(protocol::FrameType::OpenHand, std::chrono::milliseconds(200)).has_value()); } TEST_CASE("RemotePluginSession forwards OPEN_HAND, HAND_SAMPLE and CLOSE bit-exact", "[remote_device_bridge][unit]") @@ -410,20 +418,20 @@ TEST_CASE("RemotePluginSession forwards OPEN_HAND, HAND_SAMPLE and CLOSE bit-exa uint16_t channel_id = 0; { - core::HandTrackingPusher pusher(session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT)); - const auto open = bridge.wait_for(wire::FrameType::OpenHand); + const auto channel = session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT); + const auto open = bridge.wait_for(protocol::FrameType::OpenHand); REQUIRE(open.has_value()); - REQUIRE(open->header.channel_id != wire::kSessionChannelId); - REQUIRE(wire::decode_open_hand(open->payload).hand == XR_HAND_RIGHT_EXT); + REQUIRE(open->header.channel_id != protocol::kSessionChannelId); + REQUIRE(protocol::decode_open_hand(open->payload) == XR_HAND_RIGHT_EXT); channel_id = open->header.channel_id; const auto joints = make_joints(0.25f); - pusher.push(joints.data(), 123456789LL); + channel->push(joints.data(), 123456789LL); - const auto sample = bridge.wait_for(wire::FrameType::HandSample); + const auto sample = bridge.wait_for(protocol::FrameType::HandSample); REQUIRE(sample.has_value()); REQUIRE(sample->header.channel_id == channel_id); - const wire::HandSample decoded = wire::decode_hand_sample(sample->payload); + const protocol::HandSample decoded = protocol::decode_hand_sample(sample->payload); REQUIRE(decoded.sample_time_ns == 123456789LL); for (size_t i = 0; i < joints.size(); ++i) { @@ -431,7 +439,7 @@ TEST_CASE("RemotePluginSession forwards OPEN_HAND, HAND_SAMPLE and CLOSE bit-exa } } - const auto close = bridge.wait_for(wire::FrameType::Close); + const auto close = bridge.wait_for(protocol::FrameType::Close); REQUIRE(close.has_value()); REQUIRE(close->header.channel_id == channel_id); REQUIRE(close->payload.empty()); @@ -448,19 +456,17 @@ TEST_CASE("RemotePluginSession forwards schema channels", "[remote_device_bridge .localized_name = "Localized", .app_name = "test_app" }; auto channel = session.create_schema_push_channel(config); - REQUIRE(channel->config().collection_id == "collection"); - REQUIRE(channel->config().max_flatbuffer_size == 64); - const auto open = bridge.wait_for(wire::FrameType::OpenSchema); + const auto open = bridge.wait_for(protocol::FrameType::OpenSchema); REQUIRE(open.has_value()); - REQUIRE(wire::decode_open_schema(open->payload).localized_name == "Localized"); + REQUIRE(protocol::decode_open_schema(open->payload).localized_name == "Localized"); const std::vector bytes = { 1, 2, 3, 4, 5 }; channel->push_buffer(bytes.data(), bytes.size(), 77, 88); - const auto sample = bridge.wait_for(wire::FrameType::SchemaSample); + const auto sample = bridge.wait_for(protocol::FrameType::SchemaSample); REQUIRE(sample.has_value()); REQUIRE(sample->header.channel_id == open->header.channel_id); - const wire::SchemaSample decoded = wire::decode_schema_sample(sample->payload); + const protocol::SchemaSample decoded = protocol::decode_schema_sample(sample->payload); REQUIRE(decoded.bytes == bytes); REQUIRE(decoded.sample_time_local_common_clock_ns == 77); REQUIRE(decoded.sample_time_raw_device_clock_ns == 88); @@ -469,68 +475,68 @@ TEST_CASE("RemotePluginSession forwards schema channels", "[remote_device_bridge TEST_CASE("RemotePluginSession exposes bridge wrist samples through the pull channel", "[remote_device_bridge][unit]") { FakeBridge bridge; - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .wrist_tracking_pull = true }); + core::WristTrackingSourceConfig config; + config.mode = core::WristTrackingSourceMode::HandTracking; + config.left_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.5f, 0.0f, 0.0f } }; + config.right_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .wrist_tracking_pull = config }); - auto pull = session.create_pull_channel(); - const auto open_pull = bridge.wait_for(wire::FrameType::OpenPull); + const auto hello = bridge.wait_for(protocol::FrameType::Hello); + REQUIRE(hello.has_value()); + const auto declared = protocol::decode_hello(hello->payload).requirements.wrist_tracking_pull; + REQUIRE(declared.has_value()); + REQUIRE(declared->mode == core::WristTrackingSourceMode::HandTracking); + REQUIRE(declared->left_aim_to_wrist.position.x == 0.5f); + const auto open_pull = bridge.wait_for(protocol::FrameType::OpenPull); REQUIRE(open_pull.has_value()); - core::WristTrackingSourceConfig config; - config.mode = core::WristTrackingSourceMode::HandTracking; - config.left_aim_to_wrist.position.x = 0.5f; - auto source = pull->create_wrist_tracking_source(config); - REQUIRE(source != nullptr); - const auto open_wrist = bridge.wait_for(wire::FrameType::OpenWrist); - REQUIRE(open_wrist.has_value()); - REQUIRE(open_wrist->header.channel_id == open_pull->header.channel_id); - const core::WristTrackingSourceConfig decoded = wire::decode_open_wrist(open_wrist->payload); - REQUIRE(decoded.mode == core::WristTrackingSourceMode::HandTracking); - REQUIRE(decoded.left_aim_to_wrist.position.x == 0.5f); - - pull->update(); - core::WristTrackingSample before = source->query(true, 0); + core::IPluginPullChannel& pull = session.pull_channel(); + core::IWristTrackingSource& source = pull.wrist_tracking_source(); + pull.update(); + core::WristTrackingSample before = source.query(XR_HAND_LEFT_EXT, 0); REQUIRE_FALSE(before.valid); REQUIRE_FALSE(before.tracked); - wire::WristSample wrist; + protocol::WristSample wrist; wrist.sample_time_ns = 1000; - wrist.is_left = true; + wrist.hand = XR_HAND_LEFT_EXT; wrist.sample.valid = true; wrist.sample.tracked = true; wrist.sample.pose.position = { 0.1f, 1.2f, -0.4f }; - REQUIRE(bridge.send(wire::FrameType::WristSample, open_pull->header.channel_id, wire::encode(wrist))); + REQUIRE(bridge.send( + protocol::FrameType::WristSample, open_pull->header.channel_id, protocol::encode_wrist_sample(wrist))); - REQUIRE(wait_until([&] { return source->query(true, 0).valid; })); - const core::WristTrackingSample left = source->query(true, 0); + REQUIRE(wait_until([&] { return source.query(XR_HAND_LEFT_EXT, 0).valid; })); + const core::WristTrackingSample left = source.query(XR_HAND_LEFT_EXT, 0); REQUIRE(left.tracked); REQUIRE(left.pose.position.y == 1.2f); - REQUIRE_FALSE(source->query(false, 0).valid); + REQUIRE_FALSE(source.query(XR_HAND_RIGHT_EXT, 0).valid); } TEST_CASE("RemotePluginSession answers every PING with a PONG", "[remote_device_bridge][unit]") { FakeBridge bridge; RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true }); - REQUIRE(bridge.wait_for(wire::FrameType::Hello).has_value()); + REQUIRE(bridge.wait_for(protocol::FrameType::Hello).has_value()); for (int64_t t1 : { 111LL, 222LL, 333LL }) { const int64_t before = core::os_monotonic_now_ns(); - REQUIRE(bridge.send(wire::FrameType::Ping, wire::kSessionChannelId, wire::encode(wire::Ping{ t1 }))); - const auto frame = bridge.wait_for(wire::FrameType::Pong); + REQUIRE(bridge.send(protocol::FrameType::Ping, protocol::kSessionChannelId, protocol::encode_ping(t1))); + const auto frame = bridge.wait_for(protocol::FrameType::Pong); const int64_t after = core::os_monotonic_now_ns(); REQUIRE(frame.has_value()); - REQUIRE(frame->header.channel_id == wire::kSessionChannelId); - const wire::Pong pong = wire::decode_pong(frame->payload); - REQUIRE(pong.t1_ns == t1); - // Same host, same clock: before <= t2 <= t3 == header t_send <= after. - REQUIRE(before <= pong.t2_ns); - REQUIRE(pong.t2_ns <= pong.t3_ns); - REQUIRE(pong.t3_ns == frame->header.t_send_monotonic_ns); - REQUIRE(pong.t3_ns <= after); + REQUIRE(frame->header.channel_id == protocol::kSessionChannelId); + const protocol::Pong pong = protocol::decode_pong(frame->payload); + REQUIRE(pong.ping_sent_ns == t1); + // Same host, same clock: before <= ping_received <= pong_sent == header t_send <= after. + REQUIRE(before <= pong.ping_received_ns); + REQUIRE(pong.ping_received_ns <= pong.pong_sent_ns); + REQUIRE(pong.pong_sent_ns == frame->header.t_send_monotonic_ns); + REQUIRE(pong.pong_sent_ns <= after); ClockOffsetEstimator clock; - clock.add_two_way_sample(before, pong.t2_ns, pong.t3_ns, after); + clock.add_two_way_sample(before, pong.ping_received_ns, pong.pong_sent_ns, after); REQUIRE(clock.rtt_last_ns() >= 0); REQUIRE(clock.rtt_last_ns() <= after - before); } @@ -546,8 +552,8 @@ TEST_CASE("RemotePluginSession keeps the session when the bridge refuses one cha config.max_flatbuffer_size = 40960; auto refused = session.create_schema_push_channel(config); auto hand = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); - const auto open_schema = bridge.wait_for(wire::FrameType::OpenSchema); - const auto open_hand = bridge.wait_for(wire::FrameType::OpenHand); + const auto open_schema = bridge.wait_for(protocol::FrameType::OpenSchema); + const auto open_hand = bridge.wait_for(protocol::FrameType::OpenHand); REQUIRE(open_schema.has_value()); REQUIRE(open_hand.has_value()); @@ -556,7 +562,7 @@ TEST_CASE("RemotePluginSession keeps the session when the bridge refuses one cha const std::string reason = "schema 'tactile' sample size 40960 bytes exceeds bridge cap 32768 bytes (runtime " "limit); channel refused"; - REQUIRE(bridge.send(wire::FrameType::Error, open_schema->header.channel_id, wire::encode(wire::Error{ reason }))); + REQUIRE(bridge.send(protocol::FrameType::Error, open_schema->header.channel_id, protocol::encode_error(reason))); REQUIRE(wait_until([&] { return session.stats().refused_channels.count(open_schema->header.channel_id) == 1; })); REQUIRE(session.stats().refused_channels.at(open_schema->header.channel_id) == reason); REQUIRE(session.connected()); @@ -565,20 +571,20 @@ TEST_CASE("RemotePluginSession keeps the session when the bridge refuses one cha refused->push_buffer(bytes.data(), bytes.size(), 1, 1); const auto joints = make_joints(0.3f); hand->push(joints.data(), 42); - const auto sample = bridge.wait_for(wire::FrameType::HandSample); + const auto sample = bridge.wait_for(protocol::FrameType::HandSample); REQUIRE(sample.has_value()); REQUIRE(sample->header.channel_id == open_hand->header.channel_id); - REQUIRE_FALSE(bridge.wait_for(wire::FrameType::SchemaSample, std::chrono::milliseconds(200)).has_value()); + REQUIRE_FALSE(bridge.wait_for(protocol::FrameType::SchemaSample, std::chrono::milliseconds(200)).has_value()); REQUIRE(session.connected()); refused.reset(); // CLOSE still goes out so the bridge forgets the refused id - const auto close = bridge.wait_for(wire::FrameType::Close); + const auto close = bridge.wait_for(protocol::FrameType::Close); REQUIRE(close.has_value()); REQUIRE(close->header.channel_id == open_schema->header.channel_id); } SECTION("ERROR on channel 0 ends the session") { - REQUIRE(bridge.send(wire::FrameType::Error, wire::kSessionChannelId, wire::encode(wire::Error{ "fatal" }))); + REQUIRE(bridge.send(protocol::FrameType::Error, protocol::kSessionChannelId, protocol::encode_error("fatal"))); REQUIRE(wait_until([&] { return !session.connected(); })); REQUIRE(session.stats().refused_channels.empty()); } @@ -588,26 +594,23 @@ TEST_CASE("RemotePluginSession survives the bridge dropping the socket", "[remot { FakeBridge bridge; std::unique_ptr channel; - std::unique_ptr pull; { - RemotePluginSession session( - "127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true, .wrist_tracking_pull = true }); + RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", + { .hand_tracking_push = true, .wrist_tracking_pull = kWristConfig }); channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); - pull = session.create_pull_channel(); - REQUIRE(bridge.wait_for(wire::FrameType::OpenHand).has_value()); + REQUIRE(bridge.wait_for(protocol::FrameType::OpenHand).has_value()); bridge.drop_connection(); REQUIRE(wait_until([&] { return !session.connected(); })); const auto joints = make_joints(0.5f); REQUIRE_NOTHROW(channel->push(joints.data(), 1)); - REQUIRE_NOTHROW(pull->update()); + REQUIRE_NOTHROW(session.pull_channel().update()); REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT), std::runtime_error); } // Channels outliving a dead session must still be destroyable. REQUIRE_NOTHROW(channel->push(make_joints(0.5f).data(), 2)); REQUIRE_NOTHROW(channel.reset()); - REQUIRE_NOTHROW(pull.reset()); } TEST_CASE("RemotePluginSession never reconnects after the link drops", "[remote_device_bridge][unit]") @@ -621,8 +624,8 @@ TEST_CASE("RemotePluginSession never reconnects after the link drops", "[remote_ std::vector hello; if (bridge_side && bridge_side->receive(hello, kWait) == ReceiveResult::Ok) { - send_frame(*bridge_side, wire::FrameType::HelloAck, wire::kSessionChannelId, 0, - wire::encode(wire::HelloAck{ true, "" })); + send_frame(*bridge_side, protocol::FrameType::HelloAck, protocol::kSessionChannelId, 0, + protocol::encode_hello_ack(protocol::HelloAck{ true, "" })); } }); RemotePluginSession session("127.0.0.1", listener.local_port(), "test_app", { .hand_tracking_push = true }); @@ -651,9 +654,9 @@ TEST_CASE("RemotePluginSession closes remaining channels and the socket on destr { RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true }); channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); - REQUIRE(bridge.wait_for(wire::FrameType::OpenHand).has_value()); + REQUIRE(bridge.wait_for(protocol::FrameType::OpenHand).has_value()); } - REQUIRE(bridge.wait_for(wire::FrameType::Close).has_value()); + REQUIRE(bridge.wait_for(protocol::FrameType::Close).has_value()); REQUIRE(bridge.client_closed()); REQUIRE_NOTHROW(channel.reset()); } @@ -669,9 +672,9 @@ TEST_CASE("RemotePluginSession keeps the newest samples per channel while the li auto filler = session.create_schema_push_channel(filler_config); auto fast = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); // "A": 180 Hz auto slow = session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT); // "B": 60 Hz - REQUIRE(link.wait_for(wire::FrameType::OpenSchema).has_value()); - const auto open_fast = link.wait_for(wire::FrameType::OpenHand); - const auto open_slow = link.wait_for(wire::FrameType::OpenHand); + REQUIRE(link.wait_for(protocol::FrameType::OpenSchema).has_value()); + const auto open_fast = link.wait_for(protocol::FrameType::OpenHand); + const auto open_slow = link.wait_for(protocol::FrameType::OpenHand); REQUIRE(open_fast.has_value()); REQUIRE(open_slow.has_value()); const uint16_t fast_id = open_fast->header.channel_id; @@ -711,13 +714,13 @@ TEST_CASE("RemotePluginSession keeps the newest samples per channel while the li (fast_times.empty() || fast_times.back() != 1000 + kFastPushes - 1 || slow_times.empty() || slow_times.back() != 2000 + kSlowPushes - 1)) { - const auto frame = link.wait_for(wire::FrameType::HandSample, std::chrono::milliseconds(200)); + const auto frame = link.wait_for(protocol::FrameType::HandSample, std::chrono::milliseconds(200)); if (!frame.has_value()) { continue; } (frame->header.channel_id == fast_id ? fast_times : slow_times) - .push_back(wire::decode_hand_sample(frame->payload).sample_time_ns); + .push_back(protocol::decode_hand_sample(frame->payload).sample_time_ns); } // Each channel delivered its newest sample, in order, from a queue that never held more than 4. @@ -740,7 +743,7 @@ TEST_CASE("RemotePluginSession drops a channel's pending samples when the channe core::SchemaPusherConfig filler_config; filler_config.collection_id = "filler"; auto filler = session.create_schema_push_channel(filler_config); - REQUIRE(link.wait_for(wire::FrameType::OpenSchema).has_value()); + REQUIRE(link.wait_for(protocol::FrameType::OpenSchema).has_value()); // Stall the sender on a filler sample. A paused TCP peer is not enough: with 8 MiB socket buffers the kernel // absorbs the filler and the hand samples leave before the CLOSE. @@ -759,10 +762,10 @@ TEST_CASE("RemotePluginSession drops a channel's pending samples when the channe link.hold(false); // OPEN_HAND and CLOSE go out in order; the samples queued behind the stalled filler never follow the CLOSE. - const auto open = link.wait_for(wire::FrameType::OpenHand); + const auto open = link.wait_for(protocol::FrameType::OpenHand); REQUIRE(open.has_value()); - const auto close = link.wait_for(wire::FrameType::Close); + const auto close = link.wait_for(protocol::FrameType::Close); REQUIRE(close.has_value()); REQUIRE(close->header.channel_id == open->header.channel_id); - REQUIRE_FALSE(link.wait_for(wire::FrameType::HandSample, std::chrono::milliseconds(200)).has_value()); + REQUIRE_FALSE(link.wait_for(protocol::FrameType::HandSample, std::chrono::milliseconds(200)).has_value()); } From cbcd4b84990bbad1b5e8fdd4186906f349624b71 Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Fri, 30 Oct 2026 10:48:49 -0700 Subject: [PATCH 03/10] review round 1 (pr04) Applied the fact-checked (VERIFIED) findings on code this branch adds: - Sample queues get their frame type once, at OPEN; pushes to a closed or refused channel find no queue and return. send_control (unused) is gone. - sent_hand/sent_schema count frames after send_now succeeds, not when they are dequeued. - One hand_index() helper; PING answering no longer keeps a flag just to log once. - on_disconnect carries the real reason (bridge ERROR text, DecodeError text) instead of logging it separately; an unexpected frame type is an error; log lines drop prefixes the logger name already carries. - ClockOffsetEstimator takes the two-way window as a constructor parameter like the one-way window, and derives the two-way offset and RTTs from the exchange window instead of caching them; PingScheduler's two flags become one state enum. - Docs: any ITransport (not only TCP), no queue-size numbers or "P0", AGENTS.md keeps only the link rule and states which sample-path calls still throw on caller bugs; the BlockingTransport note lives with the test only. - Tests: FakeBridge takes an optional rejection; the PING test no longer re-tests the estimator; the reconnect test keeps only its own check. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- src/plugins/remote_device_bridge/AGENTS.md | 5 +- .../session/clock_offset_estimator.cpp | 38 ++++---- .../clock_offset_estimator.hpp | 25 +++--- .../remote_plugin_session.hpp | 15 ++-- .../session/remote_plugin_session.cpp | 87 ++++++++++--------- .../test_remote_plugin_session.cpp | 20 ++--- 6 files changed, 97 insertions(+), 93 deletions(-) diff --git a/src/plugins/remote_device_bridge/AGENTS.md b/src/plugins/remote_device_bridge/AGENTS.md index 27d69760b7..53cbf38fb3 100644 --- a/src/plugins/remote_device_bridge/AGENTS.md +++ b/src/plugins/remote_device_bridge/AGENTS.md @@ -11,8 +11,7 @@ SPDX-License-Identifier: Apache-2.0 - Wire format v1 (`kProtocolVersion` = 1, the only version number): 24-byte header `RPS0` magic, type u8, version u8, channel u16, seq u32, payload size u32, `t_send` i64; frame types 1..13 with no gaps; the hand byte is 0 = left, 1 = right in every payload; PING has no payload and PONG carries the PING's header `t_send` and the client's receive time. Any change to a header, payload or type number bumps `kProtocolVersion`; `decode_header` rejects other versions before reading the type, so old senders and bridges fail with a readable reason instead of mis-decoding. - A frame's `t_send` is its send time for clock exchange: call `stamp_header` where the frame is written to the transport, never when it is queued. - WebSocket is a second transport, not a second protocol: `WebSocketTransport` carries exactly one wire frame per binary message (no TLS; terminate `wss://` in a proxy). Clients pick it with a `ws://host:port[/path]` address via `parse_remote`/`connect_transport`. A server serves it on its own port; never sniff the protocol on the TCP port. The server-side HTTP upgrade runs in the first `receive()` so an accept loop never blocks on a client. -- `Teleop::remote_plugin_session` links `pusherio::pusherio` (OpenXR value types via headers) plus `log_bridge` and `Threads` privately. Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to it: a plugin that talks to a bridge through this library must run without an OpenXR runtime. +- Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to `Teleop::remote_plugin_session`: a plugin that talks to a bridge through this library must run without an OpenXR runtime. - ERROR on channel 0 ends the session; ERROR on a channel id refuses only that OPEN (schema above `--max-schema-bytes`, a hand someone else owns) and the session and its other channels continue. Refuse per channel whenever the session itself is still sound. -- `RemotePluginSession` never throws on the sample path (`push`, `push_buffer`, `query`, `update`). After a transport failure those become no-ops; only channel creation and the constructor throw. +- `RemotePluginSession` never throws on the sample path for transport failures (`push`, `push_buffer`, `query`, `update`): after one they become no-ops; only channel creation and the constructor throw. Caller bugs (null joints or buffer) still throw `std::invalid_argument`. - Timestamps cross the wire in the sender's `CLOCK_MONOTONIC`; the receiver maps them with `ClockOffsetEstimator`. The bridge's offset comes from PING/PONG (min-RTT of the last 32 exchanges, `--clock-source=two-way`); the one-way minimum of `local_recv - remote_send` hides the minimum network delay and is only the fallback (no PONG within 2 s) or the `--clock-source=one-way` A/B baseline. Do not push raw remote timestamps into OpenXR. -- Tests that need a stalled link use `BlockingTransport` in `test_remote_plugin_session.cpp`, never a TCP peer that stops reading: loopback socket buffers differ per host (416 KiB in one sandbox, 2×8 MiB on the teleop workstation), so "the sender is blocked by now" is not portable. diff --git a/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp index 88954c3324..c3edbbdd3f 100644 --- a/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp +++ b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp @@ -9,10 +9,10 @@ namespace remote_plugin_session { -ClockOffsetEstimator::ClockOffsetEstimator(size_t max_samples, int64_t window_ns) - : m_max_samples(max_samples), m_window_ns(window_ns) +ClockOffsetEstimator::ClockOffsetEstimator(size_t max_samples, int64_t window_ns, size_t two_way_window) + : m_max_samples(max_samples), m_window_ns(window_ns), m_two_way_window(two_way_window) { - if (max_samples == 0 || window_ns <= 0) + if (max_samples == 0 || window_ns <= 0 || two_way_window == 0) { throw std::invalid_argument("ClockOffsetEstimator requires a non-empty window"); } @@ -46,20 +46,21 @@ void ClockOffsetEstimator::add_two_way_sample(int64_t t1_ns, int64_t t2_ns, int6 // local - remote, the same sign as the one-way offset. const int64_t offset_ns = ((t1_ns - t2_ns) + (t4_ns - t3_ns)) / 2; m_exchanges.push_back(Exchange{ rtt_ns, offset_ns }); - if (m_exchanges.size() > kTwoWayWindow) + if (m_exchanges.size() > m_two_way_window) { m_exchanges.pop_front(); } - const Exchange& best = *std::min_element(m_exchanges.begin(), m_exchanges.end(), - [](const Exchange& a, const Exchange& b) { return a.rtt_ns < b.rtt_ns; }); - m_two_way_offset_ns = best.offset_ns; - m_rtt_min_ns = best.rtt_ns; - m_rtt_last_ns = rtt_ns; +} + +const ClockOffsetEstimator::Exchange& ClockOffsetEstimator::best_exchange() const +{ + return *std::min_element(m_exchanges.begin(), m_exchanges.end(), + [](const Exchange& a, const Exchange& b) { return a.rtt_ns < b.rtt_ns; }); } int64_t ClockOffsetEstimator::offset_ns() const { - return m_exchanges.empty() ? m_offset_ns : m_two_way_offset_ns; + return m_exchanges.empty() ? m_offset_ns : best_exchange().offset_ns; } int64_t ClockOffsetEstimator::remote_to_local(int64_t remote_ns) const @@ -69,12 +70,12 @@ int64_t ClockOffsetEstimator::remote_to_local(int64_t remote_ns) const int64_t ClockOffsetEstimator::rtt_min_ns() const { - return m_rtt_min_ns; + return m_exchanges.empty() ? 0 : best_exchange().rtt_ns; } int64_t ClockOffsetEstimator::rtt_last_ns() const { - return m_rtt_last_ns; + return m_exchanges.empty() ? 0 : m_exchanges.back().rtt_ns; } const char* ClockOffsetEstimator::offset_source() const @@ -88,13 +89,13 @@ PingScheduler::PingScheduler(int64_t start_ns) : m_start_ns(start_ns), m_next_ns PingScheduler::Action PingScheduler::poll(int64_t now_ns) { - if (m_gave_up) + if (m_state == State::GaveUp) { return Action::None; } - if (!m_got_pong && now_ns - m_start_ns >= kPongTimeoutNs) + if (m_state == State::Waiting && now_ns - m_start_ns >= kPongTimeoutNs) { - m_gave_up = true; + m_state = State::GaveUp; return Action::GiveUp; } if (now_ns < m_next_ns) @@ -108,12 +109,15 @@ PingScheduler::Action PingScheduler::poll(int64_t now_ns) void PingScheduler::on_pong() { - m_got_pong = true; + if (m_state == State::Waiting) + { + m_state = State::GotPong; + } } bool PingScheduler::active() const { - return !m_gave_up; + return m_state != State::GaveUp; } } // namespace remote_plugin_session diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp index 280b716bbb..c90f49fe51 100644 --- a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp @@ -16,16 +16,15 @@ namespace remote_plugin_session * One-way: fed with (remote send time, local receive time) of every received * frame; the offset is the running minimum of (local - remote) over a sliding * window, which hides one minimum one-way delay. Two-way: once a PING/PONG - * exchange arrives the offset comes from the minimum-RTT exchange of the last - * kTwoWayWindow instead (NTP filter; error = path asymmetry / 2) and one-way + * exchange arrives the offset comes from the minimum-RTT exchange of the + * two-way window instead (NTP filter; error = path asymmetry / 2) and one-way * samples no longer move it. Not thread-safe; the owner serializes access. */ class ClockOffsetEstimator { public: - static constexpr size_t kTwoWayWindow = 32; - - explicit ClockOffsetEstimator(size_t max_samples = 200, int64_t window_ns = 5'000'000'000); + //! One-way: at most `max_samples` within `window_ns`; two-way: the last `two_way_window` exchanges. + explicit ClockOffsetEstimator(size_t max_samples = 200, int64_t window_ns = 5'000'000'000, size_t two_way_window = 32); void add_sample(int64_t remote_t_send_ns, int64_t local_t_recv_ns); //! t1/t4: local PING send / PONG receive; t2/t3: remote PING receive / PONG send. Negative RTTs are ignored. @@ -52,14 +51,14 @@ class ClockOffsetEstimator int64_t offset_ns; }; + const Exchange& best_exchange() const; + size_t m_max_samples; int64_t m_window_ns; + size_t m_two_way_window; std::deque m_samples; int64_t m_offset_ns = 0; std::deque m_exchanges; - int64_t m_two_way_offset_ns = 0; - int64_t m_rtt_min_ns = 0; - int64_t m_rtt_last_ns = 0; }; /*! @@ -91,11 +90,17 @@ class PingScheduler bool active() const; private: + enum class State + { + Waiting, + GotPong, + GaveUp, + }; + int64_t m_start_ns; int64_t m_next_ns; int m_sent = 0; - bool m_got_pong = false; - bool m_gave_up = false; + State m_state = State::Waiting; }; //! True when a mapped sample time is older than now - max_age; max_age <= 0 disables the guard. After a link diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp index 45640c18b8..b3fecef8a2 100644 --- a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp @@ -22,18 +22,17 @@ class RemotePluginSessionImpl; } /*! - * @brief Plugin session that forwards every channel over TCP to a remote_device_bridge. + * @brief Plugin session that forwards every channel over an ITransport to a remote_device_bridge. * * Same contract as the OpenXR plugin session: the session owns its pull channel * (opened with the session, with the wrist source HELLO declared) and the owner * destroys all push channels before the session. After the transport * drops, pushes become silent no-ops and channels stay safe to destroy. - * Each push channel has a latest-wins queue (4 samples / 8 MiB); on a stalled - * link the oldest samples of that channel are dropped, other channels are - * unaffected. A channel the bridge refuses (ERROR on its channel id) turns - * into a no-op; the session and its other channels continue. P0: the pull - * channel exposes no trackers; wrist poses are the latest bridge samples (no - * interpolation). + * Each push channel has a small latest-wins queue; on a stalled link the oldest + * samples of that channel are dropped, other channels are unaffected. A channel + * the bridge refuses (ERROR on its channel id) turns into a no-op; the session + * and its other channels continue. The pull channel exposes no trackers; wrist + * poses are the latest bridge samples (no interpolation). */ class RemotePluginSession final : public core::IPluginSession { @@ -71,7 +70,7 @@ class RemotePluginSession final : public core::IPluginSession bool connected() const; //! Estimated bridge-to-local clock offset in ns (local ~= bridge + offset); 0 before the first frame. int64_t clock_offset_ns() const; - //! Cumulative counters; also logged every 5 s as one "[remote] 5s:" line. + //! Cumulative counters; also logged periodically as one "5s:" line. Stats stats() const; private: diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp index 150b4e5906..2fe709b6b4 100644 --- a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -55,9 +55,14 @@ constexpr size_t kMaxQueuedBytesPerChannel = 8u * 1024u * 1024u; using FrameBytes = std::vector; // one encoded wire message, header included +size_t hand_index(XrHandEXT hand) +{ + return hand == XR_HAND_LEFT_EXT ? 0 : 1; +} + struct ChannelQueue { - protocol::FrameType type = protocol::FrameType::HandSample; + protocol::FrameType type; // the channel's sample frame type, fixed at OPEN std::deque frames; size_t bytes = 0; uint64_t dropped = 0; @@ -192,16 +197,17 @@ class detail::RemotePluginSessionImpl channel_id = m_next_channel_id++; m_open_channels.insert(channel_id); } + if (type == protocol::FrameType::OpenHand || type == protocol::FrameType::OpenSchema) + { + std::lock_guard lock(m_queue_mutex); + m_sample_queues.emplace( + channel_id, ChannelQueue{ type == protocol::FrameType::OpenHand ? protocol::FrameType::HandSample : + protocol::FrameType::SchemaSample }); + } enqueue_control(type, channel_id, std::move(payload)); return channel_id; } - void send_control(protocol::FrameType type, uint16_t channel_id, std::vector payload) - { - require_connected(); - enqueue_control(type, channel_id, std::move(payload)); - } - void close_channel(uint16_t channel_id) { { @@ -227,7 +233,7 @@ class detail::RemotePluginSessionImpl } //! Per-channel latest-wins queue; control frames are never dropped. - void push_sample(protocol::FrameType type, uint16_t channel_id, FrameBytes frame) + void push_sample(uint16_t channel_id, FrameBytes frame) { if (!m_connected.load()) { @@ -235,12 +241,12 @@ class detail::RemotePluginSessionImpl } { std::lock_guard lock(m_queue_mutex); - if (m_refused_channels.count(channel_id) != 0) + const auto it = m_sample_queues.find(channel_id); + if (it == m_sample_queues.end()) // closed or refused { return; } - ChannelQueue& queue = m_sample_queues[channel_id]; - queue.type = type; + ChannelQueue& queue = it->second; while (!queue.frames.empty() && (queue.frames.size() >= kMaxQueuedSamplesPerChannel || queue.bytes + frame.size() > kMaxQueuedBytesPerChannel)) { @@ -260,7 +266,7 @@ class detail::RemotePluginSessionImpl core::WristTrackingSample latest_wrist(XrHandEXT hand) const { std::lock_guard lock(m_wrist_mutex); - return m_wrist[hand == XR_HAND_LEFT_EXT ? 0 : 1]; + return m_wrist[hand_index(hand)]; } void shutdown() @@ -322,7 +328,8 @@ class detail::RemotePluginSessionImpl { const int64_t t_send_ns = core::os_monotonic_now_ns(); protocol::stamp_header(frame, m_next_seq++, t_send_ns); - if (protocol::decode_header(frame).type == protocol::FrameType::Pong) + const protocol::FrameType type = protocol::decode_header(frame).type; + if (type == protocol::FrameType::Pong) { protocol::set_pong_sent_time(frame, t_send_ns); } @@ -331,6 +338,14 @@ class detail::RemotePluginSessionImpl return false; } m_tx_bytes.fetch_add(frame.size()); + if (type == protocol::FrameType::HandSample) + { + m_sent_hand.fetch_add(1); + } + else if (type == protocol::FrameType::SchemaSample) + { + m_sent_schema.fetch_add(1); + } return true; } @@ -362,7 +377,6 @@ class detail::RemotePluginSessionImpl batch.push_back(std::move(queue.frames.front())); queue.frames.pop_front(); --m_queued_samples; - (queue.type == protocol::FrameType::HandSample ? m_sent_hand : m_sent_schema).fetch_add(1); } return batch; } @@ -372,7 +386,7 @@ class detail::RemotePluginSessionImpl const RemotePluginSession::Stats now = stats(); const RemotePluginSession::Stats& last = m_logged_stats; logger()->info( - "[remote] 5s: sent hand={} schema={} wrist_rx={} hb={} | dropped queue={} | tx_bytes={} | offset_ns={} | " + "5s: sent hand={} schema={} wrist_rx={} hb={} | dropped queue={} | tx_bytes={} | offset_ns={} | " "connected={:d}", now.sent_hand - last.sent_hand, now.sent_schema - last.sent_schema, now.wrist_rx - last.wrist_rx, now.heartbeats - last.heartbeats, now.dropped_queue - last.dropped_queue, now.tx_bytes - last.tx_bytes, @@ -380,7 +394,7 @@ class detail::RemotePluginSessionImpl m_logged_stats = now; } - void on_disconnect(const char* reason) + void on_disconnect(const std::string& reason) { if (!m_connected.exchange(false)) { @@ -388,7 +402,7 @@ class detail::RemotePluginSessionImpl } if (!m_stop) { - logger()->warn("[RemotePluginSession] disconnected: {}", reason); + logger()->warn("disconnected: {}", reason); } m_transport->shutdown(); m_queue_cv.notify_all(); @@ -397,7 +411,7 @@ class detail::RemotePluginSessionImpl //! The bridge refused one channel: its pushes become no-ops, the session and other channels continue. void refuse_channel(uint16_t channel_id, const std::string& reason) { - logger()->warn("[RemotePluginSession] bridge refused channel {}: {}", channel_id, reason); + logger()->warn("bridge refused channel {}: {}", channel_id, reason); std::lock_guard lock(m_queue_mutex); m_refused_channels[channel_id] = reason; if (const auto it = m_sample_queues.find(channel_id); it != m_sample_queues.end()) @@ -488,12 +502,12 @@ class detail::RemotePluginSessionImpl { case protocol::FrameType::WristSample: { - // sample_time_ns is dropped: core::WristTrackingSample has no timestamp field and P0 serves the - // latest sample. + // sample_time_ns is dropped: core::WristTrackingSample has no timestamp field and queries return + // the latest sample. const protocol::WristSample sample = protocol::decode_wrist_sample(frame.payload); m_wrist_rx.fetch_add(1); std::lock_guard lock(m_wrist_mutex); - m_wrist[sample.hand == XR_HAND_LEFT_EXT ? 0 : 1] = sample.sample; + m_wrist[hand_index(sample.hand)] = sample.sample; break; } case protocol::FrameType::Ping: @@ -502,10 +516,6 @@ class detail::RemotePluginSessionImpl const int64_t t1_ns = protocol::decode_ping(frame.payload); enqueue_control(protocol::FrameType::Pong, protocol::kSessionChannelId, protocol::encode_pong(protocol::Pong{ t1_ns, t_recv_ns, 0 })); - if (!std::exchange(m_answered_ping, true)) - { - logger()->info("[remote] clock: answering bridge PINGs (two-way offset)"); - } break; } case protocol::FrameType::Error: @@ -516,23 +526,21 @@ class detail::RemotePluginSessionImpl refuse_channel(frame.header.channel_id, message); break; } - logger()->error("[RemotePluginSession] bridge error: {}", message); - on_disconnect("bridge reported an error"); + on_disconnect("bridge error: " + message); break; } case protocol::FrameType::Heartbeat: case protocol::FrameType::HelloAck: break; default: - logger()->warn( - "[RemotePluginSession] ignoring unexpected frame type {}", static_cast(frame.header.type)); + logger()->error( + "ignoring unexpected frame type {} (protocol mismatch)", static_cast(frame.header.type)); break; } } catch (const protocol::DecodeError& e) { - logger()->error("[RemotePluginSession] {}", e.what()); - on_disconnect("undecodable frame from the bridge"); + on_disconnect(std::string("undecodable frame from the bridge: ") + e.what()); } } @@ -562,7 +570,6 @@ class detail::RemotePluginSessionImpl std::atomic m_wrist_rx{ 0 }; std::atomic m_heartbeats{ 0 }; std::atomic m_tx_bytes{ 0 }; - bool m_answered_ping = false; // receiver thread only RemotePluginSession::Stats m_logged_stats; // sender thread only mutable std::mutex m_wrist_mutex; @@ -601,9 +608,8 @@ class RemoteHandTrackingPushChannel final : public core::IHandTrackingPushChanne { return; } - m_impl->push_sample( - protocol::FrameType::HandSample, m_channel_id, - protocol::encode_hand_sample_frame(m_channel_id, joint_locations, sample_time_local_common_clock_ns)); + m_impl->push_sample(m_channel_id, protocol::encode_hand_sample_frame( + m_channel_id, joint_locations, sample_time_local_common_clock_ns)); } private: @@ -637,10 +643,9 @@ class RemoteSchemaPushChannel final : public core::ISchemaPushChannel { return; } - m_impl->push_sample( - protocol::FrameType::SchemaSample, m_channel_id, - protocol::encode_schema_sample_frame( - m_channel_id, buffer, size, sample_time_local_common_clock_ns, sample_time_raw_device_clock_ns)); + m_impl->push_sample(m_channel_id, protocol::encode_schema_sample_frame(m_channel_id, buffer, size, + sample_time_local_common_clock_ns, + sample_time_raw_device_clock_ns)); } private: @@ -655,7 +660,7 @@ class RemoteWristTrackingSource final : public core::IWristTrackingSource { } - // P0 returns the latest bridge sample; the requested time is not interpolated. + // Returns the latest bridge sample; the requested time is not interpolated. core::WristTrackingSample query(XrHandEXT hand, int64_t /*sample_time_local_common_clock_ns*/) override { return m_impl->latest_wrist(hand); @@ -690,7 +695,7 @@ class RemotePluginPullChannel final : public core::IPluginPullChannel const core::ITrackerImpl& get_tracker_impl(const core::ITracker&) const override { - throw std::runtime_error("RemotePluginSession pull channel exposes no trackers (P0)"); + throw std::runtime_error("RemotePluginSession pull channel exposes no trackers"); } core::IWristTrackingSource& wrist_tracking_source() override diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp index 6392c2dfb2..0a5a07545f 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -69,8 +68,9 @@ std::optional take_frame(std::deque& frames, p class FakeBridge { public: - explicit FakeBridge(bool accept = true, std::string reason = "") - : m_listener(TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 1)), m_accept(accept), m_reason(std::move(reason)) + //! Answers HELLO with a refusal carrying `rejection` when set, else accepts. + explicit FakeBridge(std::optional rejection = std::nullopt) + : m_listener(TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 1)), m_rejection(std::move(rejection)) { m_thread = std::thread(&FakeBridge::serve, this); } @@ -160,7 +160,7 @@ class FakeBridge m_client = std::move(client); m_frames.push_back(std::move(hello)); send_frame(*m_client, protocol::FrameType::HelloAck, protocol::kSessionChannelId, m_seq++, - protocol::encode_hello_ack(protocol::HelloAck{ m_accept, m_reason })); + protocol::encode_hello_ack(protocol::HelloAck{ !m_rejection, m_rejection.value_or("") })); } m_cv.notify_all(); @@ -187,8 +187,7 @@ class FakeBridge } TcpSocket m_listener; - bool m_accept; - std::string m_reason; + std::optional m_rejection; std::atomic m_stop{ false }; std::thread m_thread; @@ -351,7 +350,7 @@ TEST_CASE("RemotePluginSession fails fast when the bridge refuses or is absent", { SECTION("HELLO_ACK with accepted=0 throws with the reason") { - FakeBridge bridge(false, "no OpenXR runtime"); + FakeBridge bridge(std::string("no OpenXR runtime")); try { RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", {}); @@ -534,11 +533,6 @@ TEST_CASE("RemotePluginSession answers every PING with a PONG", "[remote_device_ REQUIRE(pong.ping_received_ns <= pong.pong_sent_ns); REQUIRE(pong.pong_sent_ns == frame->header.t_send_monotonic_ns); REQUIRE(pong.pong_sent_ns <= after); - - ClockOffsetEstimator clock; - clock.add_two_way_sample(before, pong.ping_received_ns, pong.pong_sent_ns, after); - REQUIRE(clock.rtt_last_ns() >= 0); - REQUIRE(clock.rtt_last_ns() <= after - before); } } @@ -643,8 +637,6 @@ TEST_CASE("RemotePluginSession never reconnects after the link drops", "[remote_ } // A new HELLO needs a new session (a restarted plugin); nothing connects back on its own. REQUIRE(TcpTransport::accept(listener, std::chrono::milliseconds(500)) == nullptr); - REQUIRE_FALSE(session.connected()); - REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT), std::runtime_error); } TEST_CASE("RemotePluginSession closes remaining channels and the socket on destruction", "[remote_device_bridge][unit]") From 796eb550a652bb13f8d5f9d48fda1d4ec0c71662 Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Fri, 30 Oct 2026 10:58:30 -0700 Subject: [PATCH 04/10] review round 2 (pr04) Applied the fact-checked (VERIFIED) findings on code this branch adds: - The session owns its impl outright; channels and the wrist source hold a reference, as IPluginSession has the owner destroy channels first. The explicit shutdown in the destructor and the test of channels outliving the session are gone. - Removed the host/port constructor; tests connect a TcpTransport. - Dropped redundant copy/move deletions and the unread ChannelQueue::type. - require_connected is private to the impl. - A HELLO_ACK after the handshake or an unknown frame type disconnects as a protocol mismatch instead of being logged and ignored. - The stats line prefix comes from kStatsPeriod; docs no longer hardcode it. - ClockOffsetEstimator drops an always-true empty check; the two_way_window == 0 rejection is tested. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- .../session/clock_offset_estimator.cpp | 2 +- .../remote_plugin_session.hpp | 16 +-- .../session/remote_plugin_session.cpp | 108 ++++++++---------- .../test_clock_offset_estimator.cpp | 1 + .../test_remote_plugin_session.cpp | 42 +++---- 5 files changed, 74 insertions(+), 95 deletions(-) diff --git a/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp index c3edbbdd3f..86c5652dd1 100644 --- a/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp +++ b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp @@ -26,7 +26,7 @@ void ClockOffsetEstimator::add_sample(int64_t remote_t_send_ns, int64_t local_t_ { m_samples.pop_front(); } - while (!m_samples.empty() && m_samples.front().local_t_recv_ns < local_t_recv_ns - m_window_ns) + while (m_samples.front().local_t_recv_ns < local_t_recv_ns - m_window_ns) { m_samples.pop_front(); } diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp index b3fecef8a2..73959ae1d1 100644 --- a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp @@ -49,32 +49,26 @@ class RemotePluginSession final : public core::IPluginSession std::map refused_channels; //!< channel id -> the bridge's reason }; - //! Connects over TCP, sends HELLO and waits for HELLO_ACK; throws std::runtime_error on any failure. - RemotePluginSession(std::string host, uint16_t port, std::string app_name, core::PluginSessionRequirements requirements); - //! Same handshake over an already connected transport. + //! Sends HELLO over a connected transport (see connect_transport()) and waits for HELLO_ACK; throws + //! std::runtime_error on any failure. RemotePluginSession(std::unique_ptr transport, std::string app_name, core::PluginSessionRequirements requirements); ~RemotePluginSession() override; - RemotePluginSession(const RemotePluginSession&) = delete; - RemotePluginSession& operator=(const RemotePluginSession&) = delete; - RemotePluginSession(RemotePluginSession&&) = delete; - RemotePluginSession& operator=(RemotePluginSession&&) = delete; - core::IPluginPullChannel& pull_channel() override; std::unique_ptr create_schema_push_channel(const core::SchemaPusherConfig& config) override; std::unique_ptr create_hand_tracking_push_channel(XrHandEXT hand) override; //! False once the transport failed; channel creation then throws and pushes are dropped. bool connected() const; - //! Estimated bridge-to-local clock offset in ns (local ~= bridge + offset); 0 before the first frame. + //! Estimated bridge-to-local clock offset in ns (local ~= bridge + offset), measured from HELLO_ACK onward. int64_t clock_offset_ns() const; - //! Cumulative counters; also logged periodically as one "5s:" line. + //! Cumulative counters; their deltas are also logged periodically. Stats stats() const; private: - std::shared_ptr m_impl; + std::unique_ptr m_impl; std::unique_ptr m_pull_channel; }; diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp index 2fe709b6b4..ff3e288106 100644 --- a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -7,8 +7,6 @@ #include "inc/remote_plugin_session/protocol_frame.hpp" #include "inc/remote_plugin_session/protocol_payloads.hpp" #include "inc/remote_plugin_session/remote_address.hpp" -#include "inc/remote_plugin_session/tcp_socket.hpp" -#include "inc/remote_plugin_session/tcp_transport.hpp" #include #include @@ -62,7 +60,6 @@ size_t hand_index(XrHandEXT hand) struct ChannelQueue { - protocol::FrameType type; // the channel's sample frame type, fixed at OPEN std::deque frames; size_t bytes = 0; uint64_t dropped = 0; @@ -175,14 +172,6 @@ class detail::RemotePluginSessionImpl return stats; } - void require_connected() const - { - if (!m_connected.load()) - { - throw std::runtime_error("RemotePluginSession: transport is disconnected"); - } - } - //! Allocates a channel id and queues its OPEN_* frame; ids are never reused within a session. uint16_t open_channel(protocol::FrameType type, std::vector payload) { @@ -200,9 +189,7 @@ class detail::RemotePluginSessionImpl if (type == protocol::FrameType::OpenHand || type == protocol::FrameType::OpenSchema) { std::lock_guard lock(m_queue_mutex); - m_sample_queues.emplace( - channel_id, ChannelQueue{ type == protocol::FrameType::OpenHand ? protocol::FrameType::HandSample : - protocol::FrameType::SchemaSample }); + m_sample_queues.emplace(channel_id, ChannelQueue{}); } enqueue_control(type, channel_id, std::move(payload)); return channel_id; @@ -312,6 +299,14 @@ class detail::RemotePluginSessionImpl } private: + void require_connected() const + { + if (!m_connected.load()) + { + throw std::runtime_error("RemotePluginSession: transport is disconnected"); + } + } + void enqueue_control(protocol::FrameType type, uint16_t channel_id, const std::vector& payload) { FrameBytes frame = protocol::encode_frame(type, channel_id, payload); @@ -386,11 +381,11 @@ class detail::RemotePluginSessionImpl const RemotePluginSession::Stats now = stats(); const RemotePluginSession::Stats& last = m_logged_stats; logger()->info( - "5s: sent hand={} schema={} wrist_rx={} hb={} | dropped queue={} | tx_bytes={} | offset_ns={} | " + "{}s: sent hand={} schema={} wrist_rx={} hb={} | dropped queue={} | tx_bytes={} | offset_ns={} | " "connected={:d}", - now.sent_hand - last.sent_hand, now.sent_schema - last.sent_schema, now.wrist_rx - last.wrist_rx, - now.heartbeats - last.heartbeats, now.dropped_queue - last.dropped_queue, now.tx_bytes - last.tx_bytes, - clock_offset_ns(), m_connected.load()); + kStatsPeriod.count(), now.sent_hand - last.sent_hand, now.sent_schema - last.sent_schema, + now.wrist_rx - last.wrist_rx, now.heartbeats - last.heartbeats, now.dropped_queue - last.dropped_queue, + now.tx_bytes - last.tx_bytes, clock_offset_ns(), m_connected.load()); m_logged_stats = now; } @@ -530,11 +525,11 @@ class detail::RemotePluginSessionImpl break; } case protocol::FrameType::Heartbeat: - case protocol::FrameType::HelloAck: break; default: - logger()->error( - "ignoring unexpected frame type {} (protocol mismatch)", static_cast(frame.header.type)); + // HELLO_ACK after the handshake included: the peers disagree on the protocol. + on_disconnect("unexpected frame type " + std::to_string(static_cast(frame.header.type)) + + " from the bridge"); break; } } @@ -588,14 +583,14 @@ namespace class RemoteHandTrackingPushChannel final : public core::IHandTrackingPushChannel { public: - RemoteHandTrackingPushChannel(std::shared_ptr impl, uint16_t channel_id) - : m_impl(std::move(impl)), m_channel_id(channel_id) + RemoteHandTrackingPushChannel(detail::RemotePluginSessionImpl& impl, uint16_t channel_id) + : m_impl(impl), m_channel_id(channel_id) { } ~RemoteHandTrackingPushChannel() override { - m_impl->close_channel(m_channel_id); + m_impl.close_channel(m_channel_id); } void push(const XrHandJointLocationEXT* joint_locations, int64_t sample_time_local_common_clock_ns) override @@ -604,30 +599,30 @@ class RemoteHandTrackingPushChannel final : public core::IHandTrackingPushChanne { throw std::invalid_argument("Hand tracking push requires joint locations"); } - if (!m_impl->connected()) + if (!m_impl.connected()) { return; } - m_impl->push_sample(m_channel_id, protocol::encode_hand_sample_frame( - m_channel_id, joint_locations, sample_time_local_common_clock_ns)); + m_impl.push_sample(m_channel_id, protocol::encode_hand_sample_frame( + m_channel_id, joint_locations, sample_time_local_common_clock_ns)); } private: - std::shared_ptr m_impl; + detail::RemotePluginSessionImpl& m_impl; uint16_t m_channel_id; }; class RemoteSchemaPushChannel final : public core::ISchemaPushChannel { public: - RemoteSchemaPushChannel(std::shared_ptr impl, uint16_t channel_id) - : m_impl(std::move(impl)), m_channel_id(channel_id) + RemoteSchemaPushChannel(detail::RemotePluginSessionImpl& impl, uint16_t channel_id) + : m_impl(impl), m_channel_id(channel_id) { } ~RemoteSchemaPushChannel() override { - m_impl->close_channel(m_channel_id); + m_impl.close_channel(m_channel_id); } void push_buffer(const uint8_t* buffer, @@ -639,45 +634,45 @@ class RemoteSchemaPushChannel final : public core::ISchemaPushChannel { throw std::invalid_argument("push_buffer: buffer is null for a non-empty sample"); } - if (!m_impl->connected()) + if (!m_impl.connected()) { return; } - m_impl->push_sample(m_channel_id, protocol::encode_schema_sample_frame(m_channel_id, buffer, size, - sample_time_local_common_clock_ns, - sample_time_raw_device_clock_ns)); + m_impl.push_sample(m_channel_id, protocol::encode_schema_sample_frame(m_channel_id, buffer, size, + sample_time_local_common_clock_ns, + sample_time_raw_device_clock_ns)); } private: - std::shared_ptr m_impl; + detail::RemotePluginSessionImpl& m_impl; uint16_t m_channel_id; }; class RemoteWristTrackingSource final : public core::IWristTrackingSource { public: - explicit RemoteWristTrackingSource(std::shared_ptr impl) : m_impl(std::move(impl)) + explicit RemoteWristTrackingSource(detail::RemotePluginSessionImpl& impl) : m_impl(impl) { } // Returns the latest bridge sample; the requested time is not interpolated. core::WristTrackingSample query(XrHandEXT hand, int64_t /*sample_time_local_common_clock_ns*/) override { - return m_impl->latest_wrist(hand); + return m_impl.latest_wrist(hand); } private: - std::shared_ptr m_impl; + detail::RemotePluginSessionImpl& m_impl; }; class RemotePluginPullChannel final : public core::IPluginPullChannel { public: - explicit RemotePluginPullChannel(std::shared_ptr impl) - : m_impl(std::move(impl)), m_channel_id(m_impl->open_channel(protocol::FrameType::OpenPull, {})) + explicit RemotePluginPullChannel(detail::RemotePluginSessionImpl& impl) + : m_impl(impl), m_channel_id(m_impl.open_channel(protocol::FrameType::OpenPull, {})) { // The bridge streams WRIST_SAMPLEs on this channel when HELLO declared a wrist source. - if (m_impl->requirements().wrist_tracking_pull) + if (m_impl.requirements().wrist_tracking_pull) { m_wrist_source.emplace(m_impl); } @@ -685,7 +680,7 @@ class RemotePluginPullChannel final : public core::IPluginPullChannel ~RemotePluginPullChannel() override { - m_impl->close_channel(m_channel_id); + m_impl.close_channel(m_channel_id); } // The receiver thread keeps the wrist cache current; nothing to poll here. @@ -709,36 +704,23 @@ class RemotePluginPullChannel final : public core::IPluginPullChannel } private: - std::shared_ptr m_impl; + detail::RemotePluginSessionImpl& m_impl; uint16_t m_channel_id; std::optional m_wrist_source; }; } // namespace -RemotePluginSession::RemotePluginSession(std::string host, - uint16_t port, - std::string app_name, - core::PluginSessionRequirements requirements) - : RemotePluginSession( - std::make_unique(TcpSocket::connect(Endpoint{ std::move(host), port }, kConnectTimeout)), - std::move(app_name), - requirements) -{ -} - RemotePluginSession::RemotePluginSession(std::unique_ptr transport, std::string app_name, core::PluginSessionRequirements requirements) - : m_impl(std::make_shared(std::move(transport), app_name, requirements)), - m_pull_channel(std::make_unique(m_impl)) + : m_impl(std::make_unique(std::move(transport), app_name, requirements)), + m_pull_channel(std::make_unique(*m_impl)) { } -RemotePluginSession::~RemotePluginSession() -{ - m_impl->shutdown(); -} +// The pull channel closes first, then ~RemotePluginSessionImpl shuts the connection down. +RemotePluginSession::~RemotePluginSession() = default; core::IPluginPullChannel& RemotePluginSession::pull_channel() { @@ -754,7 +736,7 @@ std::unique_ptr RemotePluginSession::create_schema_pus } const uint16_t channel_id = m_impl->open_channel(protocol::FrameType::OpenSchema, protocol::encode_open_schema(config)); - return std::make_unique(m_impl, channel_id); + return std::make_unique(*m_impl, channel_id); } std::unique_ptr RemotePluginSession::create_hand_tracking_push_channel(XrHandEXT hand) @@ -764,7 +746,7 @@ std::unique_ptr RemotePluginSession::create_hand throw std::logic_error("RemotePluginSession: hand-tracking push was not declared in PluginSessionRequirements"); } const uint16_t channel_id = m_impl->open_channel(protocol::FrameType::OpenHand, protocol::encode_open_hand(hand)); - return std::make_unique(m_impl, channel_id); + return std::make_unique(*m_impl, channel_id); } bool RemotePluginSession::connected() const diff --git a/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp b/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp index c7cfe06fe4..4170950c6b 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp @@ -65,6 +65,7 @@ TEST_CASE("ClockOffsetEstimator rejects an empty window", "[remote_device_bridge { REQUIRE_THROWS_AS(ClockOffsetEstimator(0, 1), std::invalid_argument); REQUIRE_THROWS_AS(ClockOffsetEstimator(1, 0), std::invalid_argument); + REQUIRE_THROWS_AS(ClockOffsetEstimator(1, 1, 0), std::invalid_argument); } TEST_CASE("should_drop_stale keeps fresh samples and drops the backlog after a stall", "[remote_device_bridge][unit]") diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp index 0a5a07545f..055cadefde 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -326,13 +326,18 @@ bool wait_until(const std::function& predicate, std::chrono::millisecond return true; } +std::unique_ptr connect_to(uint16_t port) +{ + return std::make_unique(TcpSocket::connect(Endpoint{ "127.0.0.1", port }, kConnectTimeout)); +} + } // namespace TEST_CASE("RemotePluginSession performs the HELLO handshake", "[remote_device_bridge][unit]") { FakeBridge bridge; RemotePluginSession session( - "127.0.0.1", bridge.port(), "test_app", { .schema_push = true, .hand_tracking_push = true }); + connect_to(bridge.port()), "test_app", { .schema_push = true, .hand_tracking_push = true }); REQUIRE(session.connected()); const auto hello = bridge.wait_for(protocol::FrameType::Hello); @@ -353,7 +358,7 @@ TEST_CASE("RemotePluginSession fails fast when the bridge refuses or is absent", FakeBridge bridge(std::string("no OpenXR runtime")); try { - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", {}); + RemotePluginSession session(connect_to(bridge.port()), "test_app", {}); FAIL("expected std::runtime_error"); } catch (const std::runtime_error& e) @@ -366,7 +371,7 @@ TEST_CASE("RemotePluginSession fails fast when the bridge refuses or is absent", TcpSocket probe = TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 1); const uint16_t free_port = probe.local_port(); probe.close(); - REQUIRE_THROWS_AS(RemotePluginSession("127.0.0.1", free_port, "test_app", {}), std::runtime_error); + REQUIRE_THROWS_AS(RemotePluginSession(connect_to(free_port), "test_app", {}), std::runtime_error); } SECTION("a null transport is rejected") { @@ -390,7 +395,7 @@ TEST_CASE("RemotePluginSession enforces declared requirements", "[remote_device_ { FakeBridge bridge; RemotePluginSession session( - "127.0.0.1", bridge.port(), "test_app", { .schema_push = false, .hand_tracking_push = true }); + connect_to(bridge.port()), "test_app", { .schema_push = false, .hand_tracking_push = true }); REQUIRE_THROWS_AS(session.create_schema_push_channel(core::SchemaPusherConfig{}), std::logic_error); core::IPluginPullChannel& pull = session.pull_channel(); @@ -404,7 +409,7 @@ TEST_CASE("RemotePluginSession rejects hand-tracking push when it was not declar { FakeBridge bridge; RemotePluginSession session( - "127.0.0.1", bridge.port(), "test_app", { .schema_push = true, .hand_tracking_push = false }); + connect_to(bridge.port()), "test_app", { .schema_push = true, .hand_tracking_push = false }); REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT), std::logic_error); REQUIRE_FALSE(bridge.wait_for(protocol::FrameType::OpenHand, std::chrono::milliseconds(200)).has_value()); @@ -413,7 +418,7 @@ TEST_CASE("RemotePluginSession rejects hand-tracking push when it was not declar TEST_CASE("RemotePluginSession forwards OPEN_HAND, HAND_SAMPLE and CLOSE bit-exact", "[remote_device_bridge][unit]") { FakeBridge bridge; - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true }); + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .hand_tracking_push = true }); uint16_t channel_id = 0; { @@ -447,7 +452,7 @@ TEST_CASE("RemotePluginSession forwards OPEN_HAND, HAND_SAMPLE and CLOSE bit-exa TEST_CASE("RemotePluginSession forwards schema channels", "[remote_device_bridge][unit]") { FakeBridge bridge; - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .schema_push = true }); + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .schema_push = true }); const core::SchemaPusherConfig config{ .collection_id = "collection", .max_flatbuffer_size = 64, @@ -478,7 +483,7 @@ TEST_CASE("RemotePluginSession exposes bridge wrist samples through the pull cha config.mode = core::WristTrackingSourceMode::HandTracking; config.left_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.5f, 0.0f, 0.0f } }; config.right_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .wrist_tracking_pull = config }); + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .wrist_tracking_pull = config }); const auto hello = bridge.wait_for(protocol::FrameType::Hello); REQUIRE(hello.has_value()); @@ -515,7 +520,7 @@ TEST_CASE("RemotePluginSession exposes bridge wrist samples through the pull cha TEST_CASE("RemotePluginSession answers every PING with a PONG", "[remote_device_bridge][unit]") { FakeBridge bridge; - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true }); + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .hand_tracking_push = true }); REQUIRE(bridge.wait_for(protocol::FrameType::Hello).has_value()); for (int64_t t1 : { 111LL, 222LL, 333LL }) @@ -540,7 +545,7 @@ TEST_CASE("RemotePluginSession keeps the session when the bridge refuses one cha { FakeBridge bridge; RemotePluginSession session( - "127.0.0.1", bridge.port(), "test_app", { .schema_push = true, .hand_tracking_push = true }); + connect_to(bridge.port()), "test_app", { .schema_push = true, .hand_tracking_push = true }); core::SchemaPusherConfig config; config.collection_id = "tactile"; config.max_flatbuffer_size = 40960; @@ -587,11 +592,10 @@ TEST_CASE("RemotePluginSession keeps the session when the bridge refuses one cha TEST_CASE("RemotePluginSession survives the bridge dropping the socket", "[remote_device_bridge][unit]") { FakeBridge bridge; - std::unique_ptr channel; { - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", - { .hand_tracking_push = true, .wrist_tracking_pull = kWristConfig }); - channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + RemotePluginSession session( + connect_to(bridge.port()), "test_app", { .hand_tracking_push = true, .wrist_tracking_pull = kWristConfig }); + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); REQUIRE(bridge.wait_for(protocol::FrameType::OpenHand).has_value()); bridge.drop_connection(); @@ -601,10 +605,8 @@ TEST_CASE("RemotePluginSession survives the bridge dropping the socket", "[remot REQUIRE_NOTHROW(channel->push(joints.data(), 1)); REQUIRE_NOTHROW(session.pull_channel().update()); REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT), std::runtime_error); + REQUIRE_NOTHROW(channel.reset()); } - // Channels outliving a dead session must still be destroyable. - REQUIRE_NOTHROW(channel->push(make_joints(0.5f).data(), 2)); - REQUIRE_NOTHROW(channel.reset()); } TEST_CASE("RemotePluginSession never reconnects after the link drops", "[remote_device_bridge][unit]") @@ -622,7 +624,7 @@ TEST_CASE("RemotePluginSession never reconnects after the link drops", "[remote_ protocol::encode_hello_ack(protocol::HelloAck{ true, "" })); } }); - RemotePluginSession session("127.0.0.1", listener.local_port(), "test_app", { .hand_tracking_push = true }); + RemotePluginSession session(connect_to(listener.local_port()), "test_app", { .hand_tracking_push = true }); accept_once.join(); REQUIRE(bridge_side); auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); @@ -644,7 +646,7 @@ TEST_CASE("RemotePluginSession closes remaining channels and the socket on destr FakeBridge bridge; std::unique_ptr channel; { - RemotePluginSession session("127.0.0.1", bridge.port(), "test_app", { .hand_tracking_push = true }); + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .hand_tracking_push = true }); channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); REQUIRE(bridge.wait_for(protocol::FrameType::OpenHand).has_value()); } @@ -737,7 +739,7 @@ TEST_CASE("RemotePluginSession drops a channel's pending samples when the channe auto filler = session.create_schema_push_channel(filler_config); REQUIRE(link.wait_for(protocol::FrameType::OpenSchema).has_value()); - // Stall the sender on a filler sample. A paused TCP peer is not enough: with 8 MiB socket buffers the kernel + // Stall the sender on a filler sample. A paused TCP peer is not enough: with enlarged socket buffers the kernel // absorbs the filler and the hand samples leave before the CLOSE. link.hold(true); const std::vector filler_bytes = { 0xCD }; From a152b678075baf7ff17ac00f881c909fd7d2a1e5 Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Fri, 30 Oct 2026 11:04:53 -0700 Subject: [PATCH 05/10] review round 3 (pr04) Applied the fact-checked (VERIFIED) findings on code this branch adds: - A test destroyed a push channel after its session (use after free); channels now close inside the session's scope. - shutdown() runs once by construction: dropped the m_shutdown guard and the CLOSE sweep for channels that, by contract, are already closed. - push_sample drops a connected check its callers already make. - The stats snapshot is a local of the sender loop. - app_name is moved through to HELLO. - ClockOffsetEstimator::offset_source returns an enum. - The handshake test checks app_name; the duplicate transport test is gone. - AGENTS.md no longer describes bridge flags that are not in this branch. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- src/plugins/remote_device_bridge/AGENTS.md | 4 +- .../session/clock_offset_estimator.cpp | 4 +- .../clock_offset_estimator.hpp | 9 ++++- .../session/remote_plugin_session.cpp | 40 +++++-------------- .../test_clock_offset_estimator.cpp | 4 +- .../test_remote_plugin_session.cpp | 19 ++------- 6 files changed, 25 insertions(+), 55 deletions(-) diff --git a/src/plugins/remote_device_bridge/AGENTS.md b/src/plugins/remote_device_bridge/AGENTS.md index 53cbf38fb3..05fd34192b 100644 --- a/src/plugins/remote_device_bridge/AGENTS.md +++ b/src/plugins/remote_device_bridge/AGENTS.md @@ -12,6 +12,6 @@ SPDX-License-Identifier: Apache-2.0 - A frame's `t_send` is its send time for clock exchange: call `stamp_header` where the frame is written to the transport, never when it is queued. - WebSocket is a second transport, not a second protocol: `WebSocketTransport` carries exactly one wire frame per binary message (no TLS; terminate `wss://` in a proxy). Clients pick it with a `ws://host:port[/path]` address via `parse_remote`/`connect_transport`. A server serves it on its own port; never sniff the protocol on the TCP port. The server-side HTTP upgrade runs in the first `receive()` so an accept loop never blocks on a client. - Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to `Teleop::remote_plugin_session`: a plugin that talks to a bridge through this library must run without an OpenXR runtime. -- ERROR on channel 0 ends the session; ERROR on a channel id refuses only that OPEN (schema above `--max-schema-bytes`, a hand someone else owns) and the session and its other channels continue. Refuse per channel whenever the session itself is still sound. +- ERROR on channel 0 ends the session; ERROR on a channel id refuses only that OPEN and the session and its other channels continue. Refuse per channel whenever the session itself is still sound. - `RemotePluginSession` never throws on the sample path for transport failures (`push`, `push_buffer`, `query`, `update`): after one they become no-ops; only channel creation and the constructor throw. Caller bugs (null joints or buffer) still throw `std::invalid_argument`. -- Timestamps cross the wire in the sender's `CLOCK_MONOTONIC`; the receiver maps them with `ClockOffsetEstimator`. The bridge's offset comes from PING/PONG (min-RTT of the last 32 exchanges, `--clock-source=two-way`); the one-way minimum of `local_recv - remote_send` hides the minimum network delay and is only the fallback (no PONG within 2 s) or the `--clock-source=one-way` A/B baseline. Do not push raw remote timestamps into OpenXR. +- Timestamps cross the wire in the sender's `CLOCK_MONOTONIC`; the receiver maps them with `ClockOffsetEstimator`, which prefers PING/PONG exchanges (minimum RTT) over the one-way minimum of `local_recv - remote_send`; the one-way minimum hides the minimum network delay. Do not push raw remote timestamps into OpenXR. diff --git a/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp index 86c5652dd1..8e964fe51f 100644 --- a/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp +++ b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp @@ -78,9 +78,9 @@ int64_t ClockOffsetEstimator::rtt_last_ns() const return m_exchanges.empty() ? 0 : m_exchanges.back().rtt_ns; } -const char* ClockOffsetEstimator::offset_source() const +ClockOffsetEstimator::Source ClockOffsetEstimator::offset_source() const { - return m_exchanges.empty() ? "one-way" : "two-way"; + return m_exchanges.empty() ? Source::OneWay : Source::TwoWay; } PingScheduler::PingScheduler(int64_t start_ns) : m_start_ns(start_ns), m_next_ns(start_ns) diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp index c90f49fe51..3be8f91b3d 100644 --- a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp @@ -23,6 +23,12 @@ namespace remote_plugin_session class ClockOffsetEstimator { public: + enum class Source + { + OneWay, + TwoWay, //!< once an exchange was added + }; + //! One-way: at most `max_samples` within `window_ns`; two-way: the last `two_way_window` exchanges. explicit ClockOffsetEstimator(size_t max_samples = 200, int64_t window_ns = 5'000'000'000, size_t two_way_window = 32); @@ -36,8 +42,7 @@ class ClockOffsetEstimator //! Zero until the first two-way exchange. int64_t rtt_min_ns() const; int64_t rtt_last_ns() const; - //! "two-way" once an exchange was added, else "one-way". - const char* offset_source() const; + Source offset_source() const; private: struct Sample diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp index ff3e288106..53d9a2f3b0 100644 --- a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -71,7 +71,7 @@ class detail::RemotePluginSessionImpl { public: RemotePluginSessionImpl(std::unique_ptr transport, - const std::string& app_name, + std::string app_name, core::PluginSessionRequirements requirements) : m_requirements(requirements), m_transport(std::move(transport)) { @@ -81,7 +81,7 @@ class detail::RemotePluginSessionImpl } protocol::Hello hello; - hello.app_name = app_name; + hello.app_name = std::move(app_name); hello.requirements = requirements; FrameBytes hello_frame = protocol::encode_frame( protocol::FrameType::Hello, protocol::kSessionChannelId, protocol::encode_hello(hello)); @@ -222,10 +222,6 @@ class detail::RemotePluginSessionImpl //! Per-channel latest-wins queue; control frames are never dropped. void push_sample(uint16_t channel_id, FrameBytes frame) { - if (!m_connected.load()) - { - return; - } { std::lock_guard lock(m_queue_mutex); const auto it = m_sample_queues.find(channel_id); @@ -258,24 +254,8 @@ class detail::RemotePluginSessionImpl void shutdown() { - if (m_shutdown.exchange(true)) - { - return; - } - - std::set remaining; - { - std::lock_guard lock(m_state_mutex); - remaining.swap(m_open_channels); - } - if (m_connected.load()) - { - for (auto it = remaining.rbegin(); it != remaining.rend(); ++it) - { - enqueue_control(protocol::FrameType::Close, *it, {}); - } - } - + // Runs once: from the destructor, or from a constructor that throws (and so never destructs). Every + // channel has already sent its CLOSE, as IPluginSession has the owner destroy channels first. { std::unique_lock lock(m_queue_mutex); m_stop = true; @@ -376,17 +356,16 @@ class detail::RemotePluginSessionImpl return batch; } - void log_stats() + void log_stats(RemotePluginSession::Stats& last) { const RemotePluginSession::Stats now = stats(); - const RemotePluginSession::Stats& last = m_logged_stats; logger()->info( "{}s: sent hand={} schema={} wrist_rx={} hb={} | dropped queue={} | tx_bytes={} | offset_ns={} | " "connected={:d}", kStatsPeriod.count(), now.sent_hand - last.sent_hand, now.sent_schema - last.sent_schema, now.wrist_rx - last.wrist_rx, now.heartbeats - last.heartbeats, now.dropped_queue - last.dropped_queue, now.tx_bytes - last.tx_bytes, clock_offset_ns(), m_connected.load()); - m_logged_stats = now; + last = now; } void on_disconnect(const std::string& reason) @@ -421,6 +400,7 @@ class detail::RemotePluginSessionImpl { auto last_send = std::chrono::steady_clock::now(); auto next_stats = last_send + kStatsPeriod; + RemotePluginSession::Stats logged_stats; while (true) { std::vector batch = next_batch(std::min(last_send + kHeartbeatPeriod, next_stats)); @@ -449,7 +429,7 @@ class detail::RemotePluginSessionImpl } if (now >= next_stats) { - log_stats(); + log_stats(logged_stats); next_stats += kStatsPeriod; } } @@ -544,7 +524,6 @@ class detail::RemotePluginSessionImpl std::atomic m_connected{ false }; std::atomic m_stop{ false }; - std::atomic m_shutdown{ false }; std::mutex m_state_mutex; uint16_t m_next_channel_id = 1; @@ -565,7 +544,6 @@ class detail::RemotePluginSessionImpl std::atomic m_wrist_rx{ 0 }; std::atomic m_heartbeats{ 0 }; std::atomic m_tx_bytes{ 0 }; - RemotePluginSession::Stats m_logged_stats; // sender thread only mutable std::mutex m_wrist_mutex; core::WristTrackingSample m_wrist[2]; @@ -714,7 +692,7 @@ class RemotePluginPullChannel final : public core::IPluginPullChannel RemotePluginSession::RemotePluginSession(std::unique_ptr transport, std::string app_name, core::PluginSessionRequirements requirements) - : m_impl(std::make_unique(std::move(transport), app_name, requirements)), + : m_impl(std::make_unique(std::move(transport), std::move(app_name), requirements)), m_pull_channel(std::make_unique(*m_impl)) { } diff --git a/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp b/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp index 4170950c6b..5eb23ed303 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp @@ -104,11 +104,11 @@ TEST_CASE("Two-way exchanges recover the offset the one-way minimum biases", "[r // One-way: the fastest frame took 70 ms, so the one-way estimate is 70 ms too large. estimator.add_sample(1'000 * kMs, 1'000 * kMs + kOffset + 70 * kMs); REQUIRE(estimator.offset_ns() == kOffset + 70 * kMs); - REQUIRE(std::string(estimator.offset_source()) == "one-way"); + REQUIRE(estimator.offset_source() == ClockOffsetEstimator::Source::OneWay); REQUIRE(estimator.rtt_min_ns() == 0); exchange(estimator, 2'000 * kMs, kOffset, 70 * kMs, 70 * kMs); - REQUIRE(std::string(estimator.offset_source()) == "two-way"); + REQUIRE(estimator.offset_source() == ClockOffsetEstimator::Source::TwoWay); REQUIRE(estimator.offset_ns() == kOffset); REQUIRE(estimator.rtt_last_ns() == 140 * kMs); REQUIRE(estimator.rtt_min_ns() == 140 * kMs); diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp index 055cadefde..6b938a47b3 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -344,6 +344,7 @@ TEST_CASE("RemotePluginSession performs the HELLO handshake", "[remote_device_br REQUIRE(hello.has_value()); REQUIRE(hello->header.channel_id == protocol::kSessionChannelId); REQUIRE(hello->header.seq == 0); + REQUIRE(protocol::decode_hello(hello->payload).app_name == "test_app"); const protocol::Hello decoded = protocol::decode_hello(hello->payload); REQUIRE(decoded.app_name == "test_app"); REQUIRE(decoded.requirements.schema_push); @@ -379,18 +380,6 @@ TEST_CASE("RemotePluginSession fails fast when the bridge refuses or is absent", } } -TEST_CASE("RemotePluginSession accepts an already connected transport", "[remote_device_bridge][unit]") -{ - FakeBridge bridge; - RemotePluginSession session( - std::make_unique(TcpSocket::connect(Endpoint{ "127.0.0.1", bridge.port() }, kWait)), - "via_transport", { .hand_tracking_push = true }); - REQUIRE(session.connected()); - const auto hello = bridge.wait_for(protocol::FrameType::Hello); - REQUIRE(hello.has_value()); - REQUIRE(protocol::decode_hello(hello->payload).app_name == "via_transport"); -} - TEST_CASE("RemotePluginSession enforces declared requirements", "[remote_device_bridge][unit]") { FakeBridge bridge; @@ -641,18 +630,16 @@ TEST_CASE("RemotePluginSession never reconnects after the link drops", "[remote_ REQUIRE(TcpTransport::accept(listener, std::chrono::milliseconds(500)) == nullptr); } -TEST_CASE("RemotePluginSession closes remaining channels and the socket on destruction", "[remote_device_bridge][unit]") +TEST_CASE("RemotePluginSession closes its channels and the socket on destruction", "[remote_device_bridge][unit]") { FakeBridge bridge; - std::unique_ptr channel; { RemotePluginSession session(connect_to(bridge.port()), "test_app", { .hand_tracking_push = true }); - channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); REQUIRE(bridge.wait_for(protocol::FrameType::OpenHand).has_value()); } REQUIRE(bridge.wait_for(protocol::FrameType::Close).has_value()); REQUIRE(bridge.client_closed()); - REQUIRE_NOTHROW(channel.reset()); } TEST_CASE("RemotePluginSession keeps the newest samples per channel while the link is stalled", From 68ee0192d26c4b2d08db2b62d18b94ca63d70965 Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Wed, 18 Nov 2026 15:22:38 -0800 Subject: [PATCH 06/10] refactor(remote_device_bridge): pass sample buffers to the codec as spans (pr04) Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- .../session/remote_plugin_session.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp index 53d9a2f3b0..c8fa263db2 100644 --- a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -582,7 +583,8 @@ class RemoteHandTrackingPushChannel final : public core::IHandTrackingPushChanne return; } m_impl.push_sample(m_channel_id, protocol::encode_hand_sample_frame( - m_channel_id, joint_locations, sample_time_local_common_clock_ns)); + m_channel_id, protocol::HandJoints(joint_locations, XR_HAND_JOINT_COUNT_EXT), + sample_time_local_common_clock_ns)); } private: @@ -616,9 +618,9 @@ class RemoteSchemaPushChannel final : public core::ISchemaPushChannel { return; } - m_impl.push_sample(m_channel_id, protocol::encode_schema_sample_frame(m_channel_id, buffer, size, - sample_time_local_common_clock_ns, - sample_time_raw_device_clock_ns)); + m_impl.push_sample(m_channel_id, protocol::encode_schema_sample_frame( + m_channel_id, std::span(buffer, size), + sample_time_local_common_clock_ns, sample_time_raw_device_clock_ns)); } private: From cf020d1abae1375f28b5a20451a8c49b363bcbde Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Wed, 18 Nov 2026 15:22:43 -0800 Subject: [PATCH 07/10] refactor(remote_device_bridge): follow the sample_time_local_common_clock_ns rename (pr04) Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- .../remote_device_bridge/session/remote_plugin_session.cpp | 4 ++-- .../remote_device_bridge/test_remote_plugin_session.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp index c8fa263db2..b8be2f23ed 100644 --- a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -478,8 +478,8 @@ class detail::RemotePluginSessionImpl { case protocol::FrameType::WristSample: { - // sample_time_ns is dropped: core::WristTrackingSample has no timestamp field and queries return - // the latest sample. + // sample_time_local_common_clock_ns is dropped: core::WristTrackingSample has no timestamp field and + // queries return the latest sample. const protocol::WristSample sample = protocol::decode_wrist_sample(frame.payload); m_wrist_rx.fetch_add(1); std::lock_guard lock(m_wrist_mutex); diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp index 6b938a47b3..2d54922845 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -425,7 +425,7 @@ TEST_CASE("RemotePluginSession forwards OPEN_HAND, HAND_SAMPLE and CLOSE bit-exa REQUIRE(sample.has_value()); REQUIRE(sample->header.channel_id == channel_id); const protocol::HandSample decoded = protocol::decode_hand_sample(sample->payload); - REQUIRE(decoded.sample_time_ns == 123456789LL); + REQUIRE(decoded.sample_time_local_common_clock_ns == 123456789LL); for (size_t i = 0; i < joints.size(); ++i) { REQUIRE(std::memcmp(&decoded.joints[i], &joints[i], sizeof(XrHandJointLocationEXT)) == 0); @@ -491,7 +491,7 @@ TEST_CASE("RemotePluginSession exposes bridge wrist samples through the pull cha REQUIRE_FALSE(before.tracked); protocol::WristSample wrist; - wrist.sample_time_ns = 1000; + wrist.sample_time_local_common_clock_ns = 1000; wrist.hand = XR_HAND_LEFT_EXT; wrist.sample.valid = true; wrist.sample.tracked = true; @@ -701,7 +701,7 @@ TEST_CASE("RemotePluginSession keeps the newest samples per channel while the li continue; } (frame->header.channel_id == fast_id ? fast_times : slow_times) - .push_back(protocol::decode_hand_sample(frame->payload).sample_time_ns); + .push_back(protocol::decode_hand_sample(frame->payload).sample_time_local_common_clock_ns); } // Each channel delivered its newest sample, in order, from a queue that never held more than 4. From 5143ca66e19ed8e980ad2ba736a1a03ad73ec728 Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Wed, 18 Nov 2026 15:23:27 -0800 Subject: [PATCH 08/10] feat(remote_device_bridge)!: answer PING from header send times (pr04) Wire v1: t1 is the PING header t_send and the PONG's send time (t3) is the PONG header t_send, stamped by send_now() when the sender thread writes the frame, so a PONG that waited in the queue still reports when it left. The clock offset and RTT use the same four instants as before. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- .../session/remote_plugin_session.cpp | 17 ++---- .../test_remote_plugin_session.cpp | 56 ++++++++++++++++--- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp index b8be2f23ed..78cc56bd2a 100644 --- a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -298,17 +298,12 @@ class detail::RemotePluginSessionImpl m_queue_cv.notify_one(); } - //! Stamps seq/t_send (and a PONG's pong_sent_ns) and sends one frame. Constructor thread during the handshake, - //! sender thread after. + //! Stamps seq/t_send and sends one frame; t_send is taken here, as the frame is written, so a PONG that waited + //! in the queue still reports when it left. Constructor thread during the handshake, sender thread after. bool send_now(FrameBytes& frame) { - const int64_t t_send_ns = core::os_monotonic_now_ns(); - protocol::stamp_header(frame, m_next_seq++, t_send_ns); + protocol::stamp_header(frame, m_next_seq++, core::os_monotonic_now_ns()); const protocol::FrameType type = protocol::decode_header(frame).type; - if (type == protocol::FrameType::Pong) - { - protocol::set_pong_sent_time(frame, t_send_ns); - } if (!m_transport->send(frame.data(), frame.size())) { return false; @@ -488,10 +483,10 @@ class detail::RemotePluginSessionImpl } case protocol::FrameType::Ping: { - // Queued as control (never dropped); send_now() sets pong_sent_ns when the sender thread sends it. - const int64_t t1_ns = protocol::decode_ping(frame.payload); + // The PING's send time is its header t_send. Queued as control (never dropped); the PONG's own send + // time is stamped by send_now() when the sender thread writes it. enqueue_control(protocol::FrameType::Pong, protocol::kSessionChannelId, - protocol::encode_pong(protocol::Pong{ t1_ns, t_recv_ns, 0 })); + protocol::encode_pong(protocol::Pong{ frame.header.t_send_monotonic_ns, t_recv_ns })); break; } case protocol::FrameType::Error: diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp index 2d54922845..da326915d9 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -256,6 +256,14 @@ class BlockingTransport final : public ITransport m_cv.notify_all(); } + //! Hands `frame` to the session as if the bridge had sent it. + void deliver(std::vector frame) + { + std::lock_guard lock(m_mutex); + m_inbox.push_back(std::move(frame)); + m_cv.notify_all(); + } + //! True while a send() is waiting for hold(false). bool blocked() const { @@ -512,24 +520,56 @@ TEST_CASE("RemotePluginSession answers every PING with a PONG", "[remote_device_ RemotePluginSession session(connect_to(bridge.port()), "test_app", { .hand_tracking_push = true }); REQUIRE(bridge.wait_for(protocol::FrameType::Hello).has_value()); - for (int64_t t1 : { 111LL, 222LL, 333LL }) + for (int i = 0; i < 3; ++i) { const int64_t before = core::os_monotonic_now_ns(); - REQUIRE(bridge.send(protocol::FrameType::Ping, protocol::kSessionChannelId, protocol::encode_ping(t1))); + REQUIRE(bridge.send(protocol::FrameType::Ping, protocol::kSessionChannelId, {})); const auto frame = bridge.wait_for(protocol::FrameType::Pong); const int64_t after = core::os_monotonic_now_ns(); REQUIRE(frame.has_value()); REQUIRE(frame->header.channel_id == protocol::kSessionChannelId); const protocol::Pong pong = protocol::decode_pong(frame->payload); - REQUIRE(pong.ping_sent_ns == t1); - // Same host, same clock: before <= ping_received <= pong_sent == header t_send <= after. - REQUIRE(before <= pong.ping_received_ns); - REQUIRE(pong.ping_received_ns <= pong.pong_sent_ns); - REQUIRE(pong.pong_sent_ns == frame->header.t_send_monotonic_ns); - REQUIRE(pong.pong_sent_ns <= after); + // Same host, same clock: before <= ping_sent (the PING header t_send) <= ping_received <= the PONG header + // t_send <= after. + REQUIRE(before <= pong.ping_sent_ns); + REQUIRE(pong.ping_sent_ns <= pong.ping_received_ns); + REQUIRE(pong.ping_received_ns <= frame->header.t_send_monotonic_ns); + REQUIRE(frame->header.t_send_monotonic_ns <= after); } } +TEST_CASE("A PONG that waited in the send queue reports when it was sent, not when it was queued", + "[remote_device_bridge][unit]") +{ + auto transport = std::make_unique(); + BlockingTransport& link = *transport; + RemotePluginSession session(std::move(transport), "test_app", { .hand_tracking_push = true }); + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + const auto joints = make_joints(0.1f); + + // Stall the link: the sender thread blocks inside send() on this sample. + link.hold(true); + channel->push(joints.data(), 1); + REQUIRE(wait_until([&] { return link.blocked(); })); + + std::vector ping = protocol::encode_frame(protocol::FrameType::Ping, protocol::kSessionChannelId, {}); + const int64_t ping_sent_ns = core::os_monotonic_now_ns(); + protocol::stamp_header(ping, 1, ping_sent_ns); + link.deliver(std::move(ping)); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // the PONG is queued behind the stalled send + + const int64_t released_ns = core::os_monotonic_now_ns(); + link.hold(false); + const auto frame = link.wait_for(protocol::FrameType::Pong); + REQUIRE(frame.has_value()); + const protocol::Pong pong = protocol::decode_pong(frame->payload); + REQUIRE(pong.ping_sent_ns == ping_sent_ns); + REQUIRE(pong.ping_received_ns >= ping_sent_ns); + REQUIRE(pong.ping_received_ns < released_ns); + // The send time is when the PONG left after the stall, so t3 - t2 covers the time it waited in the queue. + REQUIRE(frame->header.t_send_monotonic_ns >= released_ns); +} + TEST_CASE("RemotePluginSession keeps the session when the bridge refuses one channel", "[remote_device_bridge][unit]") { FakeBridge bridge; From 3e15f8dc6609957bbe0d14cabdb02a5f41758460 Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Wed, 18 Nov 2026 15:23:32 -0800 Subject: [PATCH 09/10] build(remote_device_bridge): option text names the RemotePluginSession library (pr04) Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d61f8f704e..d23153b804 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,7 @@ option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF) option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF) option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF) -option(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE "Build the remote device bridge wire codec and TCP/WebSocket transports (Teleop::remote_plugin_session; POSIX sockets, skipped on Windows)" ON) +option(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE "Build the RemotePluginSession library for remote device bridge clients (Teleop::remote_plugin_session: wire codec, TCP/WebSocket transports, session; POSIX sockets, skipped on Windows)" ON) option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) option(BUILD_TESTING "Build unit tests" ON) From e4400225dfd543c9f4d9a95ea29da995d69db417 Mon Sep 17 00:00:00 2001 From: Jonghyun Kim Date: Wed, 18 Nov 2026 15:29:09 -0800 Subject: [PATCH 10/10] feat(remote_device_bridge): name the protocol version when HELLO_ACK cannot be decoded (pr04) A bridge built from the previous wire format answers with a version-0 frame; the constructor now throws "undecodable HELLO_ACK: protocol: peer speaks protocol version 0, this build speaks version 1; ..." instead of letting the bare DecodeError escape. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jonghyun Kim --- .../session/remote_plugin_session.cpp | 11 ++++++++++- .../test_remote_plugin_session.cpp | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp index 78cc56bd2a..a57cfb3956 100644 --- a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -99,7 +99,16 @@ class detail::RemotePluginSessionImpl "RemotePluginSession: timed out waiting for HELLO_ACK" : "RemotePluginSession: connection lost while waiting for HELLO_ACK"); } - const protocol::Frame frame = protocol::split_frame(std::move(bytes)); + protocol::Frame frame; + try + { + frame = protocol::split_frame(std::move(bytes)); + } + catch (const protocol::DecodeError& e) + { + // A bridge on another protocol version lands here, with both versions in the reason. + throw std::runtime_error(std::string("RemotePluginSession: undecodable HELLO_ACK: ") + e.what()); + } if (frame.header.type != protocol::FrameType::HelloAck) { throw std::runtime_error("RemotePluginSession: expected HELLO_ACK, received frame type " + diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp index da326915d9..31b99f7b57 100644 --- a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #include +#include #include #include #include @@ -204,10 +205,12 @@ class FakeBridge class BlockingTransport final : public ITransport { public: - BlockingTransport() + //! `ack_version` is the protocol version in the HELLO_ACK header, to stand in for a bridge of another version. + explicit BlockingTransport(uint8_t ack_version = protocol::kProtocolVersion) { m_inbox.push_back(protocol::encode_frame(protocol::FrameType::HelloAck, protocol::kSessionChannelId, protocol::encode_hello_ack(protocol::HelloAck{ true, "" }))); + m_inbox.back()[protocol::kVersionOffset] = ack_version; protocol::stamp_header(m_inbox.back(), 0, core::os_monotonic_now_ns()); } @@ -375,6 +378,12 @@ TEST_CASE("RemotePluginSession fails fast when the bridge refuses or is absent", REQUIRE(std::string(e.what()).find("no OpenXR runtime") != std::string::npos); } } + SECTION("a bridge on protocol version 0 is named in the error") + { + REQUIRE_THROWS_WITH(RemotePluginSession(std::make_unique(0), "test_app", {}), + Catch::Matchers::ContainsSubstring("undecodable HELLO_ACK: protocol: peer speaks protocol " + "version 0, this build speaks version 1")); + } SECTION("nothing listening throws") { TcpSocket probe = TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 1);