diff --git a/CMakeLists.txt b/CMakeLists.txt index d61f8f704e..d23153b804 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,7 @@ option(BUILD_PLUGIN_NOITOM_MOCAP "Build Noitom mocap plugin (downloads MocapApi option(BUILD_PLUGIN_OGLO "Build OGLO tactile glove plugin (BLE, Linux only; fetches SimpleBLE + nlohmann/json)" OFF) option(BUILD_PLUGIN_WUJI_GLOVE "Build Wuji glove plugin (requires the wuji_sdk C SDK)" OFF) option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; skips if the Avatar SDK is missing)" OFF) -option(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE "Build the remote device bridge wire codec and TCP/WebSocket transports (Teleop::remote_plugin_session; POSIX sockets, skipped on Windows)" ON) +option(BUILD_PLUGIN_REMOTE_DEVICE_BRIDGE "Build the RemotePluginSession library for remote device bridge clients (Teleop::remote_plugin_session: wire codec, TCP/WebSocket transports, session; POSIX sockets, skipped on Windows)" ON) option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) option(BUILD_TESTING "Build unit tests" ON) diff --git a/src/plugins/remote_device_bridge/AGENTS.md b/src/plugins/remote_device_bridge/AGENTS.md index 70b309a38d..05fd34192b 100644 --- a/src/plugins/remote_device_bridge/AGENTS.md +++ b/src/plugins/remote_device_bridge/AGENTS.md @@ -12,3 +12,6 @@ SPDX-License-Identifier: Apache-2.0 - A frame's `t_send` is its send time for clock exchange: call `stamp_header` where the frame is written to the transport, never when it is queued. - WebSocket is a second transport, not a second protocol: `WebSocketTransport` carries exactly one wire frame per binary message (no TLS; terminate `wss://` in a proxy). Clients pick it with a `ws://host:port[/path]` address via `parse_remote`/`connect_transport`. A server serves it on its own port; never sniff the protocol on the TCP port. The server-side HTTP upgrade runs in the first `receive()` so an accept loop never blocks on a client. - Never add `OpenXR::openxr_loader`, `Teleop::plugin_utils` or DeviceIO session libraries to `Teleop::remote_plugin_session`: a plugin that talks to a bridge through this library must run without an OpenXR runtime. +- ERROR on channel 0 ends the session; ERROR on a channel id refuses only that OPEN and the session and its other channels continue. Refuse per channel whenever the session itself is still sound. +- `RemotePluginSession` never throws on the sample path for transport failures (`push`, `push_buffer`, `query`, `update`): after one they become no-ops; only channel creation and the constructor throw. Caller bugs (null joints or buffer) still throw `std::invalid_argument`. +- Timestamps cross the wire in the sender's `CLOCK_MONOTONIC`; the receiver maps them with `ClockOffsetEstimator`, which prefers PING/PONG exchanges (minimum RTT) over the one-way minimum of `local_recv - remote_send`; the one-way minimum hides the minimum network delay. Do not push raw remote timestamps into OpenXR. diff --git a/src/plugins/remote_device_bridge/session/CMakeLists.txt b/src/plugins/remote_device_bridge/session/CMakeLists.txt index 4a6542aab2..3cbf20ac0c 100644 --- a/src/plugins/remote_device_bridge/session/CMakeLists.txt +++ b/src/plugins/remote_device_bridge/session/CMakeLists.txt @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +find_package(Threads REQUIRED) + add_library(remote_plugin_session STATIC + clock_offset_estimator.cpp + inc/remote_plugin_session/clock_offset_estimator.hpp protocol_frame.cpp protocol_payloads.cpp protocol_io.hpp @@ -9,6 +13,8 @@ add_library(remote_plugin_session STATIC inc/remote_plugin_session/protocol_payloads.hpp remote_address.cpp inc/remote_plugin_session/remote_address.hpp + remote_plugin_session.cpp + inc/remote_plugin_session/remote_plugin_session.hpp tcp_socket.cpp inc/remote_plugin_session/tcp_socket.hpp tcp_transport.cpp @@ -26,6 +32,9 @@ target_include_directories(remote_plugin_session target_link_libraries(remote_plugin_session PUBLIC pusherio::pusherio + PRIVATE + log_bridge::log_bridge_core + Threads::Threads ) add_library(Teleop::remote_plugin_session ALIAS remote_plugin_session) diff --git a/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp new file mode 100644 index 0000000000..8e964fe51f --- /dev/null +++ b/src/plugins/remote_device_bridge/session/clock_offset_estimator.cpp @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/remote_plugin_session/clock_offset_estimator.hpp" + +#include +#include + +namespace remote_plugin_session +{ + +ClockOffsetEstimator::ClockOffsetEstimator(size_t max_samples, int64_t window_ns, size_t two_way_window) + : m_max_samples(max_samples), m_window_ns(window_ns), m_two_way_window(two_way_window) +{ + if (max_samples == 0 || window_ns <= 0 || two_way_window == 0) + { + throw std::invalid_argument("ClockOffsetEstimator requires a non-empty window"); + } +} + +void ClockOffsetEstimator::add_sample(int64_t remote_t_send_ns, int64_t local_t_recv_ns) +{ + m_samples.push_back(Sample{ local_t_recv_ns, local_t_recv_ns - remote_t_send_ns }); + + while (m_samples.size() > m_max_samples) + { + m_samples.pop_front(); + } + while (m_samples.front().local_t_recv_ns < local_t_recv_ns - m_window_ns) + { + m_samples.pop_front(); + } + + m_offset_ns = std::min_element(m_samples.begin(), m_samples.end(), + [](const Sample& a, const Sample& b) { return a.offset_ns < b.offset_ns; }) + ->offset_ns; +} + +void ClockOffsetEstimator::add_two_way_sample(int64_t t1_ns, int64_t t2_ns, int64_t t3_ns, int64_t t4_ns) +{ + const int64_t rtt_ns = (t4_ns - t1_ns) - (t3_ns - t2_ns); + if (rtt_ns < 0) + { + return; + } + // local - remote, the same sign as the one-way offset. + const int64_t offset_ns = ((t1_ns - t2_ns) + (t4_ns - t3_ns)) / 2; + m_exchanges.push_back(Exchange{ rtt_ns, offset_ns }); + if (m_exchanges.size() > m_two_way_window) + { + m_exchanges.pop_front(); + } +} + +const ClockOffsetEstimator::Exchange& ClockOffsetEstimator::best_exchange() const +{ + return *std::min_element(m_exchanges.begin(), m_exchanges.end(), + [](const Exchange& a, const Exchange& b) { return a.rtt_ns < b.rtt_ns; }); +} + +int64_t ClockOffsetEstimator::offset_ns() const +{ + return m_exchanges.empty() ? m_offset_ns : best_exchange().offset_ns; +} + +int64_t ClockOffsetEstimator::remote_to_local(int64_t remote_ns) const +{ + return remote_ns + offset_ns(); +} + +int64_t ClockOffsetEstimator::rtt_min_ns() const +{ + return m_exchanges.empty() ? 0 : best_exchange().rtt_ns; +} + +int64_t ClockOffsetEstimator::rtt_last_ns() const +{ + return m_exchanges.empty() ? 0 : m_exchanges.back().rtt_ns; +} + +ClockOffsetEstimator::Source ClockOffsetEstimator::offset_source() const +{ + return m_exchanges.empty() ? Source::OneWay : Source::TwoWay; +} + +PingScheduler::PingScheduler(int64_t start_ns) : m_start_ns(start_ns), m_next_ns(start_ns) +{ +} + +PingScheduler::Action PingScheduler::poll(int64_t now_ns) +{ + if (m_state == State::GaveUp) + { + return Action::None; + } + if (m_state == State::Waiting && now_ns - m_start_ns >= kPongTimeoutNs) + { + m_state = State::GaveUp; + return Action::GiveUp; + } + if (now_ns < m_next_ns) + { + return Action::None; + } + ++m_sent; + m_next_ns = now_ns + (m_sent < kBurstCount ? kBurstPeriodNs : kSteadyPeriodNs); + return Action::Ping; +} + +void PingScheduler::on_pong() +{ + if (m_state == State::Waiting) + { + m_state = State::GotPong; + } +} + +bool PingScheduler::active() const +{ + return m_state != State::GaveUp; +} + +} // namespace remote_plugin_session diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp new file mode 100644 index 0000000000..3be8f91b3d --- /dev/null +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/clock_offset_estimator.hpp @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace remote_plugin_session +{ + +/*! + * @brief Maps a remote monotonic clock onto the local one (local ~= remote + offset). + * + * One-way: fed with (remote send time, local receive time) of every received + * frame; the offset is the running minimum of (local - remote) over a sliding + * window, which hides one minimum one-way delay. Two-way: once a PING/PONG + * exchange arrives the offset comes from the minimum-RTT exchange of the + * two-way window instead (NTP filter; error = path asymmetry / 2) and one-way + * samples no longer move it. Not thread-safe; the owner serializes access. + */ +class ClockOffsetEstimator +{ +public: + enum class Source + { + OneWay, + TwoWay, //!< once an exchange was added + }; + + //! One-way: at most `max_samples` within `window_ns`; two-way: the last `two_way_window` exchanges. + explicit ClockOffsetEstimator(size_t max_samples = 200, int64_t window_ns = 5'000'000'000, size_t two_way_window = 32); + + void add_sample(int64_t remote_t_send_ns, int64_t local_t_recv_ns); + //! t1/t4: local PING send / PONG receive; t2/t3: remote PING receive / PONG send. Negative RTTs are ignored. + void add_two_way_sample(int64_t t1_ns, int64_t t2_ns, int64_t t3_ns, int64_t t4_ns); + + //! Zero until the first sample arrives. + int64_t offset_ns() const; + int64_t remote_to_local(int64_t remote_ns) const; + //! Zero until the first two-way exchange. + int64_t rtt_min_ns() const; + int64_t rtt_last_ns() const; + Source offset_source() const; + +private: + struct Sample + { + int64_t local_t_recv_ns; + int64_t offset_ns; + }; + struct Exchange + { + int64_t rtt_ns; + int64_t offset_ns; + }; + + const Exchange& best_exchange() const; + + size_t m_max_samples; + int64_t m_window_ns; + size_t m_two_way_window; + std::deque m_samples; + int64_t m_offset_ns = 0; + std::deque m_exchanges; +}; + +/*! + * @brief When the bridge sends PINGs: a burst right after HELLO_ACK, then 1 Hz. + * + * Without any PONG within kPongTimeout of the start, poll() returns GiveUp once + * and nothing after that: the peer stays on the one-way estimate. + */ +class PingScheduler +{ +public: + enum class Action + { + None, + Ping, + GiveUp, + }; + + static constexpr int kBurstCount = 10; + static constexpr int64_t kBurstPeriodNs = 50'000'000; + static constexpr int64_t kSteadyPeriodNs = 1'000'000'000; + static constexpr int64_t kPongTimeoutNs = 2'000'000'000; + + explicit PingScheduler(int64_t start_ns); + + Action poll(int64_t now_ns); + void on_pong(); + //! False after GiveUp; late PONGs are then ignored. + bool active() const; + +private: + enum class State + { + Waiting, + GotPong, + GaveUp, + }; + + int64_t m_start_ns; + int64_t m_next_ns; + int m_sent = 0; + State m_state = State::Waiting; +}; + +//! True when a mapped sample time is older than now - max_age; max_age <= 0 disables the guard. After a link +//! stall the backlog arrives in a burst; pushing it into OpenXR would replay old motion instead of skipping it. +inline bool should_drop_stale(int64_t mapped_sample_ns, int64_t now_ns, int64_t max_age_ns) +{ + return max_age_ns > 0 && now_ns - mapped_sample_ns > max_age_ns; +} + +} // namespace remote_plugin_session diff --git a/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp new file mode 100644 index 0000000000..73959ae1d1 --- /dev/null +++ b/src/plugins/remote_device_bridge/session/inc/remote_plugin_session/remote_plugin_session.hpp @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "remote_address.hpp" +#include "transport.hpp" + +#include + +#include +#include +#include +#include + +namespace remote_plugin_session +{ + +namespace detail +{ +class RemotePluginSessionImpl; +} + +/*! + * @brief Plugin session that forwards every channel over an ITransport to a remote_device_bridge. + * + * Same contract as the OpenXR plugin session: the session owns its pull channel + * (opened with the session, with the wrist source HELLO declared) and the owner + * destroys all push channels before the session. After the transport + * drops, pushes become silent no-ops and channels stay safe to destroy. + * Each push channel has a small latest-wins queue; on a stalled link the oldest + * samples of that channel are dropped, other channels are unaffected. A channel + * the bridge refuses (ERROR on its channel id) turns into a no-op; the session + * and its other channels continue. The pull channel exposes no trackers; wrist + * poses are the latest bridge samples (no interpolation). + */ +class RemotePluginSession final : public core::IPluginSession +{ +public: + struct Stats + { + uint64_t sent_hand = 0; + uint64_t sent_schema = 0; + uint64_t wrist_rx = 0; + uint64_t heartbeats = 0; + uint64_t dropped_queue = 0; //!< samples evicted by the per-channel latest-wins queues + uint64_t tx_bytes = 0; + std::map dropped_by_channel; //!< open push channels only + std::map refused_channels; //!< channel id -> the bridge's reason + }; + + //! Sends HELLO over a connected transport (see connect_transport()) and waits for HELLO_ACK; throws + //! std::runtime_error on any failure. + RemotePluginSession(std::unique_ptr transport, + std::string app_name, + core::PluginSessionRequirements requirements); + ~RemotePluginSession() override; + + core::IPluginPullChannel& pull_channel() override; + std::unique_ptr create_schema_push_channel(const core::SchemaPusherConfig& config) override; + std::unique_ptr create_hand_tracking_push_channel(XrHandEXT hand) override; + + //! False once the transport failed; channel creation then throws and pushes are dropped. + bool connected() const; + //! Estimated bridge-to-local clock offset in ns (local ~= bridge + offset), measured from HELLO_ACK onward. + int64_t clock_offset_ns() const; + //! Cumulative counters; their deltas are also logged periodically. + Stats stats() const; + +private: + std::unique_ptr m_impl; + std::unique_ptr m_pull_channel; +}; + +//! Connects each session it builds to `remote`. The remote session exposes no trackers, so the factory throws +//! std::logic_error when handed any. +core::PluginSessionFactory make_remote_plugin_session_factory(std::string app_name, RemoteAddress remote); + +} // namespace remote_plugin_session diff --git a/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp new file mode 100644 index 0000000000..a57cfb3956 --- /dev/null +++ b/src/plugins/remote_device_bridge/session/remote_plugin_session.cpp @@ -0,0 +1,765 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/remote_plugin_session/remote_plugin_session.hpp" + +#include "inc/remote_plugin_session/clock_offset_estimator.hpp" +#include "inc/remote_plugin_session/protocol_frame.hpp" +#include "inc/remote_plugin_session/protocol_payloads.hpp" +#include "inc/remote_plugin_session/remote_address.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace remote_plugin_session +{ + +namespace protocol = remote_plugin_protocol; + +namespace +{ + +std::shared_ptr logger() +{ + return isaaccapture::Logger::get("isaaccapture.plugins.remote_device_bridge.RemotePluginSession"); +} + +// The bridge answers HELLO only after its OpenXR session is up, which waits for a headset; on CloudXR that +// routinely takes longer than a few seconds. +constexpr std::chrono::milliseconds kHandshakeTimeout{ 30000 }; +constexpr std::chrono::milliseconds kHeartbeatPeriod{ 500 }; +constexpr std::chrono::milliseconds kPeerTimeout{ 3000 }; +constexpr std::chrono::milliseconds kReceivePoll{ 100 }; +constexpr std::chrono::seconds kStatsPeriod{ 5 }; +// Destruction lets the sender flush its CLOSE frames this long before the transport is cut under it. +constexpr std::chrono::milliseconds kShutdownDrain{ 500 }; +// Latest-wins per channel: on a stalled link each channel keeps at most this many (or this much) of its own +// newest samples; a fast channel can never evict a slow one. +constexpr size_t kMaxQueuedSamplesPerChannel = 4; +constexpr size_t kMaxQueuedBytesPerChannel = 8u * 1024u * 1024u; + +using FrameBytes = std::vector; // one encoded wire message, header included + +size_t hand_index(XrHandEXT hand) +{ + return hand == XR_HAND_LEFT_EXT ? 0 : 1; +} + +struct ChannelQueue +{ + std::deque frames; + size_t bytes = 0; + uint64_t dropped = 0; +}; + +} // namespace + +class detail::RemotePluginSessionImpl +{ +public: + RemotePluginSessionImpl(std::unique_ptr transport, + std::string app_name, + core::PluginSessionRequirements requirements) + : m_requirements(requirements), m_transport(std::move(transport)) + { + if (!m_transport) + { + throw std::invalid_argument("RemotePluginSession: transport is null"); + } + + protocol::Hello hello; + hello.app_name = std::move(app_name); + hello.requirements = requirements; + FrameBytes hello_frame = protocol::encode_frame( + protocol::FrameType::Hello, protocol::kSessionChannelId, protocol::encode_hello(hello)); + if (!send_now(hello_frame)) + { + throw std::runtime_error("RemotePluginSession: failed to send HELLO"); + } + + FrameBytes bytes; + const ReceiveResult result = m_transport->receive(bytes, kHandshakeTimeout); + if (result != ReceiveResult::Ok) + { + throw std::runtime_error(result == ReceiveResult::Timeout ? + "RemotePluginSession: timed out waiting for HELLO_ACK" : + "RemotePluginSession: connection lost while waiting for HELLO_ACK"); + } + protocol::Frame frame; + try + { + frame = protocol::split_frame(std::move(bytes)); + } + catch (const protocol::DecodeError& e) + { + // A bridge on another protocol version lands here, with both versions in the reason. + throw std::runtime_error(std::string("RemotePluginSession: undecodable HELLO_ACK: ") + e.what()); + } + if (frame.header.type != protocol::FrameType::HelloAck) + { + throw std::runtime_error("RemotePluginSession: expected HELLO_ACK, received frame type " + + std::to_string(static_cast(frame.header.type))); + } + const protocol::HelloAck ack = protocol::decode_hello_ack(frame.payload); + if (!ack.accepted) + { + throw std::runtime_error("RemotePluginSession: bridge rejected the session: " + ack.reason); + } + m_clock.add_sample(frame.header.t_send_monotonic_ns, core::os_monotonic_now_ns()); + + m_connected.store(true); + try + { + m_sender = std::thread( + [this] + { + sender_loop(); + { + std::lock_guard lock(m_queue_mutex); + m_sender_done = true; + } + m_queue_cv.notify_all(); + }); + m_receiver = std::thread(&RemotePluginSessionImpl::receiver_loop, this); + } + catch (...) + { + shutdown(); + throw; + } + } + + ~RemotePluginSessionImpl() + { + shutdown(); + } + + const core::PluginSessionRequirements& requirements() const + { + return m_requirements; + } + + bool connected() const + { + return m_connected.load(); + } + + int64_t clock_offset_ns() const + { + std::lock_guard lock(m_clock_mutex); + return m_clock.offset_ns(); + } + + RemotePluginSession::Stats stats() const + { + RemotePluginSession::Stats stats; + stats.sent_hand = m_sent_hand.load(); + stats.sent_schema = m_sent_schema.load(); + stats.wrist_rx = m_wrist_rx.load(); + stats.heartbeats = m_heartbeats.load(); + stats.tx_bytes = m_tx_bytes.load(); + std::lock_guard lock(m_queue_mutex); + stats.dropped_queue = m_dropped_samples; + for (const auto& [channel_id, queue] : m_sample_queues) + { + stats.dropped_by_channel[channel_id] = queue.dropped; + } + stats.refused_channels = m_refused_channels; + return stats; + } + + //! Allocates a channel id and queues its OPEN_* frame; ids are never reused within a session. + uint16_t open_channel(protocol::FrameType type, std::vector payload) + { + require_connected(); + uint16_t channel_id = 0; + { + std::lock_guard lock(m_state_mutex); + if (m_next_channel_id == 0) + { + throw std::runtime_error("RemotePluginSession: channel ids exhausted"); + } + channel_id = m_next_channel_id++; + m_open_channels.insert(channel_id); + } + if (type == protocol::FrameType::OpenHand || type == protocol::FrameType::OpenSchema) + { + std::lock_guard lock(m_queue_mutex); + m_sample_queues.emplace(channel_id, ChannelQueue{}); + } + enqueue_control(type, channel_id, std::move(payload)); + return channel_id; + } + + void close_channel(uint16_t channel_id) + { + { + std::lock_guard lock(m_state_mutex); + if (m_open_channels.erase(channel_id) == 0) + { + return; + } + } + { + // Pending samples must not follow CLOSE onto the wire (the bridge would see an unknown channel). + std::lock_guard lock(m_queue_mutex); + if (const auto it = m_sample_queues.find(channel_id); it != m_sample_queues.end()) + { + m_queued_samples -= it->second.frames.size(); + m_sample_queues.erase(it); + } + } + if (m_connected.load()) + { + enqueue_control(protocol::FrameType::Close, channel_id, {}); + } + } + + //! Per-channel latest-wins queue; control frames are never dropped. + void push_sample(uint16_t channel_id, FrameBytes frame) + { + { + std::lock_guard lock(m_queue_mutex); + const auto it = m_sample_queues.find(channel_id); + if (it == m_sample_queues.end()) // closed or refused + { + return; + } + ChannelQueue& queue = it->second; + while (!queue.frames.empty() && (queue.frames.size() >= kMaxQueuedSamplesPerChannel || + queue.bytes + frame.size() > kMaxQueuedBytesPerChannel)) + { + queue.bytes -= queue.frames.front().size(); + queue.frames.pop_front(); + ++queue.dropped; + ++m_dropped_samples; + --m_queued_samples; + } + queue.bytes += frame.size(); + queue.frames.push_back(std::move(frame)); + ++m_queued_samples; + } + m_queue_cv.notify_one(); + } + + core::WristTrackingSample latest_wrist(XrHandEXT hand) const + { + std::lock_guard lock(m_wrist_mutex); + return m_wrist[hand_index(hand)]; + } + + void shutdown() + { + // Runs once: from the destructor, or from a constructor that throws (and so never destructs). Every + // channel has already sent its CLOSE, as IPluginSession has the owner destroy channels first. + { + std::unique_lock lock(m_queue_mutex); + m_stop = true; + m_queue_cv.notify_all(); + if (m_sender.joinable()) + { + // A sender stalled inside send() cannot finish on its own; the transport shutdown below cuts it off. + m_queue_cv.wait_for(lock, kShutdownDrain, [this] { return m_sender_done; }); + } + } + m_connected.store(false); + m_transport->shutdown(); + if (m_sender.joinable()) + { + m_sender.join(); + } + if (m_receiver.joinable()) + { + m_receiver.join(); + } + } + +private: + void require_connected() const + { + if (!m_connected.load()) + { + throw std::runtime_error("RemotePluginSession: transport is disconnected"); + } + } + + void enqueue_control(protocol::FrameType type, uint16_t channel_id, const std::vector& payload) + { + FrameBytes frame = protocol::encode_frame(type, channel_id, payload); + { + std::lock_guard lock(m_queue_mutex); + m_control_queue.push_back(std::move(frame)); + } + m_queue_cv.notify_one(); + } + + //! Stamps seq/t_send and sends one frame; t_send is taken here, as the frame is written, so a PONG that waited + //! in the queue still reports when it left. Constructor thread during the handshake, sender thread after. + bool send_now(FrameBytes& frame) + { + protocol::stamp_header(frame, m_next_seq++, core::os_monotonic_now_ns()); + const protocol::FrameType type = protocol::decode_header(frame).type; + if (!m_transport->send(frame.data(), frame.size())) + { + return false; + } + m_tx_bytes.fetch_add(frame.size()); + if (type == protocol::FrameType::HandSample) + { + m_sent_hand.fetch_add(1); + } + else if (type == protocol::FrameType::SchemaSample) + { + m_sent_schema.fetch_add(1); + } + return true; + } + + //! Waits until `deadline` for work. Control frames first and in order, then one sample per channel + //! (round-robin) so a 180 Hz channel cannot starve a 60 Hz one. Empty when the deadline passed. + std::vector next_batch(std::chrono::steady_clock::time_point deadline) + { + std::vector batch; + std::unique_lock lock(m_queue_mutex); + m_queue_cv.wait_until( + lock, deadline, + [this] { return m_stop || !m_connected.load() || !m_control_queue.empty() || m_queued_samples > 0; }); + for (FrameBytes& frame : m_control_queue) + { + batch.push_back(std::move(frame)); + } + m_control_queue.clear(); + if (m_stop) + { + return batch; + } + for (auto& [channel_id, queue] : m_sample_queues) + { + if (queue.frames.empty()) + { + continue; + } + queue.bytes -= queue.frames.front().size(); + batch.push_back(std::move(queue.frames.front())); + queue.frames.pop_front(); + --m_queued_samples; + } + return batch; + } + + void log_stats(RemotePluginSession::Stats& last) + { + const RemotePluginSession::Stats now = stats(); + logger()->info( + "{}s: sent hand={} schema={} wrist_rx={} hb={} | dropped queue={} | tx_bytes={} | offset_ns={} | " + "connected={:d}", + kStatsPeriod.count(), now.sent_hand - last.sent_hand, now.sent_schema - last.sent_schema, + now.wrist_rx - last.wrist_rx, now.heartbeats - last.heartbeats, now.dropped_queue - last.dropped_queue, + now.tx_bytes - last.tx_bytes, clock_offset_ns(), m_connected.load()); + last = now; + } + + void on_disconnect(const std::string& reason) + { + if (!m_connected.exchange(false)) + { + return; + } + if (!m_stop) + { + logger()->warn("disconnected: {}", reason); + } + m_transport->shutdown(); + m_queue_cv.notify_all(); + } + + //! The bridge refused one channel: its pushes become no-ops, the session and other channels continue. + void refuse_channel(uint16_t channel_id, const std::string& reason) + { + logger()->warn("bridge refused channel {}: {}", channel_id, reason); + std::lock_guard lock(m_queue_mutex); + m_refused_channels[channel_id] = reason; + if (const auto it = m_sample_queues.find(channel_id); it != m_sample_queues.end()) + { + m_queued_samples -= it->second.frames.size(); + m_sample_queues.erase(it); + } + } + + //! Sends are done outside the queue lock: a stalled send must not block the plugin's push(). + void sender_loop() + { + auto last_send = std::chrono::steady_clock::now(); + auto next_stats = last_send + kStatsPeriod; + RemotePluginSession::Stats logged_stats; + while (true) + { + std::vector batch = next_batch(std::min(last_send + kHeartbeatPeriod, next_stats)); + if (!m_connected.load()) + { + return; + } + if (batch.empty() && m_stop) + { + return; + } + const auto now = std::chrono::steady_clock::now(); + if (batch.empty() && now - last_send >= kHeartbeatPeriod) + { + batch.push_back(protocol::encode_frame(protocol::FrameType::Heartbeat, protocol::kSessionChannelId, {})); + m_heartbeats.fetch_add(1); + } + for (FrameBytes& frame : batch) + { + if (!send_now(frame)) + { + on_disconnect("send failed"); + return; + } + last_send = std::chrono::steady_clock::now(); + } + if (now >= next_stats) + { + log_stats(logged_stats); + next_stats += kStatsPeriod; + } + } + } + + void receiver_loop() + { + auto last_receive = std::chrono::steady_clock::now(); + while (m_connected.load() && !m_stop) + { + FrameBytes bytes; + switch (m_transport->receive(bytes, kReceivePoll)) + { + case ReceiveResult::Ok: + last_receive = std::chrono::steady_clock::now(); + handle_frame(std::move(bytes), core::os_monotonic_now_ns()); + break; + case ReceiveResult::Timeout: + if (std::chrono::steady_clock::now() - last_receive > kPeerTimeout) + { + on_disconnect("no frames from the bridge within the peer timeout"); + return; + } + break; + case ReceiveResult::Closed: + on_disconnect("connection closed by the bridge"); + return; + case ReceiveResult::Malformed: + on_disconnect("malformed frame from the bridge"); + return; + } + } + } + + void handle_frame(FrameBytes bytes, int64_t t_recv_ns) + { + try + { + const protocol::Frame frame = protocol::split_frame(std::move(bytes)); + { + std::lock_guard lock(m_clock_mutex); + m_clock.add_sample(frame.header.t_send_monotonic_ns, t_recv_ns); + } + switch (frame.header.type) + { + case protocol::FrameType::WristSample: + { + // sample_time_local_common_clock_ns is dropped: core::WristTrackingSample has no timestamp field and + // queries return the latest sample. + const protocol::WristSample sample = protocol::decode_wrist_sample(frame.payload); + m_wrist_rx.fetch_add(1); + std::lock_guard lock(m_wrist_mutex); + m_wrist[hand_index(sample.hand)] = sample.sample; + break; + } + case protocol::FrameType::Ping: + { + // The PING's send time is its header t_send. Queued as control (never dropped); the PONG's own send + // time is stamped by send_now() when the sender thread writes it. + enqueue_control(protocol::FrameType::Pong, protocol::kSessionChannelId, + protocol::encode_pong(protocol::Pong{ frame.header.t_send_monotonic_ns, t_recv_ns })); + break; + } + case protocol::FrameType::Error: + { + const std::string message = protocol::decode_error(frame.payload); + if (frame.header.channel_id != protocol::kSessionChannelId) + { + refuse_channel(frame.header.channel_id, message); + break; + } + on_disconnect("bridge error: " + message); + break; + } + case protocol::FrameType::Heartbeat: + break; + default: + // HELLO_ACK after the handshake included: the peers disagree on the protocol. + on_disconnect("unexpected frame type " + std::to_string(static_cast(frame.header.type)) + + " from the bridge"); + break; + } + } + catch (const protocol::DecodeError& e) + { + on_disconnect(std::string("undecodable frame from the bridge: ") + e.what()); + } + } + + core::PluginSessionRequirements m_requirements; + std::unique_ptr m_transport; + + std::atomic m_connected{ false }; + std::atomic m_stop{ false }; + + std::mutex m_state_mutex; + uint16_t m_next_channel_id = 1; + std::set m_open_channels; + + mutable std::mutex m_queue_mutex; + std::condition_variable m_queue_cv; + std::deque m_control_queue; + std::map m_sample_queues; + std::map m_refused_channels; + bool m_sender_done = false; + size_t m_queued_samples = 0; + uint64_t m_dropped_samples = 0; + uint32_t m_next_seq = 0; // handshake on the constructor thread, then the sender thread only + + std::atomic m_sent_hand{ 0 }; + std::atomic m_sent_schema{ 0 }; + std::atomic m_wrist_rx{ 0 }; + std::atomic m_heartbeats{ 0 }; + std::atomic m_tx_bytes{ 0 }; + + mutable std::mutex m_wrist_mutex; + core::WristTrackingSample m_wrist[2]; + + mutable std::mutex m_clock_mutex; + ClockOffsetEstimator m_clock; + + std::thread m_sender; + std::thread m_receiver; +}; + +namespace +{ + +class RemoteHandTrackingPushChannel final : public core::IHandTrackingPushChannel +{ +public: + RemoteHandTrackingPushChannel(detail::RemotePluginSessionImpl& impl, uint16_t channel_id) + : m_impl(impl), m_channel_id(channel_id) + { + } + + ~RemoteHandTrackingPushChannel() override + { + m_impl.close_channel(m_channel_id); + } + + void push(const XrHandJointLocationEXT* joint_locations, int64_t sample_time_local_common_clock_ns) override + { + if (joint_locations == nullptr) + { + throw std::invalid_argument("Hand tracking push requires joint locations"); + } + if (!m_impl.connected()) + { + return; + } + m_impl.push_sample(m_channel_id, protocol::encode_hand_sample_frame( + m_channel_id, protocol::HandJoints(joint_locations, XR_HAND_JOINT_COUNT_EXT), + sample_time_local_common_clock_ns)); + } + +private: + detail::RemotePluginSessionImpl& m_impl; + uint16_t m_channel_id; +}; + +class RemoteSchemaPushChannel final : public core::ISchemaPushChannel +{ +public: + RemoteSchemaPushChannel(detail::RemotePluginSessionImpl& impl, uint16_t channel_id) + : m_impl(impl), m_channel_id(channel_id) + { + } + + ~RemoteSchemaPushChannel() override + { + m_impl.close_channel(m_channel_id); + } + + void push_buffer(const uint8_t* buffer, + size_t size, + int64_t sample_time_local_common_clock_ns, + int64_t sample_time_raw_device_clock_ns) override + { + if (buffer == nullptr && size != 0) + { + throw std::invalid_argument("push_buffer: buffer is null for a non-empty sample"); + } + if (!m_impl.connected()) + { + return; + } + m_impl.push_sample(m_channel_id, protocol::encode_schema_sample_frame( + m_channel_id, std::span(buffer, size), + sample_time_local_common_clock_ns, sample_time_raw_device_clock_ns)); + } + +private: + detail::RemotePluginSessionImpl& m_impl; + uint16_t m_channel_id; +}; + +class RemoteWristTrackingSource final : public core::IWristTrackingSource +{ +public: + explicit RemoteWristTrackingSource(detail::RemotePluginSessionImpl& impl) : m_impl(impl) + { + } + + // Returns the latest bridge sample; the requested time is not interpolated. + core::WristTrackingSample query(XrHandEXT hand, int64_t /*sample_time_local_common_clock_ns*/) override + { + return m_impl.latest_wrist(hand); + } + +private: + detail::RemotePluginSessionImpl& m_impl; +}; + +class RemotePluginPullChannel final : public core::IPluginPullChannel +{ +public: + explicit RemotePluginPullChannel(detail::RemotePluginSessionImpl& impl) + : m_impl(impl), m_channel_id(m_impl.open_channel(protocol::FrameType::OpenPull, {})) + { + // The bridge streams WRIST_SAMPLEs on this channel when HELLO declared a wrist source. + if (m_impl.requirements().wrist_tracking_pull) + { + m_wrist_source.emplace(m_impl); + } + } + + ~RemotePluginPullChannel() override + { + m_impl.close_channel(m_channel_id); + } + + // The receiver thread keeps the wrist cache current; nothing to poll here. + void update() override + { + } + + const core::ITrackerImpl& get_tracker_impl(const core::ITracker&) const override + { + throw std::runtime_error("RemotePluginSession pull channel exposes no trackers"); + } + + core::IWristTrackingSource& wrist_tracking_source() override + { + if (!m_wrist_source) + { + throw std::logic_error( + "RemotePluginSession: wrist tracking pull was not declared in PluginSessionRequirements"); + } + return *m_wrist_source; + } + +private: + detail::RemotePluginSessionImpl& m_impl; + uint16_t m_channel_id; + std::optional m_wrist_source; +}; + +} // namespace + +RemotePluginSession::RemotePluginSession(std::unique_ptr transport, + std::string app_name, + core::PluginSessionRequirements requirements) + : m_impl(std::make_unique(std::move(transport), std::move(app_name), requirements)), + m_pull_channel(std::make_unique(*m_impl)) +{ +} + +// The pull channel closes first, then ~RemotePluginSessionImpl shuts the connection down. +RemotePluginSession::~RemotePluginSession() = default; + +core::IPluginPullChannel& RemotePluginSession::pull_channel() +{ + return *m_pull_channel; +} + +std::unique_ptr RemotePluginSession::create_schema_push_channel( + const core::SchemaPusherConfig& config) +{ + if (!m_impl->requirements().schema_push) + { + throw std::logic_error("RemotePluginSession: schema push was not declared in PluginSessionRequirements"); + } + const uint16_t channel_id = + m_impl->open_channel(protocol::FrameType::OpenSchema, protocol::encode_open_schema(config)); + return std::make_unique(*m_impl, channel_id); +} + +std::unique_ptr RemotePluginSession::create_hand_tracking_push_channel(XrHandEXT hand) +{ + if (!m_impl->requirements().hand_tracking_push) + { + throw std::logic_error("RemotePluginSession: hand-tracking push was not declared in PluginSessionRequirements"); + } + const uint16_t channel_id = m_impl->open_channel(protocol::FrameType::OpenHand, protocol::encode_open_hand(hand)); + return std::make_unique(*m_impl, channel_id); +} + +bool RemotePluginSession::connected() const +{ + return m_impl->connected(); +} + +int64_t RemotePluginSession::clock_offset_ns() const +{ + return m_impl->clock_offset_ns(); +} + +RemotePluginSession::Stats RemotePluginSession::stats() const +{ + return m_impl->stats(); +} + +core::PluginSessionFactory make_remote_plugin_session_factory(std::string app_name, RemoteAddress remote) +{ + return [app_name = std::move(app_name), remote = std::move(remote)]( + const core::PluginSessionRequirements& requirements, + std::vector> trackers) -> std::unique_ptr + { + if (!trackers.empty()) + { + throw std::logic_error("RemotePluginSession exposes no trackers; " + app_name + " passed some"); + } + return std::make_unique(connect_transport(remote), app_name, requirements); + }; +} + +} // namespace remote_plugin_session diff --git a/tests/cpp/plugins/remote_device_bridge/CMakeLists.txt b/tests/cpp/plugins/remote_device_bridge/CMakeLists.txt index a513b47b05..a7f9589058 100644 --- a/tests/cpp/plugins/remote_device_bridge/CMakeLists.txt +++ b/tests/cpp/plugins/remote_device_bridge/CMakeLists.txt @@ -4,7 +4,9 @@ find_package(Threads REQUIRED) add_executable(remote_plugin_session_tests + test_clock_offset_estimator.cpp test_remote_address.cpp + test_remote_plugin_session.cpp test_tcp_transport.cpp test_websocket_transport.cpp test_wire_codec.cpp diff --git a/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp b/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp new file mode 100644 index 0000000000..5eb23ed303 --- /dev/null +++ b/tests/cpp/plugins/remote_device_bridge/test_clock_offset_estimator.cpp @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include + +using remote_plugin_session::ClockOffsetEstimator; +using remote_plugin_session::PingScheduler; +using remote_plugin_session::should_drop_stale; + +TEST_CASE("ClockOffsetEstimator starts empty and follows the sign convention", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator; + REQUIRE(estimator.offset_ns() == 0); + REQUIRE(estimator.remote_to_local(42) == 42); + + // Remote clock reads 1'000 when the local clock reads 5'000 (+ 100 of delay). + estimator.add_sample(1'000, 5'100); + REQUIRE(estimator.offset_ns() == 4'100); + REQUIRE(estimator.remote_to_local(1'000) == 5'100); +} + +TEST_CASE("ClockOffsetEstimator keeps the minimum delay sample", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator; + estimator.add_sample(1'000, 5'300); // delay 300 + estimator.add_sample(2'000, 6'050); // delay 50 + estimator.add_sample(3'000, 7'400); // delay 400 + REQUIRE(estimator.offset_ns() == 4'050); +} + +TEST_CASE("ClockOffsetEstimator evicts by sample count", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator(/*max_samples=*/2, /*window_ns=*/1'000'000); + estimator.add_sample(0, 10); // offset 10 (the eventual minimum, evicted first) + estimator.add_sample(100, 150); // offset 50 + estimator.add_sample(200, 260); // offset 60 + REQUIRE(estimator.offset_ns() == 50); +} + +TEST_CASE("ClockOffsetEstimator evicts by time window", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator(/*max_samples=*/100, /*window_ns=*/1'000); + estimator.add_sample(0, 10); // local 10, offset 10 + estimator.add_sample(500, 800); // local 800, offset 300 + REQUIRE(estimator.offset_ns() == 10); + estimator.add_sample(1'500, 1'700); // local 1'700, offset 200 -> drops samples older than 700 + REQUIRE(estimator.offset_ns() == 200); +} + +TEST_CASE("ClockOffsetEstimator handles negative offsets", "[remote_device_bridge][unit]") +{ + ClockOffsetEstimator estimator; + estimator.add_sample(10'000, 2'000); + REQUIRE(estimator.offset_ns() == -8'000); + REQUIRE(estimator.remote_to_local(10'500) == 2'500); +} + +TEST_CASE("ClockOffsetEstimator rejects an empty window", "[remote_device_bridge][unit]") +{ + REQUIRE_THROWS_AS(ClockOffsetEstimator(0, 1), std::invalid_argument); + REQUIRE_THROWS_AS(ClockOffsetEstimator(1, 0), std::invalid_argument); + REQUIRE_THROWS_AS(ClockOffsetEstimator(1, 1, 0), std::invalid_argument); +} + +TEST_CASE("should_drop_stale keeps fresh samples and drops the backlog after a stall", "[remote_device_bridge][unit]") +{ + constexpr int64_t kMaxAge = 250'000'000; // 250 ms + const int64_t now = 10'000'000'000; + REQUIRE_FALSE(should_drop_stale(now, now, kMaxAge)); + REQUIRE_FALSE(should_drop_stale(now - kMaxAge, now, kMaxAge)); // exactly max_age is still fresh + REQUIRE_FALSE(should_drop_stale(now + 5'000'000, now, kMaxAge)); // clock-offset bias may map into the future + REQUIRE(should_drop_stale(now - kMaxAge - 1, now, kMaxAge)); + REQUIRE(should_drop_stale(now - 3'000'000'000, now, kMaxAge)); + REQUIRE_FALSE(should_drop_stale(now - 3'000'000'000, now, 0)); // 0 disables the guard + REQUIRE_FALSE(should_drop_stale(now - 3'000'000'000, now, -1)); +} + +namespace +{ + +constexpr int64_t kMs = 1'000'000; + +//! One exchange with true offset `offset` (local - remote), forward delay d_fwd, return delay d_back. +void exchange(ClockOffsetEstimator& estimator, int64_t t1, int64_t offset, int64_t d_fwd, int64_t d_back) +{ + const int64_t t2 = t1 + d_fwd - offset; + const int64_t t3 = t2 + 1 * kMs; // client turnaround, excluded from the RTT + const int64_t t4 = t3 + offset + d_back; + estimator.add_two_way_sample(t1, t2, t3, t4); +} + +} // namespace + +TEST_CASE("Two-way exchanges recover the offset the one-way minimum biases", "[remote_device_bridge][unit]") +{ + constexpr int64_t kOffset = 5'000 * kMs; + ClockOffsetEstimator estimator; + // One-way: the fastest frame took 70 ms, so the one-way estimate is 70 ms too large. + estimator.add_sample(1'000 * kMs, 1'000 * kMs + kOffset + 70 * kMs); + REQUIRE(estimator.offset_ns() == kOffset + 70 * kMs); + REQUIRE(estimator.offset_source() == ClockOffsetEstimator::Source::OneWay); + REQUIRE(estimator.rtt_min_ns() == 0); + + exchange(estimator, 2'000 * kMs, kOffset, 70 * kMs, 70 * kMs); + REQUIRE(estimator.offset_source() == ClockOffsetEstimator::Source::TwoWay); + REQUIRE(estimator.offset_ns() == kOffset); + REQUIRE(estimator.rtt_last_ns() == 140 * kMs); + REQUIRE(estimator.rtt_min_ns() == 140 * kMs); + REQUIRE(estimator.remote_to_local(0) == kOffset); + + // One-way samples no longer move a two-way estimate. + estimator.add_sample(3'000 * kMs, 3'000 * kMs + kOffset + 1 * kMs); + REQUIRE(estimator.offset_ns() == kOffset); +} + +TEST_CASE("Two-way offset error is half the path asymmetry", "[remote_device_bridge][unit]") +{ + constexpr int64_t kOffset = -123'456 * kMs; + ClockOffsetEstimator estimator; + // 10 ms out, 50 ms back: RTT 60 ms, estimate off by (50 - 10) / 2 = +20 ms. + exchange(estimator, 1'000 * kMs, kOffset, 10 * kMs, 50 * kMs); + REQUIRE(estimator.rtt_last_ns() == 60 * kMs); + REQUIRE(estimator.offset_ns() - kOffset == 20 * kMs); +} + +TEST_CASE("Two-way estimate follows the minimum-RTT exchange of the last 32", "[remote_device_bridge][unit]") +{ + constexpr int64_t kOffset = 7 * kMs; + ClockOffsetEstimator estimator; + // Exchange 0 has the lowest RTT and an asymmetry of +4 ms; the rest are slower and symmetric-ish. + exchange(estimator, 0, kOffset, 1 * kMs, 9 * kMs); // rtt 10, error +4 + for (int i = 1; i < 32; ++i) + { + exchange(estimator, i * 1'000 * kMs, kOffset, 30 * kMs, 50 * kMs); // rtt 80, error +10 + } + REQUIRE(estimator.rtt_min_ns() == 10 * kMs); + REQUIRE(estimator.offset_ns() == kOffset + 4 * kMs); + REQUIRE(estimator.rtt_last_ns() == 80 * kMs); + + exchange(estimator, 32'000 * kMs, kOffset, 30 * kMs, 50 * kMs); // evicts exchange 0 + REQUIRE(estimator.rtt_min_ns() == 80 * kMs); + REQUIRE(estimator.offset_ns() == kOffset + 10 * kMs); + + estimator.add_two_way_sample(0, 100, 200, 50); // negative RTT: ignored + REQUIRE(estimator.rtt_last_ns() == 80 * kMs); +} + +TEST_CASE("PingScheduler bursts 10 PINGs at 50 ms, then 1 Hz", "[remote_device_bridge][unit]") +{ + constexpr int64_t kStart = 1'000 * kMs; + PingScheduler pings(kStart); + std::vector sent; + for (int64_t t = kStart; t <= kStart + 3'500 * kMs; t += kMs) + { + const PingScheduler::Action action = pings.poll(t); + REQUIRE(action != PingScheduler::Action::GiveUp); + if (action == PingScheduler::Action::Ping) + { + sent.push_back(t - kStart); + pings.on_pong(); + } + } + const std::vector expected = { 0, 50 * kMs, 100 * kMs, 150 * kMs, 200 * kMs, + 250 * kMs, 300 * kMs, 350 * kMs, 400 * kMs, 450 * kMs, + 1450 * kMs, 2450 * kMs, 3450 * kMs }; + REQUIRE(sent == expected); + REQUIRE(pings.active()); +} + +TEST_CASE("PingScheduler gives up once when no PONG arrives within 2 s", "[remote_device_bridge][unit]") +{ + PingScheduler pings(0); + int given_up = 0; + int sent_after = 0; + for (int64_t t = 0; t <= 5'000 * kMs; t += kMs) + { + switch (pings.poll(t)) + { + case PingScheduler::Action::GiveUp: + REQUIRE(t == PingScheduler::kPongTimeoutNs); + ++given_up; + break; + case PingScheduler::Action::Ping: + sent_after += given_up; + break; + case PingScheduler::Action::None: + break; + } + } + REQUIRE(given_up == 1); + REQUIRE(sent_after == 0); + REQUIRE_FALSE(pings.active()); +} diff --git a/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp new file mode 100644 index 0000000000..31b99f7b57 --- /dev/null +++ b/tests/cpp/plugins/remote_device_bridge/test_remote_plugin_session.cpp @@ -0,0 +1,801 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace remote_plugin_session; +namespace protocol = remote_plugin_protocol; + +namespace +{ + +constexpr std::chrono::milliseconds kWait{ 2000 }; +const core::WristTrackingSourceConfig kWristConfig{ + .mode = core::WristTrackingSourceMode::Auto, + .left_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }, + .right_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } } +}; + +bool send_frame(ITransport& transport, + protocol::FrameType type, + uint16_t channel_id, + uint32_t seq, + const std::vector& payload) +{ + std::vector frame = protocol::encode_frame(type, channel_id, payload); + protocol::stamp_header(frame, seq, core::os_monotonic_now_ns()); + return transport.send(frame.data(), frame.size()); +} + +//! Removes and returns the oldest frame of `type`. +std::optional take_frame(std::deque& frames, protocol::FrameType type) +{ + const auto it = std::find_if( + frames.begin(), frames.end(), [type](const protocol::Frame& frame) { return frame.header.type == type; }); + if (it == frames.end()) + { + return std::nullopt; + } + protocol::Frame frame = std::move(*it); + frames.erase(it); + return frame; +} + +//! Minimal in-process bridge: answers HELLO, records every frame, can inject frames and drop the socket. +class FakeBridge +{ +public: + //! Answers HELLO with a refusal carrying `rejection` when set, else accepts. + explicit FakeBridge(std::optional rejection = std::nullopt) + : m_listener(TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 1)), m_rejection(std::move(rejection)) + { + m_thread = std::thread(&FakeBridge::serve, this); + } + + ~FakeBridge() + { + m_stop = true; + { + std::lock_guard lock(m_mutex); + if (m_client) + { + m_client->shutdown(); + } + } + m_thread.join(); + } + + uint16_t port() const + { + return m_listener.local_port(); + } + + std::optional wait_for(protocol::FrameType type, std::chrono::milliseconds timeout = kWait) + { + std::unique_lock lock(m_mutex); + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (true) + { + if (auto frame = take_frame(m_frames, type)) + { + return frame; + } + if (m_cv.wait_until(lock, deadline) == std::cv_status::timeout) + { + return std::nullopt; + } + } + } + + bool send(protocol::FrameType type, uint16_t channel_id, const std::vector& payload) + { + std::lock_guard lock(m_mutex); + return m_client && send_frame(*m_client, type, channel_id, m_seq++, payload); + } + + void drop_connection() + { + std::lock_guard lock(m_mutex); + if (m_client) + { + m_client->shutdown(); + } + } + + bool client_closed(std::chrono::milliseconds timeout = kWait) + { + std::unique_lock lock(m_mutex); + return m_cv.wait_for(lock, timeout, [this] { return m_closed; }); + } + +private: + void serve() + { + std::unique_ptr client; + while (!m_stop && !client) + { + client = TcpTransport::accept(m_listener, std::chrono::milliseconds(50)); + } + if (!client) + { + return; + } + + std::vector bytes; + if (client->receive(bytes, kWait) != ReceiveResult::Ok) + { + return; + } + protocol::Frame hello = protocol::split_frame(std::move(bytes)); + if (hello.header.type != protocol::FrameType::Hello) + { + return; + } + ITransport* transport = client.get(); + { + std::lock_guard lock(m_mutex); + m_client = std::move(client); + m_frames.push_back(std::move(hello)); + send_frame(*m_client, protocol::FrameType::HelloAck, protocol::kSessionChannelId, m_seq++, + protocol::encode_hello_ack(protocol::HelloAck{ !m_rejection, m_rejection.value_or("") })); + } + m_cv.notify_all(); + + while (!m_stop) + { + const ReceiveResult result = transport->receive(bytes, std::chrono::milliseconds(50)); + if (result == ReceiveResult::Timeout) + { + continue; + } + std::lock_guard lock(m_mutex); + if (result == ReceiveResult::Ok) + { + m_frames.push_back(protocol::split_frame(std::move(bytes))); + } + else + { + m_closed = true; + m_cv.notify_all(); + return; + } + m_cv.notify_all(); + } + } + + TcpSocket m_listener; + std::optional m_rejection; + std::atomic m_stop{ false }; + std::thread m_thread; + + std::mutex m_mutex; + std::condition_variable m_cv; + std::unique_ptr m_client; + std::deque m_frames; + uint32_t m_seq = 0; + bool m_closed = false; +}; + +//! In-memory ITransport standing in for a stalled TCP link: while held, send() blocks like a write into a full +//! socket; released frames are recorded. receive() serves the accepting HELLO_ACK and times out afterwards. +class BlockingTransport final : public ITransport +{ +public: + //! `ack_version` is the protocol version in the HELLO_ACK header, to stand in for a bridge of another version. + explicit BlockingTransport(uint8_t ack_version = protocol::kProtocolVersion) + { + m_inbox.push_back(protocol::encode_frame(protocol::FrameType::HelloAck, protocol::kSessionChannelId, + protocol::encode_hello_ack(protocol::HelloAck{ true, "" }))); + m_inbox.back()[protocol::kVersionOffset] = ack_version; + protocol::stamp_header(m_inbox.back(), 0, core::os_monotonic_now_ns()); + } + + bool send(const uint8_t* frame, size_t size) override + { + std::unique_lock lock(m_mutex); + m_blocked = m_held; + m_cv.wait(lock, [this] { return !m_held || m_closed; }); + m_blocked = false; + if (m_closed) + { + return false; + } + m_sent.push_back(protocol::split_frame(std::vector(frame, frame + size))); + m_cv.notify_all(); + return true; + } + + ReceiveResult receive(std::vector& frame, std::chrono::milliseconds timeout) override + { + std::unique_lock lock(m_mutex); + if (!m_cv.wait_for(lock, timeout, [this] { return !m_inbox.empty() || m_closed; })) + { + return ReceiveResult::Timeout; + } + if (m_closed) + { + return ReceiveResult::Closed; + } + frame = std::move(m_inbox.front()); + m_inbox.pop_front(); + return ReceiveResult::Ok; + } + + void shutdown() override + { + std::lock_guard lock(m_mutex); + m_closed = true; + m_cv.notify_all(); + } + + void hold(bool held) + { + std::lock_guard lock(m_mutex); + m_held = held; + m_cv.notify_all(); + } + + //! Hands `frame` to the session as if the bridge had sent it. + void deliver(std::vector frame) + { + std::lock_guard lock(m_mutex); + m_inbox.push_back(std::move(frame)); + m_cv.notify_all(); + } + + //! True while a send() is waiting for hold(false). + bool blocked() const + { + std::lock_guard lock(m_mutex); + return m_blocked; + } + + std::optional wait_for(protocol::FrameType type, std::chrono::milliseconds timeout = kWait) + { + std::unique_lock lock(m_mutex); + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (true) + { + if (auto frame = take_frame(m_sent, type)) + { + return frame; + } + if (m_cv.wait_until(lock, deadline) == std::cv_status::timeout) + { + return std::nullopt; + } + } + } + +private: + mutable std::mutex m_mutex; + std::condition_variable m_cv; + bool m_held = false; + bool m_blocked = false; + bool m_closed = false; + std::deque m_sent; + std::deque> m_inbox; +}; + +std::array make_joints(float seed) +{ + std::array joints{}; + for (size_t i = 0; i < joints.size(); ++i) + { + const float f = seed + 0.001f * static_cast(i); + joints[i].locationFlags = XR_SPACE_LOCATION_POSITION_VALID_BIT | static_cast(i) << 32; + joints[i].pose = { { f, -f, 2.0f * f, 1.0f - f }, { 3.0f * f, 4.0f * f, -5.0f * f } }; + joints[i].radius = 0.01f + f; + } + return joints; +} + +class DummyTracker final : public core::ITracker +{ +public: + std::string_view get_name() const override + { + return "dummy"; + } +}; + +bool wait_until(const std::function& predicate, std::chrono::milliseconds timeout = kWait) +{ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (!predicate()) + { + if (std::chrono::steady_clock::now() > deadline) + { + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return true; +} + +std::unique_ptr connect_to(uint16_t port) +{ + return std::make_unique(TcpSocket::connect(Endpoint{ "127.0.0.1", port }, kConnectTimeout)); +} + +} // namespace + +TEST_CASE("RemotePluginSession performs the HELLO handshake", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session( + connect_to(bridge.port()), "test_app", { .schema_push = true, .hand_tracking_push = true }); + REQUIRE(session.connected()); + + const auto hello = bridge.wait_for(protocol::FrameType::Hello); + REQUIRE(hello.has_value()); + REQUIRE(hello->header.channel_id == protocol::kSessionChannelId); + REQUIRE(hello->header.seq == 0); + REQUIRE(protocol::decode_hello(hello->payload).app_name == "test_app"); + const protocol::Hello decoded = protocol::decode_hello(hello->payload); + REQUIRE(decoded.app_name == "test_app"); + REQUIRE(decoded.requirements.schema_push); + REQUIRE(decoded.requirements.hand_tracking_push); + REQUIRE_FALSE(decoded.requirements.wrist_tracking_pull); +} + +TEST_CASE("RemotePluginSession fails fast when the bridge refuses or is absent", "[remote_device_bridge][unit]") +{ + SECTION("HELLO_ACK with accepted=0 throws with the reason") + { + FakeBridge bridge(std::string("no OpenXR runtime")); + try + { + RemotePluginSession session(connect_to(bridge.port()), "test_app", {}); + FAIL("expected std::runtime_error"); + } + catch (const std::runtime_error& e) + { + REQUIRE(std::string(e.what()).find("no OpenXR runtime") != std::string::npos); + } + } + SECTION("a bridge on protocol version 0 is named in the error") + { + REQUIRE_THROWS_WITH(RemotePluginSession(std::make_unique(0), "test_app", {}), + Catch::Matchers::ContainsSubstring("undecodable HELLO_ACK: protocol: peer speaks protocol " + "version 0, this build speaks version 1")); + } + SECTION("nothing listening throws") + { + TcpSocket probe = TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 1); + const uint16_t free_port = probe.local_port(); + probe.close(); + REQUIRE_THROWS_AS(RemotePluginSession(connect_to(free_port), "test_app", {}), std::runtime_error); + } + SECTION("a null transport is rejected") + { + REQUIRE_THROWS_AS(RemotePluginSession(std::unique_ptr(), "test_app", {}), std::invalid_argument); + } +} + +TEST_CASE("RemotePluginSession enforces declared requirements", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session( + connect_to(bridge.port()), "test_app", { .schema_push = false, .hand_tracking_push = true }); + + REQUIRE_THROWS_AS(session.create_schema_push_channel(core::SchemaPusherConfig{}), std::logic_error); + core::IPluginPullChannel& pull = session.pull_channel(); + REQUIRE_THROWS_AS(pull.wrist_tracking_source(), std::logic_error); + const DummyTracker tracker; + REQUIRE_THROWS_AS(pull.get_tracker_impl(tracker), std::runtime_error); + REQUIRE(bridge.wait_for(protocol::FrameType::OpenPull).has_value()); +} + +TEST_CASE("RemotePluginSession rejects hand-tracking push when it was not declared", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session( + connect_to(bridge.port()), "test_app", { .schema_push = true, .hand_tracking_push = false }); + + REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT), std::logic_error); + REQUIRE_FALSE(bridge.wait_for(protocol::FrameType::OpenHand, std::chrono::milliseconds(200)).has_value()); +} + +TEST_CASE("RemotePluginSession forwards OPEN_HAND, HAND_SAMPLE and CLOSE bit-exact", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .hand_tracking_push = true }); + + uint16_t channel_id = 0; + { + const auto channel = session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT); + const auto open = bridge.wait_for(protocol::FrameType::OpenHand); + REQUIRE(open.has_value()); + REQUIRE(open->header.channel_id != protocol::kSessionChannelId); + REQUIRE(protocol::decode_open_hand(open->payload) == XR_HAND_RIGHT_EXT); + channel_id = open->header.channel_id; + + const auto joints = make_joints(0.25f); + channel->push(joints.data(), 123456789LL); + + const auto sample = bridge.wait_for(protocol::FrameType::HandSample); + REQUIRE(sample.has_value()); + REQUIRE(sample->header.channel_id == channel_id); + const protocol::HandSample decoded = protocol::decode_hand_sample(sample->payload); + REQUIRE(decoded.sample_time_local_common_clock_ns == 123456789LL); + for (size_t i = 0; i < joints.size(); ++i) + { + REQUIRE(std::memcmp(&decoded.joints[i], &joints[i], sizeof(XrHandJointLocationEXT)) == 0); + } + } + + const auto close = bridge.wait_for(protocol::FrameType::Close); + REQUIRE(close.has_value()); + REQUIRE(close->header.channel_id == channel_id); + REQUIRE(close->payload.empty()); +} + +TEST_CASE("RemotePluginSession forwards schema channels", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .schema_push = true }); + + const core::SchemaPusherConfig config{ .collection_id = "collection", + .max_flatbuffer_size = 64, + .tensor_identifier = "tensor", + .localized_name = "Localized", + .app_name = "test_app" }; + auto channel = session.create_schema_push_channel(config); + + const auto open = bridge.wait_for(protocol::FrameType::OpenSchema); + REQUIRE(open.has_value()); + REQUIRE(protocol::decode_open_schema(open->payload).localized_name == "Localized"); + + const std::vector bytes = { 1, 2, 3, 4, 5 }; + channel->push_buffer(bytes.data(), bytes.size(), 77, 88); + const auto sample = bridge.wait_for(protocol::FrameType::SchemaSample); + REQUIRE(sample.has_value()); + REQUIRE(sample->header.channel_id == open->header.channel_id); + const protocol::SchemaSample decoded = protocol::decode_schema_sample(sample->payload); + REQUIRE(decoded.bytes == bytes); + REQUIRE(decoded.sample_time_local_common_clock_ns == 77); + REQUIRE(decoded.sample_time_raw_device_clock_ns == 88); +} + +TEST_CASE("RemotePluginSession exposes bridge wrist samples through the pull channel", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + core::WristTrackingSourceConfig config; + config.mode = core::WristTrackingSourceMode::HandTracking; + config.left_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.5f, 0.0f, 0.0f } }; + config.right_aim_to_wrist = { { 0.0f, 0.0f, 0.0f, 1.0f }, { 0.0f, 0.0f, 0.0f } }; + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .wrist_tracking_pull = config }); + + const auto hello = bridge.wait_for(protocol::FrameType::Hello); + REQUIRE(hello.has_value()); + const auto declared = protocol::decode_hello(hello->payload).requirements.wrist_tracking_pull; + REQUIRE(declared.has_value()); + REQUIRE(declared->mode == core::WristTrackingSourceMode::HandTracking); + REQUIRE(declared->left_aim_to_wrist.position.x == 0.5f); + const auto open_pull = bridge.wait_for(protocol::FrameType::OpenPull); + REQUIRE(open_pull.has_value()); + + core::IPluginPullChannel& pull = session.pull_channel(); + core::IWristTrackingSource& source = pull.wrist_tracking_source(); + pull.update(); + core::WristTrackingSample before = source.query(XR_HAND_LEFT_EXT, 0); + REQUIRE_FALSE(before.valid); + REQUIRE_FALSE(before.tracked); + + protocol::WristSample wrist; + wrist.sample_time_local_common_clock_ns = 1000; + wrist.hand = XR_HAND_LEFT_EXT; + wrist.sample.valid = true; + wrist.sample.tracked = true; + wrist.sample.pose.position = { 0.1f, 1.2f, -0.4f }; + REQUIRE(bridge.send( + protocol::FrameType::WristSample, open_pull->header.channel_id, protocol::encode_wrist_sample(wrist))); + + REQUIRE(wait_until([&] { return source.query(XR_HAND_LEFT_EXT, 0).valid; })); + const core::WristTrackingSample left = source.query(XR_HAND_LEFT_EXT, 0); + REQUIRE(left.tracked); + REQUIRE(left.pose.position.y == 1.2f); + REQUIRE_FALSE(source.query(XR_HAND_RIGHT_EXT, 0).valid); +} + +TEST_CASE("RemotePluginSession answers every PING with a PONG", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .hand_tracking_push = true }); + REQUIRE(bridge.wait_for(protocol::FrameType::Hello).has_value()); + + for (int i = 0; i < 3; ++i) + { + const int64_t before = core::os_monotonic_now_ns(); + REQUIRE(bridge.send(protocol::FrameType::Ping, protocol::kSessionChannelId, {})); + const auto frame = bridge.wait_for(protocol::FrameType::Pong); + const int64_t after = core::os_monotonic_now_ns(); + REQUIRE(frame.has_value()); + REQUIRE(frame->header.channel_id == protocol::kSessionChannelId); + const protocol::Pong pong = protocol::decode_pong(frame->payload); + // Same host, same clock: before <= ping_sent (the PING header t_send) <= ping_received <= the PONG header + // t_send <= after. + REQUIRE(before <= pong.ping_sent_ns); + REQUIRE(pong.ping_sent_ns <= pong.ping_received_ns); + REQUIRE(pong.ping_received_ns <= frame->header.t_send_monotonic_ns); + REQUIRE(frame->header.t_send_monotonic_ns <= after); + } +} + +TEST_CASE("A PONG that waited in the send queue reports when it was sent, not when it was queued", + "[remote_device_bridge][unit]") +{ + auto transport = std::make_unique(); + BlockingTransport& link = *transport; + RemotePluginSession session(std::move(transport), "test_app", { .hand_tracking_push = true }); + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + const auto joints = make_joints(0.1f); + + // Stall the link: the sender thread blocks inside send() on this sample. + link.hold(true); + channel->push(joints.data(), 1); + REQUIRE(wait_until([&] { return link.blocked(); })); + + std::vector ping = protocol::encode_frame(protocol::FrameType::Ping, protocol::kSessionChannelId, {}); + const int64_t ping_sent_ns = core::os_monotonic_now_ns(); + protocol::stamp_header(ping, 1, ping_sent_ns); + link.deliver(std::move(ping)); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // the PONG is queued behind the stalled send + + const int64_t released_ns = core::os_monotonic_now_ns(); + link.hold(false); + const auto frame = link.wait_for(protocol::FrameType::Pong); + REQUIRE(frame.has_value()); + const protocol::Pong pong = protocol::decode_pong(frame->payload); + REQUIRE(pong.ping_sent_ns == ping_sent_ns); + REQUIRE(pong.ping_received_ns >= ping_sent_ns); + REQUIRE(pong.ping_received_ns < released_ns); + // The send time is when the PONG left after the stall, so t3 - t2 covers the time it waited in the queue. + REQUIRE(frame->header.t_send_monotonic_ns >= released_ns); +} + +TEST_CASE("RemotePluginSession keeps the session when the bridge refuses one channel", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + RemotePluginSession session( + connect_to(bridge.port()), "test_app", { .schema_push = true, .hand_tracking_push = true }); + core::SchemaPusherConfig config; + config.collection_id = "tactile"; + config.max_flatbuffer_size = 40960; + auto refused = session.create_schema_push_channel(config); + auto hand = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + const auto open_schema = bridge.wait_for(protocol::FrameType::OpenSchema); + const auto open_hand = bridge.wait_for(protocol::FrameType::OpenHand); + REQUIRE(open_schema.has_value()); + REQUIRE(open_hand.has_value()); + + SECTION("ERROR on a channel id refuses only that channel") + { + const std::string reason = + "schema 'tactile' sample size 40960 bytes exceeds bridge cap 32768 bytes (runtime " + "limit); channel refused"; + REQUIRE(bridge.send(protocol::FrameType::Error, open_schema->header.channel_id, protocol::encode_error(reason))); + REQUIRE(wait_until([&] { return session.stats().refused_channels.count(open_schema->header.channel_id) == 1; })); + REQUIRE(session.stats().refused_channels.at(open_schema->header.channel_id) == reason); + REQUIRE(session.connected()); + + const std::vector bytes(16, 0x5A); + refused->push_buffer(bytes.data(), bytes.size(), 1, 1); + const auto joints = make_joints(0.3f); + hand->push(joints.data(), 42); + const auto sample = bridge.wait_for(protocol::FrameType::HandSample); + REQUIRE(sample.has_value()); + REQUIRE(sample->header.channel_id == open_hand->header.channel_id); + REQUIRE_FALSE(bridge.wait_for(protocol::FrameType::SchemaSample, std::chrono::milliseconds(200)).has_value()); + REQUIRE(session.connected()); + + refused.reset(); // CLOSE still goes out so the bridge forgets the refused id + const auto close = bridge.wait_for(protocol::FrameType::Close); + REQUIRE(close.has_value()); + REQUIRE(close->header.channel_id == open_schema->header.channel_id); + } + SECTION("ERROR on channel 0 ends the session") + { + REQUIRE(bridge.send(protocol::FrameType::Error, protocol::kSessionChannelId, protocol::encode_error("fatal"))); + REQUIRE(wait_until([&] { return !session.connected(); })); + REQUIRE(session.stats().refused_channels.empty()); + } +} + +TEST_CASE("RemotePluginSession survives the bridge dropping the socket", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + { + RemotePluginSession session( + connect_to(bridge.port()), "test_app", { .hand_tracking_push = true, .wrist_tracking_pull = kWristConfig }); + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + REQUIRE(bridge.wait_for(protocol::FrameType::OpenHand).has_value()); + + bridge.drop_connection(); + REQUIRE(wait_until([&] { return !session.connected(); })); + + const auto joints = make_joints(0.5f); + REQUIRE_NOTHROW(channel->push(joints.data(), 1)); + REQUIRE_NOTHROW(session.pull_channel().update()); + REQUIRE_THROWS_AS(session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT), std::runtime_error); + REQUIRE_NOTHROW(channel.reset()); + } +} + +TEST_CASE("RemotePluginSession never reconnects after the link drops", "[remote_device_bridge][unit]") +{ + TcpSocket listener = TcpSocket::listen(Endpoint{ "127.0.0.1", 0 }, 4); + std::unique_ptr bridge_side; + std::thread accept_once( + [&] + { + bridge_side = TcpTransport::accept(listener, kWait); + std::vector hello; + if (bridge_side && bridge_side->receive(hello, kWait) == ReceiveResult::Ok) + { + send_frame(*bridge_side, protocol::FrameType::HelloAck, protocol::kSessionChannelId, 0, + protocol::encode_hello_ack(protocol::HelloAck{ true, "" })); + } + }); + RemotePluginSession session(connect_to(listener.local_port()), "test_app", { .hand_tracking_push = true }); + accept_once.join(); + REQUIRE(bridge_side); + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + + bridge_side->shutdown(); + REQUIRE(wait_until([&] { return !session.connected(); })); + const auto joints = make_joints(0.4f); + for (int i = 0; i < 20; ++i) + { + channel->push(joints.data(), i); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + // A new HELLO needs a new session (a restarted plugin); nothing connects back on its own. + REQUIRE(TcpTransport::accept(listener, std::chrono::milliseconds(500)) == nullptr); +} + +TEST_CASE("RemotePluginSession closes its channels and the socket on destruction", "[remote_device_bridge][unit]") +{ + FakeBridge bridge; + { + RemotePluginSession session(connect_to(bridge.port()), "test_app", { .hand_tracking_push = true }); + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + REQUIRE(bridge.wait_for(protocol::FrameType::OpenHand).has_value()); + } + REQUIRE(bridge.wait_for(protocol::FrameType::Close).has_value()); + REQUIRE(bridge.client_closed()); +} + +TEST_CASE("RemotePluginSession keeps the newest samples per channel while the link is stalled", + "[remote_device_bridge][unit]") +{ + auto transport = std::make_unique(); + BlockingTransport& link = *transport; + RemotePluginSession session(std::move(transport), "test_app", { .schema_push = true, .hand_tracking_push = true }); + core::SchemaPusherConfig filler_config; + filler_config.collection_id = "filler"; + auto filler = session.create_schema_push_channel(filler_config); + auto fast = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); // "A": 180 Hz + auto slow = session.create_hand_tracking_push_channel(XR_HAND_RIGHT_EXT); // "B": 60 Hz + REQUIRE(link.wait_for(protocol::FrameType::OpenSchema).has_value()); + const auto open_fast = link.wait_for(protocol::FrameType::OpenHand); + const auto open_slow = link.wait_for(protocol::FrameType::OpenHand); + REQUIRE(open_fast.has_value()); + REQUIRE(open_slow.has_value()); + const uint16_t fast_id = open_fast->header.channel_id; + const uint16_t slow_id = open_slow->header.channel_id; + + // Stall the link: the sender thread blocks inside send() on the filler sample, so every hand sample pushed + // from here on can only wait in its channel's queue. + link.hold(true); + const std::vector filler_bytes = { 0xAB }; + filler->push_buffer(filler_bytes.data(), filler_bytes.size(), 0, 0); + REQUIRE(wait_until([&] { return link.blocked(); })); + + const auto joints = make_joints(0.1f); + constexpr int kFastPushes = 180; + constexpr int kSlowPushes = kFastPushes / 3; + for (int i = 0; i < kFastPushes; ++i) + { + fast->push(joints.data(), 1000 + i); + if (i % 3 == 0) + { + slow->push(joints.data(), 2000 + i / 3); + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + const RemotePluginSession::Stats stalled = session.stats(); + REQUIRE(stalled.dropped_by_channel.at(fast_id) > 0); + REQUIRE(stalled.dropped_by_channel.at(slow_id) > 0); + REQUIRE(stalled.dropped_by_channel.at(fast_id) > stalled.dropped_by_channel.at(slow_id)); + REQUIRE(stalled.dropped_queue >= stalled.dropped_by_channel.at(fast_id) + stalled.dropped_by_channel.at(slow_id)); + REQUIRE(session.connected()); + + link.hold(false); + std::vector fast_times; + std::vector slow_times; + const auto deadline = std::chrono::steady_clock::now() + kWait; + while (std::chrono::steady_clock::now() < deadline && + (fast_times.empty() || fast_times.back() != 1000 + kFastPushes - 1 || slow_times.empty() || + slow_times.back() != 2000 + kSlowPushes - 1)) + { + const auto frame = link.wait_for(protocol::FrameType::HandSample, std::chrono::milliseconds(200)); + if (!frame.has_value()) + { + continue; + } + (frame->header.channel_id == fast_id ? fast_times : slow_times) + .push_back(protocol::decode_hand_sample(frame->payload).sample_time_local_common_clock_ns); + } + + // Each channel delivered its newest sample, in order, from a queue that never held more than 4. + REQUIRE_FALSE(fast_times.empty()); + REQUIRE_FALSE(slow_times.empty()); + REQUIRE(fast_times.back() == 1000 + kFastPushes - 1); + REQUIRE(slow_times.back() == 2000 + kSlowPushes - 1); + REQUIRE(fast_times.size() <= 4); + REQUIRE(slow_times.size() <= 4); + REQUIRE(std::is_sorted(fast_times.begin(), fast_times.end())); + REQUIRE(std::is_sorted(slow_times.begin(), slow_times.end())); + REQUIRE(session.stats().sent_hand == fast_times.size() + slow_times.size()); +} + +TEST_CASE("RemotePluginSession drops a channel's pending samples when the channel closes", "[remote_device_bridge][unit]") +{ + auto transport = std::make_unique(); + BlockingTransport& link = *transport; + RemotePluginSession session(std::move(transport), "test_app", { .schema_push = true, .hand_tracking_push = true }); + core::SchemaPusherConfig filler_config; + filler_config.collection_id = "filler"; + auto filler = session.create_schema_push_channel(filler_config); + REQUIRE(link.wait_for(protocol::FrameType::OpenSchema).has_value()); + + // Stall the sender on a filler sample. A paused TCP peer is not enough: with enlarged socket buffers the kernel + // absorbs the filler and the hand samples leave before the CLOSE. + link.hold(true); + const std::vector filler_bytes = { 0xCD }; + filler->push_buffer(filler_bytes.data(), filler_bytes.size(), 0, 0); + REQUIRE(wait_until([&] { return link.blocked(); })); + { + auto channel = session.create_hand_tracking_push_channel(XR_HAND_LEFT_EXT); + const auto joints = make_joints(0.2f); + for (int i = 0; i < 3; ++i) + { + channel->push(joints.data(), i); + } + } + link.hold(false); + + // OPEN_HAND and CLOSE go out in order; the samples queued behind the stalled filler never follow the CLOSE. + const auto open = link.wait_for(protocol::FrameType::OpenHand); + REQUIRE(open.has_value()); + const auto close = link.wait_for(protocol::FrameType::Close); + REQUIRE(close.has_value()); + REQUIRE(close->header.channel_id == open->header.channel_id); + REQUIRE_FALSE(link.wait_for(protocol::FrameType::HandSample, std::chrono::milliseconds(200)).has_value()); +}