From 2dab90b149cf05b8e67c8c41ae8f978f6c3b59f8 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 14 Oct 2026 20:16:35 -0600 Subject: [PATCH 1/7] Fix "local track published for unknown sid" warning --- src/ffi_client.cpp | 19 ++++++++--- src/ffi_client.h | 6 ++-- src/local_participant.cpp | 27 +++++++-------- .../test_local_track_publish_sid.cpp | 33 +++++++++++++++++++ 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/ffi_client.cpp b/src/ffi_client.cpp index 867eba94..2a3eba73 100644 --- a/src/ffi_client.cpp +++ b/src/ffi_client.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -694,9 +695,9 @@ std::future FfiClient::simulateScenarioAsync(uintptr_t room_handle, int sc } // Participant APIs Implementation -std::future FfiClient::publishTrackAsync(std::uint64_t local_participant_handle, - std::uint64_t track_handle, - const TrackPublishOptions& options) { +std::future FfiClient::publishTrackAsync( + std::uint64_t local_participant_handle, std::uint64_t track_handle, const TrackPublishOptions& options, + std::function on_success) { // Generate client-side async_id first const AsyncId async_id = generateAsyncId(); @@ -708,7 +709,8 @@ std::future FfiClient::publishTrackAsync(std::uint return event.has_publish_track() && event.publish_track().async_id() == async_id; }, // Handler: resolve with publication or throw error - [](const proto::FfiEvent& event, std::promise& pr) { + [on_success = std::move(on_success)](const proto::FfiEvent& event, + std::promise& pr) { const auto& cb = event.publish_track(); // Oneof message { string error = 2; OwnedTrackPublication publication = @@ -723,7 +725,14 @@ std::future FfiClient::publishTrackAsync(std::uint } const proto::OwnedTrackPublication& pub = cb.publication(); - pr.set_value(pub); + try { + if (on_success) { + on_success(pub); + } + pr.set_value(pub); + } catch (...) { + pr.set_exception(std::current_exception()); + } }); // Build and send the request diff --git a/src/ffi_client.h b/src/ffi_client.h index 0104de76..4f10b1fa 100644 --- a/src/ffi_client.h +++ b/src/ffi_client.h @@ -109,9 +109,9 @@ class LIVEKIT_INTERNAL_API FfiClient { std::future simulateScenarioAsync(uintptr_t room_handle, int scenario); // Participant APIs - std::future publishTrackAsync(std::uint64_t local_participant_handle, - std::uint64_t track_handle, - const TrackPublishOptions& options); + std::future publishTrackAsync( + std::uint64_t local_participant_handle, std::uint64_t track_handle, const TrackPublishOptions& options, + std::function on_success = {}); std::future unpublishTrackAsync(std::uint64_t local_participant_handle, const std::string& track_sid, bool stop_on_unpublish); std::future publishDataAsync(std::uint64_t local_participant_handle, const std::uint8_t* data_ptr, diff --git a/src/local_participant.cpp b/src/local_participant.cpp index 8dbc1fdc..2607963d 100644 --- a/src/local_participant.cpp +++ b/src/local_participant.cpp @@ -191,21 +191,18 @@ void LocalParticipant::publishTrack(const std::shared_ptr& track, const T if (track_handle == 0) { throw std::runtime_error("LocalParticipant::publishTrack: invalid track FFI handle"); } - auto fut = FfiClient::instance().publishTrackAsync(static_cast(participant_handle), - static_cast(track_handle), options); - - // Will throw if the async op fails (error in callback). - const proto::OwnedTrackPublication owned_pub = fut.get(); - - // Construct a LocalTrackPublication from the proto publication. - auto publication = std::make_shared(owned_pub); - - const std::string sid = publication->sid(); - { - const std::scoped_lock guard(published_tracks_mutex_); - published_tracks_by_sid_[sid] = std::weak_ptr(track); - track->setPublication(publication); - } + auto fut = FfiClient::instance().publishTrackAsync( + static_cast(participant_handle), static_cast(track_handle), options, + [this, track](const proto::OwnedTrackPublication& owned_pub) { + auto publication = std::make_shared(owned_pub); + const std::scoped_lock guard(published_tracks_mutex_); + published_tracks_by_sid_[publication->sid()] = track; + track->setPublication(publication); + }); + + // Publication state is installed on the FFI thread before this future is + // completed and before Rust can emit LocalTrackPublished. + (void)fut.get(); } std::shared_ptr LocalParticipant::publishVideoTrack(const std::string& name, diff --git a/src/tests/integration/test_local_track_publish_sid.cpp b/src/tests/integration/test_local_track_publish_sid.cpp index ee767450..21707f4b 100644 --- a/src/tests/integration/test_local_track_publish_sid.cpp +++ b/src/tests/integration/test_local_track_publish_sid.cpp @@ -17,9 +17,12 @@ #include #include +#include #include +#include #include #include +#include #include "../common/audio_utils.h" #include "../common/room_test_access.h" @@ -31,6 +34,30 @@ namespace { using namespace std::chrono_literals; +class LocalPublicationDelegate final : public RoomDelegate { +public: + void onLocalTrackPublished(Room&, const LocalTrackPublishedEvent& event) override { + if (!event.publication) { + return; + } + { + const std::scoped_lock guard(mutex_); + published_sids_.insert(event.publication->sid()); + } + cv_.notify_all(); + } + + bool waitForPublished(const std::string& sid, std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, [&] { return published_sids_.count(sid) != 0; }); + } + +private: + std::mutex mutex_; + std::condition_variable cv_; + std::unordered_set published_sids_; +}; + void expectTrackSidAssigned(const Track& track, const LocalTrackPublication& publication) { const std::string& track_sid = track.sid(); const std::string& publication_sid = publication.sid(); @@ -58,6 +85,8 @@ TEST_F(LocalTrackPublishSidTest, PublishVideoTrackAssignsSid) { failIfNotConfigured(); Room room; + LocalPublicationDelegate delegate; + room.setDelegate(&delegate); const RoomOptions room_options; ASSERT_TRUE(room.connect(config_.url, config_.token_a, room_options)); @@ -69,6 +98,7 @@ TEST_F(LocalTrackPublishSidTest, PublishVideoTrackAssignsSid) { ASSERT_NE(track->publication(), nullptr); expectTrackSidAssigned(*track, *track->publication()); + EXPECT_TRUE(delegate.waitForPublished(track->sid(), 5s)); lockLocalParticipant(room)->unpublishTrack(track->publication()->sid()); } @@ -76,6 +106,8 @@ TEST_F(LocalTrackPublishSidTest, PublishAudioTrackAssignsSid) { failIfNotConfigured(); Room room; + LocalPublicationDelegate delegate; + room.setDelegate(&delegate); const RoomOptions room_options; ASSERT_TRUE(room.connect(config_.url, config_.token_a, room_options)); @@ -87,6 +119,7 @@ TEST_F(LocalTrackPublishSidTest, PublishAudioTrackAssignsSid) { ASSERT_NE(track->publication(), nullptr); expectTrackSidAssigned(*track, *track->publication()); + EXPECT_TRUE(delegate.waitForPublished(track->sid(), 5s)); lockLocalParticipant(room)->unpublishTrack(track->publication()->sid()); } From 5adaeef0c0cbe5b0a29dcdc495751ac04b0dec17 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 17:55:31 -0700 Subject: [PATCH 2/7] Memory regression test --- .github/workflows/ci.yml | 1 + .github/workflows/tests.yml | 15 ++ docs/testing.md | 26 +++ scripts/memory-regression.sh | 95 +++++++++++ src/tests/CMakeLists.txt | 19 +++ src/tests/memory/source_lifecycle_tester.cpp | 160 +++++++++++++++++++ 6 files changed, 316 insertions(+) create mode 100755 scripts/memory-regression.sh create mode 100644 src/tests/memory/source_lifecycle_tester.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c20ff88..87152589 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: - client-sdk-rust - client-sdk-rust/** - cmake/** + - scripts/memory-regression.sh - scripts/set-test-tokens.sh - scripts/run-with-backtrace.sh - CMakeLists.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index efeaf448..c38a0a69 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -425,6 +425,21 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml + - name: Run Linux memory regression gate + if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' + timeout-minutes: 10 + shell: bash + env: + MALLOC_ARENA_MAX: "1" + run: | + set -euo pipefail + scripts/memory-regression.sh \ + --build-dir "${{ env.BUILD_DIR }}" \ + --iterations 100 \ + --warmup 20 \ + --max-rss-growth-kib 8192 \ + --max-thread-growth 0 + - name: Run stress tests if: matrix.e2e-testing && inputs.run_stress_tests timeout-minutes: ${{ inputs.stress_timeout_minutes }} diff --git a/docs/testing.md b/docs/testing.md index 33919c0e..383dc48a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -106,6 +106,32 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. +## Linux memory regression gate + +The Linux PR gate repeatedly initializes the SDK, creates and destroys unused +audio/video sources and tracks, then shuts the SDK down. It compares process +RSS and thread count after cycle 20 with cycle 100, failing on thread growth or +more than 8 MiB of RSS growth. + +The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits +glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a +controlled CI signal, not evidence that the default allocator returns all +freed pages to the OS. + +To reproduce locally (no LiveKit server or tokens are required): + +```bash +./build.sh release-tests +scripts/memory-regression.sh \ + --iterations 100 --warmup 20 \ + --max-rss-growth-kib 8192 --max-thread-growth 0 +``` + +The limits are command-line options so a longer local run can use different +budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends a compact +results table when `GITHUB_STEP_SUMMARY` is set. Hardware encoding and +connected-room lifecycle coverage are separate follow-up work. + ## Memory checks (valgrind) Run `valgrind` against the test binaries to check for memory leaks and other diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh new file mode 100755 index 00000000..c3f217d4 --- /dev/null +++ b/scripts/memory-regression.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Copyright 2026 LiveKit +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: memory-regression.sh [OPTIONS] + +Run the Linux memory lifecycle regression tester and append a table to the +GitHub step summary when GITHUB_STEP_SUMMARY is set. + +Options: + --build-dir PATH Build directory (default: build-release) + --iterations N Lifecycle cycles (default: 100) + --warmup N Baseline cycle (default: 20) + --max-rss-growth-kib N Allowed RSS growth (default: 8192) + --max-thread-growth N Allowed thread growth (default: 0) + -h, --help Show this help + +Environment: + MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_ITERATIONS, + MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, + MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX +EOF +} + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" +iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" +warmup="${MEMORY_REGRESSION_WARMUP:-20}" +max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" +max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" + +while (($#)); do + case "$1" in + --build-dir) build_dir="$2"; shift 2 ;; + --iterations) iterations="$2"; shift 2 ;; + --warmup) warmup="$2"; shift 2 ;; + --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; + --max-thread-growth) max_thread_growth="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ "${build_dir}" != /* ]]; then + build_dir="${repo_root}/${build_dir}" +fi +tester="${build_dir}/bin/livekit_memory_source_lifecycle_tester" +if [[ ! -x "${tester}" ]]; then + echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2 + exit 1 +fi + +set +e +output="$( + MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ + --iterations "${iterations}" \ + --warmup "${warmup}" \ + --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-thread-growth "${max_thread_growth}" 2>&1 +)" +status=$? +set -e +printf '%s\n' "${output}" + +report_regex='memory lifecycle: RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +if [[ -n "${GITHUB_STEP_SUMMARY:-}" && "${output}" =~ ${report_regex} ]]; then + { + echo "## Linux memory regression" + echo + echo "| Metric | Baseline | Final | Growth | Limit |" + echo "|---|---:|---:|---:|---:|" + echo "| RSS | ${BASH_REMATCH[1]} KiB | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${max_rss_growth_kib} KiB |" + echo "| Threads | ${BASH_REMATCH[4]} | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${max_thread_growth} |" + echo + echo "**Verdict: ${BASH_REMATCH[7]}**" + } >> "${GITHUB_STEP_SUMMARY}" +fi + +exit "${status}" diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 9583af73..c9080662 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -348,3 +348,22 @@ endif() if(TARGET livekit_stress_tests) add_dependencies(run_all_tests livekit_stress_tests) endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_executable(livekit_memory_source_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/memory/source_lifecycle_tester.cpp" + ) + target_link_libraries(livekit_memory_source_lifecycle_tester PRIVATE livekit) + target_include_directories(livekit_memory_source_lifecycle_tester PRIVATE + ${LIVEKIT_ROOT_DIR}/include + ) + add_custom_command(TARGET livekit_memory_source_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.so" + $ + COMMENT "Copying shared libraries next to livekit_memory_source_lifecycle_tester" + ) +endif() diff --git a/src/tests/memory/source_lifecycle_tester.cpp b/src/tests/memory/source_lifecycle_tester.cpp new file mode 100644 index 00000000..197da1ee --- /dev/null +++ b/src/tests/memory/source_lifecycle_tester.cpp @@ -0,0 +1,160 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Options { + std::uint64_t iterations{100}; + std::uint64_t warmup{20}; + std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_thread_growth{0}; +}; + +struct ProcessSample { + std::uint64_t rss_kib{}; + std::uint64_t threads{}; +}; + +std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { + std::uint64_t parsed = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + throw std::runtime_error("invalid value for " + std::string(option)); + } + return parsed; +} + +Options parseOptions(int argc, char* argv[]) { + Options options; + for (int index = 1; index < argc; ++index) { + const std::string_view option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + std::string(option)); + } + const std::string_view value(argv[++index]); + if (option == "--iterations") { + options.iterations = parseUnsigned(value, option); + } else if (option == "--warmup") { + options.warmup = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-kib") { + options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-thread-growth") { + options.max_thread_growth = parseUnsigned(value, option); + } else { + throw std::runtime_error("unknown option: " + std::string(option)); + } + } + if (options.warmup == 0 || options.warmup >= options.iterations) { + throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); + } + return options; +} + +std::uint64_t statusValue(std::string_view line) { + const auto separator = line.find(':'); + if (separator == std::string_view::npos) { + throw std::runtime_error("malformed /proc/self/status value"); + } + line.remove_prefix(separator + 1); + while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { + line.remove_prefix(1); + } + const auto end = line.find_first_not_of("0123456789"); + return parseUnsigned(line.substr(0, end), "/proc/self/status"); +} + +ProcessSample sampleProcess() { + ProcessSample sample; + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmRSS:", 0) == 0) { + sample.rss_kib = statusValue(line); + } else if (line.rfind("Threads:", 0) == 0) { + sample.threads = statusValue(line); + } + } + if (sample.rss_kib == 0 || sample.threads == 0) { + throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); + } + return sample; +} + +void runCycle() { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("LiveKit initialization failed"); + } + try { + { + auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + auto video_source = std::make_shared(640, 360); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); + if (!audio_track || !video_track) { + throw std::runtime_error("failed to create local tracks"); + } + } + } catch (...) { + livekit::shutdown(); + throw; + } + livekit::shutdown(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); +} + +} // namespace + +int main(int argc, char* argv[]) { + try { + const Options options = parseOptions(argc, argv); + ProcessSample baseline; + ProcessSample final; + for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { + runCycle(); + if (iteration == options.warmup) { + baseline = sampleProcess(); + } else if (iteration == options.iterations) { + final = sampleProcess(); + } + } + + const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); + const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); + const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && + thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: RSS " << baseline.rss_kib << " -> " << final.rss_kib << " KiB (" << rss_growth + << " KiB), threads " << baseline.threads << " -> " << final.threads << " (" << thread_growth + << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + return 1; + } +} From a89e5ebaa328f8a5971c8a38b468797e263e84ef Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:01:44 -0700 Subject: [PATCH 3/7] More scenarios --- .github/workflows/tests.yml | 14 +- docs/testing.md | 35 +- scripts/memory-regression.sh | 92 +++-- src/tests/CMakeLists.txt | 16 +- src/tests/memory/lifecycle_tester.cpp | 334 +++++++++++++++++++ src/tests/memory/source_lifecycle_tester.cpp | 160 --------- 6 files changed, 451 insertions(+), 200 deletions(-) create mode 100644 src/tests/memory/lifecycle_tester.cpp delete mode 100644 src/tests/memory/source_lifecycle_tester.cpp diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c38a0a69..9accd92d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -357,7 +357,11 @@ jobs: --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml" - name: Start livekit-server - if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: >- + matrix.e2e-testing && + (inputs.integration_repeat > 0 || + inputs.run_stress_tests || + (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release')) id: livekit_server uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1 with: @@ -368,7 +372,11 @@ jobs: # Needed by token helper script - name: Install livekit-cli - if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: >- + matrix.e2e-testing && + (inputs.integration_repeat > 0 || + inputs.run_stress_tests || + (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release')) shell: bash env: # Windows installs lk via `gh api` / `gh release download`, which need this env var @@ -433,8 +441,10 @@ jobs: MALLOC_ARENA_MAX: "1" run: | set -euo pipefail + source scripts/set-test-tokens.sh scripts/memory-regression.sh \ --build-dir "${{ env.BUILD_DIR }}" \ + --scenario all \ --iterations 100 \ --warmup 20 \ --max-rss-growth-kib 8192 \ diff --git a/docs/testing.md b/docs/testing.md index 383dc48a..90ebb4d5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -44,6 +44,7 @@ __Note:__ The tests require tokens and a running LiveKit server. See the section | `livekit_unit_tests` | Pure unit tests (no server required) | | `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality | | `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) | +| `livekit_memory_lifecycle_tester` | Linux lifecycle RSS and thread regression scenarios | ## Running a local LiveKit server for tests @@ -108,29 +109,47 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate repeatedly initializes the SDK, creates and destroys unused -audio/video sources and tracks, then shuts the SDK down. It compares process -RSS and thread count after cycle 20 with cycle 100, failing on thread growth or -more than 8 MiB of RSS growth. +The Linux PR gate runs three lifecycle scenarios in separate processes: + +- `sdk-sources` repeatedly initializes the SDK, creates and destroys unused + audio/video sources and tracks, then shuts the SDK down. +- `room-client-leave` repeatedly connects, disconnects, and destroys a room. +- `room-server-delete` repeatedly connects and uses `lk` to delete the local + test room, then verifies the disconnect and end-of-stream callbacks. + +Each scenario compares process RSS and thread count after cycle 20 with cycle +100, failing on thread growth or more than 8 MiB of RSS growth. The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -To reproduce locally (no LiveKit server or tokens are required): +The offline source scenario requires no LiveKit server or tokens: ```bash ./build.sh release-tests scripts/memory-regression.sh \ + --scenario sdk-sources \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` +To run either room scenario, start `livekit-server --dev`, install the `lk` +CLI, and source the local test tokens. Use `--scenario all` to run all three +scenarios, each in a fresh tester process: + +```bash +source scripts/set-test-tokens.sh +scripts/memory-regression.sh --scenario room-client-leave +scripts/memory-regression.sh --scenario room-server-delete +scripts/memory-regression.sh --scenario all +``` + The limits are command-line options so a longer local run can use different -budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends a compact -results table when `GITHUB_STEP_SUMMARY` is set. Hardware encoding and -connected-room lifecycle coverage are separate follow-up work. +budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends one results +row per scenario when `GITHUB_STEP_SUMMARY` is set. Hardware encoding coverage +is separate follow-up work. ## Memory checks (valgrind) diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index c3f217d4..1032d769 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,6 +25,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) + --scenario NAME sdk-sources, room-client-leave, + room-server-delete, or all (default: sdk-sources) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -32,14 +34,17 @@ Options: -h, --help Show this help Environment: - MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_ITERATIONS, + MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO, + MEMORY_REGRESSION_ITERATIONS, MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, - MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX + MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. + Room scenarios also require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF } repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" +scenario="${MEMORY_REGRESSION_SCENARIO:-sdk-sources}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" @@ -48,6 +53,7 @@ max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" while (($#)); do case "$1" in --build-dir) build_dir="$2"; shift 2 ;; + --scenario) scenario="$2"; shift 2 ;; --iterations) iterations="$2"; shift 2 ;; --warmup) warmup="$2"; shift 2 ;; --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; @@ -60,36 +66,78 @@ done if [[ "${build_dir}" != /* ]]; then build_dir="${repo_root}/${build_dir}" fi -tester="${build_dir}/bin/livekit_memory_source_lifecycle_tester" +tester="${build_dir}/bin/livekit_memory_lifecycle_tester" if [[ ! -x "${tester}" ]]; then echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2 exit 1 fi -set +e -output="$( - MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ - --iterations "${iterations}" \ - --warmup "${warmup}" \ - --max-rss-growth-kib "${max_rss_growth_kib}" \ - --max-thread-growth "${max_thread_growth}" 2>&1 -)" -status=$? -set -e -printf '%s\n' "${output}" +case "${scenario}" in + sdk-sources|room-client-leave|room-server-delete) + scenarios=("${scenario}") + ;; + all) + scenarios=(sdk-sources room-client-leave room-server-delete) + ;; + *) + echo "ERROR: invalid scenario: ${scenario}" >&2 + usage >&2 + exit 2 + ;; +esac -report_regex='memory lifecycle: RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' -if [[ -n "${GITHUB_STEP_SUMMARY:-}" && "${output}" =~ ${report_regex} ]]; then +for current_scenario in "${scenarios[@]}"; do + if [[ "${current_scenario}" != "sdk-sources" ]] && + [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then + echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 + echo "Run: source scripts/set-test-tokens.sh" >&2 + exit 1 + fi +done + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { echo "## Linux memory regression" echo - echo "| Metric | Baseline | Final | Growth | Limit |" - echo "|---|---:|---:|---:|---:|" - echo "| RSS | ${BASH_REMATCH[1]} KiB | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${max_rss_growth_kib} KiB |" - echo "| Threads | ${BASH_REMATCH[4]} | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${max_thread_growth} |" + echo "Limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "**Verdict: ${BASH_REMATCH[7]}**" + echo "| Scenario | RSS baseline | RSS final | RSS growth | Threads baseline | Threads final | Thread growth | Verdict |" + echo "|---|---:|---:|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -exit "${status}" +report_regex='memory lifecycle: scenario=([a-z-]+), RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +overall_status=0 +for current_scenario in "${scenarios[@]}"; do + set +e + output="$( + MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ + --scenario "${current_scenario}" \ + --iterations "${iterations}" \ + --warmup "${warmup}" \ + --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-thread-growth "${max_thread_growth}" 2>&1 + )" + status=$? + set -e + printf '%s\n' "${output}" + + if [[ "${output}" =~ ${report_regex} ]]; then + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "| ${BASH_REMATCH[1]} | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} |" \ + >> "${GITHUB_STEP_SUMMARY}" + fi + else + echo "ERROR: ${current_scenario} did not produce a memory report." >&2 + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "| ${current_scenario} | — | — | — | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + fi + status=1 + fi + + if [[ "${status}" -ne 0 ]]; then + overall_status=1 + fi +done + +exit "${overall_status}" diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index c9080662..1204193c 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -350,20 +350,20 @@ if(TARGET livekit_stress_tests) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - add_executable(livekit_memory_source_lifecycle_tester - "${CMAKE_CURRENT_SOURCE_DIR}/memory/source_lifecycle_tester.cpp" + add_executable(livekit_memory_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/memory/lifecycle_tester.cpp" ) - target_link_libraries(livekit_memory_source_lifecycle_tester PRIVATE livekit) - target_include_directories(livekit_memory_source_lifecycle_tester PRIVATE + target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) + target_include_directories(livekit_memory_lifecycle_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include ) - add_custom_command(TARGET livekit_memory_source_lifecycle_tester POST_BUILD + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different "$/liblivekit_ffi.so" - $ - COMMENT "Copying shared libraries next to livekit_memory_source_lifecycle_tester" + $ + COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester" ) endif() diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp new file mode 100644 index 00000000..8fbb7535 --- /dev/null +++ b/src/tests/memory/lifecycle_tester.cpp @@ -0,0 +1,334 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace std::chrono_literals; + +constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; + +enum class Scenario { + SdkSources, + RoomClientLeave, + RoomServerDelete, +}; + +struct Options { + Scenario scenario{Scenario::SdkSources}; + std::uint64_t iterations{100}; + std::uint64_t warmup{20}; + std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_thread_growth{0}; +}; + +struct ProcessSample { + std::uint64_t rss_kib{}; + std::uint64_t threads{}; +}; + +struct RoomCredentials { + std::string url; + std::string token; +}; + +class SdkLifetime { +public: + SdkLifetime() { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("LiveKit initialization failed"); + } + } + + ~SdkLifetime() { livekit::shutdown(); } + + SdkLifetime(const SdkLifetime&) = delete; + SdkLifetime& operator=(const SdkLifetime&) = delete; +}; + +class DisconnectTrackingDelegate final : public livekit::RoomDelegate { +public: + void onDisconnected(livekit::Room&, const livekit::DisconnectedEvent& event) override { + { + const std::scoped_lock lock(mutex_); + reason_ = event.reason; + ++disconnect_count_; + } + condition_.notify_all(); + } + + void onRoomEos(livekit::Room&, const livekit::RoomEosEvent&) override { + { + const std::scoped_lock lock(mutex_); + ++eos_count_; + } + condition_.notify_all(); + } + + bool waitForTeardown(std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return condition_.wait_for(lock, timeout, [this]() { return disconnect_count_ > 0 && eos_count_ > 0; }); + } + + void requireDisconnect(livekit::DisconnectReason expected_reason) const { + const std::scoped_lock lock(mutex_); + if (disconnect_count_ != 1 || reason_ != expected_reason) { + throw std::runtime_error("unexpected room disconnect event"); + } + } + + void requireServerTeardown() const { + const std::scoped_lock lock(mutex_); + if (disconnect_count_ != 1 || eos_count_ != 1 || reason_ != livekit::DisconnectReason::RoomDeleted) { + throw std::runtime_error("unexpected server room teardown events"); + } + } + +private: + mutable std::mutex mutex_; + std::condition_variable condition_; + livekit::DisconnectReason reason_{livekit::DisconnectReason::Unknown}; + int disconnect_count_{0}; + int eos_count_{0}; +}; + +std::string_view scenarioName(Scenario scenario) { + switch (scenario) { + case Scenario::SdkSources: + return "sdk-sources"; + case Scenario::RoomClientLeave: + return "room-client-leave"; + case Scenario::RoomServerDelete: + return "room-server-delete"; + } + throw std::runtime_error("unknown scenario"); +} + +Scenario parseScenario(std::string_view value) { + if (value == "sdk-sources") { + return Scenario::SdkSources; + } + if (value == "room-client-leave") { + return Scenario::RoomClientLeave; + } + if (value == "room-server-delete") { + return Scenario::RoomServerDelete; + } + throw std::runtime_error("invalid value for --scenario"); +} + +std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { + std::uint64_t parsed = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + throw std::runtime_error("invalid value for " + std::string(option)); + } + return parsed; +} + +Options parseOptions(int argc, char* argv[]) { + Options options; + for (int index = 1; index < argc; ++index) { + const std::string_view option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + std::string(option)); + } + const std::string_view value(argv[++index]); + if (option == "--scenario") { + options.scenario = parseScenario(value); + } else if (option == "--iterations") { + options.iterations = parseUnsigned(value, option); + } else if (option == "--warmup") { + options.warmup = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-kib") { + options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-thread-growth") { + options.max_thread_growth = parseUnsigned(value, option); + } else { + throw std::runtime_error("unknown option: " + std::string(option)); + } + } + if (options.warmup == 0 || options.warmup >= options.iterations) { + throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); + } + return options; +} + +std::uint64_t statusValue(std::string_view line) { + const auto separator = line.find(':'); + if (separator == std::string_view::npos) { + throw std::runtime_error("malformed /proc/self/status value"); + } + line.remove_prefix(separator + 1); + while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { + line.remove_prefix(1); + } + const auto end = line.find_first_not_of("0123456789"); + return parseUnsigned(line.substr(0, end), "/proc/self/status"); +} + +ProcessSample sampleProcess() { + ProcessSample sample; + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmRSS:", 0) == 0) { + sample.rss_kib = statusValue(line); + } else if (line.rfind("Threads:", 0) == 0) { + sample.threads = statusValue(line); + } + } + if (sample.rss_kib == 0 || sample.threads == 0) { + throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); + } + return sample; +} + +RoomCredentials roomCredentials() { + const char* url = std::getenv("LIVEKIT_URL"); + const char* token = std::getenv("LIVEKIT_TOKEN_A"); + if (url == nullptr || token == nullptr || *url == '\0' || *token == '\0') { + throw std::runtime_error("room scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); + } + return {url, token}; +} + +void runSdkSourcesCycle() { + SdkLifetime sdk; + auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + auto video_source = std::make_shared(640, 360); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); + if (!audio_track || !video_track) { + throw std::runtime_error("failed to create local tracks"); + } +} + +void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { + if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { + throw std::runtime_error("room connection failed"); + } + if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { + throw std::runtime_error("room did not reach a connected state"); + } +} + +void runRoomClientLeaveCycle(const RoomCredentials& credentials) { + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("client room disconnect did not tear down local state"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); +} + +void runRoomServerDeleteCycle(const RoomCredentials& credentials) { + if (std::string_view(credentials.url) != kLocalTestLiveKitUrl) { + throw std::runtime_error("room-server-delete requires LIVEKIT_URL=ws://localhost:7880"); + } + + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + // The room name is fixed rather than interpolating untrusted environment data into the command. + if (std::system("lk --dev room delete --yes cpp_data_track_test") != 0) { + throw std::runtime_error("failed to delete local test room"); + } + if (!delegate.waitForTeardown(10s)) { + throw std::runtime_error("timed out waiting for server room teardown"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("server room deletion did not tear down local state"); + } + delegate.requireServerTeardown(); +} + +void runCycle(Scenario scenario, const RoomCredentials& credentials) { + switch (scenario) { + case Scenario::SdkSources: + runSdkSourcesCycle(); + return; + case Scenario::RoomClientLeave: + runRoomClientLeaveCycle(credentials); + return; + case Scenario::RoomServerDelete: + runRoomServerDeleteCycle(credentials); + return; + } + throw std::runtime_error("unknown scenario"); +} + +} // namespace + +int main(int argc, char* argv[]) { + try { + const Options options = parseOptions(argc, argv); + const bool room_scenario = options.scenario != Scenario::SdkSources; + const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; + std::unique_ptr room_sdk; + if (room_scenario) { + room_sdk = std::make_unique(); + } + + ProcessSample baseline; + ProcessSample final; + for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { + runCycle(options.scenario, credentials); + std::this_thread::sleep_for(100ms); + if (iteration == options.warmup) { + baseline = sampleProcess(); + } else if (iteration == options.iterations) { + final = sampleProcess(); + } + } + + const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); + const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); + const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && + thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS " << baseline.rss_kib + << " -> " << final.rss_kib << " KiB (" << rss_growth << " KiB), threads " << baseline.threads << " -> " + << final.threads << " (" << thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/tests/memory/source_lifecycle_tester.cpp b/src/tests/memory/source_lifecycle_tester.cpp deleted file mode 100644 index 197da1ee..00000000 --- a/src/tests/memory/source_lifecycle_tester.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2026 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -struct Options { - std::uint64_t iterations{100}; - std::uint64_t warmup{20}; - std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; - std::uint64_t max_thread_growth{0}; -}; - -struct ProcessSample { - std::uint64_t rss_kib{}; - std::uint64_t threads{}; -}; - -std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { - std::uint64_t parsed = 0; - const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { - throw std::runtime_error("invalid value for " + std::string(option)); - } - return parsed; -} - -Options parseOptions(int argc, char* argv[]) { - Options options; - for (int index = 1; index < argc; ++index) { - const std::string_view option(argv[index]); - if (index + 1 >= argc) { - throw std::runtime_error("missing value for " + std::string(option)); - } - const std::string_view value(argv[++index]); - if (option == "--iterations") { - options.iterations = parseUnsigned(value, option); - } else if (option == "--warmup") { - options.warmup = parseUnsigned(value, option); - } else if (option == "--max-rss-growth-kib") { - options.max_rss_growth_kib = parseUnsigned(value, option); - } else if (option == "--max-thread-growth") { - options.max_thread_growth = parseUnsigned(value, option); - } else { - throw std::runtime_error("unknown option: " + std::string(option)); - } - } - if (options.warmup == 0 || options.warmup >= options.iterations) { - throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); - } - return options; -} - -std::uint64_t statusValue(std::string_view line) { - const auto separator = line.find(':'); - if (separator == std::string_view::npos) { - throw std::runtime_error("malformed /proc/self/status value"); - } - line.remove_prefix(separator + 1); - while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { - line.remove_prefix(1); - } - const auto end = line.find_first_not_of("0123456789"); - return parseUnsigned(line.substr(0, end), "/proc/self/status"); -} - -ProcessSample sampleProcess() { - ProcessSample sample; - std::ifstream status("/proc/self/status"); - std::string line; - while (std::getline(status, line)) { - if (line.rfind("VmRSS:", 0) == 0) { - sample.rss_kib = statusValue(line); - } else if (line.rfind("Threads:", 0) == 0) { - sample.threads = statusValue(line); - } - } - if (sample.rss_kib == 0 || sample.threads == 0) { - throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); - } - return sample; -} - -void runCycle() { - if (!livekit::initialize(livekit::LogLevel::Warn)) { - throw std::runtime_error("LiveKit initialization failed"); - } - try { - { - auto audio_source = std::make_shared(48'000, 1, 100); - auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); - auto video_source = std::make_shared(640, 360); - auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); - if (!audio_track || !video_track) { - throw std::runtime_error("failed to create local tracks"); - } - } - } catch (...) { - livekit::shutdown(); - throw; - } - livekit::shutdown(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); -} - -} // namespace - -int main(int argc, char* argv[]) { - try { - const Options options = parseOptions(argc, argv); - ProcessSample baseline; - ProcessSample final; - for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { - runCycle(); - if (iteration == options.warmup) { - baseline = sampleProcess(); - } else if (iteration == options.iterations) { - final = sampleProcess(); - } - } - - const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); - const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); - const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && - thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: RSS " << baseline.rss_kib << " -> " << final.rss_kib << " KiB (" << rss_growth - << " KiB), threads " << baseline.threads << " -> " << final.threads << " (" << thread_growth - << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; - return passed ? 0 : 1; - } catch (const std::exception& error) { - std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; - return 1; - } -} From a92fea49f7a9b71f663d56b4bcaa72ce720ae751 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:31:38 -0700 Subject: [PATCH 4/7] Split audio/video track --- docs/testing.md | 18 +++++---- scripts/memory-regression.sh | 12 +++--- src/tests/memory/lifecycle_tester.cpp | 57 +++++++++++++++++++++------ 3 files changed, 61 insertions(+), 26 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 90ebb4d5..3e7c264d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -109,10 +109,12 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate runs three lifecycle scenarios in separate processes: +The Linux PR gate runs four lifecycle scenarios in separate processes: -- `sdk-sources` repeatedly initializes the SDK, creates and destroys unused - audio/video sources and tracks, then shuts the SDK down. +- `audio-track` repeatedly initializes the SDK, creates an audio source and + track, captures synthetic PCM frames, destroys them, and shuts the SDK down. +- `video-track` repeatedly initializes the SDK, creates a video source and + track, captures synthetic I420 frames, destroys them, and shuts the SDK down. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. @@ -125,18 +127,20 @@ glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -The offline source scenario requires no LiveKit server or tokens: +The offline track scenarios require no LiveKit server or tokens: ```bash ./build.sh release-tests -scripts/memory-regression.sh \ - --scenario sdk-sources \ +scripts/memory-regression.sh --scenario audio-track \ + --iterations 100 --warmup 20 \ + --max-rss-growth-kib 8192 --max-thread-growth 0 +scripts/memory-regression.sh --scenario video-track \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` To run either room scenario, start `livekit-server --dev`, install the `lk` -CLI, and source the local test tokens. Use `--scenario all` to run all three +CLI, and source the local test tokens. Use `--scenario all` to run all four scenarios, each in a fresh tester process: ```bash diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 1032d769..74ba29da 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,8 +25,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) - --scenario NAME sdk-sources, room-client-leave, - room-server-delete, or all (default: sdk-sources) + --scenario NAME audio-track, video-track, room-client-leave, + room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -44,7 +44,7 @@ EOF repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" -scenario="${MEMORY_REGRESSION_SCENARIO:-sdk-sources}" +scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" @@ -73,11 +73,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - sdk-sources|room-client-leave|room-server-delete) + audio-track|video-track|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(sdk-sources room-client-leave room-server-delete) + scenarios=(audio-track video-track room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -87,7 +87,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" != "sdk-sources" ]] && + if [[ "${current_scenario}" == room-* ]] && [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 8fbb7535..46576c9d 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -16,6 +16,7 @@ #include +#include #include #include #include @@ -36,15 +37,17 @@ namespace { using namespace std::chrono_literals; constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; +constexpr int kFramesPerCycle = 3; enum class Scenario { - SdkSources, + AudioTrack, + VideoTrack, RoomClientLeave, RoomServerDelete, }; struct Options { - Scenario scenario{Scenario::SdkSources}; + Scenario scenario{Scenario::AudioTrack}; std::uint64_t iterations{100}; std::uint64_t warmup{20}; std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; @@ -123,8 +126,10 @@ class DisconnectTrackingDelegate final : public livekit::RoomDelegate { std::string_view scenarioName(Scenario scenario) { switch (scenario) { - case Scenario::SdkSources: - return "sdk-sources"; + case Scenario::AudioTrack: + return "audio-track"; + case Scenario::VideoTrack: + return "video-track"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -134,8 +139,11 @@ std::string_view scenarioName(Scenario scenario) { } Scenario parseScenario(std::string_view value) { - if (value == "sdk-sources") { - return Scenario::SdkSources; + if (value == "audio-track") { + return Scenario::AudioTrack; + } + if (value == "video-track") { + return Scenario::VideoTrack; } if (value == "room-client-leave") { return Scenario::RoomClientLeave; @@ -222,14 +230,33 @@ RoomCredentials roomCredentials() { return {url, token}; } -void runSdkSourcesCycle() { +void runAudioTrackCycle() { SdkLifetime sdk; - auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_source = std::make_shared(48'000, 1); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + if (!audio_track) { + throw std::runtime_error("failed to create local audio track"); + } + + auto frame = livekit::AudioFrame::create(48'000, 1, 480); + std::fill(frame.data().begin(), frame.data().end(), 100); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + audio_source->captureFrame(frame, 1'000); + } +} + +void runVideoTrackCycle() { + SdkLifetime sdk; auto video_source = std::make_shared(640, 360); auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); - if (!audio_track || !video_track) { - throw std::runtime_error("failed to create local tracks"); + if (!video_track) { + throw std::runtime_error("failed to create local video track"); + } + + auto frame = livekit::VideoFrame::create(640, 360, livekit::VideoBufferType::I420); + std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + video_source->captureFrame(frame); } } @@ -282,8 +309,11 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { void runCycle(Scenario scenario, const RoomCredentials& credentials) { switch (scenario) { - case Scenario::SdkSources: - runSdkSourcesCycle(); + case Scenario::AudioTrack: + runAudioTrackCycle(); + return; + case Scenario::VideoTrack: + runVideoTrackCycle(); return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); @@ -300,7 +330,8 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool room_scenario = options.scenario != Scenario::SdkSources; + const bool room_scenario = + options.scenario == Scenario::RoomClientLeave || options.scenario == Scenario::RoomServerDelete; const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; std::unique_ptr room_sdk; if (room_scenario) { From 8ba2935cb17c70413583225ac49f356dc491150a Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:46:37 -0700 Subject: [PATCH 5/7] Warm up stats vs. final --- docs/testing.md | 15 +++-- scripts/memory-regression.sh | 25 +++++---- src/tests/memory/lifecycle_tester.cpp | 80 ++++++++++++++++++++++----- 3 files changed, 91 insertions(+), 29 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 3e7c264d..2378b9bf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -109,18 +109,22 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate runs four lifecycle scenarios in separate processes: +The Linux PR gate runs five lifecycle scenarios in separate processes: - `audio-track` repeatedly initializes the SDK, creates an audio source and track, captures synthetic PCM frames, destroys them, and shuts the SDK down. - `video-track` repeatedly initializes the SDK, creates a video source and track, captures synthetic I420 frames, destroys them, and shuts the SDK down. +- `data-track` repeatedly connects, publishes a data track, pushes synthetic + payloads, unpublishes it, and disconnects. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. -Each scenario compares process RSS and thread count after cycle 20 with cycle -100, failing on thread growth or more than 8 MiB of RSS growth. +Each scenario reports process RSS and thread count at cycle 0, after warmup +cycle 20, and after final cycle 100. The cycle 0 to warmup growth shows expected +one-time initialization; the gate applies to warmup-to-final growth and fails +on thread growth or more than 8 MiB of RSS growth. The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a @@ -139,12 +143,13 @@ scripts/memory-regression.sh --scenario video-track \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` -To run either room scenario, start `livekit-server --dev`, install the `lk` -CLI, and source the local test tokens. Use `--scenario all` to run all four +To run the data-track or room scenarios, start `livekit-server --dev`, install +the `lk` CLI, and source the local test tokens. Use `--scenario all` to run all five scenarios, each in a fresh tester process: ```bash source scripts/set-test-tokens.sh +scripts/memory-regression.sh --scenario data-track scripts/memory-regression.sh --scenario room-client-leave scripts/memory-regression.sh --scenario room-server-delete scripts/memory-regression.sh --scenario all diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 74ba29da..d805ffc2 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,8 +25,9 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) - --scenario NAME audio-track, video-track, room-client-leave, - room-server-delete, or all (default: audio-track) + --scenario NAME audio-track, video-track, data-track, + room-client-leave, room-server-delete, or all + (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -73,11 +74,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - audio-track|video-track|room-client-leave|room-server-delete) + audio-track|video-track|data-track|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(audio-track video-track room-client-leave room-server-delete) + scenarios=(audio-track video-track data-track room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -87,7 +88,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" == room-* ]] && + if [[ "${current_scenario}" == "data-track" || "${current_scenario}" == room-* ]] && [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 @@ -99,14 +100,14 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { echo "## Linux memory regression" echo - echo "Limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." + echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | RSS baseline | RSS final | RSS growth | Threads baseline | Threads final | Thread growth | Verdict |" - echo "|---|---:|---:|---:|---:|---:|---:|---|" + echo "| Scenario | Metric | Cycle 0 | Warmup (${warmup}) | Final (${iterations}) | 0 → warmup | Warmup → final | Verdict |" + echo "|---|---|---:|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -report_regex='memory lifecycle: scenario=([a-z-]+), RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' overall_status=0 for current_scenario in "${scenarios[@]}"; do set +e @@ -124,8 +125,10 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${BASH_REMATCH[1]} | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} |" \ - >> "${GITHUB_STEP_SUMMARY}" + { + echo "| ${BASH_REMATCH[1]} | RSS | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[5]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[6]} KiB | ${BASH_REMATCH[12]} |" + echo "| ${BASH_REMATCH[1]} | Threads | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} | ${BASH_REMATCH[10]} | ${BASH_REMATCH[9]} | ${BASH_REMATCH[11]} | ${BASH_REMATCH[12]} |" + } >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 46576c9d..7c8cfc9d 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -42,6 +43,7 @@ constexpr int kFramesPerCycle = 3; enum class Scenario { AudioTrack, VideoTrack, + DataTrack, RoomClientLeave, RoomServerDelete, }; @@ -130,6 +132,8 @@ std::string_view scenarioName(Scenario scenario) { return "audio-track"; case Scenario::VideoTrack: return "video-track"; + case Scenario::DataTrack: + return "data-track"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -145,6 +149,9 @@ Scenario parseScenario(std::string_view value) { if (value == "video-track") { return Scenario::VideoTrack; } + if (value == "data-track") { + return Scenario::DataTrack; + } if (value == "room-client-leave") { return Scenario::RoomClientLeave; } @@ -269,6 +276,41 @@ void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { } } +void runDataTrackCycle(const RoomCredentials& credentials) { + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing data track"); + } + auto publish_result = participant->publishDataTrack("memory-data"); + if (!publish_result) { + throw std::runtime_error("failed to publish local data track"); + } + + const auto& track = publish_result.value(); + livekit::DataTrackFrame frame; + frame.payload.assign(256, 0xFA); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + frame.user_timestamp = static_cast(frame_index); + if (!track->tryPush(frame)) { + throw std::runtime_error("failed to push local data track frame"); + } + } + + track->unpublishDataTrack(); + if (track->isPublished()) { + throw std::runtime_error("local data track remained published"); + } + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); +} + void runRoomClientLeaveCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -315,6 +357,9 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { case Scenario::VideoTrack: runVideoTrackCycle(); return; + case Scenario::DataTrack: + runDataTrackCycle(credentials); + return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); return; @@ -330,33 +375,42 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool room_scenario = - options.scenario == Scenario::RoomClientLeave || options.scenario == Scenario::RoomServerDelete; - const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; + const bool connected_scenario = options.scenario == Scenario::DataTrack || + options.scenario == Scenario::RoomClientLeave || + options.scenario == Scenario::RoomServerDelete; + const RoomCredentials credentials = connected_scenario ? roomCredentials() : RoomCredentials{}; std::unique_ptr room_sdk; - if (room_scenario) { + if (connected_scenario) { room_sdk = std::make_unique(); } - ProcessSample baseline; + const ProcessSample initial = sampleProcess(); + ProcessSample warmup; ProcessSample final; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { runCycle(options.scenario, credentials); std::this_thread::sleep_for(100ms); if (iteration == options.warmup) { - baseline = sampleProcess(); + warmup = sampleProcess(); } else if (iteration == options.iterations) { final = sampleProcess(); } } - const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); - const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); - const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && - thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS " << baseline.rss_kib - << " -> " << final.rss_kib << " KiB (" << rss_growth << " KiB), threads " << baseline.threads << " -> " - << final.threads << " (" << thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + const auto warmup_rss_growth = + static_cast(warmup.rss_kib) - static_cast(initial.rss_kib); + const auto final_rss_growth = static_cast(final.rss_kib) - static_cast(warmup.rss_kib); + const auto warmup_thread_growth = + static_cast(warmup.threads) - static_cast(initial.threads); + const auto final_thread_growth = + static_cast(final.threads) - static_cast(warmup.threads); + const bool passed = final_rss_growth <= static_cast(options.max_rss_growth_kib) && + final_thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << initial.rss_kib + << " -> warmup " << warmup.rss_kib << " KiB (" << warmup_rss_growth << " KiB) -> final " << final.rss_kib + << " KiB (" << final_rss_growth << " KiB), threads 0 " << initial.threads << " -> warmup " + << warmup.threads << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" + << final_thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; From 682f6d6e680fdf4f4fc2eed86470f65f4703882b Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 8 Oct 2026 09:39:58 -0700 Subject: [PATCH 6/7] Cleanup docs, comments, add room reuse --- .github/workflows/tests.yml | 2 - docs/testing.md | 44 +++++----- scripts/memory-regression.sh | 38 +++++--- src/tests/memory/lifecycle_tester.cpp | 121 ++++++++++++++++++++++---- 4 files changed, 150 insertions(+), 55 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9accd92d..187dacbd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -437,8 +437,6 @@ jobs: if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' timeout-minutes: 10 shell: bash - env: - MALLOC_ARENA_MAX: "1" run: | set -euo pipefail source scripts/set-test-tokens.sh diff --git a/docs/testing.md b/docs/testing.md index 2378b9bf..2e5f64b4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -107,34 +107,42 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. -## Linux memory regression gate +## Memory lifecycle tester -The Linux PR gate runs five lifecycle scenarios in separate processes: +> Note: Linux only -- `audio-track` repeatedly initializes the SDK, creates an audio source and - track, captures synthetic PCM frames, destroys them, and shuts the SDK down. -- `video-track` repeatedly initializes the SDK, creates a video source and - track, captures synthetic I420 frames, destroys them, and shuts the SDK down. +The memory lifecycle tester binary catches resource leak regressions in an +automated fashion, and is invoked in CI. This single binary runs individual +or groups of scenarios: + +- `audio-track` repeatedly initializes the SDK, connects, publishes an audio + track, captures synthetic PCM frames, unpublishes, disconnects, and shuts down. +- `video-track` repeatedly initializes the SDK, connects, publishes a video + track, captures synthetic I420 frames, unpublishes, disconnects, and shuts down. - `data-track` repeatedly connects, publishes a data track, pushes synthetic payloads, unpublishes it, and disconnects. +- `room-reuse` repeatedly connects and disconnects the same room object. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. Each scenario reports process RSS and thread count at cycle 0, after warmup -cycle 20, and after final cycle 100. The cycle 0 to warmup growth shows expected -one-time initialization; the gate applies to warmup-to-final growth and fails -on thread growth or more than 8 MiB of RSS growth. +cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth +and fails on thread growth or more than the configured amount of RSS growth. -The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits -glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a +The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, +which invokes the binary but also has GitHub actions support for a results summary table. +The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. +This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -The offline track scenarios require no LiveKit server or tokens: +Start `livekit-server --dev`, install the `lk` CLI, and source the local test +tokens before running any scenario: ```bash ./build.sh release-tests +source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario audio-track \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 @@ -143,24 +151,20 @@ scripts/memory-regression.sh --scenario video-track \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` -To run the data-track or room scenarios, start `livekit-server --dev`, install -the `lk` CLI, and source the local test tokens. Use `--scenario all` to run all five -scenarios, each in a fresh tester process: +Use `--scenario all` to run all six scenarios, each in a fresh tester process: ```bash -source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario data-track +scripts/memory-regression.sh --scenario room-reuse scripts/memory-regression.sh --scenario room-client-leave scripts/memory-regression.sh --scenario room-server-delete scripts/memory-regression.sh --scenario all ``` The limits are command-line options so a longer local run can use different -budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends one results -row per scenario when `GITHUB_STEP_SUMMARY` is set. Hardware encoding coverage -is separate follow-up work. +budgets. -## Memory checks (valgrind) +## Additional memory checks (valgrind) Run `valgrind` against the test binaries to check for memory leaks and other issues. See [tools.md](tools.md) for the recipe. diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index d805ffc2..056baa79 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -26,8 +26,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) --scenario NAME audio-track, video-track, data-track, - room-client-leave, room-server-delete, or all - (default: audio-track) + room-reuse, room-client-leave, + room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -39,7 +39,7 @@ Environment: MEMORY_REGRESSION_ITERATIONS, MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. - Room scenarios also require LIVEKIT_URL and LIVEKIT_TOKEN_A. + All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF } @@ -74,11 +74,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - audio-track|video-track|data-track|room-client-leave|room-server-delete) + audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(audio-track video-track data-track room-client-leave room-server-delete) + scenarios=(audio-track video-track data-track room-reuse room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -88,8 +88,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" == "data-track" || "${current_scenario}" == room-* ]] && - [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then + if [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 exit 1 @@ -102,12 +101,21 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then echo echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | Metric | Cycle 0 | Warmup (${warmup}) | Final (${iterations}) | 0 → warmup | Warmup → final | Verdict |" - echo "|---|---|---:|---:|---:|---:|---:|---|" + echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} |" + echo "|---|---:|---:|---:|---:|" } >> "${GITHUB_STEP_SUMMARY}" fi report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' + +format_growth() { + if [[ "$1" == "0" || "$1" == -* ]]; then + printf '%s' "$1" + else + printf '+%s' "$1" + fi +} + overall_status=0 for current_scenario in "${scenarios[@]}"; do set +e @@ -125,15 +133,17 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - { - echo "| ${BASH_REMATCH[1]} | RSS | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[5]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[6]} KiB | ${BASH_REMATCH[12]} |" - echo "| ${BASH_REMATCH[1]} | Threads | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} | ${BASH_REMATCH[10]} | ${BASH_REMATCH[9]} | ${BASH_REMATCH[11]} | ${BASH_REMATCH[12]} |" - } >> "${GITHUB_STEP_SUMMARY}" + rss_warmup_growth="$(format_growth "${BASH_REMATCH[4]}")" + rss_final_growth="$(format_growth "${BASH_REMATCH[6]}")" + thread_warmup_growth="$(format_growth "${BASH_REMATCH[9]}")" + thread_final_growth="$(format_growth "${BASH_REMATCH[11]}")" + echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} KiB | ${rss_final_growth} KiB | ${thread_warmup_growth} | ${thread_final_growth} |" \ + >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${current_scenario} | — | — | — | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + echo "| ${current_scenario} | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" fi status=1 fi diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 7c8cfc9d..fb9f0462 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -44,6 +44,7 @@ enum class Scenario { AudioTrack, VideoTrack, DataTrack, + RoomReuse, RoomClientLeave, RoomServerDelete, }; @@ -134,6 +135,8 @@ std::string_view scenarioName(Scenario scenario) { return "video-track"; case Scenario::DataTrack: return "data-track"; + case Scenario::RoomReuse: + return "room-reuse"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -152,6 +155,9 @@ Scenario parseScenario(std::string_view value) { if (value == "data-track") { return Scenario::DataTrack; } + if (value == "room-reuse") { + return Scenario::RoomReuse; + } if (value == "room-client-leave") { return Scenario::RoomClientLeave; } @@ -232,50 +238,103 @@ RoomCredentials roomCredentials() { const char* url = std::getenv("LIVEKIT_URL"); const char* token = std::getenv("LIVEKIT_TOKEN_A"); if (url == nullptr || token == nullptr || *url == '\0' || *token == '\0') { - throw std::runtime_error("room scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); + throw std::runtime_error("memory scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); } return {url, token}; } -void runAudioTrackCycle() { +void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { + if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { + throw std::runtime_error("room connection failed"); + } + if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { + throw std::runtime_error("room did not reach a connected state"); + } +} + +// Exercises SDK startup, room connection, audio publication and capture, +// explicit unpublication, client disconnect, and complete process-side teardown. +void runAudioTrackCycle(const RoomCredentials& credentials) { SdkLifetime sdk; + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + auto audio_source = std::make_shared(48'000, 1); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); if (!audio_track) { throw std::runtime_error("failed to create local audio track"); } + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing audio track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_MICROPHONE; + participant->publishTrack(audio_track, publish_options); + const auto publication = audio_track->publication(); + if (!publication) { + throw std::runtime_error("failed to publish local audio track"); + } + auto frame = livekit::AudioFrame::create(48'000, 1, 480); std::fill(frame.data().begin(), frame.data().end(), 100); for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { audio_source->captureFrame(frame, 1'000); } + + participant->unpublishTrack(publication->sid()); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } -void runVideoTrackCycle() { +// Exercises SDK startup, room connection, video publication and capture, +// encoder/resource teardown, explicit unpublication, and client disconnect. +void runVideoTrackCycle(const RoomCredentials& credentials) { SdkLifetime sdk; + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + auto video_source = std::make_shared(640, 360); auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); if (!video_track) { throw std::runtime_error("failed to create local video track"); } + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing video track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_CAMERA; + publish_options.simulcast = false; + participant->publishTrack(video_track, publish_options); + const auto publication = video_track->publication(); + if (!publication) { + throw std::runtime_error("failed to publish local video track"); + } + auto frame = livekit::VideoFrame::create(640, 360, livekit::VideoBufferType::I420); std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { video_source->captureFrame(frame); } -} -void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { - if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { - throw std::runtime_error("room connection failed"); - } - if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { - throw std::runtime_error("room did not reach a connected state"); + participant->unpublishTrack(publication->sid()); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises data-track publication, payload enqueueing, explicit unpublication, +// and room teardown while the SDK remains initialized across cycles. void runDataTrackCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -311,6 +370,20 @@ void runDataTrackCycle(const RoomCredentials& credentials) { delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises repeated connect/disconnect cycles on one Room instance to detect +// connection state, participant, listener, or FFI resources retained by reuse. +void runRoomReuseCycle(livekit::Room& room, const RoomCredentials& credentials) { + connectRoom(room, credentials); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("reused room disconnect did not tear down local state"); + } +} + +// Exercises application-initiated disconnect and destruction of a fresh Room, +// including local-participant cleanup and the expected disconnect callback. void runRoomClientLeaveCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -326,6 +399,8 @@ void runRoomClientLeaveCycle(const RoomCredentials& credentials) { delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises server-initiated room deletion and validates disconnect, event-stream +// completion, local-participant cleanup, and destruction of the torn-down Room. void runRoomServerDeleteCycle(const RoomCredentials& credentials) { if (std::string_view(credentials.url) != kLocalTestLiveKitUrl) { throw std::runtime_error("room-server-delete requires LIVEKIT_URL=ws://localhost:7880"); @@ -349,17 +424,23 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { delegate.requireServerTeardown(); } -void runCycle(Scenario scenario, const RoomCredentials& credentials) { +void runCycle(Scenario scenario, const RoomCredentials& credentials, livekit::Room* reusable_room) { switch (scenario) { case Scenario::AudioTrack: - runAudioTrackCycle(); + runAudioTrackCycle(credentials); return; case Scenario::VideoTrack: - runVideoTrackCycle(); + runVideoTrackCycle(credentials); return; case Scenario::DataTrack: runDataTrackCycle(credentials); return; + case Scenario::RoomReuse: + if (reusable_room == nullptr) { + throw std::runtime_error("room-reuse scenario is missing its persistent room"); + } + runRoomReuseCycle(*reusable_room, credentials); + return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); return; @@ -375,20 +456,22 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool connected_scenario = options.scenario == Scenario::DataTrack || - options.scenario == Scenario::RoomClientLeave || - options.scenario == Scenario::RoomServerDelete; - const RoomCredentials credentials = connected_scenario ? roomCredentials() : RoomCredentials{}; + const RoomCredentials credentials = roomCredentials(); + const bool per_cycle_sdk = options.scenario == Scenario::AudioTrack || options.scenario == Scenario::VideoTrack; std::unique_ptr room_sdk; - if (connected_scenario) { + if (!per_cycle_sdk) { room_sdk = std::make_unique(); } + std::unique_ptr reusable_room; + if (options.scenario == Scenario::RoomReuse) { + reusable_room = std::make_unique(); + } const ProcessSample initial = sampleProcess(); ProcessSample warmup; ProcessSample final; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { - runCycle(options.scenario, credentials); + runCycle(options.scenario, credentials, reusable_room.get()); std::this_thread::sleep_for(100ms); if (iteration == options.warmup) { warmup = sampleProcess(); From d638cca74f5fc1b4b53ba1c323416d0e4b0ee8c0 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 8 Oct 2026 11:33:09 -0700 Subject: [PATCH 7/7] Megabytes, emojis --- .github/workflows/tests.yml | 4 ++-- docs/testing.md | 4 ++-- scripts/memory-regression.sh | 33 ++++++++++++++++----------- src/tests/memory/lifecycle_tester.cpp | 24 +++++++++++-------- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 187dacbd..4d87c24f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -433,7 +433,7 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml - - name: Run Linux memory regression gate + - name: Run memory regression tests if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' timeout-minutes: 10 shell: bash @@ -445,7 +445,7 @@ jobs: --scenario all \ --iterations 100 \ --warmup 20 \ - --max-rss-growth-kib 8192 \ + --max-rss-growth-mib 8 \ --max-thread-growth 0 - name: Run stress tests diff --git a/docs/testing.md b/docs/testing.md index 2e5f64b4..20b7c673 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -145,10 +145,10 @@ tokens before running any scenario: source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario audio-track \ --iterations 100 --warmup 20 \ - --max-rss-growth-kib 8192 --max-thread-growth 0 + --max-rss-growth-mib 8 --max-thread-growth 0 scripts/memory-regression.sh --scenario video-track \ --iterations 100 --warmup 20 \ - --max-rss-growth-kib 8192 --max-thread-growth 0 + --max-rss-growth-mib 8 --max-thread-growth 0 ``` Use `--scenario all` to run all six scenarios, each in a fresh tester process: diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 056baa79..cfee8beb 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -30,14 +30,14 @@ Options: room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) - --max-rss-growth-kib N Allowed RSS growth (default: 8192) + --max-rss-growth-mib N Allowed RSS growth (default: 8) --max-thread-growth N Allowed thread growth (default: 0) -h, --help Show this help Environment: MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO, MEMORY_REGRESSION_ITERATIONS, - MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, + MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB, MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF @@ -48,7 +48,7 @@ build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" -max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" +max_rss_growth_mib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB:-8}" max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" while (($#)); do @@ -57,7 +57,7 @@ while (($#)); do --scenario) scenario="$2"; shift 2 ;; --iterations) iterations="$2"; shift 2 ;; --warmup) warmup="$2"; shift 2 ;; - --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; + --max-rss-growth-mib) max_rss_growth_mib="$2"; shift 2 ;; --max-thread-growth) max_thread_growth="$2"; shift 2 ;; -h|--help) usage; exit 0 ;; *) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;; @@ -97,19 +97,21 @@ done if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { - echo "## Linux memory regression" + echo "## Memory regression results" echo - echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." + echo "Iterations: ${iterations}" + echo "Warmup: ${warmup}" + echo "Threshold: RSS growth ≤ ${max_rss_growth_mib} MiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} |" - echo "|---|---:|---:|---:|---:|" + echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} | Status |" + echo "|---|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' format_growth() { - if [[ "$1" == "0" || "$1" == -* ]]; then + if [[ "$1" == "0" || "$1" == "0.00" || "$1" == -* ]]; then printf '%s' "$1" else printf '+%s' "$1" @@ -124,7 +126,7 @@ for current_scenario in "${scenarios[@]}"; do --scenario "${current_scenario}" \ --iterations "${iterations}" \ --warmup "${warmup}" \ - --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-rss-growth-mib "${max_rss_growth_mib}" \ --max-thread-growth "${max_thread_growth}" 2>&1 )" status=$? @@ -133,17 +135,22 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + if [[ "${BASH_REMATCH[12]}" == "PASS" ]]; then + verdict=":white_check_mark: PASS" + else + verdict=":x: FAIL" + fi rss_warmup_growth="$(format_growth "${BASH_REMATCH[4]}")" rss_final_growth="$(format_growth "${BASH_REMATCH[6]}")" thread_warmup_growth="$(format_growth "${BASH_REMATCH[9]}")" thread_final_growth="$(format_growth "${BASH_REMATCH[11]}")" - echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} KiB | ${rss_final_growth} KiB | ${thread_warmup_growth} | ${thread_final_growth} |" \ + echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} MiB | ${rss_final_growth} MiB | ${thread_warmup_growth} | ${thread_final_growth} | ${verdict} |" \ >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${current_scenario} | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + echo "| ${current_scenario} | — | — | — | — | :x: ERROR |" >> "${GITHUB_STEP_SUMMARY}" fi status=1 fi diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index fb9f0462..b8d279ad 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -53,7 +54,7 @@ struct Options { Scenario scenario{Scenario::AudioTrack}; std::uint64_t iterations{100}; std::uint64_t warmup{20}; - std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_rss_growth_mib{8}; std::uint64_t max_thread_growth{0}; }; @@ -190,8 +191,8 @@ Options parseOptions(int argc, char* argv[]) { options.iterations = parseUnsigned(value, option); } else if (option == "--warmup") { options.warmup = parseUnsigned(value, option); - } else if (option == "--max-rss-growth-kib") { - options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-mib") { + options.max_rss_growth_mib = parseUnsigned(value, option); } else if (option == "--max-thread-growth") { options.max_thread_growth = parseUnsigned(value, option); } else { @@ -234,6 +235,10 @@ ProcessSample sampleProcess() { return sample; } +double kibToMib(std::uint64_t kib) { return static_cast(kib) / 1024.0; } + +double kibToMib(std::int64_t kib) { return static_cast(kib) / 1024.0; } + RoomCredentials roomCredentials() { const char* url = std::getenv("LIVEKIT_URL"); const char* token = std::getenv("LIVEKIT_TOKEN_A"); @@ -487,13 +492,14 @@ int main(int argc, char* argv[]) { static_cast(warmup.threads) - static_cast(initial.threads); const auto final_thread_growth = static_cast(final.threads) - static_cast(warmup.threads); - const bool passed = final_rss_growth <= static_cast(options.max_rss_growth_kib) && + const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << initial.rss_kib - << " -> warmup " << warmup.rss_kib << " KiB (" << warmup_rss_growth << " KiB) -> final " << final.rss_kib - << " KiB (" << final_rss_growth << " KiB), threads 0 " << initial.threads << " -> warmup " - << warmup.threads << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" - << final_thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + std::cout << std::fixed << std::setprecision(2) << "memory lifecycle: scenario=" << scenarioName(options.scenario) + << ", RSS 0 " << kibToMib(initial.rss_kib) << " -> warmup " << kibToMib(warmup.rss_kib) << " MiB (" + << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.rss_kib) << " MiB (" + << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.threads << " -> warmup " << warmup.threads + << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" << final_thread_growth + << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n';