WIP: Memory regression tests

.github/workflows/ci.yml

@@ -55,6 +55,7 @@ jobs:
               - client-sdk-rust
               - client-sdk-rust/**
               - cmake/**
+              - scripts/memory-regression.sh
               - scripts/set-test-tokens.sh
               - scripts/run-with-backtrace.sh
               - CMakeLists.txt

.github/workflows/tests.yml

@@ -357,7 +357,11 @@ jobs:
             --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml"
 
       - name: Start livekit-server
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: >-
+          matrix.e2e-testing &&
+          (inputs.integration_repeat > 0 ||
+           inputs.run_stress_tests ||
+           (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'))
         id: livekit_server
         uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1
         with:
@@ -368,7 +372,11 @@ jobs:
 
       # Needed by token helper script
       - name: Install livekit-cli
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: >-
+          matrix.e2e-testing &&
+          (inputs.integration_repeat > 0 ||
+           inputs.run_stress_tests ||
+           (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'))
         shell: bash
         env:
           # Windows installs lk via `gh api` / `gh release download`, which need this env var
@@ -425,6 +433,21 @@ jobs:
             --gtest_recreate_environments_when_repeating=1 \
             --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml
 
+      - name: Run memory regression tests
+        if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release'
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          set -euo pipefail
+          source scripts/set-test-tokens.sh
+          scripts/memory-regression.sh \
+            --build-dir "${{ env.BUILD_DIR }}" \
+            --scenario all \
+            --iterations 100 \
+            --warmup 20 \
+            --max-rss-growth-mib 8 \
+            --max-thread-growth 0
+
       - name: Run stress tests
         if: matrix.e2e-testing && inputs.run_stress_tests
         timeout-minutes: ${{ inputs.stress_timeout_minutes }}

docs/testing.md

@@ -44,6 +44,7 @@ __Note:__ The tests require tokens and a running LiveKit server. See the section
 | `livekit_unit_tests` | Pure unit tests (no server required) |
 | `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality |
 | `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) |
+| `livekit_memory_lifecycle_tester` | Linux lifecycle RSS and thread regression scenarios |
 
 ## Running a local LiveKit server for tests
 
@@ -106,7 +107,64 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret -
 - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls.
 - **Stress**: high throughput, bidirectional RPC, memory pressure.
 
-## Memory checks (valgrind)
+## Memory lifecycle tester
+
+> Note: Linux only
+
+The memory lifecycle tester binary catches resource leak regressions in an
+automated fashion, and is invoked in CI. This single binary runs individual
+or groups of scenarios:
+
+- `audio-track` repeatedly initializes the SDK, connects, publishes an audio
+  track, captures synthetic PCM frames, unpublishes, disconnects, and shuts down.
+- `video-track` repeatedly initializes the SDK, connects, publishes a video
+  track, captures synthetic I420 frames, unpublishes, disconnects, and shuts down.
+- `data-track` repeatedly connects, publishes a data track, pushes synthetic
+  payloads, unpublishes it, and disconnects.
+- `room-reuse` repeatedly connects and disconnects the same room object.
+- `room-client-leave` repeatedly connects, disconnects, and destroys a room.
+- `room-server-delete` repeatedly connects and uses `lk` to delete the local
+  test room, then verifies the disconnect and end-of-stream callbacks.
+
+Each scenario reports process RSS and thread count at cycle 0, after warmup
+cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth
+and fails on thread growth or more than the configured amount of RSS growth.
+
+The binary can be run directly, or via a helper script at  `scripts/memory-regression.sh`, 
+which invokes the binary but also has GitHub actions support for a results summary table. 
+The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. 
+This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a
+controlled CI signal, not evidence that the default allocator returns all
+freed pages to the OS.
+
+Start `livekit-server --dev`, install the `lk` CLI, and source the local test
+tokens before running any scenario:
+
+```bash
+./build.sh release-tests
+source scripts/set-test-tokens.sh
+scripts/memory-regression.sh --scenario audio-track \
+  --iterations 100 --warmup 20 \
+  --max-rss-growth-mib 8 --max-thread-growth 0
+scripts/memory-regression.sh --scenario video-track \
+  --iterations 100 --warmup 20 \
+  --max-rss-growth-mib 8 --max-thread-growth 0
+```
+
+Use `--scenario all` to run all six scenarios, each in a fresh tester process:
+
+```bash
+scripts/memory-regression.sh --scenario data-track
+scripts/memory-regression.sh --scenario room-reuse
+scripts/memory-regression.sh --scenario room-client-leave
+scripts/memory-regression.sh --scenario room-server-delete
+scripts/memory-regression.sh --scenario all
+```
+
+The limits are command-line options so a longer local run can use different
+budgets.
+
+## Additional memory checks (valgrind)
 
 Run `valgrind` against the test binaries to check for memory leaks and other
 issues. See [tools.md](tools.md) for the recipe.

include/livekit/local_participant.h

@@ -296,6 +296,7 @@ class LIVEKIT_API LocalParticipant : public Participant {
   /// Publication SID → local track (@ref unpublishTrack clears the track’s
   /// cached publication). @c mutable so @ref trackPublications() const can
   /// prune expired @c weak_ptr entries.
+  mutable std::mutex published_tracks_mutex_;
   mutable TrackMap published_tracks_by_sid_;
 
   std::unordered_map<std::string, RpcHandler> rpc_handlers_;

include/livekit/local_track_publication.h

@@ -31,6 +31,9 @@ class LIVEKIT_API LocalTrackPublication : public TrackPublication {
   /// Note, this LocalTrackPublication is constructed internally only;
   /// safe to accept proto::OwnedTrackPublication.
   explicit LocalTrackPublication(const proto::OwnedTrackPublication& owned);
+
+private:
+  friend class Room;
 };
 
 } // namespace livekit

scripts/memory-regression.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+#
+# Copyright 2026 LiveKit
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage: memory-regression.sh [OPTIONS]
+
+Run the Linux memory lifecycle regression tester and append a table to the
+GitHub step summary when GITHUB_STEP_SUMMARY is set.
+
+Options:
+  --build-dir PATH            Build directory (default: build-release)
+  --scenario NAME             audio-track, video-track, data-track,
+                              room-reuse, room-client-leave,
+                              room-server-delete, or all (default: audio-track)
+  --iterations N              Lifecycle cycles (default: 100)
+  --warmup N                  Baseline cycle (default: 20)
+  --max-rss-growth-mib N      Allowed RSS growth (default: 8)
+  --max-thread-growth N       Allowed thread growth (default: 0)
+  -h, --help                  Show this help
+
+Environment:
+  MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO,
+  MEMORY_REGRESSION_ITERATIONS,
+  MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB,
+  MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX.
+  All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A.
+EOF
+}
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}"
+scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}"
+iterations="${MEMORY_REGRESSION_ITERATIONS:-100}"
+warmup="${MEMORY_REGRESSION_WARMUP:-20}"
+max_rss_growth_mib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB:-8}"
+max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}"
+
+while (($#)); do
+  case "$1" in
+    --build-dir) build_dir="$2"; shift 2 ;;
+    --scenario) scenario="$2"; shift 2 ;;
+    --iterations) iterations="$2"; shift 2 ;;
+    --warmup) warmup="$2"; shift 2 ;;
+    --max-rss-growth-mib) max_rss_growth_mib="$2"; shift 2 ;;
+    --max-thread-growth) max_thread_growth="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ "${build_dir}" != /* ]]; then
+  build_dir="${repo_root}/${build_dir}"
+fi
+tester="${build_dir}/bin/livekit_memory_lifecycle_tester"
+if [[ ! -x "${tester}" ]]; then
+  echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2
+  exit 1
+fi
+
+case "${scenario}" in
+  audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete)
+    scenarios=("${scenario}")
+    ;;
+  all)
+    scenarios=(audio-track video-track data-track room-reuse room-client-leave room-server-delete)
+    ;;
+  *)
+    echo "ERROR: invalid scenario: ${scenario}" >&2
+    usage >&2
+    exit 2
+    ;;
+esac
+
+for current_scenario in "${scenarios[@]}"; do
+  if [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then
+    echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2
+    echo "Run: source scripts/set-test-tokens.sh" >&2
+    exit 1
+  fi
+done
+
+if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+  {
+    echo "## Memory regression results"
+    echo
+    echo "Iterations: ${iterations}"
+    echo "Warmup: ${warmup}"
+    echo "Threshold: RSS growth ≤ ${max_rss_growth_mib} MiB; thread growth ≤ ${max_thread_growth}."
+    echo
+    echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} | Status |"
+    echo "|---|---:|---:|---:|---:|---|"
+  } >> "${GITHUB_STEP_SUMMARY}"
+fi
+
+report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)'
+
+format_growth() {
+  if [[ "$1" == "0" || "$1" == "0.00" || "$1" == -* ]]; then
+    printf '%s' "$1"
+  else
+    printf '+%s' "$1"
+  fi
+}
+
+overall_status=0
+for current_scenario in "${scenarios[@]}"; do
+  set +e
+  output="$(
+    MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \
+      --scenario "${current_scenario}" \
+      --iterations "${iterations}" \
+      --warmup "${warmup}" \
+      --max-rss-growth-mib "${max_rss_growth_mib}" \
+      --max-thread-growth "${max_thread_growth}" 2>&1
+  )"
+  status=$?
+  set -e
+  printf '%s\n' "${output}"
+
+  if [[ "${output}" =~ ${report_regex} ]]; then
+    if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+      if [[ "${BASH_REMATCH[12]}" == "PASS" ]]; then
+        verdict=":white_check_mark: PASS"
+      else
+        verdict=":x: FAIL"
+      fi
+      rss_warmup_growth="$(format_growth "${BASH_REMATCH[4]}")"
+      rss_final_growth="$(format_growth "${BASH_REMATCH[6]}")"
+      thread_warmup_growth="$(format_growth "${BASH_REMATCH[9]}")"
+      thread_final_growth="$(format_growth "${BASH_REMATCH[11]}")"
+      echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} MiB | ${rss_final_growth} MiB | ${thread_warmup_growth} | ${thread_final_growth} | ${verdict} |" \
+        >> "${GITHUB_STEP_SUMMARY}"
+    fi
+  else
+    echo "ERROR: ${current_scenario} did not produce a memory report." >&2
+    if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+      echo "| ${current_scenario} | — | — | — | — | :x: ERROR |" >> "${GITHUB_STEP_SUMMARY}"
+    fi
+    status=1
+  fi
+
+  if [[ "${status}" -ne 0 ]]; then
+    overall_status=1
+  fi
+done
+
+exit "${overall_status}"