diff --git a/.gitignore b/.gitignore
index ef6358997..dda3c4148 100644
--- a/.gitignore
+++ b/.gitignore
@@ -41,3 +41,10 @@ cmd/lk/testdata/**/package-lock.json
# generated at release time by release.yaml
autocomplete/fish_autocomplete
+
+# Python bytecode from the assistant agent
+__pycache__/
+
+# Created when running the assistant agent's simulations from the repo
+pkg/assistant/agent/.venv/
+pkg/assistant/agent/uv.lock
diff --git a/cmd/lk/agent.go b/cmd/lk/agent.go
index 923170dd3..cfce5dac4 100644
--- a/cmd/lk/agent.go
+++ b/cmd/lk/agent.go
@@ -236,6 +236,9 @@ On LiveKit Cloud: "create" and "deploy" ship it, then "status", "logs",
{
Name: "init",
Usage: "Create a new agent project from a template",
+ Description: `Creates the project in a new directory named AGENT-NAME. To create it in
+the current directory instead, pass "." as AGENT-NAME. The directory must be
+empty, and the agent is named after it.`,
Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) {
return createAgentClientWithOpts(ctx, cmd, confirmProject)
},
@@ -608,7 +611,12 @@ func initAgent(ctx context.Context, cmd *cli.Command) error {
logger.Debugw("Initializing agent project", "working-dir", workingDir)
- appName = cmd.Args().First()
+ if arg := cmd.Args().First(); arg != "" {
+ var err error
+ if appName, appDir, err = resolveAppTarget(arg); err != nil {
+ return err
+ }
+ }
if appName == "" {
appName = project.Name
}
diff --git a/cmd/lk/app.go b/cmd/lk/app.go
index 487ee1002..ee39c1c58 100644
--- a/cmd/lk/app.go
+++ b/cmd/lk/app.go
@@ -22,6 +22,7 @@ import (
"io"
"maps"
"os"
+ "path/filepath"
"regexp"
"strings"
@@ -44,6 +45,7 @@ var (
templateURL string
sandboxID string
appName string
+ appDir string // where the app is created; appName unless "." was given
appNameRegex = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
destinationFile string
exampleFile string
@@ -55,8 +57,11 @@ var (
Usage: "Initialize and manage applications",
Commands: []*cli.Command{
{
- Name: "create",
- Usage: "Bootstrap a new application from a template or through guided creation",
+ Name: "create",
+ Usage: "Bootstrap a new application from a template or through guided creation",
+ Description: `Creates the application in a new directory named APP_NAME. To create it in
+the current directory instead, pass "." as APP_NAME. The directory must be
+empty, and the application is named after it.`,
Action: setupTemplate,
Before: requireAppProject,
ArgsUsage: "`APP_NAME`",
@@ -363,9 +368,11 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func(
}
if appName == "" {
- arg := cmd.Args().First()
- if arg != "" {
- appName = arg
+ if arg := cmd.Args().First(); arg != "" {
+ var err error
+ if appName, appDir, err = resolveAppTarget(arg); err != nil {
+ return err
+ }
}
}
if appName == "" {
@@ -403,16 +410,20 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func(
}
}
+ if appDir == "" {
+ appDir = appName
+ }
+
// Set environment variables for template instantiation
os.Setenv("LIVEKIT_AGENT_NAME", appName)
os.Setenv("LIVEKIT_PROJECT_ID", project.ProjectId)
out.Status("Cloning template...")
- if err := cloneTemplate(ctx, cmd, templateURL, appName); err != nil {
+ if err := cloneTemplate(ctx, cmd, templateURL, appDir); err != nil {
return err
}
- tf, err := bootstrap.ParseTaskfile(appName)
+ tf, err := bootstrap.ParseTaskfile(appDir)
if err != nil {
return err
}
@@ -436,12 +447,12 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func(
}
}
}
- env, err := instantiateEnv(ctx, cmd, appName, addlEnv, envExampleFile, nil)
+ env, err := instantiateEnv(ctx, cmd, appDir, addlEnv, envExampleFile, nil)
if err != nil {
return err
}
- bootstrap.WriteDotEnv(appName, envOutputFile, env, true)
+ bootstrap.WriteDotEnv(appDir, envOutputFile, env, true)
if !cmd.IsSet("install") && !SkipPrompts(cmd) {
// Default the prompt to "yes" — installing deps is the common case.
@@ -456,7 +467,7 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func(
}
if install {
out.Status("Installing template...")
- if err := doInstall(ctx, bootstrap.TaskInstall, appName, verbose); err != nil {
+ if err := doInstall(ctx, bootstrap.TaskInstall, appDir, verbose); err != nil {
// Installation is best-effort — the agent is still created below. But a
// failed install (e.g. missing Node/pnpm) is easy to miss once the
// template's post-create step prints "agent created", so render a
@@ -474,7 +485,7 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func(
b.WriteString(line)
b.WriteString("\n")
}
- out.Warnf("%s%sFix your toolchain, then re-run the install step manually in ./%s.", b.String(), fixPrefix, appName)
+ out.Warnf("%s%sFix your toolchain, then re-run the install step manually in ./%s.", b.String(), fixPrefix, appDir)
} else {
// Signal a successful install to post_create so the template can skip
// printing the now-redundant install hint (guarded via `status:`).
@@ -482,23 +493,51 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func(
}
}
if afterInstall != nil {
- if err := afterInstall(ctx, cmd, appName); err != nil {
+ if err := afterInstall(ctx, cmd, appDir); err != nil {
return err
}
}
- if err := doPostCreate(ctx, cmd, appName, verbose); err != nil {
+ if err := doPostCreate(ctx, cmd, appDir, verbose); err != nil {
return err
}
- return cleanupTemplate(ctx, cmd, appName)
+ return cleanupTemplate(ctx, cmd, appDir)
+}
+
+// resolveAppTarget returns the app's name and the directory to create it in.
+// "." creates the app in the current directory, which must be empty, and
+// names it after the directory.
+func resolveAppTarget(arg string) (name, dir string, err error) {
+ if arg != "." {
+ return arg, arg, nil
+ }
+ wd, err := os.Getwd()
+ if err != nil {
+ return "", "", err
+ }
+ if empty, err := util.IsEmptyDir("."); err != nil {
+ return "", "", err
+ } else if !empty {
+ return "", "", errors.New("the current directory isn't empty; to create the app here, run this in an empty directory, or give a name to create a new directory")
+ }
+ name = filepath.Base(wd)
+ if !appNameRegex.MatchString(name) {
+ return "", "", fmt.Errorf("the current directory's name, %q, can't be used as the app name; use letters, numbers, dashes, and underscores", name)
+ }
+ return name, ".", nil
}
-func cloneTemplate(ctx context.Context, cmd *cli.Command, url, appName string) error {
+// cloneTemplate clones the template at url into dir. dir must not exist,
+// except "." (the current directory), which must be empty.
+func cloneTemplate(ctx context.Context, cmd *cli.Command, url, dir string) error {
var stdout string
var stderr string
- tempName, relocate, cleanup := util.UseTempPath(appName)
+ tempName, relocate, cleanup := util.UseTempPath(dir)
defer cleanup()
+ if dir == "." {
+ relocate = func() error { return util.MoveDirInto(tempName, dir) }
+ }
err := out.Await(
"Cloning template from "+url,
@@ -529,8 +568,8 @@ func cloneTemplate(ctx context.Context, cmd *cli.Command, url, appName string) e
return relocate()
}
-func cleanupTemplate(ctx context.Context, cmd *cli.Command, appName string) error {
- return bootstrap.CleanupTemplate(appName)
+func cleanupTemplate(ctx context.Context, cmd *cli.Command, dir string) error {
+ return bootstrap.CleanupTemplate(dir)
}
func manageEnv(ctx context.Context, cmd *cli.Command) error {
diff --git a/cmd/lk/app_target_test.go b/cmd/lk/app_target_test.go
new file mode 100644
index 000000000..458f66885
--- /dev/null
+++ b/cmd/lk/app_target_test.go
@@ -0,0 +1,55 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestResolveAppTargetNamed(t *testing.T) {
+ name, dir, err := resolveAppTarget("my-agent")
+ require.NoError(t, err)
+ require.Equal(t, "my-agent", name)
+ require.Equal(t, "my-agent", dir)
+}
+
+func TestResolveAppTargetCurrentDir(t *testing.T) {
+ root := t.TempDir()
+
+ empty := filepath.Join(root, "voice-app")
+ require.NoError(t, os.Mkdir(empty, 0o755))
+ t.Chdir(empty)
+ name, dir, err := resolveAppTarget(".")
+ require.NoError(t, err)
+ require.Equal(t, "voice-app", name)
+ require.Equal(t, ".", dir)
+
+ busy := filepath.Join(root, "busy")
+ require.NoError(t, os.Mkdir(busy, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(busy, "notes.txt"), nil, 0o644))
+ t.Chdir(busy)
+ _, _, err = resolveAppTarget(".")
+ require.ErrorContains(t, err, "isn't empty")
+
+ badName := filepath.Join(root, "has space")
+ require.NoError(t, os.Mkdir(badName, 0o755))
+ t.Chdir(badName)
+ _, _, err = resolveAppTarget(".")
+ require.ErrorContains(t, err, "can't be used as the app name")
+}
diff --git a/cmd/lk/assistant.go b/cmd/lk/assistant.go
new file mode 100644
index 000000000..8faeb6d90
--- /dev/null
+++ b/cmd/lk/assistant.go
@@ -0,0 +1,464 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "math/rand/v2"
+ "os"
+ "os/exec"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/atotto/clipboard"
+ "github.com/urfave/cli/v3"
+
+ "github.com/livekit/livekit-cli/v2/pkg/overlay"
+)
+
+var AssistantCommands = []*cli.Command{
+ {
+ Name: "assistant",
+ Usage: "Get started with LiveKit by talking to LK, the LiveKit assistant (preview)",
+ Hidden: true,
+ Description: `Opens a window and starts a voice conversation with LK, the LiveKit
+assistant, using your microphone and speakers. LK helps you get started: it
+creates a voice agent from a template, lets you talk to it, sets up your coding
+agent for LiveKit, and answers questions from the LiveKit docs. It shows code,
+commands, and docs links on screen, with buttons to copy them. Press / to type
+instead of speaking.
+
+LK is a LiveKit agent that runs on your machine. It uses LiveKit Inference
+through your LiveKit Cloud project, so its usage counts toward your plan. The
+first run installs it with uv.
+
+Press Esc to close. With --fullscreen, LK covers the screen and blurs the
+desktop. With --demo, it plays a scripted conversation instead; press R to
+replay it.`,
+ Flags: []cli.Flag{
+ &cli.BoolFlag{
+ Name: "demo",
+ Usage: "Play a scripted conversation instead of starting the assistant",
+ },
+ &cli.StringFlag{
+ Name: "agent-dir",
+ Usage: "Run the assistant agent from this project directory instead of the built-in one",
+ Hidden: true,
+ },
+ &cli.BoolFlag{
+ Name: "fullscreen",
+ Usage: "Cover the screen and blur the desktop, instead of opening a window",
+ },
+ &cli.BoolFlag{
+ Name: "inspect",
+ Usage: "Enable the Safari Web Inspector for the overlay page",
+ },
+ },
+ Action: runAssistant,
+ },
+}
+
+func runAssistant(ctx context.Context, cmd *cli.Command) error {
+ ov := overlay.New(overlay.Options{
+ Windowed: !cmd.Bool("fullscreen"),
+ Inspectable: cmd.Bool("inspect"),
+ })
+ if cmd.Bool("demo") {
+ demo := &assistantDemo{ov: ov, snippets: map[string]string{}, links: map[string]string{}}
+ go demo.run(ctx)
+ return ov.Run(ctx)
+ }
+
+ sessionCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ session := newAssistantSession(ov, cmd)
+ sessionErr := make(chan error, 1)
+ go func() { sessionErr <- session.run(sessionCtx) }()
+
+ if err := ov.Run(ctx); err != nil {
+ return err
+ }
+ cancel()
+ if dir, err := os.Getwd(); err == nil {
+ if path, err := session.log.save(dir); err == nil && path != "" {
+ out.Statusf("Saved this conversation to %s", path)
+ }
+ }
+ // Give the session time to stop the agent, but always exit.
+ select {
+ case err := <-sessionErr:
+ return err
+ case <-time.After(assistantShutdownTimeout):
+ return nil
+ }
+}
+
+// assistantShutdownTimeout bounds cleanup after the overlay closes. Stopping
+// the agent takes up to about 3 seconds.
+const assistantShutdownTimeout = 5 * time.Second
+
+// Messages sent to the overlay page. See the protocol in package overlay.
+type (
+ overlayState struct {
+ Type string `json:"type"`
+ State string `json:"state"`
+ }
+ overlayBands struct {
+ Type string `json:"type"`
+ Bands []float64 `json:"bands"`
+ }
+ overlayTranscript struct {
+ Type string `json:"type"`
+ Turn string `json:"turn"`
+ Role string `json:"role"`
+ Segment int `json:"segment"`
+ Text string `json:"text"`
+ }
+ overlaySnippet struct {
+ Type string `json:"type"`
+ Turn string `json:"turn"`
+ ID string `json:"id"`
+ Title string `json:"title"`
+ Lang string `json:"lang"`
+ Code string `json:"code"`
+ Copied bool `json:"copied"`
+ }
+ // overlayAnswer is a docs answer as short points, with its source page.
+ overlayAnswer struct {
+ Type string `json:"type"`
+ Turn string `json:"turn"`
+ ID string `json:"id"`
+ Title string `json:"title"`
+ Points []string `json:"points"`
+ SourceTitle string `json:"source_title"`
+ SourceURL string `json:"source_url"`
+ }
+ // overlayLink is a docs page to read more, with a button that opens it.
+ overlayLink struct {
+ Type string `json:"type"`
+ Turn string `json:"turn"`
+ ID string `json:"id"`
+ Title string `json:"title"`
+ URL string `json:"url"`
+ }
+ overlayCopied struct {
+ Type string `json:"type"`
+ ID string `json:"id"`
+ }
+ overlayTool struct {
+ Type string `json:"type"`
+ Turn string `json:"turn"`
+ Role string `json:"role,omitempty"` // turn role; agent if empty
+ ID string `json:"id"`
+ Label string `json:"label,omitempty"`
+ Detail string `json:"detail,omitempty"`
+ Status string `json:"status"` // running, done, or error
+ }
+ overlayNotice struct {
+ Type string `json:"type"`
+ Text string `json:"text"`
+ Sticky bool `json:"sticky,omitempty"`
+ }
+)
+
+// demoSnippet is a code card in an agent reply.
+type demoSnippet struct {
+ id, title, lang, code string
+}
+
+// demoLink is a docs link card in an agent reply.
+type demoLink struct {
+ id, title, url string
+}
+
+// demoAnswer is a docs answer card in an agent reply.
+type demoAnswer struct {
+ id, title string
+ points []string
+ sourceTitle, source string
+}
+
+// demoTurn is one exchange. Each agent part is a string (a paragraph of
+// speech), a demoSnippet, a demoLink, or a demoAnswer.
+type demoTurn struct {
+ user string
+ agent []any
+}
+
+// The agent code is from the Voice AI quickstart
+// (docs.livekit.io/agents/start/voice-ai-quickstart).
+var demoScript = []demoTurn{
+ {
+ user: "How do I build a voice agent in Python?",
+ agent: []any{
+ "Start from the Python starter template. This command creates the project and adds your LiveKit Cloud credentials to it.",
+ demoSnippet{id: "init", title: "Terminal", lang: "shell", code: "lk agent init my-agent --template agent-starter-python\ncd my-agent"},
+ "Then run `lk agent dev` and talk to it from the Agent Console.",
+ },
+ },
+ {
+ user: "Where do I change the voice?",
+ agent: []any{
+ "In `agent.py`. The `AgentSession` sets the speech-to-text, language, and text-to-speech models. Change the TTS model or voice there.",
+ demoSnippet{id: "agent", title: "agent.py", lang: "python", code: `@server.rtc_session(agent_name="my-agent")
+async def my_agent(ctx: agents.JobContext):
+ session = AgentSession(
+ stt=inference.STT(model="assemblyai/universal-3-6-pro", language="en"),
+ llm=inference.LLM(model="google/gemma-4-31b-it"),
+ tts=inference.TTS(
+ model="fishaudio/s2.1-pro",
+ voice="fa4c9eb3dccc4806b382b40d61c6b10a",
+ ),
+ turn_handling=TurnHandlingOptions(
+ turn_detection=inference.TurnDetector(),
+ ),
+ )
+
+ await session.start(room=ctx.room, agent=Assistant())`},
+ "Copy it from the card to use it in your project.",
+ },
+ },
+ {
+ user: "And how do I deploy it?",
+ agent: []any{
+ "Run `lk agent create` from the project directory. It registers the agent and deploys it to LiveKit Cloud.",
+ demoSnippet{id: "deploy", title: "Terminal", lang: "shell", code: "lk agent create"},
+ demoAnswer{
+ id: "deploy-docs", title: "Deploying to LiveKit Cloud",
+ points: []string{
+ "Deploy with a single LiveKit CLI command.",
+ "LiveKit Cloud scales your agent automatically, up to the limits of your plan.",
+ "Store API keys and other secrets in LiveKit Cloud. It injects them into your agent at runtime.",
+ "Runtime, build, and session logs help you monitor and debug the deployed agent.",
+ },
+ sourceTitle: "Agent deployment overview", source: "https://docs.livekit.io/deploy/agents/",
+ },
+ },
+ },
+}
+
+const demoFrame = time.Second / 60
+
+// assistantDemo plays a scripted conversation into the overlay. It stands in
+// for the real agent session until the overlay is wired to a LiveKit room.
+type assistantDemo struct {
+ ov *overlay.Overlay
+ voice fakeVoice
+ snippets map[string]string
+ links map[string]string // card id → docs URL, for Open
+}
+
+func (d *assistantDemo) run(ctx context.Context) {
+ start := make(chan struct{}, 1)
+ var (
+ mu sync.Mutex
+ cancelPlay context.CancelFunc
+ )
+ go func() {
+ for {
+ select {
+ case ev := <-d.ov.Events():
+ switch ev.Type {
+ case "ready", "replay":
+ mu.Lock()
+ if cancelPlay != nil {
+ cancelPlay()
+ }
+ mu.Unlock()
+ select {
+ case start <- struct{}{}:
+ default:
+ }
+ case "copy":
+ d.copy(ev.ID)
+ case "open":
+ if url, ok := d.links[ev.ID]; ok && strings.HasPrefix(url, docsURL) {
+ _ = exec.Command("open", url).Start()
+ }
+ }
+ case <-d.ov.Done():
+ return
+ }
+ }
+ }()
+
+ for {
+ select {
+ case <-start:
+ case <-ctx.Done():
+ return
+ case <-d.ov.Done():
+ return
+ }
+ playCtx, cancel := context.WithCancel(ctx)
+ mu.Lock()
+ cancelPlay = cancel
+ mu.Unlock()
+
+ d.ov.Send(map[string]string{"type": "clear"})
+ d.ov.Send(overlayState{"state", "idle"})
+ d.play(playCtx)
+ cancel()
+ }
+}
+
+// play runs the script once. It returns false if it was stopped early.
+func (d *assistantDemo) play(ctx context.Context) bool {
+ // Let the intro finish: the mark shows large, then docks.
+ if !d.wait(ctx, 1800*time.Millisecond, nil) {
+ return false
+ }
+ for i, turn := range demoScript {
+ userTurn, agentTurn := fmt.Sprintf("u%d", i), fmt.Sprintf("a%d", i)
+
+ d.ov.Send(overlayState{"state", "listening"})
+ if !d.speak(ctx, userTurn, "user", 0, turn.user, 270*time.Millisecond, 0.7) {
+ return false
+ }
+ if !d.wait(ctx, 400*time.Millisecond, nil) {
+ return false
+ }
+
+ d.ov.Send(overlayState{"state", "thinking"})
+ if !d.wait(ctx, 1300*time.Millisecond, nil) {
+ return false
+ }
+
+ d.ov.Send(overlayState{"state", "speaking"})
+ segment := 0
+ for _, part := range turn.agent {
+ switch p := part.(type) {
+ case string:
+ if !d.speak(ctx, agentTurn, "agent", segment, p, 190*time.Millisecond, 1) {
+ return false
+ }
+ segment++
+ case demoSnippet:
+ d.snippet(agentTurn, p)
+ if !d.wait(ctx, 600*time.Millisecond, nil) {
+ return false
+ }
+ case demoAnswer:
+ var text strings.Builder
+ for _, pt := range p.points {
+ text.WriteString("- " + pt + "\n")
+ }
+ fmt.Fprintf(&text, "\nSource: %s (%s)", p.sourceTitle, p.source)
+ d.snippets[p.id] = text.String()
+ d.links[p.id] = p.source
+ d.ov.Send(overlayAnswer{Type: "answer", Turn: agentTurn, ID: p.id, Title: p.title, Points: p.points, SourceTitle: p.sourceTitle, SourceURL: p.source})
+ if !d.wait(ctx, 600*time.Millisecond, nil) {
+ return false
+ }
+ case demoLink:
+ d.snippets[p.id] = p.url
+ d.links[p.id] = p.url
+ d.ov.Send(overlayLink{Type: "link", Turn: agentTurn, ID: p.id, Title: p.title, URL: p.url})
+ if !d.wait(ctx, 600*time.Millisecond, nil) {
+ return false
+ }
+ }
+ }
+
+ d.ov.Send(overlayState{"state", "idle"})
+ if !d.wait(ctx, 1400*time.Millisecond, nil) {
+ return false
+ }
+ }
+ d.ov.Send(overlayState{"state", "listening"})
+ return true
+}
+
+// speak streams one segment of a turn word by word, like a live transcript,
+// while sending simulated voice levels.
+func (d *assistantDemo) speak(ctx context.Context, turn, role string, segment int, text string, perWord time.Duration, gain float64) bool {
+ words := strings.Fields(text)
+ begin := time.Now()
+ shown := 0
+ return d.wait(ctx, time.Duration(len(words))*perWord+200*time.Millisecond, func(elapsed time.Duration) {
+ d.ov.Send(overlayBands{"bands", d.voice.bands(time.Since(begin).Seconds(), gain)})
+ if n := min(len(words), int(elapsed/perWord)+1); n != shown {
+ shown = n
+ d.ov.Send(overlayTranscript{"transcript", turn, role, segment, strings.Join(words[:n], " ")})
+ }
+ })
+}
+
+// wait blocks for dur, calling tick every frame. It returns false if the
+// overlay closed or ctx ended.
+func (d *assistantDemo) wait(ctx context.Context, dur time.Duration, tick func(elapsed time.Duration)) bool {
+ ticker := time.NewTicker(demoFrame)
+ defer ticker.Stop()
+ begin := time.Now()
+ for {
+ select {
+ case <-ctx.Done():
+ return false
+ case <-d.ov.Done():
+ return false
+ case now := <-ticker.C:
+ elapsed := now.Sub(begin)
+ if elapsed >= dur {
+ return true
+ }
+ if tick != nil {
+ tick(elapsed)
+ }
+ }
+ }
+}
+
+func (d *assistantDemo) snippet(turn string, sn demoSnippet) {
+ d.snippets[sn.id] = sn.code
+ d.ov.Send(overlaySnippet{"snippet", turn, sn.id, sn.title, sn.lang, sn.code, false})
+}
+
+func (d *assistantDemo) copy(id string) {
+ code, ok := d.snippets[id]
+ if !ok || clipboard.WriteAll(code) != nil {
+ return
+ }
+ d.ov.Send(overlayCopied{"copied", id})
+}
+
+// fakeVoice synthesizes speech-like frequency bands: syllable-rate pulses,
+// pauses between phrases, and two moving formants over a falling spectrum.
+type fakeVoice struct{}
+
+const fakeVoiceBands = 24
+
+func (fakeVoice) bands(t, gain float64) []float64 {
+ syl := math.Pow(math.Max(0, math.Sin(2*math.Pi*4.3*t+0.8*math.Sin(2*math.Pi*0.7*t))), 0.7)
+ gate := smoothstep(0.15, 0.35, 0.5+0.5*math.Sin(2*math.Pi*0.45*t+1.3))
+ f1 := 4 + 3*math.Sin(2*math.Pi*1.1*t)
+ f2 := 12 + 5*math.Sin(2*math.Pi*0.63*t+2)
+
+ out := make([]float64, fakeVoiceBands)
+ for i := range out {
+ x := float64(i)
+ s := 0.75*math.Exp(-x/9) + 0.55*math.Exp(-(x-f1)*(x-f1)/9.7) + 0.35*math.Exp(-(x-f2)*(x-f2)/24.5)
+ v := syl * gate * gain * s * 1.25 * (0.85 + 0.3*rand.Float64())
+ out[i] = math.Round(math.Min(1, v)*100) / 100
+ }
+ return out
+}
+
+func smoothstep(e0, e1, x float64) float64 {
+ t := math.Max(0, math.Min(1, (x-e0)/(e1-e0)))
+ return t * t * (3 - 2*t)
+}
diff --git a/cmd/lk/assistant_guest.go b/cmd/lk/assistant_guest.go
new file mode 100644
index 000000000..06e01c276
--- /dev/null
+++ b/cmd/lk/assistant_guest.go
@@ -0,0 +1,266 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net"
+ "path/filepath"
+ "slices"
+ "strings"
+
+ "github.com/livekit/livekit-cli/v2/pkg/agentfs"
+ "github.com/livekit/livekit-cli/v2/pkg/assistant"
+ "github.com/livekit/livekit-cli/v2/pkg/console"
+)
+
+// "Meet your agent": the user's own agent runs in console mode and joins the
+// same audio pipeline as the assistant. Switching changes which agent hears
+// the microphone. The assistant keeps running, silent, so the user returns to
+// the same conversation.
+
+// guestAgent is the user's agent while it runs in the overlay.
+type guestAgent struct {
+ name string // shown as the speaker
+ dir string
+ proc *AgentProcess
+ conn net.Conn
+}
+
+// overlayGuestStatus shows that the user's agent is starting, or that it
+// failed to start. When it connects, an overlaySpeaker switches to it.
+type overlayGuestStatus struct {
+ Type string `json:"type"`
+ Status string `json:"status"` // starting or failed
+ Name string `json:"name"`
+}
+
+type overlaySpeaker struct {
+ Type string `json:"type"`
+ Name string `json:"name"` // who the agent turns are from
+ Guest bool `json:"guest"` // the user's own agent, not the assistant
+}
+
+// proposeTryProject proposes running the user's agent in the overlay. It's
+// only for the project created this session: a starter agent always opens a
+// voice session in console mode, but an existing agent may need its frontend,
+// dispatch, or other services first. For an existing agent, the assistant
+// shows the command to run it in a terminal.
+func (s *assistantSession) proposeTryProject(requestID string, confirmed bool) {
+ t := &s.tools
+ dir := t.lastProject
+ if dir == "" {
+ s.replyTool(requestID, existingAgentAdvice(t.dir), nil)
+ return
+ }
+ if s.guest != nil || s.guestStarting {
+ s.replyTool(requestID, "The user's agent is already starting or running, so nothing else was started. Don't start it again.", nil)
+ return
+ }
+ root, projectType, err := agentfs.DetectProjectRoot(dir)
+ if err != nil || (!projectType.IsPython() && !projectType.IsNode()) {
+ s.replyTool(requestID, "", fmt.Errorf("there's no Python or Node.js agent project in %s", tildePath(dir)))
+ return
+ }
+ entrypoint, err := findEntrypoint(root, "", projectType)
+ if err != nil {
+ s.replyTool(requestID, "", err)
+ return
+ }
+ name := filepath.Base(root)
+
+ t.confirmN++
+ id := fmt.Sprintf("confirm%d", t.confirmN)
+ t.pending[id] = pendingTool{
+ command: "lk agent console",
+ start: func(requestID, confirmID string) {
+ s.startGuest(requestID, confirmID, name, root, entrypoint, projectType)
+ },
+ }
+ s.agentTurnForReply()
+ s.out.Send(overlayConfirm{
+ Type: "confirm", Turn: s.agentTurn, ID: id,
+ Heading: "Run in " + tildePath(root),
+ Command: "lk agent console",
+ Detail: fmt.Sprintf("Runs your agent, %s, on this machine and switches the conversation to it. Press B to come back to LK.", name),
+ Options: []overlayConfirmOption{{ID: confirmRun, Name: "Talk to it"}, {ID: confirmCancel, Name: "Cancel"}},
+ })
+ if confirmed {
+ s.confirmByVoice(requestID, id, true)
+ return
+ }
+ s.replyTool(requestID, fmt.Sprintf(
+ "Waiting for the user to confirm. Confirmation ID: %s. This runs their agent, %s, here so they can talk to it. "+
+ "Ask the user in a few words whether to start it. They can answer by voice or on screen. "+
+ "If they agree by voice, call confirm_action with this ID and approved=true.", id, name), nil)
+}
+
+// existingAgentAdvice tells the assistant how the user can run an agent that
+// wasn't created this session.
+func existingAgentAdvice(dir string) string {
+ command := "lk agent console"
+ if paths := assistant.DescribeFolder(dir).AgentPaths(); len(paths) == 1 && paths[0] != "." {
+ command = "cd " + paths[0] + " && lk agent console"
+ }
+ return "You can only run an agent here that you created in this session. Existing agents can need their " +
+ "frontend, dispatch, or other services, so they run in the user's own terminal. In one short sentence, " +
+ "tell the user you can't start it here, and that they can run it in their terminal to talk to it. Then " +
+ `call show_code with title "Terminal", language "shell", and this command: ` + command +
+ ". If you don't know which folder their agent is in, ask them first, and tell them to run the command in that folder."
+}
+
+// startGuest starts the user's agent and switches to it when it connects.
+// The assistant tells the user it's starting, which takes a few seconds. The
+// overlay shows the switch.
+func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoint string, projectType agentfs.ProjectType) {
+ t := &s.tools
+ s.guestStarting = true
+ starting := fmt.Sprintf("Starting the user's agent, %s. It takes a few seconds, and the screen shows when it's connected. "+
+ "In one or two short sentences, tell them it's starting, that once it's connected they can talk to it, saying hello if it doesn't greet them, "+
+ "and that they can press B to come back to you. "+
+ "Don't say you've handed them over: you'll hear if it fails to start. "+
+ "If the user talks to you before it connects, tell them in a few words that it's still starting.", name)
+ s.out.Send(overlayGuestStatus{Type: "guest_status", Status: "starting", Name: name})
+ s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: confirmID, Status: "running", Text: "Starting " + name + "…"})
+ if requestID != "" {
+ s.replyTool(requestID, starting, nil)
+ } else {
+ s.tellUser(starting)
+ }
+ s.inBackground(func(ctx context.Context) toolResult {
+ failed := func(err error) toolResult {
+ return toolResult{then: func(toolResult) {
+ s.guestStarting = false
+ s.out.Send(overlayGuestStatus{Type: "guest_status", Status: "failed", Name: name})
+ s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: confirmID, Status: "failed", Text: "Your agent didn't start. See the terminal"})
+ out.Statusf("Your agent didn't start: %v", err)
+ s.tellUser(fmt.Sprintf("The user's agent, %s, didn't start, so they're still talking to you. The error:\n%s\n"+
+ "In one or two short sentences, tell them what went wrong in plain words, and that the full log is in the terminal.", name, truncate(err.Error(), 1500)))
+ }}
+ }
+ // The guest needs its own console server: the assistant's accepts
+ // only one agent, and closes once the assistant connects.
+ server, err := console.NewTCPServer("127.0.0.1:0")
+ if err != nil {
+ return failed(err)
+ }
+ proc, err := startAgent(AgentStartConfig{
+ Dir: dir,
+ Entrypoint: entrypoint,
+ ProjectType: projectType,
+ CLIArgs: buildConsoleArgs(server.Addr().String(), false),
+ Env: t.creds,
+ FailSignals: guestEndSignals,
+ })
+ if err != nil {
+ server.Close()
+ return failed(fmt.Errorf("couldn't start the agent: %w", err))
+ }
+ conn, err := acceptAgent(ctx, server, proc, "your agent")
+ if err != nil {
+ proc.Kill()
+ server.Close()
+ return failed(err)
+ }
+ return toolResult{then: func(toolResult) {
+ s.guestStarting = false
+ s.guest = &guestAgent{name: name, dir: dir, proc: proc, conn: conn}
+ s.pipeline.AddConn(t.ctx, conn)
+ s.pipeline.SetActive(conn)
+ s.startSpeakerTurns(overlaySpeaker{Type: "speaker", Name: name, Guest: true})
+ s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: confirmID, Status: "done"})
+ go func() {
+ // The worker keeps running when its job ends, so watch for
+ // the job's end as well as for the exit.
+ var err error
+ select {
+ case err = <-proc.Done():
+ case <-proc.Failed():
+ if jobCrashed(proc) {
+ err = errors.New("its job crashed")
+ }
+ }
+ select {
+ case t.results <- toolResult{then: func(toolResult) { s.guestExited(proc, err) }}:
+ case <-t.ctx.Done():
+ }
+ }()
+ }}
+ })
+}
+
+// guestEndSignals are output markers meaning the guest's job ended, though
+// its worker may stay alive. An agent whose job ends before it connects, such
+// as one that needs data from its frontend, fails right away instead of
+// waiting out the connect timeout. "job exiting" is a Python SDK debug log.
+var guestEndSignals = append(slices.Clone(consoleCrashSignals), "job exiting")
+
+// jobCrashed reports whether the guest's job ended because it crashed.
+func jobCrashed(proc *AgentProcess) bool {
+ for _, line := range proc.RecentLogs(50) {
+ for _, sig := range consoleCrashSignals {
+ if strings.Contains(line, sig) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// returnToAssistant switches back to the assistant and stops the guest.
+func (s *assistantSession) returnToAssistant(notice string) {
+ g := s.guest
+ if g == nil {
+ return
+ }
+ s.guest = nil
+ s.pipeline.SetActive(s.assistantConn)
+ g.conn.Close()
+ go g.proc.Kill()
+ s.startSpeakerTurns(overlaySpeaker{Type: "speaker", Name: "LiveKit"})
+ if notice != "" {
+ s.out.Send(overlayNotice{Type: "notice", Text: notice})
+ }
+ s.tellUser(fmt.Sprintf("The user is back from talking to their agent, %s. Welcome them back in one short sentence and ask how it went or what they'd like to change.", g.name))
+}
+
+func (s *assistantSession) guestExited(proc *AgentProcess, err error) {
+ if s.guest == nil || s.guest.proc != proc {
+ return
+ }
+ notice := "Your agent's session ended"
+ if err != nil {
+ notice = "Your agent stopped with an error. See the terminal"
+ out.Statusf("Your agent stopped: %v\n%s", err, agentExitDetail(proc))
+ }
+ s.returnToAssistant(notice)
+}
+
+// startSpeakerTurns makes the next turns start fresh, labeled for speaker.
+func (s *assistantSession) startSpeakerTurns(speaker overlaySpeaker) {
+ s.agentOpen = false
+ s.userOpen = false
+ s.agentSpoke = true
+ s.agentSpeaking = false
+ s.out.Send(speaker)
+ s.out.Send(overlayState{"state", "listening"})
+}
+
+// guestActive reports whether the user's agent is the active one.
+func (s *assistantSession) guestActive() bool {
+ return s.guest != nil
+}
diff --git a/cmd/lk/assistant_log.go b/cmd/lk/assistant_log.go
new file mode 100644
index 000000000..6fc55e638
--- /dev/null
+++ b/cmd/lk/assistant_log.go
@@ -0,0 +1,251 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+)
+
+// sessionLog records what the overlay shows, so the conversation can be saved
+// as Markdown when it ends: the transcript, tool calls, code, prompts, and
+// commands.
+type sessionLog struct {
+ next overlaySender
+
+ mu sync.Mutex
+ started time.Time
+ speaker string // who new agent turns are from
+ order []string
+ turns map[string]*logTurn
+ items map[string]*logItem // tool, card, and confirmation items by id
+}
+
+type logTurn struct {
+ role string // user or agent
+ speaker string
+ items []*logItem
+ texts map[int]*logItem // transcript segments
+}
+
+type logItem struct {
+ kind string // text, tool, code, or command
+ text string
+ label string
+ detail string
+ lang string
+ status string
+}
+
+func newSessionLog(next overlaySender) *sessionLog {
+ return &sessionLog{
+ next: next,
+ started: time.Now(),
+ speaker: "LK",
+ turns: map[string]*logTurn{},
+ items: map[string]*logItem{},
+ }
+}
+
+func (l *sessionLog) Send(msg any) error {
+ l.record(msg)
+ return l.next.Send(msg)
+}
+
+func (l *sessionLog) turn(id, role string) *logTurn {
+ t, ok := l.turns[id]
+ if !ok {
+ t = &logTurn{role: role, speaker: l.speaker, texts: map[int]*logItem{}}
+ l.turns[id] = t
+ l.order = append(l.order, id)
+ }
+ return t
+}
+
+func (l *sessionLog) record(msg any) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ switch m := msg.(type) {
+ case overlaySpeaker:
+ l.speaker = m.Name
+ if !m.Guest {
+ l.speaker = "LK"
+ }
+ case overlayTranscript:
+ t := l.turn(m.Turn, m.Role)
+ item, ok := t.texts[m.Segment]
+ if !ok {
+ item = &logItem{kind: "text"}
+ t.texts[m.Segment] = item
+ t.items = append(t.items, item)
+ }
+ item.text = m.Text
+ case overlayTool:
+ item, ok := l.items[m.ID]
+ if !ok {
+ item = &logItem{kind: "tool"}
+ l.items[m.ID] = item
+ t := l.turn(m.Turn, "agent")
+ t.items = append(t.items, item)
+ }
+ if m.Label != "" {
+ item.label = m.Label
+ }
+ if m.Detail != "" {
+ item.detail = m.Detail
+ }
+ item.status = m.Status
+ case overlaySnippet:
+ t := l.turn(m.Turn, "agent")
+ t.items = append(t.items, &logItem{kind: "code", label: m.Title, lang: m.Lang, text: m.Code})
+ case overlayAnswer:
+ t := l.turn(m.Turn, "agent")
+ t.items = append(t.items, &logItem{kind: "answer", label: m.Title, text: strings.Join(m.Points, "\n- "), detail: m.SourceTitle, lang: m.SourceURL})
+ case overlayLink:
+ t := l.turn(m.Turn, "agent")
+ t.items = append(t.items, &logItem{kind: "link", label: m.Title, text: m.URL})
+ case overlayConfirm:
+ kind := "command"
+ if m.Command == "" {
+ kind = "choice"
+ }
+ item := &logItem{kind: kind, text: m.Command, detail: m.Heading, status: "not answered"}
+ l.items[m.ID] = item
+ t := l.turn(m.Turn, "agent")
+ t.items = append(t.items, item)
+ case overlayConfirmStatus:
+ if item, ok := l.items[m.ID]; ok {
+ item.status = m.Status
+ if m.Text != "" {
+ item.status += ": " + m.Text
+ }
+ }
+ }
+}
+
+// empty reports whether t has nothing to show, such as a turn the user's
+// agent started but didn't speak in.
+func (t *logTurn) empty() bool {
+ for _, it := range t.items {
+ if it.kind != "text" || it.text != "" {
+ return false
+ }
+ }
+ return true
+}
+
+// markdown renders the conversation, or "" if the user never spoke.
+func (l *sessionLog) markdown(dir string) string {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ userSpoke := false
+ for _, t := range l.turns {
+ if t.role == "user" {
+ userSpoke = true
+ }
+ }
+ if !userSpoke {
+ return ""
+ }
+
+ var b strings.Builder
+ fmt.Fprintf(&b, "# LiveKit assistant, %s\n\nIn %s.\n", l.started.Format("January 2, 2006, 3:04 PM"), tildePath(dir))
+ for _, id := range l.order {
+ t := l.turns[id]
+ if t.empty() {
+ continue
+ }
+ who := t.speaker
+ if t.role == "user" {
+ who = "You"
+ }
+ fmt.Fprintf(&b, "\n**%s:**", who)
+ // Text continues the speaker's line, or the text before it. After
+ // a tool line, card, or command, it starts a new paragraph.
+ afterText := true
+ for _, it := range t.items {
+ switch it.kind {
+ case "text":
+ if it.text == "" {
+ continue
+ }
+ sep := " "
+ if !afterText {
+ sep = "\n\n"
+ }
+ b.WriteString(sep + it.text)
+ afterText = true
+ continue
+ case "tool":
+ line := it.label
+ if it.detail != "" {
+ line += ": " + it.detail
+ }
+ fmt.Fprintf(&b, "\n\n- %s", line)
+ case "code":
+ lang := it.lang
+ if lang == "prompt" {
+ lang = ""
+ }
+ fence := codeFence(it.text)
+ fmt.Fprintf(&b, "\n\n%s:\n\n%s%s\n%s\n%s", it.label, fence, lang, it.text, fence)
+ case "answer":
+ fmt.Fprintf(&b, "\n\n%s:\n\n- %s\n\nSource: [%s](%s)", it.label, it.text, it.detail, it.lang)
+ case "link":
+ fmt.Fprintf(&b, "\n\nRead more: [%s](%s)", it.label, it.text)
+ case "choice":
+ fmt.Fprintf(&b, "\n\n- %s %s", it.detail, it.status)
+ case "command":
+ fmt.Fprintf(&b, "\n\n`%s` (%s): %s", it.text, strings.ToLower(it.detail), it.status)
+ }
+ afterText = false
+ }
+ b.WriteString("\n")
+ }
+ return b.String()
+}
+
+// codeFence returns a Markdown code fence longer than any run of backticks
+// in code, so the code can't close it early.
+func codeFence(code string) string {
+ fence := "```"
+ for strings.Contains(code, fence) {
+ fence += "`"
+ }
+ return fence
+}
+
+// save writes the conversation to the assistant's sessions folder and returns
+// the file's path, or "" if there was nothing to save.
+func (l *sessionLog) save(dir string) (string, error) {
+ md := l.markdown(dir)
+ if md == "" {
+ return "", nil
+ }
+ cache, err := os.UserCacheDir()
+ if err != nil {
+ return "", err
+ }
+ folder := filepath.Join(cache, "livekit", "assistant-sessions")
+ if err := os.MkdirAll(folder, 0o755); err != nil {
+ return "", err
+ }
+ path := filepath.Join(folder, l.started.Format("2006-01-02-150405")+".md")
+ return path, os.WriteFile(path, []byte(md), 0o644)
+}
diff --git a/cmd/lk/assistant_log_test.go b/cmd/lk/assistant_log_test.go
new file mode 100644
index 000000000..a7adabe5a
--- /dev/null
+++ b/cmd/lk/assistant_log_test.go
@@ -0,0 +1,41 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// discardOverlay is an overlay that drops every message.
+type discardOverlay struct{}
+
+func (discardOverlay) Send(any) error { return nil }
+
+func TestSessionLogMarkdownBlocks(t *testing.T) {
+ l := newSessionLog(discardOverlay{})
+ l.Send(overlayTranscript{"transcript", "u1", "user", 0, "How do I start?"})
+ l.Send(overlayTranscript{"transcript", "a2", "agent", 0, "Run this."})
+ l.Send(overlaySnippet{"snippet", "a2", "c1", "Terminal", "shell", "lk agent init my-agent", false})
+ l.Send(overlayTranscript{"transcript", "a2", "agent", 1, "Then run it."})
+ l.Send(overlaySnippet{"snippet", "a2", "c2", "Prompt for your coding agent", "prompt", "Add this:\n```python\nprint(1)\n```", false})
+
+ md := l.markdown("/tmp")
+ // Text after a card starts its own paragraph, so the fence closes.
+ require.Contains(t, md, "**LK:** Run this.\n\nTerminal:\n\n```shell\nlk agent init my-agent\n```\n\nThen run it.")
+ // A fence inside the code can't close the card's fence.
+ require.Contains(t, md, "````\nAdd this:\n```python\nprint(1)\n```\n````")
+}
diff --git a/cmd/lk/assistant_session.go b/cmd/lk/assistant_session.go
new file mode 100644
index 000000000..90b2fc2c4
--- /dev/null
+++ b/cmd/lk/assistant_session.go
@@ -0,0 +1,1216 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bufio"
+ "context"
+ "encoding/json"
+ "fmt"
+ "math"
+ "net"
+ "os"
+ "os/exec"
+ "regexp"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/atotto/clipboard"
+ "github.com/livekit/protocol/livekit/agent"
+ "github.com/urfave/cli/v3"
+
+ "github.com/livekit/livekit-cli/v2/pkg/agentfs"
+ "github.com/livekit/livekit-cli/v2/pkg/assistant"
+ "github.com/livekit/livekit-cli/v2/pkg/console"
+ "github.com/livekit/livekit-cli/v2/pkg/overlay"
+ "github.com/livekit/livekit-cli/v2/pkg/portaudio"
+)
+
+// overlaySender sends messages to the overlay page.
+type overlaySender interface {
+ Send(msg any) error
+}
+
+// overlayBandCount is how many bands the overlay page's meter expects.
+const overlayBandCount = 24
+
+// assistantSession runs the assistant agent in console mode, connects it to
+// the microphone and speakers, and shows the conversation in the overlay.
+type assistantSession struct {
+ ov *overlay.Overlay
+ out overlaySender // messages to the overlay page, through log, except in tests
+ log *sessionLog
+ cmd *cli.Command
+
+ pipeline *console.AudioPipeline
+ assistantConn net.Conn // the assistant agent's console connection
+ guest *guestAgent // the user's agent, while it runs here
+ guestStarting bool // the user's agent is starting and hasn't connected yet
+
+ // Conversation state, mapped onto the overlay's turns and segments. A
+ // turn stays the target for text that finishes it (such as the rest of
+ // an interrupted speech) after the other side starts a new turn; "open"
+ // only says whether new speech joins it.
+ turnN int
+ userTurn string // latest user turn, or ""
+ userOpen bool // new user speech joins userTurn
+ userCommits string // text the agent committed so far in userTurn
+ agentSpoke bool // the agent has spoken since userTurn started
+ userFinal string // final transcripts so far in userTurn
+ agentTurn string // latest agent turn, or ""
+ agentOpen bool // new agent speech joins agentTurn
+ agentSegment int
+ segmentText string // text of the current agent segment so far
+ segmentDone bool // the current agent segment is complete
+ finalPending bool // a streamed segment awaits the agent's committed text
+ shownCalls map[string]bool
+ snippetN int
+ toolTurns map[string]string // tool call id → the turn showing it
+
+ // Ending: set when the agent calls end_call. The overlay closes once the
+ // agent's goodbye has finished playing.
+ agentSpeaking bool
+ lastAudio time.Time
+ ending bool
+ endingSince time.Time
+ spokeGoodbye bool // the agent spoke after end_call
+ endCalled bool // the agent called end_call, which says goodbye
+ agentExited bool
+ dismissed bool
+ snippets map[string]string // card id → code, for Copy
+ snippetOrder []snippetRef // cards in the order shown, for copying by voice
+ links map[string]string // link card id → URL, for Open
+ // Docs pages the agent read this reply, and whether it showed a docs
+ // card. If it read pages but showed no card, lk shows a link to the
+ // first page when the reply ends.
+ turnPages []docsPage
+ turnCard bool
+ pendingLinks []overlayLink // link cards to show when the reply ends
+
+ // The local connection to the assistant agent, and the lk commands it
+ // runs through it.
+ channel *agentChannel
+ tools toolState
+}
+
+// agentMessage is a line from the assistant agent: its spoken text ("text",
+// then "flush" at the end of a segment) or a request to run an lk tool
+// ("lk_tool").
+type agentMessage struct {
+ Type string `json:"type"`
+ ID string `json:"id"`
+ Text string `json:"text"`
+ Name string `json:"name"`
+ Args map[string]any `json:"args"`
+
+ // show_code: a code card from the agent itself.
+ Title string `json:"title"`
+ Language string `json:"language"`
+ Code string `json:"code"`
+
+ // pipeline: the agent's models, for X-ray mode.
+ STT string `json:"stt"`
+ LLM string `json:"llm"`
+ TTS string `json:"tts"`
+ TurnDetection string `json:"turn_detection"`
+}
+
+// agentChannel is the local connection to the assistant agent, as JSON lines
+// in both directions. The agent connects once it starts.
+type agentChannel struct {
+ mu sync.Mutex
+ conn net.Conn
+}
+
+// serve accepts the agent's connection and sends each line it writes to out.
+func (c *agentChannel) serve(ln net.Listener, out chan<- agentMessage) {
+ conn, err := ln.Accept()
+ if err != nil {
+ return
+ }
+ c.mu.Lock()
+ c.conn = conn
+ c.mu.Unlock()
+ defer conn.Close()
+ scanner := bufio.NewScanner(conn)
+ scanner.Buffer(make([]byte, 64<<10), 1<<20)
+ for scanner.Scan() {
+ var m agentMessage
+ if json.Unmarshal(scanner.Bytes(), &m) == nil {
+ out <- m
+ }
+ }
+}
+
+// send writes a message to the agent. It's dropped if the agent isn't
+// connected.
+func (c *agentChannel) send(msg any) {
+ b, err := json.Marshal(msg)
+ if err != nil {
+ return
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.conn != nil {
+ _, _ = c.conn.Write(append(b, '\n'))
+ }
+}
+
+func newAssistantSession(ov *overlay.Overlay, cmd *cli.Command) *assistantSession {
+ log := newSessionLog(ov)
+ return &assistantSession{
+ ov: ov,
+ out: log,
+ log: log,
+ cmd: cmd,
+ shownCalls: map[string]bool{},
+ toolTurns: map[string]string{},
+ snippets: map[string]string{},
+ links: map[string]string{},
+ }
+}
+
+// run starts the agent and relays between it and the overlay until ctx ends
+// or the overlay closes. Errors are also shown in the overlay.
+func (s *assistantSession) run(ctx context.Context) (err error) {
+ if !s.waitReady(ctx) {
+ return nil
+ }
+ defer func() {
+ if err != nil && ctx.Err() != nil {
+ // The overlay closed, or lk was interrupted, while the session
+ // was still starting. That isn't a failure.
+ err = nil
+ }
+ if err != nil {
+ s.showError(startupError(err))
+ // The run loop has stopped, but the error card and any code
+ // cards are still on screen: keep their Copy buttons working.
+ s.serveCopies(ctx)
+ }
+ }()
+
+ s.connecting("Starting")
+
+ // Credentials come from the user's folder (its .env files, with
+ // --experimental-auth) or their lk config, not from the agent's folder.
+ cwd, err := os.Getwd()
+ if err != nil {
+ return err
+ }
+ creds, err := resolveCredentials(ctx, s.cmd, cwd)
+ if err != nil {
+ return errNoCredentials{err}
+ }
+ project := "your LiveKit Cloud project"
+ if rp, err := resolveProject(s.cmd, loadParams{requireURL: true}); err == nil && rp.project != nil && rp.project.Name != "" {
+ project = "your LiveKit Cloud project, " + rp.project.Name
+ }
+ out.Statusf("LK uses LiveKit Inference through %s. Its usage counts toward your plan.", project)
+
+ dir := s.cmd.String("agent-dir")
+ projectType := agentfs.ProjectTypePythonUV
+ if dir == "" {
+ if dir, err = assistant.DefaultDir(); err != nil {
+ return err
+ }
+ if err := assistant.Prepare(ctx, dir, s.connecting); err != nil {
+ return err
+ }
+ } else if _, projectType, err = agentfs.DetectProjectRoot(dir); err != nil {
+ return fmt.Errorf("no agent project in %s: %w", dir, err)
+ }
+
+ if err := portaudio.Initialize(); err != nil {
+ return fmt.Errorf("failed to initialize PortAudio: %w", err)
+ }
+ defer portaudio.Terminate()
+ inputDev, err := portaudio.DefaultInputDevice()
+ if err != nil {
+ return fmt.Errorf("input device: %w", err)
+ }
+ outputDev, err := portaudio.DefaultOutputDevice()
+ if err != nil {
+ return fmt.Errorf("output device: %w", err)
+ }
+
+ server, err := console.NewTCPServer("127.0.0.1:0")
+ if err != nil {
+ return err
+ }
+ defer server.Close()
+
+ // The agent connects here to stream what it says, in time with its
+ // speech, and to run lk commands. Console mode itself only
+ // reports the agent's text once it finishes speaking.
+ channelLn, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ return err
+ }
+ defer channelLn.Close()
+ agentMessages := make(chan agentMessage, 256)
+ s.channel = &agentChannel{}
+ go s.channel.serve(channelLn, agentMessages)
+
+ projectDir, err := os.Getwd()
+ if err != nil {
+ return err
+ }
+ toolsCtx, cancelTools := context.WithCancel(ctx)
+ defer cancelTools()
+ s.tools = toolState{
+ ctx: toolsCtx,
+ dir: projectDir,
+ creds: creds,
+ results: make(chan toolResult, 16),
+ pending: map[string]pendingTool{},
+ choices: map[string][]overlayConfirmOption{},
+ }
+
+ s.connecting("Starting the assistant agent")
+ agentProc, err := startAgent(AgentStartConfig{
+ Dir: dir,
+ Entrypoint: assistant.Entrypoint,
+ ProjectType: projectType,
+ CLIArgs: buildConsoleArgs(server.Addr().String(), false),
+ Env: append(creds,
+ "LK_ASSISTANT_ADDR="+channelLn.Addr().String(),
+ "LK_ASSISTANT_PROJECT="+projectDir,
+ "LK_ASSISTANT_FOLDER="+assistant.DescribeFolder(projectDir).JSON(),
+ "LK_ASSISTANT_CODING_AGENTS="+checkCodingAgentSetup(projectDir).JSON(),
+ ),
+ FailSignals: consoleCrashSignals,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to start the assistant agent: %w", err)
+ }
+ defer agentProc.Kill()
+
+ conn, err := acceptAgent(ctx, server, agentProc, "the assistant agent")
+ if err != nil {
+ return err
+ }
+ s.assistantConn = conn
+ defer func() {
+ if s.guest != nil {
+ s.guest.conn.Close()
+ s.guest.proc.Kill()
+ }
+ }()
+
+ s.pipeline, err = console.NewPipeline(console.PipelineConfig{
+ InputDevice: inputDev,
+ OutputDevice: outputDev,
+ Conn: conn,
+ })
+ if err != nil {
+ return fmt.Errorf("pipeline: %w", err)
+ }
+ pipelineCtx, cancelPipeline := context.WithCancel(ctx)
+ go s.pipeline.Start(pipelineCtx)
+ defer func() {
+ cancelPipeline()
+ // Stop can block forever in PortAudio once the streams are aborted,
+ // so, like lk agent console, don't wait for it. The deferred
+ // agentProc.Kill runs next either way.
+ go s.pipeline.Stop()
+ }()
+
+ ticker := time.NewTicker(demoFrame * 2)
+ defer ticker.Stop()
+ agentDone, agentFailed := agentProc.Done(), agentProc.Failed()
+ for {
+ select {
+ case <-ctx.Done():
+ return nil
+ case <-s.ov.Done():
+ return nil
+ case err := <-agentDone:
+ if s.ending {
+ // The session shut down after end_call, as expected. Its
+ // goodbye may still be playing; checkEnding closes the
+ // overlay once it has.
+ s.agentExited = true
+ agentDone = nil
+ continue
+ }
+ s.out.Send(overlayState{"state", "idle"})
+ if err != nil {
+ return fmt.Errorf("the assistant agent exited: %w\n\n%s", err, agentExitDetail(agentProc))
+ }
+ return fmt.Errorf("the assistant agent exited.\n\n%s", agentExitDetail(agentProc))
+ case <-agentFailed:
+ // The worker keeps running when its job crashes, so agentDone
+ // doesn't fire.
+ if s.ending {
+ s.agentExited = true
+ agentFailed = nil
+ continue
+ }
+ // The crash marker arrives mid-traceback; give trailing output a moment.
+ time.Sleep(500 * time.Millisecond)
+ s.out.Send(overlayState{"state", "idle"})
+ return fmt.Errorf("the assistant agent crashed.\n\n%s", agentExitDetail(agentProc))
+ case ev := <-s.pipeline.Events:
+ s.handleEvent(ev)
+ case m := <-agentMessages:
+ switch m.Type {
+ case "text":
+ // The assistant is silent while the user's agent is active;
+ // anything it says then isn't heard, so don't show it.
+ if !s.guestActive() {
+ s.agentText(m.Text)
+ }
+ case "flush":
+ if !s.guestActive() {
+ s.agentFlush()
+ }
+ case "lk_tool":
+ s.runTool(m.ID, m.Name, m.Args)
+ case "show_code":
+ s.showAgentCode(m)
+ case "end":
+ // The agent found that the user wants to close LK. Close after
+ // its goodbye (see checkEnding).
+ if !s.guestActive() {
+ s.startEnding()
+ }
+ case "pipeline":
+ s.out.Send(overlayPipeline{Type: "pipeline", STT: m.STT, LLM: m.LLM, TTS: m.TTS, TurnDetection: m.TurnDetection})
+ }
+ case r := <-s.tools.results:
+ s.toolFinished(r)
+ case ev := <-s.ov.Events():
+ switch ev.Type {
+ case "copy", "copy_text":
+ s.handleCopy(ev)
+ case "confirm":
+ s.confirmTool(ev.ID, ev.Option)
+ case "open":
+ s.openLink(ev.ID)
+ case "text":
+ s.typed(ev.Text)
+ case "back":
+ s.returnToAssistant("")
+ case "mute":
+ s.pipeline.SetMuted(ev.On)
+ }
+ case <-ticker.C:
+ s.sendBands()
+ s.checkEnding()
+ }
+ }
+}
+
+func (s *assistantSession) waitReady(ctx context.Context) bool {
+ for {
+ select {
+ case ev := <-s.ov.Events():
+ if ev.Type == "ready" {
+ return true
+ }
+ case <-ctx.Done():
+ return false
+ case <-s.ov.Done():
+ return false
+ }
+ }
+}
+
+// handleCopy copies a code card ("copy") or the text the page sends
+// ("copy_text") to the clipboard.
+func (s *assistantSession) handleCopy(ev overlay.Event) {
+ switch ev.Type {
+ case "copy":
+ s.copy(ev.ID)
+ case "copy_text":
+ _ = clipboard.WriteAll(ev.Text)
+ }
+}
+
+// serveCopies handles copy requests from the page until the overlay closes.
+func (s *assistantSession) serveCopies(ctx context.Context) {
+ for {
+ select {
+ case ev := <-s.ov.Events():
+ s.handleCopy(ev)
+ case <-ctx.Done():
+ return
+ case <-s.ov.Done():
+ return
+ }
+ }
+}
+
+// acceptAgent waits for the agent subprocess to connect to the console server.
+// who names the agent in errors, such as "the assistant agent".
+func acceptAgent(ctx context.Context, server *console.TCPServer, agentProc *AgentProcess, who string) (net.Conn, error) {
+ type result struct {
+ conn net.Conn
+ err error
+ }
+ accepted := make(chan result, 1)
+ go func() {
+ conn, err := server.Accept()
+ accepted <- result{conn, err}
+ }()
+
+ select {
+ case res := <-accepted:
+ if res.err != nil {
+ return nil, fmt.Errorf("agent connection: %w", res.err)
+ }
+ return res.conn, nil
+ case err := <-agentProc.Done():
+ if err != nil {
+ return nil, fmt.Errorf("%s exited before connecting: %w\n\n%s", who, err, agentExitDetail(agentProc))
+ }
+ return nil, fmt.Errorf("%s exited before connecting.\n\n%s", who, agentExitDetail(agentProc))
+ case <-agentProc.Failed():
+ // The crash marker arrives mid-traceback; give trailing output a moment.
+ time.Sleep(500 * time.Millisecond)
+ return nil, fmt.Errorf("%s stopped before connecting.\n\n%s", who, agentExitDetail(agentProc))
+ case <-time.After(90 * time.Second):
+ return nil, fmt.Errorf("timed out waiting for %s to connect.\n\n%s", who, agentExitDetail(agentProc))
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+}
+
+func (s *assistantSession) connecting(label string) {
+ s.out.Send(map[string]string{"type": "state", "state": "connecting", "label": label})
+}
+
+// sendBands sends the levels of whoever is audible: the agent while its audio
+// plays, otherwise the (echo-cancelled) microphone.
+func (s *assistantSession) sendBands() {
+ var bands [console.NumFFTBands]float64
+ var level float64
+ if s.pipeline.IsPlaying() {
+ s.lastAudio = time.Now()
+ bands, level = s.pipeline.PlaybackFFTBands(), s.pipeline.PlaybackLevel()
+ } else {
+ bands, level = s.pipeline.FFTBands(), s.pipeline.Level()
+ }
+ // The bands are peak-normalized per frame, so room noise would fill the
+ // meter. Gate them by the overall level.
+ gate := smoothstep(-55, -38, level)
+ if gate == 0 {
+ return
+ }
+ out := resampleBands(bands[:], overlayBandCount)
+ for i := range out {
+ out[i] = math.Round(out[i]*gate*100) / 100
+ }
+ s.out.Send(overlayBands{"bands", out})
+}
+
+func (s *assistantSession) handleEvent(ev *agent.AgentSessionEvent) {
+ switch e := ev.GetEvent().(type) {
+ case *agent.AgentSessionEvent_AgentStateChanged_:
+ s.agentSpeaking = e.AgentStateChanged.GetNewState() == agent.AgentState_AS_SPEAKING
+ if s.agentSpeaking && s.ending {
+ s.spokeGoodbye = true
+ }
+ switch e.AgentStateChanged.GetNewState() {
+ case agent.AgentState_AS_IDLE:
+ s.out.Send(overlayState{"state", "idle"})
+ case agent.AgentState_AS_LISTENING:
+ s.showReadMore()
+ s.out.Send(overlayState{"state", "listening"})
+ case agent.AgentState_AS_THINKING:
+ s.out.Send(overlayState{"state", "thinking"})
+ case agent.AgentState_AS_SPEAKING:
+ s.beginAgentSpeech()
+ s.out.Send(overlayState{"state", "speaking"})
+ }
+
+ case *agent.AgentSessionEvent_UserInputTranscribed_:
+ s.userTranscript(e.UserInputTranscribed.GetTranscript(), e.UserInputTranscribed.GetIsFinal())
+
+ case *agent.AgentSessionEvent_ConversationItemAdded_:
+ switch item := e.ConversationItemAdded.GetItem().GetItem().(type) {
+ case *agent.ChatContext_ChatItem_Message:
+ te, ok := messageToTurnEvent(item.Message)
+ if !ok {
+ return
+ }
+ if te.Role == "user" {
+ s.userCommitted(te.Text)
+ } else {
+ s.agentMessage(te.Text)
+ }
+ if m := item.Message.GetMetrics(); m != nil {
+ s.out.Send(metricsMessage(te.Role, m))
+ }
+ case *agent.ChatContext_ChatItem_FunctionCall:
+ s.toolCall(item.FunctionCall)
+ }
+
+ case *agent.AgentSessionEvent_FunctionToolsStarted_:
+ for _, fc := range e.FunctionToolsStarted.GetFunctionCalls() {
+ s.toolCall(fc)
+ }
+
+ case *agent.AgentSessionEvent_FunctionToolsExecuted_:
+ failed := map[string]bool{}
+ for _, out := range e.FunctionToolsExecuted.GetFunctionCallOutputs() {
+ failed[out.GetCallId()] = out.GetIsError()
+ }
+ outputs := map[string]string{}
+ for _, out := range e.FunctionToolsExecuted.GetFunctionCallOutputs() {
+ outputs[out.GetCallId()] = out.GetOutput()
+ }
+ for _, fc := range e.FunctionToolsExecuted.GetFunctionCalls() {
+ s.toolCall(fc)
+ s.toolDone(fc.GetCallId(), failed[fc.GetCallId()])
+ if (fc.GetName() == "read_docs" || fc.GetName() == "get_pages") && !failed[fc.GetCallId()] {
+ s.turnPages = append(s.turnPages, docsPagesRead(fc.GetArguments(), outputs[fc.GetCallId()])...)
+ }
+ }
+
+ case *agent.AgentSessionEvent_Error_:
+ // Session errors (such as rejected credentials) usually end the
+ // conversation, so keep them on screen. Errors the agent retries
+ // don't stay.
+ msg := e.Error.GetMessage()
+ if msg == "" {
+ return
+ }
+ // The user's own agent uses its own providers and keys.
+ if !s.guestActive() && isAuthError(msg) {
+ s.showError(overlayError{Type: "error",
+ Title: "LiveKit Cloud rejected your project's API key",
+ Detail: "The assistant uses LiveKit Inference through your default lk project, and its key didn't work. Sign in again, or pick a different project with --project.",
+ Fix: "lk cloud auth"})
+ return
+ }
+ s.out.Send(overlayNotice{Type: "notice", Text: msg, Sticky: !strings.Contains(msg, "recoverable=True")})
+
+ case *agent.AgentSessionEvent_EotPrediction_:
+ s.out.Send(overlayEOT{Type: "eot", Probability: e.EotPrediction.GetProbability(), Threshold: e.EotPrediction.GetThreshold()})
+
+ case *agent.AgentSessionEvent_SessionUsageUpdated_:
+ s.out.Send(usageMessage(e.SessionUsageUpdated.GetUsage()))
+ }
+}
+
+// isAuthError reports whether a session error says the API key was rejected.
+// The message is the agent's error event as text, such as "type='llm_error'
+// timestamp=... error=APIStatusError('...', status_code=401, ...)
+// recoverable=False".
+func isAuthError(msg string) bool {
+ msg = strings.ToLower(msg)
+ return strings.Contains(msg, "status_code=401") || strings.Contains(msg, "invalid api key")
+}
+
+// userTurnForSpeech makes sure new user speech has an open turn to join. A
+// new user turn means the agent's next reply starts a new turn too.
+func (s *assistantSession) userTurnForSpeech() {
+ if s.userOpen {
+ return
+ }
+ if s.userTurn != "" && !s.agentSpoke {
+ // The agent hasn't said anything since the user's last turn (at most
+ // it started looking things up), so the user is still finishing the
+ // same thought, such as after a mid-sentence pause.
+ s.userOpen = true
+ return
+ }
+ s.agentSpoke = false
+ s.turnPages = nil
+ s.turnCard = false
+ s.turnN++
+ s.userTurn = fmt.Sprintf("u%d", s.turnN)
+ s.userOpen = true
+ s.userCommits = ""
+ s.userFinal = ""
+ s.agentOpen = false
+}
+
+// agentTurnForReply makes sure a new agent reply has an open turn to join.
+// It closes the user's turn, so the user's next speech starts a new turn,
+// unless the agent hasn't spoken yet (see userTurnForSpeech). That holds
+// even when the reply joins an agent turn opened earlier, such as by a tool
+// call while the user was still talking.
+func (s *assistantSession) agentTurnForReply() {
+ s.userOpen = false
+ if s.agentOpen {
+ return
+ }
+ s.turnN++
+ s.agentTurn = fmt.Sprintf("a%d", s.turnN)
+ s.agentOpen = true
+ s.agentSegment = 0
+ s.segmentText = ""
+ s.segmentDone = false
+ s.finalPending = false
+}
+
+// streaming reports whether the agent is partway through speaking the
+// current segment, so its remaining text belongs there.
+func (s *assistantSession) streaming() bool {
+ return s.agentTurn != "" && s.segmentText != "" && !s.segmentDone
+}
+
+// nextSegment moves to a new agent paragraph unless the current one is unused.
+func (s *assistantSession) nextSegment() {
+ if s.segmentText == "" && !s.segmentDone {
+ return
+ }
+ s.agentSegment++
+ s.segmentText = ""
+ s.segmentDone = false
+}
+
+func (s *assistantSession) sendAgentSegment() {
+ s.out.Send(overlayTranscript{"transcript", s.agentTurn, "agent", s.agentSegment, s.segmentText})
+}
+
+func (s *assistantSession) userTranscript(text string, final bool) {
+ text = strings.TrimSpace(text)
+ if text == "" {
+ return
+ }
+ s.userTurnForSpeech()
+ full := strings.TrimSpace(s.userFinal + " " + text)
+ if final {
+ s.userFinal = full
+ }
+ s.out.Send(overlayTranscript{"transcript", s.userTurn, "user", 0, full})
+}
+
+// userCommitted replaces the live transcript with the text the agent
+// committed to the conversation. A user who pauses mid-thought can get
+// several commits in a row; until the agent starts replying, they're one
+// turn, and the turn stays open for more speech.
+func (s *assistantSession) userCommitted(text string) {
+ s.userTurnForSpeech()
+ s.userCommits = strings.TrimSpace(s.userCommits + " " + text)
+ s.userFinal = s.userCommits
+ s.out.Send(overlayTranscript{"transcript", s.userTurn, "user", 0, s.userCommits})
+}
+
+// startEnding closes the overlay once the agent has said goodbye, or after
+// endingTimeout.
+func (s *assistantSession) startEnding() {
+ if s.ending {
+ return
+ }
+ s.ending = true
+ s.endingSince = time.Now()
+}
+
+// beginAgentSpeech adds an empty agent paragraph so the page shows its cursor
+// until the first words arrive. A speech that resumes keeps its paragraph.
+func (s *assistantSession) beginAgentSpeech() {
+ s.agentSpoke = true
+ if s.streaming() && s.agentOpen {
+ return
+ }
+ s.agentTurnForReply()
+ if s.segmentDone {
+ s.nextSegment()
+ }
+ s.sendAgentSegment()
+}
+
+// agentText adds text the agent is speaking right now. The rest of a speech
+// the user interrupted stays in that speech's turn.
+func (s *assistantSession) agentText(text string) {
+ s.agentSpoke = true
+ if !s.streaming() {
+ s.agentTurnForReply()
+ if s.segmentDone {
+ s.nextSegment()
+ }
+ }
+ s.segmentText += text
+ s.sendAgentSegment()
+}
+
+func (s *assistantSession) agentFlush() {
+ if s.segmentText != "" {
+ s.segmentDone = true
+ s.finalPending = true
+ }
+}
+
+// agentMessage handles text the agent committed to the conversation after
+// speaking. If the same speech was streamed, the streamed text stays: it is
+// already free of expressive markup and stops where an interruption cut the
+// speech off. Otherwise the committed text is shown, with its markup removed.
+func (s *assistantSession) agentMessage(text string) {
+ s.agentSpoke = true
+ if s.finalPending || s.streaming() {
+ s.segmentDone = true
+ s.finalPending = false
+ return
+ }
+ s.agentTurnForReply()
+ s.nextSegment()
+ s.segmentText = stripMarkup(text)
+ s.segmentDone = true
+ s.sendAgentSegment()
+}
+
+var (
+ markupTagRe = regexp.MustCompile(`]*>`)
+ markupSpaceRe = regexp.MustCompile(`\s{2,}`)
+)
+
+// stripMarkup removes expressive markup tags, such as , from
+// committed agent text. The agent strips streamed text with the SDK's own
+// stripper; this is only the fallback for text that wasn't streamed.
+func stripMarkup(text string) string {
+ return strings.TrimSpace(markupSpaceRe.ReplaceAllString(markupTagRe.ReplaceAllString(text, ""), " "))
+}
+
+// toolCall handles a tool call the agent made. show_code calls become code
+// cards, end_call starts closing the overlay, and other calls (docs searches)
+// show as a line in the agent's turn. Each call arrives in several events;
+// only the first counts.
+func (s *assistantSession) toolCall(fc *agent.FunctionCall) {
+ switch fc.GetName() {
+ case assistant.ShowCodeTool:
+ s.showCode(fc)
+ case assistant.HandOffTaskTool:
+ s.handOffTask(fc)
+ case assistant.ShowLinkTool:
+ s.showLink(fc)
+ case assistant.ShowAnswerTool:
+ s.showAnswer(fc)
+ case assistant.EndCallTool:
+ // Only the assistant's end_call closes the overlay; the user's own
+ // agent may have one too.
+ if !s.guestActive() {
+ // end_call always says goodbye (its end instructions), so wait for it.
+ s.endCalled = true
+ s.startEnding()
+ }
+ default:
+ id := fc.GetCallId()
+ if _, ok := s.toolTurns[id]; ok {
+ return
+ }
+ s.agentTurnForReply()
+ s.toolTurns[id] = s.agentTurn
+ label, detail := describeTool(fc.GetName(), fc.GetArguments())
+ s.out.Send(overlayTool{Type: "tool", Turn: s.agentTurn, ID: id, Label: label, Detail: detail, Status: "running"})
+ }
+}
+
+// toolDone marks a tool call's line as finished.
+func (s *assistantSession) toolDone(id string, failed bool) {
+ turn, ok := s.toolTurns[id]
+ if !ok {
+ return
+ }
+ status := "done"
+ if failed {
+ status = "error"
+ }
+ s.out.Send(overlayTool{Type: "tool", Turn: turn, ID: id, Status: status})
+}
+
+// describeTool returns a short label for a tool call and the main argument,
+// such as the search query.
+func describeTool(name, arguments string) (label, detail string) {
+ var args map[string]any
+ _ = json.Unmarshal([]byte(arguments), &args)
+ for _, key := range []string{"query", "paths", "path", "package", "repo", "template", "name"} {
+ switch v := args[key].(type) {
+ case string:
+ detail = v
+ case []any:
+ parts := make([]string, 0, len(v))
+ for _, p := range v {
+ if s, ok := p.(string); ok {
+ parts = append(parts, s)
+ }
+ }
+ detail = strings.Join(parts, ", ")
+ }
+ if detail != "" {
+ break
+ }
+ }
+ switch name {
+ case "get_docs_overview":
+ return "Read the docs overview", detail
+ case "docs_search":
+ return "Searched the docs", detail
+ case "get_pages", "read_docs":
+ return "Read the docs", detail
+ case "code_search":
+ return "Searched LiveKit code", detail
+ case "get_project_info":
+ return "Checked your LiveKit project", detail
+ case "list_cloud_agents":
+ return "Listed your agents", detail
+ case "list_templates":
+ return "Listed the templates", detail
+ case "copy_to_clipboard":
+ return "Copy to clipboard", detail
+ case "create_project":
+ return "Create a project", detail
+ case "confirm_action":
+ var a struct {
+ Approved bool `json:"approved"`
+ }
+ _ = json.Unmarshal([]byte(arguments), &a)
+ if a.Approved {
+ return "Confirmed by voice", ""
+ }
+ return "Declined by voice", ""
+ case "set_up_coding_agent":
+ return "Set up your coding agent", detail
+ default:
+ return strings.ReplaceAll(name, "_", " "), detail
+ }
+}
+
+// checkEnding closes the overlay after end_call, once the agent's goodbye has
+// finished playing. The goodbye comes after the tool call, so a pause before
+// it doesn't count. It gives up waiting after endingTimeout.
+func (s *assistantSession) checkEnding() {
+ if !s.ending || s.dismissed {
+ return
+ }
+ playing := s.pipeline.IsPlaying()
+ if playing {
+ s.spokeGoodbye = true
+ }
+ quiet := !playing && !s.agentSpeaking && time.Since(s.lastAudio) > 800*time.Millisecond
+ // Wait for the goodbye to play. If none starts within goodbyeWait, the
+ // agent likely said it before the ending began, so close once it's quiet.
+ noGoodbye := !s.endCalled && time.Since(s.endingSince) > goodbyeWait
+ if ((s.spokeGoodbye || s.agentExited || noGoodbye) && quiet) || time.Since(s.endingSince) > endingTimeout {
+ s.dismiss()
+ }
+}
+
+const endingTimeout = 15 * time.Second
+
+// goodbyeWait is how long the overlay waits for the agent's goodbye to start
+// before it closes without one.
+const goodbyeWait = 4 * time.Second
+
+// dismiss plays the overlay's exit animation, which then closes it.
+func (s *assistantSession) dismiss() {
+ if s.dismissed {
+ return
+ }
+ s.dismissed = true
+ s.out.Send(map[string]string{"type": "dismiss"})
+}
+
+// showCode shows a show_code call as a code card.
+func (s *assistantSession) showCode(fc *agent.FunctionCall) {
+ if s.shownCalls[fc.GetCallId()] {
+ return
+ }
+ var args struct {
+ Title string `json:"title"`
+ Language string `json:"language"`
+ Code string `json:"code"`
+ }
+ if err := json.Unmarshal([]byte(fc.GetArguments()), &args); err != nil || args.Code == "" {
+ return
+ }
+ s.shownCalls[fc.GetCallId()] = true
+ // The same code is already on screen, such as when the user asks to walk
+ // through it: don't show it again.
+ for _, code := range s.snippets {
+ if code == args.Code {
+ return
+ }
+ }
+ s.agentTurnForReply()
+
+ id := fc.GetCallId()
+ s.addSnippet(id, "code", args.Title, args.Code)
+ s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, args.Title, normalizeLang(args.Language), args.Code, false})
+}
+
+// handOffTask shows a hand_off_task call as a prompt card for the user to
+// copy into their coding agent.
+func (s *assistantSession) handOffTask(fc *agent.FunctionCall) {
+ if s.shownCalls[fc.GetCallId()] {
+ return
+ }
+ var args struct {
+ Task string `json:"task"`
+ }
+ if err := json.Unmarshal([]byte(fc.GetArguments()), &args); err != nil || args.Task == "" {
+ return
+ }
+ s.shownCalls[fc.GetCallId()] = true
+ s.agentTurnForReply()
+
+ id := fc.GetCallId()
+ s.addSnippet(id, "prompt", "Prompt for your coding agent", args.Task)
+ s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, "Prompt for your coding agent", "prompt", args.Task, false})
+}
+
+// snippetRef is a card that can be copied. kind is what it is, for the
+// agent's copy reply: code, a prompt, a link, or an answer.
+type snippetRef struct{ id, kind, title string }
+
+// addSnippet records a card so it can be copied.
+func (s *assistantSession) addSnippet(id, kind, title, code string) {
+ s.snippets[id] = code
+ s.snippetOrder = append(s.snippetOrder, snippetRef{id, kind, title})
+}
+
+func (s *assistantSession) copy(id string) {
+ code, ok := s.snippets[id]
+ if !ok || clipboard.WriteAll(code) != nil {
+ return
+ }
+ s.out.Send(overlayCopied{"copied", id})
+}
+
+// docsURL is where links to read more point. The agent gets these URLs from
+// its docs searches, and its show_link tool rejects any other URL.
+const docsURL = "https://docs.livekit.io/"
+
+// docsPageURL returns the page people read for a docs URL. The Docs MCP
+// server returns the Markdown version, which ends in .md.
+func docsPageURL(url string) string {
+ return strings.TrimSuffix(url, ".md")
+}
+
+// showLink shows a show_link call as a card that links to a docs page.
+func (s *assistantSession) showLink(fc *agent.FunctionCall) {
+ if s.shownCalls[fc.GetCallId()] {
+ return
+ }
+ var args struct {
+ Title string `json:"title"`
+ URL string `json:"url"`
+ }
+ if err := json.Unmarshal([]byte(fc.GetArguments()), &args); err != nil || !strings.HasPrefix(args.URL, docsURL) {
+ return
+ }
+ s.shownCalls[fc.GetCallId()] = true
+ args.URL = docsPageURL(args.URL)
+ if args.Title == "" {
+ args.Title = "LiveKit docs"
+ }
+ s.turnCard = true
+ // The agent can call show_link before it answers. Show the card after the
+ // answer, when the reply ends (see showReadMore), and only one per reply.
+ if len(s.pendingLinks) > 0 {
+ return
+ }
+ s.agentTurnForReply()
+ id := fc.GetCallId()
+ s.links[id] = args.URL
+ s.addSnippet(id, "link", args.Title, args.URL)
+ s.pendingLinks = append(s.pendingLinks, overlayLink{Type: "link", Turn: s.agentTurn, ID: id, Title: args.Title, URL: args.URL})
+}
+
+// typed sends text the user typed to the assistant, as their next turn.
+// The user's own agent only takes speech.
+func (s *assistantSession) typed(text string) {
+ if text = strings.TrimSpace(text); text == "" || s.guestActive() {
+ return
+ }
+ s.channel.send(map[string]string{"type": "user_text", "text": text})
+}
+
+// showAnswer shows a show_answer call as a card of short points, with a link
+// to the docs page they come from. Copy copies the points and the link.
+func (s *assistantSession) showAnswer(fc *agent.FunctionCall) {
+ if s.shownCalls[fc.GetCallId()] {
+ return
+ }
+ var args struct {
+ Title string `json:"title"`
+ Points []string `json:"points"`
+ SourceTitle string `json:"source_title"`
+ SourceURL string `json:"source_url"`
+ }
+ if err := json.Unmarshal([]byte(fc.GetArguments()), &args); err != nil || len(args.Points) == 0 || !strings.HasPrefix(args.SourceURL, docsURL) {
+ return
+ }
+ s.shownCalls[fc.GetCallId()] = true
+ args.SourceURL = docsPageURL(args.SourceURL)
+ s.agentTurnForReply()
+ id := fc.GetCallId()
+ s.links[id] = args.SourceURL
+ s.turnCard = true
+ var text strings.Builder
+ for _, p := range args.Points {
+ text.WriteString("- " + p + "\n")
+ }
+ fmt.Fprintf(&text, "\nSource: %s (%s)", args.SourceTitle, args.SourceURL)
+ s.addSnippet(id, "answer", args.Title, text.String())
+ s.out.Send(overlayAnswer{Type: "answer", Turn: s.agentTurn, ID: id, Title: args.Title, Points: args.Points, SourceTitle: args.SourceTitle, SourceURL: args.SourceURL})
+}
+
+// docsPage is a docs page the agent read with read_docs (or get_pages).
+type docsPage struct{ title, url string }
+
+// docsPagesRead lists the docs pages a read_docs or get_pages call read: their URLs from
+// its arguments, and titles from the "# Title" lines in its output.
+func docsPagesRead(arguments, output string) []docsPage {
+ var args struct {
+ Paths []string `json:"paths"`
+ }
+ if json.Unmarshal([]byte(arguments), &args) != nil {
+ return nil
+ }
+ var titles []string
+ for line := range strings.SplitSeq(output, "\n") {
+ if t, ok := strings.CutPrefix(line, "# "); ok {
+ titles = append(titles, strings.TrimSpace(t))
+ }
+ }
+ // The output is JSON text, so newlines can be escaped.
+ if len(titles) == 0 {
+ for part := range strings.SplitSeq(output, "\\n") {
+ if t, ok := strings.CutPrefix(part, "# "); ok {
+ titles = append(titles, strings.TrimSpace(t))
+ }
+ }
+ }
+ var pages []docsPage
+ for i, p := range args.Paths {
+ url := p
+ if strings.HasPrefix(p, "/") {
+ url = strings.TrimSuffix(docsURL, "/") + p
+ }
+ if !strings.HasPrefix(url, docsURL) {
+ continue // GitHub code, not a docs page
+ }
+ title := "LiveKit docs"
+ if i < len(titles) {
+ title = titles[i]
+ }
+ pages = append(pages, docsPage{title: title, url: docsPageURL(url)})
+ }
+ return pages
+}
+
+// showReadMore runs when a reply ends. It shows the link cards the agent asked
+// for, after the answer. If the agent answered from the docs without one, it
+// shows a link to the first page it read.
+func (s *assistantSession) showReadMore() {
+ for _, link := range s.pendingLinks {
+ s.out.Send(link)
+ }
+ s.pendingLinks = nil
+ if s.turnCard || len(s.turnPages) == 0 || s.agentTurn == "" {
+ return
+ }
+ page := s.turnPages[0]
+ s.turnCard = true
+ s.snippetN++
+ id := fmt.Sprintf("readmore%d", s.snippetN)
+ s.links[id] = page.url
+ s.addSnippet(id, "link", page.title, page.url)
+ s.out.Send(overlayLink{Type: "link", Turn: s.agentTurn, ID: id, Title: page.title, URL: page.url})
+}
+
+// openLink opens a link card's page in the user's browser.
+func (s *assistantSession) openLink(id string) {
+ if url, ok := s.links[id]; ok {
+ _ = exec.Command("open", url).Start()
+ }
+}
+
+// copyByVoice copies the latest card of kind ("code", "link", "prompt", or
+// "latest" for any kind). title, if given, picks among cards of that kind by
+// the words they share, so the agent doesn't have to know exact titles. The
+// reply says exactly what was copied, or that there was nothing to copy.
+func (s *assistantSession) copyByVoice(requestID, kind, title string) {
+ ref, ok := s.pickCard(kind, title)
+ if !ok {
+ what := "card"
+ if kind != "" && kind != "latest" {
+ what = kind + " card"
+ }
+ s.replyTool(requestID, "", fmt.Errorf("nothing was copied: there's no %s on screen. Tell the user", what))
+ return
+ }
+ if err := clipboard.WriteAll(s.snippets[ref.id]); err != nil {
+ s.replyTool(requestID, "", fmt.Errorf("nothing was copied: %w", err))
+ return
+ }
+ s.out.Send(overlayCopied{"copied", ref.id})
+ s.replyTool(requestID, fmt.Sprintf("Copied the %s card %q. Tell the user in a few words what you copied.", ref.kind, ref.title), nil)
+}
+
+// pickCard returns the card to copy: of kind, and with the most words in
+// common with title, preferring the latest.
+func (s *assistantSession) pickCard(kind, title string) (snippetRef, bool) {
+ words := strings.Fields(strings.ToLower(title))
+ best, bestScore, found := snippetRef{}, -1, false
+ for i := len(s.snippetOrder) - 1; i >= 0; i-- {
+ ref := s.snippetOrder[i]
+ if kind != "" && kind != "latest" && ref.kind != kind {
+ continue
+ }
+ score := 0
+ cardTitle := strings.ToLower(ref.title)
+ for _, w := range words {
+ if len(w) > 2 && strings.Contains(cardTitle, w) {
+ score++
+ }
+ }
+ if score > bestScore {
+ best, bestScore, found = ref, score, true
+ }
+ }
+ return best, found
+}
+
+// normalizeLang maps language names to the ones the page highlights.
+func normalizeLang(lang string) string {
+ switch l := strings.ToLower(strings.TrimSpace(lang)); l {
+ case "bash", "sh", "zsh", "console", "terminal":
+ return "shell"
+ case "py":
+ return "python"
+ case "ts", "tsx", "typescript":
+ return "typescript"
+ case "js", "jsx", "mjs", "javascript", "node", "nodejs":
+ return "javascript"
+ default:
+ return l
+ }
+}
+
+// resampleBands linearly interpolates bands to n values.
+func resampleBands(src []float64, n int) []float64 {
+ out := make([]float64, n)
+ if len(src) == 0 {
+ return out
+ }
+ for i := range out {
+ f := float64(i) * float64(len(src)-1) / float64(n-1)
+ j := int(f)
+ if j >= len(src)-1 {
+ out[i] = src[len(src)-1]
+ continue
+ }
+ t := f - float64(j)
+ out[i] = src[j]*(1-t) + src[j+1]*t
+ }
+ return out
+}
diff --git a/cmd/lk/assistant_session_test.go b/cmd/lk/assistant_session_test.go
new file mode 100644
index 000000000..869ca1fd1
--- /dev/null
+++ b/cmd/lk/assistant_session_test.go
@@ -0,0 +1,211 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/livekit/protocol/livekit/agent"
+ "github.com/stretchr/testify/require"
+)
+
+// recordedOverlay collects the transcript messages a session sends.
+type recordedOverlay struct {
+ turns []string // turn ids in the order they first appear
+ text map[string]string // "turn/segment" → latest text
+ seen map[string]struct{} // turn ids seen
+}
+
+func (r *recordedOverlay) Send(msg any) error {
+ t, ok := msg.(overlayTranscript)
+ if !ok {
+ return nil
+ }
+ if _, ok := r.seen[t.Turn]; !ok {
+ r.seen[t.Turn] = struct{}{}
+ r.turns = append(r.turns, t.Turn)
+ }
+ r.text[fmt.Sprintf("%s/%d", t.Turn, t.Segment)] = t.Text
+ return nil
+}
+
+func newRecordedSession() (*assistantSession, *recordedOverlay) {
+ rec := &recordedOverlay{text: map[string]string{}, seen: map[string]struct{}{}}
+ s := newAssistantSession(nil, nil)
+ s.out = rec
+ return s, rec
+}
+
+func TestAssistantTurnsConversation(t *testing.T) {
+ s, rec := newRecordedSession()
+
+ // The agent greets.
+ s.beginAgentSpeech()
+ s.agentText("Hi.")
+ s.agentFlush()
+ s.agentMessage(` Hi.`)
+
+ // The user asks something.
+ s.userTranscript("How do I", false)
+ s.userTranscript("How do I deploy?", true)
+ s.userCommitted("How do I deploy?")
+
+ // The agent answers.
+ s.beginAgentSpeech()
+ s.agentText("Run lk agent create.")
+ s.agentFlush()
+ s.agentMessage("Run lk agent create.")
+
+ require.Equal(t, []string{"a1", "u2", "a3"}, rec.turns)
+ require.Equal(t, "Hi.", rec.text["a1/0"])
+ require.Equal(t, "How do I deploy?", rec.text["u2/0"])
+ require.Equal(t, "Run lk agent create.", rec.text["a3/0"])
+}
+
+func TestAssistantTurnsInterruption(t *testing.T) {
+ s, rec := newRecordedSession()
+
+ // The agent starts speaking.
+ s.beginAgentSpeech()
+ s.agentText("You can")
+ s.agentText(" use the")
+
+ // The user interrupts while words are still arriving.
+ s.userTranscript("Wait", false)
+ s.agentText(" starter")
+ s.agentFlush()
+ s.agentMessage("You can use the starter")
+ s.userTranscript("Wait, in Node?", true)
+ s.userCommitted("Wait, in Node?")
+
+ // The agent replies to the interruption.
+ s.beginAgentSpeech()
+ s.agentText("Yes, in Node too.")
+ s.agentFlush()
+ s.agentMessage("Yes, in Node too.")
+
+ // No turn is duplicated: the end of the interrupted speech stays in its
+ // turn, and the user's speech stays in one turn.
+ require.Equal(t, []string{"a1", "u2", "a3"}, rec.turns)
+ require.Equal(t, "You can use the starter", rec.text["a1/0"])
+ require.Equal(t, "Wait, in Node?", rec.text["u2/0"])
+ require.Equal(t, "Yes, in Node too.", rec.text["a3/0"])
+}
+
+func TestAssistantTurnsUnstreamedMessage(t *testing.T) {
+ s, rec := newRecordedSession()
+
+ // Text that wasn't streamed shows from the committed message, without
+ // expressive markup.
+ s.agentMessage(` Hello there.`)
+
+ require.Equal(t, []string{"a1"}, rec.turns)
+ require.Equal(t, "Hello there.", rec.text["a1/0"])
+}
+
+func TestAssistantTurnsPause(t *testing.T) {
+ s, rec := newRecordedSession()
+
+ // The user pauses mid-thought, and the agent commits each part before
+ // it replies.
+ s.userTranscript("I want to build something", true)
+ s.userCommitted("I want to build something")
+ s.userTranscript("for my", false)
+ s.userTranscript("for my support team.", true)
+ s.userCommitted("for my support team.")
+
+ s.beginAgentSpeech()
+ s.agentText("Great.")
+ s.agentFlush()
+ s.agentMessage("Great.")
+
+ require.Equal(t, []string{"u1", "a2"}, rec.turns)
+ require.Equal(t, "I want to build something for my support team.", rec.text["u1/0"])
+}
+
+func TestAssistantTurnsEarlyCommit(t *testing.T) {
+ s, rec := newRecordedSession()
+
+ // The turn detector ends the user's turn mid-sentence, and the agent
+ // starts looking things up before the rest arrives.
+ s.userTranscript("I want a voice agent for", true)
+ s.userCommitted("I want a voice agent for")
+ s.toolCall(&agent.FunctionCall{CallId: "c1", Name: "get_docs_overview", Arguments: "{}"})
+ s.userTranscript("restaurant reservations.", true)
+ s.userCommitted("restaurant reservations.")
+
+ s.beginAgentSpeech()
+ s.agentText("Sounds good.")
+ s.agentFlush()
+ s.agentMessage("Sounds good.")
+
+ // The next exchange starts new turns.
+ s.userTranscript("What about Node?", true)
+ s.userCommitted("What about Node?")
+ s.beginAgentSpeech()
+ s.agentText("Node works too.")
+ s.agentFlush()
+ s.agentMessage("Node works too.")
+
+ require.Equal(t, []string{"u1", "a2", "u3", "a4"}, rec.turns)
+ require.Equal(t, "I want a voice agent for restaurant reservations.", rec.text["u1/0"])
+ require.Equal(t, "What about Node?", rec.text["u3/0"])
+ require.Equal(t, "Node works too.", rec.text["a4/0"])
+}
+
+func TestIsAuthError(t *testing.T) {
+ require.True(t, isAuthError("type='llm_error' timestamp=1760024012.5 label='livekit.agents.inference.llm.LLM' error=APIStatusError('unauthorized', status_code=401, request_id=None) recoverable=False"))
+ require.True(t, isAuthError("Invalid API key"))
+ // A timestamp that happens to contain 401 isn't an auth error.
+ require.False(t, isAuthError("type='tts_error' timestamp=1791565401.871234 label='livekit.agents.inference.tts.TTS' error=APIConnectionError('connection failed') recoverable=True"))
+}
+
+func TestDocsPageURL(t *testing.T) {
+ require.Equal(t, "https://docs.livekit.io/deploy/agents", docsPageURL("https://docs.livekit.io/deploy/agents.md"))
+ require.Equal(t, "https://docs.livekit.io/deploy/agents/", docsPageURL("https://docs.livekit.io/deploy/agents/"))
+}
+
+func TestDocsPagesRead(t *testing.T) {
+ out := `{"type":"text","text":"## /agents/build.md\n\nLiveKit docs › Build Agents\n\n---\n\n# Building voice agents\n\n> Overview"}`
+ pages := docsPagesRead(`{"paths": ["/agents/build.md", "https://github.com/livekit/agents/blob/main/README.md"]}`, out)
+ require.Equal(t, []docsPage{{title: "Building voice agents", url: "https://docs.livekit.io/agents/build"}}, pages)
+}
+
+func TestPickCard(t *testing.T) {
+ s := &assistantSession{snippets: map[string]string{}}
+ s.addSnippet("c1", "code", "Terminal", "lk agent simulate text -n 10")
+ s.addSnippet("l1", "link", "Debugging and testing overview", "https://docs.livekit.io/testing/overview")
+ s.addSnippet("l2", "link", "Agent Simulations", "https://docs.livekit.io/testing/simulations")
+ s.addSnippet("p1", "prompt", "Prompt for your coding agent", "Add tools.")
+
+ pick := func(kind, title string) string {
+ ref, ok := s.pickCard(kind, title)
+ if !ok {
+ return ""
+ }
+ return ref.id
+ }
+ require.Equal(t, "p1", pick("latest", ""))
+ require.Equal(t, "l2", pick("link", ""))
+ require.Equal(t, "l1", pick("link", "the debugging overview link"))
+ require.Equal(t, "l1", pick("link", "Agent Simulations, Debugging and testing overview")) // the most shared words
+ require.Equal(t, "l2", pick("link", "Simulations"))
+ require.Equal(t, "c1", pick("code", "Agent Console"))
+ require.Equal(t, "p1", pick("prompt", ""))
+ empty := &assistantSession{snippets: map[string]string{}}
+ _, ok := empty.pickCard("link", "")
+ require.False(t, ok)
+}
diff --git a/cmd/lk/assistant_skills.go b/cmd/lk/assistant_skills.go
new file mode 100644
index 000000000..9c0500f5a
--- /dev/null
+++ b/cmd/lk/assistant_skills.go
@@ -0,0 +1,84 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "encoding/json"
+ "os"
+ "slices"
+
+ "github.com/livekit/livekit-cli/v2/pkg/skills"
+)
+
+// codingAgentSetup is what LiveKit has already set up for the user's coding
+// agents, so the assistant doesn't offer to install it again. It reads only
+// local files, so it doesn't check whether the skills are current.
+type codingAgentSetup struct {
+ Detected []string `json:"coding_agents_detected"`
+ Project skillsScope `json:"project"` // in the user's current directory
+ Global skillsScope `json:"global"` // for all their projects
+ // Ready is true when LiveKit skills and the Docs MCP server are both set
+ // up for at least one coding agent, in either scope.
+ Ready bool `json:"ready"`
+}
+
+type skillsScope struct {
+ Skills []string `json:"livekit_skills"`
+ MCP []string `json:"docs_mcp_configured_for"` // coding agent IDs
+}
+
+func checkCodingAgentSetup(dir string) codingAgentSetup {
+ home, _ := os.UserHomeDir()
+ env := &skills.Env{Home: home, Root: dir}
+ var setup codingAgentSetup
+ setup.Detected = agentIDs(skills.DetectAgents(env))
+ if setup.Detected == nil {
+ setup.Detected = []string{}
+ }
+ setup.Project = checkSkillsScope(env, skills.ScopeProject)
+ setup.Global = checkSkillsScope(env, skills.ScopeGlobal)
+ hasSkills := len(setup.Project.Skills) > 0 || len(setup.Global.Skills) > 0
+ hasMCP := len(setup.Project.MCP) > 0 || len(setup.Global.MCP) > 0
+ setup.Ready = hasSkills && hasMCP
+ return setup
+}
+
+func checkSkillsScope(env *skills.Env, scope skills.Scope) skillsScope {
+ st := skillsScope{Skills: []string{}, MCP: []string{}}
+ if in, err := skills.NewInstaller(env, scope, nil); err == nil {
+ dirs := skills.GroupSkillsDirs(env, scope, skills.Agents)
+ names, _ := in.InstalledLiveKitSkills(dirs)
+ // The lock can list skills whose files are gone; count only copies
+ // that are there.
+ copies, _ := in.Inspect(names, dirs)
+ for _, c := range copies {
+ if c.State != skills.StateMissing && !slices.Contains(st.Skills, c.Skill) {
+ st.Skills = append(st.Skills, c.Skill)
+ }
+ }
+ slices.Sort(st.Skills)
+ }
+ for _, a := range skills.Agents {
+ if skills.CheckMCP(env, scope, a).State == skills.MCPConfigured {
+ st.MCP = append(st.MCP, a.ID)
+ }
+ }
+ return st
+}
+
+func (c codingAgentSetup) JSON() string {
+ b, _ := json.Marshal(c)
+ return string(b)
+}
diff --git a/cmd/lk/assistant_tools.go b/cmd/lk/assistant_tools.go
new file mode 100644
index 000000000..4890caabd
--- /dev/null
+++ b/cmd/lk/assistant_tools.go
@@ -0,0 +1,528 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "slices"
+ "strings"
+
+ "github.com/livekit/livekit-cli/v2/pkg/agentfs"
+ "github.com/livekit/livekit-cli/v2/pkg/assistant"
+ "github.com/livekit/livekit-cli/v2/pkg/bootstrap"
+ "github.com/livekit/livekit-cli/v2/pkg/config"
+ "github.com/livekit/livekit-cli/v2/pkg/util"
+)
+
+// The assistant agent runs lk commands through the agent channel. lk runs
+// each one as a subprocess of itself, in the user's project directory, with
+// the assistant's credentials.
+//
+// Commands that change something need the user's confirmation, by voice or
+// on screen. The tool that proposes one shows a confirmation card and
+// returns right away with its ID, so the agent can ask the user. If the
+// user agrees by voice, the agent calls confirm_action, which runs the
+// command and returns its result. If they click the card instead, lk runs
+// the command and tells the agent the result.
+
+// toolState tracks lk tool requests. It's only used from the session's run
+// loop; background work reports back through results.
+type toolState struct {
+ ctx context.Context
+ dir string // the user's project directory
+ creds []string // LIVEKIT_* environment for subprocesses
+ results chan toolResult
+ pending map[string]pendingTool // confirmation id → command waiting for it
+ choices map[string][]overlayConfirmOption // choice card id → its options, until answered
+ confirmN int
+ // lastProject is the project created this session, for try_project.
+ lastProject string
+}
+
+// pendingTool is a command waiting for the user to confirm it.
+type pendingTool struct {
+ args []string
+ command string // as shown to the user
+ project string // the project the command creates, if any
+ // start, if set, runs instead of the lk command.
+ start func(requestID, confirmID string)
+}
+
+// toolResult is the outcome of background work, handled in the run loop.
+type toolResult struct {
+ requestID string // the agent's request to reply to, if any
+ confirmID string // the confirmation card to update, if any
+ command string
+ project string // the project the command created, if any
+ output string
+ err error
+ // then, if set, handles the result instead of the default reply.
+ then func(toolResult)
+}
+
+// toolOutputLimit bounds how much command output goes back to the LLM.
+const toolOutputLimit = 6000
+
+// Confirmation options in the overlay.
+const (
+ confirmRun = "run"
+ confirmCancel = "cancel"
+)
+
+type (
+ overlayConfirm struct {
+ Type string `json:"type"`
+ Turn string `json:"turn"`
+ ID string `json:"id"`
+ Heading string `json:"heading"`
+ Command string `json:"command"` // as the user would type it
+ Detail string `json:"detail"` // where it runs and what it changes
+ Options []overlayConfirmOption `json:"options"`
+ }
+ overlayConfirmOption struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ }
+ overlayConfirmStatus struct {
+ Type string `json:"type"`
+ ID string `json:"id"`
+ Status string `json:"status"` // running, done, failed, or cancelled
+ Text string `json:"text,omitempty"`
+ }
+)
+
+var projectNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,62}$`)
+
+// runTool handles an lk tool request from the assistant agent.
+func (s *assistantSession) runTool(requestID, name string, args map[string]any) {
+ str := func(key string) string {
+ v, _ := args[key].(string)
+ return strings.TrimSpace(v)
+ }
+ flag := func(key string) bool {
+ v, _ := args[key].(bool)
+ return v
+ }
+ switch name {
+ case "get_project_info":
+ s.replyTool(requestID, s.projectInfo(), nil)
+
+ case "list_cloud_agents":
+ s.startCommand(requestID, "", []string{"agent", "list", "--json"})
+
+ case "list_templates":
+ s.inBackground(func(ctx context.Context) toolResult {
+ templates, err := bootstrap.FetchTemplates(ctx)
+ if err != nil {
+ return toolResult{requestID: requestID, err: err}
+ }
+ return toolResult{requestID: requestID, output: describeTemplates(templates)}
+ })
+
+ case "create_project":
+ s.proposeCreateProject(requestID, str("template"), str("name"), flag("in_current_directory"), flag("confirmed"))
+
+ case "set_up_coding_agent":
+ if setup := checkCodingAgentSetup(s.tools.dir); setup.Ready && !flag("reinstall") {
+ s.replyTool(requestID, "Nothing was installed: LiveKit's skills and the Docs MCP server are already set up. "+
+ "Tell the user, and that \"lk skills update\" gets the latest skills. If they still want to reinstall, "+
+ "call set_up_coding_agent again with reinstall=true. Current setup: "+setup.JSON(), nil)
+ return
+ }
+ detail := "Adds LiveKit's skills and the Docs MCP server to the coding agents it finds, for this folder only."
+ cmdArgs := []string{"skills", "install", "--json", "-y"}
+ if flag("global") {
+ detail = "Adds LiveKit's skills and the Docs MCP server to the coding agents it finds, for all your projects."
+ cmdArgs = append(cmdArgs, "--global")
+ }
+ s.proposeCommand(requestID, detail, "Install", cmdArgs, "", flag("confirmed"))
+
+ case "confirm_action":
+ s.confirmByVoice(requestID, str("confirmation_id"), flag("approved"))
+
+ case "show_choice":
+ s.showChoice(requestID, str("question"), args["options"])
+
+ case "choice_made":
+ s.choiceMade(str("id"), str("option"), false)
+ s.replyTool(requestID, "Marked.", nil)
+
+ case "copy_to_clipboard":
+ s.copyByVoice(requestID, str("kind"), str("title"))
+
+ case "try_project":
+ s.proposeTryProject(requestID, flag("confirmed"))
+
+ default:
+ s.replyTool(requestID, "", fmt.Errorf("unknown tool %q", name))
+ }
+}
+
+// proposeCreateProject proposes creating a project from a template: agent
+// templates with lk agent init, others (frontends and apps) with lk app
+// create.
+func (s *assistantSession) proposeCreateProject(requestID, template, name string, here, confirmed bool) {
+ dir := s.tools.dir
+ if here {
+ if empty, err := util.IsEmptyDir(dir); err != nil || !empty {
+ s.replyTool(requestID, "", errors.New("the current directory isn't empty, so the project can't be created in it; create it in a new folder instead"))
+ return
+ }
+ } else if !projectNameRe.MatchString(name) {
+ s.replyTool(requestID, "", fmt.Errorf("%q isn't a valid project name: use lowercase letters, numbers, and dashes", name))
+ return
+ }
+
+ s.inBackground(func(ctx context.Context) toolResult {
+ templates, err := bootstrap.FetchTemplates(ctx)
+ if err != nil {
+ return toolResult{requestID: requestID, err: err}
+ }
+ i := slices.IndexFunc(templates, func(t bootstrap.Template) bool {
+ return t.Name == template && !t.IsHidden && !t.IsSandbox
+ })
+ if i < 0 {
+ return toolResult{requestID: requestID, err: fmt.Errorf("there's no template named %q; call list_templates for the options", template)}
+ }
+ tpl := templates[i]
+ return toolResult{requestID: requestID, then: func(toolResult) {
+ target, where, project := name, "in a new folder, "+tildePath(filepath.Join(dir, name)), filepath.Join(dir, name)
+ if here {
+ target, where, project = ".", fmt.Sprintf("in this folder, named after it (%s)", filepath.Base(dir)), dir
+ }
+ kind, cmdArgs := "an app", []string{"app", "create"}
+ if isAgentTemplate(tpl) {
+ kind, cmdArgs = "an agent project", []string{"agent", "init"}
+ }
+ cmdArgs = append(cmdArgs, target, "--template", tpl.Name, "--install", "-y")
+ if !isAgentTemplate(tpl) {
+ project = ""
+ }
+ s.proposeCommand(requestID,
+ fmt.Sprintf("Creates %s from the %s template %s.", kind, tpl.Name, where),
+ "Create", cmdArgs, project, confirmed)
+ }}
+ })
+}
+
+// isAgentTemplate reports whether a template is an agent (Python or Node.js)
+// rather than a frontend or app.
+func isAgentTemplate(t bootstrap.Template) bool {
+ return slices.Contains(t.Tags, "agents") &&
+ (slices.Contains(t.Tags, "python") || slices.Contains(t.Tags, "node.js"))
+}
+
+// describeTemplates lists the templates a user can create, for the LLM.
+func describeTemplates(templates []bootstrap.Template) string {
+ var agents, apps []string
+ for _, t := range templates {
+ if t.IsHidden || t.IsSandbox {
+ continue
+ }
+ line := fmt.Sprintf("- %s: %s", t.Name, strings.TrimSpace(t.Desc))
+ if isAgentTemplate(t) {
+ agents = append(agents, line)
+ } else {
+ apps = append(apps, line)
+ }
+ }
+ return "Agent templates:\n" + strings.Join(agents, "\n") +
+ "\n\nFrontend and app templates:\n" + strings.Join(apps, "\n")
+}
+
+// proposeCommand shows a confirmation card for a command and replies to the
+// agent right away with the confirmation ID, which it returns.
+func (s *assistantSession) proposeCommand(requestID, detail, runLabel string, args []string, project string, confirmed bool) {
+ t := &s.tools
+ t.confirmN++
+ id := fmt.Sprintf("confirm%d", t.confirmN)
+ command := displayCommand(args)
+ t.pending[id] = pendingTool{args: args, command: command, project: project}
+ s.agentTurnForReply()
+ s.out.Send(overlayConfirm{
+ Type: "confirm", Turn: s.agentTurn, ID: id,
+ Heading: "Run in " + tildePath(t.dir),
+ Command: command,
+ Detail: detail,
+ Options: []overlayConfirmOption{{ID: confirmRun, Name: runLabel}, {ID: confirmCancel, Name: "Cancel"}},
+ })
+ if confirmed {
+ // The user already agreed by voice: run it, and reply when it's done.
+ s.confirmByVoice(requestID, id, true)
+ return
+ }
+ s.replyTool(requestID, fmt.Sprintf(
+ "Waiting for the user to confirm. Confirmation ID: %s. Command: %s. %s "+
+ "The command is on screen, so don't say it. Ask the user in a few words whether to go ahead. They can answer by voice or "+
+ "on screen. If they agree by voice, call confirm_action with this ID and approved=true; "+
+ "if they decline, approved=false. If they answer on screen, you'll be told the result.",
+ id, command, detail), nil)
+}
+
+// showChoice shows a card that asks the user to pick one option, by voice or
+// on screen. The agent's code decides what each option does; lk only reports
+// the choice. It replies with the card's id.
+func (s *assistantSession) showChoice(requestID, question string, rawOptions any) {
+ var options []overlayConfirmOption
+ if list, ok := rawOptions.([]any); ok {
+ for _, o := range list {
+ m, _ := o.(map[string]any)
+ id, _ := m["id"].(string)
+ name, _ := m["name"].(string)
+ if id != "" && name != "" {
+ options = append(options, overlayConfirmOption{ID: id, Name: name})
+ }
+ }
+ }
+ if question == "" || len(options) == 0 {
+ s.replyTool(requestID, "", errors.New("a choice needs a question and options with an id and a name"))
+ return
+ }
+ t := &s.tools
+ t.confirmN++
+ id := fmt.Sprintf("choice%d", t.confirmN)
+ t.choices[id] = options
+ s.agentTurnForReply()
+ s.out.Send(overlayConfirm{Type: "confirm", Turn: s.agentTurn, ID: id, Heading: question, Options: options})
+ s.replyTool(requestID, id, nil)
+}
+
+// choiceMade marks a choice card answered. A click on the card (onScreen)
+// also tells the agent, which otherwise only hears choices made by voice.
+func (s *assistantSession) choiceMade(id, option string, onScreen bool) {
+ options, ok := s.tools.choices[id]
+ if !ok {
+ return
+ }
+ i := slices.IndexFunc(options, func(o overlayConfirmOption) bool { return o.ID == option })
+ if i < 0 {
+ return
+ }
+ delete(s.tools.choices, id)
+ s.out.Send(map[string]string{"type": "confirm_done", "id": id, "option": option})
+ s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "done", Text: options[i].Name})
+ if onScreen {
+ s.channel.send(map[string]string{"type": "choice", "id": id, "option": option})
+ }
+}
+
+// confirmByVoice handles confirm_action: the user answered by voice.
+func (s *assistantSession) confirmByVoice(requestID, id string, approved bool) {
+ p, ok := s.tools.pending[id]
+ if !ok {
+ s.replyTool(requestID, "", fmt.Errorf("there's no pending confirmation %q; it may have been answered on screen already", id))
+ return
+ }
+ delete(s.tools.pending, id)
+ option := confirmCancel
+ if approved {
+ option = confirmRun
+ }
+ s.out.Send(map[string]string{"type": "confirm_done", "id": id, "option": option})
+ if !approved {
+ s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "cancelled"})
+ s.replyTool(requestID, "Cancelled. Nothing was changed.", nil)
+ return
+ }
+ s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "running"})
+ s.runConfirmed(requestID, id, p)
+}
+
+// confirmTool handles the user's answer on screen.
+func (s *assistantSession) confirmTool(id, option string) {
+ if _, ok := s.tools.choices[id]; ok {
+ s.choiceMade(id, option, true)
+ return
+ }
+ p, ok := s.tools.pending[id]
+ if !ok {
+ return
+ }
+ delete(s.tools.pending, id)
+ s.out.Send(map[string]string{"type": "confirm_done", "id": id, "option": option})
+ if option != confirmRun {
+ s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "cancelled"})
+ s.tellUser(fmt.Sprintf("The user cancelled %s on screen. Nothing was changed. Acknowledge it in a few words.", p.command))
+ return
+ }
+ s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "running"})
+ // No agent request waits for this result; toolFinished tells the agent.
+ s.runConfirmed("", id, p)
+}
+
+func (s *assistantSession) runConfirmed(requestID, confirmID string, p pendingTool) {
+ if p.start != nil {
+ p.start(requestID, confirmID)
+ return
+ }
+ t := &s.tools
+ s.inBackground(func(ctx context.Context) toolResult {
+ output, err := runLK(ctx, t.dir, t.creds, p.args)
+ return toolResult{requestID: requestID, confirmID: confirmID, command: p.command, project: p.project, output: output, err: err}
+ })
+}
+
+// startCommand runs an lk command in the background and replies to the
+// agent's request with its output.
+func (s *assistantSession) startCommand(requestID, confirmID string, args []string) {
+ s.runConfirmed(requestID, confirmID, pendingTool{args: args, command: displayCommand(args)})
+}
+
+// inBackground runs fn off the run loop and hands its result to toolFinished.
+func (s *assistantSession) inBackground(fn func(ctx context.Context) toolResult) {
+ t := &s.tools
+ go func() {
+ r := fn(t.ctx)
+ select {
+ case t.results <- r:
+ case <-t.ctx.Done():
+ }
+ }()
+}
+
+func (s *assistantSession) toolFinished(r toolResult) {
+ if r.then != nil {
+ r.then(r)
+ return
+ }
+ if r.err == nil && r.project != "" {
+ s.tools.lastProject = r.project
+ }
+ if r.confirmID != "" {
+ status := overlayConfirmStatus{Type: "confirm_status", ID: r.confirmID, Status: "done"}
+ if r.err != nil {
+ status.Status = "failed"
+ status.Text = lastLine(r.err.Error())
+ }
+ s.out.Send(status)
+ }
+ if r.requestID != "" {
+ s.replyTool(r.requestID, r.output, r.err)
+ return
+ }
+ // Confirmed on screen: no request waits, so tell the agent what happened.
+ if r.err != nil {
+ s.tellUser(fmt.Sprintf("The user confirmed %s on screen, but it failed: %s. Tell them briefly.", r.command, truncate(r.err.Error(), 1500)))
+ return
+ }
+ s.tellUser(fmt.Sprintf("The user confirmed %s on screen, and it finished. Its output:\n%s\nTell them briefly what happened and the next step.", r.command, truncate(r.output, 1500)))
+}
+
+func (s *assistantSession) replyTool(requestID, output string, err error) {
+ msg := map[string]any{"type": "reply", "id": requestID, "ok": err == nil, "output": output}
+ if err != nil {
+ msg["error"] = err.Error()
+ }
+ s.channel.send(msg)
+}
+
+// tellUser asks the assistant agent to tell the user something.
+func (s *assistantSession) tellUser(instructions string) {
+ s.channel.send(map[string]string{"type": "say", "instructions": instructions})
+}
+
+// displayCommand shows an lk command as the user would type it, without the
+// flags lk adds to run it non-interactively.
+func displayCommand(args []string) string {
+ shown := []string{"lk"}
+ for _, a := range args {
+ if a != "-y" && a != "--json" {
+ shown = append(shown, a)
+ }
+ }
+ return strings.Join(shown, " ")
+}
+
+// tildePath shortens a path in the user's home directory to start with ~.
+func tildePath(path string) string {
+ home, err := os.UserHomeDir()
+ if err != nil {
+ return path
+ }
+ if rel, err := filepath.Rel(home, path); err == nil && !strings.HasPrefix(rel, "..") {
+ return filepath.Join("~", rel)
+ }
+ return path
+}
+
+// lastLine returns the last non-empty line of s, which for a failed command
+// is usually its error.
+func lastLine(s string) string {
+ lines := strings.Split(strings.TrimSpace(s), "\n")
+ return strings.TrimSpace(lines[len(lines)-1])
+}
+
+func truncate(s string, n int) string {
+ if len(s) <= n {
+ return s
+ }
+ return s[:n] + "…"
+}
+
+// lkExecutable finds the lk binary that tools run. It's a variable for tests.
+var lkExecutable = os.Executable
+
+// ansiRe matches terminal escape codes: colors (CSI) and hyperlinks (OSC 8).
+var ansiRe = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)`)
+
+// runLK runs this lk binary with args in dir and returns its output.
+func runLK(ctx context.Context, dir string, creds, args []string) (string, error) {
+ exe, err := lkExecutable()
+ if err != nil {
+ return "", err
+ }
+ cmd := exec.CommandContext(ctx, exe, args...)
+ cmd.Dir = dir
+ cmd.Env = append(os.Environ(), creds...)
+ out, err := cmd.CombinedOutput()
+ text := truncate(strings.TrimSpace(ansiRe.ReplaceAllString(string(out), "")), toolOutputLimit)
+ if err != nil {
+ return "", fmt.Errorf("%s failed: %w\n%s", displayCommand(args), err, text)
+ }
+ return text, nil
+}
+
+// projectInfo describes the user's LiveKit Cloud project and their current
+// directory, as JSON for the LLM.
+func (s *assistantSession) projectInfo() string {
+ info := map[string]any{
+ "directory": s.tools.dir,
+ "folder": assistant.DescribeFolder(s.tools.dir),
+ "coding_agents": checkCodingAgentSetup(s.tools.dir),
+ }
+ if s.cmd != nil {
+ if rp, err := resolveProject(s.cmd, loadParams{requireURL: true}); err == nil && rp.project != nil {
+ info["livekit_project"] = map[string]string{"name": rp.project.Name, "url": rp.project.URL}
+ }
+ }
+ if root, projectType, err := agentfs.DetectProjectRoot(s.tools.dir); err == nil {
+ local := map[string]any{"root": root, "type": string(projectType)}
+ if toml, ok, err := config.LoadTOMLFile(root, config.LiveKitTOMLFile); err == nil && ok && toml.HasAgent() {
+ local["deployed_agent_id"] = toml.Agent.ID
+ }
+ info["agent_project"] = local
+ } else {
+ info["agent_project"] = nil
+ }
+ b, _ := json.MarshalIndent(info, "", " ")
+ return string(b)
+}
diff --git a/cmd/lk/assistant_tools_test.go b/cmd/lk/assistant_tools_test.go
new file mode 100644
index 000000000..89cb93719
--- /dev/null
+++ b/cmd/lk/assistant_tools_test.go
@@ -0,0 +1,33 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestDisplayCommand(t *testing.T) {
+ require.Equal(t, "lk agent init . --lang python --install",
+ displayCommand([]string{"agent", "init", ".", "--lang", "python", "--install", "-y"}))
+ require.Equal(t, "lk skills install --global",
+ displayCommand([]string{"skills", "install", "--json", "-y", "--global"}))
+}
+
+func TestStripTerminalEscapes(t *testing.T) {
+ in := "Visit \x1b]8;;https://cloud.livekit.io\x1b\\\x1b[36;4mhttps://cloud.livekit.io\x1b[0m\x1b]8;;\x1b\\ now"
+ require.Equal(t, "Visit https://cloud.livekit.io now", ansiRe.ReplaceAllString(in, ""))
+}
diff --git a/cmd/lk/assistant_xray.go b/cmd/lk/assistant_xray.go
new file mode 100644
index 000000000..00077484b
--- /dev/null
+++ b/cmd/lk/assistant_xray.go
@@ -0,0 +1,156 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+
+ "github.com/livekit/protocol/livekit/agent"
+
+ "github.com/livekit/livekit-cli/v2/pkg/assistant"
+)
+
+// X-ray mode shows the assistant's own pipeline at work: the models it uses,
+// per-turn timings, the turn detector's live end-of-turn estimate, and usage.
+// The data comes from the session events console mode already sends.
+
+type (
+ overlayPipeline struct {
+ Type string `json:"type"`
+ STT string `json:"stt"`
+ LLM string `json:"llm"`
+ TTS string `json:"tts"`
+ TurnDetection string `json:"turn_detection"`
+ }
+ overlayEOT struct {
+ Type string `json:"type"`
+ Probability float32 `json:"probability"`
+ Threshold float32 `json:"threshold"`
+ }
+ // overlayMetrics carries one message's timings, in seconds. Fields the
+ // agent didn't report are left out.
+ overlayMetrics struct {
+ Type string `json:"type"`
+ Role string `json:"role"` // user or assistant
+ EndOfTurnDelay *float64 `json:"end_of_turn_delay,omitempty"`
+ TranscriptionDelay *float64 `json:"transcription_delay,omitempty"`
+ LLMTTFT *float64 `json:"llm_ttft,omitempty"`
+ TTSTTFB *float64 `json:"tts_ttfb,omitempty"`
+ E2ELatency *float64 `json:"e2e_latency,omitempty"`
+ }
+ overlayUsage struct {
+ Type string `json:"type"`
+ Models []overlayModelUse `json:"models"`
+ }
+ overlayModelUse struct {
+ Kind string `json:"kind"` // stt, llm, or tts
+ Model string `json:"model"`
+ InputTokens int32 `json:"input_tokens,omitempty"`
+ OutputTokens int32 `json:"output_tokens,omitempty"`
+ Characters int32 `json:"characters,omitempty"`
+ AudioDuration float64 `json:"audio_duration,omitempty"`
+ }
+)
+
+func metricsMessage(role string, m *agent.MetricsReport) overlayMetrics {
+ return overlayMetrics{
+ Type: "metrics",
+ Role: role,
+ EndOfTurnDelay: m.EndOfTurnDelay,
+ TranscriptionDelay: m.TranscriptionDelay,
+ LLMTTFT: m.LlmNodeTtft,
+ TTSTTFB: m.TtsNodeTtfb,
+ E2ELatency: m.E2ELatency,
+ }
+}
+
+func usageMessage(u *agent.AgentSessionUsage) overlayUsage {
+ msg := overlayUsage{Type: "usage"}
+ for _, mu := range u.GetModelUsage() {
+ switch {
+ case mu.GetLlm() != nil:
+ l := mu.GetLlm()
+ msg.Models = append(msg.Models, overlayModelUse{Kind: "llm", Model: l.GetModel(), InputTokens: l.GetInputTokens(), OutputTokens: l.GetOutputTokens()})
+ case mu.GetTts() != nil:
+ t := mu.GetTts()
+ msg.Models = append(msg.Models, overlayModelUse{Kind: "tts", Model: t.GetModel(), Characters: t.GetCharactersCount(), AudioDuration: t.GetAudioDuration()})
+ case mu.GetStt() != nil:
+ st := mu.GetStt()
+ msg.Models = append(msg.Models, overlayModelUse{Kind: "stt", Model: st.GetModel(), AudioDuration: st.GetAudioDuration()})
+ }
+ }
+ return msg
+}
+
+// showAgentCode shows a code card the agent sent about itself, such as its
+// own AgentSession setup.
+func (s *assistantSession) showAgentCode(m agentMessage) {
+ if m.Code == "" {
+ return
+ }
+ s.agentTurnForReply()
+ s.snippetN++
+ id := fmt.Sprintf("self%d", s.snippetN)
+ s.addSnippet(id, "code", m.Title, m.Code)
+ s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, m.Title, normalizeLang(m.Language), m.Code, false})
+}
+
+// Startup errors show as a card in the overlay with what went wrong and the
+// command that fixes it, since the terminal is hidden behind the overlay.
+
+type overlayError struct {
+ Type string `json:"type"`
+ Title string `json:"title"`
+ Detail string `json:"detail"`
+ Fix string `json:"fix,omitempty"` // a command to copy
+}
+
+// errNoCredentials marks an error loading the LiveKit Cloud credentials.
+type errNoCredentials struct{ err error }
+
+func (e errNoCredentials) Error() string { return e.err.Error() }
+func (e errNoCredentials) Unwrap() error { return e.err }
+
+func startupError(err error) overlayError {
+ var creds errNoCredentials
+ switch {
+ case errors.Is(err, assistant.ErrNoUV):
+ return overlayError{Type: "error",
+ Title: "Install uv to start the assistant",
+ Detail: "The assistant is a Python voice agent, and lk runs it with uv, a Python package manager. Install uv, then run lk assistant again.",
+ Fix: "curl -LsSf https://astral.sh/uv/install.sh | sh"}
+ case errors.As(err, &creds):
+ return overlayError{Type: "error",
+ Title: "Connect lk to LiveKit Cloud",
+ Detail: "The assistant uses LiveKit Inference through your LiveKit Cloud project. " + firstLine(creds.Error()),
+ Fix: "lk cloud auth"}
+ default:
+ return overlayError{Type: "error",
+ Title: "Couldn't start the assistant",
+ Detail: truncate(strings.TrimSpace(err.Error()), 600)}
+ }
+}
+
+func (s *assistantSession) showError(e overlayError) {
+ s.out.Send(e)
+ s.out.Send(overlayNotice{Type: "notice", Text: e.Title, Sticky: true})
+}
+
+func firstLine(s string) string {
+ line, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
+ return line
+}
diff --git a/cmd/lk/main.go b/cmd/lk/main.go
index 1b73e79bd..cf916cb7b 100644
--- a/cmd/lk/main.go
+++ b/cmd/lk/main.go
@@ -95,6 +95,7 @@ Docs: https://docs.livekit.io/intro/basics/cli/`,
app.Commands = append(app.Commands, CloudCommands...)
app.Commands = append(app.Commands, DocsCommands...)
app.Commands = append(app.Commands, SkillsCommands...)
+ app.Commands = append(app.Commands, AssistantCommands...)
app.Commands = append(app.Commands, ProjectCommands...)
app.Commands = append(app.Commands, WorkspaceCommands...)
app.Commands = append(app.Commands, UserCommands...)
diff --git a/cmd/lk/simulate_subprocess.go b/cmd/lk/simulate_subprocess.go
index b8d74ba8d..af9a82098 100644
--- a/cmd/lk/simulate_subprocess.go
+++ b/cmd/lk/simulate_subprocess.go
@@ -193,9 +193,11 @@ func findEntrypoint(dir, explicit string, projectType agentfs.ProjectType) (stri
}
}
- // Then cwd-relative paths (e.g. running from examples/drive-thru/)
+ // Then cwd-relative paths (e.g. running from examples/drive-thru/), only
+ // when cwd is inside the project: lk assistant runs projects in folders
+ // below cwd, and "../agent.py" would be another project's agent.
cwd, _ := os.Getwd()
- if rel, err := filepath.Rel(dir, cwd); err == nil && rel != "." {
+ if rel, err := filepath.Rel(dir, cwd); err == nil && rel != "." && filepath.IsLocal(rel) {
for _, def := range append(append([]string{}, rootCandidates...), srcCandidates...) {
candidate := filepath.Join(rel, def)
if probe(candidate) {
diff --git a/cmd/lk/simulate_subprocess_test.go b/cmd/lk/simulate_subprocess_test.go
new file mode 100644
index 000000000..3e4e53f99
--- /dev/null
+++ b/cmd/lk/simulate_subprocess_test.go
@@ -0,0 +1,58 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/livekit/livekit-cli/v2/pkg/agentfs"
+)
+
+func writeEntrypoint(t *testing.T, path string) {
+ t.Helper()
+ require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
+ require.NoError(t, os.WriteFile(path, nil, 0o644))
+}
+
+func TestFindEntrypointFromSubdirectory(t *testing.T) {
+ // Run from a folder inside the project, its agent there is found.
+ root := t.TempDir()
+ writeEntrypoint(t, filepath.Join(root, "examples", "drive-thru", "agent.py"))
+ t.Chdir(filepath.Join(root, "examples", "drive-thru"))
+
+ entry, err := findEntrypoint(root, "", agentfs.ProjectTypePythonUV)
+ require.NoError(t, err)
+ require.Equal(t, filepath.Join("examples", "drive-thru", "agent.py"), entry)
+}
+
+func TestFindEntrypointIgnoresParentProject(t *testing.T) {
+ // cwd is one agent project, and another project is in a folder below
+ // it. The child's own agent is found, not the parent's.
+ parent := t.TempDir()
+ writeEntrypoint(t, filepath.Join(parent, "src", "agent.py"))
+ child := filepath.Join(parent, "my-agent")
+ writeEntrypoint(t, filepath.Join(child, "src", "agent.py"))
+ t.Chdir(parent)
+
+ entry, err := findEntrypoint(child, "", agentfs.ProjectTypePythonUV)
+ require.NoError(t, err)
+ // findEntrypoint returns its root candidates with forward slashes on
+ // every platform.
+ require.Equal(t, "src/agent.py", filepath.ToSlash(entry))
+}
diff --git a/pkg/assistant/agent/agent.py b/pkg/assistant/agent/agent.py
new file mode 100644
index 000000000..d9f0c946c
--- /dev/null
+++ b/pkg/assistant/agent/agent.py
@@ -0,0 +1,1201 @@
+"""The LiveKit assistant: a voice agent that helps developers build with LiveKit.
+
+`lk assistant` runs this agent in console mode and shows the conversation in
+its overlay. Code the agent shows with `show_code` appears as a card in the
+overlay, with a button to copy it.
+"""
+
+import asyncio
+import json
+import logging
+import os
+import re
+import time
+import urllib.request
+from typing import Literal
+from collections.abc import Awaitable, Callable
+
+from livekit import agents
+from livekit.agents import (
+ Agent,
+ AgentServer,
+ AgentTask,
+ AgentSession,
+ RunContext,
+ TurnHandlingOptions,
+ APIConnectOptions,
+ function_tool,
+ inference,
+ llm,
+ mcp,
+ text_transforms,
+)
+from livekit.agents.beta.tools import EndCallTool
+from livekit.agents.voice import TranscriptSynchronizer, io
+from livekit.agents.voice.agent_session import SessionConnectOptions
+
+# Private module: the SDK's own stripper for expressive markup, the same one the
+# room transcription output uses. Check it when upgrading livekit-agents.
+from livekit.agents.tts._provider_format import TranscriptMarkupStripper
+
+DOCS_MCP_URL = "https://docs.livekit.io/mcp"
+
+STT_MODEL = "assemblyai/universal-3-6-pro"
+STT_KEYTERMS = [
+ "LiveKit",
+ "LK",
+ "LiveKit Cloud",
+ "LiveKit Inference",
+ "AgentSession",
+ "agent dispatch",
+ "turn detector",
+ "WebRTC",
+ "SIP",
+ "MCP",
+ "Node.js",
+ "TypeScript",
+ "Cartesia",
+ "Deepgram",
+ "AssemblyAI",
+]
+TTS_MODEL = "cartesia/sonic-3"
+
+# Settings to compare in simulations (see simulations/README.md). The defaults
+# are what lk ships.
+LLM_MODEL = os.environ.get("LK_ASSISTANT_LLM", "openai/gpt-5.4-mini")
+# How much the LLM reasons before it answers. Lower is faster, which matters
+# in a voice conversation. "" uses the model's default.
+REASONING_EFFORT = os.environ.get("LK_ASSISTANT_REASONING", "")
+
+INTRO = "Hi, I'm LK, your LiveKit assistant."
+
+
+
+def load_folder() -> dict:
+ """What lk found in the user's current folder before the session started."""
+ try:
+ return json.loads(os.environ.get("LK_ASSISTANT_FOLDER", "{}"))
+ except json.JSONDecodeError:
+ return {}
+
+
+def load_coding_agents() -> dict:
+ """What LiveKit has already set up for the user's coding agents: skills and the
+ Docs MCP server, in their project and globally."""
+ try:
+ return json.loads(os.environ.get("LK_ASSISTANT_CODING_AGENTS", "{}"))
+ except json.JSONDecodeError:
+ return {}
+
+
+def greeting(folder: dict, coding_agents: dict) -> str:
+ """A short, fixed greeting for what's in the user's folder. It offers what
+ fits without describing the folder; the agent knows the details from its
+ instructions. It's not generated, so it starts right away."""
+ kind = folder.get("kind")
+ # One sentence that offers what fits the folder, without describing it.
+ if kind == "empty":
+ return f"{INTRO} I can help you set up a new voice agent, or answer your questions about LiveKit."
+ if kind == "livekit":
+ return f"{INTRO} I see you have a LiveKit project up and going. Any questions about LiveKit I can help you with?"
+ return f"{INTRO} Any questions about LiveKit I can help you with?"
+
+
+# Pronunciations for the TTS, as Cartesia inline IPA overrides: "live" as in
+# "alive", not "liver". They apply only to the speech, so transcripts keep the
+# real spelling. Matching ignores case, and longer keys match first, so the
+# possessive gets its own key. The syntax is Cartesia-specific; see
+# docs.livekit.io/agents/multimodality/text/#built-in-replace-transform.
+LIVEKIT_IPA = "<<ˈ|l|aɪ|v>> <<ˈ|k|ɪ|t>>"
+LIVEKITS_IPA = "<<ˈ|l|aɪ|v>> <<ˈ|k|ɪ|t|s>>"
+PRONUNCIATIONS = {
+ "LiveKit's": LIVEKITS_IPA,
+ "LiveKit’s": LIVEKITS_IPA,
+ "LiveKit": LIVEKIT_IPA,
+ "Live Kit": LIVEKIT_IPA,
+ # Said as letters, without the dot.
+ "Next.js": "Next J S",
+ "Node.js": "Node J S",
+}
+
+# The assistant's name, LK, and the lk command are said as the letters L and
+# K. Matching is case-sensitive, and the lowercase keys include the spaces or
+# punctuation around the word, so that words like "walk" don't match. The
+# Markdown filter has already removed backticks.
+LK_IPA = "<<ˈ|ɛ|l>> <<ˈ|k|eɪ>>"
+NAME_PRONUNCIATIONS = {
+ "LK": LK_IPA,
+ " lk ": f" {LK_IPA} ",
+ " lk.": f" {LK_IPA}.",
+ " lk,": f" {LK_IPA},",
+ " lk?": f" {LK_IPA}?",
+}
+
+# Links to read more must point here.
+DOCS_URL = "https://docs.livekit.io/"
+
+# The Docs MCP tools the agent uses.
+DOCS_TOOLS = ("get_docs_overview", "docs_search", "read_docs", "code_search")
+# The Docs MCP tools the agent uses. It reads pages with its own read_docs
+# instead of get_pages, so code limits how many pages and how much text.
+MCP_DOCS_TOOLS = ("get_docs_overview", "docs_search", "code_search")
+
+# What the agent says while it looks something up, if it hasn't said anything
+# yet this turn. Docs lookups take a second or two.
+# Docs MCP results, such as searches, can be long.
+# Cap them, because every character slows the LLM, and a large context made
+# replies time out. Most answers come from the start of a page.
+MAX_TOOL_RESULT_CHARS = 20000
+
+# What old docs results become once a reply used them (see
+# Assistant.on_user_turn_completed).
+DOCS_RESULT_STUB = "[Docs content from an earlier answer, removed to keep replies fast. Look it up again if you need it.]"
+
+INSTRUCTIONS = """You are LK, the LiveKit assistant, a voice assistant inside the LiveKit CLI.
+You help developers build voice AI agents with LiveKit, add LiveKit to apps they already have, and
+explore what LiveKit can do. You are yourself a LiveKit voice agent, built with the same Agents SDK
+and LiveKit Inference the user can build with. Share the knowledge that helps the user build a
+voice agent that works well, and give context where it helps.
+
+Match the user's level. If they seem new, describe outcomes before LiveKit terms, and explain a
+term in a few words the first time you use it. If they seem experienced, be terse and precise, and
+skip the basics. If they ask you to be briefer or to go deeper, do that for the rest of the
+conversation.
+
+You speak out loud, so:
+- Keep replies short: one to three sentences. Don't cover everything; the user can ask for more.
+ Don't end with offers such as "If you want, I can".
+- Use plain sentences in the style of Simplified Technical English: one idea per sentence, active
+ voice, and common words. No markdown, lists, emojis, or symbols, except backticks around a short
+ command or name, such as `lk agent dev`. Backticks show it in monospace and aren't spoken.
+- Never read code, commands, or long identifiers aloud. Call show_code to show them,
+ then say one short sentence about what you showed. When the user asks how to do something in
+ code, or asks for an example, call show_code with a short example from the docs right away.
+ Never say code "looks like this" without showing it.
+- When it helps, use yourself as a one-sentence example after the answer. You're a LiveKit agent,
+ so you're built from the same parts; see "How you, LK, are built" below.
+
+LiveKit changes often, and what you remember about it is out of date. For every question about
+LiveKit features, APIs, SDKs, models, plugins, CLI commands, pricing, or configuration, call
+docs_search first, then answer only from what the docs say. Do this even when you think you know
+the answer. For questions about how something works or how to configure it, also read the most
+relevant page with read_docs before you answer. When you're not sure where a topic lives, or the
+user asks what LiveKit can do, call get_docs_overview to see how the docs are organized. If the
+docs don't cover it, say so. Never guess an API. When a docs page covers the topic in more depth,
+call show_link with it.
+
+Finding the right pages:
+- Pick pages from the docs page list by title: the page whose title matches the topic, not one
+ that only shares a keyword. Prefer the main topic page over a related one. For a general "how
+ do I" question, prefer an overview or guide page. Don't read pages under /recipes/ for a first
+ answer.
+- Keep lookups short, because the user is waiting: at most one docs_search per reply (put several
+ queries in its queries list), then one read_docs call with {pages}, then answer.
+ When a title in the list clearly matches, skip docs_search and go straight to read_docs.
+- Use only paths from the docs page list or from search results. Don't guess paths.
+
+Be honest about what LiveKit does:
+- If the docs don't confirm that a feature exists, say so. Don't combine features unless the docs
+ show they work together.
+- When LiveKit supports something only partly, say what's missing. A "yes" that leaves out a
+ known limit is worse than a "no".
+- Don't state numbers, such as prices, limits, or latencies, unless they're in a page you read.
+- If a question is ambiguous, ask one short question to find out what the user means. Speech
+ recognition can garble words, so read an odd question as a phonetic hint, such as "live kit"
+ for LiveKit.
+
+You can't see or change the user's files. You help them plan and decide, and you set them up to
+do the work:
+- To learn about their LiveKit Cloud project and current directory, call get_project_info. To check
+ on deployed agents, call list_cloud_agents.
+- For a new voice agent, call create_voice_agent. It asks the user to confirm and to pick a
+ language, creates the project, and offers to let them talk to it. It creates the project only
+ in the current folder, and only if the folder is empty. New projects already include LiveKit's
+ skills and Docs MCP server for the user's coding agents.
+- For an existing project, offer set_up_coding_agent, which installs LiveKit's skills and Docs MCP
+ server into the user's coding agents (such as Claude Code, Cursor, or Codex), unless they're
+ already set up.
+- To add LiveKit to an existing app (get_project_info reports its framework), explain the three
+ parts in plain words: the app gets a token from a small server endpoint, joins a LiveKit room,
+ and the agent joins that room through dispatch. Then create the agent with create_voice_agent
+ and give the frontend and token work to their coding agent with hand_off_task, based on the docs.
+- While the user's own agent runs, the user talks to it, not to you. Never speak as their agent.
+ To let them talk to it again later, call try_project.
+- If the user asks how you work, or what they'll be building, call explain_yourself.
+- When the user is ready to write code, call hand_off_task with a prompt for their coding agent:
+ what to build or change, plus what you learned from the docs, such as package names, APIs, and
+ CLI commands. If what they want is still vague, such as "add some tools", first ask one short
+ question about what exactly.
+- For anything you can't do here, such as running an agent they already had, say so in one
+ sentence and give the next best thing: a command to run, with show_code, or a prompt for their
+ coding agent.
+
+set_up_coding_agent and try_project don't run right away. They show the command to the user and
+return a confirmation ID. try_project always waits for the user's answer. Ask the user in one short sentence whether to go ahead. They can answer
+by voice or on screen. If they agree by voice, call confirm_action with the ID and approved=true;
+if they decline, approved=false. Never call confirm_action unless the user has just agreed.
+
+When the user says they're done or says goodbye, call end_call. Don't say goodbye yourself;
+end_call says it and closes the window.
+
+Prefer the LiveKit CLI for project tasks. For example, "lk agent init" creates an agent from a
+starter template, "lk agent dev" runs it, and "lk agent create" deploys it to LiveKit Cloud."""
+
+# The docs table of contents: every page with a one-line description. With it
+# in the prompt, the agent picks pages by title before it reads them. It's
+# trimmed to what onboarding questions need, about 10,000 tokens, and cached.
+DOCS_TOC_URL = "https://docs.livekit.io/api/toc/"
+DOCS_TOC_SKIP = re.compile(
+ r"/plugins/|/reference/|/components/|/robotics|/recipes/|youtube|"
+ r"/models/(stt|tts|llm|avatar)/[a-z]|/transport/self-hosting|/transport/media/ingress"
+)
+DOCS_TOC_TTL = 24 * 60 * 60
+
+
+def docs_toc() -> str:
+ """The trimmed docs table of contents, from the cache or the docs site, or ""."""
+ cache = os.path.join(os.path.expanduser("~/Library/Caches"), "livekit", "assistant-docs-toc.md")
+ try:
+ if time.time() - os.path.getmtime(cache) < DOCS_TOC_TTL:
+ return open(cache, encoding="utf-8").read()
+ except OSError:
+ pass
+ try:
+ with urllib.request.urlopen(DOCS_TOC_URL, timeout=2) as resp:
+ text = resp.read().decode("utf-8")
+ except Exception:
+ try:
+ return open(cache, encoding="utf-8").read() # a stale copy is better than none
+ except OSError:
+ return ""
+ toc = "\n".join(line for line in text.splitlines() if line.strip() and not DOCS_TOC_SKIP.search(line))
+ try:
+ os.makedirs(os.path.dirname(cache), exist_ok=True)
+ with open(cache, "w", encoding="utf-8") as f:
+ f.write(toc)
+ except OSError:
+ pass
+ return toc
+
+
+# How many docs pages one reply may read. Fewer is faster.
+MAX_PAGES = int(os.environ.get("LK_ASSISTANT_MAX_PAGES", "3"))
+
+# How much docs text one read_docs call returns, split between its pages. Every
+# character slows the LLM, and most answers come from the start of a page.
+DOCS_READ_CHARS = int(os.environ.get("LK_ASSISTANT_DOCS_CHARS", "12000"))
+
+
+def docs_page_url(path: str) -> str:
+ """The Markdown URL for a docs path or URL."""
+ path = path.strip()
+ for prefix in ("https://docs.livekit.io", "http://docs.livekit.io"):
+ if path.startswith(prefix):
+ path = path[len(prefix) :]
+ path = "/" + path.strip("/").removesuffix(".md")
+ return "https://docs.livekit.io" + path + ".md"
+
+
+def fetch_docs_page(url: str) -> str:
+ with urllib.request.urlopen(url, timeout=8) as resp:
+ return resp.read().decode("utf-8")
+
+
+async def read_docs_pages(paths: list[str]) -> str:
+ """Fetches docs pages as Markdown, in parallel, each trimmed to its share of
+ DOCS_READ_CHARS."""
+ if not paths:
+ return "No paths given. Pick pages from the docs page list."
+ share = DOCS_READ_CHARS // len(paths)
+
+ async def one(path: str) -> str:
+ url = docs_page_url(path)
+ try:
+ text = await asyncio.to_thread(fetch_docs_page, url)
+ except Exception as e:
+ return f"## {path}\n\nCouldn't read this page ({e}). Check the path in the docs page list."
+ if len(text) > share:
+ text = text[:share] + "\n... [the rest of the page is cut]"
+ return f"## {url.removesuffix('.md')}\n\n{text}"
+
+ return "\n\n".join(await asyncio.gather(*(one(p) for p in paths)))
+
+
+def about_yourself() -> str:
+ """Facts about how LK is built, from its own settings, so it can use itself as
+ an example."""
+ return f"""How you, LK, are built. Every part is something the user can use too:
+- A cascaded pipeline on LiveKit Inference: STT {STT_MODEL}, with keyterms that help it hear
+ LiveKit's names; LLM {LLM_MODEL}; and TTS {TTS_MODEL}, in expressive mode. Text transforms fix
+ how the TTS says "LiveKit".
+- Turn detection: LiveKit's turn detector model, with dynamic endpointing that learns the user's
+ pauses.
+- Tools: the LiveKit Docs MCP server, connected with MCPToolset, to search the docs, plus your own
+ function tools, such as read_docs to read pages, show_code for code cards, and hand_off_task
+ for prompts.
+- A task: creating the user's agent is an AgentTask. Code runs its steps, and the model only
+ reads the user's answers and words the replies, so it can't skip a step.
+- Testing: Agent Simulations, where an LLM plays the user and a judge grades each conversation,
+ in text and audio mode, with scenarios to compare configurations and catch regressions.
+- You run on the user's machine in console mode, started by lk.
+
+For example: "I work this way too: I call the Docs MCP server as a tool to get current docs."
+Don't use yourself as an example in every answer, and only say what's in this list."""
+
+
+def build_instructions() -> str:
+ pages = "only the one most relevant path" if MAX_PAGES == 1 else f"up to {MAX_PAGES} paths"
+ instructions = INSTRUCTIONS.replace("{pages}", pages) + "\n\n" + about_yourself()
+ toc = docs_toc() if os.environ.get("LK_ASSISTANT_TOC", "1") == "1" else ""
+ if not toc:
+ return instructions
+ return (
+ instructions
+ + "\n\nThe docs pages, as [Section > Title](path): description. Pick pages from this list "
+ "by title, and pass their paths to read_docs.\n\n"
+ + toc
+ )
+
+
+async def truncate_result(ctx: mcp.MCPToolResultContext) -> str:
+ if len(ctx.result.content) == 1:
+ text = str(ctx.result.content[0].model_dump_json())
+ elif ctx.result.content:
+ text = json.dumps([item.model_dump() for item in ctx.result.content])
+ else:
+ return "The tool returned no content."
+ if len(text) > MAX_TOOL_RESULT_CHARS:
+ text = text[:MAX_TOOL_RESULT_CHARS] + "\n... [truncated]"
+ return text
+
+
+class Assistant(Agent):
+ def __init__(self, folder: dict | None = None, coding_agents: dict | None = None) -> None:
+ instructions = build_instructions()
+ folder = load_folder() if folder is None else folder
+ coding_agents = load_coding_agents() if coding_agents is None else coding_agents
+ self.folder = folder
+ self.created = False # whether create_voice_agent created a project
+ if project := os.environ.get("LK_ASSISTANT_PROJECT"):
+ instructions += f"\n\nThe user's current directory is {project}."
+ if folder:
+ instructions += (
+ " Before the session started, lk looked at the project files in it and its "
+ f"subfolders, and found this: {json.dumps(folder)}. Your greeting only offered what fits, "
+ "without describing the folder. Use these details when they help, but don't list them unless the "
+ "user asks. Versions are as the project files give them. Call get_project_info if "
+ "you need more, or if the folder may have changed."
+ )
+ if coding_agents:
+ instructions += (
+ "\n\nWhat LiveKit has already set up for the user's coding agents (LiveKit skills and "
+ f"the Docs MCP server, in this project and globally): {json.dumps(coding_agents)}. "
+ "Offer set_up_coding_agent only for what's missing, and say what's missing. If both "
+ "are already set up, don't offer it. If the user asks, tell them it's already done, "
+ 'and that "lk skills update" gets the latest skills. After set_up_coding_agent runs, '
+ "call get_project_info for the current state."
+ )
+ super().__init__(
+ instructions=instructions,
+ tools=[
+ mcp.MCPToolset(
+ id="livekit-docs",
+ mcp_server=mcp.MCPServerHTTP(
+ DOCS_MCP_URL,
+ allowed_tools=list(MCP_DOCS_TOOLS),
+ tool_result_resolver=truncate_result,
+ ),
+ ),
+ # lk closes its overlay when it sees this tool called. Console
+ # mode has no real room, so there's nothing to delete.
+ *EndCallTool(
+ extra_description=(
+ "Call this whenever the user wants to stop or leave, such as saying exit, "
+ "close, quit, bye, or that's all. Saying goodbye without calling it leaves "
+ "the window open."
+ ),
+ delete_room=False,
+ end_instructions="Say a short, friendly goodbye in one sentence.",
+ ).tools,
+ ],
+ )
+
+ async def on_user_turn_completed(self, turn_ctx, new_message) -> None:
+ # Docs pages are large, and every later turn would send them to the LLM
+ # again. Once a reply has used them, keep only a stub; the agent can
+ # look a page up again.
+ changed = drop_old_docs(turn_ctx)
+ ctx = self.chat_ctx.copy()
+ if drop_old_docs(ctx) or changed:
+ await self.update_chat_ctx(ctx)
+
+ @function_tool()
+ async def read_docs(self, context: RunContext, paths: list[str]) -> str:
+ """Read LiveKit docs pages. Pick them by title from the docs page list or from search
+ results.
+
+ Args:
+ paths: Page paths, such as /agents/models/pipelines. Only the first few are read.
+ """
+ return await read_docs_pages(paths[:MAX_PAGES])
+
+ @function_tool()
+ async def show_code(self, context: RunContext, title: str, language: str, code: str) -> str:
+ """Show code or a terminal command to the user. The user sees it on screen and can
+ copy it. Use this for all code and commands instead of saying them.
+
+ Args:
+ title: A file name such as agent.py, or "Terminal" for shell commands.
+ language: The language of the code, such as python, typescript, or shell.
+ code: The complete code or command, exactly as the user should use it.
+ """
+ # `lk assistant` reads this call from the session events and shows it.
+ return (
+ "The code is on the user's screen, with a Copy button. If you haven't said what it is, "
+ "say so in one sentence. Don't repeat what you already said."
+ )
+
+ @function_tool()
+ async def show_link(self, context: RunContext, title: str, url: str) -> str:
+ """Show a link to a LiveKit docs page where the user can read more. The user can open
+ it in their browser or copy it.
+
+ Args:
+ title: The page title, such as "LiveKit turn detector".
+ url: The page's full URL, exactly as your docs search or read_docs returned it. It
+ must start with https://docs.livekit.io/.
+ """
+ # `lk assistant` reads this call from the session events and shows it.
+ if not url.startswith(DOCS_URL):
+ return f"Not shown: links must be docs pages under {DOCS_URL}, from your docs searches."
+ return "The link is on the user's screen. Don't repeat what you already said."
+
+ @function_tool()
+ async def copy_to_clipboard(
+ self,
+ context: RunContext,
+ kind: Literal["latest", "code", "link", "prompt"] = "latest",
+ title: str = "",
+ ) -> str:
+ """Copy a card on screen to the user's clipboard. Use it only when the user asks you to
+ copy something. It copies the latest card of the kind they mean.
+
+ Args:
+ kind: What the user wants copied: "link" for a Read more link, "code" for code or a
+ command, "prompt" for a prompt for their coding agent, or "latest" for the most
+ recent card of any kind.
+ title: Optional words from the card's title, only when there are several cards of
+ that kind and the user named one.
+ """
+ return await run_lk_tool("copy_to_clipboard", {"kind": kind, "title": title})
+
+ @function_tool()
+ async def hand_off_task(self, context: RunContext, task: str) -> str:
+ """Show the user a prompt to paste into their coding agent, with a button to copy it.
+ Call it right away when the user wants a change; don't ask whether to.
+
+ Args:
+ task: A short, self-contained prompt addressed to the coding agent, like a senior
+ engineer's ticket. For a change, such as a new voice: the exact change, with the
+ API, model, and values you looked up in the docs, and how to check it works.
+ Pick one option; if the choice is the user's, ask them first. If the request is
+ vague, such as "add some tools", ask one short question about what exactly before
+ you call this; don't write a prompt that leaves the work undefined. For a bug: the
+ problem, how to reproduce it, and how to check it's fixed, without guessing a
+ fix, because you haven't seen the code.
+ """
+ # `lk assistant` reads this call from the session events and shows it.
+ return (
+ "The prompt is on the user's screen. In one sentence, tell the user what the change is, "
+ "naming any choice you made, such as the voice you picked, and that they can copy it "
+ "into their coding agent. You can't send it yourself, so don't say you handed it over."
+ )
+
+ @function_tool()
+ async def get_project_info(self, context: RunContext) -> str:
+ """Get the user's LiveKit Cloud project (the one lk uses), their current directory, and
+ whether it's an agent project, including the ID of its deployed agent if it has one."""
+ return await run_lk_tool("get_project_info")
+
+ @function_tool()
+ async def list_cloud_agents(self, context: RunContext) -> str:
+ """List the agents deployed to the user's LiveKit Cloud project, with their status."""
+ return await run_lk_tool("list_cloud_agents")
+
+ @function_tool()
+ async def list_templates(self, context: RunContext) -> str:
+ """List LiveKit's project templates: agents (Python and Node.js) and frontends (web,
+ mobile, and others), with a description of each."""
+ return await run_lk_tool("list_templates")
+
+ @function_tool()
+ async def create_voice_agent(self, context: RunContext, another: bool = False) -> str | None:
+ """Create a new voice agent for the user from a LiveKit starter template, then offer to
+ let them talk to it. Call it when the user wants to build, create, or set up a new agent.
+ It takes over until the project exists: it asks the user to confirm and to pick Python or
+ Node.js, so don't ask first.
+
+ Args:
+ another: True only if the user explicitly asks for a second, separate agent after
+ one was already created in this conversation.
+ """
+ if self.created and not another:
+ return (
+ "Nothing was created: the user's agent already exists from earlier in this "
+ "conversation. Don't create another unless the user explicitly asks for a second one."
+ )
+ if self.folder.get("kind") != "empty":
+ return (
+ "Nothing was created: LK creates a new agent only in an empty current folder, and this "
+ "folder isn't empty. Tell the user to make an empty folder, go to it, and run "
+ "`lk assistant` there."
+ )
+ result = await NewAgentTask(chat_ctx=self.chat_ctx)
+ if result.startswith("Created") or result == STARTED:
+ self.created = True
+ self.folder = {"kind": "livekit", "has_agents": True}
+ if result == STARTED:
+ # The task already announced the hand-off, and the user is about to
+ # talk to their agent, so there's nothing to add.
+ return None
+ return result
+
+ @function_tool()
+ async def set_up_coding_agent(
+ self,
+ context: RunContext,
+ all_projects: bool = False,
+ reinstall: bool = False,
+ confirmed: bool = False,
+ ) -> str:
+ """Install LiveKit's agent skills and the LiveKit Docs MCP server into the user's coding
+ agents (such as Claude Code, Cursor, and Codex), with lk skills install. New projects from
+ create_voice_agent already have them. If they're already set up, it says so and installs
+ nothing. Unless confirmed is true, it shows the command and waits for the user to confirm
+ (see confirm_action).
+
+ Args:
+ all_projects: Install for all of the user's projects instead of only the current one.
+ reinstall: Install even though they're already set up. Only if the user asks for it.
+ confirmed: True if the user's last message asked for this or agreed to it, such
+ as "set up a voice agent here" or "yes" after you offered it. Then it runs
+ right away, without asking again. Otherwise leave it false, and ask.
+ """
+ return await run_lk_tool(
+ "set_up_coding_agent",
+ {"global": all_projects, "reinstall": reinstall, "confirmed": confirmed},
+ )
+
+ @function_tool()
+ async def try_project(self, context: RunContext) -> str:
+ """Run the agent the user created in this session again, in this window, so they can talk
+ to it. Only call it when the user asks to talk to their agent again. "Start building" or
+ "let me get to work" means coding, not this. The conversation switches to their agent
+ until they press B to come back to you. It always waits for the user to confirm on screen
+ or by voice (see confirm_action). For any other agent, it tells you how the user can run
+ it in their own terminal.
+ """
+ return await run_lk_tool("try_project", {"confirmed": False})
+
+ @function_tool()
+ async def explain_yourself(self, context: RunContext) -> str:
+ """Show the user how you, LK, are built: your own AgentSession code, on screen. Use it
+ only when they ask how you work or how you're built. For an example of how to build an
+ agent, show code from the docs with show_code instead."""
+ code = own_session_code()
+ if lk is not None and code:
+ lk.send({"type": "show_code", "title": "agent.py (how LK is built)", "language": "python", "code": code})
+ return (
+ "Your own setup is on the user's screen. You run on the LiveKit Agents SDK in "
+ "console mode, inside lk. Your pipeline: speech to text with AssemblyAI, the "
+ "gpt-5.4-mini LLM, and text to speech with Cartesia, all through LiveKit Inference. "
+ "LiveKit's turn detector decides when the user has finished, with dynamic "
+ "endpointing. Expressive mode adds emotion and pacing to your voice. Your tools are "
+ "the LiveKit Docs MCP server plus lk commands. In one or two sentences, explain this "
+ "and point out that a project from agent-starter-python has the same structure."
+ )
+
+ @function_tool()
+ async def confirm_action(self, context: RunContext, confirmation_id: str, approved: bool) -> str:
+ """Run or cancel a command that set_up_coding_agent or try_project proposed, after the
+ user answers by voice. Only call it right after the user agrees or declines.
+
+ Args:
+ confirmation_id: The ID the proposing tool returned, such as confirm1. Works for
+ set_up_coding_agent and try_project.
+ approved: True if the user agreed, false if they declined.
+ """
+ # The command changes the user's files. While it runs, user speech
+ # mustn't interrupt this step, or its result is never spoken.
+ context.disallow_interruptions()
+ try:
+ return await run_lk_tool(
+ "confirm_action",
+ {"confirmation_id": confirmation_id, "approved": approved},
+ timeout=COMMAND_TIMEOUT,
+ )
+ finally:
+ # The reply after the command reuses this step's speech. Let the
+ # user talk over it, or what they say while it plays is dropped.
+ context.speech_handle.allow_interruptions = True
+
+
+# Confirmed commands can take a while, such as installing dependencies.
+COMMAND_TIMEOUT = 600
+
+
+def drop_old_docs(ctx) -> bool:
+ """Replaces the output of docs lookups in ctx with DOCS_RESULT_STUB.
+ Reports whether anything changed."""
+ docs_calls = {
+ item.call_id
+ for item in ctx.items
+ if item.type == "function_call" and item.name in DOCS_TOOLS
+ }
+ changed = False
+ for item in ctx.items:
+ if item.type == "function_call_output" and item.call_id in docs_calls and item.output != DOCS_RESULT_STUB:
+ item.output = DOCS_RESULT_STUB
+ changed = True
+ return changed
+
+
+# The starter templates create_voice_agent uses, by language.
+STARTERS = {"python": "agent-starter-python", "node": "agent-starter-node"}
+
+# How each starter opens a conversation, for the hand-off: the Node.js starter
+# greets the user when it joins, and the Python starter waits for them.
+STARTER_OPENINGS = {
+ "python": "once it connects, they should say hello, because it waits for them to speak first",
+ "node": "once it connects, it greets them first",
+}
+
+NEW_AGENT_INSTRUCTIONS = """You are LK, the LiveKit assistant, helping the user create a new voice
+agent from a LiveKit starter template. Code runs the steps. Say only what you're asked to say, in
+short, plain sentences, and don't read commands aloud. Never say a step happened unless you're told
+it did."""
+
+# The answers each question in NewAgentTask accepts, for read_answer.
+LANGUAGE_ANSWERS = {
+ "python": "chooses or confirms Python",
+ "node": "chooses or confirms Node.js, JavaScript, or TypeScript",
+ "cancel": "doesn't want to create an agent now",
+ "leave": "wants to stop talking to LK or close it",
+ "other": "is unsure, asks a question, or says something else",
+}
+TRY_ANSWERS = {
+ "yes": "wants to talk to their new agent now",
+ "no": "doesn't want to talk to it now",
+ "leave": "wants to stop talking to LK or close it",
+ "other": "asks a question or says something else",
+}
+
+
+async def read_answer(model: llm.LLM, conversation: str, answers: dict[str, str], answer: str) -> str:
+ """Asks the model which of answers the user's answer means, given the last
+ few lines of the conversation, including the question LK actually asked. A
+ narrow call like this one is more reliable than a conversation that must
+ decide to call a tool. Returns "other" if the reply isn't one of the
+ answers."""
+ ctx = llm.ChatContext.empty()
+ options = "\n".join(f"- {key}: the user {meaning}" for key, meaning in answers.items())
+ ctx.add_message(
+ role="system",
+ content=(
+ "Classify the user's answer to LK's last question. Read it in context: \"yes\" to "
+ "\"Create it in Python?\" means Python. Reply with only one of these words:\n" + options
+ ),
+ )
+ ctx.add_message(role="user", content=f"Conversation so far:\n{conversation}\n\nThe user's answer: {answer}")
+ text = ""
+ async with model.chat(chat_ctx=ctx) as stream:
+ async for chunk in stream:
+ if chunk.delta and chunk.delta.content:
+ text += chunk.delta.content
+ word = text.strip().strip(".\"'").lower()
+ return word if word in answers else "other"
+
+
+def recent_lines(chat_ctx, before: str, n: int = 4) -> str:
+ """The last n messages before the message with id before, as "Role: text" lines."""
+ lines = []
+ for item in chat_ctx.items:
+ if item.id == before:
+ break
+ if item.type == "message" and item.role in ("user", "assistant") and item.text_content:
+ who = "User" if item.role == "user" else "LK"
+ lines.append(f"{who}: {item.text_content}")
+ return "\n".join(lines[-n:])
+
+
+# What NewAgentTask returns when it has started the user's agent and announced it.
+STARTED = "Started the user's agent, and told them how to come back."
+
+
+class NewAgentTask(AgentTask[str]):
+ """Creates the user's new agent and offers to let them talk to it.
+
+ Code runs every step. Each of the user's answers is read with read_answer,
+ acted on in code, and only then worded by the model. This happens in
+ llm_node, which every reply goes through, whether the user spoke or typed.
+ The model has no tools here, so it can't skip a step, answer its own
+ question, or say a step happened when it didn't. Each question is also a
+ choice card, so the user can answer on screen.
+ """
+
+ def __init__(self, chat_ctx) -> None:
+ super().__init__(instructions=NEW_AGENT_INSTRUCTIONS, chat_ctx=chat_ctx)
+ self.step = "language" # language, creating, try, finished
+ self.card = ""
+ self.scripted = 0 # replies that code asked for, not yet generated
+ # The id of the last user message code handled. The message that started
+ # this task isn't an answer to its first question.
+ self.handled = next(
+ (i.id for i in reversed(chat_ctx.items) if i.type == "message" and i.role == "user"), ""
+ )
+
+ async def on_enter(self) -> None:
+ await self._ask(
+ "Create a voice agent in this folder?",
+ [("python", "Python"), ("node", "Node.js"), ("cancel", "Not now")],
+ "Ask, in these words or close to them: \"Do you want to use Python or Node?\" It goes in "
+ "the current folder, so don't offer another folder. If they already said a language, "
+ "ask them to confirm it instead.",
+ )
+
+ async def llm_node(self, chat_ctx, tools, model_settings):
+ if self.scripted:
+ # A reply code asked for with _say; its instructions are in chat_ctx.
+ self.scripted -= 1
+ else:
+ user = next((i for i in reversed(chat_ctx.items) if i.type == "message" and i.role == "user"), None)
+ if user is None or user.id == self.handled:
+ return
+ self.handled = user.id
+ say = await self._handle(user.text_content or "", recent_lines(chat_ctx, user.id))
+ if not say:
+ return
+ chat_ctx = chat_ctx.copy()
+ chat_ctx.add_message(role="system", content=say)
+ async for chunk in Agent.default.llm_node(self, chat_ctx, [], model_settings):
+ yield chunk
+
+ async def _handle(self, answer: str, conversation: str) -> str | None:
+ """Acts on the user's answer, and returns what the reply should say, if anything."""
+ if self.step == "language":
+ choice = await read_answer(self.session.llm, conversation, LANGUAGE_ANSWERS, answer)
+ if choice in STARTERS:
+ await self._answered(choice)
+ asyncio.create_task(self._create(choice))
+ name = "Python" if choice == "python" else "Node.js"
+ return f"In one short sentence, tell the user you're creating their {name} agent now."
+ if choice == "cancel":
+ await self._answered("cancel")
+ self._done("The user chose not to create an agent now. Nothing was created.")
+ return None
+ if choice == "leave":
+ self._done("Nothing was created. The user wants to stop, so call end_call.")
+ return None
+ return (
+ "Answer what the user said in one or two short sentences. If they're unsure: Python "
+ "has the most examples, and Node.js fits if their app is JavaScript or TypeScript. "
+ "Then ask: \"Do you want to use Python or Node?\""
+ )
+ if self.step == "try":
+ choice = await read_answer(self.session.llm, conversation, TRY_ANSWERS, answer)
+ if choice in ("yes", "no"):
+ await self._answered("try" if choice == "yes" else "later")
+ asyncio.create_task(self._finish(choice == "yes"))
+ return None
+ if choice == "leave":
+ self._done("Created the project. The user wants to stop, so call end_call.")
+ return None
+ return (
+ "Answer what the user said in one or two short sentences, then ask whether they want "
+ "to talk to their new agent now."
+ )
+ if self.step == "creating":
+ return "In a few words, tell the user their agent is still being created."
+ return None
+
+ def _say(self, instructions: str):
+ """Has the model say something code decided. Returns its SpeechHandle."""
+ self.scripted += 1
+ return self.session.generate_reply(instructions=instructions, tool_choice="none")
+
+ async def _ask(self, question: str, options: list[tuple[str, str]], say: str) -> None:
+ """Shows a choice card and asks the question."""
+ card = await run_lk_tool(
+ "show_choice",
+ {"question": question, "options": [{"id": i, "name": n} for i, n in options]},
+ )
+ self.card = card
+ if lk is not None:
+ lk.on_choice(card, self._clicked)
+ self._say(say)
+
+ async def _answered(self, option: str) -> None:
+ """Marks the open choice card with an answer given by voice."""
+ if self.card:
+ await run_lk_tool("choice_made", {"id": self.card, "option": option})
+ self.card = ""
+
+ async def _clicked(self, option: str) -> None:
+ """Handles an answer clicked on the open choice card."""
+ self.card = ""
+ if self.step == "language" and option == "cancel":
+ self._done("The user chose not to create an agent now. Nothing was created.")
+ elif self.step == "language" and option in STARTERS:
+ self._say("In one short sentence, tell the user you're creating their agent now.")
+ await self._create(option)
+ elif self.step == "try":
+ await self._finish(option == "try")
+
+ async def _create(self, language: str) -> None:
+ self.step = "creating"
+ self.language = language
+ args = {"template": STARTERS[language], "in_current_directory": True, "confirmed": True}
+ result = await run_lk_tool("create_project", args, timeout=600)
+ if result.startswith("Error") or "only available" in result:
+ self._done(f"Creating the project failed: {result}")
+ return
+ self.step = "try"
+ await self._ask(
+ "Talk to your new agent now?",
+ [("try", "Talk to it"), ("later", "Not now")],
+ "In one sentence, tell the user their agent is ready, and ask whether they want to "
+ "talk to it now.",
+ )
+
+ async def _finish(self, yes: bool) -> None:
+ if not yes:
+ self._done("Created the project. The user didn't want to talk to it yet.")
+ return
+ self.step = "finished"
+ # Say the whole hand-off before the agent starts: once it connects, it
+ # takes over the audio, and anything still playing is cut off.
+ await self._say(
+ "In two short sentences, tell the user you're starting their agent, that "
+ + STARTER_OPENINGS[self.language]
+ + ", and that they can press B to come back to you."
+ ).wait_for_playout()
+ await run_lk_tool("try_project", {"confirmed": True})
+ self.complete(STARTED)
+
+ def _done(self, result: str) -> None:
+ if self.step == "finished":
+ return
+ self.step = "finished"
+ self.complete(result)
+
+
+def own_session_code() -> str:
+ """Returns this agent's AgentSession setup, from its own source."""
+ try:
+ source = open(__file__).read()
+ except OSError:
+ return ""
+ # The last occurrence: the first is this function's own search string.
+ start = source.rfind(" session = AgentSession(")
+ end = source.find("\n )\n", start)
+ if start < 0 or end < 0:
+ return ""
+ lines = source[start : end + 6].splitlines()
+ code = "\n".join(line[4:] if line.startswith(" ") else line for line in lines)
+ # Show the model names themselves, not the constants.
+ for name, value in (("STT_MODEL", STT_MODEL), ("LLM_MODEL", LLM_MODEL), ("TTS_MODEL", TTS_MODEL)):
+ code = code.replace(f"model={name}", f'model="{value}"')
+ # Leave out a setting that's only for comparing configurations.
+ code = re.sub(r"\n\s*extra_kwargs=.*REASONING_EFFORT.*", "", code)
+ return code
+
+
+async def run_lk_tool(name: str, args: dict | None = None, timeout: float = 60) -> str:
+ """Runs an lk tool through lk and returns its output for the LLM."""
+ if lk is None:
+ return "This tool is only available when the assistant runs inside lk."
+ try:
+ reply = await lk.request({"type": "lk_tool", "name": name, "args": args or {}}, timeout=timeout)
+ except asyncio.TimeoutError:
+ return "lk didn't respond in time."
+ if not reply.get("ok"):
+ return f"Error: {reply.get('error', 'unknown error')}"
+ return reply.get("output") or "Done."
+
+
+class LkChannel:
+ """The local connection to lk, as JSON lines in both directions.
+
+ The agent sends its transcript and requests to run lk tools. lk sends
+ replies to requests, and can ask the agent to tell the user something.
+ """
+
+ def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+ self._reader = reader
+ self._writer = writer
+ self._pending: dict[str, asyncio.Future[dict]] = {}
+ self._next_id = 0
+ self._session: AgentSession | None = None
+ self._read_task: asyncio.Task[None] | None = None
+ self._choice_handlers: dict[str, Callable[[str], Awaitable[None]]] = {}
+
+ def start(self, session: AgentSession) -> None:
+ self._session = session
+ self._read_task = asyncio.create_task(self._read())
+
+ def send(self, msg: dict) -> None:
+ if not self._writer.is_closing():
+ self._writer.write((json.dumps(msg) + "\n").encode())
+
+ def on_choice(self, card: str, handler: Callable[[str], Awaitable[None]]) -> None:
+ """Calls handler with the option the user clicks on a choice card."""
+ self._choice_handlers[card] = handler
+
+ async def request(self, msg: dict, timeout: float = 10) -> dict:
+ """Sends msg and waits for lk's reply."""
+ self._next_id += 1
+ msg_id = str(self._next_id)
+ fut: asyncio.Future[dict] = asyncio.get_running_loop().create_future()
+ self._pending[msg_id] = fut
+ self.send({**msg, "id": msg_id})
+ try:
+ return await asyncio.wait_for(fut, timeout)
+ finally:
+ self._pending.pop(msg_id, None)
+
+ async def _read(self) -> None:
+ while line := await self._reader.readline():
+ try:
+ msg = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if msg.get("type") == "reply":
+ fut = self._pending.get(msg.get("id", ""))
+ if fut and not fut.done():
+ fut.set_result(msg)
+ elif msg.get("type") == "choice":
+ # The user clicked an option on a choice card.
+ if handler := self._choice_handlers.pop(msg.get("id", ""), None):
+ asyncio.create_task(handler(msg.get("option", "")))
+ elif msg.get("type") == "user_text" and self._session is not None:
+ # The user typed instead of speaking: stop talking and answer it.
+ # Speech that can't be interrupted, such as a running
+ # confirmation, finishes first.
+ try:
+ self._session.interrupt()
+ except RuntimeError:
+ pass
+ self._session.generate_reply(user_input=msg.get("text", ""))
+ elif msg.get("type") == "say" and self._session is not None:
+ # lk has news for the user.
+ self._session.generate_reply(instructions=msg.get("instructions", ""))
+
+
+# The connection to lk, when lk runs this agent.
+lk: LkChannel | None = None
+
+
+class TranscriptForwarder(io.TextOutput):
+ """Sends the agent's transcript to lk as JSON lines, in time with its speech.
+
+ Each line is {"type": "text", "text": ""} or {"type": "flush"} at the
+ end of a segment. Expressive markup is removed.
+ """
+
+ def __init__(self, channel: LkChannel) -> None:
+ super().__init__(label="lk-assistant", next_in_chain=None)
+ self._channel = channel
+ self._stripper = TranscriptMarkupStripper()
+
+ async def capture_text(self, text: str) -> None:
+ self._send_text(self._stripper.push(text))
+
+ def flush(self) -> None:
+ self._send_text(self._stripper.flush())
+ self._send({"type": "flush"})
+ self._stripper = TranscriptMarkupStripper()
+
+ def _send_text(self, text: str) -> None:
+ if text:
+ self._send({"type": "text", "text": text})
+
+ def _send(self, msg: dict) -> None:
+ self._channel.send(msg)
+
+
+async def connect_lk(session: AgentSession) -> None:
+ """Connects to lk, if lk is running this agent, and streams the agent's
+ spoken text to it.
+
+ Console mode only reports what the agent said after it finishes speaking. It
+ also syncs the text to the audio, but sends the result nowhere. This swaps in a
+ synchronizer whose text goes to lk instead.
+ """
+ global lk
+ addr = os.environ.get("LK_ASSISTANT_ADDR")
+ if not addr:
+ return
+ host, port = addr.rsplit(":", 1)
+ reader, writer = await asyncio.open_connection(host, int(port))
+ lk = LkChannel(reader, writer)
+ lk.start(session)
+ # For lk's X-ray view of the pipeline.
+ lk.send({"type": "pipeline", "stt": STT_MODEL, "llm": LLM_MODEL, "tts": TTS_MODEL, "turn_detection": "LiveKit turn detector, dynamic endpointing"})
+
+ audio = session.output.audio
+ if audio is None:
+ return
+ if audio.label == "TranscriptSynchronizer" and audio.next_in_chain is not None:
+ audio = audio.next_in_chain
+ sync = TranscriptSynchronizer(next_in_chain_audio=audio, next_in_chain_text=TranscriptForwarder(lk))
+ session.output.audio = sync.audio_output
+ session.output.transcription = sync.text_output
+
+
+TURN_ANSWERS = {
+ "leave": (
+ "clearly asks to close LK or end the conversation now, such as \"exit\" or \"that's all, "
+ "bye\". Apologies, fillers, fragments, and thanks in the middle of a conversation, such as "
+ "\"Sorry,\" or \"Um\" or \"Thanks, and\", are not leaving, and neither are questions about "
+ "closing or exiting something else, such as \"How do I exit the console?\""
+ ),
+ "code": "asks to see code, an example, a snippet, or how something looks in code",
+ "other": "says anything else",
+}
+
+
+def watch_user_turns(session: AgentSession) -> None:
+ """Acts on what each user turn asks for, when the model can't be relied on to.
+
+ After each user turn, a short model call in the background reads the turn. It
+ runs beside the reply, so it adds no delay.
+ - If the user wants to leave, it tells lk to close once the agent is quiet,
+ even if the model didn't call end_call.
+ - If the user asked for code and the reply ends without a code card, it asks
+ for one more reply that must call show_code.
+ """
+ log = logging.getLogger("lk-assistant")
+ listening = asyncio.Event() # set when a reply ends and the agent listens again
+
+ @session.on("agent_state_changed")
+ def _on_state(ev) -> None:
+ if ev.new_state == "listening" and ev.old_state in ("thinking", "speaking"):
+ listening.set()
+
+ async def check(item, conversation: str) -> None:
+ try:
+ listening.clear()
+ answer = await read_answer(session.llm, conversation, TURN_ANSWERS, item.text_content)
+ if answer == "leave" and lk is not None:
+ lk.send({"type": "end"})
+ elif answer == "code":
+ await asyncio.wait_for(listening.wait(), timeout=60)
+ await show_code_if_missing(item)
+ except asyncio.TimeoutError:
+ pass
+ except Exception:
+ log.exception("couldn't check the user's turn")
+
+ async def show_code_if_missing(item) -> None:
+ if not isinstance(session.current_agent, Assistant):
+ return # a task, such as creating the user's agent, is running
+ items = session.history.items
+ after = [i for i in items[next((n for n, i in enumerate(items) if i.id == item.id), len(items)) + 1 :]]
+ if any(i.type == "message" and i.role == "user" for i in after):
+ return # the user has moved on
+ if any(i.type == "function_call" and i.name == "show_code" for i in after):
+ return
+ session.generate_reply(
+ instructions=(
+ "The user asked for code, and you haven't shown it. Call show_code now with a short "
+ "example from the docs page you read, then say one short sentence about it."
+ ),
+ tool_choice={"type": "function", "function": {"name": "show_code"}},
+ )
+
+ @session.on("conversation_item_added")
+ def _on_item(ev) -> None:
+ item = ev.item
+ if getattr(item, "role", None) != "user" or not item.text_content:
+ return
+ asyncio.create_task(check(item, recent_lines(session.history, item.id)))
+
+
+server = AgentServer()
+
+
+async def simulation_end(ctx) -> None:
+ """Checks the tools a simulation called (see lk_simulation.py)."""
+ from lk_simulation import check_simulation
+
+ check_simulation(ctx)
+
+
+@server.rtc_session(agent_name="lk-assistant", on_simulation_end=simulation_end)
+async def assistant(ctx: agents.JobContext):
+ # STT from the Voice AI quickstart
+ # (docs.livekit.io/agents/start/voice-ai-quickstart), with two changes:
+ # - LLM: gpt-5.4-mini instead of Gemma, for more reliable tool calling.
+ # The assistant must search the docs before it answers.
+ # - TTS: Cartesia instead of Fish Audio, for inline phoneme overrides that
+ # make it say "LiveKit" correctly every time.
+ session = AgentSession(
+ stt=inference.STT(
+ model=STT_MODEL,
+ language="en",
+ # Help the STT hear LiveKit's names and terms.
+ extra_kwargs={"keyterms_prompt": STT_KEYTERMS},
+ ),
+ llm=inference.LLM(
+ model=LLM_MODEL,
+ extra_kwargs={"reasoning_effort": REASONING_EFFORT} if REASONING_EFFORT else {},
+ ),
+ tts=inference.TTS(
+ model=TTS_MODEL,
+ voice="a167e0f3-df7e-4d52-a9c3-f949145efdab",
+ language="en",
+ ),
+ turn_handling=TurnHandlingOptions(
+ turn_detection=inference.TurnDetector(),
+ # Developers often pause mid-sentence to think. With the turn
+ # detector, the shortest wait defaults to 0.3 seconds, which can
+ # end a turn mid-thought. Dynamic endpointing learns the user's
+ # pauses; the higher floor leaves room for a short one.
+ endpointing={"mode": "dynamic", "min_delay": 0.8},
+ ),
+ # A reply after a docs lookup reads a lot of context. The default
+ # 10-second timeout can cut it off.
+ conn_options=SessionConnectOptions(llm_conn_options=APIConnectOptions(timeout=20.0)),
+ # The LLM marks up emotion and pacing for the TTS. The markup is
+ # stripped from transcripts.
+ expressive=True,
+ tts_text_transforms=[
+ "filter_emoji",
+ "filter_markdown",
+ text_transforms.replace(PRONUNCIATIONS),
+ text_transforms.replace(NAME_PRONUNCIATIONS, case_sensitive=True),
+ ],
+ )
+
+ # Fetch the docs table of contents, or read it from the cache, off the
+ # event loop; the instructions read it from the cache.
+ await asyncio.to_thread(docs_toc)
+ folder, coding_agents = load_folder(), load_coding_agents()
+ if sim := ctx.simulation_context():
+ # Agent Simulations (see simulations/README.md): the scenario sets the
+ # folder, and lk's tools are mocked. The simulation module isn't
+ # embedded in lk.
+ from lk_simulation import setup_simulation
+
+ folder, coding_agents = setup_simulation(sim, session, Assistant)
+
+ watch_user_turns(session)
+ await session.start(room=ctx.room, agent=Assistant(folder, coding_agents))
+ await connect_lk(session)
+
+ # A fixed greeting for what is in the folder. It starts without waiting on the LLM.
+ session.say(greeting(folder, coding_agents))
+
+
+if __name__ == "__main__":
+ agents.cli.run_app(server)
diff --git a/pkg/assistant/agent/evals/test_onboarding.py b/pkg/assistant/agent/evals/test_onboarding.py
new file mode 100644
index 000000000..d03ec35c3
--- /dev/null
+++ b/pkg/assistant/agent/evals/test_onboarding.py
@@ -0,0 +1,187 @@
+"""Evals for LK's onboarding behavior, from failures seen in real sessions.
+
+They run the real LLM with lk's tools replaced by stubs that reply the way lk
+does, so they need LiveKit Cloud credentials for LiveKit Inference
+(LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET). They aren't part of the
+embedded agent. Run them from the agent directory:
+
+ uv run --with pytest --with pytest-asyncio pytest evals -v
+
+LLM output varies, so run them a few times after changing the instructions.
+"""
+
+import json
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import agent # noqa: E402
+from livekit.agents import AgentSession, inference # noqa: E402
+
+pytestmark = [
+ pytest.mark.asyncio,
+ pytest.mark.skipif(
+ not all(os.environ.get(k) for k in ("LIVEKIT_URL", "LIVEKIT_API_KEY", "LIVEKIT_API_SECRET")),
+ reason="needs LiveKit Cloud credentials for LiveKit Inference",
+ ),
+]
+
+EMPTY = {"kind": "empty"}
+LIVEKIT_PROJECT = {
+ "kind": "livekit",
+ "summary": "a Python project that uses LiveKit Agents",
+ "projects": [{"path": ".", "language": "Python", "livekit_packages": [{"name": "livekit-agents", "version": "~=1.8"}]}],
+ "has_agents": True,
+}
+READY = {"ready": True}
+
+# What lk replies to each tool, close to the real text in cmd/lk.
+ALREADY_SET_UP = (
+ "Nothing was installed: LiveKit's skills and the Docs MCP server are already set up. Tell the user, "
+ 'and that "lk skills update" gets the latest skills. If they still want to reinstall, call '
+ "set_up_coding_agent again with reinstall=true."
+)
+CONFIRM = (
+ "Waiting for the user to confirm. Confirmation ID: confirm1. Command: {command}. "
+ "Ask the user in one short sentence whether to go ahead. They can answer by voice or on screen. If they agree by voice, call confirm_action "
+ "with this ID and approved=true; if they decline, approved=false."
+)
+EXISTING_AGENT = (
+ "You can only run an agent here that you created in this session. Existing agents can need their "
+ "frontend, dispatch, or other services, so they run in the user's own terminal. In one short sentence, "
+ "tell the user you can't start it here, and that they can run it in their terminal to talk to it. Then "
+ 'call show_code with title "Terminal", language "shell", and this command: lk agent console.'
+)
+
+
+def stub_lk(calls: list, ready: bool = False):
+ async def run_lk_tool(name: str, args: dict | None = None, timeout: float = 60) -> str:
+ calls.append((name, args or {}))
+ if name == "show_choice":
+ return f"choice{len(calls)}"
+ if name == "choice_made":
+ return "Marked."
+ if name in ("create_project", "set_up_coding_agent", "try_project") and (args or {}).get("confirmed"):
+ return "Done. The user confirmed by voice, and it ran."
+ if name == "create_project":
+ return CONFIRM.format(command="lk agent init . --template agent-starter-python --install")
+ if name == "set_up_coding_agent":
+ if ready and not (args or {}).get("reinstall"):
+ return ALREADY_SET_UP
+ return CONFIRM.format(command="lk skills install")
+ if name == "confirm_action":
+ return "Done. The project is in the current directory."
+ if name == "try_project":
+ return EXISTING_AGENT
+ if name == "get_project_info":
+ return json.dumps({"directory": "/Users/dev/my-app", "folder": EMPTY})
+ if name == "list_templates":
+ return "agent-starter-python: Python voice agent. agent-starter-node: Node.js voice agent."
+ return "Done."
+
+ return run_lk_tool
+
+
+@pytest.fixture
+def lk_calls(monkeypatch):
+ calls: list = []
+ monkeypatch.setattr(agent, "run_lk_tool", stub_lk(calls))
+ return calls
+
+
+@pytest.fixture
+def lk_calls_ready(monkeypatch):
+ """lk's tools when skills and the Docs MCP server are already set up."""
+ calls: list = []
+ monkeypatch.setattr(agent, "run_lk_tool", stub_lk(calls, ready=True))
+ return calls
+
+
+def assistant(monkeypatch, folder: dict, coding_agents: dict | None = None) -> agent.Assistant:
+ monkeypatch.setenv("LK_ASSISTANT_PROJECT", "/Users/dev/my-app")
+ monkeypatch.setenv("LK_ASSISTANT_FOLDER", json.dumps(folder))
+ monkeypatch.setenv("LK_ASSISTANT_CODING_AGENTS", json.dumps(coding_agents or {}))
+ return agent.Assistant()
+
+
+def tool_calls(result) -> list[str]:
+ return [e.item.name for e in result.events if e.type == "function_call"]
+
+
+def judge_llm() -> inference.LLM:
+ return inference.LLM(model=agent.LLM_MODEL)
+
+
+async def test_creates_project_without_asking_twice(monkeypatch, lk_calls):
+ """The user said what they want; LK creates it without asking again."""
+ async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session:
+ await session.start(assistant(monkeypatch, EMPTY))
+ await session.run(user_input="I want to build a voice agent in Python, right here.")
+ # Either path runs it in this turn: confirmed=true, or confirm_action right after.
+ ran = any(
+ (n == "create_project" and a.get("confirmed")) or (n == "confirm_action" and a.get("approved"))
+ for n, a in lk_calls
+ )
+ assert ran, f"asked again instead of running it: {lk_calls}"
+
+
+async def test_existing_agent_gets_command(monkeypatch, lk_calls):
+ """An existing agent can't run here; LK says so and shows the command."""
+ async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session:
+ await session.start(assistant(monkeypatch, LIVEKIT_PROJECT, READY))
+ result = await session.run(user_input="Can you start my agent so I can talk to it?")
+ result.expect.contains_function_call(name="show_code")
+ shown = [json.loads(e.item.arguments) for e in result.events if e.type == "function_call" and e.item.name == "show_code"]
+ assert any("lk agent console" in s.get("code", "") for s in shown), shown
+ await result.expect.contains_message(role="assistant").judge(
+ judge_llm(),
+ intent="Says it can't start the user's existing agent here, and doesn't claim the agent is running.",
+ )
+
+
+async def test_bug_fix_goes_to_coding_agent(monkeypatch, lk_calls):
+ """LK can't read code; a bug fix becomes a prompt for the coding agent."""
+ async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session:
+ await session.start(assistant(monkeypatch, LIVEKIT_PROJECT, READY))
+ result = await session.run(user_input="My agent crashes when the user hangs up. Can you fix it?")
+ await result.expect.contains_message(role="assistant").judge(
+ judge_llm(),
+ intent="Doesn't claim to have read or fixed the code. Either hands the task to the user's coding agent or offers to.",
+ )
+
+
+async def test_no_setup_offer_when_ready(monkeypatch, lk_calls_ready):
+ """Skills and the Docs MCP server are already installed."""
+ async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session:
+ await session.start(assistant(monkeypatch, LIVEKIT_PROJECT, READY))
+ result = await session.run(user_input="Should I install LiveKit's skills for Cursor?")
+ assert not any(a.get("reinstall") for n, a in lk_calls_ready if n == "set_up_coding_agent")
+ await result.expect.contains_message(role="assistant").judge(
+ judge_llm(), intent="Tells the user it's already set up, and doesn't offer to install it."
+ )
+
+
+async def test_docs_question_uses_docs(monkeypatch, lk_calls):
+ """Answers about LiveKit come from the docs, with the source on screen."""
+ async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session:
+ await session.start(assistant(monkeypatch, EMPTY))
+ result = await session.run(user_input="How does LiveKit decide when I've finished talking?")
+ calls = tool_calls(result)
+ assert any(c in calls for c in ("docs_search", "get_pages", "get_docs_overview")), calls
+ assert any(c in calls for c in ("show_answer", "show_link")), f"no source shown: {calls}"
+
+
+async def test_copy_targets_the_prompt(monkeypatch, lk_calls):
+ """"Copy it" after a prompt card copies the prompt, not an older card."""
+ async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session:
+ await session.start(assistant(monkeypatch, EMPTY))
+ await session.run(user_input="Show me the command to install LiveKit's skills.")
+ await session.run(user_input="Now write a prompt for Cursor to change my agent's voice to something calmer.")
+ result = await session.run(user_input="Copy the prompt.")
+ call = result.expect.contains_function_call(name="copy_to_clipboard")
+ title = json.loads(call.event().item.arguments).get("title", "")
+ assert title == "" or "prompt" in title.lower(), title
diff --git a/pkg/assistant/agent/lk_simulation.py b/pkg/assistant/agent/lk_simulation.py
new file mode 100644
index 000000000..c409d05f2
--- /dev/null
+++ b/pkg/assistant/agent/lk_simulation.py
@@ -0,0 +1,230 @@
+"""Agent Simulations support for LK. lk doesn't embed this file.
+
+Under a simulation, there's no lk to run commands, so lk's tools are mocked
+with replies like the ones lk sends (see cmd/lk/assistant_tools.go). The docs
+tools stay real. Each scenario's userdata sets the folder context:
+
+ userdata:
+ folder: {kind: empty} # as pkg/assistant/folder.go reports it
+ coding_agents: {ready: false} # as cmd/lk/assistant_skills.go reports it
+ expect:
+ calls: [create_project] # tools the agent must call
+ not_calls: [set_up_coding_agent] # tools it must not call
+ any_calls: [show_link, show_answer] # it must call at least one
+ show_code_contains: lk agent console
+ lk_calls: [create_project] # lk tools that must run, from code or the model
+ confirmed: [create_project] # ran without asking again
+ call_contains: {hand_off_task: voice}
+
+The simulation's judge doesn't see tool calls, so check_simulation checks them
+when the simulation ends, and fails it if they don't match.
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+
+import sys
+
+from livekit.agents import AgentSession
+
+DIR = "/Users/dev/my-app"
+
+
+def setup_simulation(sim, session: AgentSession, assistant_cls) -> tuple[dict, dict]:
+ userdata = sim.userdata() or {}
+ folder = userdata.get("folder", {"kind": "empty"})
+ coding_agents = userdata.get("coding_agents", {"ready": False})
+ # The agent's tools and its tasks reach lk through run_lk_tool. Replace it
+ # in the module that defines the agent, which runs as __main__. Each
+ # simulation job runs in its own process, so this is per job.
+ global mocks
+ mocks = LkMocks(folder, coding_agents)
+ sys.modules[assistant_cls.__module__].run_lk_tool = mocks.run
+ return folder, coding_agents
+
+
+# The job's mocks, so check_simulation can see which lk tools ran.
+mocks: "LkMocks | None" = None
+
+
+class LkMocks:
+ # Each mock takes the tool's RunContext first, as the real tools do: the
+ # SDK passes it positionally.
+ def __init__(self, folder: dict, coding_agents: dict) -> None:
+ self.folder = folder
+ self.coding_agents = coding_agents
+ self.pending: dict[str, str] = {} # confirmation id → what it runs
+ self.n = 0
+ self.created = False
+ self.calls: list[str] = [] # lk tools run, in order
+
+ async def run(self, name: str, args: dict | None = None, timeout: float = 60) -> str:
+ """Stands in for agent.run_lk_tool."""
+ args = args or {}
+ self.calls.append(name)
+ if name == "show_choice":
+ self.n += 1
+ return f"choice{self.n}"
+ if name == "choice_made":
+ return "Marked."
+ tool = self.tools().get(name)
+ if tool is None:
+ return f"Error: unknown tool {name!r}"
+ # Pass only the arguments this mock takes, so older agents work too.
+ params = inspect.signature(tool).parameters
+ return await tool(None, **{k: v for k, v in args.items() if k in params})
+
+ def tools(self) -> dict:
+ return {
+ "get_project_info": self.get_project_info,
+ "list_cloud_agents": self.list_cloud_agents,
+ "list_templates": self.list_templates,
+ "create_project": self.create_project,
+ "set_up_coding_agent": self.set_up_coding_agent,
+ "confirm_action": self.confirm_action,
+ "try_project": self.try_project,
+ "copy_to_clipboard": self.copy_to_clipboard,
+ }
+
+ def propose(self, what: str, command: str, confirmed: bool) -> str:
+ if confirmed:
+ return self.perform(what)
+ self.n += 1
+ cid = f"confirm{self.n}"
+ self.pending[cid] = what
+ return (
+ f"Waiting for the user to confirm. Confirmation ID: {cid}. Command: {command}. "
+ "The command is on screen, so don't say it. Ask the user in a few words whether to go ahead. They can answer by voice or on "
+ "screen. If they agree by voice, call confirm_action with this ID and approved=true; if "
+ "they decline, approved=false."
+ )
+
+ def perform(self, what: str) -> str:
+ if what.startswith("create"):
+ template = what.partition(":")[2] or "agent-starter-python"
+ self.created = True
+ self.folder = {"kind": "livekit", "has_agents": True}
+ return (
+ f"Created an agent project from the {template} template in the current "
+ "directory and installed its dependencies. It includes LiveKit's skills and Docs MCP "
+ "server for coding agents."
+ )
+ if what == "skills":
+ self.coding_agents = {"ready": True}
+ return "Installed LiveKit's skills and the Docs MCP server for Cursor and Claude Code."
+ if what == "try":
+ # In a simulation, the user's agent doesn't really run. This is what lk tells the
+ # assistant when the user comes back from it (see returnToAssistant).
+ return (
+ "The user's agent ran here, and the user talked to it. They're back now. Welcome them "
+ "back in one short sentence and ask how it went or what they'd like to change."
+ )
+ return "Done."
+
+ async def get_project_info(self, context) -> str:
+ return json.dumps(
+ {
+ "directory": DIR,
+ "folder": self.folder,
+ "coding_agents": self.coding_agents,
+ "livekit_project": {"name": "my-project", "url": "wss://my-project.livekit.cloud"},
+ }
+ )
+
+ async def list_cloud_agents(self, context) -> str:
+ return "[]"
+
+ async def list_templates(self, context) -> str:
+ return (
+ "Agent templates:\n- agent-starter-python: A complete voice AI starter for LiveKit Agents "
+ "with Python.\n- agent-starter-node: A complete voice AI starter for LiveKit Agents with "
+ "Node.js.\n\nFrontend and app templates:\n- agent-starter-react: A web voice AI frontend "
+ "built with Next.js."
+ )
+
+ async def create_project(
+ self, context, template: str, name: str = "", in_current_directory: bool = False, confirmed: bool = False
+ ) -> str:
+ target = "." if in_current_directory else (name or "my-agent")
+ return self.propose(f"create:{template}", f"lk agent init {target} --template {template} --install", confirmed)
+
+ async def set_up_coding_agent(
+ self, context, all_projects: bool = False, reinstall: bool = False, confirmed: bool = False
+ ) -> str:
+ if self.coding_agents.get("ready") and not reinstall:
+ return (
+ "Nothing was installed: LiveKit's skills and the Docs MCP server are already set up. "
+ 'Tell the user, and that "lk skills update" gets the latest skills.'
+ )
+ return self.propose("skills", "lk skills install" + (" --global" if all_projects else ""), confirmed)
+
+ async def confirm_action(
+ self, context, confirmation_id: str, approved: bool) -> str:
+ what = self.pending.pop(confirmation_id, None)
+ if what is None:
+ return f'Error: there\'s no pending confirmation "{confirmation_id}"'
+ if not approved:
+ return "Cancelled. Nothing was changed."
+ return self.perform(what)
+
+ async def try_project(
+ self, context, confirmed: bool = False) -> str:
+ if not self.created:
+ return (
+ "You can only run an agent here that you created in this session. In one short "
+ "sentence, tell the user you can't start it here, and that they can run it in their "
+ 'terminal. Then call show_code with title "Terminal", language "shell", and this '
+ "command: lk agent console"
+ )
+ return self.propose("try", "lk agent console", confirmed)
+
+ async def copy_to_clipboard(
+ self, context, title: str = "") -> str:
+ return f'Copied "{title or "the latest card"}" to the user\'s clipboard.'
+
+
+def check_simulation(ctx) -> None:
+ expect = (ctx.userdata() or {}).get("expect")
+ if not expect:
+ return
+ calls = [
+ item
+ for item in ctx.job_context.primary_session.history.items
+ if item.type == "function_call"
+ ]
+ names = [c.name for c in calls]
+ problems = []
+ for name in expect.get("calls", []):
+ if name not in names:
+ problems.append(f"didn't call {name}")
+ if any_of := expect.get("any_calls"):
+ if not any(name in names for name in any_of):
+ problems.append(f"didn't call any of {', '.join(any_of)}")
+ ran = mocks.calls if mocks else []
+ for name in expect.get("lk_calls", []):
+ if name not in ran:
+ problems.append(f"didn't run lk tool {name}")
+ for name in expect.get("not_calls", []):
+ if name in names:
+ problems.append(f"called {name}")
+ for name in expect.get("confirmed", []):
+ # Ran without asking again: confirmed=true, or confirm_action right after.
+ ran = any(
+ (c.name == name and json.loads(c.arguments or "{}").get("confirmed"))
+ or c.name == "confirm_action"
+ for c in calls
+ )
+ if not ran:
+ problems.append(f"asked again before running {name}")
+ for name, want in expect.get("call_contains", {}).items():
+ args = [c.arguments.lower() for c in calls if c.name == name]
+ if not any(want.lower() in a for a in args):
+ problems.append(f"didn't call {name} with {want!r}")
+ if want := expect.get("show_code_contains"):
+ shown = [c.arguments for c in calls if c.name == "show_code"]
+ if not any(want in s for s in shown):
+ problems.append(f"didn't show {want!r} with show_code")
+ if problems:
+ ctx.fail(reason="Tool calls: " + "; ".join(problems) + f". Called: {', '.join(names) or 'nothing'}.")
diff --git a/pkg/assistant/agent/pyproject.toml b/pkg/assistant/agent/pyproject.toml
new file mode 100644
index 000000000..906b2fdb8
--- /dev/null
+++ b/pkg/assistant/agent/pyproject.toml
@@ -0,0 +1,8 @@
+[project]
+name = "lk-assistant-agent"
+version = "0.1.0"
+description = "The voice agent behind lk assistant"
+requires-python = ">=3.10,<3.15"
+dependencies = [
+ "livekit-agents[mcp]~=1.8",
+]
diff --git a/pkg/assistant/agent/simulations/README.md b/pkg/assistant/agent/simulations/README.md
new file mode 100644
index 000000000..cba67de70
--- /dev/null
+++ b/pkg/assistant/agent/simulations/README.md
@@ -0,0 +1,31 @@
+# LK simulations
+
+These run LK, the agent behind `lk assistant`, through [Agent Simulations](https://docs.livekit.io/testing/simulations.md): an LLM plays the user, and a judge grades the conversation against each scenario's expectations.
+
+Under a simulation, `lk_simulation.py` mocks lk's tools, such as `create_project`, with replies like the ones lk sends. The docs tools stay real. Each scenario's `userdata` sets the folder context, and its `expect` block lists tool calls that `check_simulation` checks when the simulation ends, because the judge doesn't see tool calls. lk doesn't embed `lk_simulation.py` or this folder.
+
+## Run
+
+From `pkg/assistant/agent`, with a LiveKit Cloud project configured in `lk`:
+
+```shell
+uv sync
+lk agent simulate text --scenarios simulations/scenarios.yaml --samples 3 -y
+lk agent simulate audio --scenarios simulations/audio-scenarios.yaml --samples 3 -y
+```
+
+`--samples` must be odd, and at most 3.
+
+## Settings to compare
+
+Set these environment variables to compare configurations. The defaults are what lk ships.
+
+| Variable | Default | Values |
+|---|---|---|
+| `LK_ASSISTANT_LLM` | `openai/gpt-5.4-mini` | A LiveKit Inference LLM, such as `google/gemini-3.8-flash` |
+| `LK_ASSISTANT_MAX_PAGES` | `3` | How many docs pages one reply may read. Fewer is faster. |
+| `LK_ASSISTANT_DOCS_CHARS` | `12000` | How much docs text one `read_docs` call returns, split between its pages. |
+| `LK_ASSISTANT_TOC` | `1` | `1` adds the trimmed docs table of contents to the instructions, so the agent picks pages by title. |
+| `LK_ASSISTANT_REASONING` | `low` | The LLM's `reasoning_effort`, such as `low` or `medium`. Empty uses the model's default. Lower is faster. |
+
+Each run registers its own temporary agent name, so you can run two configurations at once.
diff --git a/pkg/assistant/agent/simulations/audio-scenarios.yaml b/pkg/assistant/agent/simulations/audio-scenarios.yaml
new file mode 100644
index 000000000..c673bd570
--- /dev/null
+++ b/pkg/assistant/agent/simulations/audio-scenarios.yaml
@@ -0,0 +1,35 @@
+id: lk-assistant-audio
+name: LK naturalness (audio)
+scenarios:
+ - id: audio-several-quick-questions
+ label: Several quick questions in a row, by voice
+ instructions: >-
+ You're exploring whether LiveKit fits your startup. Ask three quick questions, one at a
+ time: what LiveKit Inference is, whether you can use your own OpenAI key instead, and how you
+ would test an agent before launching. Keep your questions short.
+ agent_expectations: >-
+ The conversation sounds natural. LK answers each question directly, in one to three
+ sentences. It doesn't keep saying it's looking something up or checking the docs; saying
+ so once in a while is fine, but not before every answer. It doesn't start answers with the
+ same phrase each time.
+ tags:
+ area: naturalness
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+
+ - id: audio-specific-lookups
+ label: Questions that need a docs lookup, by voice
+ instructions: >-
+ You're building a voice agent. Ask which speech-to-text models LiveKit Inference offers,
+ then ask how to deploy your agent to LiveKit Cloud, then ask how to see your deployed
+ agent's logs. Keep your questions short.
+ agent_expectations: >-
+ The conversation sounds natural. LK may say briefly that it's checking when a lookup takes a
+ while, but it doesn't say so before every answer, and it doesn't repeat the same line. It
+ answers each question in one to three sentences, with no long silences.
+ tags:
+ area: naturalness
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
diff --git a/pkg/assistant/agent/simulations/scenarios.yaml b/pkg/assistant/agent/simulations/scenarios.yaml
new file mode 100644
index 000000000..22ccfda92
--- /dev/null
+++ b/pkg/assistant/agent/simulations/scenarios.yaml
@@ -0,0 +1,290 @@
+id: lk-assistant-onboarding
+name: LK onboarding
+scenarios:
+ - id: first-agent-empty-folder
+ label: New developer builds a first agent in an empty folder
+ instructions: >-
+ You're a web developer who has never built a voice agent. You're in an empty folder and
+ want a voice agent that answers questions about your bakery's hours. Ask what you need to
+ do, agree when LK offers to create the project here, and then agree to try it. Speak
+ casually, in short sentences.
+ agent_expectations: >-
+ LK tells the user it's creating a starter project in the current folder, and offers to let
+ them try it. In this simulation the user's agent doesn't really run: when the user agrees to
+ try it, LK is told that the user talked to it and came back, and LK welcomes them back. When the user agrees to something, LK goes ahead without asking the same
+ question again. Replies are one to three short spoken sentences, with no lists. Backticks
+ around commands and names are fine. Don't grade tool calls; they're checked separately.
+ tags:
+ area: getting-started
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+ expect:
+ lk_calls: [create_project, try_project]
+
+ - id: realtime-vs-cascaded
+ label: Choosing between a realtime model and the cascaded pipeline
+ instructions: >-
+ You're a backend engineer building a phone receptionist for a dental clinic. Ask whether you
+ should use a realtime speech-to-speech model or a separate STT, LLM, and TTS pipeline, and
+ why. Push back once: say realtime sounds faster, so why not use it? You want a clear
+ recommendation, not a list of options.
+ agent_expectations: >-
+ LK gives a clear recommendation with a reason, like an experienced LiveKit engineer.
+ It recommends the STT-LLM-TTS (cascaded) pipeline as the default for a receptionist, and
+ explains the tradeoff: realtime models are faster, but the cascaded pipeline has more mature
+ tool calling, transcripts, and control. It stays accurate when the user pushes back. It
+ doesn't hedge, ramble, or announce that it's looking things up.
+ tags:
+ area: expertise
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+ expect:
+ call_contains: {read_docs: pipelines}
+
+ - id: how-agents-join-rooms
+ label: Frontend developer asks how the agent and the app connect
+ instructions: >-
+ You're a frontend developer with a Next.js app. Ask how a LiveKit voice agent and your web
+ app actually connect: where the agent runs and how it ends up talking to your user. Ask one
+ follow-up about where the token comes from.
+ agent_expectations: >-
+ LK explains it accurately and simply: the app gets an access token from a server endpoint
+ and joins a room, and the agent is dispatched to that room. It explains that the token is
+ made on a server with the API key and secret, not in the browser. It's confident and
+ concrete, with no hedging, in short spoken replies.
+ tags:
+ area: expertise
+ userdata:
+ folder:
+ kind: app
+ summary: a Next.js app
+ projects: [{path: ".", language: TypeScript, framework: Next.js}]
+ coding_agents: {ready: false}
+
+ - id: phone-calls
+ label: Can my agent answer phone calls?
+ instructions: >-
+ You run a small plumbing business. Ask if a LiveKit agent can answer your business phone
+ line, and what you'd need to set that up. You don't know what SIP is.
+ agent_expectations: >-
+ LK says yes, and explains the path in plain words: a phone number from LiveKit Phone Numbers
+ or the user's own SIP provider, routed to the agent. It explains SIP in a few words if it
+ mentions it. It doesn't recite setup steps or overwhelm the user. Don't grade tool calls;
+ they're checked separately.
+ tags:
+ area: expertise
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+ expect:
+ any_calls: [show_link, show_answer]
+
+ - id: latency-expert
+ label: Experienced developer asks about turn-taking latency
+ instructions: >-
+ You're an experienced developer who has built voice agents with other frameworks. Ask, tersely,
+ how LiveKit decides when the user has finished speaking, and what you can tune if replies
+ feel slow. You dislike long explanations; say so if LK is wordy.
+ agent_expectations: >-
+ LK is terse and precise for an experienced user. It names LiveKit's turn detector model as
+ the default, says it works on top of VAD using the meaning and sound of speech, and names
+ what can be tuned, such as endpointing delay. It's accurate and doesn't pad its answers.
+ tags:
+ area: expertise
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+
+ - id: existing-project-run-agent
+ label: Existing LiveKit project, the user wants to run their agent
+ instructions: >-
+ You already have a LiveKit agent in this folder. Ask LK to start your agent so you can talk
+ to it. If LK says it can't, ask how you can.
+ agent_expectations: >-
+ LK says briefly that it can't start an existing agent here, and that the user can run it in
+ their own terminal, where a command is shown on screen. It doesn't pretend to start the agent
+ or speak as the user's agent.
+ tags:
+ area: scope
+ userdata:
+ folder:
+ kind: livekit
+ summary: a Python project that uses LiveKit Agents
+ projects: [{path: ".", language: Python, livekit_packages: [{name: livekit-agents, version: "~=1.8"}]}]
+ has_agents: true
+ coding_agents: {ready: true}
+ expect:
+ show_code_contains: lk agent console
+
+ - id: fix-my-bug
+ label: The user asks LK to fix a bug in their code
+ instructions: >-
+ You have a LiveKit agent in this folder, and it crashes when a caller hangs up. Ask LK to fix
+ it. If it suggests handing it to your coding agent, agree. You can see the cards LK shows on screen,
+ such as prompts for your coding agent, so you don't need LK to repeat them.
+ agent_expectations: >-
+ LK doesn't claim to have read or fixed the code. It's brief. The judge can't see the
+ prompt card LK shows, so don't grade whether a prompt was shown or what it says; code
+ checks that separately.
+ tags:
+ area: scope
+ userdata:
+ folder:
+ kind: livekit
+ summary: a Python project that uses LiveKit Agents
+ projects: [{path: ".", language: Python, livekit_packages: [{name: livekit-agents, version: "~=1.8"}]}]
+ has_agents: true
+ coding_agents: {ready: true}
+ expect:
+ calls: [hand_off_task]
+
+ - id: several-quick-questions
+ label: A curious user asks several quick questions in a row
+ instructions: >-
+ You're exploring whether LiveKit fits your startup. Ask three quick questions, one at a
+ time: what LiveKit Inference is, whether you can use your own OpenAI key instead, and how you
+ would test an agent before launching. Keep your questions short.
+ agent_expectations: >-
+ LK answers each question directly and accurately, in one to three sentences, and sounds like
+ an expert. It doesn't repeat the same phrases from answer to answer, such as saying it's
+ looking something up or the same opening line each time. It mentions that plugins let the
+ user bring their own provider keys, and names a testing option such as Agent Simulations or
+ unit tests.
+ tags:
+ area: naturalness
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+
+ - id: direct-request
+ label: The user asks LK to set up an agent right away
+ instructions: >-
+ You know what you want. Say, in one short sentence, to set up a voice agent here. If LK asks
+ which language, say Python. If LK asks you to confirm anything else, say yes, but note it.
+ agent_expectations: >-
+ LK asks at most one short question before it creates the project, such as Python or
+ Node.js. After the user answers, it creates the project without asking again. It doesn't
+ read the command aloud. Its replies are short. Don't grade tool calls; they're checked
+ separately.
+ tags:
+ area: naturalness
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+ expect:
+ lk_calls: [create_project]
+
+ - id: change-voice
+ label: The user wants a different voice for the agent they just made
+ instructions: >-
+ You just created a starter voice agent here and talked to it. Say you want to change its
+ voice to something warmer. Answer LK's questions briefly, and accept the first option it
+ suggests. You can see the cards LK shows on screen,
+ such as prompts for your coding agent, so you don't need LK to repeat them.
+ agent_expectations: >-
+ LK acts as the expert. It suggests one or two concrete voice options, or asks one short
+ question about the voice the user wants, and doesn't ask whether to write the change up.
+ It doesn't end its replies with offers such as "If you want, I can". It's brief. The judge
+ can't see the prompt card LK shows, so don't grade whether a prompt was shown or what it
+ says; code checks that separately.
+ tags:
+ area: expertise
+ userdata:
+ folder:
+ kind: livekit
+ summary: a Python project that uses LiveKit Agents
+ projects: [{path: ".", language: Python, livekit_packages: [{name: livekit-agents, version: "~=1.8"}]}]
+ has_agents: true
+ coding_agents: {ready: true}
+ expect:
+ calls: [hand_off_task]
+ call_contains: {hand_off_task: voice}
+
+ - id: exit
+ label: The user asks a question, then says exit
+ instructions: >-
+ Ask LK one quick question: what LiveKit Inference is. After it answers, say only "Exit."
+ agent_expectations: >-
+ LK answers the question, then ends the conversation when the user says exit, with at most
+ a short goodbye. Don't grade tool calls; they're checked separately.
+ tags:
+ area: naturalness
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+ expect:
+ calls: [end_call]
+
+ - id: mental-model
+ label: The user asks for a mental model of how LiveKit agents work
+ instructions: >-
+ You're new to LiveKit. Ask what the architecture of the framework is, or what mental model
+ you should have. You can see cards LK shows on screen.
+ agent_expectations: >-
+ LK gives a concrete, accurate mental model by walking through what happens: the user's app
+ joins a room, the agent is sent to that room, and the agent listens, detects the end of the
+ turn, thinks, and speaks. LiveKit terms such as room, dispatch, or turn detector are fine
+ when explained in a few words. It avoids vague abstractions such as "realtime bus" or
+ "orchestration". Its spoken reply is short, at most five sentences, and
+ doesn't end with an offer such as "If you want, I can". Don't grade tool calls; they're
+ checked separately.
+ tags:
+ area: expertise
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+
+ - id: confirm-before-create
+ label: The user asks to build an agent, naming a language
+ instructions: >-
+ Ask, in one sentence, whether LK can start building a hotel reception agent here in Python.
+ If LK asks you to confirm, say yes. If LK says it created a project without asking you
+ first, say you didn't want it yet.
+ agent_expectations: >-
+ Before it creates anything, LK asks the user to confirm, in one short question. It creates
+ the project only after the user says yes. It doesn't pick a different language than the user
+ asked for. Don't grade tool calls; they're checked separately.
+ tags:
+ area: getting-started
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+ expect:
+ lk_calls: [create_project]
+
+ - id: unsure-language
+ label: The user is unsure which language, then picks Python and tries it
+ instructions: >-
+ Ask LK to build a voice agent in this folder. When LK asks Python or Node.js, ask "Uh, what
+ do I choose?" After LK explains, say "Python." When LK asks whether to talk to it, say
+ "Let's chat."
+ agent_expectations: >-
+ LK helps the user choose without choosing for them. After the user says Python, LK tells the
+ user it's creating the agent, then says it's ready and asks whether to talk to it. It never
+ says a step happened before it did. Don't grade tool calls; they're checked separately.
+ tags:
+ area: getting-started
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+ expect:
+ lk_calls: [create_project, try_project]
+
+ - id: show-example
+ label: The user asks for an example of a feature
+ instructions: >-
+ Ask how LiveKit handles turn-taking. After LK answers, say "Show me an example of the
+ adaptive interruption." Then say "Copy."
+ agent_expectations: >-
+ LK shows a code example for adaptive interruption handling on screen and doesn't read code
+ aloud. When the user says copy, LK says what it copied. Don't grade tool calls; they're
+ checked separately.
+ tags:
+ area: expertise
+ userdata:
+ folder: {kind: empty}
+ coding_agents: {ready: false}
+ expect:
+ calls: [show_code]
diff --git a/pkg/assistant/assistant.go b/pkg/assistant/assistant.go
new file mode 100644
index 000000000..8c011bf37
--- /dev/null
+++ b/pkg/assistant/assistant.go
@@ -0,0 +1,117 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package assistant holds the voice agent behind `lk assistant`. The agent's
+// Python project is embedded in the CLI and installed into a cache directory
+// the first time it runs.
+package assistant
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "embed"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "os/exec"
+ "path/filepath"
+)
+
+//go:embed agent/agent.py agent/pyproject.toml
+var agentFiles embed.FS
+
+// Entrypoint is the agent's entrypoint file, relative to its project directory.
+const Entrypoint = "agent.py"
+
+// Names of agent tools that lk acts on.
+const (
+ // ShowCodeTool shows code as a card in the overlay.
+ ShowCodeTool = "show_code"
+ // HandOffTaskTool shows a prompt for the user's coding agent as a card.
+ HandOffTaskTool = "hand_off_task"
+ // ShowAnswerTool shows a docs answer as a card of short points.
+ ShowAnswerTool = "show_answer"
+ // ShowLinkTool shows a link to a docs page as a card.
+ ShowLinkTool = "show_link"
+ // EndCallTool ends the conversation. lk closes the overlay after it.
+ EndCallTool = "end_call"
+)
+
+// ErrNoUV means uv, which runs the agent, isn't installed.
+var ErrNoUV = errors.New("lk assistant runs its agent with uv. Install uv (https://docs.astral.sh/uv/) and try again")
+
+// stampFile records the hash of the agent files that were last installed.
+const stampFile = ".lk-installed"
+
+// DefaultDir is where the agent project is installed.
+func DefaultDir() (string, error) {
+ cache, err := os.UserCacheDir()
+ if err != nil {
+ return "", err
+ }
+ return filepath.Join(cache, "livekit", "lk-assistant-agent"), nil
+}
+
+// Prepare writes the agent project to dir and installs its dependencies with
+// uv. Dependencies are installed again only when the agent files change.
+// status is called with a short description before slow steps.
+func Prepare(ctx context.Context, dir string, status func(string)) error {
+ uv, err := exec.LookPath("uv")
+ if err != nil {
+ return ErrNoUV
+ }
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ return err
+ }
+
+ sum := sha256.New()
+ err = fs.WalkDir(agentFiles, "agent", func(path string, d fs.DirEntry, err error) error {
+ if err != nil || d.IsDir() {
+ return err
+ }
+ data, err := agentFiles.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ sum.Write([]byte(path))
+ sum.Write(data)
+ dst := filepath.Join(dir, filepath.Base(path))
+ if existing, err := os.ReadFile(dst); err == nil && bytes.Equal(existing, data) {
+ return nil
+ }
+ return os.WriteFile(dst, data, 0o644)
+ })
+ if err != nil {
+ return fmt.Errorf("writing the assistant agent: %w", err)
+ }
+ hash := hex.EncodeToString(sum.Sum(nil))
+
+ stamp := filepath.Join(dir, stampFile)
+ if installed, err := os.ReadFile(stamp); err == nil && string(installed) == hash {
+ if _, err := os.Stat(filepath.Join(dir, ".venv")); err == nil {
+ return nil
+ }
+ }
+
+ status("Installing the assistant agent")
+ cmd := exec.CommandContext(ctx, uv, "sync")
+ cmd.Dir = dir
+ if out, err := cmd.CombinedOutput(); err != nil {
+ return fmt.Errorf("installing the assistant agent failed: %w\n%s", err, out)
+ }
+ return os.WriteFile(stamp, []byte(hash), 0o644)
+}
diff --git a/pkg/assistant/folder.go b/pkg/assistant/folder.go
new file mode 100644
index 000000000..65734813f
--- /dev/null
+++ b/pkg/assistant/folder.go
@@ -0,0 +1,477 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package assistant
+
+import (
+ "encoding/json"
+ "fmt"
+ "maps"
+ "os"
+ "path/filepath"
+ "regexp"
+ "slices"
+ "strings"
+)
+
+// FolderKind is what the user's current folder holds.
+type FolderKind string
+
+const (
+ FolderEmpty FolderKind = "empty" // nothing in it
+ FolderLiveKit FolderKind = "livekit" // a project that uses LiveKit
+ FolderApp FolderKind = "app" // a project that doesn't use LiveKit yet
+ FolderOther FolderKind = "other" // no project found
+)
+
+// Folder describes the user's current folder, so the assistant can greet them
+// with what's in it. It reads only project manifests, such as package.json and
+// go.mod, in the folder and its direct subfolders.
+type Folder struct {
+ Kind FolderKind `json:"kind"`
+ Summary string `json:"summary,omitempty"` // spoken, such as "a Go project that uses the LiveKit Go server SDK"
+ Projects []Project `json:"projects,omitempty"`
+ // HasLiveKitTOML is true when the folder has a livekit.toml, which a
+ // LiveKit Cloud agent deployment uses.
+ HasLiveKitTOML bool `json:"has_livekit_toml,omitempty"`
+ // HasAgents is true when one or two projects use LiveKit Agents.
+ HasAgents bool `json:"has_agents,omitempty"`
+}
+
+// Project is one project in the folder.
+type Project struct {
+ Path string `json:"path"` // relative to the folder; "." is the folder itself
+ Language string `json:"language"`
+ Framework string `json:"framework,omitempty"`
+ LiveKit []Package `json:"livekit_packages,omitempty"`
+}
+
+// Package is a LiveKit package that a project depends on.
+type Package struct {
+ Name string `json:"name"`
+ Version string `json:"version,omitempty"` // as the manifest gives it, such as "^2.1.0" or "~=1.8"
+}
+
+// usesAgents reports whether p uses LiveKit Agents.
+func (p Project) usesAgents() bool {
+ return slices.ContainsFunc(p.LiveKit, func(pkg Package) bool {
+ return pkg.Name == "livekit-agents" || pkg.Name == "@livekit/agents"
+ })
+}
+
+// AgentPaths returns the paths of the projects that use LiveKit Agents.
+func (f Folder) AgentPaths() []string {
+ var paths []string
+ for _, p := range f.Projects {
+ if p.usesAgents() {
+ paths = append(paths, p.Path)
+ }
+ }
+ return paths
+}
+
+// hasAgents reports whether one or two projects use LiveKit Agents. With
+// more, it isn't clear which agent the user means.
+func (f Folder) hasAgents() bool {
+ n := 0
+ for _, p := range f.Projects {
+ if p.usesAgents() {
+ n++
+ }
+ }
+ return n > 0 && n <= 2
+}
+
+// JSON returns f as JSON for the agent.
+func (f Folder) JSON() string {
+ b, _ := json.Marshal(f)
+ return string(b)
+}
+
+// skipDirs are subfolders that never hold the user's own projects.
+var skipDirs = map[string]bool{
+ "node_modules": true, "vendor": true, "venv": true, "env": true, "dist": true,
+ "build": true, "target": true, "out": true, "__pycache__": true, "Pods": true,
+}
+
+// maxDirs limits how many folders DescribeFolder reads, so a large folder,
+// such as the home folder, stays fast.
+const maxDirs = 200
+
+// DescribeFolder describes dir: whether it's empty, and the projects in it,
+// in its subfolders, and in their subfolders. The second level finds the
+// projects in a monorepo, such as apps/web and apps/agent. It doesn't look
+// inside a project's own folders.
+func DescribeFolder(dir string) Folder {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return Folder{Kind: FolderOther}
+ }
+ empty := true
+ for _, e := range entries {
+ if e.Name() != ".DS_Store" {
+ empty = false
+ break
+ }
+ }
+ if empty {
+ return Folder{Kind: FolderEmpty}
+ }
+
+ var f Folder
+ root, rootFound := detectProject(dir)
+ visited := 0
+ var scan func(rel string, entries []os.DirEntry, depth int)
+ scan = func(rel string, entries []os.DirEntry, depth int) {
+ for _, e := range entries {
+ name := e.Name()
+ if !e.IsDir() || strings.HasPrefix(name, ".") || skipDirs[name] {
+ continue
+ }
+ if visited++; visited > maxDirs {
+ return
+ }
+ path := filepath.Join(rel, name)
+ if p, ok := detectProject(filepath.Join(dir, path)); ok {
+ p.Path = filepath.ToSlash(path)
+ f.Projects = append(f.Projects, p)
+ } else if depth < 2 {
+ if sub, err := os.ReadDir(filepath.Join(dir, path)); err == nil {
+ scan(path, sub, depth+1)
+ }
+ }
+ }
+ }
+ scan("", entries, 1)
+ // A workspace root, such as a pnpm or npm workspace, only ties its
+ // projects together, so it isn't a project of its own.
+ if rootFound && (len(f.Projects) == 0 || !isWorkspaceRoot(dir)) {
+ root.Path = "."
+ f.Projects = append([]Project{root}, f.Projects...)
+ }
+ if _, err := os.Stat(filepath.Join(dir, "livekit.toml")); err == nil {
+ f.HasLiveKitTOML = true
+ }
+
+ switch {
+ case f.HasLiveKitTOML || slices.ContainsFunc(f.Projects, func(p Project) bool { return len(p.LiveKit) > 0 }):
+ f.Kind = FolderLiveKit
+ case len(f.Projects) > 0:
+ f.Kind = FolderApp
+ default:
+ f.Kind = FolderOther
+ }
+ f.Summary = summarize(f.Projects)
+ f.HasAgents = f.hasAgents()
+ return f
+}
+
+// isWorkspaceRoot reports whether dir is the root of a JavaScript or Python
+// workspace.
+func isWorkspaceRoot(dir string) bool {
+ if _, err := os.Stat(filepath.Join(dir, "pnpm-workspace.yaml")); err == nil {
+ return true
+ }
+ if b, err := os.ReadFile(filepath.Join(dir, "package.json")); err == nil {
+ var pkg struct {
+ Workspaces json.RawMessage `json:"workspaces"`
+ }
+ if json.Unmarshal(b, &pkg) == nil && len(pkg.Workspaces) > 0 {
+ return true
+ }
+ }
+ b, _ := os.ReadFile(filepath.Join(dir, "pyproject.toml"))
+ return strings.Contains(string(b), "[tool.uv.workspace]")
+}
+
+// detectProject finds the project in dir from its manifest files.
+func detectProject(dir string) (Project, bool) {
+ read := func(name string) string {
+ b, err := os.ReadFile(filepath.Join(dir, name))
+ if err != nil {
+ return ""
+ }
+ return string(b)
+ }
+ exists := func(name string) bool {
+ _, err := os.Stat(filepath.Join(dir, name))
+ return err == nil
+ }
+
+ var p Project
+ found := false
+ if text := read("go.mod"); text != "" {
+ found = true
+ p.Language = "Go"
+ p.LiveKit = append(p.LiveKit, goPackages(text)...)
+ }
+ if text := read("package.json"); text != "" {
+ found = true
+ p.Language = "JavaScript"
+ if exists("tsconfig.json") {
+ p.Language = "TypeScript"
+ }
+ framework, pkgs := nodePackages(text)
+ p.Framework = framework
+ p.LiveKit = append(p.LiveKit, pkgs...)
+ }
+ if text := read("pyproject.toml") + "\n" + read("requirements.txt"); strings.TrimSpace(text) != "" {
+ found = true
+ if p.Language == "" {
+ p.Language = "Python"
+ }
+ lower := strings.ToLower(text)
+ for _, fw := range pythonFrameworks {
+ if p.Framework == "" && strings.Contains(lower, fw.dep) {
+ p.Framework = fw.name
+ }
+ }
+ p.LiveKit = append(p.LiveKit, pythonPackages(nameLineRe.ReplaceAllString(lower, ""))...)
+ }
+ if text := read("Cargo.toml"); text != "" {
+ found = true
+ p.Language = "Rust"
+ p.LiveKit = append(p.LiveKit, rustPackages(text)...)
+ }
+ if text := read("Package.swift"); text != "" {
+ found = true
+ p.Language = "Swift"
+ if m := swiftRe.FindStringSubmatch(text); m != nil {
+ p.LiveKit = append(p.LiveKit, Package{Name: "client-sdk-swift", Version: m[1]})
+ }
+ }
+ if text := read("pubspec.yaml"); text != "" {
+ found = true
+ p.Language = "Dart"
+ if strings.Contains(text, "sdk: flutter") {
+ p.Framework = "Flutter"
+ }
+ if m := pubspecRe.FindStringSubmatch(text); m != nil {
+ p.LiveKit = append(p.LiveKit, Package{Name: "livekit_client", Version: strings.Trim(m[1], `"'`)})
+ }
+ }
+ for _, name := range []string{"build.gradle.kts", "build.gradle", "app/build.gradle.kts", "app/build.gradle", "gradle/libs.versions.toml"} {
+ if text := read(name); text != "" {
+ found = true
+ p.Language = "Android"
+ if m := gradleRe.FindStringSubmatch(text); m != nil && len(p.LiveKit) == 0 {
+ p.LiveKit = append(p.LiveKit, Package{Name: "io.livekit:livekit-android", Version: m[1]})
+ }
+ }
+ }
+ return p, found
+}
+
+// appFrameworks maps package.json dependencies to the frameworks they mean,
+// most specific first.
+var appFrameworks = []struct{ dep, name string }{
+ {"next", "Next.js"},
+ {"expo", "Expo"},
+ {"react-native", "React Native"},
+ {"@sveltejs/kit", "SvelteKit"},
+ {"nuxt", "Nuxt"},
+ {"@angular/core", "Angular"},
+ {"svelte", "Svelte"},
+ {"vue", "Vue"},
+ {"react", "React"},
+ {"express", "Express"},
+}
+
+// pythonFrameworks are Python web frameworks found in project files.
+var pythonFrameworks = []struct{ dep, name string }{
+ {"fastapi", "FastAPI"},
+ {"django", "Django"},
+ {"flask", "Flask"},
+}
+
+func nodePackages(text string) (string, []Package) {
+ var pkg struct {
+ Dependencies map[string]string `json:"dependencies"`
+ DevDependencies map[string]string `json:"devDependencies"`
+ }
+ if json.Unmarshal([]byte(text), &pkg) != nil {
+ return "", nil
+ }
+ deps := map[string]string{}
+ maps.Copy(deps, pkg.DevDependencies)
+ maps.Copy(deps, pkg.Dependencies)
+ framework := ""
+ for _, f := range appFrameworks {
+ if _, ok := deps[f.dep]; ok {
+ framework = f.name
+ break
+ }
+ }
+ var pkgs []Package
+ for d, v := range deps {
+ if strings.HasPrefix(d, "@livekit/") || strings.HasPrefix(d, "livekit-") {
+ pkgs = append(pkgs, Package{Name: d, Version: v})
+ }
+ }
+ slices.SortFunc(pkgs, func(a, b Package) int { return strings.Compare(a.Name, b.Name) })
+ return framework, pkgs
+}
+
+var (
+ goRe = regexp.MustCompile(`(?m)^\s*(?:require\s+)?(github\.com/livekit/[\w./-]+)\s+(v[\w.+-]+)\s*$`)
+ pythonRe = regexp.MustCompile(`(?m)(?:^|["'\s])(livekit(?:-[a-z0-9]+)*)(?:\[[^\]]*\])?\s*((?:[~=<>!]=?|==)\s*[0-9][\w.*]*)?`)
+ rustRe = regexp.MustCompile(`(?m)^\s*(livekit(?:-[a-z]+)*)\s*=\s*(?:"([^"]+)"|\{[^}]*version\s*=\s*"([^"]+)")`)
+ swiftRe = regexp.MustCompile(`livekit/client-sdk-swift[^)]*?(?:from:\s*|exact:\s*)"([^"]+)"`)
+ // nameLineRe matches pyproject.toml's name and description, which can
+ // mention LiveKit without depending on it.
+ nameLineRe = regexp.MustCompile(`(?m)^\s*(?:name|description)\s*=.*$`)
+ pubspecRe = regexp.MustCompile(`(?m)^\s*livekit_client:\s*(\S+)`)
+ gradleRe = regexp.MustCompile(`io\.livekit:livekit-android(?::([\w.-]+))?`)
+)
+
+func goPackages(text string) []Package {
+ var pkgs []Package
+ for line := range strings.SplitSeq(text, "\n") {
+ if strings.Contains(line, "// indirect") {
+ continue
+ }
+ if m := goRe.FindStringSubmatch(line); m != nil {
+ pkgs = append(pkgs, Package{Name: m[1], Version: m[2]})
+ }
+ }
+ return pkgs
+}
+
+func pythonPackages(text string) []Package {
+ var pkgs []Package
+ seen := map[string]bool{}
+ for _, m := range pythonRe.FindAllStringSubmatch(text, -1) {
+ if seen[m[1]] {
+ continue
+ }
+ seen[m[1]] = true
+ pkgs = append(pkgs, Package{Name: m[1], Version: strings.ReplaceAll(m[2], " ", "")})
+ }
+ return pkgs
+}
+
+func rustPackages(text string) []Package {
+ var pkgs []Package
+ for _, m := range rustRe.FindAllStringSubmatch(text, -1) {
+ pkgs = append(pkgs, Package{Name: m[1], Version: m[2] + m[3]})
+ }
+ return pkgs
+}
+
+// packageNames are the spoken names of the main LiveKit packages. Others,
+// such as plugins, aren't named in the summary.
+var packageNames = map[string]string{
+ "livekit-agents": "LiveKit Agents",
+ "@livekit/agents": "LiveKit Agents",
+ "livekit-api": "the LiveKit server API",
+ "livekit-server-sdk": "the LiveKit server SDK",
+ "livekit-client": "the LiveKit client SDK",
+ "@livekit/components-react": "LiveKit React components",
+ "@livekit/react-native": "the LiveKit React Native SDK",
+ "github.com/livekit/server-sdk-go": "the LiveKit Go server SDK",
+ "github.com/livekit/server-sdk-go/v2": "the LiveKit Go server SDK",
+ "client-sdk-swift": "the LiveKit Swift SDK",
+ "livekit_client": "the LiveKit Flutter SDK",
+ "io.livekit:livekit-android": "the LiveKit Android SDK",
+}
+
+// summarize describes projects in a short phrase for speech, such as "a
+// Python project in agent that uses LiveKit Agents and a Next.js app in web
+// that uses the LiveKit client SDK". It names up to two projects that use
+// LiveKit, agents first, and counts the others.
+func summarize(projects []Project) string {
+ if len(projects) == 0 {
+ return ""
+ }
+ var named, others []Project
+ for _, p := range projects {
+ if len(p.LiveKit) > 0 {
+ named = append(named, p)
+ } else {
+ others = append(others, p)
+ }
+ }
+ if len(named) == 0 && len(others) <= 2 {
+ named, others = others, nil
+ }
+ if len(named) > 2 {
+ return fmt.Sprintf("%d projects, and %d of them use LiveKit", len(projects), len(named))
+ }
+ if len(named) == 0 {
+ return fmt.Sprintf("%d projects", len(projects))
+ }
+ slices.SortStableFunc(named, func(a, b Project) int {
+ switch {
+ case a.usesAgents() == b.usesAgents():
+ return 0
+ case a.usesAgents():
+ return -1
+ default:
+ return 1
+ }
+ })
+
+ var parts []string
+ for _, p := range named {
+ parts = append(parts, describe(p))
+ }
+ s := strings.Join(parts, " and ")
+ switch len(others) {
+ case 0:
+ case 1:
+ s += ", and one other project"
+ default:
+ s += fmt.Sprintf(", and %d other projects", len(others))
+ }
+ return s
+}
+
+// describe describes one project for speech.
+func describe(p Project) string {
+ what := p.Language + " project"
+ if p.Framework != "" {
+ what = p.Framework + " app"
+ }
+ s := article(what) + " " + what
+ if p.Path != "." {
+ s += " in " + p.Path
+ }
+ var names []string
+ for _, pkg := range p.LiveKit {
+ name := packageNames[pkg.Name]
+ if name == "" && strings.HasPrefix(pkg.Name, "github.com/livekit/server-sdk-go") {
+ name = packageNames["github.com/livekit/server-sdk-go"]
+ }
+ if name != "" && !slices.Contains(names, name) {
+ names = append(names, name)
+ }
+ }
+ if len(names) > 2 {
+ names = names[:2]
+ }
+ switch {
+ case len(names) > 0:
+ s += " that uses " + strings.Join(names, " and ")
+ case len(p.LiveKit) > 0:
+ s += " that uses LiveKit"
+ }
+ return s
+}
+
+func article(word string) string {
+ if strings.ContainsRune("AEIOU", rune(word[0])) {
+ return "an"
+ }
+ return "a"
+}
diff --git a/pkg/assistant/folder_test.go b/pkg/assistant/folder_test.go
new file mode 100644
index 000000000..8ceda5511
--- /dev/null
+++ b/pkg/assistant/folder_test.go
@@ -0,0 +1,138 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package assistant
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func writeFiles(t *testing.T, files map[string]string) string {
+ t.Helper()
+ dir := t.TempDir()
+ for name, content := range files {
+ path := filepath.Join(dir, name)
+ require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
+ require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
+ }
+ return dir
+}
+
+func TestDescribeFolderEmpty(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{".DS_Store": ""}))
+ require.Equal(t, FolderEmpty, f.Kind)
+}
+
+func TestDescribeFolderOther(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{"notes.txt": "hi"}))
+ require.Equal(t, FolderOther, f.Kind)
+ require.Empty(t, f.Summary)
+}
+
+func TestDescribeFolderGo(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{"go.mod": `module example.com/app
+
+go 1.24
+
+require (
+ github.com/livekit/protocol v1.39.0 // indirect
+ github.com/livekit/server-sdk-go/v2 v2.9.1
+)
+`}))
+ require.Equal(t, FolderLiveKit, f.Kind)
+ require.Equal(t, []Package{{Name: "github.com/livekit/server-sdk-go/v2", Version: "v2.9.1"}}, f.Projects[0].LiveKit)
+ require.Equal(t, "a Go project that uses the LiveKit Go server SDK", f.Summary)
+}
+
+func TestDescribeFolderPythonAgent(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{"pyproject.toml": `[project]
+name = "livekit-agent-starter"
+dependencies = [
+ "livekit-agents[silero,turn-detector]~=1.8",
+ "livekit-plugins-noise-cancellation~=0.2",
+]
+`}))
+ require.Equal(t, FolderLiveKit, f.Kind)
+ require.True(t, f.HasAgents)
+ require.Equal(t, []Package{
+ {Name: "livekit-agents", Version: "~=1.8"},
+ {Name: "livekit-plugins-noise-cancellation", Version: "~=0.2"},
+ }, f.Projects[0].LiveKit)
+ require.Equal(t, "a Python project that uses LiveKit Agents", f.Summary)
+}
+
+func TestDescribeFolderAppWithoutLiveKit(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{
+ "package.json": `{"dependencies": {"next": "15.0.0", "react": "19.0.0"}}`,
+ "tsconfig.json": "{}",
+ }))
+ require.Equal(t, FolderApp, f.Kind)
+ require.Equal(t, "TypeScript", f.Projects[0].Language)
+ require.Equal(t, "a Next.js app", f.Summary)
+}
+
+func TestDescribeFolderSubfolders(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{
+ "README.md": "",
+ "frontend/package.json": `{"dependencies": {"next": "15.0.0", "livekit-client": "^2.15.0", "@livekit/components-react": "^2.9.0"}}`,
+ "agent/requirements.txt": "livekit-agents==1.8.5\n",
+ "node_modules/x/package.json": `{}`,
+ }))
+ require.Equal(t, FolderLiveKit, f.Kind)
+ require.Len(t, f.Projects, 2)
+ require.Equal(t, "a Python project in agent that uses LiveKit Agents and a Next.js app in frontend that uses LiveKit React components and the LiveKit client SDK", f.Summary)
+ require.True(t, f.HasAgents)
+}
+
+func TestDescribeFolderManyProjects(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{
+ "a/go.mod": "module a\n",
+ "b/requirements.txt": "livekit-agents~=1.8\n",
+ "c/package.json": `{"dependencies": {"livekit-client": "^2.15.0"}}`,
+ }))
+ require.Equal(t, FolderLiveKit, f.Kind)
+ require.True(t, f.HasAgents)
+ require.Equal(t, "a Python project in b that uses LiveKit Agents and a JavaScript project in c that uses the LiveKit client SDK, and one other project", f.Summary)
+}
+
+func TestDescribeFolderManyLiveKitProjects(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{
+ "a/requirements.txt": "livekit-agents~=1.8\n",
+ "b/requirements.txt": "livekit-agents~=1.8\n",
+ "c/requirements.txt": "livekit-agents~=1.8\n",
+ "d/go.mod": "module d\n",
+ }))
+ require.False(t, f.HasAgents)
+ require.Equal(t, "4 projects, and 3 of them use LiveKit", f.Summary)
+}
+
+func TestDescribeFolderMonorepo(t *testing.T) {
+ f := DescribeFolder(writeFiles(t, map[string]string{
+ "package.json": `{"name": "pets", "private": true, "scripts": {"dev": "pnpm --filter web dev"}}`,
+ "pnpm-workspace.yaml": "packages:\n - 'apps/web'\n - 'packages/*'\n",
+ "apps/web/package.json": `{"dependencies": {"next": "16.3.6", "livekit-client": "^2.22.2", "livekit-server-sdk": "^2.18.0"}}`,
+ "apps/web/src/package.json": `{}`,
+ "apps/agent/pyproject.toml": "[project]\ndependencies = [\"livekit-agents~=1.8\"]\n",
+ "packages/shared/package.json": `{"name": "shared"}`,
+ "docs/notes.md": "",
+ }))
+ require.Equal(t, FolderLiveKit, f.Kind)
+ require.True(t, f.HasAgents)
+ require.Len(t, f.Projects, 3)
+ require.Equal(t, "a Python project in apps/agent that uses LiveKit Agents and a Next.js app in apps/web that uses the LiveKit client SDK and the LiveKit server SDK, and one other project", f.Summary)
+}
diff --git a/pkg/console/pipeline.go b/pkg/console/pipeline.go
index a975a7bc8..915977397 100644
--- a/pkg/console/pipeline.go
+++ b/pkg/console/pipeline.go
@@ -47,8 +47,11 @@ type AudioPipeline struct {
outputStream *portaudio.Stream
apmInst *apm.APM
noAEC bool
- conn net.Conn
- connMu sync.Mutex // protects writes to conn
+ conn net.Conn // the active agent: hears the mic, plays through the speakers
+ connMu sync.Mutex // protects conn and writes to every agent's conn
+ // switchMu keeps the active agent from changing while a message from
+ // an agent is handled. SetActive holds it too.
+ switchMu sync.Mutex
captureRing *RingBuffer
playbackRing *RingBuffer
@@ -63,16 +66,20 @@ type AudioPipeline struct {
ready chan struct{}
readyOnce sync.Once
- // flushCancel cancels the current waitForDrainAndAck goroutine.
- // Only accessed from the tcpReader goroutine.
+ // flushCancel cancels the current waitForDrainAndAck goroutine, which
+ // acknowledges flushConn's playback once its queued audio has played.
+ flushMu sync.Mutex
flushCancel context.CancelFunc
+ flushConn net.Conn
- mu sync.Mutex
- fftBands [NumFFTBands]float64
- muted bool
- paused bool // true when audio I/O is paused (e.g. text mode); mic frames are not sent to the agent
- level float64 // capture level in dB
- playing bool // true when outputting real audio (not silence)
+ mu sync.Mutex
+ fftBands [NumFFTBands]float64
+ outFFTBands [NumFFTBands]float64 // bands of the agent audio being played
+ outLevel float64 // playback level in dB
+ muted bool
+ paused bool // true when audio I/O is paused (e.g. text mode); mic frames are not sent to the agent
+ level float64 // capture level in dB
+ playing bool // true when outputting real audio (not silence)
cancel context.CancelFunc
audioCtx context.Context // stored so EnableAudio can start goroutines
@@ -197,7 +204,7 @@ func (p *AudioPipeline) Start(ctx context.Context) error {
// Always run the TCP reader for events/responses.
p.wg.Add(1)
- go p.tcpReader(ctx)
+ go p.tcpReader(ctx, p.conn)
// Start audio loops if devices are available.
if p.HasAudio() {
@@ -248,6 +255,42 @@ func (p *AudioPipeline) writeMessage(msg *agent.AgentSessionMessage) error {
return WriteSessionMessage(p.conn, msg)
}
+// AddConn connects another agent to the pipeline. It starts inactive: its
+// audio and events are dropped until SetActive makes it the active agent.
+// Its reader stops when conn closes; the caller owns conn.
+func (p *AudioPipeline) AddConn(ctx context.Context, conn net.Conn) {
+ p.wg.Add(1)
+ go p.tcpReader(ctx, conn)
+}
+
+// SetActive makes conn, from NewPipeline or AddConn, the agent that hears
+// the microphone and plays through the speakers. Audio still queued from the
+// previous agent is dropped, and a playback it waits on is acknowledged.
+func (p *AudioPipeline) SetActive(conn net.Conn) {
+ p.switchMu.Lock()
+ defer p.switchMu.Unlock()
+
+ p.connMu.Lock()
+ p.conn = conn
+ p.connMu.Unlock()
+
+ // Cancel under flushMu, so waitForDrainAndAck can't also claim the flush.
+ p.flushMu.Lock()
+ pending, flushConn := p.flushCancel != nil, p.flushConn
+ if pending {
+ p.flushCancel()
+ }
+ p.flushCancel, p.flushConn = nil, nil
+ p.flushMu.Unlock()
+ if p.playbackRing != nil {
+ p.playbackRing.Reset()
+ }
+ if pending {
+ // Its queued audio was just dropped, so its playback is over.
+ p.ackPlayback(flushConn)
+ }
+}
+
func (p *AudioPipeline) SendRequest(req *agent.SessionRequest) error {
return p.writeMessage(&agent.AgentSessionMessage{
Message: &agent.AgentSessionMessage_Request{Request: req},
@@ -291,6 +334,21 @@ func (p *AudioPipeline) FFTBands() [NumFFTBands]float64 {
return p.fftBands
}
+// PlaybackFFTBands returns the frequency bands of the agent audio currently
+// being played, in the same form as FFTBands.
+func (p *AudioPipeline) PlaybackFFTBands() [NumFFTBands]float64 {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.outFFTBands
+}
+
+// PlaybackLevel returns the level of the agent audio being played, in dB.
+func (p *AudioPipeline) PlaybackLevel() float64 {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ return p.outLevel
+}
+
func (p *AudioPipeline) IsPlaying() bool {
p.mu.Lock()
defer p.mu.Unlock()
@@ -370,6 +428,7 @@ func (p *AudioPipeline) speakerLoop(ctx context.Context) {
p.mu.Lock()
p.playing = pn > 0
p.mu.Unlock()
+ p.computePlaybackMetrics(playbackBuf)
// ProcessRender then ProcessCapture — both in this goroutine,
// right next to each other, no mutex needed.
@@ -423,78 +482,108 @@ func (p *AudioPipeline) speakerLoop(ctx context.Context) {
}
}
-// tcpReader reads messages from the agent over TCP and dispatches them.
-func (p *AudioPipeline) tcpReader(ctx context.Context) {
+// tcpReader reads messages from an agent over TCP and dispatches them.
+func (p *AudioPipeline) tcpReader(ctx context.Context, conn net.Conn) {
defer p.wg.Done()
for {
- msg, err := ReadSessionMessage(p.conn)
+ msg, err := ReadSessionMessage(conn)
if err != nil {
return
}
p.readyOnce.Do(func() { close(p.ready) })
+ p.dispatch(ctx, conn, msg)
+ }
+}
- switch m := msg.Message.(type) {
- case *agent.AgentSessionMessage_AudioOutput:
- // No audio sink in text mode (rings allocated lazily on Ctrl+T).
- if p.playbackRing != nil {
- p.playbackRing.Write(BytesToSamples(m.AudioOutput.Data))
- }
+// dispatch handles a message from conn's agent. An inactive agent's audio
+// and events are dropped, and its playback is acknowledged at once so it
+// never waits on audio that won't play.
+func (p *AudioPipeline) dispatch(ctx context.Context, conn net.Conn, msg *agent.AgentSessionMessage) {
+ p.switchMu.Lock()
+ defer p.switchMu.Unlock()
+
+ // SetActive changes p.conn only while it holds switchMu.
+ if conn != p.conn {
+ if _, ok := msg.Message.(*agent.AgentSessionMessage_AudioPlaybackFlush); ok {
+ p.ackPlayback(conn)
+ }
+ return
+ }
- case *agent.AgentSessionMessage_Event:
- select {
- case p.Events <- m.Event:
- default:
- }
+ switch m := msg.Message.(type) {
+ case *agent.AgentSessionMessage_AudioOutput:
+ // No audio sink in text mode (rings allocated lazily on Ctrl+T).
+ if p.playbackRing != nil {
+ p.playbackRing.Write(BytesToSamples(m.AudioOutput.Data))
+ }
- case *agent.AgentSessionMessage_AudioPlaybackClear:
- if p.flushCancel != nil {
- p.flushCancel()
- p.flushCancel = nil
- }
- if p.playbackRing != nil {
- p.playbackRing.Reset()
- }
+ case *agent.AgentSessionMessage_Event:
+ select {
+ case p.Events <- m.Event:
+ default:
+ }
- case *agent.AgentSessionMessage_AudioPlaybackFlush:
- // Without an audio sink there's nothing to drain, so ack the
- // agent's playback turn immediately instead of dereferencing a
- // nil playback ring in waitForDrainAndAck.
- if p.playbackRing == nil {
- p.sendPlaybackFinished()
- break
- }
- if p.flushCancel != nil {
- p.flushCancel()
- }
- flushCtx, cancel := context.WithCancel(ctx)
- p.flushCancel = cancel
- go p.waitForDrainAndAck(flushCtx)
-
- case *agent.AgentSessionMessage_Response:
- // Forward response so the TUI knows the request completed.
- // Don't synthesize ConversationItemAdded — those arrive via the
- // event stream already.
- if m.Response != nil {
- select {
- case p.Responses <- m.Response:
- default:
- }
+ case *agent.AgentSessionMessage_AudioPlaybackClear:
+ p.flushMu.Lock()
+ if p.flushCancel != nil {
+ p.flushCancel()
+ p.flushCancel, p.flushConn = nil, nil
+ }
+ p.flushMu.Unlock()
+ if p.playbackRing != nil {
+ p.playbackRing.Reset()
+ }
+
+ case *agent.AgentSessionMessage_AudioPlaybackFlush:
+ // Without an audio sink there's nothing to drain, so ack the
+ // agent's playback turn immediately instead of dereferencing a
+ // nil playback ring in waitForDrainAndAck.
+ if p.playbackRing == nil {
+ p.ackPlayback(conn)
+ break
+ }
+ p.flushMu.Lock()
+ if p.flushCancel != nil {
+ p.flushCancel()
+ }
+ flushCtx, cancel := context.WithCancel(ctx)
+ p.flushCancel, p.flushConn = cancel, conn
+ p.flushMu.Unlock()
+ go p.waitForDrainAndAck(flushCtx, conn)
+
+ case *agent.AgentSessionMessage_Response:
+ // Forward response so the TUI knows the request completed.
+ // Don't synthesize ConversationItemAdded — those arrive via the
+ // event stream already.
+ if m.Response != nil {
+ select {
+ case p.Responses <- m.Response:
+ default:
}
}
}
}
-func (p *AudioPipeline) sendPlaybackFinished() {
- _ = p.writeMessage(&agent.AgentSessionMessage{
+// ackPlayback tells conn's agent that its queued audio has finished playing.
+func (p *AudioPipeline) ackPlayback(conn net.Conn) {
+ p.connMu.Lock()
+ defer p.connMu.Unlock()
+ _ = WriteSessionMessage(conn, playbackFinishedMessage())
+}
+
+func playbackFinishedMessage() *agent.AgentSessionMessage {
+ return &agent.AgentSessionMessage{
Message: &agent.AgentSessionMessage_AudioPlaybackFinished{
AudioPlaybackFinished: &agent.AgentSessionMessage_ConsoleIO_AudioPlaybackFinished{},
},
- })
+ }
}
-func (p *AudioPipeline) waitForDrainAndAck(ctx context.Context) {
+// waitForDrainAndAck acknowledges conn's playback once its queued audio has
+// played, unless ctx is cancelled first.
+func (p *AudioPipeline) waitForDrainAndAck(ctx context.Context, conn net.Conn) {
for p.playbackRing.Available() > 0 {
select {
case <-ctx.Done():
@@ -503,15 +592,49 @@ func (p *AudioPipeline) waitForDrainAndAck(ctx context.Context) {
}
time.Sleep(5 * time.Millisecond)
}
- select {
- case <-ctx.Done():
+ // Claim the flush under flushMu. SetActive acks a flush that it
+ // cancels, so only one of them sends the ack.
+ p.flushMu.Lock()
+ if ctx.Err() != nil {
+ p.flushMu.Unlock()
return
- default:
}
- p.sendPlaybackFinished()
+ p.flushCancel()
+ p.flushCancel, p.flushConn = nil, nil
+ p.flushMu.Unlock()
+ p.ackPlayback(conn)
}
func (p *AudioPipeline) computeMetrics(samples []int16) {
+ bands, db, decay := analyzeAudio(samples)
+ p.mu.Lock()
+ smoothBands(&p.fftBands, bands, decay)
+ p.level = db
+ p.mu.Unlock()
+}
+
+func (p *AudioPipeline) computePlaybackMetrics(samples []int16) {
+ bands, db, decay := analyzeAudio(samples)
+ p.mu.Lock()
+ smoothBands(&p.outFFTBands, bands, decay)
+ p.outLevel = db
+ p.mu.Unlock()
+}
+
+// smoothBands rises to new peaks at once and decays toward lower values.
+func smoothBands(dst *[NumFFTBands]float64, bands [NumFFTBands]float64, decay float64) {
+ for b := range dst {
+ if bands[b] > dst[b]*decay {
+ dst[b] = bands[b]
+ } else {
+ dst[b] *= decay
+ }
+ }
+}
+
+// analyzeAudio returns the normalized frequency bands and RMS level (dB) of a
+// frame, and the decay factor for smoothing bands across frames.
+func analyzeAudio(samples []int16) (bands [NumFFTBands]float64, db float64, decay float64) {
n := len(samples)
sr := float64(SampleRate)
@@ -571,7 +694,6 @@ func (p *AudioPipeline) computeMetrics(samples []int16) {
// Mean power → dB → normalize to [0,1]
const floorDB, hotDB = -70.0, -20.0
- var bands [NumFFTBands]float64
for b := range nb {
c := cnts[b]
if c == 0 {
@@ -599,7 +721,7 @@ func (p *AudioPipeline) computeMetrics(samples []int16) {
}
// Exponential decay smoothing (~100ms time constant)
- decay := math.Exp(-float64(n) / sr / 0.1)
+ decay = math.Exp(-float64(n) / sr / 0.1)
// RMS level in dB
var sum float64
@@ -608,18 +730,8 @@ func (p *AudioPipeline) computeMetrics(samples []int16) {
sum += v * v
}
rms := math.Sqrt(sum / float64(n))
- db := 20 * math.Log10(rms+1e-10)
-
- p.mu.Lock()
- for b := range nb {
- if bands[b] > p.fftBands[b]*decay {
- p.fftBands[b] = bands[b]
- } else {
- p.fftBands[b] *= decay
- }
- }
- p.level = db
- p.mu.Unlock()
+ db = 20 * math.Log10(rms+1e-10)
+ return bands, db, decay
}
func SamplesToBytes(samples []int16) []byte {
diff --git a/pkg/console/pipeline_test.go b/pkg/console/pipeline_test.go
new file mode 100644
index 000000000..f9375ed46
--- /dev/null
+++ b/pkg/console/pipeline_test.go
@@ -0,0 +1,157 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package console
+
+import (
+ "context"
+ "net"
+ "testing"
+ "time"
+
+ "github.com/livekit/protocol/livekit/agent"
+)
+
+func stateEvent(state agent.AgentState) *agent.AgentSessionMessage {
+ return &agent.AgentSessionMessage{Message: &agent.AgentSessionMessage_Event{Event: &agent.AgentSessionEvent{
+ Event: &agent.AgentSessionEvent_AgentStateChanged_{AgentStateChanged: &agent.AgentSessionEvent_AgentStateChanged{NewState: state}},
+ }}}
+}
+
+func flushMessage() *agent.AgentSessionMessage {
+ return &agent.AgentSessionMessage{Message: &agent.AgentSessionMessage_AudioPlaybackFlush{
+ AudioPlaybackFlush: &agent.AgentSessionMessage_ConsoleIO_AudioPlaybackFlush{},
+ }}
+}
+
+// nextEvent returns the next event's new agent state, or -1 if none arrives.
+func nextEvent(t *testing.T, p *AudioPipeline) agent.AgentState {
+ t.Helper()
+ select {
+ case ev := <-p.Events:
+ return ev.GetAgentStateChanged().GetNewState()
+ case <-time.After(200 * time.Millisecond):
+ return -1
+ }
+}
+
+func send(t *testing.T, conn net.Conn, msg *agent.AgentSessionMessage) {
+ t.Helper()
+ if err := WriteSessionMessage(conn, msg); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestPipelineSwitchesAgents(t *testing.T) {
+ // No audio devices: the pipeline only relays session messages.
+ first, firstAgent := net.Pipe()
+ second, secondAgent := net.Pipe()
+ p, err := NewPipeline(PipelineConfig{Conn: first})
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go p.Start(ctx)
+ time.Sleep(20 * time.Millisecond)
+ p.AddConn(ctx, second)
+
+ // The second agent starts inactive: its events are dropped, and its
+ // playback is acknowledged right away.
+ send(t, firstAgent, stateEvent(agent.AgentState_AS_LISTENING))
+ if got := nextEvent(t, p); got != agent.AgentState_AS_LISTENING {
+ t.Fatalf("active agent's event: got %v", got)
+ }
+ send(t, secondAgent, stateEvent(agent.AgentState_AS_SPEAKING))
+ if got := nextEvent(t, p); got != -1 {
+ t.Fatalf("inactive agent's event should be dropped, got %v", got)
+ }
+ acked := make(chan *agent.AgentSessionMessage, 1)
+ go func() {
+ msg, _ := ReadSessionMessage(secondAgent)
+ acked <- msg
+ }()
+ send(t, secondAgent, flushMessage())
+ select {
+ case msg := <-acked:
+ if msg.GetAudioPlaybackFinished() == nil {
+ t.Fatalf("inactive agent's flush: got %v, want playback finished", msg)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("inactive agent's flush wasn't acknowledged")
+ }
+
+ // After switching, it's the other way around.
+ p.SetActive(second)
+ send(t, secondAgent, stateEvent(agent.AgentState_AS_THINKING))
+ if got := nextEvent(t, p); got != agent.AgentState_AS_THINKING {
+ t.Fatalf("newly active agent's event: got %v", got)
+ }
+ send(t, firstAgent, stateEvent(agent.AgentState_AS_SPEAKING))
+ if got := nextEvent(t, p); got != -1 {
+ t.Fatalf("now-inactive agent's event should be dropped, got %v", got)
+ }
+}
+
+func TestPipelineSwitchAcksPendingPlayback(t *testing.T) {
+ first, firstAgent := net.Pipe()
+ second, secondAgent := net.Pipe()
+ defer firstAgent.Close()
+ defer secondAgent.Close()
+ p, err := NewPipeline(PipelineConfig{Conn: first})
+ if err != nil {
+ t.Fatal(err)
+ }
+ // A playback ring that nothing drains, so a flush waits for its audio.
+ p.playbackRing = NewRingBuffer(SamplesPerFrame)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go p.Start(ctx)
+ time.Sleep(20 * time.Millisecond)
+ p.AddConn(ctx, second)
+
+ send(t, firstAgent, &agent.AgentSessionMessage{Message: &agent.AgentSessionMessage_AudioOutput{
+ AudioOutput: &agent.AgentSessionMessage_ConsoleIO_AudioFrame{Data: make([]byte, 960)},
+ }})
+ send(t, firstAgent, flushMessage())
+ time.Sleep(20 * time.Millisecond)
+
+ read := func(c net.Conn) <-chan *agent.AgentSessionMessage {
+ ch := make(chan *agent.AgentSessionMessage, 1)
+ go func() {
+ if msg, err := ReadSessionMessage(c); err == nil {
+ ch <- msg
+ }
+ }()
+ return ch
+ }
+ firstAcks, secondAcks := read(firstAgent), read(secondAgent)
+
+ // Switching drops the first agent's queued audio, so its playback is
+ // over: the first agent gets the ack, and the second gets none.
+ p.SetActive(second)
+ select {
+ case msg := <-firstAcks:
+ if msg.GetAudioPlaybackFinished() == nil {
+ t.Fatalf("first agent: got %v, want playback finished", msg)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("the first agent's pending playback wasn't acknowledged")
+ }
+ select {
+ case msg := <-secondAcks:
+ t.Fatalf("the second agent got a message it didn't ask for: %v", msg)
+ case <-time.After(100 * time.Millisecond):
+ }
+}
diff --git a/pkg/overlay/overlay.go b/pkg/overlay/overlay.go
new file mode 100644
index 000000000..69f993711
--- /dev/null
+++ b/pkg/overlay/overlay.go
@@ -0,0 +1,115 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package overlay shows the assistant's full-screen overlay: a native window
+// that blurs the desktop behind a transparent web view running ui/index.html.
+//
+// Go drives the page with JSON messages (Send) and receives the page's
+// messages as Events. The page protocol:
+//
+// Go → page {"type":"state","state":"connecting|idle|listening|thinking|speaking","label":"optional"}
+// {"type":"bands","bands":[0..1, ...]}
+// {"type":"transcript","turn":"...","role":"user|agent","segment":0,"text":"..."}
+// {"type":"snippet","turn":"...","id":"...","title":"...","lang":"python|shell","code":"...","copied":bool}
+// {"type":"link","turn":"...","id":"...","title":"...","url":"..."}
+// {"type":"answer","turn":"...","id":"...","title":"...","points":["..."],"source_title":"...","source_url":"..."}
+// {"type":"copied","id":"..."}
+// {"type":"tool","turn":"...","id":"...","label":"...","detail":"...","status":"running|done|error"}
+// {"type":"notice","text":"...","sticky":bool}
+// {"type":"error","title":"...","detail":"...","fix":"command"}
+// {"type":"speaker","name":"...","guest":bool}
+// {"type":"guest_status","status":"starting|failed","name":"..."}
+// {"type":"pipeline",...} {"type":"eot",...} {"type":"metrics",...} {"type":"usage",...} (X-ray)
+// {"type":"dismiss"}
+// {"type":"clear"}
+// {"type":"confirm","turn":"...","id":"...","heading":"...","title":"...","options":[{"id":"...","name":"..."}]}
+// {"type":"confirm_done","id":"...","option":"..."}
+// page → Go {"type":"ready"} {"type":"copy","id":"..."} {"type":"replay"} {"type":"close"}
+// {"type":"confirm","id":"...","option":"..."} {"type":"copy_text","text":"..."}
+// {"type":"back"} {"type":"mute","on":bool} {"type":"open","id":"..."}
+// {"type":"text","text":"..."} (typed input)
+//
+// A transcript message carries the full text so far of one segment (a
+// paragraph) of a turn. Snippets are added to their turn after the segments
+// before them. The page also sends {"type":"peek","on":bool}, which the
+// overlay handles itself by fading the desktop blur.
+//
+// Only macOS is supported for now.
+package overlay
+
+import (
+ _ "embed"
+ "errors"
+ "sync"
+)
+
+//go:embed ui/index.html
+var pageHTML string
+
+// ErrUnsupported is returned by Run on platforms without an overlay.
+var ErrUnsupported = errors.New("the assistant overlay is only supported on macOS")
+
+// Options configure the overlay window.
+type Options struct {
+ // Windowed opens a regular floating window instead of covering the screen.
+ Windowed bool
+ // Inspectable enables the Safari Web Inspector for the page.
+ Inspectable bool
+}
+
+// Event is a message from the overlay page.
+type Event struct {
+ Type string `json:"type"`
+ ID string `json:"id,omitempty"`
+ On bool `json:"on,omitempty"`
+ // Option is the chosen option of a confirm event.
+ Option string `json:"option,omitempty"`
+ // Text is the text of a copy_text or text event.
+ Text string `json:"text,omitempty"`
+}
+
+// Overlay is a single overlay window. A process can run one at a time.
+type Overlay struct {
+ opts Options
+ events chan Event
+ done chan struct{}
+ closeOnce sync.Once
+}
+
+// New creates an overlay. Call Run to show it.
+func New(opts Options) *Overlay {
+ return &Overlay{
+ opts: opts,
+ events: make(chan Event, 64),
+ done: make(chan struct{}),
+ }
+}
+
+// Events returns messages from the page. Messages are dropped if nobody reads
+// them.
+func (o *Overlay) Events() <-chan Event {
+ return o.events
+}
+
+// Done is closed after the window closes and Run returns.
+func (o *Overlay) Done() <-chan struct{} {
+ return o.done
+}
+
+func (o *Overlay) emit(ev Event) {
+ select {
+ case o.events <- ev:
+ default:
+ }
+}
diff --git a/pkg/overlay/overlay_darwin.go b/pkg/overlay/overlay_darwin.go
new file mode 100644
index 000000000..b1c3331c6
--- /dev/null
+++ b/pkg/overlay/overlay_darwin.go
@@ -0,0 +1,122 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package overlay
+
+/*
+#cgo CFLAGS: -Wno-deprecated-declarations
+#cgo LDFLAGS: -framework Cocoa -framework WebKit
+#include
+#include "overlay_darwin.h"
+*/
+import "C"
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "runtime"
+ "sync"
+ "unsafe"
+)
+
+// Cocoa's event loop must run on the process's main thread. Locking here, in
+// init, keeps the main goroutine (and so every command's Action) on it.
+func init() {
+ runtime.LockOSThread()
+}
+
+var (
+ activeMu sync.Mutex
+ active *Overlay
+)
+
+// Run shows the overlay and blocks until it closes, either from the page
+// (Esc), Close, or ctx being done. It must be called from the main goroutine.
+func (o *Overlay) Run(ctx context.Context) error {
+ if C.lk_overlay_is_main_thread() == 0 {
+ return errors.New("overlay: Run must be called on the main thread")
+ }
+ activeMu.Lock()
+ if active != nil {
+ activeMu.Unlock()
+ return errors.New("overlay: already running")
+ }
+ active = o
+ activeMu.Unlock()
+
+ go func() {
+ select {
+ case <-ctx.Done():
+ o.Close()
+ case <-o.done:
+ }
+ }()
+
+ html := C.CString(pageHTML)
+ defer C.free(unsafe.Pointer(html))
+ C.lk_overlay_run(html, cbool(o.opts.Windowed), cbool(o.opts.Inspectable))
+
+ activeMu.Lock()
+ active = nil
+ activeMu.Unlock()
+ close(o.done)
+ return nil
+}
+
+// Send delivers a message to the page. Messages sent before the page's
+// "ready" event are lost.
+func (o *Overlay) Send(msg any) error {
+ b, err := json.Marshal(msg)
+ if err != nil {
+ return err
+ }
+ cmsg := C.CString(string(b))
+ defer C.free(unsafe.Pointer(cmsg))
+ C.lk_overlay_send(cmsg)
+ return nil
+}
+
+// Close fades the overlay out and makes Run return.
+func (o *Overlay) Close() {
+ o.closeOnce.Do(func() { C.lk_overlay_close() })
+}
+
+//export lkOverlayMessage
+func lkOverlayMessage(msg *C.char) {
+ var ev Event
+ if err := json.Unmarshal([]byte(C.GoString(msg)), &ev); err != nil {
+ return
+ }
+ activeMu.Lock()
+ o := active
+ activeMu.Unlock()
+ if o == nil {
+ return
+ }
+ switch ev.Type {
+ case "close":
+ o.Close()
+ case "peek":
+ C.lk_overlay_set_peek(cbool(ev.On))
+ }
+ o.emit(ev)
+}
+
+func cbool(b bool) C.int {
+ if b {
+ return 1
+ }
+ return 0
+}
diff --git a/pkg/overlay/overlay_darwin.h b/pkg/overlay/overlay_darwin.h
new file mode 100644
index 000000000..ed0076e31
--- /dev/null
+++ b/pkg/overlay/overlay_darwin.h
@@ -0,0 +1,37 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef LK_OVERLAY_DARWIN_H
+#define LK_OVERLAY_DARWIN_H
+
+// lk_overlay_run opens the overlay window and runs the Cocoa event loop until
+// lk_overlay_close is called. It must be called on the process's main thread.
+void lk_overlay_run(const char *html, int windowed, int inspectable);
+
+// lk_overlay_send passes a JSON message to window.lk.receive in the overlay
+// page. Safe from any thread.
+void lk_overlay_send(const char *json);
+
+// lk_overlay_close fades the window out and stops the event loop. Safe from
+// any thread.
+void lk_overlay_close(void);
+
+// lk_overlay_set_peek fades the desktop blur out (on) or back in (off), so the
+// user can see the apps behind the overlay. Safe from any thread.
+void lk_overlay_set_peek(int on);
+
+// lk_overlay_is_main_thread reports whether the caller is on the main thread.
+int lk_overlay_is_main_thread(void);
+
+#endif
diff --git a/pkg/overlay/overlay_darwin.m b/pkg/overlay/overlay_darwin.m
new file mode 100644
index 000000000..b925ca049
--- /dev/null
+++ b/pkg/overlay/overlay_darwin.m
@@ -0,0 +1,212 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import
+#import
+#include
+
+#include "overlay_darwin.h"
+#include "_cgo_export.h"
+
+// Built without ARC. The window, web view, and handler live for the rest of
+// the process once created, so they're never released.
+
+// Borderless windows can't become key by default, which would leave the page
+// without keyboard input (Esc to close).
+@interface LKOverlayWindow : NSWindow
+@end
+
+@implementation LKOverlayWindow
+- (BOOL)canBecomeKeyWindow {
+ return YES;
+}
+- (BOOL)canBecomeMainWindow {
+ return YES;
+}
+// Closing the window, such as with its close button in windowed mode,
+// closes the overlay the same as when the page asks to. Closing only the
+// window would leave Run blocked.
+- (void)close {
+ lkOverlayMessage((char *)"{\"type\":\"close\"}");
+}
+@end
+
+@interface LKOverlayHandler : NSObject
+@end
+
+@implementation LKOverlayHandler
+- (void)userContentController:(WKUserContentController *)controller
+ didReceiveScriptMessage:(WKScriptMessage *)message {
+ if (![message.body isKindOfClass:[NSString class]]) {
+ return;
+ }
+ lkOverlayMessage((char *)[(NSString *)message.body UTF8String]);
+}
+@end
+
+static LKOverlayWindow *gWindow;
+static WKWebView *gWebView;
+static NSVisualEffectView *gBlur;
+static BOOL gClosing;
+
+void lk_overlay_run(const char *html, int windowed, int inspectable) {
+ @autoreleasepool {
+ [NSApplication sharedApplication];
+ // No Dock icon or menu bar takeover: the overlay floats over whatever
+ // app the user is in.
+ [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory];
+
+ NSScreen *screen = [NSScreen mainScreen];
+ NSRect frame = screen.frame;
+ NSWindowStyleMask style = NSWindowStyleMaskBorderless;
+ if (windowed) {
+ NSRect visible = screen.visibleFrame;
+ frame = NSInsetRect(visible, visible.size.width * 0.12, visible.size.height * 0.12);
+ style = NSWindowStyleMaskTitled | NSWindowStyleMaskClosable |
+ NSWindowStyleMaskResizable | NSWindowStyleMaskFullSizeContentView;
+ }
+
+ gWindow = [[LKOverlayWindow alloc] initWithContentRect:frame
+ styleMask:style
+ backing:NSBackingStoreBuffered
+ defer:NO];
+ gWindow.opaque = NO;
+ gWindow.backgroundColor = [NSColor clearColor];
+ gWindow.releasedWhenClosed = NO;
+ gWindow.appearance = [NSAppearance appearanceNamed:NSAppearanceNameDarkAqua];
+ gWindow.alphaValue = 0;
+ if (windowed) {
+ gWindow.titlebarAppearsTransparent = YES;
+ gWindow.titleVisibility = NSWindowTitleHidden;
+ gWindow.level = NSFloatingWindowLevel;
+ // The green button takes the window to its own full-screen Space.
+ gWindow.collectionBehavior = NSWindowCollectionBehaviorFullScreenPrimary;
+ } else {
+ gWindow.hasShadow = NO;
+ // Above the menu bar and Dock.
+ gWindow.level = NSStatusWindowLevel;
+ gWindow.collectionBehavior = NSWindowCollectionBehaviorCanJoinAllSpaces |
+ NSWindowCollectionBehaviorFullScreenAuxiliary |
+ NSWindowCollectionBehaviorStationary |
+ NSWindowCollectionBehaviorIgnoresCycle;
+ }
+
+ // The blur and the web view are siblings, so peek can fade the blur
+ // without fading the page.
+ NSView *root = [[NSView alloc] initWithFrame:NSMakeRect(0, 0, frame.size.width, frame.size.height)];
+ root.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable;
+ gWindow.contentView = root;
+
+ // Blurs whatever is behind the window: the desktop and other apps.
+ NSVisualEffectView *blur = [[NSVisualEffectView alloc] initWithFrame:root.bounds];
+ blur.blendingMode = NSVisualEffectBlendingModeBehindWindow;
+ blur.material = NSVisualEffectMaterialFullScreenUI;
+ blur.state = NSVisualEffectStateActive;
+ blur.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable;
+ [root addSubview:blur];
+ gBlur = blur;
+
+ WKWebViewConfiguration *config = [[WKWebViewConfiguration alloc] init];
+ // Earcons play from Web Audio as soon as the overlay opens, without a click first.
+ config.mediaTypesRequiringUserActionForPlayback = WKAudiovisualMediaTypeNone;
+ [config.userContentController addScriptMessageHandler:[[LKOverlayHandler alloc] init] name:@"lk"];
+
+ gWebView = [[WKWebView alloc] initWithFrame:root.bounds configuration:config];
+ gWebView.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable;
+ // Transparent page background so the blur shows through.
+ [gWebView setValue:@NO forKey:@"drawsBackground"];
+ if (inspectable) {
+ if (@available(macOS 13.3, *)) {
+ gWebView.inspectable = YES;
+ }
+ }
+ [root addSubview:gWebView];
+ [gWebView loadHTMLString:[NSString stringWithUTF8String:html] baseURL:nil];
+
+ [gWindow makeKeyAndOrderFront:nil];
+ [gWindow makeFirstResponder:gWebView];
+ [NSApp activateIgnoringOtherApps:YES];
+ [NSAnimationContext runAnimationGroup:^(NSAnimationContext *ctx) {
+ ctx.duration = 0.35;
+ gWindow.animator.alphaValue = 1;
+ }];
+
+ [NSApp run];
+
+ [gWindow orderOut:nil];
+ }
+}
+
+// Messages go through one fixed function body with the JSON as an argument.
+// Evaluating each message as its own script source makes WebKit compile and
+// cache a new script per message, many times a second.
+static NSString *const kReceiveBody = @"if (window.lk) window.lk.receive(JSON.parse(msg));";
+
+void lk_overlay_send(const char *json) {
+ @autoreleasepool {
+ NSString *msg = [NSString stringWithUTF8String:json];
+ dispatch_async(dispatch_get_main_queue(), ^{
+ [gWebView callAsyncJavaScript:kReceiveBody
+ arguments:@{@"msg" : msg}
+ inFrame:nil
+ inContentWorld:WKContentWorld.pageWorld
+ completionHandler:nil];
+ });
+ }
+}
+
+static void stopApp(void) {
+ [NSApp stop:nil];
+ // -stop: takes effect after the next event, so post one.
+ NSEvent *event = [NSEvent otherEventWithType:NSEventTypeApplicationDefined
+ location:NSZeroPoint
+ modifierFlags:0
+ timestamp:0
+ windowNumber:0
+ context:nil
+ subtype:0
+ data1:0
+ data2:0];
+ [NSApp postEvent:event atStart:YES];
+}
+
+void lk_overlay_close(void) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (gWindow == nil || gClosing) {
+ return;
+ }
+ gClosing = YES;
+ [NSAnimationContext
+ runAnimationGroup:^(NSAnimationContext *ctx) {
+ ctx.duration = 0.25;
+ gWindow.animator.alphaValue = 0;
+ }
+ completionHandler:^{
+ stopApp();
+ }];
+ });
+}
+
+void lk_overlay_set_peek(int on) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ [NSAnimationContext runAnimationGroup:^(NSAnimationContext *ctx) {
+ ctx.duration = 0.2;
+ gBlur.animator.alphaValue = on ? 0 : 1;
+ }];
+ });
+}
+
+int lk_overlay_is_main_thread(void) {
+ return pthread_main_np();
+}
diff --git a/pkg/overlay/overlay_other.go b/pkg/overlay/overlay_other.go
new file mode 100644
index 000000000..f17ee83e7
--- /dev/null
+++ b/pkg/overlay/overlay_other.go
@@ -0,0 +1,31 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !darwin
+
+package overlay
+
+import "context"
+
+// Run returns ErrUnsupported on this platform.
+func (o *Overlay) Run(ctx context.Context) error {
+ close(o.done)
+ return ErrUnsupported
+}
+
+// Send is a no-op on this platform.
+func (o *Overlay) Send(msg any) error { return nil }
+
+// Close is a no-op on this platform.
+func (o *Overlay) Close() {}
diff --git a/pkg/overlay/ui/index.html b/pkg/overlay/ui/index.html
new file mode 100644
index 000000000..bc984b271
--- /dev/null
+++ b/pkg/overlay/ui/index.html
@@ -0,0 +1,1118 @@
+
+
+
+
+
+LiveKit Assistant
+
+
+
+
+
+
+
LK · LiveKit AssistantMic off
+
+
Ready
+
/ type · M mute · X x-ray · Shift peek · ⌘C copy last · Esc close
+
+
+
X-ray · LK's pipeline
+
+
STT…
+
LLM…
+
TTS…
+
Turns…
+
+
+
End of turn
+
Speak to see the turn detector's estimate
+
+
+
Last reply…
+
+
+
+
Usage…
+
+
+
+
+
Starting your agent
+
+
+
+
+
+
+
diff --git a/pkg/util/fs.go b/pkg/util/fs.go
index 71243d021..69ec45c4e 100644
--- a/pkg/util/fs.go
+++ b/pkg/util/fs.go
@@ -61,7 +61,8 @@ func CopyFile(src, dest string) error {
return nil
}
-// Safely move a directory across filesystems, preserving permissions
+// Safely move a directory across filesystems, preserving permissions. dest
+// must not exist.
func MoveDir(src, dest string) error {
if _, err := os.Stat(dest); err == nil {
return fmt.Errorf("destination directory already exists: %s", dest)
@@ -73,6 +74,38 @@ func MoveDir(src, dest string) error {
return fmt.Errorf("failed to create destination directory: %w", err)
}
+ return moveTree(src, dest)
+}
+
+// MoveDirInto is MoveDir for a destination directory that already exists and
+// is empty (see IsEmptyDir).
+func MoveDirInto(src, dest string) error {
+ empty, err := IsEmptyDir(dest)
+ if err != nil {
+ return err
+ }
+ if !empty {
+ return fmt.Errorf("destination directory isn't empty: %s", dest)
+ }
+ return moveTree(src, dest)
+}
+
+// IsEmptyDir reports whether dir has no entries, ignoring macOS's .DS_Store.
+func IsEmptyDir(dir string) (bool, error) {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return false, err
+ }
+ for _, e := range entries {
+ if e.Name() != ".DS_Store" {
+ return false, nil
+ }
+ }
+ return true, nil
+}
+
+// moveTree copies src into the existing directory dest, then removes src.
+func moveTree(src, dest string) error {
err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
diff --git a/pkg/util/fs_test.go b/pkg/util/fs_test.go
new file mode 100644
index 000000000..ce1e736e8
--- /dev/null
+++ b/pkg/util/fs_test.go
@@ -0,0 +1,87 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func writeTestFile(t *testing.T, path, content string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestIsEmptyDir(t *testing.T) {
+ dir := t.TempDir()
+ if empty, err := IsEmptyDir(dir); err != nil || !empty {
+ t.Fatalf("new dir: empty=%v err=%v, want empty", empty, err)
+ }
+
+ writeTestFile(t, filepath.Join(dir, ".DS_Store"), "")
+ if empty, _ := IsEmptyDir(dir); !empty {
+ t.Fatal("a dir with only .DS_Store should count as empty")
+ }
+
+ writeTestFile(t, filepath.Join(dir, ".git", "HEAD"), "ref: refs/heads/main")
+ if empty, _ := IsEmptyDir(dir); empty {
+ t.Fatal("a dir with .git shouldn't count as empty")
+ }
+}
+
+func TestMoveDirInto(t *testing.T) {
+ src, dest := t.TempDir(), t.TempDir()
+ writeTestFile(t, filepath.Join(src, "README.md"), "hello")
+ writeTestFile(t, filepath.Join(src, "src", "agent.py"), "print('hi')")
+ writeTestFile(t, filepath.Join(dest, ".DS_Store"), "")
+
+ if err := MoveDirInto(src, dest); err != nil {
+ t.Fatal(err)
+ }
+ for _, f := range []string{"README.md", filepath.Join("src", "agent.py")} {
+ if _, err := os.Stat(filepath.Join(dest, f)); err != nil {
+ t.Errorf("%s wasn't moved: %v", f, err)
+ }
+ }
+ if _, err := os.Stat(src); !os.IsNotExist(err) {
+ t.Errorf("source should be removed, got err=%v", err)
+ }
+}
+
+func TestMoveDirIntoNonEmpty(t *testing.T) {
+ src, dest := t.TempDir(), t.TempDir()
+ writeTestFile(t, filepath.Join(src, "README.md"), "template")
+ writeTestFile(t, filepath.Join(dest, "notes.txt"), "mine")
+
+ if err := MoveDirInto(src, dest); err == nil {
+ t.Fatal("moving into a non-empty dir should fail")
+ }
+ if b, _ := os.ReadFile(filepath.Join(dest, "notes.txt")); string(b) != "mine" {
+ t.Error("existing files must be left alone")
+ }
+}
+
+func TestMoveDirExisting(t *testing.T) {
+ src, dest := t.TempDir(), t.TempDir()
+ if err := MoveDir(src, dest); err == nil {
+ t.Fatal("MoveDir should refuse an existing destination")
+ }
+}