From 080512aa07d444a2d1ada4b9ec93fdbde17c54b9 Mon Sep 17 00:00:00 2001 From: royalfig Date: Fri, 9 Oct 2026 17:42:00 -0400 Subject: [PATCH 01/12] fix(simulate): find entrypoints only inside the project directory Co-Authored-By: Claude Opus 5.5 --- cmd/lk/simulate_subprocess.go | 6 ++-- cmd/lk/simulate_subprocess_test.go | 56 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 cmd/lk/simulate_subprocess_test.go diff --git a/cmd/lk/simulate_subprocess.go b/cmd/lk/simulate_subprocess.go index b8d74ba8..af9a8209 100644 --- a/cmd/lk/simulate_subprocess.go +++ b/cmd/lk/simulate_subprocess.go @@ -193,9 +193,11 @@ func findEntrypoint(dir, explicit string, projectType agentfs.ProjectType) (stri } } - // Then cwd-relative paths (e.g. running from examples/drive-thru/) + // Then cwd-relative paths (e.g. running from examples/drive-thru/), only + // when cwd is inside the project: lk assistant runs projects in folders + // below cwd, and "../agent.py" would be another project's agent. cwd, _ := os.Getwd() - if rel, err := filepath.Rel(dir, cwd); err == nil && rel != "." { + if rel, err := filepath.Rel(dir, cwd); err == nil && rel != "." && filepath.IsLocal(rel) { for _, def := range append(append([]string{}, rootCandidates...), srcCandidates...) { candidate := filepath.Join(rel, def) if probe(candidate) { diff --git a/cmd/lk/simulate_subprocess_test.go b/cmd/lk/simulate_subprocess_test.go new file mode 100644 index 00000000..a5a7e60a --- /dev/null +++ b/cmd/lk/simulate_subprocess_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/livekit/livekit-cli/v2/pkg/agentfs" +) + +func writeEntrypoint(t *testing.T, path string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, nil, 0o644)) +} + +func TestFindEntrypointFromSubdirectory(t *testing.T) { + // Run from a folder inside the project, its agent there is found. + root := t.TempDir() + writeEntrypoint(t, filepath.Join(root, "examples", "drive-thru", "agent.py")) + t.Chdir(filepath.Join(root, "examples", "drive-thru")) + + entry, err := findEntrypoint(root, "", agentfs.ProjectTypePythonUV) + require.NoError(t, err) + require.Equal(t, filepath.Join("examples", "drive-thru", "agent.py"), entry) +} + +func TestFindEntrypointIgnoresParentProject(t *testing.T) { + // cwd is one agent project, and another project is in a folder below + // it. The child's own agent is found, not the parent's. + parent := t.TempDir() + writeEntrypoint(t, filepath.Join(parent, "src", "agent.py")) + child := filepath.Join(parent, "my-agent") + writeEntrypoint(t, filepath.Join(child, "src", "agent.py")) + t.Chdir(parent) + + entry, err := findEntrypoint(child, "", agentfs.ProjectTypePythonUV) + require.NoError(t, err) + require.Equal(t, filepath.Join("src", "agent.py"), entry) +} From a7e90908b122e0894298728dd0e95e74ca886d51 Mon Sep 17 00:00:00 2001 From: royalfig Date: Fri, 9 Oct 2026 17:42:01 -0400 Subject: [PATCH 02/12] feat(agent): create projects in the current directory with `lk agent init .` `lk agent init .` and `lk app create .` fill the empty current directory and name the project after it. Co-Authored-By: Claude Opus 5.5 --- cmd/lk/agent.go | 10 ++++- cmd/lk/app.go | 75 +++++++++++++++++++++++++-------- cmd/lk/app_target_test.go | 55 +++++++++++++++++++++++++ pkg/util/fs.go | 35 +++++++++++++++- pkg/util/fs_test.go | 87 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 242 insertions(+), 20 deletions(-) create mode 100644 cmd/lk/app_target_test.go create mode 100644 pkg/util/fs_test.go diff --git a/cmd/lk/agent.go b/cmd/lk/agent.go index 923170dd..cfce5dac 100644 --- a/cmd/lk/agent.go +++ b/cmd/lk/agent.go @@ -236,6 +236,9 @@ On LiveKit Cloud: "create" and "deploy" ship it, then "status", "logs", { Name: "init", Usage: "Create a new agent project from a template", + Description: `Creates the project in a new directory named AGENT-NAME. To create it in +the current directory instead, pass "." as AGENT-NAME. The directory must be +empty, and the agent is named after it.`, Before: func(ctx context.Context, cmd *cli.Command) (context.Context, error) { return createAgentClientWithOpts(ctx, cmd, confirmProject) }, @@ -608,7 +611,12 @@ func initAgent(ctx context.Context, cmd *cli.Command) error { logger.Debugw("Initializing agent project", "working-dir", workingDir) - appName = cmd.Args().First() + if arg := cmd.Args().First(); arg != "" { + var err error + if appName, appDir, err = resolveAppTarget(arg); err != nil { + return err + } + } if appName == "" { appName = project.Name } diff --git a/cmd/lk/app.go b/cmd/lk/app.go index 487ee100..ee39c1c5 100644 --- a/cmd/lk/app.go +++ b/cmd/lk/app.go @@ -22,6 +22,7 @@ import ( "io" "maps" "os" + "path/filepath" "regexp" "strings" @@ -44,6 +45,7 @@ var ( templateURL string sandboxID string appName string + appDir string // where the app is created; appName unless "." was given appNameRegex = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`) destinationFile string exampleFile string @@ -55,8 +57,11 @@ var ( Usage: "Initialize and manage applications", Commands: []*cli.Command{ { - Name: "create", - Usage: "Bootstrap a new application from a template or through guided creation", + Name: "create", + Usage: "Bootstrap a new application from a template or through guided creation", + Description: `Creates the application in a new directory named APP_NAME. To create it in +the current directory instead, pass "." as APP_NAME. The directory must be +empty, and the application is named after it.`, Action: setupTemplate, Before: requireAppProject, ArgsUsage: "`APP_NAME`", @@ -363,9 +368,11 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func( } if appName == "" { - arg := cmd.Args().First() - if arg != "" { - appName = arg + if arg := cmd.Args().First(); arg != "" { + var err error + if appName, appDir, err = resolveAppTarget(arg); err != nil { + return err + } } } if appName == "" { @@ -403,16 +410,20 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func( } } + if appDir == "" { + appDir = appName + } + // Set environment variables for template instantiation os.Setenv("LIVEKIT_AGENT_NAME", appName) os.Setenv("LIVEKIT_PROJECT_ID", project.ProjectId) out.Status("Cloning template...") - if err := cloneTemplate(ctx, cmd, templateURL, appName); err != nil { + if err := cloneTemplate(ctx, cmd, templateURL, appDir); err != nil { return err } - tf, err := bootstrap.ParseTaskfile(appName) + tf, err := bootstrap.ParseTaskfile(appDir) if err != nil { return err } @@ -436,12 +447,12 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func( } } } - env, err := instantiateEnv(ctx, cmd, appName, addlEnv, envExampleFile, nil) + env, err := instantiateEnv(ctx, cmd, appDir, addlEnv, envExampleFile, nil) if err != nil { return err } - bootstrap.WriteDotEnv(appName, envOutputFile, env, true) + bootstrap.WriteDotEnv(appDir, envOutputFile, env, true) if !cmd.IsSet("install") && !SkipPrompts(cmd) { // Default the prompt to "yes" — installing deps is the common case. @@ -456,7 +467,7 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func( } if install { out.Status("Installing template...") - if err := doInstall(ctx, bootstrap.TaskInstall, appName, verbose); err != nil { + if err := doInstall(ctx, bootstrap.TaskInstall, appDir, verbose); err != nil { // Installation is best-effort — the agent is still created below. But a // failed install (e.g. missing Node/pnpm) is easy to miss once the // template's post-create step prints "agent created", so render a @@ -474,7 +485,7 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func( b.WriteString(line) b.WriteString("\n") } - out.Warnf("%s%sFix your toolchain, then re-run the install step manually in ./%s.", b.String(), fixPrefix, appName) + out.Warnf("%s%sFix your toolchain, then re-run the install step manually in ./%s.", b.String(), fixPrefix, appDir) } else { // Signal a successful install to post_create so the template can skip // printing the now-redundant install hint (guarded via `status:`). @@ -482,23 +493,51 @@ func setupTemplateWith(ctx context.Context, cmd *cli.Command, afterInstall func( } } if afterInstall != nil { - if err := afterInstall(ctx, cmd, appName); err != nil { + if err := afterInstall(ctx, cmd, appDir); err != nil { return err } } - if err := doPostCreate(ctx, cmd, appName, verbose); err != nil { + if err := doPostCreate(ctx, cmd, appDir, verbose); err != nil { return err } - return cleanupTemplate(ctx, cmd, appName) + return cleanupTemplate(ctx, cmd, appDir) +} + +// resolveAppTarget returns the app's name and the directory to create it in. +// "." creates the app in the current directory, which must be empty, and +// names it after the directory. +func resolveAppTarget(arg string) (name, dir string, err error) { + if arg != "." { + return arg, arg, nil + } + wd, err := os.Getwd() + if err != nil { + return "", "", err + } + if empty, err := util.IsEmptyDir("."); err != nil { + return "", "", err + } else if !empty { + return "", "", errors.New("the current directory isn't empty; to create the app here, run this in an empty directory, or give a name to create a new directory") + } + name = filepath.Base(wd) + if !appNameRegex.MatchString(name) { + return "", "", fmt.Errorf("the current directory's name, %q, can't be used as the app name; use letters, numbers, dashes, and underscores", name) + } + return name, ".", nil } -func cloneTemplate(ctx context.Context, cmd *cli.Command, url, appName string) error { +// cloneTemplate clones the template at url into dir. dir must not exist, +// except "." (the current directory), which must be empty. +func cloneTemplate(ctx context.Context, cmd *cli.Command, url, dir string) error { var stdout string var stderr string - tempName, relocate, cleanup := util.UseTempPath(appName) + tempName, relocate, cleanup := util.UseTempPath(dir) defer cleanup() + if dir == "." { + relocate = func() error { return util.MoveDirInto(tempName, dir) } + } err := out.Await( "Cloning template from "+url, @@ -529,8 +568,8 @@ func cloneTemplate(ctx context.Context, cmd *cli.Command, url, appName string) e return relocate() } -func cleanupTemplate(ctx context.Context, cmd *cli.Command, appName string) error { - return bootstrap.CleanupTemplate(appName) +func cleanupTemplate(ctx context.Context, cmd *cli.Command, dir string) error { + return bootstrap.CleanupTemplate(dir) } func manageEnv(ctx context.Context, cmd *cli.Command) error { diff --git a/cmd/lk/app_target_test.go b/cmd/lk/app_target_test.go new file mode 100644 index 00000000..458f6688 --- /dev/null +++ b/cmd/lk/app_target_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestResolveAppTargetNamed(t *testing.T) { + name, dir, err := resolveAppTarget("my-agent") + require.NoError(t, err) + require.Equal(t, "my-agent", name) + require.Equal(t, "my-agent", dir) +} + +func TestResolveAppTargetCurrentDir(t *testing.T) { + root := t.TempDir() + + empty := filepath.Join(root, "voice-app") + require.NoError(t, os.Mkdir(empty, 0o755)) + t.Chdir(empty) + name, dir, err := resolveAppTarget(".") + require.NoError(t, err) + require.Equal(t, "voice-app", name) + require.Equal(t, ".", dir) + + busy := filepath.Join(root, "busy") + require.NoError(t, os.Mkdir(busy, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(busy, "notes.txt"), nil, 0o644)) + t.Chdir(busy) + _, _, err = resolveAppTarget(".") + require.ErrorContains(t, err, "isn't empty") + + badName := filepath.Join(root, "has space") + require.NoError(t, os.Mkdir(badName, 0o755)) + t.Chdir(badName) + _, _, err = resolveAppTarget(".") + require.ErrorContains(t, err, "can't be used as the app name") +} diff --git a/pkg/util/fs.go b/pkg/util/fs.go index 71243d02..69ec45c4 100644 --- a/pkg/util/fs.go +++ b/pkg/util/fs.go @@ -61,7 +61,8 @@ func CopyFile(src, dest string) error { return nil } -// Safely move a directory across filesystems, preserving permissions +// Safely move a directory across filesystems, preserving permissions. dest +// must not exist. func MoveDir(src, dest string) error { if _, err := os.Stat(dest); err == nil { return fmt.Errorf("destination directory already exists: %s", dest) @@ -73,6 +74,38 @@ func MoveDir(src, dest string) error { return fmt.Errorf("failed to create destination directory: %w", err) } + return moveTree(src, dest) +} + +// MoveDirInto is MoveDir for a destination directory that already exists and +// is empty (see IsEmptyDir). +func MoveDirInto(src, dest string) error { + empty, err := IsEmptyDir(dest) + if err != nil { + return err + } + if !empty { + return fmt.Errorf("destination directory isn't empty: %s", dest) + } + return moveTree(src, dest) +} + +// IsEmptyDir reports whether dir has no entries, ignoring macOS's .DS_Store. +func IsEmptyDir(dir string) (bool, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return false, err + } + for _, e := range entries { + if e.Name() != ".DS_Store" { + return false, nil + } + } + return true, nil +} + +// moveTree copies src into the existing directory dest, then removes src. +func moveTree(src, dest string) error { err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error { if err != nil { return err diff --git a/pkg/util/fs_test.go b/pkg/util/fs_test.go new file mode 100644 index 00000000..ce1e736e --- /dev/null +++ b/pkg/util/fs_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "os" + "path/filepath" + "testing" +) + +func writeTestFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestIsEmptyDir(t *testing.T) { + dir := t.TempDir() + if empty, err := IsEmptyDir(dir); err != nil || !empty { + t.Fatalf("new dir: empty=%v err=%v, want empty", empty, err) + } + + writeTestFile(t, filepath.Join(dir, ".DS_Store"), "") + if empty, _ := IsEmptyDir(dir); !empty { + t.Fatal("a dir with only .DS_Store should count as empty") + } + + writeTestFile(t, filepath.Join(dir, ".git", "HEAD"), "ref: refs/heads/main") + if empty, _ := IsEmptyDir(dir); empty { + t.Fatal("a dir with .git shouldn't count as empty") + } +} + +func TestMoveDirInto(t *testing.T) { + src, dest := t.TempDir(), t.TempDir() + writeTestFile(t, filepath.Join(src, "README.md"), "hello") + writeTestFile(t, filepath.Join(src, "src", "agent.py"), "print('hi')") + writeTestFile(t, filepath.Join(dest, ".DS_Store"), "") + + if err := MoveDirInto(src, dest); err != nil { + t.Fatal(err) + } + for _, f := range []string{"README.md", filepath.Join("src", "agent.py")} { + if _, err := os.Stat(filepath.Join(dest, f)); err != nil { + t.Errorf("%s wasn't moved: %v", f, err) + } + } + if _, err := os.Stat(src); !os.IsNotExist(err) { + t.Errorf("source should be removed, got err=%v", err) + } +} + +func TestMoveDirIntoNonEmpty(t *testing.T) { + src, dest := t.TempDir(), t.TempDir() + writeTestFile(t, filepath.Join(src, "README.md"), "template") + writeTestFile(t, filepath.Join(dest, "notes.txt"), "mine") + + if err := MoveDirInto(src, dest); err == nil { + t.Fatal("moving into a non-empty dir should fail") + } + if b, _ := os.ReadFile(filepath.Join(dest, "notes.txt")); string(b) != "mine" { + t.Error("existing files must be left alone") + } +} + +func TestMoveDirExisting(t *testing.T) { + src, dest := t.TempDir(), t.TempDir() + if err := MoveDir(src, dest); err == nil { + t.Fatal("MoveDir should refuse an existing destination") + } +} From 3efe8e002a73bafde8651c0c4c9494283429bdfa Mon Sep 17 00:00:00 2001 From: royalfig Date: Fri, 9 Oct 2026 17:42:01 -0400 Subject: [PATCH 03/12] feat(console): playback audio metrics and multiple agent connections The pipeline measures the agent's playback audio as well as the microphone, and can hold several agent connections and switch the active one. Co-Authored-By: Claude Opus 5.5 --- pkg/console/pipeline.go | 270 +++++++++++++++++++++++++---------- pkg/console/pipeline_test.go | 157 ++++++++++++++++++++ 2 files changed, 348 insertions(+), 79 deletions(-) create mode 100644 pkg/console/pipeline_test.go diff --git a/pkg/console/pipeline.go b/pkg/console/pipeline.go index a975a7bc..91597739 100644 --- a/pkg/console/pipeline.go +++ b/pkg/console/pipeline.go @@ -47,8 +47,11 @@ type AudioPipeline struct { outputStream *portaudio.Stream apmInst *apm.APM noAEC bool - conn net.Conn - connMu sync.Mutex // protects writes to conn + conn net.Conn // the active agent: hears the mic, plays through the speakers + connMu sync.Mutex // protects conn and writes to every agent's conn + // switchMu keeps the active agent from changing while a message from + // an agent is handled. SetActive holds it too. + switchMu sync.Mutex captureRing *RingBuffer playbackRing *RingBuffer @@ -63,16 +66,20 @@ type AudioPipeline struct { ready chan struct{} readyOnce sync.Once - // flushCancel cancels the current waitForDrainAndAck goroutine. - // Only accessed from the tcpReader goroutine. + // flushCancel cancels the current waitForDrainAndAck goroutine, which + // acknowledges flushConn's playback once its queued audio has played. + flushMu sync.Mutex flushCancel context.CancelFunc + flushConn net.Conn - mu sync.Mutex - fftBands [NumFFTBands]float64 - muted bool - paused bool // true when audio I/O is paused (e.g. text mode); mic frames are not sent to the agent - level float64 // capture level in dB - playing bool // true when outputting real audio (not silence) + mu sync.Mutex + fftBands [NumFFTBands]float64 + outFFTBands [NumFFTBands]float64 // bands of the agent audio being played + outLevel float64 // playback level in dB + muted bool + paused bool // true when audio I/O is paused (e.g. text mode); mic frames are not sent to the agent + level float64 // capture level in dB + playing bool // true when outputting real audio (not silence) cancel context.CancelFunc audioCtx context.Context // stored so EnableAudio can start goroutines @@ -197,7 +204,7 @@ func (p *AudioPipeline) Start(ctx context.Context) error { // Always run the TCP reader for events/responses. p.wg.Add(1) - go p.tcpReader(ctx) + go p.tcpReader(ctx, p.conn) // Start audio loops if devices are available. if p.HasAudio() { @@ -248,6 +255,42 @@ func (p *AudioPipeline) writeMessage(msg *agent.AgentSessionMessage) error { return WriteSessionMessage(p.conn, msg) } +// AddConn connects another agent to the pipeline. It starts inactive: its +// audio and events are dropped until SetActive makes it the active agent. +// Its reader stops when conn closes; the caller owns conn. +func (p *AudioPipeline) AddConn(ctx context.Context, conn net.Conn) { + p.wg.Add(1) + go p.tcpReader(ctx, conn) +} + +// SetActive makes conn, from NewPipeline or AddConn, the agent that hears +// the microphone and plays through the speakers. Audio still queued from the +// previous agent is dropped, and a playback it waits on is acknowledged. +func (p *AudioPipeline) SetActive(conn net.Conn) { + p.switchMu.Lock() + defer p.switchMu.Unlock() + + p.connMu.Lock() + p.conn = conn + p.connMu.Unlock() + + // Cancel under flushMu, so waitForDrainAndAck can't also claim the flush. + p.flushMu.Lock() + pending, flushConn := p.flushCancel != nil, p.flushConn + if pending { + p.flushCancel() + } + p.flushCancel, p.flushConn = nil, nil + p.flushMu.Unlock() + if p.playbackRing != nil { + p.playbackRing.Reset() + } + if pending { + // Its queued audio was just dropped, so its playback is over. + p.ackPlayback(flushConn) + } +} + func (p *AudioPipeline) SendRequest(req *agent.SessionRequest) error { return p.writeMessage(&agent.AgentSessionMessage{ Message: &agent.AgentSessionMessage_Request{Request: req}, @@ -291,6 +334,21 @@ func (p *AudioPipeline) FFTBands() [NumFFTBands]float64 { return p.fftBands } +// PlaybackFFTBands returns the frequency bands of the agent audio currently +// being played, in the same form as FFTBands. +func (p *AudioPipeline) PlaybackFFTBands() [NumFFTBands]float64 { + p.mu.Lock() + defer p.mu.Unlock() + return p.outFFTBands +} + +// PlaybackLevel returns the level of the agent audio being played, in dB. +func (p *AudioPipeline) PlaybackLevel() float64 { + p.mu.Lock() + defer p.mu.Unlock() + return p.outLevel +} + func (p *AudioPipeline) IsPlaying() bool { p.mu.Lock() defer p.mu.Unlock() @@ -370,6 +428,7 @@ func (p *AudioPipeline) speakerLoop(ctx context.Context) { p.mu.Lock() p.playing = pn > 0 p.mu.Unlock() + p.computePlaybackMetrics(playbackBuf) // ProcessRender then ProcessCapture — both in this goroutine, // right next to each other, no mutex needed. @@ -423,78 +482,108 @@ func (p *AudioPipeline) speakerLoop(ctx context.Context) { } } -// tcpReader reads messages from the agent over TCP and dispatches them. -func (p *AudioPipeline) tcpReader(ctx context.Context) { +// tcpReader reads messages from an agent over TCP and dispatches them. +func (p *AudioPipeline) tcpReader(ctx context.Context, conn net.Conn) { defer p.wg.Done() for { - msg, err := ReadSessionMessage(p.conn) + msg, err := ReadSessionMessage(conn) if err != nil { return } p.readyOnce.Do(func() { close(p.ready) }) + p.dispatch(ctx, conn, msg) + } +} - switch m := msg.Message.(type) { - case *agent.AgentSessionMessage_AudioOutput: - // No audio sink in text mode (rings allocated lazily on Ctrl+T). - if p.playbackRing != nil { - p.playbackRing.Write(BytesToSamples(m.AudioOutput.Data)) - } +// dispatch handles a message from conn's agent. An inactive agent's audio +// and events are dropped, and its playback is acknowledged at once so it +// never waits on audio that won't play. +func (p *AudioPipeline) dispatch(ctx context.Context, conn net.Conn, msg *agent.AgentSessionMessage) { + p.switchMu.Lock() + defer p.switchMu.Unlock() + + // SetActive changes p.conn only while it holds switchMu. + if conn != p.conn { + if _, ok := msg.Message.(*agent.AgentSessionMessage_AudioPlaybackFlush); ok { + p.ackPlayback(conn) + } + return + } - case *agent.AgentSessionMessage_Event: - select { - case p.Events <- m.Event: - default: - } + switch m := msg.Message.(type) { + case *agent.AgentSessionMessage_AudioOutput: + // No audio sink in text mode (rings allocated lazily on Ctrl+T). + if p.playbackRing != nil { + p.playbackRing.Write(BytesToSamples(m.AudioOutput.Data)) + } - case *agent.AgentSessionMessage_AudioPlaybackClear: - if p.flushCancel != nil { - p.flushCancel() - p.flushCancel = nil - } - if p.playbackRing != nil { - p.playbackRing.Reset() - } + case *agent.AgentSessionMessage_Event: + select { + case p.Events <- m.Event: + default: + } - case *agent.AgentSessionMessage_AudioPlaybackFlush: - // Without an audio sink there's nothing to drain, so ack the - // agent's playback turn immediately instead of dereferencing a - // nil playback ring in waitForDrainAndAck. - if p.playbackRing == nil { - p.sendPlaybackFinished() - break - } - if p.flushCancel != nil { - p.flushCancel() - } - flushCtx, cancel := context.WithCancel(ctx) - p.flushCancel = cancel - go p.waitForDrainAndAck(flushCtx) - - case *agent.AgentSessionMessage_Response: - // Forward response so the TUI knows the request completed. - // Don't synthesize ConversationItemAdded — those arrive via the - // event stream already. - if m.Response != nil { - select { - case p.Responses <- m.Response: - default: - } + case *agent.AgentSessionMessage_AudioPlaybackClear: + p.flushMu.Lock() + if p.flushCancel != nil { + p.flushCancel() + p.flushCancel, p.flushConn = nil, nil + } + p.flushMu.Unlock() + if p.playbackRing != nil { + p.playbackRing.Reset() + } + + case *agent.AgentSessionMessage_AudioPlaybackFlush: + // Without an audio sink there's nothing to drain, so ack the + // agent's playback turn immediately instead of dereferencing a + // nil playback ring in waitForDrainAndAck. + if p.playbackRing == nil { + p.ackPlayback(conn) + break + } + p.flushMu.Lock() + if p.flushCancel != nil { + p.flushCancel() + } + flushCtx, cancel := context.WithCancel(ctx) + p.flushCancel, p.flushConn = cancel, conn + p.flushMu.Unlock() + go p.waitForDrainAndAck(flushCtx, conn) + + case *agent.AgentSessionMessage_Response: + // Forward response so the TUI knows the request completed. + // Don't synthesize ConversationItemAdded — those arrive via the + // event stream already. + if m.Response != nil { + select { + case p.Responses <- m.Response: + default: } } } } -func (p *AudioPipeline) sendPlaybackFinished() { - _ = p.writeMessage(&agent.AgentSessionMessage{ +// ackPlayback tells conn's agent that its queued audio has finished playing. +func (p *AudioPipeline) ackPlayback(conn net.Conn) { + p.connMu.Lock() + defer p.connMu.Unlock() + _ = WriteSessionMessage(conn, playbackFinishedMessage()) +} + +func playbackFinishedMessage() *agent.AgentSessionMessage { + return &agent.AgentSessionMessage{ Message: &agent.AgentSessionMessage_AudioPlaybackFinished{ AudioPlaybackFinished: &agent.AgentSessionMessage_ConsoleIO_AudioPlaybackFinished{}, }, - }) + } } -func (p *AudioPipeline) waitForDrainAndAck(ctx context.Context) { +// waitForDrainAndAck acknowledges conn's playback once its queued audio has +// played, unless ctx is cancelled first. +func (p *AudioPipeline) waitForDrainAndAck(ctx context.Context, conn net.Conn) { for p.playbackRing.Available() > 0 { select { case <-ctx.Done(): @@ -503,15 +592,49 @@ func (p *AudioPipeline) waitForDrainAndAck(ctx context.Context) { } time.Sleep(5 * time.Millisecond) } - select { - case <-ctx.Done(): + // Claim the flush under flushMu. SetActive acks a flush that it + // cancels, so only one of them sends the ack. + p.flushMu.Lock() + if ctx.Err() != nil { + p.flushMu.Unlock() return - default: } - p.sendPlaybackFinished() + p.flushCancel() + p.flushCancel, p.flushConn = nil, nil + p.flushMu.Unlock() + p.ackPlayback(conn) } func (p *AudioPipeline) computeMetrics(samples []int16) { + bands, db, decay := analyzeAudio(samples) + p.mu.Lock() + smoothBands(&p.fftBands, bands, decay) + p.level = db + p.mu.Unlock() +} + +func (p *AudioPipeline) computePlaybackMetrics(samples []int16) { + bands, db, decay := analyzeAudio(samples) + p.mu.Lock() + smoothBands(&p.outFFTBands, bands, decay) + p.outLevel = db + p.mu.Unlock() +} + +// smoothBands rises to new peaks at once and decays toward lower values. +func smoothBands(dst *[NumFFTBands]float64, bands [NumFFTBands]float64, decay float64) { + for b := range dst { + if bands[b] > dst[b]*decay { + dst[b] = bands[b] + } else { + dst[b] *= decay + } + } +} + +// analyzeAudio returns the normalized frequency bands and RMS level (dB) of a +// frame, and the decay factor for smoothing bands across frames. +func analyzeAudio(samples []int16) (bands [NumFFTBands]float64, db float64, decay float64) { n := len(samples) sr := float64(SampleRate) @@ -571,7 +694,6 @@ func (p *AudioPipeline) computeMetrics(samples []int16) { // Mean power → dB → normalize to [0,1] const floorDB, hotDB = -70.0, -20.0 - var bands [NumFFTBands]float64 for b := range nb { c := cnts[b] if c == 0 { @@ -599,7 +721,7 @@ func (p *AudioPipeline) computeMetrics(samples []int16) { } // Exponential decay smoothing (~100ms time constant) - decay := math.Exp(-float64(n) / sr / 0.1) + decay = math.Exp(-float64(n) / sr / 0.1) // RMS level in dB var sum float64 @@ -608,18 +730,8 @@ func (p *AudioPipeline) computeMetrics(samples []int16) { sum += v * v } rms := math.Sqrt(sum / float64(n)) - db := 20 * math.Log10(rms+1e-10) - - p.mu.Lock() - for b := range nb { - if bands[b] > p.fftBands[b]*decay { - p.fftBands[b] = bands[b] - } else { - p.fftBands[b] *= decay - } - } - p.level = db - p.mu.Unlock() + db = 20 * math.Log10(rms+1e-10) + return bands, db, decay } func SamplesToBytes(samples []int16) []byte { diff --git a/pkg/console/pipeline_test.go b/pkg/console/pipeline_test.go new file mode 100644 index 00000000..f9375ed4 --- /dev/null +++ b/pkg/console/pipeline_test.go @@ -0,0 +1,157 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package console + +import ( + "context" + "net" + "testing" + "time" + + "github.com/livekit/protocol/livekit/agent" +) + +func stateEvent(state agent.AgentState) *agent.AgentSessionMessage { + return &agent.AgentSessionMessage{Message: &agent.AgentSessionMessage_Event{Event: &agent.AgentSessionEvent{ + Event: &agent.AgentSessionEvent_AgentStateChanged_{AgentStateChanged: &agent.AgentSessionEvent_AgentStateChanged{NewState: state}}, + }}} +} + +func flushMessage() *agent.AgentSessionMessage { + return &agent.AgentSessionMessage{Message: &agent.AgentSessionMessage_AudioPlaybackFlush{ + AudioPlaybackFlush: &agent.AgentSessionMessage_ConsoleIO_AudioPlaybackFlush{}, + }} +} + +// nextEvent returns the next event's new agent state, or -1 if none arrives. +func nextEvent(t *testing.T, p *AudioPipeline) agent.AgentState { + t.Helper() + select { + case ev := <-p.Events: + return ev.GetAgentStateChanged().GetNewState() + case <-time.After(200 * time.Millisecond): + return -1 + } +} + +func send(t *testing.T, conn net.Conn, msg *agent.AgentSessionMessage) { + t.Helper() + if err := WriteSessionMessage(conn, msg); err != nil { + t.Fatal(err) + } +} + +func TestPipelineSwitchesAgents(t *testing.T) { + // No audio devices: the pipeline only relays session messages. + first, firstAgent := net.Pipe() + second, secondAgent := net.Pipe() + p, err := NewPipeline(PipelineConfig{Conn: first}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Start(ctx) + time.Sleep(20 * time.Millisecond) + p.AddConn(ctx, second) + + // The second agent starts inactive: its events are dropped, and its + // playback is acknowledged right away. + send(t, firstAgent, stateEvent(agent.AgentState_AS_LISTENING)) + if got := nextEvent(t, p); got != agent.AgentState_AS_LISTENING { + t.Fatalf("active agent's event: got %v", got) + } + send(t, secondAgent, stateEvent(agent.AgentState_AS_SPEAKING)) + if got := nextEvent(t, p); got != -1 { + t.Fatalf("inactive agent's event should be dropped, got %v", got) + } + acked := make(chan *agent.AgentSessionMessage, 1) + go func() { + msg, _ := ReadSessionMessage(secondAgent) + acked <- msg + }() + send(t, secondAgent, flushMessage()) + select { + case msg := <-acked: + if msg.GetAudioPlaybackFinished() == nil { + t.Fatalf("inactive agent's flush: got %v, want playback finished", msg) + } + case <-time.After(time.Second): + t.Fatal("inactive agent's flush wasn't acknowledged") + } + + // After switching, it's the other way around. + p.SetActive(second) + send(t, secondAgent, stateEvent(agent.AgentState_AS_THINKING)) + if got := nextEvent(t, p); got != agent.AgentState_AS_THINKING { + t.Fatalf("newly active agent's event: got %v", got) + } + send(t, firstAgent, stateEvent(agent.AgentState_AS_SPEAKING)) + if got := nextEvent(t, p); got != -1 { + t.Fatalf("now-inactive agent's event should be dropped, got %v", got) + } +} + +func TestPipelineSwitchAcksPendingPlayback(t *testing.T) { + first, firstAgent := net.Pipe() + second, secondAgent := net.Pipe() + defer firstAgent.Close() + defer secondAgent.Close() + p, err := NewPipeline(PipelineConfig{Conn: first}) + if err != nil { + t.Fatal(err) + } + // A playback ring that nothing drains, so a flush waits for its audio. + p.playbackRing = NewRingBuffer(SamplesPerFrame) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Start(ctx) + time.Sleep(20 * time.Millisecond) + p.AddConn(ctx, second) + + send(t, firstAgent, &agent.AgentSessionMessage{Message: &agent.AgentSessionMessage_AudioOutput{ + AudioOutput: &agent.AgentSessionMessage_ConsoleIO_AudioFrame{Data: make([]byte, 960)}, + }}) + send(t, firstAgent, flushMessage()) + time.Sleep(20 * time.Millisecond) + + read := func(c net.Conn) <-chan *agent.AgentSessionMessage { + ch := make(chan *agent.AgentSessionMessage, 1) + go func() { + if msg, err := ReadSessionMessage(c); err == nil { + ch <- msg + } + }() + return ch + } + firstAcks, secondAcks := read(firstAgent), read(secondAgent) + + // Switching drops the first agent's queued audio, so its playback is + // over: the first agent gets the ack, and the second gets none. + p.SetActive(second) + select { + case msg := <-firstAcks: + if msg.GetAudioPlaybackFinished() == nil { + t.Fatalf("first agent: got %v, want playback finished", msg) + } + case <-time.After(time.Second): + t.Fatal("the first agent's pending playback wasn't acknowledged") + } + select { + case msg := <-secondAcks: + t.Fatalf("the second agent got a message it didn't ask for: %v", msg) + case <-time.After(100 * time.Millisecond): + } +} From cb434b33b19776ff51dfa35a329369a32be9965b Mon Sep 17 00:00:00 2001 From: royalfig Date: Fri, 9 Oct 2026 17:42:05 -0400 Subject: [PATCH 04/12] feat(overlay): native macOS overlay with a blurred backdrop and web UI Co-Authored-By: Claude Opus 5.5 --- pkg/overlay/overlay.go | 111 ++++ pkg/overlay/overlay_darwin.go | 122 +++++ pkg/overlay/overlay_darwin.h | 37 ++ pkg/overlay/overlay_darwin.m | 210 ++++++++ pkg/overlay/overlay_other.go | 31 ++ pkg/overlay/ui/index.html | 930 ++++++++++++++++++++++++++++++++++ 6 files changed, 1441 insertions(+) create mode 100644 pkg/overlay/overlay.go create mode 100644 pkg/overlay/overlay_darwin.go create mode 100644 pkg/overlay/overlay_darwin.h create mode 100644 pkg/overlay/overlay_darwin.m create mode 100644 pkg/overlay/overlay_other.go create mode 100644 pkg/overlay/ui/index.html diff --git a/pkg/overlay/overlay.go b/pkg/overlay/overlay.go new file mode 100644 index 00000000..4d63767c --- /dev/null +++ b/pkg/overlay/overlay.go @@ -0,0 +1,111 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package overlay shows the assistant's full-screen overlay: a native window +// that blurs the desktop behind a transparent web view running ui/index.html. +// +// Go drives the page with JSON messages (Send) and receives the page's +// messages as Events. The page protocol: +// +// Go → page {"type":"state","state":"connecting|idle|listening|thinking|speaking","label":"optional"} +// {"type":"bands","bands":[0..1, ...]} +// {"type":"transcript","turn":"...","role":"user|agent","segment":0,"text":"..."} +// {"type":"snippet","turn":"...","id":"...","title":"...","lang":"python|shell","code":"...","copied":bool} +// {"type":"copied","id":"..."} +// {"type":"tool","turn":"...","id":"...","label":"...","detail":"...","status":"running|done|error"} +// {"type":"notice","text":"...","sticky":bool} +// {"type":"error","title":"...","detail":"...","fix":"command"} +// {"type":"speaker","name":"...","guest":bool} +// {"type":"pipeline",...} {"type":"eot",...} {"type":"metrics",...} {"type":"usage",...} (X-ray) +// {"type":"dismiss"} +// {"type":"clear"} +// {"type":"confirm","turn":"...","id":"...","heading":"...","title":"...","options":[{"id":"...","name":"..."}]} +// {"type":"confirm_done","id":"...","option":"..."} +// page → Go {"type":"ready"} {"type":"copy","id":"..."} {"type":"replay"} {"type":"close"} +// {"type":"confirm","id":"...","option":"..."} {"type":"copy_text","text":"..."} +// {"type":"back"} {"type":"mute","on":bool} +// +// A transcript message carries the full text so far of one segment (a +// paragraph) of a turn. Snippets are added to their turn after the segments +// before them. The page also sends {"type":"peek","on":bool}, which the +// overlay handles itself by fading the desktop blur. +// +// Only macOS is supported for now. +package overlay + +import ( + _ "embed" + "errors" + "sync" +) + +//go:embed ui/index.html +var pageHTML string + +// ErrUnsupported is returned by Run on platforms without an overlay. +var ErrUnsupported = errors.New("the assistant overlay is only supported on macOS") + +// Options configure the overlay window. +type Options struct { + // Windowed opens a regular floating window instead of covering the screen. + Windowed bool + // Inspectable enables the Safari Web Inspector for the page. + Inspectable bool +} + +// Event is a message from the overlay page. +type Event struct { + Type string `json:"type"` + ID string `json:"id,omitempty"` + On bool `json:"on,omitempty"` + // Option is the chosen option of a confirm event. + Option string `json:"option,omitempty"` + // Text is the text of a copy_text event. + Text string `json:"text,omitempty"` +} + +// Overlay is a single overlay window. A process can run one at a time. +type Overlay struct { + opts Options + events chan Event + done chan struct{} + closeOnce sync.Once +} + +// New creates an overlay. Call Run to show it. +func New(opts Options) *Overlay { + return &Overlay{ + opts: opts, + events: make(chan Event, 64), + done: make(chan struct{}), + } +} + +// Events returns messages from the page. Messages are dropped if nobody reads +// them. +func (o *Overlay) Events() <-chan Event { + return o.events +} + +// Done is closed after the window closes and Run returns. +func (o *Overlay) Done() <-chan struct{} { + return o.done +} + +func (o *Overlay) emit(ev Event) { + select { + case o.events <- ev: + default: + } +} diff --git a/pkg/overlay/overlay_darwin.go b/pkg/overlay/overlay_darwin.go new file mode 100644 index 00000000..b1c3331c --- /dev/null +++ b/pkg/overlay/overlay_darwin.go @@ -0,0 +1,122 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package overlay + +/* +#cgo CFLAGS: -Wno-deprecated-declarations +#cgo LDFLAGS: -framework Cocoa -framework WebKit +#include +#include "overlay_darwin.h" +*/ +import "C" + +import ( + "context" + "encoding/json" + "errors" + "runtime" + "sync" + "unsafe" +) + +// Cocoa's event loop must run on the process's main thread. Locking here, in +// init, keeps the main goroutine (and so every command's Action) on it. +func init() { + runtime.LockOSThread() +} + +var ( + activeMu sync.Mutex + active *Overlay +) + +// Run shows the overlay and blocks until it closes, either from the page +// (Esc), Close, or ctx being done. It must be called from the main goroutine. +func (o *Overlay) Run(ctx context.Context) error { + if C.lk_overlay_is_main_thread() == 0 { + return errors.New("overlay: Run must be called on the main thread") + } + activeMu.Lock() + if active != nil { + activeMu.Unlock() + return errors.New("overlay: already running") + } + active = o + activeMu.Unlock() + + go func() { + select { + case <-ctx.Done(): + o.Close() + case <-o.done: + } + }() + + html := C.CString(pageHTML) + defer C.free(unsafe.Pointer(html)) + C.lk_overlay_run(html, cbool(o.opts.Windowed), cbool(o.opts.Inspectable)) + + activeMu.Lock() + active = nil + activeMu.Unlock() + close(o.done) + return nil +} + +// Send delivers a message to the page. Messages sent before the page's +// "ready" event are lost. +func (o *Overlay) Send(msg any) error { + b, err := json.Marshal(msg) + if err != nil { + return err + } + cmsg := C.CString(string(b)) + defer C.free(unsafe.Pointer(cmsg)) + C.lk_overlay_send(cmsg) + return nil +} + +// Close fades the overlay out and makes Run return. +func (o *Overlay) Close() { + o.closeOnce.Do(func() { C.lk_overlay_close() }) +} + +//export lkOverlayMessage +func lkOverlayMessage(msg *C.char) { + var ev Event + if err := json.Unmarshal([]byte(C.GoString(msg)), &ev); err != nil { + return + } + activeMu.Lock() + o := active + activeMu.Unlock() + if o == nil { + return + } + switch ev.Type { + case "close": + o.Close() + case "peek": + C.lk_overlay_set_peek(cbool(ev.On)) + } + o.emit(ev) +} + +func cbool(b bool) C.int { + if b { + return 1 + } + return 0 +} diff --git a/pkg/overlay/overlay_darwin.h b/pkg/overlay/overlay_darwin.h new file mode 100644 index 00000000..ed0076e3 --- /dev/null +++ b/pkg/overlay/overlay_darwin.h @@ -0,0 +1,37 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef LK_OVERLAY_DARWIN_H +#define LK_OVERLAY_DARWIN_H + +// lk_overlay_run opens the overlay window and runs the Cocoa event loop until +// lk_overlay_close is called. It must be called on the process's main thread. +void lk_overlay_run(const char *html, int windowed, int inspectable); + +// lk_overlay_send passes a JSON message to window.lk.receive in the overlay +// page. Safe from any thread. +void lk_overlay_send(const char *json); + +// lk_overlay_close fades the window out and stops the event loop. Safe from +// any thread. +void lk_overlay_close(void); + +// lk_overlay_set_peek fades the desktop blur out (on) or back in (off), so the +// user can see the apps behind the overlay. Safe from any thread. +void lk_overlay_set_peek(int on); + +// lk_overlay_is_main_thread reports whether the caller is on the main thread. +int lk_overlay_is_main_thread(void); + +#endif diff --git a/pkg/overlay/overlay_darwin.m b/pkg/overlay/overlay_darwin.m new file mode 100644 index 00000000..323799a0 --- /dev/null +++ b/pkg/overlay/overlay_darwin.m @@ -0,0 +1,210 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import +#include + +#include "overlay_darwin.h" +#include "_cgo_export.h" + +// Built without ARC. The window, web view, and handler live for the rest of +// the process once created, so they're never released. + +// Borderless windows can't become key by default, which would leave the page +// without keyboard input (Esc to close). +@interface LKOverlayWindow : NSWindow +@end + +@implementation LKOverlayWindow +- (BOOL)canBecomeKeyWindow { + return YES; +} +- (BOOL)canBecomeMainWindow { + return YES; +} +// Closing the window, such as with its close button in windowed mode, +// closes the overlay the same as when the page asks to. Closing only the +// window would leave Run blocked. +- (void)close { + lkOverlayMessage((char *)"{\"type\":\"close\"}"); +} +@end + +@interface LKOverlayHandler : NSObject +@end + +@implementation LKOverlayHandler +- (void)userContentController:(WKUserContentController *)controller + didReceiveScriptMessage:(WKScriptMessage *)message { + if (![message.body isKindOfClass:[NSString class]]) { + return; + } + lkOverlayMessage((char *)[(NSString *)message.body UTF8String]); +} +@end + +static LKOverlayWindow *gWindow; +static WKWebView *gWebView; +static NSVisualEffectView *gBlur; +static BOOL gClosing; + +void lk_overlay_run(const char *html, int windowed, int inspectable) { + @autoreleasepool { + [NSApplication sharedApplication]; + // No Dock icon or menu bar takeover: the overlay floats over whatever + // app the user is in. + [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory]; + + NSScreen *screen = [NSScreen mainScreen]; + NSRect frame = screen.frame; + NSWindowStyleMask style = NSWindowStyleMaskBorderless; + if (windowed) { + NSRect visible = screen.visibleFrame; + frame = NSInsetRect(visible, visible.size.width * 0.12, visible.size.height * 0.12); + style = NSWindowStyleMaskTitled | NSWindowStyleMaskClosable | + NSWindowStyleMaskResizable | NSWindowStyleMaskFullSizeContentView; + } + + gWindow = [[LKOverlayWindow alloc] initWithContentRect:frame + styleMask:style + backing:NSBackingStoreBuffered + defer:NO]; + gWindow.opaque = NO; + gWindow.backgroundColor = [NSColor clearColor]; + gWindow.releasedWhenClosed = NO; + gWindow.appearance = [NSAppearance appearanceNamed:NSAppearanceNameDarkAqua]; + gWindow.alphaValue = 0; + if (windowed) { + gWindow.titlebarAppearsTransparent = YES; + gWindow.titleVisibility = NSWindowTitleHidden; + gWindow.level = NSFloatingWindowLevel; + } else { + gWindow.hasShadow = NO; + // Above the menu bar and Dock. + gWindow.level = NSStatusWindowLevel; + gWindow.collectionBehavior = NSWindowCollectionBehaviorCanJoinAllSpaces | + NSWindowCollectionBehaviorFullScreenAuxiliary | + NSWindowCollectionBehaviorStationary | + NSWindowCollectionBehaviorIgnoresCycle; + } + + // The blur and the web view are siblings, so peek can fade the blur + // without fading the page. + NSView *root = [[NSView alloc] initWithFrame:NSMakeRect(0, 0, frame.size.width, frame.size.height)]; + root.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable; + gWindow.contentView = root; + + // Blurs whatever is behind the window: the desktop and other apps. + NSVisualEffectView *blur = [[NSVisualEffectView alloc] initWithFrame:root.bounds]; + blur.blendingMode = NSVisualEffectBlendingModeBehindWindow; + blur.material = NSVisualEffectMaterialFullScreenUI; + blur.state = NSVisualEffectStateActive; + blur.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable; + [root addSubview:blur]; + gBlur = blur; + + WKWebViewConfiguration *config = [[WKWebViewConfiguration alloc] init]; + // Earcons play from Web Audio as soon as the overlay opens, without a click first. + config.mediaTypesRequiringUserActionForPlayback = WKAudiovisualMediaTypeNone; + [config.userContentController addScriptMessageHandler:[[LKOverlayHandler alloc] init] name:@"lk"]; + + gWebView = [[WKWebView alloc] initWithFrame:root.bounds configuration:config]; + gWebView.autoresizingMask = NSViewWidthSizable | NSViewHeightSizable; + // Transparent page background so the blur shows through. + [gWebView setValue:@NO forKey:@"drawsBackground"]; + if (inspectable) { + if (@available(macOS 13.3, *)) { + gWebView.inspectable = YES; + } + } + [root addSubview:gWebView]; + [gWebView loadHTMLString:[NSString stringWithUTF8String:html] baseURL:nil]; + + [gWindow makeKeyAndOrderFront:nil]; + [gWindow makeFirstResponder:gWebView]; + [NSApp activateIgnoringOtherApps:YES]; + [NSAnimationContext runAnimationGroup:^(NSAnimationContext *ctx) { + ctx.duration = 0.35; + gWindow.animator.alphaValue = 1; + }]; + + [NSApp run]; + + [gWindow orderOut:nil]; + } +} + +// Messages go through one fixed function body with the JSON as an argument. +// Evaluating each message as its own script source makes WebKit compile and +// cache a new script per message, many times a second. +static NSString *const kReceiveBody = @"if (window.lk) window.lk.receive(JSON.parse(msg));"; + +void lk_overlay_send(const char *json) { + @autoreleasepool { + NSString *msg = [NSString stringWithUTF8String:json]; + dispatch_async(dispatch_get_main_queue(), ^{ + [gWebView callAsyncJavaScript:kReceiveBody + arguments:@{@"msg" : msg} + inFrame:nil + inContentWorld:WKContentWorld.pageWorld + completionHandler:nil]; + }); + } +} + +static void stopApp(void) { + [NSApp stop:nil]; + // -stop: takes effect after the next event, so post one. + NSEvent *event = [NSEvent otherEventWithType:NSEventTypeApplicationDefined + location:NSZeroPoint + modifierFlags:0 + timestamp:0 + windowNumber:0 + context:nil + subtype:0 + data1:0 + data2:0]; + [NSApp postEvent:event atStart:YES]; +} + +void lk_overlay_close(void) { + dispatch_async(dispatch_get_main_queue(), ^{ + if (gWindow == nil || gClosing) { + return; + } + gClosing = YES; + [NSAnimationContext + runAnimationGroup:^(NSAnimationContext *ctx) { + ctx.duration = 0.25; + gWindow.animator.alphaValue = 0; + } + completionHandler:^{ + stopApp(); + }]; + }); +} + +void lk_overlay_set_peek(int on) { + dispatch_async(dispatch_get_main_queue(), ^{ + [NSAnimationContext runAnimationGroup:^(NSAnimationContext *ctx) { + ctx.duration = 0.2; + gBlur.animator.alphaValue = on ? 0 : 1; + }]; + }); +} + +int lk_overlay_is_main_thread(void) { + return pthread_main_np(); +} diff --git a/pkg/overlay/overlay_other.go b/pkg/overlay/overlay_other.go new file mode 100644 index 00000000..f17ee83e --- /dev/null +++ b/pkg/overlay/overlay_other.go @@ -0,0 +1,31 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !darwin + +package overlay + +import "context" + +// Run returns ErrUnsupported on this platform. +func (o *Overlay) Run(ctx context.Context) error { + close(o.done) + return ErrUnsupported +} + +// Send is a no-op on this platform. +func (o *Overlay) Send(msg any) error { return nil } + +// Close is a no-op on this platform. +func (o *Overlay) Close() {} diff --git a/pkg/overlay/ui/index.html b/pkg/overlay/ui/index.html new file mode 100644 index 00000000..c6775304 --- /dev/null +++ b/pkg/overlay/ui/index.html @@ -0,0 +1,930 @@ + + + + + +LiveKit Assistant + + + +
+ +
+
LK · LiveKit AssistantMic off
+
Talking to
+
Ready
+
M mute · X x-ray · Shift peek · ⌘C copy last · Esc close
+
+
X-ray · LK's pipeline
+
+
STT…
+
LLM…
+
TTS…
+
Turns…
+
+
+
End of turn
+
Speak to see the turn detector's estimate
+
+
+
Last reply…
+
+
+
+
Usage…
+
+
+
+ + + + + + + From 599bf5c7d7cd8a4ae8db073ae01f7ca54378e56e Mon Sep 17 00:00:00 2001 From: royalfig Date: Fri, 9 Oct 2026 17:42:05 -0400 Subject: [PATCH 05/12] feat(assistant): add hidden `lk assistant` voice assistant LK is a voice assistant in a macOS overlay. It answers from the LiveKit docs, shows code and prompts to copy, runs lk commands after the user confirms, and lets the user talk to the agent they created. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + cmd/lk/assistant.go | 396 ++++++++++++ cmd/lk/assistant_guest.go | 202 +++++++ cmd/lk/assistant_log.go | 221 +++++++ cmd/lk/assistant_log_test.go | 41 ++ cmd/lk/assistant_session.go | 932 +++++++++++++++++++++++++++++ cmd/lk/assistant_session_test.go | 174 ++++++ cmd/lk/assistant_tools.go | 542 +++++++++++++++++ cmd/lk/assistant_tools_test.go | 33 + cmd/lk/assistant_xray.go | 156 +++++ cmd/lk/main.go | 1 + pkg/assistant/agent/agent.py | 524 ++++++++++++++++ pkg/assistant/agent/pyproject.toml | 8 + pkg/assistant/assistant.go | 113 ++++ 14 files changed, 3346 insertions(+) create mode 100644 cmd/lk/assistant.go create mode 100644 cmd/lk/assistant_guest.go create mode 100644 cmd/lk/assistant_log.go create mode 100644 cmd/lk/assistant_log_test.go create mode 100644 cmd/lk/assistant_session.go create mode 100644 cmd/lk/assistant_session_test.go create mode 100644 cmd/lk/assistant_tools.go create mode 100644 cmd/lk/assistant_tools_test.go create mode 100644 cmd/lk/assistant_xray.go create mode 100644 pkg/assistant/agent/agent.py create mode 100644 pkg/assistant/agent/pyproject.toml create mode 100644 pkg/assistant/assistant.go diff --git a/.gitignore b/.gitignore index ef635899..d96f5184 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ cmd/lk/testdata/**/package-lock.json # generated at release time by release.yaml autocomplete/fish_autocomplete + +# Python bytecode from the assistant agent +__pycache__/ diff --git a/cmd/lk/assistant.go b/cmd/lk/assistant.go new file mode 100644 index 00000000..b997b3ed --- /dev/null +++ b/cmd/lk/assistant.go @@ -0,0 +1,396 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "math" + "math/rand/v2" + "os" + "strings" + "sync" + "time" + + "github.com/atotto/clipboard" + "github.com/urfave/cli/v3" + + "github.com/livekit/livekit-cli/v2/pkg/overlay" +) + +var AssistantCommands = []*cli.Command{ + { + Name: "assistant", + Usage: "Talk to the LiveKit voice assistant (preview)", + Hidden: true, + Description: `Opens the assistant overlay over the desktop and starts a voice conversation +with the LiveKit assistant, using your microphone and speakers. The assistant +answers from the LiveKit docs and shows code on screen, with a button to copy +it. + +The assistant is a LiveKit agent that runs on your machine and uses LiveKit +Inference through your LiveKit Cloud project. The first run installs it with +uv. + +Press Esc to close the overlay. With --demo, the overlay plays a scripted +conversation instead; press R to replay it.`, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "demo", + Usage: "Play a scripted conversation instead of starting the assistant", + }, + &cli.StringFlag{ + Name: "agent-dir", + Usage: "Run the assistant agent from this project directory instead of the built-in one", + Hidden: true, + }, + &cli.BoolFlag{ + Name: "windowed", + Usage: "Open in a window instead of covering the screen", + }, + &cli.BoolFlag{ + Name: "inspect", + Usage: "Enable the Safari Web Inspector for the overlay page", + }, + }, + Action: runAssistant, + }, +} + +func runAssistant(ctx context.Context, cmd *cli.Command) error { + ov := overlay.New(overlay.Options{ + Windowed: cmd.Bool("windowed"), + Inspectable: cmd.Bool("inspect"), + }) + if cmd.Bool("demo") { + demo := &assistantDemo{ov: ov, snippets: map[string]string{}} + go demo.run(ctx) + return ov.Run(ctx) + } + + sessionCtx, cancel := context.WithCancel(ctx) + defer cancel() + session := newAssistantSession(ov, cmd) + sessionErr := make(chan error, 1) + go func() { sessionErr <- session.run(sessionCtx) }() + + if err := ov.Run(ctx); err != nil { + return err + } + cancel() + if dir, err := os.Getwd(); err == nil { + if path, err := session.log.save(dir); err == nil && path != "" { + out.Statusf("Saved this conversation to %s", path) + } + } + // Give the session time to stop the agent, but always exit. + select { + case err := <-sessionErr: + return err + case <-time.After(assistantShutdownTimeout): + return nil + } +} + +// assistantShutdownTimeout bounds cleanup after the overlay closes. Stopping +// the agent takes up to about 3 seconds. +const assistantShutdownTimeout = 5 * time.Second + +// Messages sent to the overlay page. See the protocol in package overlay. +type ( + overlayState struct { + Type string `json:"type"` + State string `json:"state"` + } + overlayBands struct { + Type string `json:"type"` + Bands []float64 `json:"bands"` + } + overlayTranscript struct { + Type string `json:"type"` + Turn string `json:"turn"` + Role string `json:"role"` + Segment int `json:"segment"` + Text string `json:"text"` + } + overlaySnippet struct { + Type string `json:"type"` + Turn string `json:"turn"` + ID string `json:"id"` + Title string `json:"title"` + Lang string `json:"lang"` + Code string `json:"code"` + Copied bool `json:"copied"` + } + overlayCopied struct { + Type string `json:"type"` + ID string `json:"id"` + } + overlayTool struct { + Type string `json:"type"` + Turn string `json:"turn"` + Role string `json:"role,omitempty"` // turn role; agent if empty + ID string `json:"id"` + Label string `json:"label,omitempty"` + Detail string `json:"detail,omitempty"` + Status string `json:"status"` // running, done, or error + } + overlayNotice struct { + Type string `json:"type"` + Text string `json:"text"` + Sticky bool `json:"sticky,omitempty"` + } +) + +// demoSnippet is a code card in an agent reply. +type demoSnippet struct { + id, title, lang, code string +} + +// demoTurn is one exchange. Each agent part is a string (a paragraph of +// speech) or a demoSnippet. +type demoTurn struct { + user string + agent []any +} + +// The agent code is from the Voice AI quickstart +// (docs.livekit.io/agents/start/voice-ai-quickstart). +var demoScript = []demoTurn{ + { + user: "How do I build a voice agent in Python?", + agent: []any{ + "Start from the Python starter template. This command creates the project and adds your LiveKit Cloud credentials to it.", + demoSnippet{id: "init", title: "Terminal", lang: "shell", code: "lk agent init my-agent --template agent-starter-python\ncd my-agent"}, + "Then run lk agent dev and talk to it from the Agent Console.", + }, + }, + { + user: "Where do I change the voice?", + agent: []any{ + "In agent.py. The agent session sets the speech-to-text, language, and text-to-speech models. Change the TTS model or voice there.", + demoSnippet{id: "agent", title: "agent.py", lang: "python", code: `@server.rtc_session(agent_name="my-agent") +async def my_agent(ctx: agents.JobContext): + session = AgentSession( + stt=inference.STT(model="assemblyai/universal-3-6-pro", language="en"), + llm=inference.LLM(model="google/gemma-4-31b-it"), + tts=inference.TTS( + model="fishaudio/s2.1-pro", + voice="fa4c9eb3dccc4806b382b40d61c6b10a", + ), + turn_handling=TurnHandlingOptions( + turn_detection=inference.TurnDetector(), + ), + ) + + await session.start(room=ctx.room, agent=Assistant())`}, + "Copy it from the card to use it in your project.", + }, + }, + { + user: "And how do I deploy it?", + agent: []any{ + "Run this from the project directory. It registers the agent and deploys it to LiveKit Cloud.", + demoSnippet{id: "deploy", title: "Terminal", lang: "shell", code: "lk agent create"}, + }, + }, +} + +const demoFrame = time.Second / 60 + +// assistantDemo plays a scripted conversation into the overlay. It stands in +// for the real agent session until the overlay is wired to a LiveKit room. +type assistantDemo struct { + ov *overlay.Overlay + voice fakeVoice + snippets map[string]string +} + +func (d *assistantDemo) run(ctx context.Context) { + start := make(chan struct{}, 1) + var ( + mu sync.Mutex + cancelPlay context.CancelFunc + ) + go func() { + for { + select { + case ev := <-d.ov.Events(): + switch ev.Type { + case "ready", "replay": + mu.Lock() + if cancelPlay != nil { + cancelPlay() + } + mu.Unlock() + select { + case start <- struct{}{}: + default: + } + case "copy": + d.copy(ev.ID) + } + case <-d.ov.Done(): + return + } + } + }() + + for { + select { + case <-start: + case <-ctx.Done(): + return + case <-d.ov.Done(): + return + } + playCtx, cancel := context.WithCancel(ctx) + mu.Lock() + cancelPlay = cancel + mu.Unlock() + + d.ov.Send(map[string]string{"type": "clear"}) + d.ov.Send(overlayState{"state", "idle"}) + d.play(playCtx) + cancel() + } +} + +// play runs the script once. It returns false if it was stopped early. +func (d *assistantDemo) play(ctx context.Context) bool { + // Let the intro finish: the mark shows large, then docks. + if !d.wait(ctx, 1800*time.Millisecond, nil) { + return false + } + for i, turn := range demoScript { + userTurn, agentTurn := fmt.Sprintf("u%d", i), fmt.Sprintf("a%d", i) + + d.ov.Send(overlayState{"state", "listening"}) + if !d.speak(ctx, userTurn, "user", 0, turn.user, 270*time.Millisecond, 0.7) { + return false + } + if !d.wait(ctx, 400*time.Millisecond, nil) { + return false + } + + d.ov.Send(overlayState{"state", "thinking"}) + if !d.wait(ctx, 1300*time.Millisecond, nil) { + return false + } + + d.ov.Send(overlayState{"state", "speaking"}) + segment := 0 + for _, part := range turn.agent { + switch p := part.(type) { + case string: + if !d.speak(ctx, agentTurn, "agent", segment, p, 190*time.Millisecond, 1) { + return false + } + segment++ + case demoSnippet: + d.snippet(agentTurn, p) + if !d.wait(ctx, 600*time.Millisecond, nil) { + return false + } + } + } + + d.ov.Send(overlayState{"state", "idle"}) + if !d.wait(ctx, 1400*time.Millisecond, nil) { + return false + } + } + d.ov.Send(overlayState{"state", "listening"}) + return true +} + +// speak streams one segment of a turn word by word, like a live transcript, +// while sending simulated voice levels. +func (d *assistantDemo) speak(ctx context.Context, turn, role string, segment int, text string, perWord time.Duration, gain float64) bool { + words := strings.Fields(text) + begin := time.Now() + shown := 0 + return d.wait(ctx, time.Duration(len(words))*perWord+200*time.Millisecond, func(elapsed time.Duration) { + d.ov.Send(overlayBands{"bands", d.voice.bands(time.Since(begin).Seconds(), gain)}) + if n := min(len(words), int(elapsed/perWord)+1); n != shown { + shown = n + d.ov.Send(overlayTranscript{"transcript", turn, role, segment, strings.Join(words[:n], " ")}) + } + }) +} + +// wait blocks for dur, calling tick every frame. It returns false if the +// overlay closed or ctx ended. +func (d *assistantDemo) wait(ctx context.Context, dur time.Duration, tick func(elapsed time.Duration)) bool { + ticker := time.NewTicker(demoFrame) + defer ticker.Stop() + begin := time.Now() + for { + select { + case <-ctx.Done(): + return false + case <-d.ov.Done(): + return false + case now := <-ticker.C: + elapsed := now.Sub(begin) + if elapsed >= dur { + return true + } + if tick != nil { + tick(elapsed) + } + } + } +} + +func (d *assistantDemo) snippet(turn string, sn demoSnippet) { + d.snippets[sn.id] = sn.code + d.ov.Send(overlaySnippet{"snippet", turn, sn.id, sn.title, sn.lang, sn.code, false}) +} + +func (d *assistantDemo) copy(id string) { + code, ok := d.snippets[id] + if !ok || clipboard.WriteAll(code) != nil { + return + } + d.ov.Send(overlayCopied{"copied", id}) +} + +// fakeVoice synthesizes speech-like frequency bands: syllable-rate pulses, +// pauses between phrases, and two moving formants over a falling spectrum. +type fakeVoice struct{} + +const fakeVoiceBands = 24 + +func (fakeVoice) bands(t, gain float64) []float64 { + syl := math.Pow(math.Max(0, math.Sin(2*math.Pi*4.3*t+0.8*math.Sin(2*math.Pi*0.7*t))), 0.7) + gate := smoothstep(0.15, 0.35, 0.5+0.5*math.Sin(2*math.Pi*0.45*t+1.3)) + f1 := 4 + 3*math.Sin(2*math.Pi*1.1*t) + f2 := 12 + 5*math.Sin(2*math.Pi*0.63*t+2) + + out := make([]float64, fakeVoiceBands) + for i := range out { + x := float64(i) + s := 0.75*math.Exp(-x/9) + 0.55*math.Exp(-(x-f1)*(x-f1)/9.7) + 0.35*math.Exp(-(x-f2)*(x-f2)/24.5) + v := syl * gate * gain * s * 1.25 * (0.85 + 0.3*rand.Float64()) + out[i] = math.Round(math.Min(1, v)*100) / 100 + } + return out +} + +func smoothstep(e0, e1, x float64) float64 { + t := math.Max(0, math.Min(1, (x-e0)/(e1-e0))) + return t * t * (3 - 2*t) +} diff --git a/cmd/lk/assistant_guest.go b/cmd/lk/assistant_guest.go new file mode 100644 index 00000000..c03a3fe9 --- /dev/null +++ b/cmd/lk/assistant_guest.go @@ -0,0 +1,202 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "net" + "path/filepath" + + "github.com/livekit/livekit-cli/v2/pkg/agentfs" + "github.com/livekit/livekit-cli/v2/pkg/console" +) + +// "Meet your agent": the user's own agent runs in console mode and joins the +// same audio pipeline as the assistant. Switching changes which agent hears +// the microphone. The assistant keeps running, silent, so the user returns to +// the same conversation. + +// guestAgent is the user's agent while it runs in the overlay. +type guestAgent struct { + name string // shown as the speaker + dir string + proc *AgentProcess + conn net.Conn +} + +type overlaySpeaker struct { + Type string `json:"type"` + Name string `json:"name"` // who the agent turns are from + Guest bool `json:"guest"` // the user's own agent, not the assistant +} + +// proposeTryProject proposes running the user's agent in the overlay. path +// is relative to the project directory; empty means the project created this +// session, or the current directory. +func (s *assistantSession) proposeTryProject(requestID, path string) { + t := &s.tools + dir := t.dir + switch { + case path != "": + dir = filepath.Join(t.dir, path) + case t.lastProject != "": + dir = t.lastProject + } + if s.guest != nil { + s.replyTool(requestID, "", errors.New("the user's agent is already running; they can press B to come back first")) + return + } + root, projectType, err := agentfs.DetectProjectRoot(dir) + if err != nil || (!projectType.IsPython() && !projectType.IsNode()) { + s.replyTool(requestID, "", fmt.Errorf("there's no Python or Node.js agent project in %s", tildePath(dir))) + return + } + entrypoint, err := findEntrypoint(root, "", projectType) + if err != nil { + s.replyTool(requestID, "", err) + return + } + name := filepath.Base(root) + + t.confirmN++ + id := fmt.Sprintf("confirm%d", t.confirmN) + t.pending[id] = pendingTool{ + command: "lk agent console", + start: func(requestID, confirmID string) { + s.startGuest(requestID, confirmID, name, root, entrypoint, projectType) + }, + } + s.agentTurnForReply() + s.out.Send(overlayConfirm{ + Type: "confirm", Turn: s.agentTurn, ID: id, + Heading: "Run in " + tildePath(root), + Command: "lk agent console", + Detail: fmt.Sprintf("Runs your agent, %s, on this machine and switches the conversation to it. Press B to come back to LK.", name), + Options: []overlayConfirmOption{{ID: confirmRun, Name: "Talk to it"}, {ID: confirmCancel, Name: "Cancel"}}, + }) + s.replyTool(requestID, fmt.Sprintf( + "Waiting for the user to confirm. Confirmation ID: %s. This runs their agent, %s, here so they can talk to it. "+ + "Ask the user, in one short sentence, whether to start it. They can answer by voice or on screen. "+ + "If they agree by voice, call confirm_action with this ID and approved=true.", id, name), nil) +} + +// startGuest starts the user's agent and switches to it as soon as it +// connects. It greets right away, so there's no waiting: the assistant says +// its hand-off sentence while the agent starts up, which takes a few seconds. +func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoint string, projectType agentfs.ProjectType) { + t := &s.tools + handOff := fmt.Sprintf("Starting the user's agent, %s. It takes a few seconds. In one or two short sentences, tell them you're handing them over to it, that they should say hello to start (starter agents wait for the user to speak first), and that they can press B to come back to you.", name) + if requestID != "" { + s.replyTool(requestID, handOff, nil) + } else { + s.tellUser(handOff) + } + s.inBackground(func(ctx context.Context) toolResult { + failed := func(err error) toolResult { + return toolResult{confirmID: confirmID, command: "lk agent console", err: err} + } + // The guest needs its own console server: the assistant's accepts + // only one agent, and closes once the assistant connects. + server, err := console.NewTCPServer("127.0.0.1:0") + if err != nil { + return failed(err) + } + proc, err := startAgent(AgentStartConfig{ + Dir: dir, + Entrypoint: entrypoint, + ProjectType: projectType, + CLIArgs: buildConsoleArgs(server.Addr().String(), false), + Env: t.creds, + FailSignals: consoleCrashSignals, + }) + if err != nil { + server.Close() + return failed(fmt.Errorf("couldn't start the agent: %w", err)) + } + conn, err := acceptAgent(ctx, server, proc) + if err != nil { + proc.Kill() + server.Close() + return failed(err) + } + return toolResult{then: func(toolResult) { + s.guest = &guestAgent{name: name, dir: dir, proc: proc, conn: conn} + s.pipeline.AddConn(t.ctx, conn) + s.pipeline.SetActive(conn) + s.startSpeakerTurns(overlaySpeaker{Type: "speaker", Name: name, Guest: true}) + s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: confirmID, Status: "done"}) + go func() { + // The worker keeps running when its job crashes, so watch + // for the crash as well as for the exit. + var err error + select { + case err = <-proc.Done(): + case <-proc.Failed(): + err = errors.New("its job crashed") + } + select { + case t.results <- toolResult{then: func(toolResult) { s.guestExited(proc, err) }}: + case <-t.ctx.Done(): + } + }() + }} + }) +} + +// returnToAssistant switches back to the assistant and stops the guest. +func (s *assistantSession) returnToAssistant(notice string) { + g := s.guest + if g == nil { + return + } + s.guest = nil + s.pipeline.SetActive(s.assistantConn) + g.conn.Close() + go g.proc.Kill() + s.startSpeakerTurns(overlaySpeaker{Type: "speaker", Name: "LiveKit"}) + if notice != "" { + s.out.Send(overlayNotice{Type: "notice", Text: notice}) + } + s.tellUser(fmt.Sprintf("The user is back from talking to their agent, %s. Welcome them back in one short sentence and ask how it went or what they'd like to change.", g.name)) +} + +func (s *assistantSession) guestExited(proc *AgentProcess, err error) { + if s.guest == nil || s.guest.proc != proc { + return + } + notice := "Your agent stopped" + if err != nil { + notice = "Your agent stopped with an error. See the terminal" + out.Statusf("Your agent stopped: %v\n%s", err, agentExitDetail(proc)) + } + s.returnToAssistant(notice) +} + +// startSpeakerTurns makes the next turns start fresh, labeled for speaker. +func (s *assistantSession) startSpeakerTurns(speaker overlaySpeaker) { + s.agentOpen = false + s.userOpen = false + s.agentSpoke = true + s.agentSpeaking = false + s.out.Send(speaker) + s.out.Send(overlayState{"state", "listening"}) +} + +// guestActive reports whether the user's agent is the active one. +func (s *assistantSession) guestActive() bool { + return s.guest != nil +} diff --git a/cmd/lk/assistant_log.go b/cmd/lk/assistant_log.go new file mode 100644 index 00000000..08160a80 --- /dev/null +++ b/cmd/lk/assistant_log.go @@ -0,0 +1,221 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// sessionLog records what the overlay shows, so the conversation can be saved +// as Markdown when it ends: the transcript, tool calls, code, prompts, and +// commands. +type sessionLog struct { + next overlaySender + + mu sync.Mutex + started time.Time + speaker string // who new agent turns are from + order []string + turns map[string]*logTurn + items map[string]*logItem // tool, card, and confirmation items by id +} + +type logTurn struct { + role string // user or agent + speaker string + items []*logItem + texts map[int]*logItem // transcript segments +} + +type logItem struct { + kind string // text, tool, code, or command + text string + label string + detail string + lang string + status string +} + +func newSessionLog(next overlaySender) *sessionLog { + return &sessionLog{ + next: next, + started: time.Now(), + speaker: "LK", + turns: map[string]*logTurn{}, + items: map[string]*logItem{}, + } +} + +func (l *sessionLog) Send(msg any) error { + l.record(msg) + return l.next.Send(msg) +} + +func (l *sessionLog) turn(id, role string) *logTurn { + t, ok := l.turns[id] + if !ok { + t = &logTurn{role: role, speaker: l.speaker, texts: map[int]*logItem{}} + l.turns[id] = t + l.order = append(l.order, id) + } + return t +} + +func (l *sessionLog) record(msg any) { + l.mu.Lock() + defer l.mu.Unlock() + switch m := msg.(type) { + case overlaySpeaker: + l.speaker = m.Name + if !m.Guest { + l.speaker = "LK" + } + case overlayTranscript: + t := l.turn(m.Turn, m.Role) + item, ok := t.texts[m.Segment] + if !ok { + item = &logItem{kind: "text"} + t.texts[m.Segment] = item + t.items = append(t.items, item) + } + item.text = m.Text + case overlayTool: + item, ok := l.items[m.ID] + if !ok { + item = &logItem{kind: "tool"} + l.items[m.ID] = item + t := l.turn(m.Turn, "agent") + t.items = append(t.items, item) + } + if m.Label != "" { + item.label = m.Label + } + if m.Detail != "" { + item.detail = m.Detail + } + item.status = m.Status + case overlaySnippet: + t := l.turn(m.Turn, "agent") + t.items = append(t.items, &logItem{kind: "code", label: m.Title, lang: m.Lang, text: m.Code}) + case overlayConfirm: + item := &logItem{kind: "command", text: m.Command, detail: m.Heading, status: "not answered"} + l.items[m.ID] = item + t := l.turn(m.Turn, "agent") + t.items = append(t.items, item) + case overlayConfirmStatus: + if item, ok := l.items[m.ID]; ok { + item.status = m.Status + if m.Text != "" { + item.status += ": " + m.Text + } + } + } +} + +// markdown renders the conversation, or "" if the user never spoke. +func (l *sessionLog) markdown(dir string) string { + l.mu.Lock() + defer l.mu.Unlock() + userSpoke := false + for _, t := range l.turns { + if t.role == "user" { + userSpoke = true + } + } + if !userSpoke { + return "" + } + + var b strings.Builder + fmt.Fprintf(&b, "# LiveKit assistant, %s\n\nIn %s.\n", l.started.Format("January 2, 2006, 3:04 PM"), tildePath(dir)) + for _, id := range l.order { + t := l.turns[id] + who := t.speaker + if t.role == "user" { + who = "You" + } + fmt.Fprintf(&b, "\n**%s:**", who) + // Text continues the speaker's line, or the text before it. After + // a tool line, card, or command, it starts a new paragraph. + afterText := true + for _, it := range t.items { + switch it.kind { + case "text": + if it.text == "" { + continue + } + sep := " " + if !afterText { + sep = "\n\n" + } + b.WriteString(sep + it.text) + afterText = true + continue + case "tool": + line := it.label + if it.detail != "" { + line += ": " + it.detail + } + fmt.Fprintf(&b, "\n\n- %s", line) + case "code": + lang := it.lang + if lang == "prompt" { + lang = "" + } + fence := codeFence(it.text) + fmt.Fprintf(&b, "\n\n%s:\n\n%s%s\n%s\n%s", it.label, fence, lang, it.text, fence) + case "command": + fmt.Fprintf(&b, "\n\n`%s` (%s): %s", it.text, strings.ToLower(it.detail), it.status) + } + afterText = false + } + b.WriteString("\n") + } + return b.String() +} + +// codeFence returns a Markdown code fence longer than any run of backticks +// in code, so the code can't close it early. +func codeFence(code string) string { + fence := "```" + for strings.Contains(code, fence) { + fence += "`" + } + return fence +} + +// save writes the conversation to the assistant's sessions folder and returns +// the file's path, or "" if there was nothing to save. +func (l *sessionLog) save(dir string) (string, error) { + md := l.markdown(dir) + if md == "" { + return "", nil + } + cache, err := os.UserCacheDir() + if err != nil { + return "", err + } + folder := filepath.Join(cache, "livekit", "assistant-sessions") + if err := os.MkdirAll(folder, 0o755); err != nil { + return "", err + } + path := filepath.Join(folder, l.started.Format("2006-01-02-150405")+".md") + return path, os.WriteFile(path, []byte(md), 0o644) +} diff --git a/cmd/lk/assistant_log_test.go b/cmd/lk/assistant_log_test.go new file mode 100644 index 00000000..a7adabe5 --- /dev/null +++ b/cmd/lk/assistant_log_test.go @@ -0,0 +1,41 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// discardOverlay is an overlay that drops every message. +type discardOverlay struct{} + +func (discardOverlay) Send(any) error { return nil } + +func TestSessionLogMarkdownBlocks(t *testing.T) { + l := newSessionLog(discardOverlay{}) + l.Send(overlayTranscript{"transcript", "u1", "user", 0, "How do I start?"}) + l.Send(overlayTranscript{"transcript", "a2", "agent", 0, "Run this."}) + l.Send(overlaySnippet{"snippet", "a2", "c1", "Terminal", "shell", "lk agent init my-agent", false}) + l.Send(overlayTranscript{"transcript", "a2", "agent", 1, "Then run it."}) + l.Send(overlaySnippet{"snippet", "a2", "c2", "Prompt for your coding agent", "prompt", "Add this:\n```python\nprint(1)\n```", false}) + + md := l.markdown("/tmp") + // Text after a card starts its own paragraph, so the fence closes. + require.Contains(t, md, "**LK:** Run this.\n\nTerminal:\n\n```shell\nlk agent init my-agent\n```\n\nThen run it.") + // A fence inside the code can't close the card's fence. + require.Contains(t, md, "````\nAdd this:\n```python\nprint(1)\n```\n````") +} diff --git a/cmd/lk/assistant_session.go b/cmd/lk/assistant_session.go new file mode 100644 index 00000000..59528ea7 --- /dev/null +++ b/cmd/lk/assistant_session.go @@ -0,0 +1,932 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "math" + "net" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/atotto/clipboard" + "github.com/livekit/protocol/livekit/agent" + "github.com/urfave/cli/v3" + + "github.com/livekit/livekit-cli/v2/pkg/agentfs" + "github.com/livekit/livekit-cli/v2/pkg/assistant" + "github.com/livekit/livekit-cli/v2/pkg/console" + "github.com/livekit/livekit-cli/v2/pkg/overlay" + "github.com/livekit/livekit-cli/v2/pkg/portaudio" +) + +// overlaySender sends messages to the overlay page. +type overlaySender interface { + Send(msg any) error +} + +// overlayBandCount is how many bands the overlay page's meter expects. +const overlayBandCount = 24 + +// assistantSession runs the assistant agent in console mode, connects it to +// the microphone and speakers, and shows the conversation in the overlay. +type assistantSession struct { + ov *overlay.Overlay + out overlaySender // messages to the overlay page, through log, except in tests + log *sessionLog + cmd *cli.Command + + pipeline *console.AudioPipeline + assistantConn net.Conn // the assistant agent's console connection + guest *guestAgent // the user's agent, while it runs here + + // Conversation state, mapped onto the overlay's turns and segments. A + // turn stays the target for text that finishes it (such as the rest of + // an interrupted speech) after the other side starts a new turn; "open" + // only says whether new speech joins it. + turnN int + userTurn string // latest user turn, or "" + userOpen bool // new user speech joins userTurn + userCommits string // text the agent committed so far in userTurn + agentSpoke bool // the agent has spoken since userTurn started + userFinal string // final transcripts so far in userTurn + agentTurn string // latest agent turn, or "" + agentOpen bool // new agent speech joins agentTurn + agentSegment int + segmentText string // text of the current agent segment so far + segmentDone bool // the current agent segment is complete + finalPending bool // a streamed segment awaits the agent's committed text + shownCalls map[string]bool + snippetN int + toolTurns map[string]string // tool call id → the turn showing it + + // Ending: set when the agent calls end_call. The overlay closes once the + // agent's goodbye has finished playing. + agentSpeaking bool + lastAudio time.Time + ending bool + endingSince time.Time + spokeGoodbye bool // the agent spoke after end_call + agentExited bool + dismissed bool + snippets map[string]string + + // The local connection to the assistant agent, and the lk commands it + // runs through it. + channel *agentChannel + tools toolState +} + +// agentMessage is a line from the assistant agent: its spoken text ("text", +// then "flush" at the end of a segment) or a request to run an lk tool +// ("lk_tool"). +type agentMessage struct { + Type string `json:"type"` + ID string `json:"id"` + Text string `json:"text"` + Name string `json:"name"` + Args map[string]any `json:"args"` + + // show_code: a code card from the agent itself. + Title string `json:"title"` + Language string `json:"language"` + Code string `json:"code"` + + // pipeline: the agent's models, for X-ray mode. + STT string `json:"stt"` + LLM string `json:"llm"` + TTS string `json:"tts"` + TurnDetection string `json:"turn_detection"` +} + +// agentChannel is the local connection to the assistant agent, as JSON lines +// in both directions. The agent connects once it starts. +type agentChannel struct { + mu sync.Mutex + conn net.Conn +} + +// serve accepts the agent's connection and sends each line it writes to out. +func (c *agentChannel) serve(ln net.Listener, out chan<- agentMessage) { + conn, err := ln.Accept() + if err != nil { + return + } + c.mu.Lock() + c.conn = conn + c.mu.Unlock() + defer conn.Close() + scanner := bufio.NewScanner(conn) + scanner.Buffer(make([]byte, 64<<10), 1<<20) + for scanner.Scan() { + var m agentMessage + if json.Unmarshal(scanner.Bytes(), &m) == nil { + out <- m + } + } +} + +// send writes a message to the agent. It's dropped if the agent isn't +// connected. +func (c *agentChannel) send(msg any) { + b, err := json.Marshal(msg) + if err != nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.conn != nil { + _, _ = c.conn.Write(append(b, '\n')) + } +} + +func newAssistantSession(ov *overlay.Overlay, cmd *cli.Command) *assistantSession { + log := newSessionLog(ov) + return &assistantSession{ + ov: ov, + out: log, + log: log, + cmd: cmd, + shownCalls: map[string]bool{}, + toolTurns: map[string]string{}, + snippets: map[string]string{}, + } +} + +// run starts the agent and relays between it and the overlay until ctx ends +// or the overlay closes. Errors are also shown in the overlay. +func (s *assistantSession) run(ctx context.Context) (err error) { + if !s.waitReady(ctx) { + return nil + } + defer func() { + if err != nil && ctx.Err() != nil { + // The overlay closed, or lk was interrupted, while the session + // was still starting. That isn't a failure. + err = nil + } + if err != nil { + s.showError(startupError(err)) + // The run loop has stopped, but the error card and any code + // cards are still on screen: keep their Copy buttons working. + s.serveCopies(ctx) + } + }() + + s.connecting("Starting") + + creds, err := resolveCredentials(s.cmd) + if err != nil { + return errNoCredentials{err} + } + + dir := s.cmd.String("agent-dir") + projectType := agentfs.ProjectTypePythonUV + if dir == "" { + if dir, err = assistant.DefaultDir(); err != nil { + return err + } + if err := assistant.Prepare(ctx, dir, s.connecting); err != nil { + return err + } + } else if _, projectType, err = agentfs.DetectProjectRoot(dir); err != nil { + return fmt.Errorf("no agent project in %s: %w", dir, err) + } + + if err := portaudio.Initialize(); err != nil { + return fmt.Errorf("failed to initialize PortAudio: %w", err) + } + defer portaudio.Terminate() + inputDev, err := portaudio.DefaultInputDevice() + if err != nil { + return fmt.Errorf("input device: %w", err) + } + outputDev, err := portaudio.DefaultOutputDevice() + if err != nil { + return fmt.Errorf("output device: %w", err) + } + + server, err := console.NewTCPServer("127.0.0.1:0") + if err != nil { + return err + } + defer server.Close() + + // The agent connects here to stream what it says, in time with its + // speech, and to run lk commands. Console mode itself only + // reports the agent's text once it finishes speaking. + channelLn, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return err + } + defer channelLn.Close() + agentMessages := make(chan agentMessage, 256) + s.channel = &agentChannel{} + go s.channel.serve(channelLn, agentMessages) + + projectDir, err := os.Getwd() + if err != nil { + return err + } + toolsCtx, cancelTools := context.WithCancel(ctx) + defer cancelTools() + s.tools = toolState{ + ctx: toolsCtx, + dir: projectDir, + creds: creds, + results: make(chan toolResult, 16), + pending: map[string]pendingTool{}, + } + + s.connecting("Starting the assistant agent") + agentProc, err := startAgent(AgentStartConfig{ + Dir: dir, + Entrypoint: assistant.Entrypoint, + ProjectType: projectType, + CLIArgs: buildConsoleArgs(server.Addr().String(), false), + Env: append(creds, + "LK_ASSISTANT_ADDR="+channelLn.Addr().String(), + "LK_ASSISTANT_PROJECT="+projectDir, + ), + FailSignals: consoleCrashSignals, + }) + if err != nil { + return fmt.Errorf("failed to start the assistant agent: %w", err) + } + defer agentProc.Kill() + + conn, err := acceptAgent(ctx, server, agentProc) + if err != nil { + return err + } + s.assistantConn = conn + defer func() { + if s.guest != nil { + s.guest.conn.Close() + s.guest.proc.Kill() + } + }() + + s.pipeline, err = console.NewPipeline(console.PipelineConfig{ + InputDevice: inputDev, + OutputDevice: outputDev, + Conn: conn, + }) + if err != nil { + return fmt.Errorf("pipeline: %w", err) + } + pipelineCtx, cancelPipeline := context.WithCancel(ctx) + go s.pipeline.Start(pipelineCtx) + defer func() { + cancelPipeline() + // Stop can block forever in PortAudio once the streams are aborted, + // so, like lk agent console, don't wait for it. The deferred + // agentProc.Kill runs next either way. + go s.pipeline.Stop() + }() + + ticker := time.NewTicker(demoFrame * 2) + defer ticker.Stop() + agentDone, agentFailed := agentProc.Done(), agentProc.Failed() + for { + select { + case <-ctx.Done(): + return nil + case <-s.ov.Done(): + return nil + case err := <-agentDone: + if s.ending { + // The session shut down after end_call, as expected. Its + // goodbye may still be playing; checkEnding closes the + // overlay once it has. + s.agentExited = true + agentDone = nil + continue + } + s.out.Send(overlayState{"state", "idle"}) + if err != nil { + return fmt.Errorf("the assistant agent exited: %w\n\n%s", err, agentExitDetail(agentProc)) + } + return fmt.Errorf("the assistant agent exited.\n\n%s", agentExitDetail(agentProc)) + case <-agentFailed: + // The worker keeps running when its job crashes, so agentDone + // doesn't fire. + if s.ending { + s.agentExited = true + agentFailed = nil + continue + } + // The crash marker arrives mid-traceback; give trailing output a moment. + time.Sleep(500 * time.Millisecond) + s.out.Send(overlayState{"state", "idle"}) + return fmt.Errorf("the assistant agent crashed.\n\n%s", agentExitDetail(agentProc)) + case ev := <-s.pipeline.Events: + s.handleEvent(ev) + case m := <-agentMessages: + switch m.Type { + case "text": + // The assistant is silent while the user's agent is active; + // anything it says then isn't heard, so don't show it. + if !s.guestActive() { + s.agentText(m.Text) + } + case "flush": + if !s.guestActive() { + s.agentFlush() + } + case "lk_tool": + s.runTool(m.ID, m.Name, m.Args) + case "show_code": + s.showAgentCode(m) + case "pipeline": + s.out.Send(overlayPipeline{Type: "pipeline", STT: m.STT, LLM: m.LLM, TTS: m.TTS, TurnDetection: m.TurnDetection}) + } + case r := <-s.tools.results: + s.toolFinished(r) + case ev := <-s.ov.Events(): + switch ev.Type { + case "copy", "copy_text": + s.handleCopy(ev) + case "confirm": + s.confirmTool(ev.ID, ev.Option) + case "back": + s.returnToAssistant("") + case "mute": + s.pipeline.SetMuted(ev.On) + } + case <-ticker.C: + s.sendBands() + s.checkEnding() + } + } +} + +func (s *assistantSession) waitReady(ctx context.Context) bool { + for { + select { + case ev := <-s.ov.Events(): + if ev.Type == "ready" { + return true + } + case <-ctx.Done(): + return false + case <-s.ov.Done(): + return false + } + } +} + +// handleCopy copies a code card ("copy") or the text the page sends +// ("copy_text") to the clipboard. +func (s *assistantSession) handleCopy(ev overlay.Event) { + switch ev.Type { + case "copy": + s.copy(ev.ID) + case "copy_text": + _ = clipboard.WriteAll(ev.Text) + } +} + +// serveCopies handles copy requests from the page until the overlay closes. +func (s *assistantSession) serveCopies(ctx context.Context) { + for { + select { + case ev := <-s.ov.Events(): + s.handleCopy(ev) + case <-ctx.Done(): + return + case <-s.ov.Done(): + return + } + } +} + +// acceptAgent waits for the agent subprocess to connect to the console server. +func acceptAgent(ctx context.Context, server *console.TCPServer, agentProc *AgentProcess) (net.Conn, error) { + type result struct { + conn net.Conn + err error + } + accepted := make(chan result, 1) + go func() { + conn, err := server.Accept() + accepted <- result{conn, err} + }() + + select { + case res := <-accepted: + if res.err != nil { + return nil, fmt.Errorf("agent connection: %w", res.err) + } + return res.conn, nil + case err := <-agentProc.Done(): + if err != nil { + return nil, fmt.Errorf("the assistant agent exited before connecting: %w\n\n%s", err, agentExitDetail(agentProc)) + } + return nil, fmt.Errorf("the assistant agent exited before connecting.\n\n%s", agentExitDetail(agentProc)) + case <-agentProc.Failed(): + // The crash marker arrives mid-traceback; give trailing output a moment. + time.Sleep(500 * time.Millisecond) + return nil, fmt.Errorf("the assistant agent crashed before connecting.\n\n%s", agentExitDetail(agentProc)) + case <-time.After(90 * time.Second): + return nil, fmt.Errorf("timed out waiting for the assistant agent to connect.\n\n%s", agentExitDetail(agentProc)) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (s *assistantSession) connecting(label string) { + s.out.Send(map[string]string{"type": "state", "state": "connecting", "label": label}) +} + +// sendBands sends the levels of whoever is audible: the agent while its audio +// plays, otherwise the (echo-cancelled) microphone. +func (s *assistantSession) sendBands() { + var bands [console.NumFFTBands]float64 + var level float64 + if s.pipeline.IsPlaying() { + s.lastAudio = time.Now() + bands, level = s.pipeline.PlaybackFFTBands(), s.pipeline.PlaybackLevel() + } else { + bands, level = s.pipeline.FFTBands(), s.pipeline.Level() + } + // The bands are peak-normalized per frame, so room noise would fill the + // meter. Gate them by the overall level. + gate := smoothstep(-55, -38, level) + if gate == 0 { + return + } + out := resampleBands(bands[:], overlayBandCount) + for i := range out { + out[i] = math.Round(out[i]*gate*100) / 100 + } + s.out.Send(overlayBands{"bands", out}) +} + +func (s *assistantSession) handleEvent(ev *agent.AgentSessionEvent) { + switch e := ev.GetEvent().(type) { + case *agent.AgentSessionEvent_AgentStateChanged_: + s.agentSpeaking = e.AgentStateChanged.GetNewState() == agent.AgentState_AS_SPEAKING + if s.agentSpeaking && s.ending { + s.spokeGoodbye = true + } + switch e.AgentStateChanged.GetNewState() { + case agent.AgentState_AS_IDLE: + s.out.Send(overlayState{"state", "idle"}) + case agent.AgentState_AS_LISTENING: + s.out.Send(overlayState{"state", "listening"}) + case agent.AgentState_AS_THINKING: + s.out.Send(overlayState{"state", "thinking"}) + case agent.AgentState_AS_SPEAKING: + s.beginAgentSpeech() + s.out.Send(overlayState{"state", "speaking"}) + } + + case *agent.AgentSessionEvent_UserInputTranscribed_: + s.userTranscript(e.UserInputTranscribed.GetTranscript(), e.UserInputTranscribed.GetIsFinal()) + + case *agent.AgentSessionEvent_ConversationItemAdded_: + switch item := e.ConversationItemAdded.GetItem().GetItem().(type) { + case *agent.ChatContext_ChatItem_Message: + te, ok := messageToTurnEvent(item.Message) + if !ok { + return + } + if te.Role == "user" { + s.userCommitted(te.Text) + } else { + s.agentMessage(te.Text) + } + if m := item.Message.GetMetrics(); m != nil { + s.out.Send(metricsMessage(te.Role, m)) + } + case *agent.ChatContext_ChatItem_FunctionCall: + s.toolCall(item.FunctionCall) + } + + case *agent.AgentSessionEvent_FunctionToolsStarted_: + for _, fc := range e.FunctionToolsStarted.GetFunctionCalls() { + s.toolCall(fc) + } + + case *agent.AgentSessionEvent_FunctionToolsExecuted_: + failed := map[string]bool{} + for _, out := range e.FunctionToolsExecuted.GetFunctionCallOutputs() { + failed[out.GetCallId()] = out.GetIsError() + } + for _, fc := range e.FunctionToolsExecuted.GetFunctionCalls() { + s.toolCall(fc) + s.toolDone(fc.GetCallId(), failed[fc.GetCallId()]) + } + + case *agent.AgentSessionEvent_Error_: + // Session errors (such as rejected credentials) usually end the + // conversation, so keep them on screen. Errors the agent retries + // don't stay. + msg := e.Error.GetMessage() + if msg == "" { + return + } + // The user's own agent uses its own providers and keys. + if !s.guestActive() && isAuthError(msg) { + s.showError(overlayError{Type: "error", + Title: "LiveKit Cloud rejected your project's API key", + Detail: "The assistant uses LiveKit Inference through your default lk project, and its key didn't work. Sign in again, or pick a different project with --project.", + Fix: "lk cloud auth"}) + return + } + s.out.Send(overlayNotice{Type: "notice", Text: msg, Sticky: !strings.Contains(msg, "recoverable=True")}) + + case *agent.AgentSessionEvent_EotPrediction_: + s.out.Send(overlayEOT{Type: "eot", Probability: e.EotPrediction.GetProbability(), Threshold: e.EotPrediction.GetThreshold()}) + + case *agent.AgentSessionEvent_SessionUsageUpdated_: + s.out.Send(usageMessage(e.SessionUsageUpdated.GetUsage())) + } +} + +// isAuthError reports whether a session error says the API key was rejected. +// The message is the agent's error event as text, such as "type='llm_error' +// timestamp=... error=APIStatusError('...', status_code=401, ...) +// recoverable=False". +func isAuthError(msg string) bool { + msg = strings.ToLower(msg) + return strings.Contains(msg, "status_code=401") || strings.Contains(msg, "invalid api key") +} + +// userTurnForSpeech makes sure new user speech has an open turn to join. A +// new user turn means the agent's next reply starts a new turn too. +func (s *assistantSession) userTurnForSpeech() { + if s.userOpen { + return + } + if s.userTurn != "" && !s.agentSpoke { + // The agent hasn't said anything since the user's last turn (at most + // it started looking things up), so the user is still finishing the + // same thought, such as after a mid-sentence pause. + s.userOpen = true + return + } + s.agentSpoke = false + s.turnN++ + s.userTurn = fmt.Sprintf("u%d", s.turnN) + s.userOpen = true + s.userCommits = "" + s.userFinal = "" + s.agentOpen = false +} + +// agentTurnForReply makes sure a new agent reply has an open turn to join. +// It closes the user's turn, so the user's next speech starts a new turn, +// unless the agent hasn't spoken yet (see userTurnForSpeech). That holds +// even when the reply joins an agent turn opened earlier, such as by a tool +// call while the user was still talking. +func (s *assistantSession) agentTurnForReply() { + s.userOpen = false + if s.agentOpen { + return + } + s.turnN++ + s.agentTurn = fmt.Sprintf("a%d", s.turnN) + s.agentOpen = true + s.agentSegment = 0 + s.segmentText = "" + s.segmentDone = false + s.finalPending = false +} + +// streaming reports whether the agent is partway through speaking the +// current segment, so its remaining text belongs there. +func (s *assistantSession) streaming() bool { + return s.agentTurn != "" && s.segmentText != "" && !s.segmentDone +} + +// nextSegment moves to a new agent paragraph unless the current one is unused. +func (s *assistantSession) nextSegment() { + if s.segmentText == "" && !s.segmentDone { + return + } + s.agentSegment++ + s.segmentText = "" + s.segmentDone = false +} + +func (s *assistantSession) sendAgentSegment() { + s.out.Send(overlayTranscript{"transcript", s.agentTurn, "agent", s.agentSegment, s.segmentText}) +} + +func (s *assistantSession) userTranscript(text string, final bool) { + text = strings.TrimSpace(text) + if text == "" { + return + } + s.userTurnForSpeech() + full := strings.TrimSpace(s.userFinal + " " + text) + if final { + s.userFinal = full + } + s.out.Send(overlayTranscript{"transcript", s.userTurn, "user", 0, full}) +} + +// userCommitted replaces the live transcript with the text the agent +// committed to the conversation. A user who pauses mid-thought can get +// several commits in a row; until the agent starts replying, they're one +// turn, and the turn stays open for more speech. +func (s *assistantSession) userCommitted(text string) { + s.userTurnForSpeech() + s.userCommits = strings.TrimSpace(s.userCommits + " " + text) + s.userFinal = s.userCommits + s.out.Send(overlayTranscript{"transcript", s.userTurn, "user", 0, s.userCommits}) +} + +// beginAgentSpeech adds an empty agent paragraph so the page shows its cursor +// until the first words arrive. A speech that resumes keeps its paragraph. +func (s *assistantSession) beginAgentSpeech() { + s.agentSpoke = true + if s.streaming() && s.agentOpen { + return + } + s.agentTurnForReply() + if s.segmentDone { + s.nextSegment() + } + s.sendAgentSegment() +} + +// agentText adds text the agent is speaking right now. The rest of a speech +// the user interrupted stays in that speech's turn. +func (s *assistantSession) agentText(text string) { + s.agentSpoke = true + if !s.streaming() { + s.agentTurnForReply() + if s.segmentDone { + s.nextSegment() + } + } + s.segmentText += text + s.sendAgentSegment() +} + +func (s *assistantSession) agentFlush() { + if s.segmentText != "" { + s.segmentDone = true + s.finalPending = true + } +} + +// agentMessage handles text the agent committed to the conversation after +// speaking. If the same speech was streamed, the streamed text stays: it is +// already free of expressive markup and stops where an interruption cut the +// speech off. Otherwise the committed text is shown, with its markup removed. +func (s *assistantSession) agentMessage(text string) { + s.agentSpoke = true + if s.finalPending || s.streaming() { + s.segmentDone = true + s.finalPending = false + return + } + s.agentTurnForReply() + s.nextSegment() + s.segmentText = stripMarkup(text) + s.segmentDone = true + s.sendAgentSegment() +} + +var ( + markupTagRe = regexp.MustCompile(`]*>`) + markupSpaceRe = regexp.MustCompile(`\s{2,}`) +) + +// stripMarkup removes expressive markup tags, such as , from +// committed agent text. The agent strips streamed text with the SDK's own +// stripper; this is only the fallback for text that wasn't streamed. +func stripMarkup(text string) string { + return strings.TrimSpace(markupSpaceRe.ReplaceAllString(markupTagRe.ReplaceAllString(text, ""), " ")) +} + +// toolCall handles a tool call the agent made. show_code calls become code +// cards, end_call starts closing the overlay, and other calls (docs searches) +// show as a line in the agent's turn. Each call arrives in several events; +// only the first counts. +func (s *assistantSession) toolCall(fc *agent.FunctionCall) { + switch fc.GetName() { + case assistant.ShowCodeTool: + s.showCode(fc) + case assistant.HandOffTaskTool: + s.handOffTask(fc) + case assistant.EndCallTool: + // Only the assistant's end_call closes the overlay; the user's own + // agent may have one too. + if !s.ending && !s.guestActive() { + s.ending = true + s.endingSince = time.Now() + } + default: + id := fc.GetCallId() + if _, ok := s.toolTurns[id]; ok { + return + } + s.agentTurnForReply() + s.toolTurns[id] = s.agentTurn + label, detail := describeTool(fc.GetName(), fc.GetArguments()) + s.out.Send(overlayTool{Type: "tool", Turn: s.agentTurn, ID: id, Label: label, Detail: detail, Status: "running"}) + } +} + +// toolDone marks a tool call's line as finished. +func (s *assistantSession) toolDone(id string, failed bool) { + turn, ok := s.toolTurns[id] + if !ok { + return + } + status := "done" + if failed { + status = "error" + } + s.out.Send(overlayTool{Type: "tool", Turn: turn, ID: id, Status: status}) +} + +// describeTool returns a short label for a tool call and the main argument, +// such as the search query. +func describeTool(name, arguments string) (label, detail string) { + var args map[string]any + _ = json.Unmarshal([]byte(arguments), &args) + for _, key := range []string{"query", "paths", "path", "package", "repo", "template", "name"} { + switch v := args[key].(type) { + case string: + detail = v + case []any: + parts := make([]string, 0, len(v)) + for _, p := range v { + if s, ok := p.(string); ok { + parts = append(parts, s) + } + } + detail = strings.Join(parts, ", ") + } + if detail != "" { + break + } + } + switch name { + case "get_docs_overview": + return "Read the docs overview", detail + case "docs_search": + return "Searched the docs", detail + case "get_pages": + return "Read the docs", detail + case "code_search": + return "Searched LiveKit code", detail + case "get_project_info": + return "Checked your LiveKit project", detail + case "list_cloud_agents": + return "Listed your agents", detail + case "list_templates": + return "Listed the templates", detail + case "create_project": + return "Create a project", detail + case "confirm_action": + var a struct { + Approved bool `json:"approved"` + } + _ = json.Unmarshal([]byte(arguments), &a) + if a.Approved { + return "Confirmed by voice", "" + } + return "Declined by voice", "" + case "set_up_coding_agent": + return "Set up your coding agent", detail + default: + return strings.ReplaceAll(name, "_", " "), detail + } +} + +// checkEnding closes the overlay after end_call, once the agent's goodbye has +// finished playing. The goodbye comes after the tool call, so a pause before +// it doesn't count. It gives up waiting after endingTimeout. +func (s *assistantSession) checkEnding() { + if !s.ending || s.dismissed { + return + } + playing := s.pipeline.IsPlaying() + if playing { + s.spokeGoodbye = true + } + quiet := !playing && !s.agentSpeaking && time.Since(s.lastAudio) > 800*time.Millisecond + if ((s.spokeGoodbye || s.agentExited) && quiet) || time.Since(s.endingSince) > endingTimeout { + s.dismiss() + } +} + +const endingTimeout = 15 * time.Second + +// dismiss plays the overlay's exit animation, which then closes it. +func (s *assistantSession) dismiss() { + if s.dismissed { + return + } + s.dismissed = true + s.out.Send(map[string]string{"type": "dismiss"}) +} + +// showCode shows a show_code call as a code card. +func (s *assistantSession) showCode(fc *agent.FunctionCall) { + if s.shownCalls[fc.GetCallId()] { + return + } + var args struct { + Title string `json:"title"` + Language string `json:"language"` + Code string `json:"code"` + } + if err := json.Unmarshal([]byte(fc.GetArguments()), &args); err != nil || args.Code == "" { + return + } + s.shownCalls[fc.GetCallId()] = true + s.agentTurnForReply() + + id := fc.GetCallId() + s.snippets[id] = args.Code + s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, args.Title, normalizeLang(args.Language), args.Code, false}) +} + +// handOffTask shows a hand_off_task call as a prompt card for the user to +// copy into their coding agent. +func (s *assistantSession) handOffTask(fc *agent.FunctionCall) { + if s.shownCalls[fc.GetCallId()] { + return + } + var args struct { + Task string `json:"task"` + } + if err := json.Unmarshal([]byte(fc.GetArguments()), &args); err != nil || args.Task == "" { + return + } + s.shownCalls[fc.GetCallId()] = true + s.agentTurnForReply() + + id := fc.GetCallId() + s.snippets[id] = args.Task + s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, "Prompt for your coding agent", "prompt", args.Task, false}) +} + +func (s *assistantSession) copy(id string) { + code, ok := s.snippets[id] + if !ok || clipboard.WriteAll(code) != nil { + return + } + s.out.Send(overlayCopied{"copied", id}) +} + +// normalizeLang maps language names to the ones the page highlights. +func normalizeLang(lang string) string { + switch l := strings.ToLower(strings.TrimSpace(lang)); l { + case "bash", "sh", "zsh", "console", "terminal": + return "shell" + case "py": + return "python" + case "ts", "tsx", "typescript": + return "typescript" + case "js", "jsx", "mjs", "javascript", "node", "nodejs": + return "javascript" + default: + return l + } +} + +// resampleBands linearly interpolates bands to n values. +func resampleBands(src []float64, n int) []float64 { + out := make([]float64, n) + if len(src) == 0 { + return out + } + for i := range out { + f := float64(i) * float64(len(src)-1) / float64(n-1) + j := int(f) + if j >= len(src)-1 { + out[i] = src[len(src)-1] + continue + } + t := f - float64(j) + out[i] = src[j]*(1-t) + src[j+1]*t + } + return out +} diff --git a/cmd/lk/assistant_session_test.go b/cmd/lk/assistant_session_test.go new file mode 100644 index 00000000..f0ec61b1 --- /dev/null +++ b/cmd/lk/assistant_session_test.go @@ -0,0 +1,174 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "testing" + + "github.com/livekit/protocol/livekit/agent" + "github.com/stretchr/testify/require" +) + +// recordedOverlay collects the transcript messages a session sends. +type recordedOverlay struct { + turns []string // turn ids in the order they first appear + text map[string]string // "turn/segment" → latest text + seen map[string]struct{} // turn ids seen +} + +func (r *recordedOverlay) Send(msg any) error { + t, ok := msg.(overlayTranscript) + if !ok { + return nil + } + if _, ok := r.seen[t.Turn]; !ok { + r.seen[t.Turn] = struct{}{} + r.turns = append(r.turns, t.Turn) + } + r.text[fmt.Sprintf("%s/%d", t.Turn, t.Segment)] = t.Text + return nil +} + +func newRecordedSession() (*assistantSession, *recordedOverlay) { + rec := &recordedOverlay{text: map[string]string{}, seen: map[string]struct{}{}} + s := newAssistantSession(nil, nil) + s.out = rec + return s, rec +} + +func TestAssistantTurnsConversation(t *testing.T) { + s, rec := newRecordedSession() + + // The agent greets. + s.beginAgentSpeech() + s.agentText("Hi.") + s.agentFlush() + s.agentMessage(` Hi.`) + + // The user asks something. + s.userTranscript("How do I", false) + s.userTranscript("How do I deploy?", true) + s.userCommitted("How do I deploy?") + + // The agent answers. + s.beginAgentSpeech() + s.agentText("Run lk agent create.") + s.agentFlush() + s.agentMessage("Run lk agent create.") + + require.Equal(t, []string{"a1", "u2", "a3"}, rec.turns) + require.Equal(t, "Hi.", rec.text["a1/0"]) + require.Equal(t, "How do I deploy?", rec.text["u2/0"]) + require.Equal(t, "Run lk agent create.", rec.text["a3/0"]) +} + +func TestAssistantTurnsInterruption(t *testing.T) { + s, rec := newRecordedSession() + + // The agent starts speaking. + s.beginAgentSpeech() + s.agentText("You can") + s.agentText(" use the") + + // The user interrupts while words are still arriving. + s.userTranscript("Wait", false) + s.agentText(" starter") + s.agentFlush() + s.agentMessage("You can use the starter") + s.userTranscript("Wait, in Node?", true) + s.userCommitted("Wait, in Node?") + + // The agent replies to the interruption. + s.beginAgentSpeech() + s.agentText("Yes, in Node too.") + s.agentFlush() + s.agentMessage("Yes, in Node too.") + + // No turn is duplicated: the end of the interrupted speech stays in its + // turn, and the user's speech stays in one turn. + require.Equal(t, []string{"a1", "u2", "a3"}, rec.turns) + require.Equal(t, "You can use the starter", rec.text["a1/0"]) + require.Equal(t, "Wait, in Node?", rec.text["u2/0"]) + require.Equal(t, "Yes, in Node too.", rec.text["a3/0"]) +} + +func TestAssistantTurnsUnstreamedMessage(t *testing.T) { + s, rec := newRecordedSession() + + // Text that wasn't streamed shows from the committed message, without + // expressive markup. + s.agentMessage(` Hello there.`) + + require.Equal(t, []string{"a1"}, rec.turns) + require.Equal(t, "Hello there.", rec.text["a1/0"]) +} + +func TestAssistantTurnsPause(t *testing.T) { + s, rec := newRecordedSession() + + // The user pauses mid-thought, and the agent commits each part before + // it replies. + s.userTranscript("I want to build something", true) + s.userCommitted("I want to build something") + s.userTranscript("for my", false) + s.userTranscript("for my support team.", true) + s.userCommitted("for my support team.") + + s.beginAgentSpeech() + s.agentText("Great.") + s.agentFlush() + s.agentMessage("Great.") + + require.Equal(t, []string{"u1", "a2"}, rec.turns) + require.Equal(t, "I want to build something for my support team.", rec.text["u1/0"]) +} + +func TestAssistantTurnsEarlyCommit(t *testing.T) { + s, rec := newRecordedSession() + + // The turn detector ends the user's turn mid-sentence, and the agent + // starts looking things up before the rest arrives. + s.userTranscript("I want a voice agent for", true) + s.userCommitted("I want a voice agent for") + s.toolCall(&agent.FunctionCall{CallId: "c1", Name: "get_docs_overview", Arguments: "{}"}) + s.userTranscript("restaurant reservations.", true) + s.userCommitted("restaurant reservations.") + + s.beginAgentSpeech() + s.agentText("Sounds good.") + s.agentFlush() + s.agentMessage("Sounds good.") + + // The next exchange starts new turns. + s.userTranscript("What about Node?", true) + s.userCommitted("What about Node?") + s.beginAgentSpeech() + s.agentText("Node works too.") + s.agentFlush() + s.agentMessage("Node works too.") + + require.Equal(t, []string{"u1", "a2", "u3", "a4"}, rec.turns) + require.Equal(t, "I want a voice agent for restaurant reservations.", rec.text["u1/0"]) + require.Equal(t, "What about Node?", rec.text["u3/0"]) + require.Equal(t, "Node works too.", rec.text["a4/0"]) +} + +func TestIsAuthError(t *testing.T) { + require.True(t, isAuthError("type='llm_error' timestamp=1760024012.5 label='livekit.agents.inference.llm.LLM' error=APIStatusError('unauthorized', status_code=401, request_id=None) recoverable=False")) + require.True(t, isAuthError("Invalid API key")) + // A timestamp that happens to contain 401 isn't an auth error. + require.False(t, isAuthError("type='tts_error' timestamp=1791565401.871234 label='livekit.agents.inference.tts.TTS' error=APIConnectionError('connection failed') recoverable=True")) +} diff --git a/cmd/lk/assistant_tools.go b/cmd/lk/assistant_tools.go new file mode 100644 index 00000000..f54c04de --- /dev/null +++ b/cmd/lk/assistant_tools.go @@ -0,0 +1,542 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/livekit/livekit-cli/v2/pkg/agentfs" + "github.com/livekit/livekit-cli/v2/pkg/bootstrap" + "github.com/livekit/livekit-cli/v2/pkg/config" + "github.com/livekit/livekit-cli/v2/pkg/util" +) + +// The assistant agent runs lk commands through the agent channel. lk runs +// each one as a subprocess of itself, in the user's project directory, with +// the assistant's credentials. +// +// Commands that change something need the user's confirmation, by voice or +// on screen. The tool that proposes one shows a confirmation card and +// returns right away with its ID, so the agent can ask the user. If the +// user agrees by voice, the agent calls confirm_action, which runs the +// command and returns its result. If they click the card instead, lk runs +// the command and tells the agent the result. + +// toolState tracks lk tool requests. It's only used from the session's run +// loop; background work reports back through results. +type toolState struct { + ctx context.Context + dir string // the user's project directory + creds []string // LIVEKIT_* environment for subprocesses + results chan toolResult + pending map[string]pendingTool // confirmation id → command waiting for it + confirmN int + // lastProject is the project created this session, for try_project. + lastProject string +} + +// pendingTool is a command waiting for the user to confirm it. +type pendingTool struct { + args []string + command string // as shown to the user + project string // the project the command creates, if any + // start, if set, runs instead of the lk command. + start func(requestID, confirmID string) +} + +// toolResult is the outcome of background work, handled in the run loop. +type toolResult struct { + requestID string // the agent's request to reply to, if any + confirmID string // the confirmation card to update, if any + command string + project string // the project the command created, if any + output string + err error + // then, if set, handles the result instead of the default reply. + then func(toolResult) +} + +// toolOutputLimit bounds how much command output goes back to the LLM. +const toolOutputLimit = 6000 + +// Confirmation options in the overlay. +const ( + confirmRun = "run" + confirmCancel = "cancel" +) + +type ( + overlayConfirm struct { + Type string `json:"type"` + Turn string `json:"turn"` + ID string `json:"id"` + Heading string `json:"heading"` + Command string `json:"command"` // as the user would type it + Detail string `json:"detail"` // where it runs and what it changes + Options []overlayConfirmOption `json:"options"` + } + overlayConfirmOption struct { + ID string `json:"id"` + Name string `json:"name"` + } + overlayConfirmStatus struct { + Type string `json:"type"` + ID string `json:"id"` + Status string `json:"status"` // running, done, failed, or cancelled + Text string `json:"text,omitempty"` + } +) + +var projectNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]{0,62}$`) + +// runTool handles an lk tool request from the assistant agent. +func (s *assistantSession) runTool(requestID, name string, args map[string]any) { + str := func(key string) string { + v, _ := args[key].(string) + return strings.TrimSpace(v) + } + flag := func(key string) bool { + v, _ := args[key].(bool) + return v + } + switch name { + case "get_project_info": + s.replyTool(requestID, s.projectInfo(), nil) + + case "list_cloud_agents": + s.startCommand(requestID, "", []string{"agent", "list", "--json"}) + + case "list_templates": + s.inBackground(func(ctx context.Context) toolResult { + templates, err := bootstrap.FetchTemplates(ctx) + if err != nil { + return toolResult{requestID: requestID, err: err} + } + return toolResult{requestID: requestID, output: describeTemplates(templates)} + }) + + case "create_project": + s.proposeCreateProject(requestID, str("template"), str("name"), flag("in_current_directory")) + + case "set_up_coding_agent": + detail := "Adds LiveKit's skills and the Docs MCP server to the coding agents it finds, for this folder only." + cmdArgs := []string{"skills", "install", "--json", "-y"} + if flag("global") { + detail = "Adds LiveKit's skills and the Docs MCP server to the coding agents it finds, for all your projects." + cmdArgs = append(cmdArgs, "--global") + } + s.proposeCommand(requestID, detail, "Install", cmdArgs) + + case "confirm_action": + s.confirmByVoice(requestID, str("confirmation_id"), flag("approved")) + + case "try_project": + s.proposeTryProject(requestID, str("path")) + + default: + s.replyTool(requestID, "", fmt.Errorf("unknown tool %q", name)) + } +} + +// proposeCreateProject proposes creating a project from a template: agent +// templates with lk agent init, others (frontends and apps) with lk app +// create. +func (s *assistantSession) proposeCreateProject(requestID, template, name string, here bool) { + dir := s.tools.dir + if here { + if empty, err := util.IsEmptyDir(dir); err != nil || !empty { + s.replyTool(requestID, "", errors.New("the current directory isn't empty, so the project can't be created in it; create it in a new folder instead")) + return + } + } else if !projectNameRe.MatchString(name) { + s.replyTool(requestID, "", fmt.Errorf("%q isn't a valid project name: use lowercase letters, numbers, and dashes", name)) + return + } + + s.inBackground(func(ctx context.Context) toolResult { + templates, err := bootstrap.FetchTemplates(ctx) + if err != nil { + return toolResult{requestID: requestID, err: err} + } + i := slices.IndexFunc(templates, func(t bootstrap.Template) bool { + return t.Name == template && !t.IsHidden && !t.IsSandbox + }) + if i < 0 { + return toolResult{requestID: requestID, err: fmt.Errorf("there's no template named %q; call list_templates for the options", template)} + } + tpl := templates[i] + return toolResult{requestID: requestID, then: func(toolResult) { + target, where, project := name, "in a new folder, "+tildePath(filepath.Join(dir, name)), filepath.Join(dir, name) + if here { + target, where, project = ".", fmt.Sprintf("in this folder, named after it (%s)", filepath.Base(dir)), dir + } + kind, cmdArgs := "an app", []string{"app", "create"} + if isAgentTemplate(tpl) { + kind, cmdArgs = "an agent project", []string{"agent", "init"} + } + cmdArgs = append(cmdArgs, target, "--template", tpl.Name, "--install", "-y") + id := s.proposeCommand(requestID, + fmt.Sprintf("Creates %s from the %s template %s.", kind, tpl.Name, where), + "Create", cmdArgs) + if isAgentTemplate(tpl) { + p := s.tools.pending[id] + p.project = project + s.tools.pending[id] = p + } + }} + }) +} + +// isAgentTemplate reports whether a template is an agent (Python or Node.js) +// rather than a frontend or app. +func isAgentTemplate(t bootstrap.Template) bool { + return slices.Contains(t.Tags, "agents") && + (slices.Contains(t.Tags, "python") || slices.Contains(t.Tags, "node.js")) +} + +// describeTemplates lists the templates a user can create, for the LLM. +func describeTemplates(templates []bootstrap.Template) string { + var agents, apps []string + for _, t := range templates { + if t.IsHidden || t.IsSandbox { + continue + } + line := fmt.Sprintf("- %s: %s", t.Name, strings.TrimSpace(t.Desc)) + if isAgentTemplate(t) { + agents = append(agents, line) + } else { + apps = append(apps, line) + } + } + return "Agent templates:\n" + strings.Join(agents, "\n") + + "\n\nFrontend and app templates:\n" + strings.Join(apps, "\n") +} + +// proposeCommand shows a confirmation card for a command and replies to the +// agent right away with the confirmation ID, which it returns. +func (s *assistantSession) proposeCommand(requestID, detail, runLabel string, args []string) string { + t := &s.tools + t.confirmN++ + id := fmt.Sprintf("confirm%d", t.confirmN) + command := displayCommand(args) + t.pending[id] = pendingTool{args: args, command: command} + s.agentTurnForReply() + s.out.Send(overlayConfirm{ + Type: "confirm", Turn: s.agentTurn, ID: id, + Heading: "Run in " + tildePath(t.dir), + Command: command, + Detail: detail, + Options: []overlayConfirmOption{{ID: confirmRun, Name: runLabel}, {ID: confirmCancel, Name: "Cancel"}}, + }) + s.replyTool(requestID, fmt.Sprintf( + "Waiting for the user to confirm. Confirmation ID: %s. Command: %s. %s "+ + "Ask the user, in one short sentence, whether to go ahead. They can answer by voice or "+ + "on screen. If they agree by voice, call confirm_action with this ID and approved=true; "+ + "if they decline, approved=false. If they answer on screen, you'll be told the result.", + id, command, detail), nil) + return id +} + +// confirmByVoice handles confirm_action: the user answered by voice. +func (s *assistantSession) confirmByVoice(requestID, id string, approved bool) { + p, ok := s.tools.pending[id] + if !ok { + s.replyTool(requestID, "", fmt.Errorf("there's no pending confirmation %q; it may have been answered on screen already", id)) + return + } + delete(s.tools.pending, id) + option := confirmCancel + if approved { + option = confirmRun + } + s.out.Send(map[string]string{"type": "confirm_done", "id": id, "option": option}) + if !approved { + s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "cancelled"}) + s.replyTool(requestID, "Cancelled. Nothing was changed.", nil) + return + } + s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "running"}) + s.runConfirmed(requestID, id, p) +} + +// confirmTool handles the user's answer on screen. +func (s *assistantSession) confirmTool(id, option string) { + p, ok := s.tools.pending[id] + if !ok { + return + } + delete(s.tools.pending, id) + s.out.Send(map[string]string{"type": "confirm_done", "id": id, "option": option}) + if option != confirmRun { + s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "cancelled"}) + s.tellUser(fmt.Sprintf("The user cancelled %s on screen. Nothing was changed. Acknowledge it in a few words.", p.command)) + return + } + s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "running"}) + // No agent request waits for this result; toolFinished tells the agent. + s.runConfirmed("", id, p) +} + +func (s *assistantSession) runConfirmed(requestID, confirmID string, p pendingTool) { + if p.start != nil { + p.start(requestID, confirmID) + return + } + t := &s.tools + s.inBackground(func(ctx context.Context) toolResult { + output, err := runLK(ctx, t.dir, t.creds, p.args) + return toolResult{requestID: requestID, confirmID: confirmID, command: p.command, project: p.project, output: output, err: err} + }) +} + +// startCommand runs an lk command in the background and replies to the +// agent's request with its output. +func (s *assistantSession) startCommand(requestID, confirmID string, args []string) { + s.runConfirmed(requestID, confirmID, pendingTool{args: args, command: displayCommand(args)}) +} + +// inBackground runs fn off the run loop and hands its result to toolFinished. +func (s *assistantSession) inBackground(fn func(ctx context.Context) toolResult) { + t := &s.tools + go func() { + r := fn(t.ctx) + select { + case t.results <- r: + case <-t.ctx.Done(): + } + }() +} + +func (s *assistantSession) toolFinished(r toolResult) { + if r.then != nil { + r.then(r) + return + } + if r.err == nil && r.project != "" { + s.tools.lastProject = r.project + } + if r.confirmID != "" { + status := overlayConfirmStatus{Type: "confirm_status", ID: r.confirmID, Status: "done"} + if r.err != nil { + status.Status = "failed" + status.Text = lastLine(r.err.Error()) + } + s.out.Send(status) + } + if r.requestID != "" { + s.replyTool(r.requestID, r.output, r.err) + return + } + // Confirmed on screen: no request waits, so tell the agent what happened. + if r.err != nil { + s.tellUser(fmt.Sprintf("The user confirmed %s on screen, but it failed: %s. Tell them briefly.", r.command, truncate(r.err.Error(), 1500))) + return + } + s.tellUser(fmt.Sprintf("The user confirmed %s on screen, and it finished. Its output:\n%s\nTell them briefly what happened and the next step.", r.command, truncate(r.output, 1500))) +} + +func (s *assistantSession) replyTool(requestID, output string, err error) { + msg := map[string]any{"type": "reply", "id": requestID, "ok": err == nil, "output": output} + if err != nil { + msg["error"] = err.Error() + } + s.channel.send(msg) +} + +// tellUser asks the assistant agent to tell the user something. +func (s *assistantSession) tellUser(instructions string) { + s.channel.send(map[string]string{"type": "say", "instructions": instructions}) +} + +// displayCommand shows an lk command as the user would type it, without the +// flags lk adds to run it non-interactively. +func displayCommand(args []string) string { + shown := []string{"lk"} + for _, a := range args { + if a != "-y" && a != "--json" { + shown = append(shown, a) + } + } + return strings.Join(shown, " ") +} + +// tildePath shortens a path in the user's home directory to start with ~. +func tildePath(path string) string { + home, err := os.UserHomeDir() + if err != nil { + return path + } + if rel, err := filepath.Rel(home, path); err == nil && !strings.HasPrefix(rel, "..") { + return filepath.Join("~", rel) + } + return path +} + +// lastLine returns the last non-empty line of s, which for a failed command +// is usually its error. +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return strings.TrimSpace(lines[len(lines)-1]) +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} + +// lkExecutable finds the lk binary that tools run. It's a variable for tests. +var lkExecutable = os.Executable + +// ansiRe matches terminal escape codes: colors (CSI) and hyperlinks (OSC 8). +var ansiRe = regexp.MustCompile(`\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)`) + +// runLK runs this lk binary with args in dir and returns its output. +func runLK(ctx context.Context, dir string, creds, args []string) (string, error) { + exe, err := lkExecutable() + if err != nil { + return "", err + } + cmd := exec.CommandContext(ctx, exe, args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), creds...) + out, err := cmd.CombinedOutput() + text := truncate(strings.TrimSpace(ansiRe.ReplaceAllString(string(out), "")), toolOutputLimit) + if err != nil { + return "", fmt.Errorf("%s failed: %w\n%s", displayCommand(args), err, text) + } + return text, nil +} + +// projectInfo describes the user's LiveKit Cloud project and their current +// directory, as JSON for the LLM. +func (s *assistantSession) projectInfo() string { + empty, _ := util.IsEmptyDir(s.tools.dir) + info := map[string]any{"directory": s.tools.dir, "directory_is_empty": empty} + if s.cmd != nil { + if rp, err := resolveProject(s.cmd, loadParams{requireURL: true}); err == nil && rp.project != nil { + info["livekit_project"] = map[string]string{"name": rp.project.Name, "url": rp.project.URL} + } + } + if app := detectApp(s.tools.dir); app != nil { + info["app"] = app + } + if root, projectType, err := agentfs.DetectProjectRoot(s.tools.dir); err == nil { + local := map[string]any{"root": root, "type": string(projectType)} + if toml, ok, err := config.LoadTOMLFile(root, config.LiveKitTOMLFile); err == nil && ok && toml.HasAgent() { + local["deployed_agent_id"] = toml.Agent.ID + } + info["agent_project"] = local + } else { + info["agent_project"] = nil + } + b, _ := json.MarshalIndent(info, "", " ") + return string(b) +} + +// appFrameworks maps package.json dependencies to the frameworks they mean, +// most specific first. +var appFrameworks = []struct{ dep, name string }{ + {"next", "Next.js"}, + {"expo", "Expo (React Native)"}, + {"react-native", "React Native"}, + {"@angular/core", "Angular"}, + {"svelte", "Svelte"}, + {"vue", "Vue"}, + {"react", "React"}, +} + +// pythonFrameworks are Python web frameworks found in project files. +var pythonFrameworks = []struct{ dep, name string }{ + {"fastapi", "FastAPI"}, + {"django", "Django"}, + {"flask", "Flask"}, +} + +// detectApp describes an existing app in dir: its framework and any LiveKit +// packages it already uses. It returns nil if dir isn't an app it recognizes. +func detectApp(dir string) map[string]any { + app := map[string]any{} + var livekit []string + + if b, err := os.ReadFile(filepath.Join(dir, "package.json")); err == nil { + var pkg struct { + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + } + if json.Unmarshal(b, &pkg) == nil { + deps := map[string]bool{} + for d := range pkg.Dependencies { + deps[d] = true + } + for d := range pkg.DevDependencies { + deps[d] = true + } + for _, f := range appFrameworks { + if deps[f.dep] { + app["framework"] = f.name + break + } + } + for d := range deps { + if strings.HasPrefix(d, "@livekit/") || d == "livekit-client" || d == "livekit-server-sdk" { + livekit = append(livekit, d) + } + } + app["language"] = "JavaScript or TypeScript" + } + } + + var py strings.Builder + for _, f := range []string{"pyproject.toml", "requirements.txt"} { + if b, err := os.ReadFile(filepath.Join(dir, f)); err == nil { + py.Write(b) + } + } + if pyText := strings.ToLower(py.String()); pyText != "" { + if _, ok := app["language"]; !ok { + app["language"] = "Python" + } + for _, f := range pythonFrameworks { + if strings.Contains(pyText, f.dep) { + app["backend"] = f.name + break + } + } + for _, d := range []string{"livekit-agents", "livekit-api", "livekit"} { + if strings.Contains(pyText, d) { + livekit = append(livekit, d) + break + } + } + } + + if len(app) == 0 { + return nil + } + slices.Sort(livekit) + app["livekit_packages"] = livekit + return app +} diff --git a/cmd/lk/assistant_tools_test.go b/cmd/lk/assistant_tools_test.go new file mode 100644 index 00000000..89cb9371 --- /dev/null +++ b/cmd/lk/assistant_tools_test.go @@ -0,0 +1,33 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDisplayCommand(t *testing.T) { + require.Equal(t, "lk agent init . --lang python --install", + displayCommand([]string{"agent", "init", ".", "--lang", "python", "--install", "-y"})) + require.Equal(t, "lk skills install --global", + displayCommand([]string{"skills", "install", "--json", "-y", "--global"})) +} + +func TestStripTerminalEscapes(t *testing.T) { + in := "Visit \x1b]8;;https://cloud.livekit.io\x1b\\\x1b[36;4mhttps://cloud.livekit.io\x1b[0m\x1b]8;;\x1b\\ now" + require.Equal(t, "Visit https://cloud.livekit.io now", ansiRe.ReplaceAllString(in, "")) +} diff --git a/cmd/lk/assistant_xray.go b/cmd/lk/assistant_xray.go new file mode 100644 index 00000000..1cef7d55 --- /dev/null +++ b/cmd/lk/assistant_xray.go @@ -0,0 +1,156 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "fmt" + "strings" + + "github.com/livekit/protocol/livekit/agent" + + "github.com/livekit/livekit-cli/v2/pkg/assistant" +) + +// X-ray mode shows the assistant's own pipeline at work: the models it uses, +// per-turn timings, the turn detector's live end-of-turn estimate, and usage. +// The data comes from the session events console mode already sends. + +type ( + overlayPipeline struct { + Type string `json:"type"` + STT string `json:"stt"` + LLM string `json:"llm"` + TTS string `json:"tts"` + TurnDetection string `json:"turn_detection"` + } + overlayEOT struct { + Type string `json:"type"` + Probability float32 `json:"probability"` + Threshold float32 `json:"threshold"` + } + // overlayMetrics carries one message's timings, in seconds. Fields the + // agent didn't report are left out. + overlayMetrics struct { + Type string `json:"type"` + Role string `json:"role"` // user or assistant + EndOfTurnDelay *float64 `json:"end_of_turn_delay,omitempty"` + TranscriptionDelay *float64 `json:"transcription_delay,omitempty"` + LLMTTFT *float64 `json:"llm_ttft,omitempty"` + TTSTTFB *float64 `json:"tts_ttfb,omitempty"` + E2ELatency *float64 `json:"e2e_latency,omitempty"` + } + overlayUsage struct { + Type string `json:"type"` + Models []overlayModelUse `json:"models"` + } + overlayModelUse struct { + Kind string `json:"kind"` // stt, llm, or tts + Model string `json:"model"` + InputTokens int32 `json:"input_tokens,omitempty"` + OutputTokens int32 `json:"output_tokens,omitempty"` + Characters int32 `json:"characters,omitempty"` + AudioDuration float64 `json:"audio_duration,omitempty"` + } +) + +func metricsMessage(role string, m *agent.MetricsReport) overlayMetrics { + return overlayMetrics{ + Type: "metrics", + Role: role, + EndOfTurnDelay: m.EndOfTurnDelay, + TranscriptionDelay: m.TranscriptionDelay, + LLMTTFT: m.LlmNodeTtft, + TTSTTFB: m.TtsNodeTtfb, + E2ELatency: m.E2ELatency, + } +} + +func usageMessage(u *agent.AgentSessionUsage) overlayUsage { + msg := overlayUsage{Type: "usage"} + for _, mu := range u.GetModelUsage() { + switch { + case mu.GetLlm() != nil: + l := mu.GetLlm() + msg.Models = append(msg.Models, overlayModelUse{Kind: "llm", Model: l.GetModel(), InputTokens: l.GetInputTokens(), OutputTokens: l.GetOutputTokens()}) + case mu.GetTts() != nil: + t := mu.GetTts() + msg.Models = append(msg.Models, overlayModelUse{Kind: "tts", Model: t.GetModel(), Characters: t.GetCharactersCount(), AudioDuration: t.GetAudioDuration()}) + case mu.GetStt() != nil: + st := mu.GetStt() + msg.Models = append(msg.Models, overlayModelUse{Kind: "stt", Model: st.GetModel(), AudioDuration: st.GetAudioDuration()}) + } + } + return msg +} + +// showAgentCode shows a code card the agent sent about itself, such as its +// own AgentSession setup. +func (s *assistantSession) showAgentCode(m agentMessage) { + if m.Code == "" { + return + } + s.agentTurnForReply() + s.snippetN++ + id := fmt.Sprintf("self%d", s.snippetN) + s.snippets[id] = m.Code + s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, m.Title, normalizeLang(m.Language), m.Code, false}) +} + +// Startup errors show as a card in the overlay with what went wrong and the +// command that fixes it, since the terminal is hidden behind the overlay. + +type overlayError struct { + Type string `json:"type"` + Title string `json:"title"` + Detail string `json:"detail"` + Fix string `json:"fix,omitempty"` // a command to copy +} + +// errNoCredentials marks an error loading the LiveKit Cloud credentials. +type errNoCredentials struct{ err error } + +func (e errNoCredentials) Error() string { return e.err.Error() } +func (e errNoCredentials) Unwrap() error { return e.err } + +func startupError(err error) overlayError { + var creds errNoCredentials + switch { + case errors.Is(err, assistant.ErrNoUV): + return overlayError{Type: "error", + Title: "Install uv to start the assistant", + Detail: "The assistant is a Python voice agent, and lk runs it with uv, a Python package manager. Install uv, then run lk assistant again.", + Fix: "curl -LsSf https://astral.sh/uv/install.sh | sh"} + case errors.As(err, &creds): + return overlayError{Type: "error", + Title: "Connect lk to LiveKit Cloud", + Detail: "The assistant uses LiveKit Inference through your LiveKit Cloud project. " + firstLine(creds.Error()), + Fix: "lk cloud auth"} + default: + return overlayError{Type: "error", + Title: "Couldn't start the assistant", + Detail: truncate(strings.TrimSpace(err.Error()), 600)} + } +} + +func (s *assistantSession) showError(e overlayError) { + s.out.Send(e) + s.out.Send(overlayNotice{Type: "notice", Text: e.Title, Sticky: true}) +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return line +} diff --git a/cmd/lk/main.go b/cmd/lk/main.go index 1b73e79b..cf916cb7 100644 --- a/cmd/lk/main.go +++ b/cmd/lk/main.go @@ -95,6 +95,7 @@ Docs: https://docs.livekit.io/intro/basics/cli/`, app.Commands = append(app.Commands, CloudCommands...) app.Commands = append(app.Commands, DocsCommands...) app.Commands = append(app.Commands, SkillsCommands...) + app.Commands = append(app.Commands, AssistantCommands...) app.Commands = append(app.Commands, ProjectCommands...) app.Commands = append(app.Commands, WorkspaceCommands...) app.Commands = append(app.Commands, UserCommands...) diff --git a/pkg/assistant/agent/agent.py b/pkg/assistant/agent/agent.py new file mode 100644 index 00000000..3cb99aae --- /dev/null +++ b/pkg/assistant/agent/agent.py @@ -0,0 +1,524 @@ +"""The LiveKit assistant: a voice agent that helps developers build with LiveKit. + +`lk assistant` runs this agent in console mode and shows the conversation in +its overlay. Code the agent shows with `show_code` appears as a card in the +overlay, with a button to copy it. +""" + +import asyncio +import json +import os +import random + +from livekit import agents +from livekit.agents import ( + Agent, + AgentServer, + AgentSession, + RunContext, + TurnHandlingOptions, + function_tool, + inference, + mcp, + text_transforms, +) +from livekit.agents.beta.tools import EndCallTool +from livekit.agents.voice import TranscriptSynchronizer, io + +# Private module: the SDK's own stripper for expressive markup, the same one the +# room transcription output uses. Check it when upgrading livekit-agents. +from livekit.agents.tts._provider_format import TranscriptMarkupStripper + +DOCS_MCP_URL = "https://docs.livekit.io/mcp" + +STT_MODEL = "assemblyai/universal-3-6-pro" +LLM_MODEL = "openai/gpt-5.4-mini" +TTS_MODEL = "cartesia/sonic-3" + +GREETING = ( + "Hi, I'm LK, your LiveKit assistant. I can help you build a voice agent, add LiveKit to an " + "app you already have, or explore what LiveKit can do, like realtime voice models, phone " + "calls, and simulations for testing your agents. What would you like to do?" +) + +# Pronunciations for the TTS, as Cartesia inline IPA overrides: "live" as in +# "alive", not "liver". They apply only to the speech, so transcripts keep the +# real spelling. Matching ignores case, and longer keys match first, so the +# possessive gets its own key. The syntax is Cartesia-specific; see +# docs.livekit.io/agents/multimodality/text/#built-in-replace-transform. +LIVEKIT_IPA = "<<ˈ|l|aɪ|v>> <<ˈ|k|ɪ|t>>" +LIVEKITS_IPA = "<<ˈ|l|aɪ|v>> <<ˈ|k|ɪ|t|s>>" +PRONUNCIATIONS = { + "LiveKit's": LIVEKITS_IPA, + "LiveKit’s": LIVEKITS_IPA, + "LiveKit": LIVEKIT_IPA, + "Live Kit": LIVEKIT_IPA, +} + +# The assistant's name, LK, is said as the letters L and K. Matching is +# case-sensitive so that words like "walk" don't match. +NAME_PRONUNCIATIONS = {"LK": "<<ˈ|ɛ|l>> <<ˈ|k|eɪ>>"} + +# The Docs MCP tools the agent uses. +DOCS_TOOLS = ("get_docs_overview", "docs_search", "get_pages", "code_search") + +# What the agent says while it looks something up, if it hasn't said anything +# yet this turn. Docs lookups take a second or two. +LOOKUP_FILLERS = ( + "Let me check the docs.", + "One moment, I'll look that up.", + "Let me look that up in the docs.", +) + +# Docs MCP results can be whole pages. Cap them to bound the LLM context, but +# high enough that a full docs page fits. The longest pages, such as +# /agents/build/turns/, are over 37,000 characters. +MAX_TOOL_RESULT_CHARS = 50000 + +INSTRUCTIONS = """You are LK, the LiveKit assistant, a voice assistant inside the LiveKit CLI. +You help developers build voice AI agents with LiveKit, add LiveKit to apps they already have, and +explore what LiveKit can do. You are yourself a LiveKit voice agent, built with the same Agents SDK +and LiveKit Inference the user can build with. + +Match the user's level. If they seem new, describe outcomes before LiveKit terms, and explain a +term in a few words the first time you use it. If they seem experienced, be terse and precise, and +skip the basics. If they ask you to be briefer or to go deeper, do that for the rest of the +conversation. + +You speak out loud, so: +- Keep replies short: one to three sentences. Offer to go deeper instead of covering everything. +- Use plain sentences. No markdown, lists, emojis, or symbols. +- Never read code, commands, or long identifiers aloud. Call show_code to show them, + then say one short sentence about what you showed. + +LiveKit changes often, and what you remember about it is out of date. For every question about +LiveKit features, APIs, SDKs, models, plugins, CLI commands, pricing, or configuration, call +docs_search first, then answer only from what the docs say. Do this even when you think you know +the answer. For questions about how something works or how to configure it, also read the most +relevant page with get_pages before you answer. When you're not sure where a topic lives, or the +user asks what LiveKit can do, call get_docs_overview to see how the docs are organized. If the +docs don't cover it, say so. Never guess an API. + +You can't see or change the user's files. You help them plan and decide, and you set them up to +do the work: +- To learn about their LiveKit Cloud project and current directory, call get_project_info. To check + on deployed agents, call list_cloud_agents. +- For a new project, start it from a template with create_project. If the user hasn't said which + template, call list_templates and suggest the best fit: for a voice agent, agent-starter-python + or agent-starter-node. Call get_project_info first: if the current directory is empty, ask + whether to create the project right there or in a new folder. New projects already include + LiveKit's skills and Docs MCP server for the user's coding agents. +- For an existing project, offer set_up_coding_agent, which installs LiveKit's skills and Docs MCP + server into the user's coding agents (such as Claude Code, Cursor, or Codex). +- To add LiveKit to an existing app (get_project_info reports its framework), explain the three + parts in plain words: the app gets a token from a small server endpoint, joins a LiveKit room, + and the agent joins that room through dispatch. Then create the agent with create_project and + give the frontend and token work to their coding agent with hand_off_task, based on the docs. +- After a project is created, offer to try it right here with try_project, so the user can talk + to the agent they just made. +- If the user asks how you work, or what they'll be building, call explain_yourself. +- When the user is ready to write code, call hand_off_task with a prompt for their coding agent: + what to build or change, plus what you learned from the docs, such as package names, APIs, and + CLI commands. + +create_project and set_up_coding_agent don't run right away. They show the command to the user and +return a confirmation ID. Ask the user in one short sentence whether to go ahead. They can answer +by voice or on screen. If they agree by voice, call confirm_action with the ID and approved=true; +if they decline, approved=false. Never call confirm_action unless the user has just agreed. + +When the user says they're done or says goodbye, call end_call. + +Prefer the LiveKit CLI for project tasks. For example, "lk agent init" creates an agent from a +starter template, "lk agent dev" runs it, and "lk agent create" deploys it to LiveKit Cloud.""" + + +async def truncate_result(ctx: mcp.MCPToolResultContext) -> str: + if len(ctx.result.content) == 1: + text = str(ctx.result.content[0].model_dump_json()) + elif ctx.result.content: + text = json.dumps([item.model_dump() for item in ctx.result.content]) + else: + return "The tool returned no content." + if len(text) > MAX_TOOL_RESULT_CHARS: + text = text[:MAX_TOOL_RESULT_CHARS] + "\n... [truncated]" + return text + + +class Assistant(Agent): + def __init__(self) -> None: + instructions = INSTRUCTIONS + if project := os.environ.get("LK_ASSISTANT_PROJECT"): + instructions += f"\n\nThe user's current directory is {project}." + super().__init__( + instructions=instructions, + tools=[ + mcp.MCPToolset( + id="livekit-docs", + mcp_server=mcp.MCPServerHTTP( + DOCS_MCP_URL, + allowed_tools=list(DOCS_TOOLS), + tool_result_resolver=truncate_result, + ), + ), + # lk closes its overlay when it sees this tool called. Console + # mode has no real room, so there's nothing to delete. + *EndCallTool( + extra_description="Use this when the user says they're done, says goodbye, or asks to close the assistant.", + delete_room=False, + end_instructions="Say a short, friendly goodbye in one sentence.", + ).tools, + ], + ) + + @function_tool() + async def show_code(self, context: RunContext, title: str, language: str, code: str) -> str: + """Show code or a terminal command to the user. The user sees it on screen and can + copy it. Use this for all code and commands instead of saying them. + + Args: + title: A file name such as agent.py, or "Terminal" for shell commands. + language: The language of the code, such as python, typescript, or shell. + code: The complete code or command, exactly as the user should use it. + """ + # `lk assistant` reads this call from the session events and shows it. + return "The code is on the user's screen. They can copy it with the Copy button." + + @function_tool() + async def hand_off_task(self, context: RunContext, task: str) -> str: + """Show the user a prompt to paste into their coding agent, with a button to copy it. + + Args: + task: A complete, self-contained prompt: what to build or change in the user's + project, plus details from the LiveKit docs such as package names, APIs, and CLI + commands. + """ + # `lk assistant` reads this call from the session events and shows it. + return "The prompt is on the user's screen. They can copy it into their coding agent." + + @function_tool() + async def get_project_info(self, context: RunContext) -> str: + """Get the user's LiveKit Cloud project (the one lk uses), their current directory, and + whether it's an agent project, including the ID of its deployed agent if it has one.""" + return await run_lk_tool("get_project_info") + + @function_tool() + async def list_cloud_agents(self, context: RunContext) -> str: + """List the agents deployed to the user's LiveKit Cloud project, with their status.""" + return await run_lk_tool("list_cloud_agents") + + @function_tool() + async def list_templates(self, context: RunContext) -> str: + """List LiveKit's project templates: agents (Python and Node.js) and frontends (web, + mobile, and others), with a description of each.""" + return await run_lk_tool("list_templates") + + @function_tool() + async def create_project( + self, context: RunContext, template: str, name: str = "", in_current_directory: bool = False + ) -> str: + """Propose creating a project from a LiveKit template, with lk agent init for agent + templates or lk app create for frontends, including installing dependencies. It doesn't run + until the user confirms (see confirm_action). Afterward, tell them how to run it: for an + agent, lk agent dev in the project folder. + + Args: + template: The template name, from list_templates, such as agent-starter-python. + name: The name for the new folder: lowercase letters, numbers, and dashes. Not used + with in_current_directory. + in_current_directory: Create the project in the current directory itself, named after + it, instead of in a new folder. Only works if the directory is empty, and only if + the user wants it. + """ + return await run_lk_tool( + "create_project", + {"template": template, "name": name, "in_current_directory": in_current_directory}, + ) + + @function_tool() + async def set_up_coding_agent(self, context: RunContext, all_projects: bool = False) -> str: + """Propose installing LiveKit's agent skills and the LiveKit Docs MCP server into the + user's coding agents (such as Claude Code, Cursor, and Codex), with lk skills install. Use + it for existing projects; new projects from create_project already have them. It doesn't + run until the user confirms (see confirm_action). + + Args: + all_projects: Install for all of the user's projects instead of only the current one. + """ + return await run_lk_tool("set_up_coding_agent", {"global": all_projects}) + + @function_tool() + async def try_project(self, context: RunContext, path: str = "") -> str: + """Propose running the user's agent in this window with lk agent console, so they can talk + to it. The conversation switches to their agent until they press B to come back to you. + It doesn't run until the user confirms (see confirm_action). + + Args: + path: The project folder, relative to the user's current directory. Leave it empty + for the project created in this session, or the current directory. + """ + return await run_lk_tool("try_project", {"path": path}) + + @function_tool() + async def explain_yourself(self, context: RunContext) -> str: + """Show the user how you're built: your own AgentSession code, on screen. Use it when they + ask how you work, or to show what a LiveKit voice agent looks like.""" + code = own_session_code() + if lk is not None and code: + lk.send({"type": "show_code", "title": "agent.py (how LK is built)", "language": "python", "code": code}) + return ( + "Your own setup is on the user's screen. You run on the LiveKit Agents SDK in " + "console mode, inside lk. Your pipeline: speech to text with AssemblyAI, the " + "gpt-5.4-mini LLM, and text to speech with Cartesia, all through LiveKit Inference. " + "LiveKit's turn detector decides when the user has finished, with dynamic " + "endpointing. Expressive mode adds emotion and pacing to your voice. Your tools are " + "the LiveKit Docs MCP server plus lk commands. In one or two sentences, explain this " + "and point out that a project from agent-starter-python has the same structure." + ) + + @function_tool() + async def confirm_action(self, context: RunContext, confirmation_id: str, approved: bool) -> str: + """Run or cancel a command that create_project or set_up_coding_agent proposed, after the + user answers by voice. Only call it right after the user agrees or declines. + + Args: + confirmation_id: The ID the proposing tool returned, such as confirm1. Works for + create_project, set_up_coding_agent, and try_project. + approved: True if the user agreed, false if they declined. + """ + # The command changes the user's files. While it runs, user speech + # mustn't interrupt this step, or its result is never spoken. + context.disallow_interruptions() + try: + return await run_lk_tool( + "confirm_action", + {"confirmation_id": confirmation_id, "approved": approved}, + timeout=COMMAND_TIMEOUT, + ) + finally: + # The reply after the command reuses this step's speech. Let the + # user talk over it, or what they say while it plays is dropped. + context.speech_handle.allow_interruptions = True + + +# Confirmed commands can take a while, such as installing dependencies. +COMMAND_TIMEOUT = 600 + + +def own_session_code() -> str: + """Returns this agent's AgentSession setup, from its own source.""" + try: + source = open(__file__).read() + except OSError: + return "" + # The last occurrence: the first is this function's own search string. + start = source.rfind(" session = AgentSession(") + end = source.find("\n )\n", start) + if start < 0 or end < 0: + return "" + lines = source[start : end + 6].splitlines() + code = "\n".join(line[4:] if line.startswith(" ") else line for line in lines) + # Show the model names themselves, not the constants. + for name, value in (("STT_MODEL", STT_MODEL), ("LLM_MODEL", LLM_MODEL), ("TTS_MODEL", TTS_MODEL)): + code = code.replace(f"model={name}", f'model="{value}"') + return code + + +async def run_lk_tool(name: str, args: dict | None = None, timeout: float = 60) -> str: + """Runs an lk tool through lk and returns its output for the LLM.""" + if lk is None: + return "This tool is only available when the assistant runs inside lk." + try: + reply = await lk.request({"type": "lk_tool", "name": name, "args": args or {}}, timeout=timeout) + except asyncio.TimeoutError: + return "lk didn't respond in time." + if not reply.get("ok"): + return f"Error: {reply.get('error', 'unknown error')}" + return reply.get("output") or "Done." + + +class LkChannel: + """The local connection to lk, as JSON lines in both directions. + + The agent sends its transcript and requests to run lk tools. lk sends + replies to requests, and can ask the agent to tell the user something. + """ + + def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + self._reader = reader + self._writer = writer + self._pending: dict[str, asyncio.Future[dict]] = {} + self._next_id = 0 + self._session: AgentSession | None = None + self._read_task: asyncio.Task[None] | None = None + + def start(self, session: AgentSession) -> None: + self._session = session + self._read_task = asyncio.create_task(self._read()) + + def send(self, msg: dict) -> None: + if not self._writer.is_closing(): + self._writer.write((json.dumps(msg) + "\n").encode()) + + async def request(self, msg: dict, timeout: float = 10) -> dict: + """Sends msg and waits for lk's reply.""" + self._next_id += 1 + msg_id = str(self._next_id) + fut: asyncio.Future[dict] = asyncio.get_running_loop().create_future() + self._pending[msg_id] = fut + self.send({**msg, "id": msg_id}) + try: + return await asyncio.wait_for(fut, timeout) + finally: + self._pending.pop(msg_id, None) + + async def _read(self) -> None: + while line := await self._reader.readline(): + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + if msg.get("type") == "reply": + fut = self._pending.get(msg.get("id", "")) + if fut and not fut.done(): + fut.set_result(msg) + elif msg.get("type") == "say" and self._session is not None: + # lk has news for the user. + self._session.generate_reply(instructions=msg.get("instructions", "")) + + +# The connection to lk, when lk runs this agent. +lk: LkChannel | None = None + + +class TranscriptForwarder(io.TextOutput): + """Sends the agent's transcript to lk as JSON lines, in time with its speech. + + Each line is {"type": "text", "text": ""} or {"type": "flush"} at the + end of a segment. Expressive markup is removed. + """ + + def __init__(self, channel: LkChannel) -> None: + super().__init__(label="lk-assistant", next_in_chain=None) + self._channel = channel + self._stripper = TranscriptMarkupStripper() + + async def capture_text(self, text: str) -> None: + self._send_text(self._stripper.push(text)) + + def flush(self) -> None: + self._send_text(self._stripper.flush()) + self._send({"type": "flush"}) + self._stripper = TranscriptMarkupStripper() + + def _send_text(self, text: str) -> None: + if text: + self._send({"type": "text", "text": text}) + + def _send(self, msg: dict) -> None: + self._channel.send(msg) + + +async def connect_lk(session: AgentSession) -> None: + """Connects to lk, if lk is running this agent, and streams the agent's + spoken text to it. + + Console mode only reports what the agent said after it finishes speaking. It + also syncs the text to the audio, but sends the result nowhere. This swaps in a + synchronizer whose text goes to lk instead. + """ + global lk + addr = os.environ.get("LK_ASSISTANT_ADDR") + if not addr: + return + host, port = addr.rsplit(":", 1) + reader, writer = await asyncio.open_connection(host, int(port)) + lk = LkChannel(reader, writer) + lk.start(session) + # For lk's X-ray view of the pipeline. + lk.send({"type": "pipeline", "stt": STT_MODEL, "llm": LLM_MODEL, "tts": TTS_MODEL, "turn_detection": "LiveKit turn detector, dynamic endpointing"}) + + audio = session.output.audio + if audio is None: + return + if audio.label == "TranscriptSynchronizer" and audio.next_in_chain is not None: + audio = audio.next_in_chain + sync = TranscriptSynchronizer(next_in_chain_audio=audio, next_in_chain_text=TranscriptForwarder(lk)) + session.output.audio = sync.audio_output + session.output.transcription = sync.text_output + + +def speak_during_lookups(session: AgentSession) -> None: + """Says a short line when a docs lookup starts, so the user doesn't sit in + silence. Skipped if the agent already spoke this turn.""" + spoke = False + + @session.on("conversation_item_added") + def _on_item(ev) -> None: + nonlocal spoke + if getattr(ev.item, "role", None) == "user": + spoke = False + + @session.on("agent_state_changed") + def _on_state(ev) -> None: + nonlocal spoke + if ev.new_state == "speaking": + spoke = True + + @session.on("tool_execution_updated") + def _on_tool(ev) -> None: + nonlocal spoke + update = ev.update + if update.type != "tool_call_started" or update.function_call.name not in DOCS_TOOLS or spoke: + return + spoke = True + session.say(random.choice(LOOKUP_FILLERS), add_to_chat_ctx=False) + + +server = AgentServer() + + +@server.rtc_session(agent_name="lk-assistant") +async def assistant(ctx: agents.JobContext): + # STT from the Voice AI quickstart + # (docs.livekit.io/agents/start/voice-ai-quickstart), with two changes: + # - LLM: gpt-5.4-mini instead of Gemma, for more reliable tool calling. + # The assistant must search the docs before it answers. + # - TTS: Cartesia instead of Fish Audio, for inline phoneme overrides that + # make it say "LiveKit" correctly every time. + session = AgentSession( + stt=inference.STT(model=STT_MODEL, language="en"), + llm=inference.LLM(model=LLM_MODEL), + tts=inference.TTS( + model=TTS_MODEL, + voice="a167e0f3-df7e-4d52-a9c3-f949145efdab", + language="en", + ), + turn_handling=TurnHandlingOptions( + turn_detection=inference.TurnDetector(), + # Developers often pause mid-sentence to think. With the turn + # detector, the shortest wait defaults to 0.3 seconds, which can + # end a turn mid-thought. Dynamic endpointing learns the user's + # pauses; the higher floor leaves room for a short one. + endpointing={"mode": "dynamic", "min_delay": 0.8}, + ), + # The LLM marks up emotion and pacing for the TTS. The markup is + # stripped from transcripts. + expressive=True, + tts_text_transforms=[ + "filter_emoji", + "filter_markdown", + text_transforms.replace(PRONUNCIATIONS), + text_transforms.replace(NAME_PRONUNCIATIONS, case_sensitive=True), + ], + ) + + speak_during_lookups(session) + await session.start(room=ctx.room, agent=Assistant()) + await connect_lk(session) + + # A fixed greeting: the same every time, and it starts without waiting on the LLM. + session.say(GREETING) + + +if __name__ == "__main__": + agents.cli.run_app(server) diff --git a/pkg/assistant/agent/pyproject.toml b/pkg/assistant/agent/pyproject.toml new file mode 100644 index 00000000..906b2fdb --- /dev/null +++ b/pkg/assistant/agent/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "lk-assistant-agent" +version = "0.1.0" +description = "The voice agent behind lk assistant" +requires-python = ">=3.10,<3.15" +dependencies = [ + "livekit-agents[mcp]~=1.8", +] diff --git a/pkg/assistant/assistant.go b/pkg/assistant/assistant.go new file mode 100644 index 00000000..3d676a2a --- /dev/null +++ b/pkg/assistant/assistant.go @@ -0,0 +1,113 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package assistant holds the voice agent behind `lk assistant`. The agent's +// Python project is embedded in the CLI and installed into a cache directory +// the first time it runs. +package assistant + +import ( + "bytes" + "context" + "crypto/sha256" + "embed" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" +) + +//go:embed agent/agent.py agent/pyproject.toml +var agentFiles embed.FS + +// Entrypoint is the agent's entrypoint file, relative to its project directory. +const Entrypoint = "agent.py" + +// Names of agent tools that lk acts on. +const ( + // ShowCodeTool shows code as a card in the overlay. + ShowCodeTool = "show_code" + // HandOffTaskTool shows a prompt for the user's coding agent as a card. + HandOffTaskTool = "hand_off_task" + // EndCallTool ends the conversation. lk closes the overlay after it. + EndCallTool = "end_call" +) + +// ErrNoUV means uv, which runs the agent, isn't installed. +var ErrNoUV = errors.New("lk assistant runs its agent with uv. Install uv (https://docs.astral.sh/uv/) and try again") + +// stampFile records the hash of the agent files that were last installed. +const stampFile = ".lk-installed" + +// DefaultDir is where the agent project is installed. +func DefaultDir() (string, error) { + cache, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(cache, "livekit", "lk-assistant-agent"), nil +} + +// Prepare writes the agent project to dir and installs its dependencies with +// uv. Dependencies are installed again only when the agent files change. +// status is called with a short description before slow steps. +func Prepare(ctx context.Context, dir string, status func(string)) error { + uv, err := exec.LookPath("uv") + if err != nil { + return ErrNoUV + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + + sum := sha256.New() + err = fs.WalkDir(agentFiles, "agent", func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + data, err := agentFiles.ReadFile(path) + if err != nil { + return err + } + sum.Write([]byte(path)) + sum.Write(data) + dst := filepath.Join(dir, filepath.Base(path)) + if existing, err := os.ReadFile(dst); err == nil && bytes.Equal(existing, data) { + return nil + } + return os.WriteFile(dst, data, 0o644) + }) + if err != nil { + return fmt.Errorf("writing the assistant agent: %w", err) + } + hash := hex.EncodeToString(sum.Sum(nil)) + + stamp := filepath.Join(dir, stampFile) + if installed, err := os.ReadFile(stamp); err == nil && string(installed) == hash { + if _, err := os.Stat(filepath.Join(dir, ".venv")); err == nil { + return nil + } + } + + status("Installing the assistant agent") + cmd := exec.CommandContext(ctx, uv, "sync") + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("installing the assistant agent failed: %w\n%s", err, out) + } + return os.WriteFile(stamp, []byte(hash), 0o644) +} From f78a40c09cd1c00a0e2ea49525c78506d1c90dc6 Mon Sep 17 00:00:00 2001 From: royalfig Date: Mon, 12 Oct 2026 18:07:33 -0400 Subject: [PATCH 06/12] feat(overlay): typed input, docs answer and link cards, inline code, guest status Press / to type instead of speaking. Answer cards show short points with their docs source, link cards open a docs page, and backticked text in the transcript shows as code. The guest bar shows when the user's agent is starting. Co-Authored-By: Claude Opus 5.5 --- pkg/overlay/overlay.go | 8 +- pkg/overlay/ui/index.html | 152 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 151 insertions(+), 9 deletions(-) diff --git a/pkg/overlay/overlay.go b/pkg/overlay/overlay.go index 4d63767c..69f99371 100644 --- a/pkg/overlay/overlay.go +++ b/pkg/overlay/overlay.go @@ -22,11 +22,14 @@ // {"type":"bands","bands":[0..1, ...]} // {"type":"transcript","turn":"...","role":"user|agent","segment":0,"text":"..."} // {"type":"snippet","turn":"...","id":"...","title":"...","lang":"python|shell","code":"...","copied":bool} +// {"type":"link","turn":"...","id":"...","title":"...","url":"..."} +// {"type":"answer","turn":"...","id":"...","title":"...","points":["..."],"source_title":"...","source_url":"..."} // {"type":"copied","id":"..."} // {"type":"tool","turn":"...","id":"...","label":"...","detail":"...","status":"running|done|error"} // {"type":"notice","text":"...","sticky":bool} // {"type":"error","title":"...","detail":"...","fix":"command"} // {"type":"speaker","name":"...","guest":bool} +// {"type":"guest_status","status":"starting|failed","name":"..."} // {"type":"pipeline",...} {"type":"eot",...} {"type":"metrics",...} {"type":"usage",...} (X-ray) // {"type":"dismiss"} // {"type":"clear"} @@ -34,7 +37,8 @@ // {"type":"confirm_done","id":"...","option":"..."} // page → Go {"type":"ready"} {"type":"copy","id":"..."} {"type":"replay"} {"type":"close"} // {"type":"confirm","id":"...","option":"..."} {"type":"copy_text","text":"..."} -// {"type":"back"} {"type":"mute","on":bool} +// {"type":"back"} {"type":"mute","on":bool} {"type":"open","id":"..."} +// {"type":"text","text":"..."} (typed input) // // A transcript message carries the full text so far of one segment (a // paragraph) of a turn. Snippets are added to their turn after the segments @@ -71,7 +75,7 @@ type Event struct { On bool `json:"on,omitempty"` // Option is the chosen option of a confirm event. Option string `json:"option,omitempty"` - // Text is the text of a copy_text event. + // Text is the text of a copy_text or text event. Text string `json:"text,omitempty"` } diff --git a/pkg/overlay/ui/index.html b/pkg/overlay/ui/index.html index c6775304..453b58a9 100644 --- a/pkg/overlay/ui/index.html +++ b/pkg/overlay/ui/index.html @@ -60,6 +60,15 @@ #status[data-state="thinking"], #status[data-state="speaking"], #status.notice { color: var(--accent); } #hints { position: fixed; right: 32px; bottom: 28px; color: var(--fg4); } + /* Typed input, for when speaking isn't an option. */ + #compose { position: fixed; left: 50%; bottom: 16px; transform: translateX(-50%); width: min(560px, calc(100vw - 32px)); display: none; } + body.composing #compose { display: block; } + body.composing #status { visibility: hidden; } + #compose input { + width: 100%; box-sizing: border-box; padding: 10px 14px; font: 15px/1.4 var(--sans); color: var(--fg0); + background: rgba(19, 19, 19, 0.9); border: 1px solid rgba(31, 213, 249, 0.35); border-radius: var(--radius); outline: none; + } + #compose input::placeholder { color: var(--fg4); } #brand .mic { display: none; margin-left: 6px; padding: 2px 6px; border: 1px solid rgba(245, 101, 92, 0.5); border-radius: 3px; color: var(--remove); } body.mic-off #brand .mic { display: inline; } @@ -73,6 +82,11 @@ body.guest #guest { display: flex; } #guest .dot { width: 6px; height: 6px; background: var(--purple); animation: blink 1.2s steps(2) infinite; } #guest .name { color: var(--purple); } + #guest .hint { color: var(--fg3); } + #guest .hint:empty { display: none; } + /* While the user's agent starts, the bar shows it, without Back. */ + body.guest-starting #guest { display: flex; } + body.guest-starting #guest button { display: none; } #guest button { padding: 5px 9px; cursor: pointer; color: var(--fg1); background: rgba(255, 255, 255, 0.05); border: 1px solid rgba(255, 255, 255, 0.1); border-radius: 4px; @@ -127,6 +141,7 @@ .turn.agent .who { color: var(--accent); } .turn .body > * + * { margin-top: 14px; } .turn p { margin: 0; font-size: 18px; line-height: 1.55; } + .turn p code { padding: 1px 5px; font: 0.82em var(--mono); color: var(--fg0); background: rgba(255, 255, 255, 0.07); border: 1px solid var(--separator); border-radius: 4px; overflow-wrap: anywhere; } .turn.user p { color: var(--fg3); } .turn.agent p { color: var(--fg0); } /* Tool calls the agent made, such as docs searches. */ @@ -158,6 +173,19 @@ } .card .copy:hover { background: rgba(255, 255, 255, 0.1); } .card[data-copied="true"] .copy { color: var(--accent); border-color: rgba(31, 213, 249, 0.35); } + /* A docs page to read more. */ + .card.link header { border-bottom: 0; } + .card.link .kind { flex: none; color: var(--accent); } + .card.link .page { min-width: 0; display: flex; flex-direction: column; gap: 2px; } + .card.link .title { font: 14px/1.4 var(--sans); color: var(--fg0); } + .card.link .url { font: 11.5px/1.4 var(--mono); color: var(--fg4); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } + .card.link .open { margin-left: 0; } + /* A docs answer: short points, with the page they come from. */ + .card.answer .kind { color: var(--accent); } + .card.answer ul { margin: 0; padding: 12px 16px 12px 32px; display: flex; flex-direction: column; gap: 6px; font: 15px/1.5 var(--sans); color: var(--fg1); user-select: text; } + .card.answer ul code { padding: 1px 5px; font: 0.85em var(--mono); color: var(--fg0); background: rgba(255, 255, 255, 0.07); border: 1px solid var(--separator); border-radius: 4px; } + .card.answer footer { display: flex; align-items: center; gap: 10px; padding: 8px 10px 8px 14px; border-top: 1px solid var(--separator); } + .card.answer footer .source { min-width: 0; flex: 1; font: 12px/1.4 var(--mono); color: var(--fg3); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .card .code { position: relative; max-height: 190px; overflow: hidden; } .card.expanded .code { max-height: none; } /* Prompts are prose, so they wrap; code scrolls sideways. */ @@ -215,9 +243,10 @@
LK · LiveKit AssistantMic off
-
Talking to
+
Ready
-
M mute · X x-ray · Shift peek · ⌘C copy last · Esc close
+
/ type · M mute · X x-ray · Shift peek · ⌘C copy last · Esc close
+
X-ray · LK's pipeline
@@ -531,6 +560,24 @@ if (atBottom) streamEl.scrollTop = streamEl.scrollHeight; } + // guestWhat sets the guest bar's text, such as "Talking to my-agent". + function guestWhat(text, name) { + const what = guestEl.querySelector('.what'); + what.textContent = text + ' '; + const n = document.createElement('span'); + n.className = 'name'; + n.textContent = name; + what.append(n); + } + + function setGuestStatus({ status, name }) { + body.classList.toggle('guest-starting', status === 'starting'); + if (status === 'starting') { + guestWhat('Starting', name + '…'); + guestEl.querySelector('.hint').textContent = ''; + } + } + // Who agent turns are from: the assistant, or the user's own agent. const speaker = { name: 'LK', guest: false }; const guestEl = document.getElementById('guest'); @@ -539,7 +586,12 @@ function setSpeaker({ name, guest }) { speaker.name = guest ? name : 'LK'; speaker.guest = !!guest; - guestEl.querySelector('.name').textContent = name; + body.classList.remove('guest-starting'); + if (guest) { + guestWhat('Talking to', name); + // The user's agent waits for them to speak first. + guestEl.querySelector('.hint').textContent = 'Say hello to start'; + } body.classList.toggle('guest', speaker.guest); // X-ray follows whichever agent is talking. xrayEl.querySelector('.head').textContent = `X-ray · ${speaker.name}'s pipeline`; @@ -638,19 +690,20 @@ } function setTranscript({ turn, role, segment = 0, text }) { + if (role === 'user' && speaker.guest && text) guestEl.querySelector('.hint').textContent = ''; const key = `${turn}/${segment}`; let p = segments.get(key); if (!p) { p = document.createElement('p'); p.target = ''; p.shown = 0; - p.node = document.createTextNode(''); + p.node = document.createElement('span'); p.append(p.node); stickToBottom(() => turnBody(turn, role).append(p)); segments.set(key, p); } if (role !== 'agent' || reducedMotion) { - stickToBottom(() => { p.node.data = text; }); + stickToBottom(() => setText(p, text)); return; } // Agent text types out toward the latest transcript, catching up if behind. @@ -658,6 +711,14 @@ if (!revealing.includes(p)) revealing.push(p); } + // setText shows a segment's text, with `backticked` commands and names in + // monospace. While text types out, an unclosed backtick shows as code + // already, so the code doesn't change style when the closing one arrives. + function setText(p, text) { + if (!text.includes('`')) { p.node.textContent = text; return; } + p.node.innerHTML = inlineCode(text); + } + const cursor = document.createElement('span'); cursor.className = 'cursor'; @@ -667,7 +728,7 @@ const behind = p.target.length - p.shown; if (behind > 0) { p.shown = Math.min(p.target.length, p.shown + Math.max(1, Math.round(behind / 12))); - stickToBottom(() => { p.node.data = p.target.slice(0, p.shown); }); + stickToBottom(() => setText(p, p.target.slice(0, p.shown))); } if (cursor.parentNode !== p) p.append(cursor); if (p.shown >= p.target.length && (revealing.length > 1 || state !== 'speaking')) revealing.shift(); @@ -688,7 +749,7 @@ card.querySelector('.more').addEventListener('click', () => card.classList.add('expanded')); card.code = code; // Finish typing earlier text first, so the card lands after it. - for (const p of revealing) { p.shown = p.target.length; p.node.data = p.target; } + for (const p of revealing) { p.shown = p.target.length; setText(p, p.target); } revealing.length = 0; stickToBottom(() => turnBody(turn, 'agent').append(card)); if (card.querySelector('.code').scrollHeight <= 192) card.classList.add('expanded'); @@ -697,6 +758,52 @@ if (copied) markCopied(id); } + // inlineCode escapes text and shows `backticked` parts in monospace. + const inlineCode = (text) => text.split('`').map((part, i) => (i % 2 ? `${esc(part)}` : esc(part))).join(''); + + function addAnswer({ turn, id, title, points, source_title, source_url }) { + const card = document.createElement('section'); + card.className = 'card answer'; + card.dataset.id = id; + card.innerHTML = + '
Answer
' + + '
    '; + card.querySelector('.title').textContent = title; + card.querySelector('ul').innerHTML = points.map((p) => `
  • ${inlineCode(p)}
    • `).join(''); + card.querySelector('.source').textContent = `${source_title} · ${source_url.replace(/^https:\/\//, '')}`; + card.querySelector('.copy').addEventListener('click', () => requestCopy(id)); + card.querySelector('.open').addEventListener('click', () => { + if (host) post({ type: 'open', id }); + else window.open(source_url, '_blank'); + }); + card.code = points.map((p) => `- ${p}`).join('\n') + `\n\nSource: ${source_title} (${source_url})`; + for (const p of revealing) { p.shown = p.target.length; setText(p, p.target); } + revealing.length = 0; + stickToBottom(() => turnBody(turn, 'agent').append(card)); + snippets.set(id, card); + } + + function addLink({ turn, id, title, url }) { + const card = document.createElement('section'); + card.className = 'card link'; + card.dataset.id = id; + card.innerHTML = + '
    Read more' + + '
    '; + card.querySelector('.title').textContent = title; + card.querySelector('.url').textContent = url.replace(/^https:\/\//, ''); + card.querySelector('.copy').addEventListener('click', () => requestCopy(id)); + card.querySelector('.open').addEventListener('click', () => { + if (host) post({ type: 'open', id }); + else window.open(url, '_blank'); + }); + card.code = url; + for (const p of revealing) { p.shown = p.target.length; setText(p, p.target); } + revealing.length = 0; + stickToBottom(() => turnBody(turn, 'agent').append(card)); + snippets.set(id, card); + } + const tools = new Map(); // call id → line function setTool({ turn, role = 'agent', id, label, detail, status }) { @@ -839,6 +946,9 @@ break; case 'transcript': setTranscript(msg); break; case 'snippet': addSnippet(msg); break; + case 'link': addLink(msg); break; + case 'answer': addAnswer(msg); break; + case 'guest_status': setGuestStatus(msg); break; case 'tool': setTool(msg); break; case 'confirm': askConfirm(msg); break; case 'confirm_done': markConfirm(msg.id, msg.option); break; @@ -877,7 +987,35 @@ post({ type: 'peek', on }); } + const composeEl = document.getElementById('compose'); + const composeInput = composeEl.querySelector('input'); + function openCompose() { + if (speaker.guest) { showNotice('Typing works with LK only. Press B to come back'); return; } + body.classList.add('composing'); + composeInput.focus(); + } + function closeCompose() { + body.classList.remove('composing'); + composeInput.value = ''; + composeInput.blur(); + } + composeEl.addEventListener('submit', (e) => { + e.preventDefault(); + const text = composeInput.value.trim(); + if (text) post({ type: 'text', text }); + closeCompose(); + }); + addEventListener('keydown', (e) => { + // While typing, keys go to the input, not to the shortcuts. + if (e.target === composeInput) { + if (e.key === 'Escape') { e.preventDefault(); closeCompose(); } + return; + } + if (e.key === '/' || (e.key === 'Enter' && !e.metaKey && !e.ctrlKey && document.activeElement === document.body)) { + e.preventDefault(); + return openCompose(); + } if (e.key === 'Escape') return dismiss(); if (e.key === 'Shift' && !e.repeat) return setPeek(true); if ((e.metaKey || e.ctrlKey) && e.key === 'c') { From e504864c5f9e8671b790bc1ab93a468b28fbaeb5 Mon Sep 17 00:00:00 2001 From: royalfig Date: Mon, 12 Oct 2026 18:07:33 -0400 Subject: [PATCH 07/12] feat(assistant): focus LK on onboarding - Scan the folder and coding agent setup at startup, and greet for what's there. - Run only agents created this session; give the command for existing ones. - Fail fast when a guest agent's job ends before it connects. - Docs answers on screen with their source, copy by voice, and typed input. - Confirm in one step when the user already agreed. - Open in a window by default; --fullscreen covers the screen. - Pronounce LK, lk, and LiveKit's correctly. - Add onboarding evals, run with real Inference and stubbed lk tools. Co-Authored-By: Claude Opus 5.5 --- cmd/lk/assistant.go | 108 ++++- cmd/lk/assistant_guest.go | 108 ++++- cmd/lk/assistant_log.go | 24 + cmd/lk/assistant_session.go | 148 +++++- cmd/lk/assistant_skills.go | 84 ++++ cmd/lk/assistant_tools.go | 137 ++---- cmd/lk/assistant_xray.go | 2 +- pkg/assistant/agent/agent.py | 343 ++++++++++--- pkg/assistant/agent/evals/test_onboarding.py | 184 +++++++ pkg/assistant/assistant.go | 4 + pkg/assistant/folder.go | 477 +++++++++++++++++++ pkg/assistant/folder_test.go | 138 ++++++ 12 files changed, 1537 insertions(+), 220 deletions(-) create mode 100644 cmd/lk/assistant_skills.go create mode 100644 pkg/assistant/agent/evals/test_onboarding.py create mode 100644 pkg/assistant/folder.go create mode 100644 pkg/assistant/folder_test.go diff --git a/cmd/lk/assistant.go b/cmd/lk/assistant.go index b997b3ed..8faeb6d9 100644 --- a/cmd/lk/assistant.go +++ b/cmd/lk/assistant.go @@ -20,6 +20,7 @@ import ( "math" "math/rand/v2" "os" + "os/exec" "strings" "sync" "time" @@ -33,19 +34,22 @@ import ( var AssistantCommands = []*cli.Command{ { Name: "assistant", - Usage: "Talk to the LiveKit voice assistant (preview)", + Usage: "Get started with LiveKit by talking to LK, the LiveKit assistant (preview)", Hidden: true, - Description: `Opens the assistant overlay over the desktop and starts a voice conversation -with the LiveKit assistant, using your microphone and speakers. The assistant -answers from the LiveKit docs and shows code on screen, with a button to copy -it. - -The assistant is a LiveKit agent that runs on your machine and uses LiveKit -Inference through your LiveKit Cloud project. The first run installs it with -uv. - -Press Esc to close the overlay. With --demo, the overlay plays a scripted -conversation instead; press R to replay it.`, + Description: `Opens a window and starts a voice conversation with LK, the LiveKit +assistant, using your microphone and speakers. LK helps you get started: it +creates a voice agent from a template, lets you talk to it, sets up your coding +agent for LiveKit, and answers questions from the LiveKit docs. It shows code, +commands, and docs links on screen, with buttons to copy them. Press / to type +instead of speaking. + +LK is a LiveKit agent that runs on your machine. It uses LiveKit Inference +through your LiveKit Cloud project, so its usage counts toward your plan. The +first run installs it with uv. + +Press Esc to close. With --fullscreen, LK covers the screen and blurs the +desktop. With --demo, it plays a scripted conversation instead; press R to +replay it.`, Flags: []cli.Flag{ &cli.BoolFlag{ Name: "demo", @@ -57,8 +61,8 @@ conversation instead; press R to replay it.`, Hidden: true, }, &cli.BoolFlag{ - Name: "windowed", - Usage: "Open in a window instead of covering the screen", + Name: "fullscreen", + Usage: "Cover the screen and blur the desktop, instead of opening a window", }, &cli.BoolFlag{ Name: "inspect", @@ -71,11 +75,11 @@ conversation instead; press R to replay it.`, func runAssistant(ctx context.Context, cmd *cli.Command) error { ov := overlay.New(overlay.Options{ - Windowed: cmd.Bool("windowed"), + Windowed: !cmd.Bool("fullscreen"), Inspectable: cmd.Bool("inspect"), }) if cmd.Bool("demo") { - demo := &assistantDemo{ov: ov, snippets: map[string]string{}} + demo := &assistantDemo{ov: ov, snippets: map[string]string{}, links: map[string]string{}} go demo.run(ctx) return ov.Run(ctx) } @@ -134,6 +138,24 @@ type ( Code string `json:"code"` Copied bool `json:"copied"` } + // overlayAnswer is a docs answer as short points, with its source page. + overlayAnswer struct { + Type string `json:"type"` + Turn string `json:"turn"` + ID string `json:"id"` + Title string `json:"title"` + Points []string `json:"points"` + SourceTitle string `json:"source_title"` + SourceURL string `json:"source_url"` + } + // overlayLink is a docs page to read more, with a button that opens it. + overlayLink struct { + Type string `json:"type"` + Turn string `json:"turn"` + ID string `json:"id"` + Title string `json:"title"` + URL string `json:"url"` + } overlayCopied struct { Type string `json:"type"` ID string `json:"id"` @@ -159,8 +181,20 @@ type demoSnippet struct { id, title, lang, code string } +// demoLink is a docs link card in an agent reply. +type demoLink struct { + id, title, url string +} + +// demoAnswer is a docs answer card in an agent reply. +type demoAnswer struct { + id, title string + points []string + sourceTitle, source string +} + // demoTurn is one exchange. Each agent part is a string (a paragraph of -// speech) or a demoSnippet. +// speech), a demoSnippet, a demoLink, or a demoAnswer. type demoTurn struct { user string agent []any @@ -174,13 +208,13 @@ var demoScript = []demoTurn{ agent: []any{ "Start from the Python starter template. This command creates the project and adds your LiveKit Cloud credentials to it.", demoSnippet{id: "init", title: "Terminal", lang: "shell", code: "lk agent init my-agent --template agent-starter-python\ncd my-agent"}, - "Then run lk agent dev and talk to it from the Agent Console.", + "Then run `lk agent dev` and talk to it from the Agent Console.", }, }, { user: "Where do I change the voice?", agent: []any{ - "In agent.py. The agent session sets the speech-to-text, language, and text-to-speech models. Change the TTS model or voice there.", + "In `agent.py`. The `AgentSession` sets the speech-to-text, language, and text-to-speech models. Change the TTS model or voice there.", demoSnippet{id: "agent", title: "agent.py", lang: "python", code: `@server.rtc_session(agent_name="my-agent") async def my_agent(ctx: agents.JobContext): session = AgentSession( @@ -202,8 +236,18 @@ async def my_agent(ctx: agents.JobContext): { user: "And how do I deploy it?", agent: []any{ - "Run this from the project directory. It registers the agent and deploys it to LiveKit Cloud.", + "Run `lk agent create` from the project directory. It registers the agent and deploys it to LiveKit Cloud.", demoSnippet{id: "deploy", title: "Terminal", lang: "shell", code: "lk agent create"}, + demoAnswer{ + id: "deploy-docs", title: "Deploying to LiveKit Cloud", + points: []string{ + "Deploy with a single LiveKit CLI command.", + "LiveKit Cloud scales your agent automatically, up to the limits of your plan.", + "Store API keys and other secrets in LiveKit Cloud. It injects them into your agent at runtime.", + "Runtime, build, and session logs help you monitor and debug the deployed agent.", + }, + sourceTitle: "Agent deployment overview", source: "https://docs.livekit.io/deploy/agents/", + }, }, }, } @@ -216,6 +260,7 @@ type assistantDemo struct { ov *overlay.Overlay voice fakeVoice snippets map[string]string + links map[string]string // card id → docs URL, for Open } func (d *assistantDemo) run(ctx context.Context) { @@ -241,6 +286,10 @@ func (d *assistantDemo) run(ctx context.Context) { } case "copy": d.copy(ev.ID) + case "open": + if url, ok := d.links[ev.ID]; ok && strings.HasPrefix(url, docsURL) { + _ = exec.Command("open", url).Start() + } } case <-d.ov.Done(): return @@ -304,6 +353,25 @@ func (d *assistantDemo) play(ctx context.Context) bool { if !d.wait(ctx, 600*time.Millisecond, nil) { return false } + case demoAnswer: + var text strings.Builder + for _, pt := range p.points { + text.WriteString("- " + pt + "\n") + } + fmt.Fprintf(&text, "\nSource: %s (%s)", p.sourceTitle, p.source) + d.snippets[p.id] = text.String() + d.links[p.id] = p.source + d.ov.Send(overlayAnswer{Type: "answer", Turn: agentTurn, ID: p.id, Title: p.title, Points: p.points, SourceTitle: p.sourceTitle, SourceURL: p.source}) + if !d.wait(ctx, 600*time.Millisecond, nil) { + return false + } + case demoLink: + d.snippets[p.id] = p.url + d.links[p.id] = p.url + d.ov.Send(overlayLink{Type: "link", Turn: agentTurn, ID: p.id, Title: p.title, URL: p.url}) + if !d.wait(ctx, 600*time.Millisecond, nil) { + return false + } } } diff --git a/cmd/lk/assistant_guest.go b/cmd/lk/assistant_guest.go index c03a3fe9..b741b5f3 100644 --- a/cmd/lk/assistant_guest.go +++ b/cmd/lk/assistant_guest.go @@ -20,8 +20,11 @@ import ( "fmt" "net" "path/filepath" + "slices" + "strings" "github.com/livekit/livekit-cli/v2/pkg/agentfs" + "github.com/livekit/livekit-cli/v2/pkg/assistant" "github.com/livekit/livekit-cli/v2/pkg/console" ) @@ -38,23 +41,31 @@ type guestAgent struct { conn net.Conn } +// overlayGuestStatus shows that the user's agent is starting, or that it +// failed to start. When it connects, an overlaySpeaker switches to it. +type overlayGuestStatus struct { + Type string `json:"type"` + Status string `json:"status"` // starting or failed + Name string `json:"name"` +} + type overlaySpeaker struct { Type string `json:"type"` Name string `json:"name"` // who the agent turns are from Guest bool `json:"guest"` // the user's own agent, not the assistant } -// proposeTryProject proposes running the user's agent in the overlay. path -// is relative to the project directory; empty means the project created this -// session, or the current directory. -func (s *assistantSession) proposeTryProject(requestID, path string) { +// proposeTryProject proposes running the user's agent in the overlay. It's +// only for the project created this session: a starter agent always opens a +// voice session in console mode, but an existing agent may need its frontend, +// dispatch, or other services first. For an existing agent, the assistant +// shows the command to run it in a terminal. +func (s *assistantSession) proposeTryProject(requestID string, confirmed bool) { t := &s.tools - dir := t.dir - switch { - case path != "": - dir = filepath.Join(t.dir, path) - case t.lastProject != "": - dir = t.lastProject + dir := t.lastProject + if dir == "" { + s.replyTool(requestID, existingAgentAdvice(t.dir), nil) + return } if s.guest != nil { s.replyTool(requestID, "", errors.New("the user's agent is already running; they can press B to come back first")) @@ -88,26 +99,55 @@ func (s *assistantSession) proposeTryProject(requestID, path string) { Detail: fmt.Sprintf("Runs your agent, %s, on this machine and switches the conversation to it. Press B to come back to LK.", name), Options: []overlayConfirmOption{{ID: confirmRun, Name: "Talk to it"}, {ID: confirmCancel, Name: "Cancel"}}, }) + if confirmed { + s.confirmByVoice(requestID, id, true) + return + } s.replyTool(requestID, fmt.Sprintf( "Waiting for the user to confirm. Confirmation ID: %s. This runs their agent, %s, here so they can talk to it. "+ - "Ask the user, in one short sentence, whether to start it. They can answer by voice or on screen. "+ + "Ask the user in one short sentence whether to start it. They can answer by voice or on screen. "+ "If they agree by voice, call confirm_action with this ID and approved=true.", id, name), nil) } -// startGuest starts the user's agent and switches to it as soon as it -// connects. It greets right away, so there's no waiting: the assistant says -// its hand-off sentence while the agent starts up, which takes a few seconds. +// existingAgentAdvice tells the assistant how the user can run an agent that +// wasn't created this session. +func existingAgentAdvice(dir string) string { + command := "lk agent console" + if paths := assistant.DescribeFolder(dir).AgentPaths(); len(paths) == 1 && paths[0] != "." { + command = "cd " + paths[0] + " && lk agent console" + } + return "You can only run an agent here that you created in this session. Existing agents can need their " + + "frontend, dispatch, or other services, so they run in the user's own terminal. In one short sentence, " + + "tell the user you can't start it here, and that they can run it in their terminal to talk to it. Then " + + `call show_code with title "Terminal", language "shell", and this command: ` + command + + ". If you don't know which folder their agent is in, ask them first, and tell them to run the command in that folder." +} + +// startGuest starts the user's agent and switches to it when it connects. +// The assistant tells the user it's starting, which takes a few seconds. The +// overlay shows the switch. func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoint string, projectType agentfs.ProjectType) { t := &s.tools - handOff := fmt.Sprintf("Starting the user's agent, %s. It takes a few seconds. In one or two short sentences, tell them you're handing them over to it, that they should say hello to start (starter agents wait for the user to speak first), and that they can press B to come back to you.", name) + starting := fmt.Sprintf("Starting the user's agent, %s. It takes a few seconds, and the screen shows when it's connected. "+ + "In one short sentence, tell them it's starting, and that once it's connected they should say hello, because starter agents wait for the user to speak first. "+ + "Don't say you've handed them over: you'll hear if it fails to start. "+ + "If the user talks to you before it connects, tell them in a few words that it's still starting.", name) + s.out.Send(overlayGuestStatus{Type: "guest_status", Status: "starting", Name: name}) + s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: confirmID, Status: "running", Text: "Starting " + name + "…"}) if requestID != "" { - s.replyTool(requestID, handOff, nil) + s.replyTool(requestID, starting, nil) } else { - s.tellUser(handOff) + s.tellUser(starting) } s.inBackground(func(ctx context.Context) toolResult { failed := func(err error) toolResult { - return toolResult{confirmID: confirmID, command: "lk agent console", err: err} + return toolResult{then: func(toolResult) { + s.out.Send(overlayGuestStatus{Type: "guest_status", Status: "failed", Name: name}) + s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: confirmID, Status: "failed", Text: "Your agent didn't start. See the terminal"}) + out.Statusf("Your agent didn't start: %v", err) + s.tellUser(fmt.Sprintf("The user's agent, %s, didn't start, so they're still talking to you. The error:\n%s\n"+ + "In one or two short sentences, tell them what went wrong in plain words, and that the full log is in the terminal.", name, truncate(err.Error(), 1500))) + }} } // The guest needs its own console server: the assistant's accepts // only one agent, and closes once the assistant connects. @@ -121,13 +161,13 @@ func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoin ProjectType: projectType, CLIArgs: buildConsoleArgs(server.Addr().String(), false), Env: t.creds, - FailSignals: consoleCrashSignals, + FailSignals: guestEndSignals, }) if err != nil { server.Close() return failed(fmt.Errorf("couldn't start the agent: %w", err)) } - conn, err := acceptAgent(ctx, server, proc) + conn, err := acceptAgent(ctx, server, proc, "your agent") if err != nil { proc.Kill() server.Close() @@ -140,13 +180,15 @@ func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoin s.startSpeakerTurns(overlaySpeaker{Type: "speaker", Name: name, Guest: true}) s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: confirmID, Status: "done"}) go func() { - // The worker keeps running when its job crashes, so watch - // for the crash as well as for the exit. + // The worker keeps running when its job ends, so watch for + // the job's end as well as for the exit. var err error select { case err = <-proc.Done(): case <-proc.Failed(): - err = errors.New("its job crashed") + if jobCrashed(proc) { + err = errors.New("its job crashed") + } } select { case t.results <- toolResult{then: func(toolResult) { s.guestExited(proc, err) }}: @@ -157,6 +199,24 @@ func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoin }) } +// guestEndSignals are output markers meaning the guest's job ended, though +// its worker may stay alive. An agent whose job ends before it connects, such +// as one that needs data from its frontend, fails right away instead of +// waiting out the connect timeout. "job exiting" is a Python SDK debug log. +var guestEndSignals = append(slices.Clone(consoleCrashSignals), "job exiting") + +// jobCrashed reports whether the guest's job ended because it crashed. +func jobCrashed(proc *AgentProcess) bool { + for _, line := range proc.RecentLogs(50) { + for _, sig := range consoleCrashSignals { + if strings.Contains(line, sig) { + return true + } + } + } + return false +} + // returnToAssistant switches back to the assistant and stops the guest. func (s *assistantSession) returnToAssistant(notice string) { g := s.guest @@ -178,7 +238,7 @@ func (s *assistantSession) guestExited(proc *AgentProcess, err error) { if s.guest == nil || s.guest.proc != proc { return } - notice := "Your agent stopped" + notice := "Your agent's session ended" if err != nil { notice = "Your agent stopped with an error. See the terminal" out.Statusf("Your agent stopped: %v\n%s", err, agentExitDetail(proc)) diff --git a/cmd/lk/assistant_log.go b/cmd/lk/assistant_log.go index 08160a80..5c8e4d01 100644 --- a/cmd/lk/assistant_log.go +++ b/cmd/lk/assistant_log.go @@ -114,6 +114,12 @@ func (l *sessionLog) record(msg any) { case overlaySnippet: t := l.turn(m.Turn, "agent") t.items = append(t.items, &logItem{kind: "code", label: m.Title, lang: m.Lang, text: m.Code}) + case overlayAnswer: + t := l.turn(m.Turn, "agent") + t.items = append(t.items, &logItem{kind: "answer", label: m.Title, text: strings.Join(m.Points, "\n- "), detail: m.SourceTitle, lang: m.SourceURL}) + case overlayLink: + t := l.turn(m.Turn, "agent") + t.items = append(t.items, &logItem{kind: "link", label: m.Title, text: m.URL}) case overlayConfirm: item := &logItem{kind: "command", text: m.Command, detail: m.Heading, status: "not answered"} l.items[m.ID] = item @@ -129,6 +135,17 @@ func (l *sessionLog) record(msg any) { } } +// empty reports whether t has nothing to show, such as a turn the user's +// agent started but didn't speak in. +func (t *logTurn) empty() bool { + for _, it := range t.items { + if it.kind != "text" || it.text != "" { + return false + } + } + return true +} + // markdown renders the conversation, or "" if the user never spoke. func (l *sessionLog) markdown(dir string) string { l.mu.Lock() @@ -147,6 +164,9 @@ func (l *sessionLog) markdown(dir string) string { fmt.Fprintf(&b, "# LiveKit assistant, %s\n\nIn %s.\n", l.started.Format("January 2, 2006, 3:04 PM"), tildePath(dir)) for _, id := range l.order { t := l.turns[id] + if t.empty() { + continue + } who := t.speaker if t.role == "user" { who = "You" @@ -181,6 +201,10 @@ func (l *sessionLog) markdown(dir string) string { } fence := codeFence(it.text) fmt.Fprintf(&b, "\n\n%s:\n\n%s%s\n%s\n%s", it.label, fence, lang, it.text, fence) + case "answer": + fmt.Fprintf(&b, "\n\n%s:\n\n- %s\n\nSource: [%s](%s)", it.label, it.text, it.detail, it.lang) + case "link": + fmt.Fprintf(&b, "\n\nRead more: [%s](%s)", it.label, it.text) case "command": fmt.Fprintf(&b, "\n\n`%s` (%s): %s", it.text, strings.ToLower(it.detail), it.status) } diff --git a/cmd/lk/assistant_session.go b/cmd/lk/assistant_session.go index 59528ea7..4450c516 100644 --- a/cmd/lk/assistant_session.go +++ b/cmd/lk/assistant_session.go @@ -18,10 +18,12 @@ import ( "bufio" "context" "encoding/json" + "errors" "fmt" "math" "net" "os" + "os/exec" "regexp" "strings" "sync" @@ -87,7 +89,9 @@ type assistantSession struct { spokeGoodbye bool // the agent spoke after end_call agentExited bool dismissed bool - snippets map[string]string + snippets map[string]string // card id → code, for Copy + snippetOrder []snippetRef // cards in the order shown, for copying by voice + links map[string]string // link card id → URL, for Open // The local connection to the assistant agent, and the lk commands it // runs through it. @@ -168,6 +172,7 @@ func newAssistantSession(ov *overlay.Overlay, cmd *cli.Command) *assistantSessio shownCalls: map[string]bool{}, toolTurns: map[string]string{}, snippets: map[string]string{}, + links: map[string]string{}, } } @@ -197,6 +202,11 @@ func (s *assistantSession) run(ctx context.Context) (err error) { if err != nil { return errNoCredentials{err} } + project := "your LiveKit Cloud project" + if rp, err := resolveProject(s.cmd, loadParams{requireURL: true}); err == nil && rp.project != nil && rp.project.Name != "" { + project = "your LiveKit Cloud project, " + rp.project.Name + } + out.Statusf("LK uses LiveKit Inference through %s. Its usage counts toward your plan.", project) dir := s.cmd.String("agent-dir") projectType := agentfs.ProjectTypePythonUV @@ -265,6 +275,8 @@ func (s *assistantSession) run(ctx context.Context) (err error) { Env: append(creds, "LK_ASSISTANT_ADDR="+channelLn.Addr().String(), "LK_ASSISTANT_PROJECT="+projectDir, + "LK_ASSISTANT_FOLDER="+assistant.DescribeFolder(projectDir).JSON(), + "LK_ASSISTANT_CODING_AGENTS="+checkCodingAgentSetup(projectDir).JSON(), ), FailSignals: consoleCrashSignals, }) @@ -273,7 +285,7 @@ func (s *assistantSession) run(ctx context.Context) (err error) { } defer agentProc.Kill() - conn, err := acceptAgent(ctx, server, agentProc) + conn, err := acceptAgent(ctx, server, agentProc, "the assistant agent") if err != nil { return err } @@ -367,6 +379,10 @@ func (s *assistantSession) run(ctx context.Context) (err error) { s.handleCopy(ev) case "confirm": s.confirmTool(ev.ID, ev.Option) + case "open": + s.openLink(ev.ID) + case "text": + s.typed(ev.Text) case "back": s.returnToAssistant("") case "mute": @@ -420,7 +436,8 @@ func (s *assistantSession) serveCopies(ctx context.Context) { } // acceptAgent waits for the agent subprocess to connect to the console server. -func acceptAgent(ctx context.Context, server *console.TCPServer, agentProc *AgentProcess) (net.Conn, error) { +// who names the agent in errors, such as "the assistant agent". +func acceptAgent(ctx context.Context, server *console.TCPServer, agentProc *AgentProcess, who string) (net.Conn, error) { type result struct { conn net.Conn err error @@ -439,15 +456,15 @@ func acceptAgent(ctx context.Context, server *console.TCPServer, agentProc *Agen return res.conn, nil case err := <-agentProc.Done(): if err != nil { - return nil, fmt.Errorf("the assistant agent exited before connecting: %w\n\n%s", err, agentExitDetail(agentProc)) + return nil, fmt.Errorf("%s exited before connecting: %w\n\n%s", who, err, agentExitDetail(agentProc)) } - return nil, fmt.Errorf("the assistant agent exited before connecting.\n\n%s", agentExitDetail(agentProc)) + return nil, fmt.Errorf("%s exited before connecting.\n\n%s", who, agentExitDetail(agentProc)) case <-agentProc.Failed(): // The crash marker arrives mid-traceback; give trailing output a moment. time.Sleep(500 * time.Millisecond) - return nil, fmt.Errorf("the assistant agent crashed before connecting.\n\n%s", agentExitDetail(agentProc)) + return nil, fmt.Errorf("%s stopped before connecting.\n\n%s", who, agentExitDetail(agentProc)) case <-time.After(90 * time.Second): - return nil, fmt.Errorf("timed out waiting for the assistant agent to connect.\n\n%s", agentExitDetail(agentProc)) + return nil, fmt.Errorf("timed out waiting for %s to connect.\n\n%s", who, agentExitDetail(agentProc)) case <-ctx.Done(): return nil, ctx.Err() } @@ -732,6 +749,10 @@ func (s *assistantSession) toolCall(fc *agent.FunctionCall) { s.showCode(fc) case assistant.HandOffTaskTool: s.handOffTask(fc) + case assistant.ShowLinkTool: + s.showLink(fc) + case assistant.ShowAnswerTool: + s.showAnswer(fc) case assistant.EndCallTool: // Only the assistant's end_call closes the overlay; the user's own // agent may have one too. @@ -801,6 +822,8 @@ func describeTool(name, arguments string) (label, detail string) { return "Listed your agents", detail case "list_templates": return "Listed the templates", detail + case "copy_to_clipboard": + return "Copied to your clipboard", detail case "create_project": return "Create a project", detail case "confirm_action": @@ -864,7 +887,7 @@ func (s *assistantSession) showCode(fc *agent.FunctionCall) { s.agentTurnForReply() id := fc.GetCallId() - s.snippets[id] = args.Code + s.addSnippet(id, args.Title, args.Code) s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, args.Title, normalizeLang(args.Language), args.Code, false}) } @@ -884,10 +907,18 @@ func (s *assistantSession) handOffTask(fc *agent.FunctionCall) { s.agentTurnForReply() id := fc.GetCallId() - s.snippets[id] = args.Task + s.addSnippet(id, "Prompt for your coding agent", args.Task) s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, "Prompt for your coding agent", "prompt", args.Task, false}) } +type snippetRef struct{ id, title string } + +// addSnippet records a code or prompt card so it can be copied. +func (s *assistantSession) addSnippet(id, title, code string) { + s.snippets[id] = code + s.snippetOrder = append(s.snippetOrder, snippetRef{id, title}) +} + func (s *assistantSession) copy(id string) { code, ok := s.snippets[id] if !ok || clipboard.WriteAll(code) != nil { @@ -896,6 +927,105 @@ func (s *assistantSession) copy(id string) { s.out.Send(overlayCopied{"copied", id}) } +// docsURL is where links to read more point. The agent gets these URLs from +// its docs searches, and its show_link tool rejects any other URL. +const docsURL = "https://docs.livekit.io/" + +// showLink shows a show_link call as a card that links to a docs page. +func (s *assistantSession) showLink(fc *agent.FunctionCall) { + if s.shownCalls[fc.GetCallId()] { + return + } + var args struct { + Title string `json:"title"` + URL string `json:"url"` + } + if err := json.Unmarshal([]byte(fc.GetArguments()), &args); err != nil || !strings.HasPrefix(args.URL, docsURL) { + return + } + s.shownCalls[fc.GetCallId()] = true + if args.Title == "" { + args.Title = "LiveKit docs" + } + s.agentTurnForReply() + id := fc.GetCallId() + s.links[id] = args.URL + s.addSnippet(id, args.Title, args.URL) + s.out.Send(overlayLink{Type: "link", Turn: s.agentTurn, ID: id, Title: args.Title, URL: args.URL}) +} + +// typed sends text the user typed to the assistant, as their next turn. +// The user's own agent only takes speech. +func (s *assistantSession) typed(text string) { + if text = strings.TrimSpace(text); text == "" || s.guestActive() { + return + } + s.channel.send(map[string]string{"type": "user_text", "text": text}) +} + +// showAnswer shows a show_answer call as a card of short points, with a link +// to the docs page they come from. Copy copies the points and the link. +func (s *assistantSession) showAnswer(fc *agent.FunctionCall) { + if s.shownCalls[fc.GetCallId()] { + return + } + var args struct { + Title string `json:"title"` + Points []string `json:"points"` + SourceTitle string `json:"source_title"` + SourceURL string `json:"source_url"` + } + if err := json.Unmarshal([]byte(fc.GetArguments()), &args); err != nil || len(args.Points) == 0 || !strings.HasPrefix(args.SourceURL, docsURL) { + return + } + s.shownCalls[fc.GetCallId()] = true + s.agentTurnForReply() + id := fc.GetCallId() + s.links[id] = args.SourceURL + var text strings.Builder + for _, p := range args.Points { + text.WriteString("- " + p + "\n") + } + fmt.Fprintf(&text, "\nSource: %s (%s)", args.SourceTitle, args.SourceURL) + s.addSnippet(id, args.Title, text.String()) + s.out.Send(overlayAnswer{Type: "answer", Turn: s.agentTurn, ID: id, Title: args.Title, Points: args.Points, SourceTitle: args.SourceTitle, SourceURL: args.SourceURL}) +} + +// openLink opens a link card's page in the user's browser. +func (s *assistantSession) openLink(id string) { + if url, ok := s.links[id]; ok { + _ = exec.Command("open", url).Start() + } +} + +// copyByVoice copies the latest card whose title contains title, or the +// latest card if title is empty, and replies with the title it copied. +func (s *assistantSession) copyByVoice(requestID, title string) { + want := strings.ToLower(strings.TrimSpace(title)) + for i := len(s.snippetOrder) - 1; i >= 0; i-- { + ref := s.snippetOrder[i] + if want != "" && !strings.Contains(strings.ToLower(ref.title), want) { + continue + } + if err := clipboard.WriteAll(s.snippets[ref.id]); err != nil { + s.replyTool(requestID, "", fmt.Errorf("couldn't copy to the clipboard: %w", err)) + return + } + s.out.Send(overlayCopied{"copied", ref.id}) + s.replyTool(requestID, fmt.Sprintf("Copied %q to the user's clipboard.", ref.title), nil) + return + } + if len(s.snippetOrder) == 0 { + s.replyTool(requestID, "", errors.New("there's nothing on screen to copy yet; show it with show_code or hand_off_task first")) + return + } + var titles []string + for _, ref := range s.snippetOrder { + titles = append(titles, ref.title) + } + s.replyTool(requestID, "", fmt.Errorf("no card on screen has %q in its title; the cards are: %s", title, strings.Join(titles, ", "))) +} + // normalizeLang maps language names to the ones the page highlights. func normalizeLang(lang string) string { switch l := strings.ToLower(strings.TrimSpace(lang)); l { diff --git a/cmd/lk/assistant_skills.go b/cmd/lk/assistant_skills.go new file mode 100644 index 00000000..9c0500f5 --- /dev/null +++ b/cmd/lk/assistant_skills.go @@ -0,0 +1,84 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "os" + "slices" + + "github.com/livekit/livekit-cli/v2/pkg/skills" +) + +// codingAgentSetup is what LiveKit has already set up for the user's coding +// agents, so the assistant doesn't offer to install it again. It reads only +// local files, so it doesn't check whether the skills are current. +type codingAgentSetup struct { + Detected []string `json:"coding_agents_detected"` + Project skillsScope `json:"project"` // in the user's current directory + Global skillsScope `json:"global"` // for all their projects + // Ready is true when LiveKit skills and the Docs MCP server are both set + // up for at least one coding agent, in either scope. + Ready bool `json:"ready"` +} + +type skillsScope struct { + Skills []string `json:"livekit_skills"` + MCP []string `json:"docs_mcp_configured_for"` // coding agent IDs +} + +func checkCodingAgentSetup(dir string) codingAgentSetup { + home, _ := os.UserHomeDir() + env := &skills.Env{Home: home, Root: dir} + var setup codingAgentSetup + setup.Detected = agentIDs(skills.DetectAgents(env)) + if setup.Detected == nil { + setup.Detected = []string{} + } + setup.Project = checkSkillsScope(env, skills.ScopeProject) + setup.Global = checkSkillsScope(env, skills.ScopeGlobal) + hasSkills := len(setup.Project.Skills) > 0 || len(setup.Global.Skills) > 0 + hasMCP := len(setup.Project.MCP) > 0 || len(setup.Global.MCP) > 0 + setup.Ready = hasSkills && hasMCP + return setup +} + +func checkSkillsScope(env *skills.Env, scope skills.Scope) skillsScope { + st := skillsScope{Skills: []string{}, MCP: []string{}} + if in, err := skills.NewInstaller(env, scope, nil); err == nil { + dirs := skills.GroupSkillsDirs(env, scope, skills.Agents) + names, _ := in.InstalledLiveKitSkills(dirs) + // The lock can list skills whose files are gone; count only copies + // that are there. + copies, _ := in.Inspect(names, dirs) + for _, c := range copies { + if c.State != skills.StateMissing && !slices.Contains(st.Skills, c.Skill) { + st.Skills = append(st.Skills, c.Skill) + } + } + slices.Sort(st.Skills) + } + for _, a := range skills.Agents { + if skills.CheckMCP(env, scope, a).State == skills.MCPConfigured { + st.MCP = append(st.MCP, a.ID) + } + } + return st +} + +func (c codingAgentSetup) JSON() string { + b, _ := json.Marshal(c) + return string(b) +} diff --git a/cmd/lk/assistant_tools.go b/cmd/lk/assistant_tools.go index f54c04de..56a6f7fc 100644 --- a/cmd/lk/assistant_tools.go +++ b/cmd/lk/assistant_tools.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/livekit/livekit-cli/v2/pkg/agentfs" + "github.com/livekit/livekit-cli/v2/pkg/assistant" "github.com/livekit/livekit-cli/v2/pkg/bootstrap" "github.com/livekit/livekit-cli/v2/pkg/config" "github.com/livekit/livekit-cli/v2/pkg/util" @@ -137,22 +138,31 @@ func (s *assistantSession) runTool(requestID, name string, args map[string]any) }) case "create_project": - s.proposeCreateProject(requestID, str("template"), str("name"), flag("in_current_directory")) + s.proposeCreateProject(requestID, str("template"), str("name"), flag("in_current_directory"), flag("confirmed")) case "set_up_coding_agent": + if setup := checkCodingAgentSetup(s.tools.dir); setup.Ready && !flag("reinstall") { + s.replyTool(requestID, "Nothing was installed: LiveKit's skills and the Docs MCP server are already set up. "+ + "Tell the user, and that \"lk skills update\" gets the latest skills. If they still want to reinstall, "+ + "call set_up_coding_agent again with reinstall=true. Current setup: "+setup.JSON(), nil) + return + } detail := "Adds LiveKit's skills and the Docs MCP server to the coding agents it finds, for this folder only." cmdArgs := []string{"skills", "install", "--json", "-y"} if flag("global") { detail = "Adds LiveKit's skills and the Docs MCP server to the coding agents it finds, for all your projects." cmdArgs = append(cmdArgs, "--global") } - s.proposeCommand(requestID, detail, "Install", cmdArgs) + s.proposeCommand(requestID, detail, "Install", cmdArgs, "", flag("confirmed")) case "confirm_action": s.confirmByVoice(requestID, str("confirmation_id"), flag("approved")) + case "copy_to_clipboard": + s.copyByVoice(requestID, str("title")) + case "try_project": - s.proposeTryProject(requestID, str("path")) + s.proposeTryProject(requestID, flag("confirmed")) default: s.replyTool(requestID, "", fmt.Errorf("unknown tool %q", name)) @@ -162,7 +172,7 @@ func (s *assistantSession) runTool(requestID, name string, args map[string]any) // proposeCreateProject proposes creating a project from a template: agent // templates with lk agent init, others (frontends and apps) with lk app // create. -func (s *assistantSession) proposeCreateProject(requestID, template, name string, here bool) { +func (s *assistantSession) proposeCreateProject(requestID, template, name string, here, confirmed bool) { dir := s.tools.dir if here { if empty, err := util.IsEmptyDir(dir); err != nil || !empty { @@ -196,14 +206,12 @@ func (s *assistantSession) proposeCreateProject(requestID, template, name string kind, cmdArgs = "an agent project", []string{"agent", "init"} } cmdArgs = append(cmdArgs, target, "--template", tpl.Name, "--install", "-y") - id := s.proposeCommand(requestID, - fmt.Sprintf("Creates %s from the %s template %s.", kind, tpl.Name, where), - "Create", cmdArgs) - if isAgentTemplate(tpl) { - p := s.tools.pending[id] - p.project = project - s.tools.pending[id] = p + if !isAgentTemplate(tpl) { + project = "" } + s.proposeCommand(requestID, + fmt.Sprintf("Creates %s from the %s template %s.", kind, tpl.Name, where), + "Create", cmdArgs, project, confirmed) }} }) } @@ -235,12 +243,12 @@ func describeTemplates(templates []bootstrap.Template) string { // proposeCommand shows a confirmation card for a command and replies to the // agent right away with the confirmation ID, which it returns. -func (s *assistantSession) proposeCommand(requestID, detail, runLabel string, args []string) string { +func (s *assistantSession) proposeCommand(requestID, detail, runLabel string, args []string, project string, confirmed bool) { t := &s.tools t.confirmN++ id := fmt.Sprintf("confirm%d", t.confirmN) command := displayCommand(args) - t.pending[id] = pendingTool{args: args, command: command} + t.pending[id] = pendingTool{args: args, command: command, project: project} s.agentTurnForReply() s.out.Send(overlayConfirm{ Type: "confirm", Turn: s.agentTurn, ID: id, @@ -249,13 +257,17 @@ func (s *assistantSession) proposeCommand(requestID, detail, runLabel string, ar Detail: detail, Options: []overlayConfirmOption{{ID: confirmRun, Name: runLabel}, {ID: confirmCancel, Name: "Cancel"}}, }) + if confirmed { + // The user already agreed by voice: run it, and reply when it's done. + s.confirmByVoice(requestID, id, true) + return + } s.replyTool(requestID, fmt.Sprintf( "Waiting for the user to confirm. Confirmation ID: %s. Command: %s. %s "+ - "Ask the user, in one short sentence, whether to go ahead. They can answer by voice or "+ + "Ask the user in one short sentence whether to go ahead. They can answer by voice or "+ "on screen. If they agree by voice, call confirm_action with this ID and approved=true; "+ "if they decline, approved=false. If they answer on screen, you'll be told the result.", id, command, detail), nil) - return id } // confirmByVoice handles confirm_action: the user answered by voice. @@ -433,16 +445,16 @@ func runLK(ctx context.Context, dir string, creds, args []string) (string, error // projectInfo describes the user's LiveKit Cloud project and their current // directory, as JSON for the LLM. func (s *assistantSession) projectInfo() string { - empty, _ := util.IsEmptyDir(s.tools.dir) - info := map[string]any{"directory": s.tools.dir, "directory_is_empty": empty} + info := map[string]any{ + "directory": s.tools.dir, + "folder": assistant.DescribeFolder(s.tools.dir), + "coding_agents": checkCodingAgentSetup(s.tools.dir), + } if s.cmd != nil { if rp, err := resolveProject(s.cmd, loadParams{requireURL: true}); err == nil && rp.project != nil { info["livekit_project"] = map[string]string{"name": rp.project.Name, "url": rp.project.URL} } } - if app := detectApp(s.tools.dir); app != nil { - info["app"] = app - } if root, projectType, err := agentfs.DetectProjectRoot(s.tools.dir); err == nil { local := map[string]any{"root": root, "type": string(projectType)} if toml, ok, err := config.LoadTOMLFile(root, config.LiveKitTOMLFile); err == nil && ok && toml.HasAgent() { @@ -455,88 +467,3 @@ func (s *assistantSession) projectInfo() string { b, _ := json.MarshalIndent(info, "", " ") return string(b) } - -// appFrameworks maps package.json dependencies to the frameworks they mean, -// most specific first. -var appFrameworks = []struct{ dep, name string }{ - {"next", "Next.js"}, - {"expo", "Expo (React Native)"}, - {"react-native", "React Native"}, - {"@angular/core", "Angular"}, - {"svelte", "Svelte"}, - {"vue", "Vue"}, - {"react", "React"}, -} - -// pythonFrameworks are Python web frameworks found in project files. -var pythonFrameworks = []struct{ dep, name string }{ - {"fastapi", "FastAPI"}, - {"django", "Django"}, - {"flask", "Flask"}, -} - -// detectApp describes an existing app in dir: its framework and any LiveKit -// packages it already uses. It returns nil if dir isn't an app it recognizes. -func detectApp(dir string) map[string]any { - app := map[string]any{} - var livekit []string - - if b, err := os.ReadFile(filepath.Join(dir, "package.json")); err == nil { - var pkg struct { - Dependencies map[string]string `json:"dependencies"` - DevDependencies map[string]string `json:"devDependencies"` - } - if json.Unmarshal(b, &pkg) == nil { - deps := map[string]bool{} - for d := range pkg.Dependencies { - deps[d] = true - } - for d := range pkg.DevDependencies { - deps[d] = true - } - for _, f := range appFrameworks { - if deps[f.dep] { - app["framework"] = f.name - break - } - } - for d := range deps { - if strings.HasPrefix(d, "@livekit/") || d == "livekit-client" || d == "livekit-server-sdk" { - livekit = append(livekit, d) - } - } - app["language"] = "JavaScript or TypeScript" - } - } - - var py strings.Builder - for _, f := range []string{"pyproject.toml", "requirements.txt"} { - if b, err := os.ReadFile(filepath.Join(dir, f)); err == nil { - py.Write(b) - } - } - if pyText := strings.ToLower(py.String()); pyText != "" { - if _, ok := app["language"]; !ok { - app["language"] = "Python" - } - for _, f := range pythonFrameworks { - if strings.Contains(pyText, f.dep) { - app["backend"] = f.name - break - } - } - for _, d := range []string{"livekit-agents", "livekit-api", "livekit"} { - if strings.Contains(pyText, d) { - livekit = append(livekit, d) - break - } - } - } - - if len(app) == 0 { - return nil - } - slices.Sort(livekit) - app["livekit_packages"] = livekit - return app -} diff --git a/cmd/lk/assistant_xray.go b/cmd/lk/assistant_xray.go index 1cef7d55..bf27d689 100644 --- a/cmd/lk/assistant_xray.go +++ b/cmd/lk/assistant_xray.go @@ -105,7 +105,7 @@ func (s *assistantSession) showAgentCode(m agentMessage) { s.agentTurnForReply() s.snippetN++ id := fmt.Sprintf("self%d", s.snippetN) - s.snippets[id] = m.Code + s.addSnippet(id, m.Title, m.Code) s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, m.Title, normalizeLang(m.Language), m.Code, false}) } diff --git a/pkg/assistant/agent/agent.py b/pkg/assistant/agent/agent.py index 3cb99aae..4d3262c2 100644 --- a/pkg/assistant/agent/agent.py +++ b/pkg/assistant/agent/agent.py @@ -35,12 +35,66 @@ LLM_MODEL = "openai/gpt-5.4-mini" TTS_MODEL = "cartesia/sonic-3" +INTRO = "Hi, I'm LK, your LiveKit assistant." + +# The greeting for a folder lk doesn't recognize as a project. GREETING = ( - "Hi, I'm LK, your LiveKit assistant. I can help you build a voice agent, add LiveKit to an " - "app you already have, or explore what LiveKit can do, like realtime voice models, phone " - "calls, and simulations for testing your agents. What would you like to do?" + f"{INTRO} I can help you build your first voice agent: create it from a template, talk to it, " + "and set up your coding agent for LiveKit. What would you like to build?" ) + +def load_folder() -> dict: + """What lk found in the user's current folder before the session started.""" + try: + return json.loads(os.environ.get("LK_ASSISTANT_FOLDER", "{}")) + except json.JSONDecodeError: + return {} + + +def load_coding_agents() -> dict: + """What LiveKit has already set up for the user's coding agents: skills and the + Docs MCP server, in their project and globally.""" + try: + return json.loads(os.environ.get("LK_ASSISTANT_CODING_AGENTS", "{}")) + except json.JSONDecodeError: + return {} + + +def greeting(folder: dict, coding_agents: dict) -> str: + """A short, fixed greeting for what's in the user's folder. It doesn't list the + details; the agent knows them from its instructions. It's not generated, so it + starts right away.""" + kind = folder.get("kind") + # Offer to set up coding agents only when it isn't done already. + ready = coding_agents.get("ready") + if kind == "empty": + return ( + f"{INTRO} I can help you build your first voice agent. This folder is empty, so I can " + "create one here and you can talk to it in about a minute. Or we can talk about what " + "you want to build first. What would you like to do?" + ) + if kind == "livekit": + if ready: + return ( + f"{INTRO} This folder already has a LiveKit project, and your coding agent is set up " + "for it, so that's the best place to work on it. I can answer quick questions from " + "the docs, or create a new agent in a new folder. What would you like to do?" + ) + return ( + f"{INTRO} This folder already has a LiveKit project. The most useful thing I can do " + "here is set up your coding agent for LiveKit, so it has current docs and skills. I can " + "also answer quick questions from the docs. What would you like to do?" + ) + if kind == "app": + return ( + f"{INTRO} I see a project here that doesn't use LiveKit yet. I can create a voice agent " + "for it and give your coding agent the steps to connect the two. What would you like to " + "do?" + ) + return GREETING + + # Pronunciations for the TTS, as Cartesia inline IPA overrides: "live" as in # "alive", not "liver". They apply only to the speech, so transcripts keep the # real spelling. Matching ignores case, and longer keys match first, so the @@ -53,11 +107,26 @@ "LiveKit’s": LIVEKITS_IPA, "LiveKit": LIVEKIT_IPA, "Live Kit": LIVEKIT_IPA, + # Said as letters, without the dot. + "Next.js": "Next J S", + "Node.js": "Node J S", +} + +# The assistant's name, LK, and the lk command are said as the letters L and +# K. Matching is case-sensitive, and the lowercase keys include the spaces or +# punctuation around the word, so that words like "walk" don't match. The +# Markdown filter has already removed backticks. +LK_IPA = "<<ˈ|ɛ|l>> <<ˈ|k|eɪ>>" +NAME_PRONUNCIATIONS = { + "LK": LK_IPA, + " lk ": f" {LK_IPA} ", + " lk.": f" {LK_IPA}.", + " lk,": f" {LK_IPA},", + " lk?": f" {LK_IPA}?", } -# The assistant's name, LK, is said as the letters L and K. Matching is -# case-sensitive so that words like "walk" don't match. -NAME_PRONUNCIATIONS = {"LK": "<<ˈ|ɛ|l>> <<ˈ|k|eɪ>>"} +# Links to read more must point here. +DOCS_URL = "https://docs.livekit.io/" # The Docs MCP tools the agent uses. DOCS_TOOLS = ("get_docs_overview", "docs_search", "get_pages", "code_search") @@ -76,20 +145,40 @@ MAX_TOOL_RESULT_CHARS = 50000 INSTRUCTIONS = """You are LK, the LiveKit assistant, a voice assistant inside the LiveKit CLI. -You help developers build voice AI agents with LiveKit, add LiveKit to apps they already have, and -explore what LiveKit can do. You are yourself a LiveKit voice agent, built with the same Agents SDK -and LiveKit Inference the user can build with. +You help developers get started with LiveKit: build their first voice agent, talk to it, set up +their coding agent for LiveKit, and understand the basics they need. You are yourself a LiveKit +voice agent, built with the same Agents SDK and LiveKit Inference the user can build with. + +Getting started is your job. Ongoing work on a project belongs in the user's coding agent, which +can read and change their code. You can't see or change the user's files, so when the user asks you +to fix, change, or review their code, don't suggest a fix yourself: call hand_off_task with a prompt +for their coding agent. Match the user's level. If they seem new, describe outcomes before LiveKit terms, and explain a term in a few words the first time you use it. If they seem experienced, be terse and precise, and skip the basics. If they ask you to be briefer or to go deeper, do that for the rest of the -conversation. +conversation. Be plain and calm: no slang, no exclamations, no flattery. + +Don't repeat yourself. Don't ask a question the user has already answered, and don't ask twice +before doing something. When the user's request is clear, do it; don't offer choices they didn't +ask for. Don't restate what's already on screen. + +Say only what's true. Tell the user something worked only after a tool result says so. Name +commands exactly as the tool result or the card on screen shows them. Don't say you can't do +something and then do it. Don't promise anything you can't show, such as a diagram or a file. + +When you talk about the STT-LLM-TTS pipeline, say once that it's a cascaded pipeline, three models +in sequence: speech-to-text, an LLM, and text-to-speech. After that, call it the cascaded pipeline. You speak out loud, so: - Keep replies short: one to three sentences. Offer to go deeper instead of covering everything. -- Use plain sentences. No markdown, lists, emojis, or symbols. +- Use plain sentences. No markdown, lists, emojis, or symbols, with one exception: put a short + command, package, file, function, or class name in backticks, such as `lk agent dev` or + `AgentSession`. On screen it shows in monospace. The backticks aren't spoken. - Never read code, commands, or long identifiers aloud. Call show_code to show them, then say one short sentence about what you showed. +- If the user asks you to copy something on screen, call copy_to_clipboard. Don't copy unless + they ask. LiveKit changes often, and what you remember about it is out of date. For every question about LiveKit features, APIs, SDKs, models, plugins, CLI commands, pricing, or configuration, call @@ -99,37 +188,63 @@ user asks what LiveKit can do, call get_docs_overview to see how the docs are organized. If the docs don't cover it, say so. Never guess an API. -You can't see or change the user's files. You help them plan and decide, and you set them up to -do the work: -- To learn about their LiveKit Cloud project and current directory, call get_project_info. To check - on deployed agents, call list_cloud_agents. -- For a new project, start it from a template with create_project. If the user hasn't said which - template, call list_templates and suggest the best fit: for a voice agent, agent-starter-python - or agent-starter-node. Call get_project_info first: if the current directory is empty, ask - whether to create the project right there or in a new folder. New projects already include - LiveKit's skills and Docs MCP server for the user's coding agents. -- For an existing project, offer set_up_coding_agent, which installs LiveKit's skills and Docs MCP - server into the user's coding agents (such as Claude Code, Cursor, or Codex). -- To add LiveKit to an existing app (get_project_info reports its framework), explain the three - parts in plain words: the app gets a token from a small server endpoint, joins a LiveKit room, - and the agent joins that room through dispatch. Then create the agent with create_project and - give the frontend and token work to their coding agent with hand_off_task, based on the docs. -- After a project is created, offer to try it right here with try_project, so the user can talk - to the agent they just made. +Show docs answers on screen, not only in speech. When an answer has more than one point, call +show_answer with a few short points and the docs page they come from, then say one sentence that +sums it up. For a one-sentence answer, just say it, and call show_link with the page it comes +from when the user would want to read more. Use page titles and URLs exactly as your docs search +returned them, and never read a URL aloud. + +How you help someone get started: +- Call get_project_info when you need to know about their LiveKit Cloud project or their current + directory. +- For a new voice agent, start it from a template with create_project. If the user hasn't said + which template, suggest agent-starter-python or agent-starter-node, or call list_templates if + they want something else. If the current directory is empty, create it right there. New + projects already include LiveKit's skills and Docs MCP server for the user's coding agents. +- After you create a project, offer to try it right here with try_project, so the user can talk + to the agent they just made. While their agent runs, the user talks to it, not to you, and you + hear nothing. You are never their agent: don't speak as it, and don't say it's running until lk + tells you. You can't stop it; the user presses B. +- When they want to change their new agent, such as its instructions, voice, or tools, call + hand_off_task with a prompt for their coding agent: what to change, plus what you learned from + the docs, such as package names, APIs, and CLI commands. Write the prompt itself, addressed to + the coding agent, not a prompt about writing a prompt. +- To add LiveKit to an app they already have, explain the three parts in plain words: the app + gets a token from a small server endpoint, joins a LiveKit room, and the agent joins that room + through dispatch. Then create the agent with create_project in a new folder, and give the + frontend and token work to their coding agent with hand_off_task, based on the docs. +- If their coding agent isn't set up for LiveKit, offer set_up_coding_agent, which installs + LiveKit's skills and Docs MCP server into it (such as Claude Code, Cursor, or Codex). +- If the folder already has a LiveKit project, the user is past getting started. Point them to + their coding agent for work on it, and help with what you can: setting up their coding agent, + quick questions, or a new agent in a new folder. - If the user asks how you work, or what they'll be building, call explain_yourself. -- When the user is ready to write code, call hand_off_task with a prompt for their coding agent: - what to build or change, plus what you learned from the docs, such as package names, APIs, and - CLI commands. -create_project and set_up_coding_agent don't run right away. They show the command to the user and -return a confirmation ID. Ask the user in one short sentence whether to go ahead. They can answer -by voice or on screen. If they agree by voice, call confirm_action with the ID and approved=true; -if they decline, approved=false. Never call confirm_action unless the user has just agreed. +When the user asks for something outside that, don't pretend and don't improvise. Say in one short +sentence that you can't do it here, then give them the next best thing: +- A command they can run themselves, with show_code. For example, to run an agent they already + had, try_project tells you the command. To deploy, "lk agent create" the first time and "lk + agent deploy" after that. To see logs, "lk agent logs". Check the docs for the exact command. +- A prompt for their coding agent, with hand_off_task, for anything that reads or changes their + code: fixing a bug, adding a feature, or reviewing their agent. +- The docs page that covers it, with show_link, for anything else about LiveKit. +For something unrelated to LiveKit, say it's outside what you help with, and offer what you can. + +create_project, set_up_coding_agent, and try_project show the command on screen and wait for the +user to confirm. If the user's last message already agreed to exactly that action, pass +confirmed=true so it runs right away, and don't ask again. Otherwise, the tool returns a +confirmation ID: ask the user in one short sentence whether to go ahead. They can answer by voice +or on screen. If they agree by voice, call confirm_action with the ID and approved=true; if they +decline, approved=false. Never confirm anything the user hasn't agreed to. + +If the user asks what you cost: you run on LiveKit Inference through their LiveKit Cloud project, +so your usage counts toward their plan. When the user says they're done or says goodbye, call end_call. -Prefer the LiveKit CLI for project tasks. For example, "lk agent init" creates an agent from a -starter template, "lk agent dev" runs it, and "lk agent create" deploys it to LiveKit Cloud.""" +Prefer the LiveKit CLI for project tasks. "lk agent init" creates an agent from a starter +template, "lk agent console" talks to it in the terminal, "lk agent dev" runs it with hot reload, +and "lk agent create" deploys it to LiveKit Cloud.""" async def truncate_result(ctx: mcp.MCPToolResultContext) -> str: @@ -149,6 +264,23 @@ def __init__(self) -> None: instructions = INSTRUCTIONS if project := os.environ.get("LK_ASSISTANT_PROJECT"): instructions += f"\n\nThe user's current directory is {project}." + if folder := load_folder(): + instructions += ( + " Before the session started, lk looked at the project files in it and its " + f"subfolders, and found this: {json.dumps(folder)}. Your greeting only said whether " + "it uses LiveKit. Use these details when they help, but don't list them unless the " + "user asks. Versions are as the project files give them. Call get_project_info if " + "you need more, or if the folder may have changed." + ) + if coding_agents := load_coding_agents(): + instructions += ( + "\n\nWhat LiveKit has already set up for the user's coding agents (LiveKit skills and " + f"the Docs MCP server, in this project and globally): {json.dumps(coding_agents)}. " + "Offer set_up_coding_agent only for what's missing, and say what's missing. If both " + "are already set up, don't offer it. If the user asks, tell them it's already done, " + 'and that "lk skills update" gets the latest skills. After set_up_coding_agent runs, ' + "call get_project_info for the current state." + ) super().__init__( instructions=instructions, tools=[ @@ -183,6 +315,58 @@ async def show_code(self, context: RunContext, title: str, language: str, code: # `lk assistant` reads this call from the session events and shows it. return "The code is on the user's screen. They can copy it with the Copy button." + @function_tool() + async def show_answer( + self, + context: RunContext, + title: str, + points: list[str], + source_title: str, + source_url: str, + ) -> str: + """Show a docs answer on screen as a few short points, with the docs page they come + from. The user can read it, copy it, and open the page. After calling it, say one + sentence that sums it up; don't read the points aloud. + + Args: + title: What the answer is about, such as "Turn detection". + points: Two to five short points, one sentence each. Put commands and names in + backticks. + source_title: The title of the docs page the points come from. + source_url: The page's full URL, exactly as your docs search or get_pages returned + it. It must start with https://docs.livekit.io/. + """ + # `lk assistant` reads this call from the session events and shows it. + if not source_url.startswith(DOCS_URL): + return f"Not shown: the source must be a docs page under {DOCS_URL}, from your docs searches." + return "The answer is on the user's screen. Say one sentence that sums it up." + + @function_tool() + async def show_link(self, context: RunContext, title: str, url: str) -> str: + """Show a link to a LiveKit docs page where the user can read more. The user can open + it in their browser or copy it. + + Args: + title: The page title, such as "LiveKit turn detector". + url: The page's full URL, exactly as your docs search or get_pages returned it. It + must start with https://docs.livekit.io/. + """ + # `lk assistant` reads this call from the session events and shows it. + if not url.startswith(DOCS_URL): + return f"Not shown: links must be docs pages under {DOCS_URL}, from your docs searches." + return "The link is on the user's screen." + + @function_tool() + async def copy_to_clipboard(self, context: RunContext, title: str = "") -> str: + """Copy a code, command, or prompt card on screen to the user's clipboard. Use it only + when the user asks you to copy something. + + Args: + title: Part of the card's title, such as "agent.py", "Terminal", or "Prompt". Leave it + empty for the most recent card. + """ + return await run_lk_tool("copy_to_clipboard", {"title": title}) + @function_tool() async def hand_off_task(self, context: RunContext, task: str) -> str: """Show the user a prompt to paste into their coding agent, with a button to copy it. @@ -214,49 +398,77 @@ async def list_templates(self, context: RunContext) -> str: @function_tool() async def create_project( - self, context: RunContext, template: str, name: str = "", in_current_directory: bool = False + self, + context: RunContext, + template: str, + name: str = "", + in_current_directory: bool = False, + confirmed: bool = False, ) -> str: - """Propose creating a project from a LiveKit template, with lk agent init for agent - templates or lk app create for frontends, including installing dependencies. It doesn't run - until the user confirms (see confirm_action). Afterward, tell them how to run it: for an - agent, lk agent dev in the project folder. + """Create a project from a LiveKit template, with lk agent init for agent templates or lk + app create for frontends, including installing dependencies. Unless confirmed is true, it + shows the command and waits for the user to confirm (see confirm_action). Afterward, offer + to try the new agent with try_project. Args: - template: The template name, from list_templates, such as agent-starter-python. + template: The template name, such as agent-starter-python. name: The name for the new folder: lowercase letters, numbers, and dashes. Not used with in_current_directory. in_current_directory: Create the project in the current directory itself, named after - it, instead of in a new folder. Only works if the directory is empty, and only if - the user wants it. + it, instead of in a new folder. Only works if the directory is empty. + confirmed: True only if the user's last message already agreed to exactly this, + such as "yes, create it here" after you offered it. Then it runs right away, + without asking again. Otherwise leave it false, and ask. """ return await run_lk_tool( "create_project", - {"template": template, "name": name, "in_current_directory": in_current_directory}, + { + "template": template, + "name": name, + "in_current_directory": in_current_directory, + "confirmed": confirmed, + }, ) @function_tool() - async def set_up_coding_agent(self, context: RunContext, all_projects: bool = False) -> str: - """Propose installing LiveKit's agent skills and the LiveKit Docs MCP server into the - user's coding agents (such as Claude Code, Cursor, and Codex), with lk skills install. Use - it for existing projects; new projects from create_project already have them. It doesn't - run until the user confirms (see confirm_action). + async def set_up_coding_agent( + self, + context: RunContext, + all_projects: bool = False, + reinstall: bool = False, + confirmed: bool = False, + ) -> str: + """Install LiveKit's agent skills and the LiveKit Docs MCP server into the user's coding + agents (such as Claude Code, Cursor, and Codex), with lk skills install. New projects from + create_project already have them. If they're already set up, it says so and installs + nothing. Unless confirmed is true, it shows the command and waits for the user to confirm + (see confirm_action). Args: all_projects: Install for all of the user's projects instead of only the current one. + reinstall: Install even though they're already set up. Only if the user asks for it. + confirmed: True only if the user's last message already agreed to exactly this, + such as "yes, create it here" after you offered it. Then it runs right away, + without asking again. Otherwise leave it false, and ask. """ - return await run_lk_tool("set_up_coding_agent", {"global": all_projects}) + return await run_lk_tool( + "set_up_coding_agent", + {"global": all_projects, "reinstall": reinstall, "confirmed": confirmed}, + ) @function_tool() - async def try_project(self, context: RunContext, path: str = "") -> str: - """Propose running the user's agent in this window with lk agent console, so they can talk - to it. The conversation switches to their agent until they press B to come back to you. - It doesn't run until the user confirms (see confirm_action). + async def try_project(self, context: RunContext, confirmed: bool = False) -> str: + """Run the agent the user created in this session, in this window, so they can talk to + it. The conversation switches to their agent until they press B to come back to you. + Unless confirmed is true, it waits for the user to confirm (see confirm_action). For any + other agent, it tells you how the user can run it in their own terminal. Args: - path: The project folder, relative to the user's current directory. Leave it empty - for the project created in this session, or the current directory. + confirmed: True only if the user's last message already agreed to exactly this, + such as "yes, create it here" after you offered it. Then it runs right away, + without asking again. Otherwise leave it false, and ask. """ - return await run_lk_tool("try_project", {"path": path}) + return await run_lk_tool("try_project", {"confirmed": confirmed}) @function_tool() async def explain_yourself(self, context: RunContext) -> str: @@ -381,6 +593,15 @@ async def _read(self) -> None: fut = self._pending.get(msg.get("id", "")) if fut and not fut.done(): fut.set_result(msg) + elif msg.get("type") == "user_text" and self._session is not None: + # The user typed instead of speaking: stop talking and answer it. + # Speech that can't be interrupted, such as a running + # confirmation, finishes first. + try: + self._session.interrupt() + except RuntimeError: + pass + self._session.generate_reply(user_input=msg.get("text", "")) elif msg.get("type") == "say" and self._session is not None: # lk has news for the user. self._session.generate_reply(instructions=msg.get("instructions", "")) @@ -516,8 +737,8 @@ async def assistant(ctx: agents.JobContext): await session.start(room=ctx.room, agent=Assistant()) await connect_lk(session) - # A fixed greeting: the same every time, and it starts without waiting on the LLM. - session.say(GREETING) + # A fixed greeting for what is in the folder. It starts without waiting on the LLM. + session.say(greeting(load_folder(), load_coding_agents())) if __name__ == "__main__": diff --git a/pkg/assistant/agent/evals/test_onboarding.py b/pkg/assistant/agent/evals/test_onboarding.py new file mode 100644 index 00000000..c4a14884 --- /dev/null +++ b/pkg/assistant/agent/evals/test_onboarding.py @@ -0,0 +1,184 @@ +"""Evals for LK's onboarding behavior, from failures seen in real sessions. + +They run the real LLM with lk's tools replaced by stubs that reply the way lk +does, so they need LiveKit Cloud credentials for LiveKit Inference +(LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET). They aren't part of the +embedded agent. Run them from the agent directory: + + uv run --with pytest --with pytest-asyncio pytest evals -v + +LLM output varies, so run them a few times after changing the instructions. +""" + +import json +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import agent # noqa: E402 +from livekit.agents import AgentSession, inference # noqa: E402 + +pytestmark = [ + pytest.mark.asyncio, + pytest.mark.skipif( + not all(os.environ.get(k) for k in ("LIVEKIT_URL", "LIVEKIT_API_KEY", "LIVEKIT_API_SECRET")), + reason="needs LiveKit Cloud credentials for LiveKit Inference", + ), +] + +EMPTY = {"kind": "empty"} +LIVEKIT_PROJECT = { + "kind": "livekit", + "summary": "a Python project that uses LiveKit Agents", + "projects": [{"path": ".", "language": "Python", "livekit_packages": [{"name": "livekit-agents", "version": "~=1.8"}]}], + "has_agents": True, +} +READY = {"ready": True} + +# What lk replies to each tool, close to the real text in cmd/lk. +ALREADY_SET_UP = ( + "Nothing was installed: LiveKit's skills and the Docs MCP server are already set up. Tell the user, " + 'and that "lk skills update" gets the latest skills. If they still want to reinstall, call ' + "set_up_coding_agent again with reinstall=true." +) +CONFIRM = ( + "Waiting for the user to confirm. Confirmation ID: confirm1. Command: {command}. " + "Ask the user in one short sentence whether to go ahead. They can answer by voice or on screen. If they agree by voice, call confirm_action " + "with this ID and approved=true; if they decline, approved=false." +) +EXISTING_AGENT = ( + "You can only run an agent here that you created in this session. Existing agents can need their " + "frontend, dispatch, or other services, so they run in the user's own terminal. In one short sentence, " + "tell the user you can't start it here, and that they can run it in their terminal to talk to it. Then " + 'call show_code with title "Terminal", language "shell", and this command: lk agent console.' +) + + +def stub_lk(calls: list, ready: bool = False): + async def run_lk_tool(name: str, args: dict | None = None, timeout: float = 60) -> str: + calls.append((name, args or {})) + if name in ("create_project", "set_up_coding_agent", "try_project") and (args or {}).get("confirmed"): + return "Done. The user confirmed by voice, and it ran." + if name == "create_project": + return CONFIRM.format(command="lk agent init . --template agent-starter-python --install") + if name == "set_up_coding_agent": + if ready and not (args or {}).get("reinstall"): + return ALREADY_SET_UP + return CONFIRM.format(command="lk skills install") + if name == "confirm_action": + return "Done. The project is in the current directory." + if name == "try_project": + return EXISTING_AGENT + if name == "get_project_info": + return json.dumps({"directory": "/Users/dev/my-app", "folder": EMPTY}) + if name == "list_templates": + return "agent-starter-python: Python voice agent. agent-starter-node: Node.js voice agent." + return "Done." + + return run_lk_tool + + +@pytest.fixture +def lk_calls(monkeypatch): + calls: list = [] + monkeypatch.setattr(agent, "run_lk_tool", stub_lk(calls)) + return calls + + +@pytest.fixture +def lk_calls_ready(monkeypatch): + """lk's tools when skills and the Docs MCP server are already set up.""" + calls: list = [] + monkeypatch.setattr(agent, "run_lk_tool", stub_lk(calls, ready=True)) + return calls + + +def assistant(monkeypatch, folder: dict, coding_agents: dict | None = None) -> agent.Assistant: + monkeypatch.setenv("LK_ASSISTANT_PROJECT", "/Users/dev/my-app") + monkeypatch.setenv("LK_ASSISTANT_FOLDER", json.dumps(folder)) + monkeypatch.setenv("LK_ASSISTANT_CODING_AGENTS", json.dumps(coding_agents or {})) + return agent.Assistant() + + +def tool_calls(result) -> list[str]: + return [e.item.name for e in result.events if e.type == "function_call"] + + +def judge_llm() -> inference.LLM: + return inference.LLM(model=agent.LLM_MODEL) + + +async def test_creates_project_without_asking_twice(monkeypatch, lk_calls): + """The user agreed to an offer; LK shouldn't ask again before running it.""" + async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session: + await session.start(assistant(monkeypatch, EMPTY)) + await session.run(user_input="I want to build a voice agent in Python.") + result = await session.run(user_input="Yes, create it here.") + # Either path runs it in this turn: confirmed=true, or confirm_action right after. + ran = any( + (n == "create_project" and a.get("confirmed")) or (n == "confirm_action" and a.get("approved")) + for n, a in lk_calls + ) + assert ran, f"asked again instead of running it: {lk_calls}" + + +async def test_existing_agent_gets_command(monkeypatch, lk_calls): + """An existing agent can't run here; LK says so and shows the command.""" + async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session: + await session.start(assistant(monkeypatch, LIVEKIT_PROJECT, READY)) + result = await session.run(user_input="Can you start my agent so I can talk to it?") + result.expect.contains_function_call(name="show_code") + shown = [json.loads(e.item.arguments) for e in result.events if e.type == "function_call" and e.item.name == "show_code"] + assert any("lk agent console" in s.get("code", "") for s in shown), shown + await result.expect.contains_message(role="assistant").judge( + judge_llm(), + intent="Says it can't start the user's existing agent here, and doesn't claim the agent is running.", + ) + + +async def test_bug_fix_goes_to_coding_agent(monkeypatch, lk_calls): + """LK can't read code; a bug fix becomes a prompt for the coding agent.""" + async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session: + await session.start(assistant(monkeypatch, LIVEKIT_PROJECT, READY)) + result = await session.run(user_input="My agent crashes when the user hangs up. Can you fix it?") + await result.expect.contains_message(role="assistant").judge( + judge_llm(), + intent="Doesn't claim to have read or fixed the code. Either hands the task to the user's coding agent or offers to.", + ) + + +async def test_no_setup_offer_when_ready(monkeypatch, lk_calls_ready): + """Skills and the Docs MCP server are already installed.""" + async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session: + await session.start(assistant(monkeypatch, LIVEKIT_PROJECT, READY)) + result = await session.run(user_input="Should I install LiveKit's skills for Cursor?") + assert not any(a.get("reinstall") for n, a in lk_calls_ready if n == "set_up_coding_agent") + await result.expect.contains_message(role="assistant").judge( + judge_llm(), intent="Tells the user it's already set up, and doesn't offer to install it." + ) + + +async def test_docs_question_uses_docs(monkeypatch, lk_calls): + """Answers about LiveKit come from the docs, with the source on screen.""" + async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session: + await session.start(assistant(monkeypatch, EMPTY)) + result = await session.run(user_input="How does LiveKit decide when I've finished talking?") + calls = tool_calls(result) + assert any(c in calls for c in ("docs_search", "get_pages", "get_docs_overview")), calls + assert any(c in calls for c in ("show_answer", "show_link")), f"no source shown: {calls}" + + +async def test_copy_targets_the_prompt(monkeypatch, lk_calls): + """"Copy it" after a prompt card copies the prompt, not an older card.""" + async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session: + await session.start(assistant(monkeypatch, EMPTY)) + await session.run(user_input="Show me the command to install LiveKit's skills.") + await session.run(user_input="Now write a prompt for Cursor to change my agent's voice to something calmer.") + result = await session.run(user_input="Copy the prompt.") + call = result.expect.contains_function_call(name="copy_to_clipboard") + title = json.loads(call.event().item.arguments).get("title", "") + assert title == "" or "prompt" in title.lower(), title diff --git a/pkg/assistant/assistant.go b/pkg/assistant/assistant.go index 3d676a2a..8c011bf3 100644 --- a/pkg/assistant/assistant.go +++ b/pkg/assistant/assistant.go @@ -43,6 +43,10 @@ const ( ShowCodeTool = "show_code" // HandOffTaskTool shows a prompt for the user's coding agent as a card. HandOffTaskTool = "hand_off_task" + // ShowAnswerTool shows a docs answer as a card of short points. + ShowAnswerTool = "show_answer" + // ShowLinkTool shows a link to a docs page as a card. + ShowLinkTool = "show_link" // EndCallTool ends the conversation. lk closes the overlay after it. EndCallTool = "end_call" ) diff --git a/pkg/assistant/folder.go b/pkg/assistant/folder.go new file mode 100644 index 00000000..5348db8b --- /dev/null +++ b/pkg/assistant/folder.go @@ -0,0 +1,477 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package assistant + +import ( + "encoding/json" + "fmt" + "maps" + "os" + "path/filepath" + "regexp" + "slices" + "strings" +) + +// FolderKind is what the user's current folder holds. +type FolderKind string + +const ( + FolderEmpty FolderKind = "empty" // nothing in it + FolderLiveKit FolderKind = "livekit" // a project that uses LiveKit + FolderApp FolderKind = "app" // a project that doesn't use LiveKit yet + FolderOther FolderKind = "other" // no project found +) + +// Folder describes the user's current folder, so the assistant can greet them +// with what's in it. It reads only project manifests, such as package.json and +// go.mod, in the folder and its direct subfolders. +type Folder struct { + Kind FolderKind `json:"kind"` + Summary string `json:"summary,omitempty"` // spoken, such as "a Go project that uses the LiveKit Go server SDK" + Projects []Project `json:"projects,omitempty"` + // HasLiveKitTOML is true when the folder has a livekit.toml, which a + // LiveKit Cloud agent deployment uses. + HasLiveKitTOML bool `json:"has_livekit_toml,omitempty"` + // HasAgents is true when one or two projects use LiveKit Agents. + HasAgents bool `json:"has_agents,omitempty"` +} + +// Project is one project in the folder. +type Project struct { + Path string `json:"path"` // relative to the folder; "." is the folder itself + Language string `json:"language"` + Framework string `json:"framework,omitempty"` + LiveKit []Package `json:"livekit_packages,omitempty"` +} + +// Package is a LiveKit package that a project depends on. +type Package struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` // as the manifest gives it, such as "^2.1.0" or "~=1.8" +} + +// usesAgents reports whether p uses LiveKit Agents. +func (p Project) usesAgents() bool { + return slices.ContainsFunc(p.LiveKit, func(pkg Package) bool { + return pkg.Name == "livekit-agents" || pkg.Name == "@livekit/agents" + }) +} + +// AgentPaths returns the paths of the projects that use LiveKit Agents. +func (f Folder) AgentPaths() []string { + var paths []string + for _, p := range f.Projects { + if p.usesAgents() { + paths = append(paths, p.Path) + } + } + return paths +} + +// hasAgents reports whether one or two projects use LiveKit Agents. With +// more, it isn't clear which agent the user means. +func (f Folder) hasAgents() bool { + n := 0 + for _, p := range f.Projects { + if p.usesAgents() { + n++ + } + } + return n > 0 && n <= 2 +} + +// JSON returns f as JSON for the agent. +func (f Folder) JSON() string { + b, _ := json.Marshal(f) + return string(b) +} + +// skipDirs are subfolders that never hold the user's own projects. +var skipDirs = map[string]bool{ + "node_modules": true, "vendor": true, "venv": true, "env": true, "dist": true, + "build": true, "target": true, "out": true, "__pycache__": true, "Pods": true, +} + +// maxDirs limits how many folders DescribeFolder reads, so a large folder, +// such as the home folder, stays fast. +const maxDirs = 200 + +// DescribeFolder describes dir: whether it's empty, and the projects in it, +// in its subfolders, and in their subfolders. The second level finds the +// projects in a monorepo, such as apps/web and apps/agent. It doesn't look +// inside a project's own folders. +func DescribeFolder(dir string) Folder { + entries, err := os.ReadDir(dir) + if err != nil { + return Folder{Kind: FolderOther} + } + empty := true + for _, e := range entries { + if e.Name() != ".DS_Store" { + empty = false + break + } + } + if empty { + return Folder{Kind: FolderEmpty} + } + + var f Folder + root, rootFound := detectProject(dir) + visited := 0 + var scan func(rel string, entries []os.DirEntry, depth int) + scan = func(rel string, entries []os.DirEntry, depth int) { + for _, e := range entries { + name := e.Name() + if !e.IsDir() || strings.HasPrefix(name, ".") || skipDirs[name] { + continue + } + if visited++; visited > maxDirs { + return + } + path := filepath.Join(rel, name) + if p, ok := detectProject(filepath.Join(dir, path)); ok { + p.Path = filepath.ToSlash(path) + f.Projects = append(f.Projects, p) + } else if depth < 2 { + if sub, err := os.ReadDir(filepath.Join(dir, path)); err == nil { + scan(path, sub, depth+1) + } + } + } + } + scan("", entries, 1) + // A workspace root, such as a pnpm or npm workspace, only ties its + // projects together, so it isn't a project of its own. + if rootFound && !(len(f.Projects) > 0 && isWorkspaceRoot(dir)) { + root.Path = "." + f.Projects = append([]Project{root}, f.Projects...) + } + if _, err := os.Stat(filepath.Join(dir, "livekit.toml")); err == nil { + f.HasLiveKitTOML = true + } + + switch { + case f.HasLiveKitTOML || slices.ContainsFunc(f.Projects, func(p Project) bool { return len(p.LiveKit) > 0 }): + f.Kind = FolderLiveKit + case len(f.Projects) > 0: + f.Kind = FolderApp + default: + f.Kind = FolderOther + } + f.Summary = summarize(f.Projects) + f.HasAgents = f.hasAgents() + return f +} + +// isWorkspaceRoot reports whether dir is the root of a JavaScript or Python +// workspace. +func isWorkspaceRoot(dir string) bool { + if _, err := os.Stat(filepath.Join(dir, "pnpm-workspace.yaml")); err == nil { + return true + } + if b, err := os.ReadFile(filepath.Join(dir, "package.json")); err == nil { + var pkg struct { + Workspaces json.RawMessage `json:"workspaces"` + } + if json.Unmarshal(b, &pkg) == nil && len(pkg.Workspaces) > 0 { + return true + } + } + b, _ := os.ReadFile(filepath.Join(dir, "pyproject.toml")) + return strings.Contains(string(b), "[tool.uv.workspace]") +} + +// detectProject finds the project in dir from its manifest files. +func detectProject(dir string) (Project, bool) { + read := func(name string) string { + b, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return "" + } + return string(b) + } + exists := func(name string) bool { + _, err := os.Stat(filepath.Join(dir, name)) + return err == nil + } + + var p Project + found := false + if text := read("go.mod"); text != "" { + found = true + p.Language = "Go" + p.LiveKit = append(p.LiveKit, goPackages(text)...) + } + if text := read("package.json"); text != "" { + found = true + p.Language = "JavaScript" + if exists("tsconfig.json") { + p.Language = "TypeScript" + } + framework, pkgs := nodePackages(text) + p.Framework = framework + p.LiveKit = append(p.LiveKit, pkgs...) + } + if text := read("pyproject.toml") + "\n" + read("requirements.txt"); strings.TrimSpace(text) != "" { + found = true + if p.Language == "" { + p.Language = "Python" + } + lower := strings.ToLower(text) + for _, fw := range pythonFrameworks { + if p.Framework == "" && strings.Contains(lower, fw.dep) { + p.Framework = fw.name + } + } + p.LiveKit = append(p.LiveKit, pythonPackages(nameLineRe.ReplaceAllString(lower, ""))...) + } + if text := read("Cargo.toml"); text != "" { + found = true + p.Language = "Rust" + p.LiveKit = append(p.LiveKit, rustPackages(text)...) + } + if text := read("Package.swift"); text != "" { + found = true + p.Language = "Swift" + if m := swiftRe.FindStringSubmatch(text); m != nil { + p.LiveKit = append(p.LiveKit, Package{Name: "client-sdk-swift", Version: m[1]}) + } + } + if text := read("pubspec.yaml"); text != "" { + found = true + p.Language = "Dart" + if strings.Contains(text, "sdk: flutter") { + p.Framework = "Flutter" + } + if m := pubspecRe.FindStringSubmatch(text); m != nil { + p.LiveKit = append(p.LiveKit, Package{Name: "livekit_client", Version: strings.Trim(m[1], `"'`)}) + } + } + for _, name := range []string{"build.gradle.kts", "build.gradle", "app/build.gradle.kts", "app/build.gradle", "gradle/libs.versions.toml"} { + if text := read(name); text != "" { + found = true + p.Language = "Android" + if m := gradleRe.FindStringSubmatch(text); m != nil && len(p.LiveKit) == 0 { + p.LiveKit = append(p.LiveKit, Package{Name: "io.livekit:livekit-android", Version: m[1]}) + } + } + } + return p, found +} + +// appFrameworks maps package.json dependencies to the frameworks they mean, +// most specific first. +var appFrameworks = []struct{ dep, name string }{ + {"next", "Next.js"}, + {"expo", "Expo"}, + {"react-native", "React Native"}, + {"@sveltejs/kit", "SvelteKit"}, + {"nuxt", "Nuxt"}, + {"@angular/core", "Angular"}, + {"svelte", "Svelte"}, + {"vue", "Vue"}, + {"react", "React"}, + {"express", "Express"}, +} + +// pythonFrameworks are Python web frameworks found in project files. +var pythonFrameworks = []struct{ dep, name string }{ + {"fastapi", "FastAPI"}, + {"django", "Django"}, + {"flask", "Flask"}, +} + +func nodePackages(text string) (string, []Package) { + var pkg struct { + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + } + if json.Unmarshal([]byte(text), &pkg) != nil { + return "", nil + } + deps := map[string]string{} + maps.Copy(deps, pkg.DevDependencies) + maps.Copy(deps, pkg.Dependencies) + framework := "" + for _, f := range appFrameworks { + if _, ok := deps[f.dep]; ok { + framework = f.name + break + } + } + var pkgs []Package + for d, v := range deps { + if strings.HasPrefix(d, "@livekit/") || strings.HasPrefix(d, "livekit-") { + pkgs = append(pkgs, Package{Name: d, Version: v}) + } + } + slices.SortFunc(pkgs, func(a, b Package) int { return strings.Compare(a.Name, b.Name) }) + return framework, pkgs +} + +var ( + goRe = regexp.MustCompile(`(?m)^\s*(?:require\s+)?(github\.com/livekit/[\w./-]+)\s+(v[\w.+-]+)\s*$`) + pythonRe = regexp.MustCompile(`(?m)(?:^|["'\s])(livekit(?:-[a-z0-9]+)*)(?:\[[^\]]*\])?\s*((?:[~=<>!]=?|==)\s*[0-9][\w.*]*)?`) + rustRe = regexp.MustCompile(`(?m)^\s*(livekit(?:-[a-z]+)*)\s*=\s*(?:"([^"]+)"|\{[^}]*version\s*=\s*"([^"]+)")`) + swiftRe = regexp.MustCompile(`livekit/client-sdk-swift[^)]*?(?:from:\s*|exact:\s*)"([^"]+)"`) + // nameLineRe matches pyproject.toml's name and description, which can + // mention LiveKit without depending on it. + nameLineRe = regexp.MustCompile(`(?m)^\s*(?:name|description)\s*=.*$`) + pubspecRe = regexp.MustCompile(`(?m)^\s*livekit_client:\s*(\S+)`) + gradleRe = regexp.MustCompile(`io\.livekit:livekit-android(?::([\w.-]+))?`) +) + +func goPackages(text string) []Package { + var pkgs []Package + for line := range strings.SplitSeq(text, "\n") { + if strings.Contains(line, "// indirect") { + continue + } + if m := goRe.FindStringSubmatch(line); m != nil { + pkgs = append(pkgs, Package{Name: m[1], Version: m[2]}) + } + } + return pkgs +} + +func pythonPackages(text string) []Package { + var pkgs []Package + seen := map[string]bool{} + for _, m := range pythonRe.FindAllStringSubmatch(text, -1) { + if seen[m[1]] { + continue + } + seen[m[1]] = true + pkgs = append(pkgs, Package{Name: m[1], Version: strings.ReplaceAll(m[2], " ", "")}) + } + return pkgs +} + +func rustPackages(text string) []Package { + var pkgs []Package + for _, m := range rustRe.FindAllStringSubmatch(text, -1) { + pkgs = append(pkgs, Package{Name: m[1], Version: m[2] + m[3]}) + } + return pkgs +} + +// packageNames are the spoken names of the main LiveKit packages. Others, +// such as plugins, aren't named in the summary. +var packageNames = map[string]string{ + "livekit-agents": "LiveKit Agents", + "@livekit/agents": "LiveKit Agents", + "livekit-api": "the LiveKit server API", + "livekit-server-sdk": "the LiveKit server SDK", + "livekit-client": "the LiveKit client SDK", + "@livekit/components-react": "LiveKit React components", + "@livekit/react-native": "the LiveKit React Native SDK", + "github.com/livekit/server-sdk-go": "the LiveKit Go server SDK", + "github.com/livekit/server-sdk-go/v2": "the LiveKit Go server SDK", + "client-sdk-swift": "the LiveKit Swift SDK", + "livekit_client": "the LiveKit Flutter SDK", + "io.livekit:livekit-android": "the LiveKit Android SDK", +} + +// summarize describes projects in a short phrase for speech, such as "a +// Python project in agent that uses LiveKit Agents and a Next.js app in web +// that uses the LiveKit client SDK". It names up to two projects that use +// LiveKit, agents first, and counts the others. +func summarize(projects []Project) string { + if len(projects) == 0 { + return "" + } + var named, others []Project + for _, p := range projects { + if len(p.LiveKit) > 0 { + named = append(named, p) + } else { + others = append(others, p) + } + } + if len(named) == 0 && len(others) <= 2 { + named, others = others, nil + } + if len(named) > 2 { + return fmt.Sprintf("%d projects, and %d of them use LiveKit", len(projects), len(named)) + } + if len(named) == 0 { + return fmt.Sprintf("%d projects", len(projects)) + } + slices.SortStableFunc(named, func(a, b Project) int { + switch { + case a.usesAgents() == b.usesAgents(): + return 0 + case a.usesAgents(): + return -1 + default: + return 1 + } + }) + + var parts []string + for _, p := range named { + parts = append(parts, describe(p)) + } + s := strings.Join(parts, " and ") + switch len(others) { + case 0: + case 1: + s += ", and one other project" + default: + s += fmt.Sprintf(", and %d other projects", len(others)) + } + return s +} + +// describe describes one project for speech. +func describe(p Project) string { + what := p.Language + " project" + if p.Framework != "" { + what = p.Framework + " app" + } + s := article(what) + " " + what + if p.Path != "." { + s += " in " + p.Path + } + var names []string + for _, pkg := range p.LiveKit { + name := packageNames[pkg.Name] + if name == "" && strings.HasPrefix(pkg.Name, "github.com/livekit/server-sdk-go") { + name = packageNames["github.com/livekit/server-sdk-go"] + } + if name != "" && !slices.Contains(names, name) { + names = append(names, name) + } + } + if len(names) > 2 { + names = names[:2] + } + switch { + case len(names) > 0: + s += " that uses " + strings.Join(names, " and ") + case len(p.LiveKit) > 0: + s += " that uses LiveKit" + } + return s +} + +func article(word string) string { + if strings.ContainsRune("AEIOU", rune(word[0])) { + return "an" + } + return "a" +} diff --git a/pkg/assistant/folder_test.go b/pkg/assistant/folder_test.go new file mode 100644 index 00000000..8ceda551 --- /dev/null +++ b/pkg/assistant/folder_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package assistant + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeFiles(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for name, content := range files { + path := filepath.Join(dir, name) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + } + return dir +} + +func TestDescribeFolderEmpty(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{".DS_Store": ""})) + require.Equal(t, FolderEmpty, f.Kind) +} + +func TestDescribeFolderOther(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{"notes.txt": "hi"})) + require.Equal(t, FolderOther, f.Kind) + require.Empty(t, f.Summary) +} + +func TestDescribeFolderGo(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{"go.mod": `module example.com/app + +go 1.24 + +require ( + github.com/livekit/protocol v1.39.0 // indirect + github.com/livekit/server-sdk-go/v2 v2.9.1 +) +`})) + require.Equal(t, FolderLiveKit, f.Kind) + require.Equal(t, []Package{{Name: "github.com/livekit/server-sdk-go/v2", Version: "v2.9.1"}}, f.Projects[0].LiveKit) + require.Equal(t, "a Go project that uses the LiveKit Go server SDK", f.Summary) +} + +func TestDescribeFolderPythonAgent(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{"pyproject.toml": `[project] +name = "livekit-agent-starter" +dependencies = [ + "livekit-agents[silero,turn-detector]~=1.8", + "livekit-plugins-noise-cancellation~=0.2", +] +`})) + require.Equal(t, FolderLiveKit, f.Kind) + require.True(t, f.HasAgents) + require.Equal(t, []Package{ + {Name: "livekit-agents", Version: "~=1.8"}, + {Name: "livekit-plugins-noise-cancellation", Version: "~=0.2"}, + }, f.Projects[0].LiveKit) + require.Equal(t, "a Python project that uses LiveKit Agents", f.Summary) +} + +func TestDescribeFolderAppWithoutLiveKit(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{ + "package.json": `{"dependencies": {"next": "15.0.0", "react": "19.0.0"}}`, + "tsconfig.json": "{}", + })) + require.Equal(t, FolderApp, f.Kind) + require.Equal(t, "TypeScript", f.Projects[0].Language) + require.Equal(t, "a Next.js app", f.Summary) +} + +func TestDescribeFolderSubfolders(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{ + "README.md": "", + "frontend/package.json": `{"dependencies": {"next": "15.0.0", "livekit-client": "^2.15.0", "@livekit/components-react": "^2.9.0"}}`, + "agent/requirements.txt": "livekit-agents==1.8.5\n", + "node_modules/x/package.json": `{}`, + })) + require.Equal(t, FolderLiveKit, f.Kind) + require.Len(t, f.Projects, 2) + require.Equal(t, "a Python project in agent that uses LiveKit Agents and a Next.js app in frontend that uses LiveKit React components and the LiveKit client SDK", f.Summary) + require.True(t, f.HasAgents) +} + +func TestDescribeFolderManyProjects(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{ + "a/go.mod": "module a\n", + "b/requirements.txt": "livekit-agents~=1.8\n", + "c/package.json": `{"dependencies": {"livekit-client": "^2.15.0"}}`, + })) + require.Equal(t, FolderLiveKit, f.Kind) + require.True(t, f.HasAgents) + require.Equal(t, "a Python project in b that uses LiveKit Agents and a JavaScript project in c that uses the LiveKit client SDK, and one other project", f.Summary) +} + +func TestDescribeFolderManyLiveKitProjects(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{ + "a/requirements.txt": "livekit-agents~=1.8\n", + "b/requirements.txt": "livekit-agents~=1.8\n", + "c/requirements.txt": "livekit-agents~=1.8\n", + "d/go.mod": "module d\n", + })) + require.False(t, f.HasAgents) + require.Equal(t, "4 projects, and 3 of them use LiveKit", f.Summary) +} + +func TestDescribeFolderMonorepo(t *testing.T) { + f := DescribeFolder(writeFiles(t, map[string]string{ + "package.json": `{"name": "pets", "private": true, "scripts": {"dev": "pnpm --filter web dev"}}`, + "pnpm-workspace.yaml": "packages:\n - 'apps/web'\n - 'packages/*'\n", + "apps/web/package.json": `{"dependencies": {"next": "16.3.6", "livekit-client": "^2.22.2", "livekit-server-sdk": "^2.18.0"}}`, + "apps/web/src/package.json": `{}`, + "apps/agent/pyproject.toml": "[project]\ndependencies = [\"livekit-agents~=1.8\"]\n", + "packages/shared/package.json": `{"name": "shared"}`, + "docs/notes.md": "", + })) + require.Equal(t, FolderLiveKit, f.Kind) + require.True(t, f.HasAgents) + require.Len(t, f.Projects, 3) + require.Equal(t, "a Python project in apps/agent that uses LiveKit Agents and a Next.js app in apps/web that uses the LiveKit client SDK and the LiveKit server SDK, and one other project", f.Summary) +} From d3d100aed58133da997881b29be19816c2097eb2 Mon Sep 17 00:00:00 2001 From: royalfig Date: Tue, 13 Oct 2026 11:08:47 -0400 Subject: [PATCH 08/12] feat(assistant): simulations, LiveKit primer, and conversation fixes - Run LK in Agent Simulations: scenarios, mocked lk tools, and tool-call checks. - Add a primer so basic questions get an expert answer without a lookup. - Say a lookup line only when a lookup is slow. - Short greetings that offer what fits the folder. - Speak in Simplified Technical English; explain concretely. - Close on a spoken exit even if the agent doesn't call end_call. - Cover the screen while the user's agent starts. - Open docs pages, not their Markdown, and darken the backdrop. - Allow native full screen in windowed mode. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 4 + cmd/lk/assistant_guest.go | 2 +- cmd/lk/assistant_session.go | 38 ++- cmd/lk/assistant_session_test.go | 14 + cmd/lk/assistant_tools.go | 2 +- pkg/assistant/agent/agent.py | 302 +++++++++++++----- pkg/assistant/agent/lk_simulation.py | 196 ++++++++++++ pkg/assistant/agent/simulations/README.md | 29 ++ .../agent/simulations/audio-scenarios.yaml | 35 ++ .../agent/simulations/scenarios.yaml | 237 ++++++++++++++ pkg/overlay/overlay_darwin.m | 2 + pkg/overlay/ui/index.html | 61 +++- 12 files changed, 834 insertions(+), 88 deletions(-) create mode 100644 pkg/assistant/agent/lk_simulation.py create mode 100644 pkg/assistant/agent/simulations/README.md create mode 100644 pkg/assistant/agent/simulations/audio-scenarios.yaml create mode 100644 pkg/assistant/agent/simulations/scenarios.yaml diff --git a/.gitignore b/.gitignore index d96f5184..dda3c414 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ autocomplete/fish_autocomplete # Python bytecode from the assistant agent __pycache__/ + +# Created when running the assistant agent's simulations from the repo +pkg/assistant/agent/.venv/ +pkg/assistant/agent/uv.lock diff --git a/cmd/lk/assistant_guest.go b/cmd/lk/assistant_guest.go index b741b5f3..71f340d1 100644 --- a/cmd/lk/assistant_guest.go +++ b/cmd/lk/assistant_guest.go @@ -105,7 +105,7 @@ func (s *assistantSession) proposeTryProject(requestID string, confirmed bool) { } s.replyTool(requestID, fmt.Sprintf( "Waiting for the user to confirm. Confirmation ID: %s. This runs their agent, %s, here so they can talk to it. "+ - "Ask the user in one short sentence whether to start it. They can answer by voice or on screen. "+ + "Ask the user in a few words whether to start it. They can answer by voice or on screen. "+ "If they agree by voice, call confirm_action with this ID and approved=true.", id, name), nil) } diff --git a/cmd/lk/assistant_session.go b/cmd/lk/assistant_session.go index 4450c516..427209a7 100644 --- a/cmd/lk/assistant_session.go +++ b/cmd/lk/assistant_session.go @@ -25,6 +25,7 @@ import ( "os" "os/exec" "regexp" + "slices" "strings" "sync" "time" @@ -672,6 +673,30 @@ func (s *assistantSession) userCommitted(text string) { s.userCommits = strings.TrimSpace(s.userCommits + " " + text) s.userFinal = s.userCommits s.out.Send(overlayTranscript{"transcript", s.userTurn, "user", 0, s.userCommits}) + // The agent usually calls end_call, but it can just say goodbye. A turn + // that's only "exit" closes the overlay either way. + if isExitCommand(s.userCommits) && !s.guestActive() { + s.startEnding() + } +} + +// exitCommands are whole turns that mean the user wants to close LK. +var exitCommands = []string{"exit", "quit", "close", "stop", "bye", "goodbye", "close it", "exit please", "that's all", "i'm done"} + +// isExitCommand reports whether a user turn is only a request to close LK. +func isExitCommand(text string) bool { + t := strings.Trim(strings.ToLower(strings.TrimSpace(text)), ".!?, ") + return slices.Contains(exitCommands, t) +} + +// startEnding closes the overlay once the agent has said goodbye, or after +// endingTimeout. +func (s *assistantSession) startEnding() { + if s.ending { + return + } + s.ending = true + s.endingSince = time.Now() } // beginAgentSpeech adds an empty agent paragraph so the page shows its cursor @@ -756,9 +781,8 @@ func (s *assistantSession) toolCall(fc *agent.FunctionCall) { case assistant.EndCallTool: // Only the assistant's end_call closes the overlay; the user's own // agent may have one too. - if !s.ending && !s.guestActive() { - s.ending = true - s.endingSince = time.Now() + if !s.guestActive() { + s.startEnding() } default: id := fc.GetCallId() @@ -931,6 +955,12 @@ func (s *assistantSession) copy(id string) { // its docs searches, and its show_link tool rejects any other URL. const docsURL = "https://docs.livekit.io/" +// docsPageURL returns the page people read for a docs URL. The Docs MCP +// server returns the Markdown version, which ends in .md. +func docsPageURL(url string) string { + return strings.TrimSuffix(url, ".md") +} + // showLink shows a show_link call as a card that links to a docs page. func (s *assistantSession) showLink(fc *agent.FunctionCall) { if s.shownCalls[fc.GetCallId()] { @@ -944,6 +974,7 @@ func (s *assistantSession) showLink(fc *agent.FunctionCall) { return } s.shownCalls[fc.GetCallId()] = true + args.URL = docsPageURL(args.URL) if args.Title == "" { args.Title = "LiveKit docs" } @@ -979,6 +1010,7 @@ func (s *assistantSession) showAnswer(fc *agent.FunctionCall) { return } s.shownCalls[fc.GetCallId()] = true + args.SourceURL = docsPageURL(args.SourceURL) s.agentTurnForReply() id := fc.GetCallId() s.links[id] = args.SourceURL diff --git a/cmd/lk/assistant_session_test.go b/cmd/lk/assistant_session_test.go index f0ec61b1..0fb70845 100644 --- a/cmd/lk/assistant_session_test.go +++ b/cmd/lk/assistant_session_test.go @@ -172,3 +172,17 @@ func TestIsAuthError(t *testing.T) { // A timestamp that happens to contain 401 isn't an auth error. require.False(t, isAuthError("type='tts_error' timestamp=1791565401.871234 label='livekit.agents.inference.tts.TTS' error=APIConnectionError('connection failed') recoverable=True")) } + +func TestIsExitCommand(t *testing.T) { + for _, text := range []string{"Exit.", "close", "Bye!", "That's all."} { + require.True(t, isExitCommand(text), text) + } + for _, text := range []string{"How do I exit the console?", "Close the loop on dispatch", ""} { + require.False(t, isExitCommand(text), text) + } +} + +func TestDocsPageURL(t *testing.T) { + require.Equal(t, "https://docs.livekit.io/deploy/agents", docsPageURL("https://docs.livekit.io/deploy/agents.md")) + require.Equal(t, "https://docs.livekit.io/deploy/agents/", docsPageURL("https://docs.livekit.io/deploy/agents/")) +} diff --git a/cmd/lk/assistant_tools.go b/cmd/lk/assistant_tools.go index 56a6f7fc..baf13b6e 100644 --- a/cmd/lk/assistant_tools.go +++ b/cmd/lk/assistant_tools.go @@ -264,7 +264,7 @@ func (s *assistantSession) proposeCommand(requestID, detail, runLabel string, ar } s.replyTool(requestID, fmt.Sprintf( "Waiting for the user to confirm. Confirmation ID: %s. Command: %s. %s "+ - "Ask the user in one short sentence whether to go ahead. They can answer by voice or "+ + "The command is on screen, so don't say it. Ask the user in a few words whether to go ahead. They can answer by voice or "+ "on screen. If they agree by voice, call confirm_action with this ID and approved=true; "+ "if they decline, approved=false. If they answer on screen, you'll be told the result.", id, command, detail), nil) diff --git a/pkg/assistant/agent/agent.py b/pkg/assistant/agent/agent.py index 4d3262c2..cdd542ab 100644 --- a/pkg/assistant/agent/agent.py +++ b/pkg/assistant/agent/agent.py @@ -32,16 +32,22 @@ DOCS_MCP_URL = "https://docs.livekit.io/mcp" STT_MODEL = "assemblyai/universal-3-6-pro" -LLM_MODEL = "openai/gpt-5.4-mini" TTS_MODEL = "cartesia/sonic-3" +# Settings to compare in simulations (see simulations/README.md). The defaults +# are what lk ships. +LLM_MODEL = os.environ.get("LK_ASSISTANT_LLM", "openai/gpt-5.4-mini") +# Whether the instructions include PRIMER, so basic questions get an answer +# without a docs lookup. +USE_PRIMER = os.environ.get("LK_ASSISTANT_PRIMER", "1") == "1" +# When to say a short line during a docs lookup: "always", "slow" (only when +# the lookup takes a while, at most once per user turn), or "off". +FILLER = os.environ.get("LK_ASSISTANT_FILLER", "slow") + INTRO = "Hi, I'm LK, your LiveKit assistant." # The greeting for a folder lk doesn't recognize as a project. -GREETING = ( - f"{INTRO} I can help you build your first voice agent: create it from a template, talk to it, " - "and set up your coding agent for LiveKit. What would you like to build?" -) +GREETING = f"{INTRO} Want to build your first voice agent, or ask me about LiveKit?" def load_folder() -> dict: @@ -62,36 +68,19 @@ def load_coding_agents() -> dict: def greeting(folder: dict, coding_agents: dict) -> str: - """A short, fixed greeting for what's in the user's folder. It doesn't list the - details; the agent knows them from its instructions. It's not generated, so it - starts right away.""" + """A short, fixed greeting for what's in the user's folder. It offers what + fits without describing the folder; the agent knows the details from its + instructions. It's not generated, so it starts right away.""" kind = folder.get("kind") - # Offer to set up coding agents only when it isn't done already. - ready = coding_agents.get("ready") + # One sentence that offers what fits the folder, without describing it. if kind == "empty": - return ( - f"{INTRO} I can help you build your first voice agent. This folder is empty, so I can " - "create one here and you can talk to it in about a minute. Or we can talk about what " - "you want to build first. What would you like to do?" - ) + return f"{INTRO} Want to get a voice agent up and running here, or ask me about LiveKit?" if kind == "livekit": - if ready: - return ( - f"{INTRO} This folder already has a LiveKit project, and your coding agent is set up " - "for it, so that's the best place to work on it. I can answer quick questions from " - "the docs, or create a new agent in a new folder. What would you like to do?" - ) - return ( - f"{INTRO} This folder already has a LiveKit project. The most useful thing I can do " - "here is set up your coding agent for LiveKit, so it has current docs and skills. I can " - "also answer quick questions from the docs. What would you like to do?" - ) + if coding_agents.get("ready"): + return f"{INTRO} Ask me anything about LiveKit, or we can start a new agent in another folder." + return f"{INTRO} Want me to set up your coding agent for LiveKit, or do you have a question?" if kind == "app": - return ( - f"{INTRO} I see a project here that doesn't use LiveKit yet. I can create a voice agent " - "for it and give your coding agent the steps to connect the two. What would you like to " - "do?" - ) + return f"{INTRO} Want to add a voice agent to this project, or ask me about LiveKit first?" return GREETING @@ -139,6 +128,9 @@ def greeting(folder: dict, coding_agents: dict) -> str: "Let me look that up in the docs.", ) +# With FILLER "slow", how long a lookup runs before the agent says a line. +SLOW_LOOKUP_SECONDS = 2.5 + # Docs MCP results can be whole pages. Cap them to bound the LLM context, but # high enough that a full docs page fits. The longest pages, such as # /agents/build/turns/, are over 37,000 characters. @@ -149,10 +141,23 @@ def greeting(folder: dict, coding_agents: dict) -> str: their coding agent for LiveKit, and understand the basics they need. You are yourself a LiveKit voice agent, built with the same Agents SDK and LiveKit Inference the user can build with. +Be generous with what you know. Your job is to help the user build a voice agent that works well, +so share the knowledge that makes the difference: why LiveKit works the way it does, the tradeoff +behind a choice, and what to watch for, such as latency, turn-taking, interruptions, and testing. +Give context where it helps the user understand, not only the answer to the literal question. +Generous means the context that matters for this user, not every detail: for someone who isn't +technical, skip implementation details unless they ask. +Explain concretely. Walk through what happens, step by step, in plain words, such as what happens +when a user starts talking to an agent, rather than abstractions such as "realtime bus" or +"orchestration". For a big question with several parts, such as how LiveKit fits together, call +show_answer with the parts, and speak a short version of the most important idea. + Getting started is your job. Ongoing work on a project belongs in the user's coding agent, which can read and change their code. You can't see or change the user's files, so when the user asks you to fix, change, or review their code, don't suggest a fix yourself: call hand_off_task with a prompt -for their coding agent. +for their coding agent. Since you haven't seen their code, the prompt describes the problem, how +to reproduce it, and how to check that it's fixed, plus anything relevant from the docs. It doesn't +prescribe a fix; the coding agent finds the cause. Match the user's level. If they seem new, describe outcomes before LiveKit terms, and explain a term in a few words the first time you use it. If they seem experienced, be terse and precise, and @@ -160,8 +165,11 @@ def greeting(folder: dict, coding_agents: dict) -> str: conversation. Be plain and calm: no slang, no exclamations, no flattery. Don't repeat yourself. Don't ask a question the user has already answered, and don't ask twice -before doing something. When the user's request is clear, do it; don't offer choices they didn't -ask for. Don't restate what's already on screen. +before doing something. Don't announce that you're looking something up; just look it up. Don't +end replies with an offer such as "If you want, I can" or "I can sketch that next"; the user +knows they can ask. When the next step is clear, ask in a few +words, such as "Want to try it?", or just do it. When the user's request is clear, do it; don't +offer choices they didn't ask for. Don't restate what's already on screen. Say only what's true. Tell the user something worked only after a tool result says so. Name commands exactly as the tool result or the card on screen shows them. Don't say you can't do @@ -170,8 +178,16 @@ def greeting(folder: dict, coding_agents: dict) -> str: When you talk about the STT-LLM-TTS pipeline, say once that it's a cascaded pipeline, three models in sequence: speech-to-text, an LLM, and text-to-speech. After that, call it the cascaded pipeline. +Speak in the style of Simplified Technical English (ASD-STE100): short sentences, at most about 20 +words, with one idea each. Use active voice, simple tenses, and common words with one clear +meaning. Don't use idioms or figures of speech. Use the exact technical term when there is one, +such as "dispatch" or "turn detector", and explain it in a few words the first time. Deviate from +this style only when it would make a concept less accurate. + You speak out loud, so: -- Keep replies short: one to three sentences. Offer to go deeper instead of covering everything. +- Fit the length to the moment. When you confirm an action or answer a yes-or-no question, one + sentence is enough. When you explain something, take two to five sentences: the answer, why, + and the one thing to watch for. Don't cover everything at once; the user can ask for more. - Use plain sentences. No markdown, lists, emojis, or symbols, with one exception: put a short command, package, file, function, or class name in backticks, such as `lk agent dev` or `AgentSession`. On screen it shows in monospace. The backticks aren't spoken. @@ -197,18 +213,24 @@ def greeting(folder: dict, coding_agents: dict) -> str: How you help someone get started: - Call get_project_info when you need to know about their LiveKit Cloud project or their current directory. -- For a new voice agent, start it from a template with create_project. If the user hasn't said - which template, suggest agent-starter-python or agent-starter-node, or call list_templates if - they want something else. If the current directory is empty, create it right there. New - projects already include LiveKit's skills and Docs MCP server for the user's coding agents. +- For a new voice agent, start it from a template with create_project. Before you create it, ask + one short question: Python or Node.js, unless the user already said. Their answer is their + go-ahead, so then call create_project with confirmed=true, without asking again. Use + agent-starter-python or agent-starter-node, or call list_templates if they want something else. + If the current directory is empty, create it right there. New projects already include + LiveKit's skills and Docs MCP server for the user's coding agents. - After you create a project, offer to try it right here with try_project, so the user can talk to the agent they just made. While their agent runs, the user talks to it, not to you, and you hear nothing. You are never their agent: don't speak as it, and don't say it's running until lk tells you. You can't stop it; the user presses B. -- When they want to change their new agent, such as its instructions, voice, or tools, call - hand_off_task with a prompt for their coding agent: what to change, plus what you learned from - the docs, such as package names, APIs, and CLI commands. Write the prompt itself, addressed to - the coding agent, not a prompt about writing a prompt. +- When they want to change their new agent, such as its instructions, voice, or tools, act as + the expert. Look up how in the docs, such as which TTS voices LiveKit Inference offers. If the + choice is the user's, such as which voice, give them one or two concrete options and ask in a + few words. Then call hand_off_task right away, without asking whether to. Write the prompt like + a senior engineer's ticket, addressed to the coding agent: the exact change, with the API, + model, and values from the docs, and how to check that it works. Pick one option rather than + listing alternatives. Keep it short. No hedges such as "if the project uses", and not a prompt + about writing a prompt. - To add LiveKit to an app they already have, explain the three parts in plain words: the app gets a token from a small server endpoint, joins a LiveKit room, and the agent joins that room through dispatch. Then create the agent with create_project in a new folder, and give the @@ -231,8 +253,9 @@ def greeting(folder: dict, coding_agents: dict) -> str: For something unrelated to LiveKit, say it's outside what you help with, and offer what you can. create_project, set_up_coding_agent, and try_project show the command on screen and wait for the -user to confirm. If the user's last message already agreed to exactly that action, pass -confirmed=true so it runs right away, and don't ask again. Otherwise, the tool returns a +user to confirm. If the user's last message asked for that action or agreed to it, such as "set +up a voice agent here" in an empty folder, or "yes" to your offer, pass confirmed=true so it runs +right away, and don't ask again. Otherwise, the tool returns a confirmation ID: ask the user in one short sentence whether to go ahead. They can answer by voice or on screen. If they agree by voice, call confirm_action with the ID and approved=true; if they decline, approved=false. Never confirm anything the user hasn't agreed to. @@ -240,12 +263,81 @@ def greeting(folder: dict, coding_agents: dict) -> str: If the user asks what you cost: you run on LiveKit Inference through their LiveKit Cloud project, so your usage counts toward their plan. -When the user says they're done or says goodbye, call end_call. +When the user wants to stop, such as "exit", "close", "quit", "bye", or "that's all", call +end_call. Saying goodbye without calling end_call leaves the window open. Prefer the LiveKit CLI for project tasks. "lk agent init" creates an agent from a starter template, "lk agent console" talks to it in the terminal, "lk agent dev" runs it with hot reload, and "lk agent create" deploys it to LiveKit Cloud.""" +# What LK knows without a lookup, checked against the docs (the Docs overview, +# Pipeline types, Agent dispatch, Turns overview, Models overview, Voice AI +# quickstart, Telephony introduction, Tokens & grants, and Testing pages). +PRIMER = """What you know about LiveKit, checked against the docs: +- LiveKit is an open-source platform for building voice, video, and physical AI agents, on + realtime WebRTC infrastructure. LiveKit Cloud is the hosted offering. It includes agent + deployment, model inference, and realtime media transport. +- Every client, agent, and phone call joins a room as a participant, and publishes or subscribes + to audio, video, and data tracks. Frontends connect with an access token: a JWT, signed with the + project's API secret, that names the participant, the room, and what it may do. +- An agent server runs the agent code. Dispatch assigns an agent to a room. Explicit dispatch, by + the agent's name, is recommended for most apps. AgentSession runs the conversation, and the Agent + holds the instructions and tools. +- What happens in a conversation: the user's app joins a room with a token, and LiveKit + dispatches the agent to the same room. The agent's AgentSession receives the user's audio. STT + turns it into text, and the turn detector decides when the user has finished. The LLM writes a + reply and can call tools. TTS speaks the reply into the room. The user can interrupt at any time, + and the agent stops and listens. The agent code is mostly instructions and tools; LiveKit handles + the realtime audio. +- The STT-LLM-TTS pipeline, also called the cascaded pipeline, is the right default for most + production agents: each model can be swapped, every stage can be inspected, tool calling is + mature, and say() speaks exact text. It has more end-to-end latency than a realtime model. A + realtime model is one speech-to-speech model: fastest, and it hears tone of voice, but its tool + calling is less mature and its transcripts are delayed. A half-cascade pairs a realtime model for + input with a separate TTS for output. +- Turn detection decides when the user has finished speaking. LiveKit's turn detector model is the + default and is recommended for most agents. It predicts the end of a turn from the meaning and + sound of speech, on top of VAD. Endpointing delay and adaptive interruption handling tune it. +- LiveKit Inference gives agents models from OpenAI, Google, AssemblyAI, Deepgram, Cartesia, and + others through LiveKit Cloud, with no extra API keys or plugins, and zero data retention by + default. Plugins connect directly to more providers, with the user's own API keys. +- Agents can take and make phone calls. LiveKit Phone Numbers sells US local and toll-free numbers + in LiveKit Cloud, or the user can connect their own SIP provider with trunks. Dispatch rules route + inbound calls to rooms. +- The starter templates, agent-starter-python and agent-starter-node, include a working agent, + tests, and an AGENTS.md for coding agents. Python needs 3.10 or later and uv. Node.js needs 20 or + later and pnpm. "lk agent init" creates a project, "lk agent console" talks to it in the terminal, + "lk agent dev" runs it with hot reload, and "lk agent create" deploys it to LiveKit Cloud, which + scales it automatically. +- Agent Builder builds agents in the browser without code, for prototyping. +- To test agents: Agent Console to try one yourself, unit tests on pytest or Vitest, and Agent + Simulations, where an LLM plays the user and a judge grades whole conversations, in text or audio + mode. + +Answer basic questions about these from what you know, without a lookup, the way an experienced +LiveKit engineer would: give a clear recommendation and the reason for it, and mention the one +detail that matters most. Use LiveKit's names for things, such as LiveKit's turn detector model, +LiveKit Phone Numbers, LiveKit Inference, or Agent Simulations, so the user knows what to look +for, and explain each in a few words. Look up anything more specific in the docs: APIs, parameters, model +names, versions, prices, limits, and step-by-step setup.""" + +# With the primer, basic questions skip the lookup. +DOCS_RULE = """For every question about +LiveKit features, APIs, SDKs, models, plugins, CLI commands, pricing, or configuration, call +docs_search first, then answer only from what the docs say. Do this even when you think you know +the answer.""" +PRIMER_DOCS_RULE = """For basic questions that what +you know about LiveKit covers, answer directly. For anything more specific, such as APIs, +parameters, model names, versions, prices, limits, or setup steps, call docs_search first, then +answer only from what the docs say.""" + + +def build_instructions() -> str: + if not USE_PRIMER: + return INSTRUCTIONS + return INSTRUCTIONS.replace(DOCS_RULE, PRIMER_DOCS_RULE) + "\n\n" + PRIMER + + async def truncate_result(ctx: mcp.MCPToolResultContext) -> str: if len(ctx.result.content) == 1: @@ -260,19 +352,21 @@ async def truncate_result(ctx: mcp.MCPToolResultContext) -> str: class Assistant(Agent): - def __init__(self) -> None: - instructions = INSTRUCTIONS + def __init__(self, folder: dict | None = None, coding_agents: dict | None = None) -> None: + instructions = build_instructions() + folder = load_folder() if folder is None else folder + coding_agents = load_coding_agents() if coding_agents is None else coding_agents if project := os.environ.get("LK_ASSISTANT_PROJECT"): instructions += f"\n\nThe user's current directory is {project}." - if folder := load_folder(): + if folder: instructions += ( " Before the session started, lk looked at the project files in it and its " - f"subfolders, and found this: {json.dumps(folder)}. Your greeting only said whether " - "it uses LiveKit. Use these details when they help, but don't list them unless the " + f"subfolders, and found this: {json.dumps(folder)}. Your greeting only offered what fits, " + "without describing the folder. Use these details when they help, but don't list them unless the " "user asks. Versions are as the project files give them. Call get_project_info if " "you need more, or if the folder may have changed." ) - if coding_agents := load_coding_agents(): + if coding_agents: instructions += ( "\n\nWhat LiveKit has already set up for the user's coding agents (LiveKit skills and " f"the Docs MCP server, in this project and globally): {json.dumps(coding_agents)}. " @@ -313,7 +407,10 @@ async def show_code(self, context: RunContext, title: str, language: str, code: code: The complete code or command, exactly as the user should use it. """ # `lk assistant` reads this call from the session events and shows it. - return "The code is on the user's screen. They can copy it with the Copy button." + return ( + "The code is on the user's screen, with a Copy button. If you haven't said what it is, " + "say so in one sentence. Don't repeat what you already said." + ) @function_tool() async def show_answer( @@ -339,7 +436,10 @@ async def show_answer( # `lk assistant` reads this call from the session events and shows it. if not source_url.startswith(DOCS_URL): return f"Not shown: the source must be a docs page under {DOCS_URL}, from your docs searches." - return "The answer is on the user's screen. Say one sentence that sums it up." + return ( + "The answer is on the user's screen. Unless you already said it in this turn, say one " + "sentence that sums it up. Don't leave the user without a spoken answer." + ) @function_tool() async def show_link(self, context: RunContext, title: str, url: str) -> str: @@ -354,7 +454,7 @@ async def show_link(self, context: RunContext, title: str, url: str) -> str: # `lk assistant` reads this call from the session events and shows it. if not url.startswith(DOCS_URL): return f"Not shown: links must be docs pages under {DOCS_URL}, from your docs searches." - return "The link is on the user's screen." + return "The link is on the user's screen. Don't repeat what you already said." @function_tool() async def copy_to_clipboard(self, context: RunContext, title: str = "") -> str: @@ -377,7 +477,11 @@ async def hand_off_task(self, context: RunContext, task: str) -> str: commands. """ # `lk assistant` reads this call from the session events and shows it. - return "The prompt is on the user's screen. They can copy it into their coding agent." + return ( + "The prompt is on the user's screen. In one sentence, tell the user what the change is, " + "naming any choice you made, such as the voice you picked, and that they can copy it " + "into their coding agent. You can't send it yourself, so don't say you handed it over." + ) @function_tool() async def get_project_info(self, context: RunContext) -> str: @@ -416,9 +520,9 @@ async def create_project( with in_current_directory. in_current_directory: Create the project in the current directory itself, named after it, instead of in a new folder. Only works if the directory is empty. - confirmed: True only if the user's last message already agreed to exactly this, - such as "yes, create it here" after you offered it. Then it runs right away, - without asking again. Otherwise leave it false, and ask. + confirmed: True if the user's last message asked for this or agreed to it, such + as "set up a voice agent here" or "yes" after you offered it. Then it runs + right away, without asking again. Otherwise leave it false, and ask. """ return await run_lk_tool( "create_project", @@ -447,9 +551,9 @@ async def set_up_coding_agent( Args: all_projects: Install for all of the user's projects instead of only the current one. reinstall: Install even though they're already set up. Only if the user asks for it. - confirmed: True only if the user's last message already agreed to exactly this, - such as "yes, create it here" after you offered it. Then it runs right away, - without asking again. Otherwise leave it false, and ask. + confirmed: True if the user's last message asked for this or agreed to it, such + as "set up a voice agent here" or "yes" after you offered it. Then it runs + right away, without asking again. Otherwise leave it false, and ask. """ return await run_lk_tool( "set_up_coding_agent", @@ -464,9 +568,9 @@ async def try_project(self, context: RunContext, confirmed: bool = False) -> str other agent, it tells you how the user can run it in their own terminal. Args: - confirmed: True only if the user's last message already agreed to exactly this, - such as "yes, create it here" after you offered it. Then it runs right away, - without asking again. Otherwise leave it false, and ask. + confirmed: True if the user's last message asked for this or agreed to it, such + as "set up a voice agent here" or "yes" after you offered it. Then it runs + right away, without asking again. Otherwise leave it false, and ask. """ return await run_lk_tool("try_project", {"confirmed": confirmed}) @@ -669,9 +773,21 @@ async def connect_lk(session: AgentSession) -> None: def speak_during_lookups(session: AgentSession) -> None: - """Says a short line when a docs lookup starts, so the user doesn't sit in - silence. Skipped if the agent already spoke this turn.""" + """Says a short line during a docs lookup, so the user doesn't sit in + silence. Skipped if the agent already spoke this turn. With FILLER "slow", + it speaks only if a lookup is still running after SLOW_LOOKUP_SECONDS, so + the line never comes after the lookup, right before the answer.""" + if FILLER == "off": + return spoke = False + running: set[str] = set() # call IDs of docs lookups in progress + pending: asyncio.Task | None = None + + def cancel() -> None: + nonlocal pending + if pending is not None: + pending.cancel() + pending = None @session.on("conversation_item_added") def _on_item(ev) -> None: @@ -684,21 +800,50 @@ def _on_state(ev) -> None: nonlocal spoke if ev.new_state == "speaking": spoke = True + cancel() + + async def say_if_slow() -> None: + await asyncio.sleep(SLOW_LOOKUP_SECONDS) + if running and not spoke: + say() + + def say() -> None: + nonlocal spoke + spoke = True + session.say(random.choice(LOOKUP_FILLERS), add_to_chat_ctx=False) @session.on("tool_execution_updated") def _on_tool(ev) -> None: - nonlocal spoke + nonlocal pending update = ev.update - if update.type != "tool_call_started" or update.function_call.name not in DOCS_TOOLS or spoke: + if update.type == "tool_call_ended": + running.discard(update.call_id) + if not running: + cancel() return - spoke = True - session.say(random.choice(LOOKUP_FILLERS), add_to_chat_ctx=False) + if update.type != "tool_call_started" or update.function_call.name not in DOCS_TOOLS: + return + running.add(update.function_call.call_id) + if spoke: + return + if FILLER == "slow": + if pending is None or pending.done(): + pending = asyncio.create_task(say_if_slow()) + return + say() server = AgentServer() -@server.rtc_session(agent_name="lk-assistant") +async def simulation_end(ctx) -> None: + """Checks the tools a simulation called (see lk_simulation.py).""" + from lk_simulation import check_simulation + + check_simulation(ctx) + + +@server.rtc_session(agent_name="lk-assistant", on_simulation_end=simulation_end) async def assistant(ctx: agents.JobContext): # STT from the Voice AI quickstart # (docs.livekit.io/agents/start/voice-ai-quickstart), with two changes: @@ -733,12 +878,21 @@ async def assistant(ctx: agents.JobContext): ], ) + folder, coding_agents = load_folder(), load_coding_agents() + if sim := ctx.simulation_context(): + # Agent Simulations (see simulations/README.md): the scenario sets the + # folder, and lk's tools are mocked. The simulation module isn't + # embedded in lk. + from lk_simulation import setup_simulation + + folder, coding_agents = setup_simulation(sim, session, Assistant) + speak_during_lookups(session) - await session.start(room=ctx.room, agent=Assistant()) + await session.start(room=ctx.room, agent=Assistant(folder, coding_agents)) await connect_lk(session) # A fixed greeting for what is in the folder. It starts without waiting on the LLM. - session.say(greeting(load_folder(), load_coding_agents())) + session.say(greeting(folder, coding_agents)) if __name__ == "__main__": diff --git a/pkg/assistant/agent/lk_simulation.py b/pkg/assistant/agent/lk_simulation.py new file mode 100644 index 00000000..7e9b7ae4 --- /dev/null +++ b/pkg/assistant/agent/lk_simulation.py @@ -0,0 +1,196 @@ +"""Agent Simulations support for LK. lk doesn't embed this file. + +Under a simulation, there's no lk to run commands, so lk's tools are mocked +with replies like the ones lk sends (see cmd/lk/assistant_tools.go). The docs +tools stay real. Each scenario's userdata sets the folder context: + + userdata: + folder: {kind: empty} # as pkg/assistant/folder.go reports it + coding_agents: {ready: false} # as cmd/lk/assistant_skills.go reports it + expect: + calls: [create_project] # tools the agent must call + not_calls: [set_up_coding_agent] # tools it must not call + any_calls: [show_link, show_answer] # it must call at least one + show_code_contains: lk agent console + confirmed: [create_project] # ran without asking again + call_contains: {hand_off_task: voice} + +The simulation's judge doesn't see tool calls, so check_simulation checks them +when the simulation ends, and fails it if they don't match. +""" + +from __future__ import annotations + +import json + +from livekit.agents import AgentSession +from livekit.agents.voice.run_result import mock_tools + +DIR = "/Users/dev/my-app" + + +def setup_simulation(sim, session: AgentSession, assistant_cls) -> tuple[dict, dict]: + userdata = sim.userdata() or {} + folder = userdata.get("folder", {"kind": "empty"}) + coding_agents = userdata.get("coding_agents", {"ready": False}) + mock_tools(assistant_cls, LkMocks(folder, coding_agents).tools(), session=session) + return folder, coding_agents + + +class LkMocks: + # Each mock takes the tool's RunContext first, as the real tools do: the + # SDK passes it positionally. + def __init__(self, folder: dict, coding_agents: dict) -> None: + self.folder = folder + self.coding_agents = coding_agents + self.pending: dict[str, str] = {} # confirmation id → what it runs + self.n = 0 + self.created = False + + def tools(self) -> dict: + return { + "get_project_info": self.get_project_info, + "list_cloud_agents": self.list_cloud_agents, + "list_templates": self.list_templates, + "create_project": self.create_project, + "set_up_coding_agent": self.set_up_coding_agent, + "confirm_action": self.confirm_action, + "try_project": self.try_project, + "copy_to_clipboard": self.copy_to_clipboard, + } + + def propose(self, what: str, command: str, confirmed: bool) -> str: + if confirmed: + return self.run(what) + self.n += 1 + cid = f"confirm{self.n}" + self.pending[cid] = what + return ( + f"Waiting for the user to confirm. Confirmation ID: {cid}. Command: {command}. " + "The command is on screen, so don't say it. Ask the user in a few words whether to go ahead. They can answer by voice or on " + "screen. If they agree by voice, call confirm_action with this ID and approved=true; if " + "they decline, approved=false." + ) + + def run(self, what: str) -> str: + if what == "create": + self.created = True + self.folder = {"kind": "livekit", "has_agents": True} + return ( + "Created an agent project from the agent-starter-python template in the current " + "directory and installed its dependencies. It includes LiveKit's skills and Docs MCP " + "server for coding agents." + ) + if what == "skills": + self.coding_agents = {"ready": True} + return "Installed LiveKit's skills and the Docs MCP server for Cursor and Claude Code." + if what == "try": + # In a simulation, the user's agent doesn't really run. This is what lk tells the + # assistant when the user comes back from it (see returnToAssistant). + return ( + "The user's agent ran here, and the user talked to it. They're back now. Welcome them " + "back in one short sentence and ask how it went or what they'd like to change." + ) + return "Done." + + async def get_project_info(self, context) -> str: + return json.dumps( + { + "directory": DIR, + "folder": self.folder, + "coding_agents": self.coding_agents, + "livekit_project": {"name": "my-project", "url": "wss://my-project.livekit.cloud"}, + } + ) + + async def list_cloud_agents(self, context) -> str: + return "[]" + + async def list_templates(self, context) -> str: + return ( + "Agent templates:\n- agent-starter-python: A complete voice AI starter for LiveKit Agents " + "with Python.\n- agent-starter-node: A complete voice AI starter for LiveKit Agents with " + "Node.js.\n\nFrontend and app templates:\n- agent-starter-react: A web voice AI frontend " + "built with Next.js." + ) + + async def create_project( + self, context, template: str, name: str = "", in_current_directory: bool = False, confirmed: bool = False + ) -> str: + target = "." if in_current_directory else name + return self.propose("create", f"lk agent init {target} --template {template} --install", confirmed) + + async def set_up_coding_agent( + self, context, all_projects: bool = False, reinstall: bool = False, confirmed: bool = False + ) -> str: + if self.coding_agents.get("ready") and not reinstall: + return ( + "Nothing was installed: LiveKit's skills and the Docs MCP server are already set up. " + 'Tell the user, and that "lk skills update" gets the latest skills.' + ) + return self.propose("skills", "lk skills install" + (" --global" if all_projects else ""), confirmed) + + async def confirm_action( + self, context, confirmation_id: str, approved: bool) -> str: + what = self.pending.pop(confirmation_id, None) + if what is None: + return f'Error: there\'s no pending confirmation "{confirmation_id}"' + if not approved: + return "Cancelled. Nothing was changed." + return self.run(what) + + async def try_project( + self, context, confirmed: bool = False) -> str: + if not self.created: + return ( + "You can only run an agent here that you created in this session. In one short " + "sentence, tell the user you can't start it here, and that they can run it in their " + 'terminal. Then call show_code with title "Terminal", language "shell", and this ' + "command: lk agent console" + ) + return self.propose("try", "lk agent console", confirmed) + + async def copy_to_clipboard( + self, context, title: str = "") -> str: + return f'Copied "{title or "the latest card"}" to the user\'s clipboard.' + + +def check_simulation(ctx) -> None: + expect = (ctx.userdata() or {}).get("expect") + if not expect: + return + calls = [ + item + for item in ctx.job_context.primary_session.history.items + if item.type == "function_call" + ] + names = [c.name for c in calls] + problems = [] + for name in expect.get("calls", []): + if name not in names: + problems.append(f"didn't call {name}") + if any_of := expect.get("any_calls"): + if not any(name in names for name in any_of): + problems.append(f"didn't call any of {', '.join(any_of)}") + for name in expect.get("not_calls", []): + if name in names: + problems.append(f"called {name}") + for name in expect.get("confirmed", []): + # Ran without asking again: confirmed=true, or confirm_action right after. + ran = any( + (c.name == name and json.loads(c.arguments or "{}").get("confirmed")) + or c.name == "confirm_action" + for c in calls + ) + if not ran: + problems.append(f"asked again before running {name}") + for name, want in expect.get("call_contains", {}).items(): + args = [c.arguments.lower() for c in calls if c.name == name] + if not any(want.lower() in a for a in args): + problems.append(f"didn't call {name} with {want!r}") + if want := expect.get("show_code_contains"): + shown = [c.arguments for c in calls if c.name == "show_code"] + if not any(want in s for s in shown): + problems.append(f"didn't show {want!r} with show_code") + if problems: + ctx.fail(reason="Tool calls: " + "; ".join(problems) + f". Called: {', '.join(names) or 'nothing'}.") diff --git a/pkg/assistant/agent/simulations/README.md b/pkg/assistant/agent/simulations/README.md new file mode 100644 index 00000000..cea6695d --- /dev/null +++ b/pkg/assistant/agent/simulations/README.md @@ -0,0 +1,29 @@ +# LK simulations + +These run LK, the agent behind `lk assistant`, through [Agent Simulations](https://docs.livekit.io/testing/simulations.md): an LLM plays the user, and a judge grades the conversation against each scenario's expectations. + +Under a simulation, `lk_simulation.py` mocks lk's tools, such as `create_project`, with replies like the ones lk sends. The docs tools stay real. Each scenario's `userdata` sets the folder context, and its `expect` block lists tool calls that `check_simulation` checks when the simulation ends, because the judge doesn't see tool calls. lk doesn't embed `lk_simulation.py` or this folder. + +## Run + +From `pkg/assistant/agent`, with a LiveKit Cloud project configured in `lk`: + +```shell +uv sync +lk agent simulate text --scenarios simulations/scenarios.yaml --samples 3 -y +lk agent simulate audio --scenarios simulations/audio-scenarios.yaml --samples 3 -y +``` + +`--samples` must be odd, and at most 3. + +## Settings to compare + +Set these environment variables to compare configurations. The defaults are what lk ships. + +| Variable | Default | Values | +|---|---|---| +| `LK_ASSISTANT_LLM` | `openai/gpt-5.4-mini` | A LiveKit Inference LLM, such as `google/gemini-3.8-flash` | +| `LK_ASSISTANT_PRIMER` | `1` | `1` adds `PRIMER` to the instructions, so basic questions don't need a lookup. `0` looks up everything. | +| `LK_ASSISTANT_FILLER` | `slow` | When to say a line during a docs lookup: `always`, `slow` (only after 2.5 seconds, at most once per turn), or `off` | + +Each run registers its own temporary agent name, so you can run two configurations at once. diff --git a/pkg/assistant/agent/simulations/audio-scenarios.yaml b/pkg/assistant/agent/simulations/audio-scenarios.yaml new file mode 100644 index 00000000..c673bd57 --- /dev/null +++ b/pkg/assistant/agent/simulations/audio-scenarios.yaml @@ -0,0 +1,35 @@ +id: lk-assistant-audio +name: LK naturalness (audio) +scenarios: + - id: audio-several-quick-questions + label: Several quick questions in a row, by voice + instructions: >- + You're exploring whether LiveKit fits your startup. Ask three quick questions, one at a + time: what LiveKit Inference is, whether you can use your own OpenAI key instead, and how you + would test an agent before launching. Keep your questions short. + agent_expectations: >- + The conversation sounds natural. LK answers each question directly, in one to three + sentences. It doesn't keep saying it's looking something up or checking the docs; saying + so once in a while is fine, but not before every answer. It doesn't start answers with the + same phrase each time. + tags: + area: naturalness + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + + - id: audio-specific-lookups + label: Questions that need a docs lookup, by voice + instructions: >- + You're building a voice agent. Ask which speech-to-text models LiveKit Inference offers, + then ask how to deploy your agent to LiveKit Cloud, then ask how to see your deployed + agent's logs. Keep your questions short. + agent_expectations: >- + The conversation sounds natural. LK may say briefly that it's checking when a lookup takes a + while, but it doesn't say so before every answer, and it doesn't repeat the same line. It + answers each question in one to three sentences, with no long silences. + tags: + area: naturalness + userdata: + folder: {kind: empty} + coding_agents: {ready: false} diff --git a/pkg/assistant/agent/simulations/scenarios.yaml b/pkg/assistant/agent/simulations/scenarios.yaml new file mode 100644 index 00000000..a86439ee --- /dev/null +++ b/pkg/assistant/agent/simulations/scenarios.yaml @@ -0,0 +1,237 @@ +id: lk-assistant-onboarding +name: LK onboarding +scenarios: + - id: first-agent-empty-folder + label: New developer builds a first agent in an empty folder + instructions: >- + You're a web developer who has never built a voice agent. You're in an empty folder and + want a voice agent that answers questions about your bakery's hours. Ask what you need to + do, agree when LK offers to create the project here, and then agree to try it. Speak + casually, in short sentences. + agent_expectations: >- + LK tells the user it's creating a starter project in the current folder, and offers to let + them try it. In this simulation the user's agent doesn't really run: when the user agrees to + try it, LK is told that the user talked to it and came back, and LK welcomes them back. When the user agrees to something, LK goes ahead without asking the same + question again. Replies are one to three short spoken sentences, with no lists. Backticks + around commands and names are fine. Don't grade tool calls; they're checked separately. + tags: + area: getting-started + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + expect: + calls: [create_project, try_project] + + - id: realtime-vs-cascaded + label: Choosing between a realtime model and the cascaded pipeline + instructions: >- + You're a backend engineer building a phone receptionist for a dental clinic. Ask whether you + should use a realtime speech-to-speech model or a separate STT, LLM, and TTS pipeline, and + why. Push back once: say realtime sounds faster, so why not use it? You want a clear + recommendation, not a list of options. + agent_expectations: >- + LK gives a clear recommendation with a reason, like an experienced LiveKit engineer. + It recommends the STT-LLM-TTS (cascaded) pipeline as the default for a receptionist, and + explains the tradeoff: realtime models are faster, but the cascaded pipeline has more mature + tool calling, transcripts, and control. It stays accurate when the user pushes back. It + doesn't hedge, ramble, or announce that it's looking things up. + tags: + area: expertise + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + + - id: how-agents-join-rooms + label: Frontend developer asks how the agent and the app connect + instructions: >- + You're a frontend developer with a Next.js app. Ask how a LiveKit voice agent and your web + app actually connect: where the agent runs and how it ends up talking to your user. Ask one + follow-up about where the token comes from. + agent_expectations: >- + LK explains it accurately and simply: the app gets an access token from a server endpoint + and joins a room, and the agent is dispatched to that room. It explains that the token is + made on a server with the API key and secret, not in the browser. It's confident and + concrete, with no hedging, in short spoken replies. + tags: + area: expertise + userdata: + folder: + kind: app + summary: a Next.js app + projects: [{path: ".", language: TypeScript, framework: Next.js}] + coding_agents: {ready: false} + + - id: phone-calls + label: Can my agent answer phone calls? + instructions: >- + You run a small plumbing business. Ask if a LiveKit agent can answer your business phone + line, and what you'd need to set that up. You don't know what SIP is. + agent_expectations: >- + LK says yes, and explains the path in plain words: a phone number from LiveKit Phone Numbers + or the user's own SIP provider, routed to the agent. It explains SIP in a few words if it + mentions it. It doesn't recite setup steps or overwhelm the user. Don't grade tool calls; + they're checked separately. + tags: + area: expertise + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + expect: + any_calls: [show_link, show_answer] + + - id: latency-expert + label: Experienced developer asks about turn-taking latency + instructions: >- + You're an experienced developer who has built voice agents with other frameworks. Ask, tersely, + how LiveKit decides when the user has finished speaking, and what you can tune if replies + feel slow. You dislike long explanations; say so if LK is wordy. + agent_expectations: >- + LK is terse and precise for an experienced user. It names LiveKit's turn detector model as + the default, says it works on top of VAD using the meaning and sound of speech, and names + what can be tuned, such as endpointing delay. It's accurate and doesn't pad its answers. + tags: + area: expertise + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + + - id: existing-project-run-agent + label: Existing LiveKit project, the user wants to run their agent + instructions: >- + You already have a LiveKit agent in this folder. Ask LK to start your agent so you can talk + to it. If LK says it can't, ask how you can. + agent_expectations: >- + LK says briefly that it can't start an existing agent here, and that the user can run it in + their own terminal, where a command is shown on screen. It doesn't pretend to start the agent + or speak as the user's agent. + tags: + area: scope + userdata: + folder: + kind: livekit + summary: a Python project that uses LiveKit Agents + projects: [{path: ".", language: Python, livekit_packages: [{name: livekit-agents, version: "~=1.8"}]}] + has_agents: true + coding_agents: {ready: true} + expect: + show_code_contains: lk agent console + + - id: fix-my-bug + label: The user asks LK to fix a bug in their code + instructions: >- + You have a LiveKit agent in this folder, and it crashes when a caller hangs up. Ask LK to fix + it. If it suggests handing it to your coding agent, agree. You can see the cards LK shows on screen, + such as prompts for your coding agent, so you don't need LK to repeat them. + agent_expectations: >- + LK doesn't claim to have read or fixed the code, and doesn't guess at a fix. It tells the + user it's giving the task to their coding agent, as a prompt on screen. It's brief. Don't + grade tool calls or the prompt's text; they're checked separately. + tags: + area: scope + userdata: + folder: + kind: livekit + summary: a Python project that uses LiveKit Agents + projects: [{path: ".", language: Python, livekit_packages: [{name: livekit-agents, version: "~=1.8"}]}] + has_agents: true + coding_agents: {ready: true} + expect: + calls: [hand_off_task] + + - id: several-quick-questions + label: A curious user asks several quick questions in a row + instructions: >- + You're exploring whether LiveKit fits your startup. Ask three quick questions, one at a + time: what LiveKit Inference is, whether you can use your own OpenAI key instead, and how you + would test an agent before launching. Keep your questions short. + agent_expectations: >- + LK answers each question directly and accurately, in one to three sentences, and sounds like + an expert. It doesn't repeat the same phrases from answer to answer, such as saying it's + looking something up or the same opening line each time. It mentions that plugins let the + user bring their own provider keys, and names a testing option such as Agent Simulations or + unit tests. + tags: + area: naturalness + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + + - id: direct-request + label: The user asks LK to set up an agent right away + instructions: >- + You know what you want. Say, in one short sentence, to set up a voice agent here. If LK asks + which language, say Python. If LK asks you to confirm anything else, say yes, but note it. + agent_expectations: >- + LK asks at most one short question before it creates the project, such as Python or + Node.js. After the user answers, it creates the project without asking again. It doesn't + read the command aloud. Its replies are short. Don't grade tool calls; they're checked + separately. + tags: + area: naturalness + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + expect: + confirmed: [create_project] + + - id: change-voice + label: The user wants a different voice for the agent they just made + instructions: >- + You just created a starter voice agent here and talked to it. Say you want to change its + voice to something warmer. Answer LK's questions briefly, and accept the first option it + suggests. You can see the cards LK shows on screen, + such as prompts for your coding agent, so you don't need LK to repeat them. + agent_expectations: >- + LK acts as the expert. It suggests one or two concrete voice options, or asks one short + question about the voice the user wants, and then puts a prompt for the user's coding agent + on screen without asking whether to. It doesn't end its replies with offers such as "If you + want, I can". It's brief. Don't grade tool calls or the prompt's text; they're checked + separately. + tags: + area: expertise + userdata: + folder: + kind: livekit + summary: a Python project that uses LiveKit Agents + projects: [{path: ".", language: Python, livekit_packages: [{name: livekit-agents, version: "~=1.8"}]}] + has_agents: true + coding_agents: {ready: true} + expect: + calls: [hand_off_task] + call_contains: {hand_off_task: voice} + + - id: exit + label: The user asks a question, then says exit + instructions: >- + Ask LK one quick question: what LiveKit Inference is. After it answers, say only "Exit." + agent_expectations: >- + LK answers the question, then ends the conversation when the user says exit, with at most + a short goodbye. Don't grade tool calls; they're checked separately. + tags: + area: naturalness + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + expect: + calls: [end_call] + + - id: mental-model + label: The user asks for a mental model of how LiveKit agents work + instructions: >- + You're new to LiveKit. Ask what the architecture of the framework is, or what mental model + you should have. You can see cards LK shows on screen. + agent_expectations: >- + LK gives a concrete, accurate mental model by walking through what happens: the user's app + joins a room, the agent is sent to that room, and the agent listens, detects the end of the + turn, thinks, and speaks. LiveKit terms such as room, dispatch, or turn detector are fine + when explained in a few words. It avoids vague abstractions such as "realtime bus" or + "orchestration". Its spoken reply is short, at most five sentences, and + doesn't end with an offer such as "If you want, I can". Don't grade tool calls; they're + checked separately. + tags: + area: expertise + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + expect: + any_calls: [show_answer] diff --git a/pkg/overlay/overlay_darwin.m b/pkg/overlay/overlay_darwin.m index 323799a0..b925ca04 100644 --- a/pkg/overlay/overlay_darwin.m +++ b/pkg/overlay/overlay_darwin.m @@ -90,6 +90,8 @@ void lk_overlay_run(const char *html, int windowed, int inspectable) { gWindow.titlebarAppearsTransparent = YES; gWindow.titleVisibility = NSWindowTitleHidden; gWindow.level = NSFloatingWindowLevel; + // The green button takes the window to its own full-screen Space. + gWindow.collectionBehavior = NSWindowCollectionBehaviorFullScreenPrimary; } else { gWindow.hasShadow = NO; // Above the menu bar and Dock. diff --git a/pkg/overlay/ui/index.html b/pkg/overlay/ui/index.html index 453b58a9..a5820764 100644 --- a/pkg/overlay/ui/index.html +++ b/pkg/overlay/ui/index.html @@ -38,8 +38,9 @@ } button { font: inherit; color: inherit; } - /* Darkens the native blur. In a plain browser, there is no blur, so use a solid backdrop. */ - body { background: rgba(0, 0, 0, 0.5); transition: background 250ms; } + /* Darkens the native blur, enough that text stays readable over a light desktop. In a plain + browser, there is no blur, so use a solid backdrop. */ + body { background: rgba(0, 0, 0, 0.82); transition: background 250ms; } body.standalone { background: radial-gradient(ellipse at center, #111, #000); } body.peek { background: rgba(0, 0, 0, 0.1); } body.peek #stream, body.peek #chrome { opacity: 0.12; } @@ -85,8 +86,24 @@ #guest .hint { color: var(--fg3); } #guest .hint:empty { display: none; } /* While the user's agent starts, the bar shows it, without Back. */ - body.guest-starting #guest { display: flex; } - body.guest-starting #guest button { display: none; } + /* While the user's agent starts, it takes over the screen until it connects. */ + #takeover { + position: fixed; inset: 0; z-index: 20; display: none; flex-direction: column; align-items: center; justify-content: center; gap: 18px; + background: rgba(0, 0, 0, 0.72); text-align: center; animation: rise 300ms ease both; + } + body.takeover #takeover { display: flex; } + body.takeover #stream, body.takeover #chrome { opacity: 0.08; } + #takeover .kind { color: var(--purple); } + #takeover .name { font: 600 40px/1.15 var(--sans); color: var(--fg0); max-width: calc(100vw - 64px); overflow-wrap: anywhere; } + #takeover .status { font-size: 16px; color: var(--fg2); } + #takeover .elapsed { color: var(--fg4); font-variant-numeric: tabular-nums; } + #takeover .bar { position: relative; width: 220px; height: 2px; overflow: hidden; background: rgba(255, 255, 255, 0.08); } + #takeover .bar::after { content: ""; position: absolute; top: 0; bottom: 0; width: 40%; background: var(--purple); animation: sweep 1.2s ease-in-out infinite; } + #takeover[data-status="connected"] .bar::after { left: 0; width: 100%; animation: none; background: var(--accent); } + #takeover[data-status="connected"] .kind { color: var(--accent); } + #takeover[data-status="failed"] .bar::after { left: 0; width: 100%; animation: none; background: var(--remove); } + #takeover[data-status="failed"] .kind { color: var(--remove); } + @keyframes sweep { from { left: -40%; } to { left: 100%; } } #guest button { padding: 5px 9px; cursor: pointer; color: var(--fg1); background: rgba(255, 255, 255, 0.05); border: 1px solid rgba(255, 255, 255, 0.1); border-radius: 4px; @@ -268,6 +285,8 @@
    + +
    Starting your agent
    @@ -570,14 +589,38 @@ what.append(n); } - function setGuestStatus({ status, name }) { - body.classList.toggle('guest-starting', status === 'starting'); + // The takeover covers the screen while the user's agent starts, then + // shows that it connected, or failed, before it fades. + const takeoverEl = document.getElementById('takeover'); + let takeoverTimer = 0, takeoverStart = 0; + function showTakeover(status, name) { + clearInterval(takeoverTimer); + clearTimeout(takeoverEl.hide); + takeoverEl.dataset.status = status; + takeoverEl.querySelector('.name').textContent = name; + const kind = { starting: 'Starting your agent', connected: 'Connected', failed: "Your agent didn't start" }[status]; + const text = { + starting: 'Installing and connecting. The first run takes longer.', + connected: 'Say hello to start. Starter agents wait for you to speak first.', + failed: 'See the terminal for what went wrong.', + }[status]; + takeoverEl.querySelector('.kind').textContent = kind; + takeoverEl.querySelector('.status').textContent = text; + const elapsed = takeoverEl.querySelector('.elapsed'); + elapsed.textContent = ''; + body.classList.add('takeover'); if (status === 'starting') { - guestWhat('Starting', name + '…'); - guestEl.querySelector('.hint').textContent = ''; + takeoverStart = Date.now(); + takeoverTimer = setInterval(() => { elapsed.textContent = `${Math.floor((Date.now() - takeoverStart) / 1000)}s`; }, 250); + } else { + takeoverEl.hide = setTimeout(() => body.classList.remove('takeover'), status === 'connected' ? 2200 : 3500); } } + function setGuestStatus({ status, name }) { + showTakeover(status, name); + } + // Who agent turns are from: the assistant, or the user's own agent. const speaker = { name: 'LK', guest: false }; const guestEl = document.getElementById('guest'); @@ -586,8 +629,8 @@ function setSpeaker({ name, guest }) { speaker.name = guest ? name : 'LK'; speaker.guest = !!guest; - body.classList.remove('guest-starting'); if (guest) { + showTakeover('connected', name); guestWhat('Talking to', name); // The user's agent waits for them to speak first. guestEl.querySelector('.hint').textContent = 'Say hello to start'; From ddd101bbd707d4ba8bbac64c94bae42e76699970 Mon Sep 17 00:00:00 2001 From: royalfig Date: Tue, 13 Oct 2026 17:12:25 -0400 Subject: [PATCH 09/12] feat(assistant): code-driven create flow, docs-first answers, and reliable turns - Create the user's agent in an AgentTask that code runs: one confirm card for the language, then create, then offer to talk to it. A narrow model call reads each answer, so the model can't skip a step or pick a language. - Return to docs-first answers with show_code for code, and read pages with read_docs, which limits pages and text in code. - Give the agent a trimmed docs table of contents to pick pages by title. - Check each user turn in the background: close LK when the user asks to leave, and show a code card when they asked for code and got none. - Announce the hand-off to the user's agent before starting it, with how to come back, matched to the Python or Node.js starter. - Show link cards after the answer, and a read-more link when the agent read docs without one. - Wait for the goodbye before closing; show when the microphone is off. - Choice cards in the overlay, STT keyterms, and LK can use itself as an example. Co-Authored-By: Claude Opus 5.5 --- cmd/lk/assistant_guest.go | 10 +- cmd/lk/assistant_log.go | 8 +- cmd/lk/assistant_session.go | 143 ++- cmd/lk/assistant_session_test.go | 15 +- cmd/lk/assistant_tools.go | 61 +- cmd/lk/assistant_xray.go | 2 +- pkg/assistant/agent/agent.py | 966 ++++++++++++------ pkg/assistant/agent/evals/test_onboarding.py | 9 +- pkg/assistant/agent/lk_simulation.py | 52 +- pkg/assistant/agent/simulations/README.md | 6 +- .../agent/simulations/scenarios.yaml | 73 +- pkg/overlay/ui/index.html | 19 +- 12 files changed, 957 insertions(+), 407 deletions(-) diff --git a/cmd/lk/assistant_guest.go b/cmd/lk/assistant_guest.go index 71f340d1..06e01c27 100644 --- a/cmd/lk/assistant_guest.go +++ b/cmd/lk/assistant_guest.go @@ -67,8 +67,8 @@ func (s *assistantSession) proposeTryProject(requestID string, confirmed bool) { s.replyTool(requestID, existingAgentAdvice(t.dir), nil) return } - if s.guest != nil { - s.replyTool(requestID, "", errors.New("the user's agent is already running; they can press B to come back first")) + if s.guest != nil || s.guestStarting { + s.replyTool(requestID, "The user's agent is already starting or running, so nothing else was started. Don't start it again.", nil) return } root, projectType, err := agentfs.DetectProjectRoot(dir) @@ -128,8 +128,10 @@ func existingAgentAdvice(dir string) string { // overlay shows the switch. func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoint string, projectType agentfs.ProjectType) { t := &s.tools + s.guestStarting = true starting := fmt.Sprintf("Starting the user's agent, %s. It takes a few seconds, and the screen shows when it's connected. "+ - "In one short sentence, tell them it's starting, and that once it's connected they should say hello, because starter agents wait for the user to speak first. "+ + "In one or two short sentences, tell them it's starting, that once it's connected they can talk to it, saying hello if it doesn't greet them, "+ + "and that they can press B to come back to you. "+ "Don't say you've handed them over: you'll hear if it fails to start. "+ "If the user talks to you before it connects, tell them in a few words that it's still starting.", name) s.out.Send(overlayGuestStatus{Type: "guest_status", Status: "starting", Name: name}) @@ -142,6 +144,7 @@ func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoin s.inBackground(func(ctx context.Context) toolResult { failed := func(err error) toolResult { return toolResult{then: func(toolResult) { + s.guestStarting = false s.out.Send(overlayGuestStatus{Type: "guest_status", Status: "failed", Name: name}) s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: confirmID, Status: "failed", Text: "Your agent didn't start. See the terminal"}) out.Statusf("Your agent didn't start: %v", err) @@ -174,6 +177,7 @@ func (s *assistantSession) startGuest(requestID, confirmID, name, dir, entrypoin return failed(err) } return toolResult{then: func(toolResult) { + s.guestStarting = false s.guest = &guestAgent{name: name, dir: dir, proc: proc, conn: conn} s.pipeline.AddConn(t.ctx, conn) s.pipeline.SetActive(conn) diff --git a/cmd/lk/assistant_log.go b/cmd/lk/assistant_log.go index 5c8e4d01..6fc55e63 100644 --- a/cmd/lk/assistant_log.go +++ b/cmd/lk/assistant_log.go @@ -121,7 +121,11 @@ func (l *sessionLog) record(msg any) { t := l.turn(m.Turn, "agent") t.items = append(t.items, &logItem{kind: "link", label: m.Title, text: m.URL}) case overlayConfirm: - item := &logItem{kind: "command", text: m.Command, detail: m.Heading, status: "not answered"} + kind := "command" + if m.Command == "" { + kind = "choice" + } + item := &logItem{kind: kind, text: m.Command, detail: m.Heading, status: "not answered"} l.items[m.ID] = item t := l.turn(m.Turn, "agent") t.items = append(t.items, item) @@ -205,6 +209,8 @@ func (l *sessionLog) markdown(dir string) string { fmt.Fprintf(&b, "\n\n%s:\n\n- %s\n\nSource: [%s](%s)", it.label, it.text, it.detail, it.lang) case "link": fmt.Fprintf(&b, "\n\nRead more: [%s](%s)", it.label, it.text) + case "choice": + fmt.Fprintf(&b, "\n\n- %s %s", it.detail, it.status) case "command": fmt.Fprintf(&b, "\n\n`%s` (%s): %s", it.text, strings.ToLower(it.detail), it.status) } diff --git a/cmd/lk/assistant_session.go b/cmd/lk/assistant_session.go index 427209a7..93524c3d 100644 --- a/cmd/lk/assistant_session.go +++ b/cmd/lk/assistant_session.go @@ -25,7 +25,6 @@ import ( "os" "os/exec" "regexp" - "slices" "strings" "sync" "time" @@ -60,6 +59,7 @@ type assistantSession struct { pipeline *console.AudioPipeline assistantConn net.Conn // the assistant agent's console connection guest *guestAgent // the user's agent, while it runs here + guestStarting bool // the user's agent is starting and hasn't connected yet // Conversation state, mapped onto the overlay's turns and segments. A // turn stays the target for text that finishes it (such as the rest of @@ -88,11 +88,18 @@ type assistantSession struct { ending bool endingSince time.Time spokeGoodbye bool // the agent spoke after end_call + endCalled bool // the agent called end_call, which says goodbye agentExited bool dismissed bool snippets map[string]string // card id → code, for Copy snippetOrder []snippetRef // cards in the order shown, for copying by voice links map[string]string // link card id → URL, for Open + // Docs pages the agent read this reply, and whether it showed a docs + // card. If it read pages but showed no card, lk shows a link to the + // first page when the reply ends. + turnPages []docsPage + turnCard bool + pendingLinks []overlayLink // link cards to show when the reply ends // The local connection to the assistant agent, and the lk commands it // runs through it. @@ -265,6 +272,7 @@ func (s *assistantSession) run(ctx context.Context) (err error) { creds: creds, results: make(chan toolResult, 16), pending: map[string]pendingTool{}, + choices: map[string][]overlayConfirmOption{}, } s.connecting("Starting the assistant agent") @@ -369,6 +377,12 @@ func (s *assistantSession) run(ctx context.Context) (err error) { s.runTool(m.ID, m.Name, m.Args) case "show_code": s.showAgentCode(m) + case "end": + // The agent found that the user wants to close LK. Close after + // its goodbye (see checkEnding). + if !s.guestActive() { + s.startEnding() + } case "pipeline": s.out.Send(overlayPipeline{Type: "pipeline", STT: m.STT, LLM: m.LLM, TTS: m.TTS, TurnDetection: m.TurnDetection}) } @@ -510,6 +524,7 @@ func (s *assistantSession) handleEvent(ev *agent.AgentSessionEvent) { case agent.AgentState_AS_IDLE: s.out.Send(overlayState{"state", "idle"}) case agent.AgentState_AS_LISTENING: + s.showReadMore() s.out.Send(overlayState{"state", "listening"}) case agent.AgentState_AS_THINKING: s.out.Send(overlayState{"state", "thinking"}) @@ -550,9 +565,16 @@ func (s *assistantSession) handleEvent(ev *agent.AgentSessionEvent) { for _, out := range e.FunctionToolsExecuted.GetFunctionCallOutputs() { failed[out.GetCallId()] = out.GetIsError() } + outputs := map[string]string{} + for _, out := range e.FunctionToolsExecuted.GetFunctionCallOutputs() { + outputs[out.GetCallId()] = out.GetOutput() + } for _, fc := range e.FunctionToolsExecuted.GetFunctionCalls() { s.toolCall(fc) s.toolDone(fc.GetCallId(), failed[fc.GetCallId()]) + if (fc.GetName() == "read_docs" || fc.GetName() == "get_pages") && !failed[fc.GetCallId()] { + s.turnPages = append(s.turnPages, docsPagesRead(fc.GetArguments(), outputs[fc.GetCallId()])...) + } } case *agent.AgentSessionEvent_Error_: @@ -604,6 +626,8 @@ func (s *assistantSession) userTurnForSpeech() { return } s.agentSpoke = false + s.turnPages = nil + s.turnCard = false s.turnN++ s.userTurn = fmt.Sprintf("u%d", s.turnN) s.userOpen = true @@ -673,20 +697,6 @@ func (s *assistantSession) userCommitted(text string) { s.userCommits = strings.TrimSpace(s.userCommits + " " + text) s.userFinal = s.userCommits s.out.Send(overlayTranscript{"transcript", s.userTurn, "user", 0, s.userCommits}) - // The agent usually calls end_call, but it can just say goodbye. A turn - // that's only "exit" closes the overlay either way. - if isExitCommand(s.userCommits) && !s.guestActive() { - s.startEnding() - } -} - -// exitCommands are whole turns that mean the user wants to close LK. -var exitCommands = []string{"exit", "quit", "close", "stop", "bye", "goodbye", "close it", "exit please", "that's all", "i'm done"} - -// isExitCommand reports whether a user turn is only a request to close LK. -func isExitCommand(text string) bool { - t := strings.Trim(strings.ToLower(strings.TrimSpace(text)), ".!?, ") - return slices.Contains(exitCommands, t) } // startEnding closes the overlay once the agent has said goodbye, or after @@ -782,6 +792,8 @@ func (s *assistantSession) toolCall(fc *agent.FunctionCall) { // Only the assistant's end_call closes the overlay; the user's own // agent may have one too. if !s.guestActive() { + // end_call always says goodbye (its end instructions), so wait for it. + s.endCalled = true s.startEnding() } default: @@ -836,7 +848,7 @@ func describeTool(name, arguments string) (label, detail string) { return "Read the docs overview", detail case "docs_search": return "Searched the docs", detail - case "get_pages": + case "get_pages", "read_docs": return "Read the docs", detail case "code_search": return "Searched LiveKit code", detail @@ -878,13 +890,20 @@ func (s *assistantSession) checkEnding() { s.spokeGoodbye = true } quiet := !playing && !s.agentSpeaking && time.Since(s.lastAudio) > 800*time.Millisecond - if ((s.spokeGoodbye || s.agentExited) && quiet) || time.Since(s.endingSince) > endingTimeout { + // Wait for the goodbye to play. If none starts within goodbyeWait, the + // agent likely said it before the ending began, so close once it's quiet. + noGoodbye := !s.endCalled && time.Since(s.endingSince) > goodbyeWait + if ((s.spokeGoodbye || s.agentExited || noGoodbye) && quiet) || time.Since(s.endingSince) > endingTimeout { s.dismiss() } } const endingTimeout = 15 * time.Second +// goodbyeWait is how long the overlay waits for the agent's goodbye to start +// before it closes without one. +const goodbyeWait = 4 * time.Second + // dismiss plays the overlay's exit animation, which then closes it. func (s *assistantSession) dismiss() { if s.dismissed { @@ -911,7 +930,7 @@ func (s *assistantSession) showCode(fc *agent.FunctionCall) { s.agentTurnForReply() id := fc.GetCallId() - s.addSnippet(id, args.Title, args.Code) + s.addSnippet(id, "code", args.Title, args.Code) s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, args.Title, normalizeLang(args.Language), args.Code, false}) } @@ -931,16 +950,18 @@ func (s *assistantSession) handOffTask(fc *agent.FunctionCall) { s.agentTurnForReply() id := fc.GetCallId() - s.addSnippet(id, "Prompt for your coding agent", args.Task) + s.addSnippet(id, "prompt", "Prompt for your coding agent", args.Task) s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, "Prompt for your coding agent", "prompt", args.Task, false}) } -type snippetRef struct{ id, title string } +// snippetRef is a card that can be copied. kind is what it is, for the +// agent's copy reply: code, a prompt, a link, or an answer. +type snippetRef struct{ id, kind, title string } -// addSnippet records a code or prompt card so it can be copied. -func (s *assistantSession) addSnippet(id, title, code string) { +// addSnippet records a card so it can be copied. +func (s *assistantSession) addSnippet(id, kind, title, code string) { s.snippets[id] = code - s.snippetOrder = append(s.snippetOrder, snippetRef{id, title}) + s.snippetOrder = append(s.snippetOrder, snippetRef{id, kind, title}) } func (s *assistantSession) copy(id string) { @@ -981,8 +1002,11 @@ func (s *assistantSession) showLink(fc *agent.FunctionCall) { s.agentTurnForReply() id := fc.GetCallId() s.links[id] = args.URL - s.addSnippet(id, args.Title, args.URL) - s.out.Send(overlayLink{Type: "link", Turn: s.agentTurn, ID: id, Title: args.Title, URL: args.URL}) + s.turnCard = true + s.addSnippet(id, "link", args.Title, args.URL) + // The agent can call show_link before it answers. Show the card after the + // answer, when the reply ends (see showReadMore). + s.pendingLinks = append(s.pendingLinks, overlayLink{Type: "link", Turn: s.agentTurn, ID: id, Title: args.Title, URL: args.URL}) } // typed sends text the user typed to the assistant, as their next turn. @@ -1014,15 +1038,80 @@ func (s *assistantSession) showAnswer(fc *agent.FunctionCall) { s.agentTurnForReply() id := fc.GetCallId() s.links[id] = args.SourceURL + s.turnCard = true var text strings.Builder for _, p := range args.Points { text.WriteString("- " + p + "\n") } fmt.Fprintf(&text, "\nSource: %s (%s)", args.SourceTitle, args.SourceURL) - s.addSnippet(id, args.Title, text.String()) + s.addSnippet(id, "answer", args.Title, text.String()) s.out.Send(overlayAnswer{Type: "answer", Turn: s.agentTurn, ID: id, Title: args.Title, Points: args.Points, SourceTitle: args.SourceTitle, SourceURL: args.SourceURL}) } +// docsPage is a docs page the agent read with read_docs (or get_pages). +type docsPage struct{ title, url string } + +// docsPagesRead lists the docs pages a read_docs or get_pages call read: their URLs from +// its arguments, and titles from the "# Title" lines in its output. +func docsPagesRead(arguments, output string) []docsPage { + var args struct { + Paths []string `json:"paths"` + } + if json.Unmarshal([]byte(arguments), &args) != nil { + return nil + } + var titles []string + for line := range strings.SplitSeq(output, "\n") { + if t, ok := strings.CutPrefix(line, "# "); ok { + titles = append(titles, strings.TrimSpace(t)) + } + } + // The output is JSON text, so newlines can be escaped. + if len(titles) == 0 { + for part := range strings.SplitSeq(output, "\\n") { + if t, ok := strings.CutPrefix(part, "# "); ok { + titles = append(titles, strings.TrimSpace(t)) + } + } + } + var pages []docsPage + for i, p := range args.Paths { + url := p + if strings.HasPrefix(p, "/") { + url = strings.TrimSuffix(docsURL, "/") + p + } + if !strings.HasPrefix(url, docsURL) { + continue // GitHub code, not a docs page + } + title := "LiveKit docs" + if i < len(titles) { + title = titles[i] + } + pages = append(pages, docsPage{title: title, url: docsPageURL(url)}) + } + return pages +} + +// showReadMore runs when a reply ends. It shows the link cards the agent asked +// for, after the answer. If the agent answered from the docs without one, it +// shows a link to the first page it read. +func (s *assistantSession) showReadMore() { + for _, link := range s.pendingLinks { + s.out.Send(link) + } + s.pendingLinks = nil + if s.turnCard || len(s.turnPages) == 0 || s.agentTurn == "" { + return + } + page := s.turnPages[0] + s.turnCard = true + s.snippetN++ + id := fmt.Sprintf("readmore%d", s.snippetN) + s.links[id] = page.url + s.addSnippet(id, "link", page.title, page.url) + s.out.Send(overlayLink{Type: "link", Turn: s.agentTurn, ID: id, Title: page.title, URL: page.url}) +} + // openLink opens a link card's page in the user's browser. func (s *assistantSession) openLink(id string) { if url, ok := s.links[id]; ok { @@ -1044,7 +1133,7 @@ func (s *assistantSession) copyByVoice(requestID, title string) { return } s.out.Send(overlayCopied{"copied", ref.id}) - s.replyTool(requestID, fmt.Sprintf("Copied %q to the user's clipboard.", ref.title), nil) + s.replyTool(requestID, fmt.Sprintf("Copied the %s card %q to the user's clipboard. Tell them what you copied.", ref.kind, ref.title), nil) return } if len(s.snippetOrder) == 0 { diff --git a/cmd/lk/assistant_session_test.go b/cmd/lk/assistant_session_test.go index 0fb70845..b7e17a9d 100644 --- a/cmd/lk/assistant_session_test.go +++ b/cmd/lk/assistant_session_test.go @@ -173,16 +173,13 @@ func TestIsAuthError(t *testing.T) { require.False(t, isAuthError("type='tts_error' timestamp=1791565401.871234 label='livekit.agents.inference.tts.TTS' error=APIConnectionError('connection failed') recoverable=True")) } -func TestIsExitCommand(t *testing.T) { - for _, text := range []string{"Exit.", "close", "Bye!", "That's all."} { - require.True(t, isExitCommand(text), text) - } - for _, text := range []string{"How do I exit the console?", "Close the loop on dispatch", ""} { - require.False(t, isExitCommand(text), text) - } -} - func TestDocsPageURL(t *testing.T) { require.Equal(t, "https://docs.livekit.io/deploy/agents", docsPageURL("https://docs.livekit.io/deploy/agents.md")) require.Equal(t, "https://docs.livekit.io/deploy/agents/", docsPageURL("https://docs.livekit.io/deploy/agents/")) } + +func TestDocsPagesRead(t *testing.T) { + out := `{"type":"text","text":"## /agents/build.md\n\nLiveKit docs › Build Agents\n\n---\n\n# Building voice agents\n\n> Overview"}` + pages := docsPagesRead(`{"paths": ["/agents/build.md", "https://github.com/livekit/agents/blob/main/README.md"]}`, out) + require.Equal(t, []docsPage{{title: "Building voice agents", url: "https://docs.livekit.io/agents/build"}}, pages) +} diff --git a/cmd/lk/assistant_tools.go b/cmd/lk/assistant_tools.go index baf13b6e..5f501400 100644 --- a/cmd/lk/assistant_tools.go +++ b/cmd/lk/assistant_tools.go @@ -51,7 +51,8 @@ type toolState struct { dir string // the user's project directory creds []string // LIVEKIT_* environment for subprocesses results chan toolResult - pending map[string]pendingTool // confirmation id → command waiting for it + pending map[string]pendingTool // confirmation id → command waiting for it + choices map[string][]overlayConfirmOption // choice card id → its options, until answered confirmN int // lastProject is the project created this session, for try_project. lastProject string @@ -158,6 +159,13 @@ func (s *assistantSession) runTool(requestID, name string, args map[string]any) case "confirm_action": s.confirmByVoice(requestID, str("confirmation_id"), flag("approved")) + case "show_choice": + s.showChoice(requestID, str("question"), args["options"]) + + case "choice_made": + s.choiceMade(str("id"), str("option"), false) + s.replyTool(requestID, "Marked.", nil) + case "copy_to_clipboard": s.copyByVoice(requestID, str("title")) @@ -270,6 +278,53 @@ func (s *assistantSession) proposeCommand(requestID, detail, runLabel string, ar id, command, detail), nil) } +// showChoice shows a card that asks the user to pick one option, by voice or +// on screen. The agent's code decides what each option does; lk only reports +// the choice. It replies with the card's id. +func (s *assistantSession) showChoice(requestID, question string, rawOptions any) { + var options []overlayConfirmOption + if list, ok := rawOptions.([]any); ok { + for _, o := range list { + m, _ := o.(map[string]any) + id, _ := m["id"].(string) + name, _ := m["name"].(string) + if id != "" && name != "" { + options = append(options, overlayConfirmOption{ID: id, Name: name}) + } + } + } + if question == "" || len(options) == 0 { + s.replyTool(requestID, "", errors.New("a choice needs a question and options with an id and a name")) + return + } + t := &s.tools + t.confirmN++ + id := fmt.Sprintf("choice%d", t.confirmN) + t.choices[id] = options + s.agentTurnForReply() + s.out.Send(overlayConfirm{Type: "confirm", Turn: s.agentTurn, ID: id, Heading: question, Options: options}) + s.replyTool(requestID, id, nil) +} + +// choiceMade marks a choice card answered. A click on the card (onScreen) +// also tells the agent, which otherwise only hears choices made by voice. +func (s *assistantSession) choiceMade(id, option string, onScreen bool) { + options, ok := s.tools.choices[id] + if !ok { + return + } + i := slices.IndexFunc(options, func(o overlayConfirmOption) bool { return o.ID == option }) + if i < 0 { + return + } + delete(s.tools.choices, id) + s.out.Send(map[string]string{"type": "confirm_done", "id": id, "option": option}) + s.out.Send(overlayConfirmStatus{Type: "confirm_status", ID: id, Status: "done", Text: options[i].Name}) + if onScreen { + s.channel.send(map[string]string{"type": "choice", "id": id, "option": option}) + } +} + // confirmByVoice handles confirm_action: the user answered by voice. func (s *assistantSession) confirmByVoice(requestID, id string, approved bool) { p, ok := s.tools.pending[id] @@ -294,6 +349,10 @@ func (s *assistantSession) confirmByVoice(requestID, id string, approved bool) { // confirmTool handles the user's answer on screen. func (s *assistantSession) confirmTool(id, option string) { + if _, ok := s.tools.choices[id]; ok { + s.choiceMade(id, option, true) + return + } p, ok := s.tools.pending[id] if !ok { return diff --git a/cmd/lk/assistant_xray.go b/cmd/lk/assistant_xray.go index bf27d689..00077484 100644 --- a/cmd/lk/assistant_xray.go +++ b/cmd/lk/assistant_xray.go @@ -105,7 +105,7 @@ func (s *assistantSession) showAgentCode(m agentMessage) { s.agentTurnForReply() s.snippetN++ id := fmt.Sprintf("self%d", s.snippetN) - s.addSnippet(id, m.Title, m.Code) + s.addSnippet(id, "code", m.Title, m.Code) s.out.Send(overlaySnippet{"snippet", s.agentTurn, id, m.Title, normalizeLang(m.Language), m.Code, false}) } diff --git a/pkg/assistant/agent/agent.py b/pkg/assistant/agent/agent.py index cdd542ab..3ff370ea 100644 --- a/pkg/assistant/agent/agent.py +++ b/pkg/assistant/agent/agent.py @@ -7,23 +7,31 @@ import asyncio import json +import logging import os -import random +import re +import time +import urllib.request +from collections.abc import Awaitable, Callable from livekit import agents from livekit.agents import ( Agent, AgentServer, + AgentTask, AgentSession, RunContext, TurnHandlingOptions, + APIConnectOptions, function_tool, inference, + llm, mcp, text_transforms, ) from livekit.agents.beta.tools import EndCallTool from livekit.agents.voice import TranscriptSynchronizer, io +from livekit.agents.voice.agent_session import SessionConnectOptions # Private module: the SDK's own stripper for expressive markup, the same one the # room transcription output uses. Check it when upgrading livekit-agents. @@ -32,22 +40,34 @@ DOCS_MCP_URL = "https://docs.livekit.io/mcp" STT_MODEL = "assemblyai/universal-3-6-pro" +STT_KEYTERMS = [ + "LiveKit", + "LK", + "LiveKit Cloud", + "LiveKit Inference", + "AgentSession", + "agent dispatch", + "turn detector", + "WebRTC", + "SIP", + "MCP", + "Node.js", + "TypeScript", + "Cartesia", + "Deepgram", + "AssemblyAI", +] TTS_MODEL = "cartesia/sonic-3" # Settings to compare in simulations (see simulations/README.md). The defaults # are what lk ships. LLM_MODEL = os.environ.get("LK_ASSISTANT_LLM", "openai/gpt-5.4-mini") -# Whether the instructions include PRIMER, so basic questions get an answer -# without a docs lookup. -USE_PRIMER = os.environ.get("LK_ASSISTANT_PRIMER", "1") == "1" -# When to say a short line during a docs lookup: "always", "slow" (only when -# the lookup takes a while, at most once per user turn), or "off". -FILLER = os.environ.get("LK_ASSISTANT_FILLER", "slow") +# How much the LLM reasons before it answers. Lower is faster, which matters +# in a voice conversation. "" uses the model's default. +REASONING_EFFORT = os.environ.get("LK_ASSISTANT_REASONING", "") INTRO = "Hi, I'm LK, your LiveKit assistant." -# The greeting for a folder lk doesn't recognize as a project. -GREETING = f"{INTRO} Want to build your first voice agent, or ask me about LiveKit?" def load_folder() -> dict: @@ -74,14 +94,10 @@ def greeting(folder: dict, coding_agents: dict) -> str: kind = folder.get("kind") # One sentence that offers what fits the folder, without describing it. if kind == "empty": - return f"{INTRO} Want to get a voice agent up and running here, or ask me about LiveKit?" + return f"{INTRO} I can help you set up a new voice agent, or answer your questions about LiveKit." if kind == "livekit": - if coding_agents.get("ready"): - return f"{INTRO} Ask me anything about LiveKit, or we can start a new agent in another folder." - return f"{INTRO} Want me to set up your coding agent for LiveKit, or do you have a question?" - if kind == "app": - return f"{INTRO} Want to add a voice agent to this project, or ask me about LiveKit first?" - return GREETING + return f"{INTRO} I see you have a LiveKit project up and going. Any questions about LiveKit I can help you with?" + return f"{INTRO} Any questions about LiveKit I can help you with?" # Pronunciations for the TTS, as Cartesia inline IPA overrides: "live" as in @@ -118,225 +134,226 @@ def greeting(folder: dict, coding_agents: dict) -> str: DOCS_URL = "https://docs.livekit.io/" # The Docs MCP tools the agent uses. -DOCS_TOOLS = ("get_docs_overview", "docs_search", "get_pages", "code_search") +DOCS_TOOLS = ("get_docs_overview", "docs_search", "read_docs", "code_search") +# The Docs MCP tools the agent uses. It reads pages with its own read_docs +# instead of get_pages, so code limits how many pages and how much text. +MCP_DOCS_TOOLS = ("get_docs_overview", "docs_search", "code_search") # What the agent says while it looks something up, if it hasn't said anything # yet this turn. Docs lookups take a second or two. -LOOKUP_FILLERS = ( - "Let me check the docs.", - "One moment, I'll look that up.", - "Let me look that up in the docs.", -) - -# With FILLER "slow", how long a lookup runs before the agent says a line. -SLOW_LOOKUP_SECONDS = 2.5 +# Docs MCP results, such as searches, can be long. +# Cap them, because every character slows the LLM, and a large context made +# replies time out. Most answers come from the start of a page. +MAX_TOOL_RESULT_CHARS = 20000 -# Docs MCP results can be whole pages. Cap them to bound the LLM context, but -# high enough that a full docs page fits. The longest pages, such as -# /agents/build/turns/, are over 37,000 characters. -MAX_TOOL_RESULT_CHARS = 50000 +# What old docs results become once a reply used them (see +# Assistant.on_user_turn_completed). +DOCS_RESULT_STUB = "[Docs content from an earlier answer, removed to keep replies fast. Look it up again if you need it.]" INSTRUCTIONS = """You are LK, the LiveKit assistant, a voice assistant inside the LiveKit CLI. -You help developers get started with LiveKit: build their first voice agent, talk to it, set up -their coding agent for LiveKit, and understand the basics they need. You are yourself a LiveKit -voice agent, built with the same Agents SDK and LiveKit Inference the user can build with. - -Be generous with what you know. Your job is to help the user build a voice agent that works well, -so share the knowledge that makes the difference: why LiveKit works the way it does, the tradeoff -behind a choice, and what to watch for, such as latency, turn-taking, interruptions, and testing. -Give context where it helps the user understand, not only the answer to the literal question. -Generous means the context that matters for this user, not every detail: for someone who isn't -technical, skip implementation details unless they ask. -Explain concretely. Walk through what happens, step by step, in plain words, such as what happens -when a user starts talking to an agent, rather than abstractions such as "realtime bus" or -"orchestration". For a big question with several parts, such as how LiveKit fits together, call -show_answer with the parts, and speak a short version of the most important idea. - -Getting started is your job. Ongoing work on a project belongs in the user's coding agent, which -can read and change their code. You can't see or change the user's files, so when the user asks you -to fix, change, or review their code, don't suggest a fix yourself: call hand_off_task with a prompt -for their coding agent. Since you haven't seen their code, the prompt describes the problem, how -to reproduce it, and how to check that it's fixed, plus anything relevant from the docs. It doesn't -prescribe a fix; the coding agent finds the cause. +You help developers build voice AI agents with LiveKit, add LiveKit to apps they already have, and +explore what LiveKit can do. You are yourself a LiveKit voice agent, built with the same Agents SDK +and LiveKit Inference the user can build with. Share the knowledge that helps the user build a +voice agent that works well, and give context where it helps. Match the user's level. If they seem new, describe outcomes before LiveKit terms, and explain a term in a few words the first time you use it. If they seem experienced, be terse and precise, and skip the basics. If they ask you to be briefer or to go deeper, do that for the rest of the -conversation. Be plain and calm: no slang, no exclamations, no flattery. - -Don't repeat yourself. Don't ask a question the user has already answered, and don't ask twice -before doing something. Don't announce that you're looking something up; just look it up. Don't -end replies with an offer such as "If you want, I can" or "I can sketch that next"; the user -knows they can ask. When the next step is clear, ask in a few -words, such as "Want to try it?", or just do it. When the user's request is clear, do it; don't -offer choices they didn't ask for. Don't restate what's already on screen. - -Say only what's true. Tell the user something worked only after a tool result says so. Name -commands exactly as the tool result or the card on screen shows them. Don't say you can't do -something and then do it. Don't promise anything you can't show, such as a diagram or a file. - -When you talk about the STT-LLM-TTS pipeline, say once that it's a cascaded pipeline, three models -in sequence: speech-to-text, an LLM, and text-to-speech. After that, call it the cascaded pipeline. - -Speak in the style of Simplified Technical English (ASD-STE100): short sentences, at most about 20 -words, with one idea each. Use active voice, simple tenses, and common words with one clear -meaning. Don't use idioms or figures of speech. Use the exact technical term when there is one, -such as "dispatch" or "turn detector", and explain it in a few words the first time. Deviate from -this style only when it would make a concept less accurate. +conversation. You speak out loud, so: -- Fit the length to the moment. When you confirm an action or answer a yes-or-no question, one - sentence is enough. When you explain something, take two to five sentences: the answer, why, - and the one thing to watch for. Don't cover everything at once; the user can ask for more. -- Use plain sentences. No markdown, lists, emojis, or symbols, with one exception: put a short - command, package, file, function, or class name in backticks, such as `lk agent dev` or - `AgentSession`. On screen it shows in monospace. The backticks aren't spoken. +- Keep replies short: one to three sentences. Don't cover everything; the user can ask for more. + Don't end with offers such as "If you want, I can". +- Use plain sentences in the style of Simplified Technical English: one idea per sentence, active + voice, and common words. No markdown, lists, emojis, or symbols, except backticks around a short + command or name, such as `lk agent dev`. Backticks show it in monospace and aren't spoken. - Never read code, commands, or long identifiers aloud. Call show_code to show them, - then say one short sentence about what you showed. -- If the user asks you to copy something on screen, call copy_to_clipboard. Don't copy unless - they ask. + then say one short sentence about what you showed. When the user asks how to do something in + code, or asks for an example, call show_code with a short example from the docs right away. + Never say code "looks like this" without showing it. +- When it helps, use yourself as a one-sentence example after the answer. You're a LiveKit agent, + so you're built from the same parts; see "How you, LK, are built" below. LiveKit changes often, and what you remember about it is out of date. For every question about LiveKit features, APIs, SDKs, models, plugins, CLI commands, pricing, or configuration, call docs_search first, then answer only from what the docs say. Do this even when you think you know the answer. For questions about how something works or how to configure it, also read the most -relevant page with get_pages before you answer. When you're not sure where a topic lives, or the +relevant page with read_docs before you answer. When you're not sure where a topic lives, or the user asks what LiveKit can do, call get_docs_overview to see how the docs are organized. If the -docs don't cover it, say so. Never guess an API. - -Show docs answers on screen, not only in speech. When an answer has more than one point, call -show_answer with a few short points and the docs page they come from, then say one sentence that -sums it up. For a one-sentence answer, just say it, and call show_link with the page it comes -from when the user would want to read more. Use page titles and URLs exactly as your docs search -returned them, and never read a URL aloud. - -How you help someone get started: -- Call get_project_info when you need to know about their LiveKit Cloud project or their current - directory. -- For a new voice agent, start it from a template with create_project. Before you create it, ask - one short question: Python or Node.js, unless the user already said. Their answer is their - go-ahead, so then call create_project with confirmed=true, without asking again. Use - agent-starter-python or agent-starter-node, or call list_templates if they want something else. - If the current directory is empty, create it right there. New projects already include - LiveKit's skills and Docs MCP server for the user's coding agents. -- After you create a project, offer to try it right here with try_project, so the user can talk - to the agent they just made. While their agent runs, the user talks to it, not to you, and you - hear nothing. You are never their agent: don't speak as it, and don't say it's running until lk - tells you. You can't stop it; the user presses B. -- When they want to change their new agent, such as its instructions, voice, or tools, act as - the expert. Look up how in the docs, such as which TTS voices LiveKit Inference offers. If the - choice is the user's, such as which voice, give them one or two concrete options and ask in a - few words. Then call hand_off_task right away, without asking whether to. Write the prompt like - a senior engineer's ticket, addressed to the coding agent: the exact change, with the API, - model, and values from the docs, and how to check that it works. Pick one option rather than - listing alternatives. Keep it short. No hedges such as "if the project uses", and not a prompt - about writing a prompt. -- To add LiveKit to an app they already have, explain the three parts in plain words: the app - gets a token from a small server endpoint, joins a LiveKit room, and the agent joins that room - through dispatch. Then create the agent with create_project in a new folder, and give the - frontend and token work to their coding agent with hand_off_task, based on the docs. -- If their coding agent isn't set up for LiveKit, offer set_up_coding_agent, which installs - LiveKit's skills and Docs MCP server into it (such as Claude Code, Cursor, or Codex). -- If the folder already has a LiveKit project, the user is past getting started. Point them to - their coding agent for work on it, and help with what you can: setting up their coding agent, - quick questions, or a new agent in a new folder. +docs don't cover it, say so. Never guess an API. When a docs page covers the topic in more depth, +call show_link with it. + +Finding the right pages: +- Pick pages from the docs page list by title: the page whose title matches the topic, not one + that only shares a keyword. Prefer the main topic page over a related one. For a general "how + do I" question, prefer an overview or guide page. Don't read pages under /recipes/ for a first + answer. +- Keep lookups short, because the user is waiting: at most one docs_search per reply (put several + queries in its queries list), then one read_docs call with {pages}, then answer. + When a title in the list clearly matches, skip docs_search and go straight to read_docs. +- Use only paths from the docs page list or from search results. Don't guess paths. + +Be honest about what LiveKit does: +- If the docs don't confirm that a feature exists, say so. Don't combine features unless the docs + show they work together. +- When LiveKit supports something only partly, say what's missing. A "yes" that leaves out a + known limit is worse than a "no". +- Don't state numbers, such as prices, limits, or latencies, unless they're in a page you read. +- If a question is ambiguous, ask one short question to find out what the user means. Speech + recognition can garble words, so read an odd question as a phonetic hint, such as "live kit" + for LiveKit. + +You can't see or change the user's files. You help them plan and decide, and you set them up to +do the work: +- To learn about their LiveKit Cloud project and current directory, call get_project_info. To check + on deployed agents, call list_cloud_agents. +- For a new voice agent, call create_voice_agent. It asks the user to confirm and to pick a + language, creates the project, and offers to let them talk to it. It creates the project only + in the current folder, and only if the folder is empty. New projects already include LiveKit's + skills and Docs MCP server for the user's coding agents. +- For an existing project, offer set_up_coding_agent, which installs LiveKit's skills and Docs MCP + server into the user's coding agents (such as Claude Code, Cursor, or Codex), unless they're + already set up. +- To add LiveKit to an existing app (get_project_info reports its framework), explain the three + parts in plain words: the app gets a token from a small server endpoint, joins a LiveKit room, + and the agent joins that room through dispatch. Then create the agent with create_voice_agent + and give the frontend and token work to their coding agent with hand_off_task, based on the docs. +- While the user's own agent runs, the user talks to it, not to you. Never speak as their agent. + To let them talk to it again later, call try_project. - If the user asks how you work, or what they'll be building, call explain_yourself. +- When the user is ready to write code, call hand_off_task with a prompt for their coding agent: + what to build or change, plus what you learned from the docs, such as package names, APIs, and + CLI commands. If what they want is still vague, such as "add some tools", first ask one short + question about what exactly. +- For anything you can't do here, such as running an agent they already had, say so in one + sentence and give the next best thing: a command to run, with show_code, or a prompt for their + coding agent. + +set_up_coding_agent and try_project don't run right away. They show the command to the user and +return a confirmation ID. Ask the user in one short sentence whether to go ahead. They can answer +by voice or on screen. If they agree by voice, call confirm_action with the ID and approved=true; +if they decline, approved=false. Never call confirm_action unless the user has just agreed. + +When the user says they're done or says goodbye, call end_call. Don't say goodbye yourself; +end_call says it and closes the window. + +Prefer the LiveKit CLI for project tasks. For example, "lk agent init" creates an agent from a +starter template, "lk agent dev" runs it, and "lk agent create" deploys it to LiveKit Cloud.""" + +# The docs table of contents: every page with a one-line description. With it +# in the prompt, the agent picks pages by title before it reads them. It's +# trimmed to what onboarding questions need, about 10,000 tokens, and cached. +DOCS_TOC_URL = "https://docs.livekit.io/api/toc/" +DOCS_TOC_SKIP = re.compile( + r"/plugins/|/reference/|/components/|/robotics|/recipes/|youtube|" + r"/models/(stt|tts|llm|avatar)/[a-z]|/transport/self-hosting|/transport/media/ingress" +) +DOCS_TOC_TTL = 24 * 60 * 60 -When the user asks for something outside that, don't pretend and don't improvise. Say in one short -sentence that you can't do it here, then give them the next best thing: -- A command they can run themselves, with show_code. For example, to run an agent they already - had, try_project tells you the command. To deploy, "lk agent create" the first time and "lk - agent deploy" after that. To see logs, "lk agent logs". Check the docs for the exact command. -- A prompt for their coding agent, with hand_off_task, for anything that reads or changes their - code: fixing a bug, adding a feature, or reviewing their agent. -- The docs page that covers it, with show_link, for anything else about LiveKit. -For something unrelated to LiveKit, say it's outside what you help with, and offer what you can. - -create_project, set_up_coding_agent, and try_project show the command on screen and wait for the -user to confirm. If the user's last message asked for that action or agreed to it, such as "set -up a voice agent here" in an empty folder, or "yes" to your offer, pass confirmed=true so it runs -right away, and don't ask again. Otherwise, the tool returns a -confirmation ID: ask the user in one short sentence whether to go ahead. They can answer by voice -or on screen. If they agree by voice, call confirm_action with the ID and approved=true; if they -decline, approved=false. Never confirm anything the user hasn't agreed to. - -If the user asks what you cost: you run on LiveKit Inference through their LiveKit Cloud project, -so your usage counts toward their plan. - -When the user wants to stop, such as "exit", "close", "quit", "bye", or "that's all", call -end_call. Saying goodbye without calling end_call leaves the window open. - -Prefer the LiveKit CLI for project tasks. "lk agent init" creates an agent from a starter -template, "lk agent console" talks to it in the terminal, "lk agent dev" runs it with hot reload, -and "lk agent create" deploys it to LiveKit Cloud.""" - -# What LK knows without a lookup, checked against the docs (the Docs overview, -# Pipeline types, Agent dispatch, Turns overview, Models overview, Voice AI -# quickstart, Telephony introduction, Tokens & grants, and Testing pages). -PRIMER = """What you know about LiveKit, checked against the docs: -- LiveKit is an open-source platform for building voice, video, and physical AI agents, on - realtime WebRTC infrastructure. LiveKit Cloud is the hosted offering. It includes agent - deployment, model inference, and realtime media transport. -- Every client, agent, and phone call joins a room as a participant, and publishes or subscribes - to audio, video, and data tracks. Frontends connect with an access token: a JWT, signed with the - project's API secret, that names the participant, the room, and what it may do. -- An agent server runs the agent code. Dispatch assigns an agent to a room. Explicit dispatch, by - the agent's name, is recommended for most apps. AgentSession runs the conversation, and the Agent - holds the instructions and tools. -- What happens in a conversation: the user's app joins a room with a token, and LiveKit - dispatches the agent to the same room. The agent's AgentSession receives the user's audio. STT - turns it into text, and the turn detector decides when the user has finished. The LLM writes a - reply and can call tools. TTS speaks the reply into the room. The user can interrupt at any time, - and the agent stops and listens. The agent code is mostly instructions and tools; LiveKit handles - the realtime audio. -- The STT-LLM-TTS pipeline, also called the cascaded pipeline, is the right default for most - production agents: each model can be swapped, every stage can be inspected, tool calling is - mature, and say() speaks exact text. It has more end-to-end latency than a realtime model. A - realtime model is one speech-to-speech model: fastest, and it hears tone of voice, but its tool - calling is less mature and its transcripts are delayed. A half-cascade pairs a realtime model for - input with a separate TTS for output. -- Turn detection decides when the user has finished speaking. LiveKit's turn detector model is the - default and is recommended for most agents. It predicts the end of a turn from the meaning and - sound of speech, on top of VAD. Endpointing delay and adaptive interruption handling tune it. -- LiveKit Inference gives agents models from OpenAI, Google, AssemblyAI, Deepgram, Cartesia, and - others through LiveKit Cloud, with no extra API keys or plugins, and zero data retention by - default. Plugins connect directly to more providers, with the user's own API keys. -- Agents can take and make phone calls. LiveKit Phone Numbers sells US local and toll-free numbers - in LiveKit Cloud, or the user can connect their own SIP provider with trunks. Dispatch rules route - inbound calls to rooms. -- The starter templates, agent-starter-python and agent-starter-node, include a working agent, - tests, and an AGENTS.md for coding agents. Python needs 3.10 or later and uv. Node.js needs 20 or - later and pnpm. "lk agent init" creates a project, "lk agent console" talks to it in the terminal, - "lk agent dev" runs it with hot reload, and "lk agent create" deploys it to LiveKit Cloud, which - scales it automatically. -- Agent Builder builds agents in the browser without code, for prototyping. -- To test agents: Agent Console to try one yourself, unit tests on pytest or Vitest, and Agent - Simulations, where an LLM plays the user and a judge grades whole conversations, in text or audio - mode. - -Answer basic questions about these from what you know, without a lookup, the way an experienced -LiveKit engineer would: give a clear recommendation and the reason for it, and mention the one -detail that matters most. Use LiveKit's names for things, such as LiveKit's turn detector model, -LiveKit Phone Numbers, LiveKit Inference, or Agent Simulations, so the user knows what to look -for, and explain each in a few words. Look up anything more specific in the docs: APIs, parameters, model -names, versions, prices, limits, and step-by-step setup.""" - -# With the primer, basic questions skip the lookup. -DOCS_RULE = """For every question about -LiveKit features, APIs, SDKs, models, plugins, CLI commands, pricing, or configuration, call -docs_search first, then answer only from what the docs say. Do this even when you think you know -the answer.""" -PRIMER_DOCS_RULE = """For basic questions that what -you know about LiveKit covers, answer directly. For anything more specific, such as APIs, -parameters, model names, versions, prices, limits, or setup steps, call docs_search first, then -answer only from what the docs say.""" +def docs_toc() -> str: + """The trimmed docs table of contents, from the cache or the docs site, or "".""" + cache = os.path.join(os.path.expanduser("~/Library/Caches"), "livekit", "assistant-docs-toc.md") + try: + if time.time() - os.path.getmtime(cache) < DOCS_TOC_TTL: + return open(cache, encoding="utf-8").read() + except OSError: + pass + try: + with urllib.request.urlopen(DOCS_TOC_URL, timeout=2) as resp: + text = resp.read().decode("utf-8") + except Exception: + try: + return open(cache, encoding="utf-8").read() # a stale copy is better than none + except OSError: + return "" + toc = "\n".join(line for line in text.splitlines() if line.strip() and not DOCS_TOC_SKIP.search(line)) + try: + os.makedirs(os.path.dirname(cache), exist_ok=True) + with open(cache, "w", encoding="utf-8") as f: + f.write(toc) + except OSError: + pass + return toc + + +# How many docs pages one reply may read. Fewer is faster. +MAX_PAGES = int(os.environ.get("LK_ASSISTANT_MAX_PAGES", "3")) + +# How much docs text one read_docs call returns, split between its pages. Every +# character slows the LLM, and most answers come from the start of a page. +DOCS_READ_CHARS = int(os.environ.get("LK_ASSISTANT_DOCS_CHARS", "12000")) + + +def docs_page_url(path: str) -> str: + """The Markdown URL for a docs path or URL.""" + path = path.strip() + for prefix in ("https://docs.livekit.io", "http://docs.livekit.io"): + if path.startswith(prefix): + path = path[len(prefix) :] + path = "/" + path.strip("/").removesuffix(".md") + return "https://docs.livekit.io" + path + ".md" + + +def fetch_docs_page(url: str) -> str: + with urllib.request.urlopen(url, timeout=8) as resp: + return resp.read().decode("utf-8") + + +async def read_docs_pages(paths: list[str]) -> str: + """Fetches docs pages as Markdown, in parallel, each trimmed to its share of + DOCS_READ_CHARS.""" + if not paths: + return "No paths given. Pick pages from the docs page list." + share = DOCS_READ_CHARS // len(paths) + + async def one(path: str) -> str: + url = docs_page_url(path) + try: + text = await asyncio.to_thread(fetch_docs_page, url) + except Exception as e: + return f"## {path}\n\nCouldn't read this page ({e}). Check the path in the docs page list." + if len(text) > share: + text = text[:share] + "\n... [the rest of the page is cut]" + return f"## {url.removesuffix('.md')}\n\n{text}" + + return "\n\n".join(await asyncio.gather(*(one(p) for p in paths))) + + +def about_yourself() -> str: + """Facts about how LK is built, from its own settings, so it can use itself as + an example.""" + return f"""How you, LK, are built. Every part is something the user can use too: +- A cascaded pipeline on LiveKit Inference: STT {STT_MODEL}, with keyterms that help it hear + LiveKit's names; LLM {LLM_MODEL}; and TTS {TTS_MODEL}, in expressive mode. Text transforms fix + how the TTS says "LiveKit". +- Turn detection: LiveKit's turn detector model, with dynamic endpointing that learns the user's + pauses. +- Tools: the LiveKit Docs MCP server, connected with MCPToolset, to search the docs, plus your own + function tools, such as read_docs to read pages, show_code for code cards, and hand_off_task + for prompts. +- A task: creating the user's agent is an AgentTask. Code runs its steps, and the model only + reads the user's answers and words the replies, so it can't skip a step. +- Testing: Agent Simulations, where an LLM plays the user and a judge grades each conversation, + in text and audio mode, with scenarios to compare configurations and catch regressions. +- You run on the user's machine in console mode, started by lk. + +For example: "I work this way too: I call the Docs MCP server as a tool to get current docs." +Don't use yourself as an example in every answer, and only say what's in this list.""" -def build_instructions() -> str: - if not USE_PRIMER: - return INSTRUCTIONS - return INSTRUCTIONS.replace(DOCS_RULE, PRIMER_DOCS_RULE) + "\n\n" + PRIMER +def build_instructions() -> str: + pages = "only the one most relevant path" if MAX_PAGES == 1 else f"up to {MAX_PAGES} paths" + instructions = INSTRUCTIONS.replace("{pages}", pages) + "\n\n" + about_yourself() + toc = docs_toc() if os.environ.get("LK_ASSISTANT_TOC", "1") == "1" else "" + if not toc: + return instructions + return ( + instructions + + "\n\nThe docs pages, as [Section > Title](path): description. Pick pages from this list " + "by title, and pass their paths to read_docs.\n\n" + + toc + ) async def truncate_result(ctx: mcp.MCPToolResultContext) -> str: @@ -356,6 +373,8 @@ def __init__(self, folder: dict | None = None, coding_agents: dict | None = None instructions = build_instructions() folder = load_folder() if folder is None else folder coding_agents = load_coding_agents() if coding_agents is None else coding_agents + self.folder = folder + self.created = False # whether create_voice_agent created a project if project := os.environ.get("LK_ASSISTANT_PROJECT"): instructions += f"\n\nThe user's current directory is {project}." if folder: @@ -382,20 +401,43 @@ def __init__(self, folder: dict | None = None, coding_agents: dict | None = None id="livekit-docs", mcp_server=mcp.MCPServerHTTP( DOCS_MCP_URL, - allowed_tools=list(DOCS_TOOLS), + allowed_tools=list(MCP_DOCS_TOOLS), tool_result_resolver=truncate_result, ), ), # lk closes its overlay when it sees this tool called. Console # mode has no real room, so there's nothing to delete. *EndCallTool( - extra_description="Use this when the user says they're done, says goodbye, or asks to close the assistant.", + extra_description=( + "Call this whenever the user wants to stop or leave, such as saying exit, " + "close, quit, bye, or that's all. Saying goodbye without calling it leaves " + "the window open." + ), delete_room=False, end_instructions="Say a short, friendly goodbye in one sentence.", ).tools, ], ) + async def on_user_turn_completed(self, turn_ctx, new_message) -> None: + # Docs pages are large, and every later turn would send them to the LLM + # again. Once a reply has used them, keep only a stub; the agent can + # look a page up again. + changed = drop_old_docs(turn_ctx) + ctx = self.chat_ctx.copy() + if drop_old_docs(ctx) or changed: + await self.update_chat_ctx(ctx) + + @function_tool() + async def read_docs(self, context: RunContext, paths: list[str]) -> str: + """Read LiveKit docs pages. Pick them by title from the docs page list or from search + results. + + Args: + paths: Page paths, such as /agents/models/pipelines. Only the first few are read. + """ + return await read_docs_pages(paths[:MAX_PAGES]) + @function_tool() async def show_code(self, context: RunContext, title: str, language: str, code: str) -> str: """Show code or a terminal command to the user. The user sees it on screen and can @@ -412,35 +454,6 @@ async def show_code(self, context: RunContext, title: str, language: str, code: "say so in one sentence. Don't repeat what you already said." ) - @function_tool() - async def show_answer( - self, - context: RunContext, - title: str, - points: list[str], - source_title: str, - source_url: str, - ) -> str: - """Show a docs answer on screen as a few short points, with the docs page they come - from. The user can read it, copy it, and open the page. After calling it, say one - sentence that sums it up; don't read the points aloud. - - Args: - title: What the answer is about, such as "Turn detection". - points: Two to five short points, one sentence each. Put commands and names in - backticks. - source_title: The title of the docs page the points come from. - source_url: The page's full URL, exactly as your docs search or get_pages returned - it. It must start with https://docs.livekit.io/. - """ - # `lk assistant` reads this call from the session events and shows it. - if not source_url.startswith(DOCS_URL): - return f"Not shown: the source must be a docs page under {DOCS_URL}, from your docs searches." - return ( - "The answer is on the user's screen. Unless you already said it in this turn, say one " - "sentence that sums it up. Don't leave the user without a spoken answer." - ) - @function_tool() async def show_link(self, context: RunContext, title: str, url: str) -> str: """Show a link to a LiveKit docs page where the user can read more. The user can open @@ -448,7 +461,7 @@ async def show_link(self, context: RunContext, title: str, url: str) -> str: Args: title: The page title, such as "LiveKit turn detector". - url: The page's full URL, exactly as your docs search or get_pages returned it. It + url: The page's full URL, exactly as your docs search or read_docs returned it. It must start with https://docs.livekit.io/. """ # `lk assistant` reads this call from the session events and shows it. @@ -470,11 +483,17 @@ async def copy_to_clipboard(self, context: RunContext, title: str = "") -> str: @function_tool() async def hand_off_task(self, context: RunContext, task: str) -> str: """Show the user a prompt to paste into their coding agent, with a button to copy it. + Call it right away when the user wants a change; don't ask whether to. Args: - task: A complete, self-contained prompt: what to build or change in the user's - project, plus details from the LiveKit docs such as package names, APIs, and CLI - commands. + task: A short, self-contained prompt addressed to the coding agent, like a senior + engineer's ticket. For a change, such as a new voice: the exact change, with the + API, model, and values you looked up in the docs, and how to check it works. + Pick one option; if the choice is the user's, ask them first. If the request is + vague, such as "add some tools", ask one short question about what exactly before + you call this; don't write a prompt that leaves the work undefined. For a bug: the + problem, how to reproduce it, and how to check it's fixed, without guessing a + fix, because you haven't seen the code. """ # `lk assistant` reads this call from the session events and shows it. return ( @@ -501,38 +520,36 @@ async def list_templates(self, context: RunContext) -> str: return await run_lk_tool("list_templates") @function_tool() - async def create_project( - self, - context: RunContext, - template: str, - name: str = "", - in_current_directory: bool = False, - confirmed: bool = False, - ) -> str: - """Create a project from a LiveKit template, with lk agent init for agent templates or lk - app create for frontends, including installing dependencies. Unless confirmed is true, it - shows the command and waits for the user to confirm (see confirm_action). Afterward, offer - to try the new agent with try_project. + async def create_voice_agent(self, context: RunContext, another: bool = False) -> str | None: + """Create a new voice agent for the user from a LiveKit starter template, then offer to + let them talk to it. Call it when the user wants to build, create, or set up a new agent. + It takes over until the project exists: it asks the user to confirm and to pick Python or + Node.js, so don't ask first. Args: - template: The template name, such as agent-starter-python. - name: The name for the new folder: lowercase letters, numbers, and dashes. Not used - with in_current_directory. - in_current_directory: Create the project in the current directory itself, named after - it, instead of in a new folder. Only works if the directory is empty. - confirmed: True if the user's last message asked for this or agreed to it, such - as "set up a voice agent here" or "yes" after you offered it. Then it runs - right away, without asking again. Otherwise leave it false, and ask. + another: True only if the user explicitly asks for a second, separate agent after + one was already created in this conversation. """ - return await run_lk_tool( - "create_project", - { - "template": template, - "name": name, - "in_current_directory": in_current_directory, - "confirmed": confirmed, - }, - ) + if self.created and not another: + return ( + "Nothing was created: the user's agent already exists from earlier in this " + "conversation. Don't create another unless the user explicitly asks for a second one." + ) + if self.folder.get("kind") != "empty": + return ( + "Nothing was created: LK creates a new agent only in an empty current folder, and this " + "folder isn't empty. Tell the user to make an empty folder, go to it, and run " + "`lk assistant` there." + ) + result = await NewAgentTask(chat_ctx=self.chat_ctx) + if result.startswith("Created") or result == STARTED: + self.created = True + self.folder = {"kind": "livekit", "has_agents": True} + if result == STARTED: + # The task already announced the hand-off, and the user is about to + # talk to their agent, so there's nothing to add. + return None + return result @function_tool() async def set_up_coding_agent( @@ -544,7 +561,7 @@ async def set_up_coding_agent( ) -> str: """Install LiveKit's agent skills and the LiveKit Docs MCP server into the user's coding agents (such as Claude Code, Cursor, and Codex), with lk skills install. New projects from - create_project already have them. If they're already set up, it says so and installs + create_voice_agent already have them. If they're already set up, it says so and installs nothing. Unless confirmed is true, it shows the command and waits for the user to confirm (see confirm_action). @@ -576,8 +593,9 @@ async def try_project(self, context: RunContext, confirmed: bool = False) -> str @function_tool() async def explain_yourself(self, context: RunContext) -> str: - """Show the user how you're built: your own AgentSession code, on screen. Use it when they - ask how you work, or to show what a LiveKit voice agent looks like.""" + """Show the user how you, LK, are built: your own AgentSession code, on screen. Use it + only when they ask how you work or how you're built. For an example of how to build an + agent, show code from the docs with show_code instead.""" code = own_session_code() if lk is not None and code: lk.send({"type": "show_code", "title": "agent.py (how LK is built)", "language": "python", "code": code}) @@ -593,12 +611,12 @@ async def explain_yourself(self, context: RunContext) -> str: @function_tool() async def confirm_action(self, context: RunContext, confirmation_id: str, approved: bool) -> str: - """Run or cancel a command that create_project or set_up_coding_agent proposed, after the + """Run or cancel a command that set_up_coding_agent or try_project proposed, after the user answers by voice. Only call it right after the user agrees or declines. Args: confirmation_id: The ID the proposing tool returned, such as confirm1. Works for - create_project, set_up_coding_agent, and try_project. + set_up_coding_agent and try_project. approved: True if the user agreed, false if they declined. """ # The command changes the user's files. While it runs, user speech @@ -620,6 +638,251 @@ async def confirm_action(self, context: RunContext, confirmation_id: str, approv COMMAND_TIMEOUT = 600 +def drop_old_docs(ctx) -> bool: + """Replaces the output of docs lookups in ctx with DOCS_RESULT_STUB. + Reports whether anything changed.""" + docs_calls = { + item.call_id + for item in ctx.items + if item.type == "function_call" and item.name in DOCS_TOOLS + } + changed = False + for item in ctx.items: + if item.type == "function_call_output" and item.call_id in docs_calls and item.output != DOCS_RESULT_STUB: + item.output = DOCS_RESULT_STUB + changed = True + return changed + + +# The starter templates create_voice_agent uses, by language. +STARTERS = {"python": "agent-starter-python", "node": "agent-starter-node"} + +# How each starter opens a conversation, for the hand-off: the Node.js starter +# greets the user when it joins, and the Python starter waits for them. +STARTER_OPENINGS = { + "python": "once it connects, they should say hello, because it waits for them to speak first", + "node": "once it connects, it greets them first", +} + +NEW_AGENT_INSTRUCTIONS = """You are LK, the LiveKit assistant, helping the user create a new voice +agent from a LiveKit starter template. Code runs the steps. Say only what you're asked to say, in +short, plain sentences, and don't read commands aloud. Never say a step happened unless you're told +it did.""" + +# The answers each question in NewAgentTask accepts, for read_answer. +LANGUAGE_ANSWERS = { + "python": "chooses or confirms Python", + "node": "chooses or confirms Node.js, JavaScript, or TypeScript", + "cancel": "doesn't want to create an agent now", + "leave": "wants to stop talking to LK or close it", + "other": "is unsure, asks a question, or says something else", +} +TRY_ANSWERS = { + "yes": "wants to talk to their new agent now", + "no": "doesn't want to talk to it now", + "leave": "wants to stop talking to LK or close it", + "other": "asks a question or says something else", +} + + +async def read_answer(model: llm.LLM, conversation: str, answers: dict[str, str], answer: str) -> str: + """Asks the model which of answers the user's answer means, given the last + few lines of the conversation, including the question LK actually asked. A + narrow call like this one is more reliable than a conversation that must + decide to call a tool. Returns "other" if the reply isn't one of the + answers.""" + ctx = llm.ChatContext.empty() + options = "\n".join(f"- {key}: the user {meaning}" for key, meaning in answers.items()) + ctx.add_message( + role="system", + content=( + "Classify the user's answer to LK's last question. Read it in context: \"yes\" to " + "\"Create it in Python?\" means Python. Reply with only one of these words:\n" + options + ), + ) + ctx.add_message(role="user", content=f"Conversation so far:\n{conversation}\n\nThe user's answer: {answer}") + text = "" + async with model.chat(chat_ctx=ctx) as stream: + async for chunk in stream: + if chunk.delta and chunk.delta.content: + text += chunk.delta.content + word = text.strip().strip(".\"'").lower() + return word if word in answers else "other" + + +def recent_lines(chat_ctx, before: str, n: int = 4) -> str: + """The last n messages before the message with id before, as "Role: text" lines.""" + lines = [] + for item in chat_ctx.items: + if item.id == before: + break + if item.type == "message" and item.role in ("user", "assistant") and item.text_content: + who = "User" if item.role == "user" else "LK" + lines.append(f"{who}: {item.text_content}") + return "\n".join(lines[-n:]) + + +# What NewAgentTask returns when it has started the user's agent and announced it. +STARTED = "Started the user's agent, and told them how to come back." + + +class NewAgentTask(AgentTask[str]): + """Creates the user's new agent and offers to let them talk to it. + + Code runs every step. Each of the user's answers is read with read_answer, + acted on in code, and only then worded by the model. This happens in + llm_node, which every reply goes through, whether the user spoke or typed. + The model has no tools here, so it can't skip a step, answer its own + question, or say a step happened when it didn't. Each question is also a + choice card, so the user can answer on screen. + """ + + def __init__(self, chat_ctx) -> None: + super().__init__(instructions=NEW_AGENT_INSTRUCTIONS, chat_ctx=chat_ctx) + self.step = "language" # language, creating, try, finished + self.card = "" + self.scripted = 0 # replies that code asked for, not yet generated + # The id of the last user message code handled. The message that started + # this task isn't an answer to its first question. + self.handled = next( + (i.id for i in reversed(chat_ctx.items) if i.type == "message" and i.role == "user"), "" + ) + + async def on_enter(self) -> None: + await self._ask( + "Create a voice agent in this folder?", + [("python", "Python"), ("node", "Node.js"), ("cancel", "Not now")], + "Ask, in these words or close to them: \"Do you want to use Python or Node?\" It goes in " + "the current folder, so don't offer another folder. If they already said a language, " + "ask them to confirm it instead.", + ) + + async def llm_node(self, chat_ctx, tools, model_settings): + if self.scripted: + # A reply code asked for with _say; its instructions are in chat_ctx. + self.scripted -= 1 + else: + user = next((i for i in reversed(chat_ctx.items) if i.type == "message" and i.role == "user"), None) + if user is None or user.id == self.handled: + return + self.handled = user.id + say = await self._handle(user.text_content or "", recent_lines(chat_ctx, user.id)) + if not say: + return + chat_ctx = chat_ctx.copy() + chat_ctx.add_message(role="system", content=say) + async for chunk in Agent.default.llm_node(self, chat_ctx, [], model_settings): + yield chunk + + async def _handle(self, answer: str, conversation: str) -> str | None: + """Acts on the user's answer, and returns what the reply should say, if anything.""" + if self.step == "language": + choice = await read_answer(self.session.llm, conversation, LANGUAGE_ANSWERS, answer) + if choice in STARTERS: + await self._answered(choice) + asyncio.create_task(self._create(choice)) + name = "Python" if choice == "python" else "Node.js" + return f"In one short sentence, tell the user you're creating their {name} agent now." + if choice == "cancel": + await self._answered("cancel") + self._done("The user chose not to create an agent now. Nothing was created.") + return None + if choice == "leave": + self._done("Nothing was created. The user wants to stop, so call end_call.") + return None + return ( + "Answer what the user said in one or two short sentences. If they're unsure: Python " + "has the most examples, and Node.js fits if their app is JavaScript or TypeScript. " + "Then ask: \"Do you want to use Python or Node?\"" + ) + if self.step == "try": + choice = await read_answer(self.session.llm, conversation, TRY_ANSWERS, answer) + if choice in ("yes", "no"): + await self._answered("try" if choice == "yes" else "later") + asyncio.create_task(self._finish(choice == "yes")) + return None + if choice == "leave": + self._done("Created the project. The user wants to stop, so call end_call.") + return None + return ( + "Answer what the user said in one or two short sentences, then ask whether they want " + "to talk to their new agent now." + ) + if self.step == "creating": + return "In a few words, tell the user their agent is still being created." + return None + + def _say(self, instructions: str): + """Has the model say something code decided. Returns its SpeechHandle.""" + self.scripted += 1 + return self.session.generate_reply(instructions=instructions, tool_choice="none") + + async def _ask(self, question: str, options: list[tuple[str, str]], say: str) -> None: + """Shows a choice card and asks the question.""" + card = await run_lk_tool( + "show_choice", + {"question": question, "options": [{"id": i, "name": n} for i, n in options]}, + ) + self.card = card + if lk is not None: + lk.on_choice(card, self._clicked) + self._say(say) + + async def _answered(self, option: str) -> None: + """Marks the open choice card with an answer given by voice.""" + if self.card: + await run_lk_tool("choice_made", {"id": self.card, "option": option}) + self.card = "" + + async def _clicked(self, option: str) -> None: + """Handles an answer clicked on the open choice card.""" + self.card = "" + if self.step == "language" and option == "cancel": + self._done("The user chose not to create an agent now. Nothing was created.") + elif self.step == "language" and option in STARTERS: + self._say("In one short sentence, tell the user you're creating their agent now.") + await self._create(option) + elif self.step == "try": + await self._finish(option == "try") + + async def _create(self, language: str) -> None: + self.step = "creating" + self.language = language + args = {"template": STARTERS[language], "in_current_directory": True, "confirmed": True} + result = await run_lk_tool("create_project", args, timeout=600) + if result.startswith("Error") or "only available" in result: + self._done(f"Creating the project failed: {result}") + return + self.step = "try" + await self._ask( + "Talk to your new agent now?", + [("try", "Talk to it"), ("later", "Not now")], + "In one sentence, tell the user their agent is ready, and ask whether they want to " + "talk to it now.", + ) + + async def _finish(self, yes: bool) -> None: + if not yes: + self._done("Created the project. The user didn't want to talk to it yet.") + return + self.step = "finished" + # Say the whole hand-off before the agent starts: once it connects, it + # takes over the audio, and anything still playing is cut off. + await self._say( + "In two short sentences, tell the user you're starting their agent, that " + + STARTER_OPENINGS[self.language] + + ", and that they can press B to come back to you." + ).wait_for_playout() + await run_lk_tool("try_project", {"confirmed": True}) + self.complete(STARTED) + + def _done(self, result: str) -> None: + if self.step == "finished": + return + self.step = "finished" + self.complete(result) + + def own_session_code() -> str: """Returns this agent's AgentSession setup, from its own source.""" try: @@ -636,6 +899,8 @@ def own_session_code() -> str: # Show the model names themselves, not the constants. for name, value in (("STT_MODEL", STT_MODEL), ("LLM_MODEL", LLM_MODEL), ("TTS_MODEL", TTS_MODEL)): code = code.replace(f"model={name}", f'model="{value}"') + # Leave out a setting that's only for comparing configurations. + code = re.sub(r"\n\s*extra_kwargs=.*REASONING_EFFORT.*", "", code) return code @@ -666,6 +931,7 @@ def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) - self._next_id = 0 self._session: AgentSession | None = None self._read_task: asyncio.Task[None] | None = None + self._choice_handlers: dict[str, Callable[[str], Awaitable[None]]] = {} def start(self, session: AgentSession) -> None: self._session = session @@ -675,6 +941,10 @@ def send(self, msg: dict) -> None: if not self._writer.is_closing(): self._writer.write((json.dumps(msg) + "\n").encode()) + def on_choice(self, card: str, handler: Callable[[str], Awaitable[None]]) -> None: + """Calls handler with the option the user clicks on a choice card.""" + self._choice_handlers[card] = handler + async def request(self, msg: dict, timeout: float = 10) -> dict: """Sends msg and waits for lk's reply.""" self._next_id += 1 @@ -697,6 +967,10 @@ async def _read(self) -> None: fut = self._pending.get(msg.get("id", "")) if fut and not fut.done(): fut.set_result(msg) + elif msg.get("type") == "choice": + # The user clicked an option on a choice card. + if handler := self._choice_handlers.pop(msg.get("id", ""), None): + asyncio.create_task(handler(msg.get("option", ""))) elif msg.get("type") == "user_text" and self._session is not None: # The user typed instead of speaking: stop talking and answer it. # Speech that can't be interrupted, such as a running @@ -772,65 +1046,73 @@ async def connect_lk(session: AgentSession) -> None: session.output.transcription = sync.text_output -def speak_during_lookups(session: AgentSession) -> None: - """Says a short line during a docs lookup, so the user doesn't sit in - silence. Skipped if the agent already spoke this turn. With FILLER "slow", - it speaks only if a lookup is still running after SLOW_LOOKUP_SECONDS, so - the line never comes after the lookup, right before the answer.""" - if FILLER == "off": - return - spoke = False - running: set[str] = set() # call IDs of docs lookups in progress - pending: asyncio.Task | None = None +TURN_ANSWERS = { + "leave": ( + "clearly asks to close LK or end the conversation now, such as \"exit\" or \"that's all, " + "bye\". Apologies, fillers, fragments, and thanks in the middle of a conversation, such as " + "\"Sorry,\" or \"Um\" or \"Thanks, and\", are not leaving, and neither are questions about " + "closing or exiting something else, such as \"How do I exit the console?\"" + ), + "code": "asks to see code, an example, a snippet, or how something looks in code", + "other": "says anything else", +} - def cancel() -> None: - nonlocal pending - if pending is not None: - pending.cancel() - pending = None - @session.on("conversation_item_added") - def _on_item(ev) -> None: - nonlocal spoke - if getattr(ev.item, "role", None) == "user": - spoke = False +def watch_user_turns(session: AgentSession) -> None: + """Acts on what each user turn asks for, when the model can't be relied on to. + + After each user turn, a short model call in the background reads the turn. It + runs beside the reply, so it adds no delay. + - If the user wants to leave, it tells lk to close once the agent is quiet, + even if the model didn't call end_call. + - If the user asked for code and the reply ends without a code card, it asks + for one more reply that must call show_code. + """ + log = logging.getLogger("lk-assistant") + listening = asyncio.Event() # set when a reply ends and the agent listens again @session.on("agent_state_changed") def _on_state(ev) -> None: - nonlocal spoke - if ev.new_state == "speaking": - spoke = True - cancel() - - async def say_if_slow() -> None: - await asyncio.sleep(SLOW_LOOKUP_SECONDS) - if running and not spoke: - say() - - def say() -> None: - nonlocal spoke - spoke = True - session.say(random.choice(LOOKUP_FILLERS), add_to_chat_ctx=False) - - @session.on("tool_execution_updated") - def _on_tool(ev) -> None: - nonlocal pending - update = ev.update - if update.type == "tool_call_ended": - running.discard(update.call_id) - if not running: - cancel() - return - if update.type != "tool_call_started" or update.function_call.name not in DOCS_TOOLS: - return - running.add(update.function_call.call_id) - if spoke: + if ev.new_state == "listening" and ev.old_state in ("thinking", "speaking"): + listening.set() + + async def check(item, conversation: str) -> None: + try: + listening.clear() + answer = await read_answer(session.llm, conversation, TURN_ANSWERS, item.text_content) + if answer == "leave" and lk is not None: + lk.send({"type": "end"}) + elif answer == "code": + await asyncio.wait_for(listening.wait(), timeout=60) + await show_code_if_missing(item) + except asyncio.TimeoutError: + pass + except Exception: + log.exception("couldn't check the user's turn") + + async def show_code_if_missing(item) -> None: + if not isinstance(session.current_agent, Assistant): + return # a task, such as creating the user's agent, is running + items = session.history.items + after = [i for i in items[next((n for n, i in enumerate(items) if i.id == item.id), len(items)) + 1 :]] + if any(i.type == "message" and i.role == "user" for i in after): + return # the user has moved on + if any(i.type == "function_call" and i.name == "show_code" for i in after): return - if FILLER == "slow": - if pending is None or pending.done(): - pending = asyncio.create_task(say_if_slow()) + session.generate_reply( + instructions=( + "The user asked for code, and you haven't shown it. Call show_code now with a short " + "example from the docs page you read, then say one short sentence about it." + ), + tool_choice={"type": "function", "function": {"name": "show_code"}}, + ) + + @session.on("conversation_item_added") + def _on_item(ev) -> None: + item = ev.item + if getattr(item, "role", None) != "user" or not item.text_content: return - say() + asyncio.create_task(check(item, recent_lines(session.history, item.id))) server = AgentServer() @@ -852,8 +1134,16 @@ async def assistant(ctx: agents.JobContext): # - TTS: Cartesia instead of Fish Audio, for inline phoneme overrides that # make it say "LiveKit" correctly every time. session = AgentSession( - stt=inference.STT(model=STT_MODEL, language="en"), - llm=inference.LLM(model=LLM_MODEL), + stt=inference.STT( + model=STT_MODEL, + language="en", + # Help the STT hear LiveKit's names and terms. + extra_kwargs={"keyterms_prompt": STT_KEYTERMS}, + ), + llm=inference.LLM( + model=LLM_MODEL, + extra_kwargs={"reasoning_effort": REASONING_EFFORT} if REASONING_EFFORT else {}, + ), tts=inference.TTS( model=TTS_MODEL, voice="a167e0f3-df7e-4d52-a9c3-f949145efdab", @@ -867,6 +1157,9 @@ async def assistant(ctx: agents.JobContext): # pauses; the higher floor leaves room for a short one. endpointing={"mode": "dynamic", "min_delay": 0.8}, ), + # A reply after a docs lookup reads a lot of context. The default + # 10-second timeout can cut it off. + conn_options=SessionConnectOptions(llm_conn_options=APIConnectOptions(timeout=20.0)), # The LLM marks up emotion and pacing for the TTS. The markup is # stripped from transcripts. expressive=True, @@ -878,6 +1171,9 @@ async def assistant(ctx: agents.JobContext): ], ) + # Fetch the docs table of contents, or read it from the cache, off the + # event loop; the instructions read it from the cache. + await asyncio.to_thread(docs_toc) folder, coding_agents = load_folder(), load_coding_agents() if sim := ctx.simulation_context(): # Agent Simulations (see simulations/README.md): the scenario sets the @@ -887,7 +1183,7 @@ async def assistant(ctx: agents.JobContext): folder, coding_agents = setup_simulation(sim, session, Assistant) - speak_during_lookups(session) + watch_user_turns(session) await session.start(room=ctx.room, agent=Assistant(folder, coding_agents)) await connect_lk(session) diff --git a/pkg/assistant/agent/evals/test_onboarding.py b/pkg/assistant/agent/evals/test_onboarding.py index c4a14884..d03ec35c 100644 --- a/pkg/assistant/agent/evals/test_onboarding.py +++ b/pkg/assistant/agent/evals/test_onboarding.py @@ -61,6 +61,10 @@ def stub_lk(calls: list, ready: bool = False): async def run_lk_tool(name: str, args: dict | None = None, timeout: float = 60) -> str: calls.append((name, args or {})) + if name == "show_choice": + return f"choice{len(calls)}" + if name == "choice_made": + return "Marked." if name in ("create_project", "set_up_coding_agent", "try_project") and (args or {}).get("confirmed"): return "Done. The user confirmed by voice, and it ran." if name == "create_project": @@ -113,11 +117,10 @@ def judge_llm() -> inference.LLM: async def test_creates_project_without_asking_twice(monkeypatch, lk_calls): - """The user agreed to an offer; LK shouldn't ask again before running it.""" + """The user said what they want; LK creates it without asking again.""" async with AgentSession(llm=inference.LLM(model=agent.LLM_MODEL)) as session: await session.start(assistant(monkeypatch, EMPTY)) - await session.run(user_input="I want to build a voice agent in Python.") - result = await session.run(user_input="Yes, create it here.") + await session.run(user_input="I want to build a voice agent in Python, right here.") # Either path runs it in this turn: confirmed=true, or confirm_action right after. ran = any( (n == "create_project" and a.get("confirmed")) or (n == "confirm_action" and a.get("approved")) diff --git a/pkg/assistant/agent/lk_simulation.py b/pkg/assistant/agent/lk_simulation.py index 7e9b7ae4..c409d05f 100644 --- a/pkg/assistant/agent/lk_simulation.py +++ b/pkg/assistant/agent/lk_simulation.py @@ -12,6 +12,7 @@ not_calls: [set_up_coding_agent] # tools it must not call any_calls: [show_link, show_answer] # it must call at least one show_code_contains: lk agent console + lk_calls: [create_project] # lk tools that must run, from code or the model confirmed: [create_project] # ran without asking again call_contains: {hand_off_task: voice} @@ -21,10 +22,12 @@ from __future__ import annotations +import inspect import json +import sys + from livekit.agents import AgentSession -from livekit.agents.voice.run_result import mock_tools DIR = "/Users/dev/my-app" @@ -33,10 +36,19 @@ def setup_simulation(sim, session: AgentSession, assistant_cls) -> tuple[dict, d userdata = sim.userdata() or {} folder = userdata.get("folder", {"kind": "empty"}) coding_agents = userdata.get("coding_agents", {"ready": False}) - mock_tools(assistant_cls, LkMocks(folder, coding_agents).tools(), session=session) + # The agent's tools and its tasks reach lk through run_lk_tool. Replace it + # in the module that defines the agent, which runs as __main__. Each + # simulation job runs in its own process, so this is per job. + global mocks + mocks = LkMocks(folder, coding_agents) + sys.modules[assistant_cls.__module__].run_lk_tool = mocks.run return folder, coding_agents +# The job's mocks, so check_simulation can see which lk tools ran. +mocks: "LkMocks | None" = None + + class LkMocks: # Each mock takes the tool's RunContext first, as the real tools do: the # SDK passes it positionally. @@ -46,6 +58,23 @@ def __init__(self, folder: dict, coding_agents: dict) -> None: self.pending: dict[str, str] = {} # confirmation id → what it runs self.n = 0 self.created = False + self.calls: list[str] = [] # lk tools run, in order + + async def run(self, name: str, args: dict | None = None, timeout: float = 60) -> str: + """Stands in for agent.run_lk_tool.""" + args = args or {} + self.calls.append(name) + if name == "show_choice": + self.n += 1 + return f"choice{self.n}" + if name == "choice_made": + return "Marked." + tool = self.tools().get(name) + if tool is None: + return f"Error: unknown tool {name!r}" + # Pass only the arguments this mock takes, so older agents work too. + params = inspect.signature(tool).parameters + return await tool(None, **{k: v for k, v in args.items() if k in params}) def tools(self) -> dict: return { @@ -61,7 +90,7 @@ def tools(self) -> dict: def propose(self, what: str, command: str, confirmed: bool) -> str: if confirmed: - return self.run(what) + return self.perform(what) self.n += 1 cid = f"confirm{self.n}" self.pending[cid] = what @@ -72,12 +101,13 @@ def propose(self, what: str, command: str, confirmed: bool) -> str: "they decline, approved=false." ) - def run(self, what: str) -> str: - if what == "create": + def perform(self, what: str) -> str: + if what.startswith("create"): + template = what.partition(":")[2] or "agent-starter-python" self.created = True self.folder = {"kind": "livekit", "has_agents": True} return ( - "Created an agent project from the agent-starter-python template in the current " + f"Created an agent project from the {template} template in the current " "directory and installed its dependencies. It includes LiveKit's skills and Docs MCP " "server for coding agents." ) @@ -117,8 +147,8 @@ async def list_templates(self, context) -> str: async def create_project( self, context, template: str, name: str = "", in_current_directory: bool = False, confirmed: bool = False ) -> str: - target = "." if in_current_directory else name - return self.propose("create", f"lk agent init {target} --template {template} --install", confirmed) + target = "." if in_current_directory else (name or "my-agent") + return self.propose(f"create:{template}", f"lk agent init {target} --template {template} --install", confirmed) async def set_up_coding_agent( self, context, all_projects: bool = False, reinstall: bool = False, confirmed: bool = False @@ -137,7 +167,7 @@ async def confirm_action( return f'Error: there\'s no pending confirmation "{confirmation_id}"' if not approved: return "Cancelled. Nothing was changed." - return self.run(what) + return self.perform(what) async def try_project( self, context, confirmed: bool = False) -> str: @@ -172,6 +202,10 @@ def check_simulation(ctx) -> None: if any_of := expect.get("any_calls"): if not any(name in names for name in any_of): problems.append(f"didn't call any of {', '.join(any_of)}") + ran = mocks.calls if mocks else [] + for name in expect.get("lk_calls", []): + if name not in ran: + problems.append(f"didn't run lk tool {name}") for name in expect.get("not_calls", []): if name in names: problems.append(f"called {name}") diff --git a/pkg/assistant/agent/simulations/README.md b/pkg/assistant/agent/simulations/README.md index cea6695d..cba67de7 100644 --- a/pkg/assistant/agent/simulations/README.md +++ b/pkg/assistant/agent/simulations/README.md @@ -23,7 +23,9 @@ Set these environment variables to compare configurations. The defaults are what | Variable | Default | Values | |---|---|---| | `LK_ASSISTANT_LLM` | `openai/gpt-5.4-mini` | A LiveKit Inference LLM, such as `google/gemini-3.8-flash` | -| `LK_ASSISTANT_PRIMER` | `1` | `1` adds `PRIMER` to the instructions, so basic questions don't need a lookup. `0` looks up everything. | -| `LK_ASSISTANT_FILLER` | `slow` | When to say a line during a docs lookup: `always`, `slow` (only after 2.5 seconds, at most once per turn), or `off` | +| `LK_ASSISTANT_MAX_PAGES` | `3` | How many docs pages one reply may read. Fewer is faster. | +| `LK_ASSISTANT_DOCS_CHARS` | `12000` | How much docs text one `read_docs` call returns, split between its pages. | +| `LK_ASSISTANT_TOC` | `1` | `1` adds the trimmed docs table of contents to the instructions, so the agent picks pages by title. | +| `LK_ASSISTANT_REASONING` | `low` | The LLM's `reasoning_effort`, such as `low` or `medium`. Empty uses the model's default. Lower is faster. | Each run registers its own temporary agent name, so you can run two configurations at once. diff --git a/pkg/assistant/agent/simulations/scenarios.yaml b/pkg/assistant/agent/simulations/scenarios.yaml index a86439ee..22ccfda9 100644 --- a/pkg/assistant/agent/simulations/scenarios.yaml +++ b/pkg/assistant/agent/simulations/scenarios.yaml @@ -20,7 +20,7 @@ scenarios: folder: {kind: empty} coding_agents: {ready: false} expect: - calls: [create_project, try_project] + lk_calls: [create_project, try_project] - id: realtime-vs-cascaded label: Choosing between a realtime model and the cascaded pipeline @@ -40,6 +40,8 @@ scenarios: userdata: folder: {kind: empty} coding_agents: {ready: false} + expect: + call_contains: {read_docs: pipelines} - id: how-agents-join-rooms label: Frontend developer asks how the agent and the app connect @@ -123,9 +125,9 @@ scenarios: it. If it suggests handing it to your coding agent, agree. You can see the cards LK shows on screen, such as prompts for your coding agent, so you don't need LK to repeat them. agent_expectations: >- - LK doesn't claim to have read or fixed the code, and doesn't guess at a fix. It tells the - user it's giving the task to their coding agent, as a prompt on screen. It's brief. Don't - grade tool calls or the prompt's text; they're checked separately. + LK doesn't claim to have read or fixed the code. It's brief. The judge can't see the + prompt card LK shows, so don't grade whether a prompt was shown or what it says; code + checks that separately. tags: area: scope userdata: @@ -172,7 +174,7 @@ scenarios: folder: {kind: empty} coding_agents: {ready: false} expect: - confirmed: [create_project] + lk_calls: [create_project] - id: change-voice label: The user wants a different voice for the agent they just made @@ -183,10 +185,10 @@ scenarios: such as prompts for your coding agent, so you don't need LK to repeat them. agent_expectations: >- LK acts as the expert. It suggests one or two concrete voice options, or asks one short - question about the voice the user wants, and then puts a prompt for the user's coding agent - on screen without asking whether to. It doesn't end its replies with offers such as "If you - want, I can". It's brief. Don't grade tool calls or the prompt's text; they're checked - separately. + question about the voice the user wants, and doesn't ask whether to write the change up. + It doesn't end its replies with offers such as "If you want, I can". It's brief. The judge + can't see the prompt card LK shows, so don't grade whether a prompt was shown or what it + says; code checks that separately. tags: area: expertise userdata: @@ -233,5 +235,56 @@ scenarios: userdata: folder: {kind: empty} coding_agents: {ready: false} + + - id: confirm-before-create + label: The user asks to build an agent, naming a language + instructions: >- + Ask, in one sentence, whether LK can start building a hotel reception agent here in Python. + If LK asks you to confirm, say yes. If LK says it created a project without asking you + first, say you didn't want it yet. + agent_expectations: >- + Before it creates anything, LK asks the user to confirm, in one short question. It creates + the project only after the user says yes. It doesn't pick a different language than the user + asked for. Don't grade tool calls; they're checked separately. + tags: + area: getting-started + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + expect: + lk_calls: [create_project] + + - id: unsure-language + label: The user is unsure which language, then picks Python and tries it + instructions: >- + Ask LK to build a voice agent in this folder. When LK asks Python or Node.js, ask "Uh, what + do I choose?" After LK explains, say "Python." When LK asks whether to talk to it, say + "Let's chat." + agent_expectations: >- + LK helps the user choose without choosing for them. After the user says Python, LK tells the + user it's creating the agent, then says it's ready and asks whether to talk to it. It never + says a step happened before it did. Don't grade tool calls; they're checked separately. + tags: + area: getting-started + userdata: + folder: {kind: empty} + coding_agents: {ready: false} + expect: + lk_calls: [create_project, try_project] + + - id: show-example + label: The user asks for an example of a feature + instructions: >- + Ask how LiveKit handles turn-taking. After LK answers, say "Show me an example of the + adaptive interruption." Then say "Copy." + agent_expectations: >- + LK shows a code example for adaptive interruption handling on screen and doesn't read code + aloud. When the user says copy, LK says what it copied. Don't grade tool calls; they're + checked separately. + tags: + area: expertise + userdata: + folder: {kind: empty} + coding_agents: {ready: false} expect: - any_calls: [show_answer] + calls: [show_code] diff --git a/pkg/overlay/ui/index.html b/pkg/overlay/ui/index.html index a5820764..bc984b27 100644 --- a/pkg/overlay/ui/index.html +++ b/pkg/overlay/ui/index.html @@ -58,6 +58,7 @@ #status { position: fixed; left: 0; right: 0; bottom: 28px; text-align: center; color: var(--fg4); transition: color 200ms; } #status[data-state="listening"] { color: var(--fg1); } + body.mic-off #status[data-state="listening"], body.mic-off #status[data-state="idle"] { color: var(--remove); } #status[data-state="thinking"], #status[data-state="speaking"], #status.notice { color: var(--accent); } #hints { position: fixed; right: 32px; bottom: 28px; color: var(--fg4); } @@ -552,7 +553,9 @@ if (!LABELS[next]) return; const changed = next !== state; state = next; - stateLabel = label || LABELS[next]; + // With the microphone off, LK can't hear the user, so don't say it's listening. + const deaf = micOff && (next === 'listening' || next === 'idle'); + stateLabel = label || (deaf ? 'Microphone off · Press M' : LABELS[next]); statusEl.dataset.state = next; if (!notice) statusEl.textContent = stateLabel; if (changed && EARCONS[next]) EARCONS[next](); @@ -601,7 +604,7 @@ const kind = { starting: 'Starting your agent', connected: 'Connected', failed: "Your agent didn't start" }[status]; const text = { starting: 'Installing and connecting. The first run takes longer.', - connected: 'Say hello to start. Starter agents wait for you to speak first.', + connected: "Talk to it. Say hello if it doesn't greet you. Press B to come back.", failed: 'See the terminal for what went wrong.', }[status]; takeoverEl.querySelector('.kind').textContent = kind; @@ -633,7 +636,7 @@ showTakeover('connected', name); guestWhat('Talking to', name); // The user's agent waits for them to speak first. - guestEl.querySelector('.hint').textContent = 'Say hello to start'; + guestEl.querySelector('.hint').textContent = "Say hello if it doesn't greet you"; } body.classList.toggle('guest', speaker.guest); // X-ray follows whichever agent is talking. @@ -879,7 +882,9 @@ card.innerHTML = '
    ' +
       '
    ';
     card.querySelector('.title').textContent = heading;
-    card.querySelector('.command').textContent = command;
+    // A choice card asks a question and has no command.
+    if (command) card.querySelector('.command').textContent = command;
+    else card.querySelector('.command').remove();
     card.querySelector('.detail').textContent = detail;
     const row = card.querySelector('.options');
     options.forEach((o, i) => {
@@ -898,7 +903,7 @@
     stickToBottom(() => turnBody(turn, 'agent').append(card));
     confirms.set(id, card);
     pendingConfirm = id;
-    if (notice !== -1) showNotice('Waiting for you to confirm');
+    if (notice !== -1) showNotice(command ? 'Waiting for you to confirm' : 'Waiting for your choice');
     EARCONS.listening();
   }
 
@@ -915,7 +920,8 @@
     const card = confirms.get(id);
     if (!card) return;
     card.dataset.status = status;
-    card.querySelector('.result .label').textContent = CONFIRM_STATUS[status] || status;
+    const choice = !card.querySelector('.command');
+    card.querySelector('.result .label').textContent = choice && status === 'done' ? 'Chose' : CONFIRM_STATUS[status] || status;
     card.querySelector('.result .text').textContent = text || '';
   }
 
@@ -1076,6 +1082,7 @@
       micOff = !micOff;
       body.classList.toggle('mic-off', micOff);
       post({ type: 'mute', on: micOff });
+      setState(state);
       if (notice !== -1) showNotice(micOff ? 'Microphone off' : 'Microphone on');
       return;
     }

From 8fe4cd75cd7802018b0299e4a5d86226c2a3f25a Mon Sep 17 00:00:00 2001
From: royalfig 
Date: Thu, 15 Oct 2026 10:52:49 -0400
Subject: [PATCH 10/12] fix(assistant): copy cards by kind, confirm restarts,
 and tidy cards

- Copy the latest card of the kind the user asked for (link, code, or
  prompt), matching titles loosely, and report exactly what was copied.
- Always confirm before restarting the user's agent.
- Don't repeat a code card that's already on screen, and show one link card
  per reply.

Co-Authored-By: Claude Opus 5.5 
---
 cmd/lk/assistant_session.go      | 77 +++++++++++++++++++++-----------
 cmd/lk/assistant_session_test.go | 26 +++++++++++
 cmd/lk/assistant_tools.go        |  2 +-
 pkg/assistant/agent/agent.py     | 42 +++++++++--------
 4 files changed, 103 insertions(+), 44 deletions(-)

diff --git a/cmd/lk/assistant_session.go b/cmd/lk/assistant_session.go
index 93524c3d..1456b196 100644
--- a/cmd/lk/assistant_session.go
+++ b/cmd/lk/assistant_session.go
@@ -18,7 +18,6 @@ import (
 	"bufio"
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math"
 	"net"
@@ -859,7 +858,7 @@ func describeTool(name, arguments string) (label, detail string) {
 	case "list_templates":
 		return "Listed the templates", detail
 	case "copy_to_clipboard":
-		return "Copied to your clipboard", detail
+		return "Copy to clipboard", detail
 	case "create_project":
 		return "Create a project", detail
 	case "confirm_action":
@@ -927,6 +926,13 @@ func (s *assistantSession) showCode(fc *agent.FunctionCall) {
 		return
 	}
 	s.shownCalls[fc.GetCallId()] = true
+	// The same code is already on screen, such as when the user asks to walk
+	// through it: don't show it again.
+	for _, code := range s.snippets {
+		if code == args.Code {
+			return
+		}
+	}
 	s.agentTurnForReply()
 
 	id := fc.GetCallId()
@@ -999,13 +1005,16 @@ func (s *assistantSession) showLink(fc *agent.FunctionCall) {
 	if args.Title == "" {
 		args.Title = "LiveKit docs"
 	}
+	s.turnCard = true
+	// The agent can call show_link before it answers. Show the card after the
+	// answer, when the reply ends (see showReadMore), and only one per reply.
+	if len(s.pendingLinks) > 0 {
+		return
+	}
 	s.agentTurnForReply()
 	id := fc.GetCallId()
 	s.links[id] = args.URL
-	s.turnCard = true
 	s.addSnippet(id, "link", args.Title, args.URL)
-	// The agent can call show_link before it answers. Show the card after the
-	// answer, when the reply ends (see showReadMore).
 	s.pendingLinks = append(s.pendingLinks, overlayLink{Type: "link", Turn: s.agentTurn, ID: id, Title: args.Title, URL: args.URL})
 }
 
@@ -1119,32 +1128,50 @@ func (s *assistantSession) openLink(id string) {
 	}
 }
 
-// copyByVoice copies the latest card whose title contains title, or the
-// latest card if title is empty, and replies with the title it copied.
-func (s *assistantSession) copyByVoice(requestID, title string) {
-	want := strings.ToLower(strings.TrimSpace(title))
-	for i := len(s.snippetOrder) - 1; i >= 0; i-- {
-		ref := s.snippetOrder[i]
-		if want != "" && !strings.Contains(strings.ToLower(ref.title), want) {
-			continue
-		}
-		if err := clipboard.WriteAll(s.snippets[ref.id]); err != nil {
-			s.replyTool(requestID, "", fmt.Errorf("couldn't copy to the clipboard: %w", err))
-			return
+// copyByVoice copies the latest card of kind ("code", "link", "prompt", or
+// "latest" for any kind). title, if given, picks among cards of that kind by
+// the words they share, so the agent doesn't have to know exact titles. The
+// reply says exactly what was copied, or that there was nothing to copy.
+func (s *assistantSession) copyByVoice(requestID, kind, title string) {
+	ref, ok := s.pickCard(kind, title)
+	if !ok {
+		what := "card"
+		if kind != "" && kind != "latest" {
+			what = kind + " card"
 		}
-		s.out.Send(overlayCopied{"copied", ref.id})
-		s.replyTool(requestID, fmt.Sprintf("Copied the %s card %q to the user's clipboard. Tell them what you copied.", ref.kind, ref.title), nil)
+		s.replyTool(requestID, "", fmt.Errorf("nothing was copied: there's no %s on screen. Tell the user", what))
 		return
 	}
-	if len(s.snippetOrder) == 0 {
-		s.replyTool(requestID, "", errors.New("there's nothing on screen to copy yet; show it with show_code or hand_off_task first"))
+	if err := clipboard.WriteAll(s.snippets[ref.id]); err != nil {
+		s.replyTool(requestID, "", fmt.Errorf("nothing was copied: %w", err))
 		return
 	}
-	var titles []string
-	for _, ref := range s.snippetOrder {
-		titles = append(titles, ref.title)
+	s.out.Send(overlayCopied{"copied", ref.id})
+	s.replyTool(requestID, fmt.Sprintf("Copied the %s card %q. Tell the user in a few words what you copied.", ref.kind, ref.title), nil)
+}
+
+// pickCard returns the card to copy: of kind, and with the most words in
+// common with title, preferring the latest.
+func (s *assistantSession) pickCard(kind, title string) (snippetRef, bool) {
+	words := strings.Fields(strings.ToLower(title))
+	best, bestScore, found := snippetRef{}, -1, false
+	for i := len(s.snippetOrder) - 1; i >= 0; i-- {
+		ref := s.snippetOrder[i]
+		if kind != "" && kind != "latest" && ref.kind != kind {
+			continue
+		}
+		score := 0
+		cardTitle := strings.ToLower(ref.title)
+		for _, w := range words {
+			if len(w) > 2 && strings.Contains(cardTitle, w) {
+				score++
+			}
+		}
+		if score > bestScore {
+			best, bestScore, found = ref, score, true
+		}
 	}
-	s.replyTool(requestID, "", fmt.Errorf("no card on screen has %q in its title; the cards are: %s", title, strings.Join(titles, ", ")))
+	return best, found
 }
 
 // normalizeLang maps language names to the ones the page highlights.
diff --git a/cmd/lk/assistant_session_test.go b/cmd/lk/assistant_session_test.go
index b7e17a9d..869ca1fd 100644
--- a/cmd/lk/assistant_session_test.go
+++ b/cmd/lk/assistant_session_test.go
@@ -183,3 +183,29 @@ func TestDocsPagesRead(t *testing.T) {
 	pages := docsPagesRead(`{"paths": ["/agents/build.md", "https://github.com/livekit/agents/blob/main/README.md"]}`, out)
 	require.Equal(t, []docsPage{{title: "Building voice agents", url: "https://docs.livekit.io/agents/build"}}, pages)
 }
+
+func TestPickCard(t *testing.T) {
+	s := &assistantSession{snippets: map[string]string{}}
+	s.addSnippet("c1", "code", "Terminal", "lk agent simulate text -n 10")
+	s.addSnippet("l1", "link", "Debugging and testing overview", "https://docs.livekit.io/testing/overview")
+	s.addSnippet("l2", "link", "Agent Simulations", "https://docs.livekit.io/testing/simulations")
+	s.addSnippet("p1", "prompt", "Prompt for your coding agent", "Add tools.")
+
+	pick := func(kind, title string) string {
+		ref, ok := s.pickCard(kind, title)
+		if !ok {
+			return ""
+		}
+		return ref.id
+	}
+	require.Equal(t, "p1", pick("latest", ""))
+	require.Equal(t, "l2", pick("link", ""))
+	require.Equal(t, "l1", pick("link", "the debugging overview link"))
+	require.Equal(t, "l1", pick("link", "Agent Simulations, Debugging and testing overview")) // the most shared words
+	require.Equal(t, "l2", pick("link", "Simulations"))
+	require.Equal(t, "c1", pick("code", "Agent Console"))
+	require.Equal(t, "p1", pick("prompt", ""))
+	empty := &assistantSession{snippets: map[string]string{}}
+	_, ok := empty.pickCard("link", "")
+	require.False(t, ok)
+}
diff --git a/cmd/lk/assistant_tools.go b/cmd/lk/assistant_tools.go
index 5f501400..4890caab 100644
--- a/cmd/lk/assistant_tools.go
+++ b/cmd/lk/assistant_tools.go
@@ -167,7 +167,7 @@ func (s *assistantSession) runTool(requestID, name string, args map[string]any)
 		s.replyTool(requestID, "Marked.", nil)
 
 	case "copy_to_clipboard":
-		s.copyByVoice(requestID, str("title"))
+		s.copyByVoice(requestID, str("kind"), str("title"))
 
 	case "try_project":
 		s.proposeTryProject(requestID, flag("confirmed"))
diff --git a/pkg/assistant/agent/agent.py b/pkg/assistant/agent/agent.py
index 3ff370ea..d9f0c946 100644
--- a/pkg/assistant/agent/agent.py
+++ b/pkg/assistant/agent/agent.py
@@ -12,6 +12,7 @@
 import re
 import time
 import urllib.request
+from typing import Literal
 from collections.abc import Awaitable, Callable
 
 from livekit import agents
@@ -230,7 +231,7 @@ def greeting(folder: dict, coding_agents: dict) -> str:
   coding agent.
 
 set_up_coding_agent and try_project don't run right away. They show the command to the user and
-return a confirmation ID. Ask the user in one short sentence whether to go ahead. They can answer
+return a confirmation ID. try_project always waits for the user's answer. Ask the user in one short sentence whether to go ahead. They can answer
 by voice or on screen. If they agree by voice, call confirm_action with the ID and approved=true;
 if they decline, approved=false. Never call confirm_action unless the user has just agreed.
 
@@ -470,15 +471,23 @@ async def show_link(self, context: RunContext, title: str, url: str) -> str:
         return "The link is on the user's screen. Don't repeat what you already said."
 
     @function_tool()
-    async def copy_to_clipboard(self, context: RunContext, title: str = "") -> str:
-        """Copy a code, command, or prompt card on screen to the user's clipboard. Use it only
-        when the user asks you to copy something.
+    async def copy_to_clipboard(
+        self,
+        context: RunContext,
+        kind: Literal["latest", "code", "link", "prompt"] = "latest",
+        title: str = "",
+    ) -> str:
+        """Copy a card on screen to the user's clipboard. Use it only when the user asks you to
+        copy something. It copies the latest card of the kind they mean.
 
         Args:
-            title: Part of the card's title, such as "agent.py", "Terminal", or "Prompt". Leave it
-                empty for the most recent card.
+            kind: What the user wants copied: "link" for a Read more link, "code" for code or a
+                command, "prompt" for a prompt for their coding agent, or "latest" for the most
+                recent card of any kind.
+            title: Optional words from the card's title, only when there are several cards of
+                that kind and the user named one.
         """
-        return await run_lk_tool("copy_to_clipboard", {"title": title})
+        return await run_lk_tool("copy_to_clipboard", {"kind": kind, "title": title})
 
     @function_tool()
     async def hand_off_task(self, context: RunContext, task: str) -> str:
@@ -578,18 +587,15 @@ async def set_up_coding_agent(
         )
 
     @function_tool()
-    async def try_project(self, context: RunContext, confirmed: bool = False) -> str:
-        """Run the agent the user created in this session, in this window, so they can talk to
-        it. The conversation switches to their agent until they press B to come back to you.
-        Unless confirmed is true, it waits for the user to confirm (see confirm_action). For any
-        other agent, it tells you how the user can run it in their own terminal.
-
-        Args:
-            confirmed: True if the user's last message asked for this or agreed to it, such
-                as "set up a voice agent here" or "yes" after you offered it. Then it runs
-                right away, without asking again. Otherwise leave it false, and ask.
+    async def try_project(self, context: RunContext) -> str:
+        """Run the agent the user created in this session again, in this window, so they can talk
+        to it. Only call it when the user asks to talk to their agent again. "Start building" or
+        "let me get to work" means coding, not this. The conversation switches to their agent
+        until they press B to come back to you. It always waits for the user to confirm on screen
+        or by voice (see confirm_action). For any other agent, it tells you how the user can run
+        it in their own terminal.
         """
-        return await run_lk_tool("try_project", {"confirmed": confirmed})
+        return await run_lk_tool("try_project", {"confirmed": False})
 
     @function_tool()
     async def explain_yourself(self, context: RunContext) -> str:

From 86b43bd7c0f3bc38cfd072a4f4bad53172cd3372 Mon Sep 17 00:00:00 2001
From: royalfig 
Date: Thu, 15 Oct 2026 10:54:16 -0400
Subject: [PATCH 11/12] fix: build against main and fix the entrypoint test on
 Windows

- Pass the context and the user's folder to resolveCredentials, which now
  reads .env files from that folder.
- Compare findEntrypoint's result with forward slashes, which it returns on
  every platform.

Co-Authored-By: Claude Opus 5.5 
---
 cmd/lk/assistant_session.go        | 8 +++++++-
 cmd/lk/simulate_subprocess_test.go | 4 +++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/cmd/lk/assistant_session.go b/cmd/lk/assistant_session.go
index 1456b196..90b2fc2c 100644
--- a/cmd/lk/assistant_session.go
+++ b/cmd/lk/assistant_session.go
@@ -205,7 +205,13 @@ func (s *assistantSession) run(ctx context.Context) (err error) {
 
 	s.connecting("Starting")
 
-	creds, err := resolveCredentials(s.cmd)
+	// Credentials come from the user's folder (its .env files, with
+	// --experimental-auth) or their lk config, not from the agent's folder.
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	creds, err := resolveCredentials(ctx, s.cmd, cwd)
 	if err != nil {
 		return errNoCredentials{err}
 	}
diff --git a/cmd/lk/simulate_subprocess_test.go b/cmd/lk/simulate_subprocess_test.go
index a5a7e60a..3e4e53f9 100644
--- a/cmd/lk/simulate_subprocess_test.go
+++ b/cmd/lk/simulate_subprocess_test.go
@@ -52,5 +52,7 @@ func TestFindEntrypointIgnoresParentProject(t *testing.T) {
 
 	entry, err := findEntrypoint(child, "", agentfs.ProjectTypePythonUV)
 	require.NoError(t, err)
-	require.Equal(t, filepath.Join("src", "agent.py"), entry)
+	// findEntrypoint returns its root candidates with forward slashes on
+	// every platform.
+	require.Equal(t, "src/agent.py", filepath.ToSlash(entry))
 }

From 80ed7d2942603074f0f0162c098cafcb93c690f0 Mon Sep 17 00:00:00 2001
From: royalfig 
Date: Thu, 15 Oct 2026 11:04:22 -0400
Subject: [PATCH 12/12] fix(assistant): apply De Morgan's law in
 DescribeFolder, for lint

Co-Authored-By: Claude Opus 5.5 
---
 pkg/assistant/folder.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/assistant/folder.go b/pkg/assistant/folder.go
index 5348db8b..65734813 100644
--- a/pkg/assistant/folder.go
+++ b/pkg/assistant/folder.go
@@ -156,7 +156,7 @@ func DescribeFolder(dir string) Folder {
 	scan("", entries, 1)
 	// A workspace root, such as a pnpm or npm workspace, only ties its
 	// projects together, so it isn't a project of its own.
-	if rootFound && !(len(f.Projects) > 0 && isWorkspaceRoot(dir)) {
+	if rootFound && (len(f.Projects) == 0 || !isWorkspaceRoot(dir)) {
 		root.Path = "."
 		f.Projects = append([]Project{root}, f.Projects...)
 	}