Merge pull request #2822 from bmahabirbu/sandbox-router

Add router mode support to ramalama sandbox

Author: mikebonnet

container-images/pi-agent/Containerfile

@@ -2,7 +2,7 @@ FROM quay.io/fedora/fedora:44
 
 ARG PI_VERSION=0.80.7
 ARG NODE_VERSION=26.5.0
-ARG PI_LLAMA_CPP_VERSION=0.8.1
+ARG PI_LLAMA_SERVER_VERSION=1.1.0
 ARG PI_WEB_ACCESS_VERSION=0.13.0
 ARG TARGETARCH
 
@@ -43,7 +43,7 @@ RUN echo "Include /home/node/.ssh/config" >> /etc/ssh/ssh_config \
     && echo "StrictHostKeyChecking accept-new" >> /etc/ssh/ssh_config
 
 RUN npm install -g --ignore-scripts @earendil-works/pi-coding-agent@${PI_VERSION} \
-    && pi install npm:pi-llama-cpp@${PI_LLAMA_CPP_VERSION} \
+    && pi install npm:pi-llama-server@${PI_LLAMA_SERVER_VERSION} \
     && pi install npm:pi-web-access@${PI_WEB_ACCESS_VERSION} \
     && npm cache clean --force \
     && echo '{"workflow":"none"}' > /home/node/.pi/web-search.json \

docs/options/models-max.md

@@ -1,7 +1,7 @@
 ####> This option file is used in:
-####>   ramalama serve
+####>   ramalama sandbox goose, ramalama sandbox opencode, ramalama sandbox pi, ramalama serve
 ####> If this file is edited, make sure the changes
 ####> are applicable to all of those.
 #### **--models-max**=*integer*
 Maximum number of models to load concurrently in router mode (default: 4).
-Only used when `ramalama serve` is invoked in router mode (zero or multiple models).
+Only used when invoked in router mode (zero or multiple models).

docs/options/prompt.md

@@ -0,0 +1,6 @@
+####> This option file is used in:
+####>   ramalama sandbox goose, ramalama sandbox opencode, ramalama sandbox pi
+####> If this file is edited, make sure the changes
+####> are applicable to all of those.
+#### **--prompt**
+Instructions for the sandbox agent to process non-interactively.

docs/ramalama-sandbox-goose.1.md.in

@@ -4,16 +4,19 @@
 ramalama\-sandbox\-goose - run Goose in a sandbox, backed by a local AI Model
 
 ## SYNOPSIS
-**ramalama sandbox goose** [*options*] *model* [arg ...]
+**ramalama sandbox goose** [*options*] [*model* ...]
 
 ## DESCRIPTION
 Run Goose in a container, connected to a local model server also running
 in a container. Goose uses the model for reasoning and tool calling.
 
-When run with no arguments after the model, an interactive session is
-launched. If one or more arguments are provided, they are passed to the agent
-as instructions to process non-interactively. Commands may also be passed via
-stdin.
+When a single model is specified, the server loads that model directly.
+When zero or multiple models are specified, the server starts in router mode
+using llama.cpp's --models-dir, dynamically loading and unloading models.
+With no models, all GGUF models in the store are served.
+
+Use **--prompt** to pass instructions for non-interactive processing.
+Commands may also be passed via stdin.
 
 If the url is not given then two containers are started: a model server (llama-server) and the agent
 container. They communicate via container networking. When the agent session
@@ -54,6 +57,8 @@ Otherwise only the agent container starts using the given url for the openai com
 
 @@option model-draft
 
+@@option models-max
+
 @@option name
 
 @@option ncmoe
@@ -68,6 +73,8 @@ Otherwise only the agent container starts using the given url for the openai com
 
 @@option privileged
 
+@@option prompt
+
 @@option pull
 
 @@option runtime-args
@@ -122,7 +129,7 @@ ramalama sandbox goose -w ./src qwen3:4b
 
 Request the agent to perform actions non-interactively:
 ```
-ramalama sandbox goose -w ./src qwen3:4b Please analyze the source code in the current directory
+ramalama sandbox goose -w ./src qwen3:4b --prompt "Please analyze the source code in the current directory"
 ```
 
 Send instructions to the agent via stdin:

docs/ramalama-sandbox-opencode.1.md.in

@@ -4,16 +4,19 @@
 ramalama\-sandbox\-opencode - run OpenCode in a sandbox, backed by a local AI Model
 
 ## SYNOPSIS
-**ramalama sandbox opencode** [*options*] *model* [arg ...]
+**ramalama sandbox opencode** [*options*] [*model* ...]
 
 ## DESCRIPTION
 Run OpenCode in a container, connected to a local model server also running
 in a container. OpenCode uses the model for reasoning and tool calling.
 
-When run with no arguments after the model, an interactive TUI session is
-launched. If one or more arguments are provided, they are passed to the agent
-as instructions to process non-interactively. Commands may also be passed via
-stdin.
+When a single model is specified, the server loads that model directly.
+When zero or multiple models are specified, the server starts in router mode
+using llama.cpp's --models-dir, dynamically loading and unloading models.
+With no models, all GGUF models in the store are served.
+
+Use **--prompt** to pass instructions for non-interactive processing.
+Commands may also be passed via stdin.
 
 If the url is not given then two containers are started: a model server (llama-server) and the agent
 container. They communicate via container networking. When the agent session
@@ -52,6 +55,8 @@ Otherwise only the agent container starts using the given url for the openai com
 
 @@option model-draft
 
+@@option models-max
+
 @@option name
 
 @@option ncmoe
@@ -68,6 +73,8 @@ Otherwise only the agent container starts using the given url for the openai com
 
 @@option privileged
 
+@@option prompt
+
 @@option pull
 
 @@option runtime-args
@@ -122,7 +129,7 @@ ramalama sandbox opencode -w ./src qwen3:4b
 
 Request the agent to perform actions non-interactively:
 ```
-ramalama sandbox opencode -w ./src qwen3:4b Please analyze the source code in the current directory
+ramalama sandbox opencode -w ./src qwen3:4b --prompt "Please analyze the source code in the current directory"
 ```
 
 Send instructions to the agent via stdin:

docs/ramalama-sandbox-pi.1.md.in

@@ -4,16 +4,19 @@
 ramalama\-sandbox\-pi - run Pi in a sandbox, backed by a local AI Model
 
 ## SYNOPSIS
-**ramalama sandbox pi** [*options*] *model* [arg ...]
+**ramalama sandbox pi** [*options*] [*model* ...]
 
 ## DESCRIPTION
 Run Pi in a container, connected to a local model server also running
 in a container. Pi uses the model for reasoning and tool calling.
 
-When run with no arguments after the model, an interactive session is
-launched. If one or more arguments are provided, they are passed to the agent
-as instructions to process non-interactively. Commands may also be passed via
-stdin.
+When a single model is specified, the server loads that model directly.
+When zero or multiple models are specified, the server starts in router mode
+using llama.cpp's --models-dir, dynamically loading and unloading models.
+With no models, all GGUF models in the store are served.
+
+Use **--prompt** to pass instructions for non-interactive processing.
+Commands may also be passed via stdin.
 
 Two containers are started: a model server (llama-server) and the agent
 container. They communicate via container networking. When the agent session
@@ -56,6 +59,8 @@ Otherwise only the agent container starts using the given url for the openai com
 
 @@option model-draft
 
+@@option models-max
+
 @@option name
 
 @@option ncmoe
@@ -72,6 +77,8 @@ Otherwise only the agent container starts using the given url for the openai com
 
 @@option privileged
 
+@@option prompt
+
 @@option pull
 
 @@option runtime-args
@@ -109,6 +116,16 @@ Run the Pi agent with default settings:
 ramalama sandbox pi qwen3:4b
 ```
 
+Run the Pi agent in router mode (serve all models in the store):
+```
+ramalama sandbox pi
+```
+
+Run the Pi agent with multiple models in router mode:
+```
+ramalama sandbox pi qwen3:4b granite-code:3b
+```
+
 Run the Pi agent with a custom image:
 ```
 ramalama sandbox pi --pi-image myimage:v1 qwen3:4b
@@ -126,7 +143,7 @@ ramalama sandbox pi -w ./src qwen3:4b
 
 Request the agent to perform actions non-interactively:
 ```
-ramalama sandbox pi -w ./src qwen3:4b Please analyze the source code in the current directory
+ramalama sandbox pi -w ./src qwen3:4b --prompt "Please analyze the source code in the current directory"
 ```
 
 Send instructions to the agent via stdin:

ramalama/plugins/runtimes/inference/llama_cpp.py

@@ -6,6 +6,7 @@
 import os
 import platform
 import shutil
+import subprocess
 import sys
 import tempfile
 from collections.abc import Mapping
@@ -312,7 +313,6 @@ def service_ready_check(self, conn: HTTPConnection, args: Any, model_name: Optio
         except ValueError:
             logger.debug(f"{self.name} {container_name} /models does not include a model list in the response")
             return False
-
         if not model_name:
             if hasattr(args, 'MODEL') and isinstance(args.MODEL, str):
                 model_name = New(args.MODEL, args).model_alias
@@ -539,13 +539,12 @@ def _serve_handler(self, args: argparse.Namespace) -> None:
 
         super()._serve_handler(args)
 
-    def _serve_router(self, args: argparse.Namespace) -> None:
-        """Serve multiple models using llama.cpp router mode (container-only)."""
+    def _build_router_engine(self, args: argparse.Namespace) -> Engine:
+        """Resolve models and build an Engine with bind mounts for router mode."""
         if not args.container:
             sys.exit("Error: multi-model router mode requires a container runtime.")
 
         set_accel_env_vars()
-        args.port = compute_serving_port(args)
         args.router_mode = True
 
         if args.MODEL:
@@ -563,28 +562,46 @@ def _serve_router(self, args: argparse.Namespace) -> None:
         if not models:
             sys.exit("Error: no GGUF models found in the model store. Pull a model first with: ramalama pull <model>")
 
-        if args.container and not args.dryrun:
+        if not args.dryrun and not getattr(args, "image", None):
             config = ActiveConfig()
             should_pull = config.pull in ["always", "missing", "newer"]
             args.image = ensure_image(config.engine, accel_image(config), should_pull=should_pull)
 
-        cmd = assemble_command(args)
+        cmd = self.handle_subcommand("serve", args)
         engine = Engine(args)
         name = getattr(args, "name", None) or genname()
+        args.name = name
         engine.add(["--label", "ai.ramalama", "--name", name, "--env=HOME=/tmp", "--init"])
 
         for host_path, container_name in models:
             mount_path = f"/mnt/models/{container_name}"
             container_host_path = get_container_mount_path(host_path)
-            engine.add([f"--mount=type=bind,src={container_host_path},destination={mount_path},ro"])
+            engine.add([f"--mount=type=bind,src={container_host_path},destination={mount_path},ro{engine.relabel()}"])
 
         engine.add([args.image] + cmd)
+        return engine
+
+    def _serve_router(self, args: argparse.Namespace) -> None:
+        """Serve multiple models using llama.cpp router mode (container-only)."""
+        args.port = compute_serving_port(args)
+        engine = self._build_router_engine(args)
 
         if args.dryrun:
             engine.dryrun()
             return
         engine.exec()
 
+    def serve_router_nonblocking(self, args: argparse.Namespace) -> None:
+        """Start the router-mode server non-blocking (for sandbox use)."""
+        args.detach = True
+        engine = self._build_router_engine(args)
+
+        if args.dryrun:
+            engine.dryrun()
+            return
+
+        subprocess.Popen(engine.exec_args)
+
     @staticmethod
     def _migrate_store_ref_files(store: Any) -> None:
         """Migrate any old-format ref files to JSON before enumeration."""