From 3f0a77d9b3d3984aa6e165712d02cdc9c2692424 Mon Sep 17 00:00:00 2001 From: Nathan Van Gheem <524413+vangheem@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:01:16 -0400 Subject: [PATCH 1/4] Simplify Code Mode with active tools and Python utilities --- docs/harness-sdk.md | 126 ++--- hyperforge/src/hyperforge/codemode/modules.py | 15 + hyperforge/src/hyperforge/codemode/worker.py | 20 +- .../src/hyperforge/harness_sdk/__init__.py | 11 +- .../src/hyperforge/harness_sdk/harness.py | 8 +- .../hyperforge/harness_sdk/tools/__init__.py | 2 - .../hyperforge/harness_sdk/tools/codemode.py | 362 +++++------- .../src/hyperforge/harness_sdk/tools/utils.py | 10 - hyperforge/tests/harness_sdk/test_codemode.py | 526 ++++++++++++------ hyperforge/tests/harness_sdk/test_harness.py | 7 +- 10 files changed, 583 insertions(+), 504 deletions(-) create mode 100644 hyperforge/src/hyperforge/codemode/modules.py diff --git a/docs/harness-sdk.md b/docs/harness-sdk.md index 59e16a23..d9225f7e 100644 --- a/docs/harness-sdk.md +++ b/docs/harness-sdk.md @@ -192,70 +192,42 @@ handlers always receive a validated model instance; they never receive `None`. Tools can attach typed context to their result by setting `context_type` and, if needed, registering a schema and formatter with `register_context()`. -Tools are inherited by spawned sub-agents by default. Set -`inheritance=ToolInheritancePolicy.DO_NOT_INHERIT` on `@tool` or `HarnessTool` -when a tool must remain on the current agent. Scoped Code Mode tools use this -policy by default. +Tools, including Code Mode, are inherited by spawned sub-agents. Code Mode +resolves tools against the harness executing it, not a captured parent tool list. -## Scoped Code Mode +## Code Mode -Use `create_codemode_tool()` when generated Python should orchestrate a small, -explicit capability set without exposing those capabilities as top-level model -tools. The capability list must be an immutable tuple. Scoped Code Mode never -calls `harness.iter_tools()` and never includes core or external tools unless the -caller explicitly passes them. +Use `create_codemode_tool()` to orchestrate the agent's currently active tools +with restricted Python. Core tools and active external tools are available +automatically; inactive lazy tools are excluded. The tool set is resolved on +each invocation. Code Mode instances are excluded to prevent recursive execution. -```python -from pydantic import BaseModel, Field +Optional `capabilities` are additional trusted Python utility functions, not +`HarnessTool` objects. Both synchronous and asynchronous functions are supported; +generated code calls either as an ordinary function, without `await`. Any iterable +is accepted and captured when the Code Mode tool is created. Utilities are not +registered as top-level agent tools. Their Python signatures, rather than JSON +schemas or repeated tool descriptions, are included in the Code Mode description. +```python from hyperforge.harness_sdk import ( AgentHarness, - CodeModeCapability, CodeModeExecutionLimiter, CodeModeLimits, - HarnessTool, - ToolCallContext, create_codemode_tool, - tool, ) -class SearchInput(BaseModel): - query: str = Field(description="Read-only catalog search query") +def double_string(input_string: str) -> str: + return input_string * 2 -class SearchOutput(BaseModel): - matches: list[dict[str, str]] - internal_cursor: str | None = None - - -@tool(description="Search the approved catalog without modifying it.") -async def search_catalog( - context: ToolCallContext, - input_value: SearchInput, -) -> SearchOutput: - return SearchOutput( - matches=[{"id": "item-1", "title": input_value.query}], - internal_cursor="do-not-expose", - ) - - -def project_search_result( - capability: HarnessTool, - output: BaseModel, -) -> dict[str, object]: - del capability - validated = SearchOutput.model_validate(output) - return {"matches": validated.matches} +def times_two(input: int) -> int: + return input * 2 code_mode = create_codemode_tool( - capabilities=( - CodeModeCapability( - search_catalog, - result_adapter=project_search_result, - ), - ), + capabilities=[double_string, times_two], limits=CodeModeLimits( max_source_bytes=64 * 1024, max_result_bytes=256 * 1024, @@ -270,30 +242,31 @@ code_mode = create_codemode_tool( agent = AgentHarness( model="your-model", model_client=model_client, - tools=[code_mode], + tools=[*application_tools, code_mode], ) ``` -Only `code_mode` is registered with the harness in this example. -`search_catalog` is callable from generated code but is not advertised as a -top-level model tool. The Code Mode tool description includes each capability's -description and argument schema so the model can write valid calls. Nested -arguments still pass through `HarnessTool.execute()`, including JSON Schema, -Pydantic input, and output validation. - -Every `CodeModeCapability` has a result adapter. The safe default returns the -tool's formatted model-facing context as a string. Prefer an application adapter, -as above, when generated code needs selected structured fields. Projected values -must contain only JSON worker values; the SDK rejects non-serializable values, -non-finite numbers, Pydantic models at any nesting depth, and the reserved -`__model__` transport key. Use -`raw_codemode_result_adapter` only after explicitly deciding that the complete -JSON-mode Pydantic output is safe for generated code. A raw adapter does not -bypass serialization or result-size validation. +No capabilities are required: `create_codemode_tool()` makes all active agent +tools usable. Active tool calls pass through `HarnessTool.execute()`, preserving +JSON Schema and Pydantic input/output validation. Results are the tool's complete +JSON-mode output, not its formatted context. **Do not put private fields in tool +outputs** just because a context formatter hides them. Remove such fields in the +tool itself if generated code must not see them. + +Utility calls use normal Python argument binding, including defaults, positional- +only, and keyword-only arguments. Type annotations document utilities; they are +not runtime validators. Utilities run on the host and must be trusted, bounded +application functions; the sandbox does not restrict their implementation. +Utility results must contain only JSON worker values. Non-serializable values, +non-finite numbers, nested Pydantic models, and the reserved `__model__` key are +rejected. Tool and utility results share the same serialization and size limits. Capability names must be unique ASCII public, non-keyword Python identifiers and must not conflict with worker names such as `codemode`, `output`, `save`, `question`, -`agent_id`, `dataclass`, `Chunk`, `Context`, `List`, `Any`, or `Dict`. +`agent_id`, `dataclass`, `Chunk`, `Context`, `List`, `Any`, `Dict`, `re`, or `math`. +Utility names must not collide with active tool names; collisions fail rather +than silently shadowing a tool. Function arguments named `agent_id` pass through +unchanged. `CodeModeLimits` applies source, per-call projected-result, cumulative projected-result, final-output, and nested-call limits to each invocation. @@ -327,6 +300,18 @@ built-ins: `abs`, `bool`, `bytes`, `chr`, `complex`, `divmod`, `float`, `hash`, `hex`, `id`, `int`, `isinstance`, `issubclass`, `len`, `oct`, `ord`, `pow`, `range`, `repr`, `round`, `slice`, `sorted`, `str`, `sum`, `tuple`, and `zip`. Other helpers, including `all`, `any`, `min`, and `max`, are unavailable. +`math`, `re`, `statistics`, `itertools`, and `decimal` are available without +imports. Public module attributes are accessible, including functions, constants, +and classes (for example, +`math.sqrt(4)`, `re.findall(pattern, text)`, or `re.Pattern`). Arbitrary object +attributes, private module attributes, and attributes of unapproved modules are +not. `ALLOWED_GLOBAL_MODULES` in `hyperforge/codemode/modules.py` is the shared +registry for worker globals, attribute access, reserved names, and the tool's +module list. Comprehensions and generator expressions remain supported by +the existing restricted Python interpreter. +Convert `decimal.Decimal` results to strings (to preserve precision) or floats +before returning them through `output()`, since the worker protocol carries JSON +values rather than Decimal objects. Authorization, nested/global call limits, projected-result limits, and invalid or repeated `output` attempts remain terminal even if generated code catches the immediate callback error. An output attempt rejected by worker @@ -367,9 +352,14 @@ contract and persists its raw `code` and `question` arguments. Applications must not embed secrets in either field. The nested events described above are the sanitized audit surface for capability calls and results. -The existing exported `codemode` tool remains available for compatibility. It -discovers registered tools and returns raw JSON-mode outputs. New applications -that require capability isolation should use `create_codemode_tool()`. +The exported `codemode` convenience tool uses the same implementation and has +no additional utilities. It retains automatic remote/local selection for +compatibility: remote when a socket is configured, otherwise isolated local +execution (equivalent to `remote_required=None`). +Use the factory's default `remote_required=True` for production fail-closed +remote execution. `CodeModeCapability`, result adapters, and +`ToolInheritancePolicy` have been removed; register tools on the agent and pass +plain Python functions as additional capabilities instead. Remote and isolated-process Code Mode execution move values over the bounded JSON worker protocol. Values crossing those worker boundaries must be JSON diff --git a/hyperforge/src/hyperforge/codemode/modules.py b/hyperforge/src/hyperforge/codemode/modules.py new file mode 100644 index 00000000..d0048c1b --- /dev/null +++ b/hyperforge/src/hyperforge/codemode/modules.py @@ -0,0 +1,15 @@ +import decimal +import itertools +import math +import re +import statistics +from types import ModuleType + +# Shared by the worker's globals/attribute guard and the harness tool description. +ALLOWED_GLOBAL_MODULES: dict[str, ModuleType] = { + "re": re, + "math": math, + "statistics": statistics, + "itertools": itertools, + "decimal": decimal, +} diff --git a/hyperforge/src/hyperforge/codemode/worker.py b/hyperforge/src/hyperforge/codemode/worker.py index 0adda404..852713f6 100644 --- a/hyperforge/src/hyperforge/codemode/worker.py +++ b/hyperforge/src/hyperforge/codemode/worker.py @@ -38,6 +38,7 @@ decode_protocol_value, encode_protocol_value, ) +from .modules import ALLOWED_GLOBAL_MODULES BLOCKED_EXCEPTIONS = { "BaseException", @@ -48,6 +49,15 @@ REDACTED_EXECUTION_ERROR = "Generated code execution failed" +def guarded_module_attribute(obj: Any, name: str) -> Any: + """Allow public attributes of registered modules, not arbitrary objects.""" + if not any( + obj is module for module in ALLOWED_GLOBAL_MODULES.values() + ) or name.startswith("_"): + raise AttributeError("Object attributes are unavailable") + return getattr(obj, name) + + def _set_memory_limit(max_memory_bytes: int | None) -> None: if max_memory_bytes is None or sys.platform == "darwin": return @@ -118,6 +128,7 @@ def __init__( ): self.pipe = pipe self.functions_agent_id: Dict[str, List[str]] = {} + self._harness_functions = False self.debug = debug self.json_protocol = json_protocol self._redact_errors = False @@ -135,6 +146,7 @@ def _process_question_context_sync( redact_errors: bool = False, ): self._redact_errors = redact_errors + self._harness_functions = redact_errors and set(function_names) == {"harness"} self._output_attempted = False self._output_send_failed = False try: @@ -161,6 +173,7 @@ def _process_question_context_sync( "sum": sum, }, "_getitem_": lambda obj, index: obj[index], + "_getattr_": guarded_module_attribute, "_getiter_": iter, "_inplacevar_": guarded_inplace, "dataclass": dataclass, @@ -173,6 +186,7 @@ def _process_question_context_sync( "_unpack_sequence_": guarded_unpack_sequence, "save": self.save, "question": question, + **ALLOWED_GLOBAL_MODULES, } ) if self.debug: @@ -240,7 +254,11 @@ def save( def valid_functions_call( self, function_name: str, *args: Any, **kwargs: Any ) -> Any: - agent_id = kwargs.pop("agent_id", None) + # Harness function arguments must pass through unchanged. Legacy published + # agent calls still use agent_id to select the callback destination. + agent_id = ( + "harness" if self._harness_functions else kwargs.pop("agent_id", None) + ) if agent_id is not None: if agent_id not in self.functions_agent_id.get(function_name, []): raise ValueError( diff --git a/hyperforge/src/hyperforge/harness_sdk/__init__.py b/hyperforge/src/hyperforge/harness_sdk/__init__.py index d38cdeec..b1837831 100644 --- a/hyperforge/src/hyperforge/harness_sdk/__init__.py +++ b/hyperforge/src/hyperforge/harness_sdk/__init__.py @@ -39,20 +39,16 @@ HarnessToolCall, ) from .storage import HarnessStorageProtocol, InMemoryHarnessStorage -from .tools import HarnessTool, ToolCallContext, ToolInheritancePolicy, tool +from .tools import HarnessTool, ToolCallContext, tool from .tools.codemode import ( - CodeModeCapability, CodeModeDispatch, CodeModeExecutionLimiter, CodemodeInput, CodeModeLimits, CodemodeOutput, - CodeModeResultAdapter, CodeModeRunner, codemode, - context_codemode_result_adapter, create_codemode_tool, - raw_codemode_result_adapter, ) from .usage import HarnessUsage, UsageLimitExceeded, UsageLimits @@ -69,11 +65,9 @@ "ChatCompletionToolCallDelta", "ChatCompletionToolCallFunctionDelta", "ChatCompletionUsage", - "CodeModeCapability", "CodeModeDispatch", "CodeModeExecutionLimiter", "CodeModeLimits", - "CodeModeResultAdapter", "CodeModeRunner", "CodemodeInput", "CodemodeOutput", @@ -98,18 +92,15 @@ "NucliaChatCompletionsClient", "NucliaChatCompletionsError", "ReasoningEffort", - "ToolInheritancePolicy", "UsageLimitExceeded", "UsageLimits", "context_schema", "codemode", - "context_codemode_result_adapter", "create_agent", "create_codemode_tool", "format_context", "make_context", "register_context", - "raw_codemode_result_adapter", "tool", "published_agent_to_tools", "preload_published_agent_to_tools", diff --git a/hyperforge/src/hyperforge/harness_sdk/harness.py b/hyperforge/src/hyperforge/harness_sdk/harness.py index f43b66f9..1f4cc1f4 100644 --- a/hyperforge/src/hyperforge/harness_sdk/harness.py +++ b/hyperforge/src/hyperforge/harness_sdk/harness.py @@ -23,7 +23,7 @@ HarnessToolCall, ) from .storage import HarnessStorageProtocol, InMemoryHarnessStorage -from .tools import HarnessTool, ToolCallContext, ToolInheritancePolicy +from .tools import HarnessTool, ToolCallContext from .tools.core import DictOutput, SendMessageInput, SpawnAgentInput, create_core_tools from .usage import HarnessUsage, UsageLimitExceeded, UsageLimits @@ -1133,11 +1133,7 @@ def _create_child(self, child_id: str, *, include_history: bool) -> AgentHarness model=self._config.model, model_client=self._config.model_client, reasoning_effort=self._config.reasoning_effort, - tools=( - tool - for tool in self._config.tools - if tool.inheritance == ToolInheritancePolicy.INHERIT - ), + tools=self._config.tools, system_prompt=self._config.system_prompt, title=self._config.title, conversation_id=self.conversation_id, diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/__init__.py b/hyperforge/src/hyperforge/harness_sdk/tools/__init__.py index 7d0c592a..bd281c61 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/__init__.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/__init__.py @@ -3,7 +3,6 @@ HarnessTool, ToolCallContext, ToolHandler, - ToolInheritancePolicy, tool, ) @@ -12,6 +11,5 @@ "HarnessTool", "ToolCallContext", "ToolHandler", - "ToolInheritancePolicy", "tool", ] diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py index 6e943d8c..3b0c2b62 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py @@ -6,6 +6,7 @@ import threading import time import uuid +from collections.abc import Iterable from dataclasses import dataclass from typing import Any, Awaitable, Callable, NoReturn, Protocol, cast @@ -19,24 +20,22 @@ encode_protocol_value, encode_sandbox_message, ) +from hyperforge.codemode.modules import ALLOWED_GLOBAL_MODULES from hyperforge.codemode.sandbox import MAX_PACKET_BYTES, settings from hyperforge.definition import FunctionDefinition -from ..context import format_context from ..models import HarnessEventType from ..usage import UsageLimitExceeded -from . import HarnessTool, ToolCallContext, ToolInheritancePolicy, tool +from . import HarnessTool, ToolCallContext CODEMODE_TOOL_NAME = "codemode" OUTPUT_FUNCTION_NAME = "output" CODEMODE_USAGE_GUIDANCE = ( - "Return the final result by calling output(value) exactly once. " - "Do not use print(); it is unavailable. " - "Available common built-ins are abs, bool, bytes, chr, complex, divmod, float, " - "hash, hex, id, int, isinstance, issubclass, len, oct, ord, pow, range, repr, " - "round, slice, sorted, str, sum, tuple, and zip; helpers such as all, any, min, " - "and max are unavailable. " - "For example, assign the result to a variable and finish with output(result)." + "Execute restricted Python code. Each currently active tool is available as a " + "function with the same name and arguments. Tool results are JSON data (dicts/lists). " + "Supported: assignments, loops, conditionals, indexing, and builtins including len, " + "range, sum, and sorted. Imports and arbitrary object attributes are unavailable. " + "Return the final result by calling output(value) exactly once; do not use print()." ) DEFAULT_MAX_SOURCE_BYTES = 64 * 1024 DEFAULT_MAX_RESULT_BYTES = 1024 * 1024 @@ -65,6 +64,7 @@ "print", "printed", "question", + *ALLOWED_GLOBAL_MODULES, "save", } ) @@ -91,9 +91,6 @@ _SOURCE_SIZE_CHUNK_CHARS = 4096 _CAPABILITY_EVENT_FAILURE = "Code Mode capability failed" _POLICY_CHECK_FAILURE = "Code Mode policy check failed" -type CodeModeResultAdapter[OutputT: BaseModel] = Callable[ - [HarnessTool[Any, OutputT], OutputT], Any | Awaitable[Any] -] type CodeModeDispatch = Callable[[RestrictedPythonTask], Awaitable[Any]] @@ -110,20 +107,6 @@ class CodemodeOutput(BaseModel): value: Any = None -def context_codemode_result_adapter[OutputT: BaseModel]( - capability: HarnessTool[Any, OutputT], output: OutputT -) -> Any: - """Project a result to the same formatted context shown to the model.""" - return format_context(capability.context(output)) - - -def raw_codemode_result_adapter[OutputT: BaseModel]( - _capability: HarnessTool[Any, OutputT], output: OutputT -) -> Any: - """Explicitly expose the capability's complete JSON-mode output.""" - return output.model_dump(mode="json") - - class CodeModeRunner(Protocol): """Execute a request and route worker callbacks through dispatch. @@ -138,9 +121,10 @@ async def run( @dataclass(frozen=True) -class CodeModeCapability[OutputT: BaseModel]: - tool: HarnessTool[Any, OutputT] - result_adapter: CodeModeResultAdapter[OutputT] = context_codemode_result_adapter +class _CodeModeFunction: + name: str + parameters: dict[str, Any] + execute: Callable[[ToolCallContext, RestrictedPythonTask], Awaitable[Any]] @dataclass(frozen=True) @@ -196,155 +180,61 @@ def release(self) -> None: _process_execution_limiter = CodeModeExecutionLimiter() -@tool( - name=CODEMODE_TOOL_NAME, - description=( - "Execute restricted Python code. Registered tools are available as functions; " - f"{CODEMODE_USAGE_GUIDANCE}" - ), -) -async def codemode( - context: ToolCallContext, input_value: CodemodeInput -) -> CodemodeOutput: - harness = context.harness - tools = { - tool.name: tool - for tool in harness.iter_tools() - if tool.name != CODEMODE_TOOL_NAME - } - result = CodemodeOutput() - - async def dispatch(task: RestrictedPythonTask) -> Any: - if task.function == OUTPUT_FUNCTION_NAME: - if task.args and task.keyword_args: - raise ValueError("output accepts either a positional or keyword value") - if len(task.args) > 1: - raise ValueError("output accepts one value") - result.value = task.args[0] if task.args else task.keyword_args.get("value") - return None - - tool = tools.get(task.function) - if tool is None: - raise ValueError(f"Unknown codemode function: {task.function}") - harness.usage.tool_calls += 1 - harness._check_limit("max_tool_calls", harness.usage.tool_calls) - arguments = _tool_arguments(tool, task.args, task.keyword_args) - output = await tool.execute( - ToolCallContext(harness=harness, name=tool.name), arguments - ) - return output.model_dump(mode="json") - - runner = ( - SandboxRunner.remote(settings.sandbox_socket, dispatch) - if settings.sandbox_socket is not None - else SandboxRunner.isolated_process(dispatch) - ) - definitions = { - name: FunctionDefinition( - name=name, - description=tool.description, - parameters=tool.parameters.get("properties", {}), - ) - for name, tool in tools.items() - } - definitions[OUTPUT_FUNCTION_NAME] = FunctionDefinition( - name=OUTPUT_FUNCTION_NAME, - description="Set the value returned by codemode.", - parameters={"value": {}}, - ) - await runner.run( - WorkerExecutionRequest( - code=input_value.code, - question=input_value.question, - local_vars={}, - global_vars={}, - function_names={"harness": definitions}, - max_runtime_seconds=harness.usage_limits.max_codemode_runtime_seconds, - max_memory_bytes=harness.usage_limits.max_codemode_memory_bytes, - ) - ) - return result - - def create_codemode_tool( *, - capabilities: tuple[CodeModeCapability[Any], ...], + capabilities: Iterable[Callable[..., Any]] = (), limits: CodeModeLimits = CodeModeLimits(), execution_limiter: CodeModeExecutionLimiter = _process_execution_limiter, - remote_required: bool = True, + remote_required: bool | None = True, runner: CodeModeRunner | None = None, - inheritance: ToolInheritancePolicy = ToolInheritancePolicy.DO_NOT_INHERIT, name: str = CODEMODE_TOOL_NAME, description: str | None = None, ) -> HarnessTool[CodemodeInput, CodemodeOutput]: - """Create a Code Mode tool with only the explicitly supplied capabilities. + """Create Code Mode with active agent tools and optional Python utilities. + + Capabilities are trusted host-side sync or async functions, not agent tools. + Only their signatures are advertised; their results must be JSON worker values. ``runner`` defaults to the remote sandbox (or the isolated local process when ``remote_required=False``); inject a ``CodeModeRunner`` for - deterministic tests without any sandbox environment. + deterministic tests without any sandbox environment. ``remote_required=None`` + selects remote execution when a sandbox socket is configured, otherwise local. """ - if not isinstance(capabilities, tuple): - raise TypeError("capabilities must be an immutable tuple") - capability_map: dict[str, CodeModeCapability[Any]] = {} + capability_map: dict[str, _CodeModeFunction] = {} + signatures: list[str] = [] for capability in capabilities: - if not isinstance(capability, CodeModeCapability): - raise TypeError("capabilities must contain CodeModeCapability values") - capability_name = capability.tool.name - if ( - not capability_name.isascii() - or not capability_name.isidentifier() - or keyword.iskeyword(capability_name) - or capability_name.startswith("_") - ): - raise ValueError( - f"Code Mode capability name {capability_name!r} must be an ASCII " - "public Python identifier" - ) - if capability_name in RESERVED_CAPABILITY_NAMES: - raise ValueError( - f"Code Mode capability name {capability_name!r} is reserved" - ) + if not callable(capability) or isinstance(capability, HarnessTool): + raise TypeError("capabilities must contain Python functions, not tools") + capability_name = getattr(capability, "__name__", "") + _validate_function_name(capability_name) if capability_name in capability_map: raise ValueError( f"Code Mode capability names must be unique: {capability_name}" ) - if ( - "agent_id" in capability.tool.input_model.model_fields - or "agent_id" in capability.tool.parameters.get("properties", {}) - ): - raise ValueError( - f"Code Mode capability {capability_name!r} uses the reserved " - "'agent_id' input field" - ) - capability_map[capability_name] = capability - scoped_capabilities = tuple(capabilities) - if inheritance == ToolInheritancePolicy.INHERIT: - non_inherited = sorted( - capability.tool.name - for capability in capabilities - if capability.tool.inheritance == ToolInheritancePolicy.DO_NOT_INHERIT - ) - if non_inherited: - raise ValueError( - "An inherited Code Mode tool cannot include capabilities marked " - f"DO_NOT_INHERIT: {', '.join(non_inherited)}" - ) + signature = inspect.signature(capability) + capability_map[capability_name] = _utility_function(capability, signature) + signatures.append(f"- {capability_name}{signature}") async def execute( context: ToolCallContext, input_value: CodemodeInput ) -> CodemodeOutput: harness = context.harness turn_id = harness.turn_id + requires_remote = ( + remote_required + if remote_required is not None + else settings.sandbox_socket is not None + ) if _utf8_exceeds_limit(input_value.code, limits.max_source_bytes): raise ValueError( "Code Mode source exceeds maximum size: " f"> {limits.max_source_bytes} bytes" ) - socket = settings.sandbox_socket if runner is None and remote_required else None + socket = settings.sandbox_socket if runner is None and requires_remote else None sandbox_token = ( - settings.sandbox_token if runner is None and remote_required else None + settings.sandbox_token if runner is None and requires_remote else None ) - if runner is None and remote_required: + if runner is None and requires_remote: if socket is None: raise RuntimeError( "Remote Code Mode execution is required but SANDBOX_SOCKET " @@ -354,35 +244,88 @@ async def execute( raise RuntimeError( "Remote Code Mode execution is required but SANDBOX_TOKEN is absent" ) + functions = dict(capability_map) + for active_tool in harness.iter_tools(): + # Exclude every Code Mode instance, including custom-named instances. + if getattr(active_tool.handler, "_codemode", False): + continue + _validate_function_name(active_tool.name) + if active_tool.name in functions: + raise ValueError( + f"Code Mode utility conflicts with active tool: {active_tool.name}" + ) + functions[active_tool.name] = _active_tool_function(active_tool) execution_limiter.acquire() return await _execute_scoped_codemode( harness, input_value, - scoped_capabilities, + functions, limits, socket, sandbox_token, execution_limiter, runner, - remote_required, + requires_remote, parent_call_id=context.id, turn_id=turn_id, ) execute.__name__ = name - tool_description = _scoped_description(scoped_capabilities, description) + setattr(execute, "_codemode", True) + tool_description = _codemode_description(signatures, description) return HarnessTool( name=name, handler=execute, description=tool_description, - inheritance=inheritance, + ) + + +def _validate_function_name(name: str) -> None: + if ( + not name.isascii() + or not name.isidentifier() + or keyword.iskeyword(name) + or name.startswith("_") + ): + raise ValueError( + f"Code Mode capability name {name!r} must be an ASCII public Python identifier" + ) + if name in RESERVED_CAPABILITY_NAMES: + raise ValueError(f"Code Mode capability name {name!r} is reserved") + + +def _active_tool_function(active_tool: HarnessTool[Any, Any]) -> _CodeModeFunction: + async def execute(context: ToolCallContext, task: RestrictedPythonTask) -> Any: + arguments = _scoped_tool_arguments(active_tool, task.args, task.keyword_args) + output = await active_tool.execute(context, arguments) + return output.model_dump(mode="json") + + return _CodeModeFunction( + active_tool.name, active_tool.parameters.get("properties", {}), execute + ) + + +def _utility_function( + function: Callable[..., Any], signature: inspect.Signature +) -> _CodeModeFunction: + async def execute(_context: ToolCallContext, task: RestrictedPythonTask) -> Any: + bound = signature.bind(*task.args, **task.keyword_args) + result = function(*bound.args, **bound.kwargs) + if inspect.isawaitable(result): + result = await result + return result + + return _CodeModeFunction( + getattr(function, "__name__"), + {name: {} for name in signature.parameters}, + execute, ) async def _execute_scoped_codemode( harness: Any, input_value: CodemodeInput, - capabilities: tuple[CodeModeCapability[Any], ...], + capability_map: dict[str, _CodeModeFunction], limits: CodeModeLimits, socket: str | None, sandbox_token: str | None, @@ -393,7 +336,6 @@ async def _execute_scoped_codemode( parent_call_id: str | None = None, turn_id: str | None = None, ) -> CodemodeOutput: - capability_map = {capability.tool.name: capability for capability in capabilities} result = CodemodeOutput() invocation_state = _InvocationState() output_state = _OutputState(invocation_state) @@ -464,44 +406,36 @@ async def dispatch(task: RestrictedPythonTask) -> Any: { "call": { "id": call_id, - "name": capability.tool.name, - "arguments": _sanitize_task_arguments(capability.tool, task), + "name": capability.name, + "arguments": _sanitize_task_arguments(capability, task), }, **marker, }, ) started = time.perf_counter() try: - arguments = _scoped_tool_arguments( - capability.tool, task.args, task.keyword_args - ) - output = await capability.tool.execute( + projected = await capability.execute( ToolCallContext( harness=harness, - name=capability.tool.name, + name=capability.name, id=call_id, turn_id=turn_id, _emit_failure=event_failure, ), - arguments, + task, ) invocation_state.raise_if_failed() - projected = capability.result_adapter(capability.tool, output) - if inspect.isawaitable(projected): - projected = await projected try: normalized, result_bytes = _normalize_worker_value( projected, limits.max_result_bytes, - f"Code Mode result from {capability.tool.name}", + f"Code Mode result from {capability.name}", ) except (TypeError, ValueError) as exc: invocation_state.latch(exc) raise try: - transport_label = ( - f"Code Mode result transport from {capability.tool.name}" - ) + transport_label = f"Code Mode result transport from {capability.name}" if remote_required: encode_sandbox_message( SandboxMessage.Response(result=normalized), @@ -533,7 +467,7 @@ async def dispatch(task: RestrictedPythonTask) -> Any: HarnessEventType.TOOL_FAILED, { "call_id": call_id, - "tool": capability.tool.name, + "tool": capability.name, "result": {"error": type(exc).__name__}, "duration_ms": round((time.perf_counter() - started) * 1000, 3), **marker, @@ -544,7 +478,7 @@ async def dispatch(task: RestrictedPythonTask) -> Any: if isinstance(exc, _CapabilityEventFailure): raise RuntimeError(_CAPABILITY_EVENT_FAILURE) from None raise RuntimeError( - f"Code Mode capability {capability.tool.name!r} failed " + f"Code Mode capability {capability.name!r} failed " f"({type(exc).__name__})" ) from None else: @@ -552,7 +486,7 @@ async def dispatch(task: RestrictedPythonTask) -> Any: HarnessEventType.TOOL_COMPLETED, { "call_id": call_id, - "tool": capability.tool.name, + "tool": capability.name, "result": sanitized_result, "duration_ms": round((time.perf_counter() - started) * 1000, 3), "result_bytes": result_bytes, @@ -571,12 +505,12 @@ async def dispatch(task: RestrictedPythonTask) -> Any: else SandboxRunner.isolated_process(dispatch) ) definitions = { - capability.tool.name: FunctionDefinition( - name=capability.tool.name, - description=capability.tool.description, - parameters=capability.tool.parameters.get("properties", {}), + capability.name: FunctionDefinition( + name=capability.name, + description="", + parameters=capability.parameters, ) - for capability in capabilities + for capability in capability_map.values() } definitions[OUTPUT_FUNCTION_NAME] = FunctionDefinition( name=OUTPUT_FUNCTION_NAME, @@ -812,10 +746,12 @@ def _utf8_size(value: str) -> int: def _sanitize_task_arguments( - capability: HarnessTool[Any, Any], task: RestrictedPythonTask + capability: _CodeModeFunction, task: RestrictedPythonTask ) -> Any: try: - arguments = _scoped_tool_arguments(capability, task.args, task.keyword_args) + arguments = _function_arguments( + capability.name, capability.parameters, task.args, task.keyword_args + ) except ValueError: return {"invalid": True} return _sanitize_event_value(arguments) @@ -864,60 +800,40 @@ def _nested_event_context(harness: Any) -> dict[str, Any]: return {"execution_context": context} if context else {} -def _scoped_description( - capabilities: tuple[CodeModeCapability[Any], ...], description: str | None = None -) -> str: - introduction = ( - description - or "Execute restricted Python code using only the scoped capabilities below." +def _codemode_description(signatures: list[str], description: str | None = None) -> str: + introduction = CODEMODE_USAGE_GUIDANCE + if description: + introduction = f"{description}\n\n{introduction}" + introduction += ( + "\n\nAvailable Python modules (public attributes only):\n" + + "\n".join(f"- {name}" for name in ALLOWED_GLOBAL_MODULES) ) - introduction = f"{introduction} {CODEMODE_USAGE_GUIDANCE}" - if not capabilities: - return introduction - definitions = [] - for capability in capabilities: - schema = json.dumps( - capability.tool.parameters, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ) - definitions.append( - f"{capability.tool.name}: {capability.tool.description or 'No description.'} " - f"Arguments JSON Schema: {schema}" - ) - return f"{introduction}\nCapability functions:\n" + "\n".join(definitions) + if signatures: + introduction += "\n\nAvailable utility functions:\n" + "\n".join(signatures) + return introduction + "\n\nExample:\n```python\nx = 1\nx = x * 2\noutput(x)\n```" -def _tool_arguments( +def _scoped_tool_arguments( tool: HarnessTool[Any, Any], args: tuple[Any, ...], keyword_args: dict[str, Any], ) -> dict[str, Any]: - if not args: - return keyword_args - names = list(tool.input_model.model_fields) - if len(args) > len(names): - raise ValueError(f"Too many positional arguments for {tool.name}") - arguments = dict(zip(names, args, strict=False)) - duplicates = arguments.keys() & keyword_args.keys() - if duplicates: - duplicate = next(iter(duplicates)) - raise ValueError(f"Multiple values for argument {duplicate!r}") - arguments.update(keyword_args) - return arguments + return _function_arguments( + tool.name, tool.parameters.get("properties", {}), args, keyword_args + ) -def _scoped_tool_arguments( - tool: HarnessTool[Any, Any], +def _function_arguments( + name: str, + parameters: dict[str, Any], args: tuple[Any, ...], keyword_args: dict[str, Any], ) -> dict[str, Any]: if not args: return keyword_args - names = list(tool.parameters.get("properties", {})) + names = list(parameters) if len(args) > len(names): - raise ValueError(f"Too many positional arguments for {tool.name}") + raise ValueError(f"Too many positional arguments for {name}") arguments = dict(zip(names, args, strict=False)) duplicates = arguments.keys() & keyword_args.keys() if duplicates: @@ -927,18 +843,20 @@ def _scoped_tool_arguments( return arguments +# The convenience tool shares the factory's dispatch, limits, and output contract. +# Preserve automatic remote/local selection for the convenience tool; production +# callers should use the factory's fail-closed remote default. +codemode = create_codemode_tool(remote_required=None) + + __all__ = [ - "CodeModeCapability", "CodeModeDispatch", "CodeModeExecutionLimiter", "CodeModeLimits", - "CodeModeResultAdapter", "CodeModeRunner", "CodemodeInput", "CodemodeOutput", "RESERVED_CAPABILITY_NAMES", "codemode", - "context_codemode_result_adapter", "create_codemode_tool", - "raw_codemode_result_adapter", ] diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/utils.py b/hyperforge/src/hyperforge/harness_sdk/tools/utils.py index 9e0b5951..0551b52f 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/utils.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/utils.py @@ -2,7 +2,6 @@ import inspect from dataclasses import dataclass, field -from enum import StrEnum from typing import TYPE_CHECKING, Any, Awaitable, Callable, cast, get_type_hints from jsonschema import ValidationError as JsonSchemaValidationError @@ -34,11 +33,6 @@ class _CurrentTurn: _CURRENT_TURN = _CurrentTurn() -class ToolInheritancePolicy(StrEnum): - INHERIT = "inherit" - DO_NOT_INHERIT = "do_not_inherit" - - @dataclass(frozen=True) class ToolCallContext: """Execution context passed to a tool handler. @@ -93,7 +87,6 @@ class HarnessTool[InputT: BaseModel, OutputT: BaseModel]: parameters_schema: dict[str, Any] | None = None context_factory: ContextFactory[OutputT] | None = None lazy_load: bool = False - inheritance: ToolInheritancePolicy = ToolInheritancePolicy.INHERIT input_model: type[InputT] = field(init=False) output_model: type[OutputT] = field(init=False) _parameters: dict[str, Any] = field(init=False, repr=False) @@ -167,7 +160,6 @@ def tool( parameters_schema: dict[str, Any] | None = None, context_factory: ContextFactory[Any] | None = None, lazy_load: bool = False, - inheritance: ToolInheritancePolicy = ToolInheritancePolicy.INHERIT, ) -> Callable[[ToolHandler[Any, Any]], HarnessTool[Any, Any]]: """Create a harness tool from an annotated async handler.""" @@ -180,7 +172,6 @@ def decorate(handler: ToolHandler[Any, Any]) -> HarnessTool[Any, Any]: parameters_schema=parameters_schema, context_factory=context_factory, lazy_load=lazy_load, - inheritance=inheritance, ) return decorate @@ -195,6 +186,5 @@ def _is_model_type(value: Any) -> bool: "HarnessTool", "ToolCallContext", "ToolHandler", - "ToolInheritancePolicy", "tool", ] diff --git a/hyperforge/tests/harness_sdk/test_codemode.py b/hyperforge/tests/harness_sdk/test_codemode.py index eb85dce8..bfce0b5b 100644 --- a/hyperforge/tests/harness_sdk/test_codemode.py +++ b/hyperforge/tests/harness_sdk/test_codemode.py @@ -1,4 +1,5 @@ import asyncio +import inspect import logging import sys import tempfile @@ -6,7 +7,7 @@ from collections.abc import AsyncIterator from contextvars import ContextVar from pathlib import Path -from typing import Any +from typing import Any, Callable import pytest from pydantic import BaseModel, Field @@ -15,7 +16,6 @@ from hyperforge.codemode import sandbox as sandbox_module from hyperforge.harness_sdk import ( AgentHarness, - CodeModeCapability, CodeModeDispatch, CodeModeExecutionLimiter, CodemodeInput, @@ -27,11 +27,9 @@ HarnessToolCall, ModelDelta, ToolCallContext, - ToolInheritancePolicy, UsageLimits, codemode, create_codemode_tool, - raw_codemode_result_adapter, tool, ) from hyperforge.harness_sdk.tools import codemode as codemode_module @@ -118,7 +116,7 @@ def worker_task(function: str, *args: Any, **kwargs: Any) -> RestrictedPythonTas def local_codemode( - *capabilities: CodeModeCapability[Any], + *capabilities: Callable[..., Any], limits: CodeModeLimits | None = None, execution_limiter: CodeModeExecutionLimiter | None = None, ) -> HarnessTool[Any, Any]: @@ -136,10 +134,34 @@ def local_codemode( ) -def value_adapter( - _tool: HarnessTool[Any, UpperOutput], output: UpperOutput -) -> dict[str, str]: - return {"value": UpperOutput.model_validate(output).value} +def utility_from_tool( + wrapped: HarnessTool, result_adapter: Callable[..., Any] +) -> Callable[..., Any]: + """Reuse result-policy fixtures as plain utilities (no harness context needed).""" + + async def utility(*args: Any, **kwargs: Any) -> Any: + arguments = codemode_module._scoped_tool_arguments(wrapped, args, kwargs) + value = wrapped.input_model.model_validate(arguments) + context = ToolCallContext( + harness=AgentHarness(model="test", model_client=UnusedModel()), + name=wrapped.name, + ) + output = await wrapped.handler(context, value) + return result_adapter(wrapped, output) + + utility.__name__ = wrapped.name + setattr( + utility, + "__signature__", + inspect.Signature( + [ + inspect.Parameter(name, inspect.Parameter.POSITIONAL_OR_KEYWORD) + for name in wrapped.parameters.get("properties", {}) + ] + + [inspect.Parameter("extra", inspect.Parameter.VAR_KEYWORD)] + ), + ) + return utility @pytest.mark.asyncio @@ -181,7 +203,7 @@ async def test_codemode_counts_nested_tool_calls() -> None: usage_limits=UsageLimits(max_tool_calls=1), ) - with pytest.raises(RuntimeError, match="max_tool_calls"): + with pytest.raises(RuntimeError, match="Generated code execution failed"): await codemode.execute( ToolCallContext(harness=harness, name=codemode.name), CodemodeInput(code="upper(value='one')\nupper(value='two')").model_dump(), @@ -196,7 +218,7 @@ async def test_codemode_propagates_tool_validation_errors() -> None: tools=[upper], ) - with pytest.raises(RuntimeError, match="Invalid upper arguments"): + with pytest.raises(RuntimeError, match="Generated code execution failed"): await codemode.execute( ToolCallContext(harness=harness, name=codemode.name), CodemodeInput(code="upper(missing='value')").model_dump(), @@ -254,7 +276,9 @@ async def read_context( request_context.reset(token) assert result.value == "available" - assert call_ids == [None] + assert len(call_ids) == 1 + assert call_ids[0] is not None + assert call_ids[0] != "outer-call" @pytest.mark.asyncio @@ -276,7 +300,7 @@ async def test_codemode_enforces_runtime_limit() -> None: async def test_codemode_blocks_process_control_exceptions() -> None: harness = AgentHarness(model="test", model_client=UnusedModel()) - with pytest.raises(RuntimeError, match="SystemExit.*not defined"): + with pytest.raises(RuntimeError, match="Generated code execution failed"): await codemode.execute( ToolCallContext(harness=harness, name=codemode.name), CodemodeInput(code="raise SystemExit(1)").model_dump(), @@ -300,9 +324,7 @@ async def test_codemode_enforces_memory_limit() -> None: @pytest.mark.asyncio -async def test_scoped_codemode_uses_only_explicit_hidden_capabilities( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_codemode_includes_active_tools_and_additional_utilities() -> None: seen_call_ids: list[str | None] = [] @tool(name="hidden_upper") @@ -316,27 +338,36 @@ async def hidden_upper( async def lower(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value.lower()) + def double_string(input_string: str) -> str: + return input_string * 2 + runner = RecordingRunner( - [worker_task("hidden_upper", "hello"), worker_task("output", "done")] + [ + worker_task("hidden_upper", "hello"), + worker_task("double_string", "hello"), + worker_task("output", "done"), + ] ) scoped = create_codemode_tool( - capabilities=( - CodeModeCapability( - hidden_upper, - result_adapter=raw_codemode_result_adapter, - ), - ), + capabilities=[double_string], runner=runner, ) harness = AgentHarness( model="test", model_client=UnusedModel(), - tools=[scoped, lower], - ) - monkeypatch.setattr( - harness, - "iter_tools", - lambda *args, **kwargs: pytest.fail("scoped Code Mode discovered tools"), + tools=[scoped, lower, hidden_upper], + disabled_core_tools=[ + "remember", + "recall", + "forget", + "spawn_agent", + "send_message", + "wait_agent", + "compact", + "search_tools", + "activate_tools", + "call_tool", + ], ) result = await scoped.execute( @@ -348,15 +379,17 @@ async def lower(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: assert len(seen_call_ids) == 1 assert seen_call_ids[0] is not None assert seen_call_ids[0] != "outer-call" - assert hidden_upper.name not in harness._tools assert set(runner.requests[0].function_names["harness"]) == { "hidden_upper", + "lower", + "double_string", "output", } assert runner.requests[0].code == "hidden_upper('hello'); output('done')" assert runner.requests[0].question == "Why?" assert runner.requests[0].redact_errors is True assert runner.results[0] == ("hidden_upper", {"value": "HELLO"}) + assert runner.results[1] == ("double_string", "hellohello") @pytest.mark.asyncio @@ -373,18 +406,12 @@ async def lower(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value.lower()) scoped = create_codemode_tool( - capabilities=( - CodeModeCapability( - upper, - result_adapter=raw_codemode_result_adapter, - ), - ), remote_required=False, ) harness = AgentHarness( model="test", model_client=UnusedModel(), - tools=[scoped, lower], + tools=[scoped, lower, upper], ) result = await scoped.execute( @@ -395,58 +422,197 @@ async def lower(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: ) assert result.value == "HELLO" - assert upper.name not in harness._tools - with pytest.raises(RuntimeError, match="Generated code execution failed"): - await scoped.execute( - ToolCallContext(harness=harness, name=scoped.name), - {"code": "lower(value='HELLO')"}, - ) + result = await scoped.execute( + _ctx(harness), {"code": "output(lower(value='HELLO'))"} + ) + assert result.value == {"value": "hello"} @pytest.mark.asyncio -async def test_scoped_codemode_default_projection_uses_formatted_context() -> None: +async def test_codemode_active_tool_results_are_raw_json() -> None: runner = RecordingRunner( [worker_task("lookup", value="hello"), worker_task("output", None)] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(lookup),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[lookup]) await scoped.execute( ToolCallContext(harness=harness, name=scoped.name), {"code": "lookup(value='hello'); output('done')"}, ) - assert runner.results == [("lookup", '{"value":"HELLO"}'), ("output", None)] + assert runner.results == [ + ("lookup", {"value": "HELLO", "secret": "internal"}), + ("output", None), + ] @pytest.mark.asyncio -async def test_scoped_codemode_raw_projection_must_be_explicit() -> None: - runner = RecordingRunner( - [worker_task("lookup", value="hello"), worker_task("output", None)] +async def test_codemode_orchestrates_tools_sync_and_async_utilities() -> None: + def double_string(input_string: str, /, *, suffix: str = "!") -> str: + return input_string * 2 + suffix + + async def times_two(input: int = 2) -> int: + await asyncio.sleep(0) + return input * 2 + + utilities = [double_string, times_two] + scoped = create_codemode_tool(capabilities=utilities, remote_required=False) + utilities.clear() # The factory captures the iterable, not the mutable list. + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, upper] ) - scoped = create_codemode_tool( - capabilities=( - CodeModeCapability( - lookup, - result_adapter=raw_codemode_result_adapter, - ), - ), - runner=runner, + result = await scoped.execute( + _ctx(harness), + { + "code": "text = double_string('hi')\ntext = upper(value=text)\noutput([text['value'], times_two(), times_two(input=3)])" + }, + ) + assert result.value == ["HIHI!", 4, 6] + assert harness.usage.tool_calls == 4 + + +@pytest.mark.asyncio +async def test_codemode_resolves_lazy_tools_for_each_invocation() -> None: + lazy = HarnessTool("lazy_upper", upper.handler, lazy_load=True) + scoped = local_codemode() + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, lazy] + ) + + with pytest.raises(RuntimeError, match="Generated code execution failed"): + await scoped.execute( + _ctx(harness), {"code": "output(lazy_upper(value='hello'))"} + ) + + await harness.activate_tools([lazy.name]) + result = await scoped.execute( + _ctx(harness), {"code": "output(lazy_upper(value='hello'))"} + ) + assert result.value == {"value": "HELLO"} + + +@pytest.mark.asyncio +async def test_codemode_excludes_other_codemode_instances_and_includes_core_tools() -> ( + None +): + runner = RecordingRunner([worker_task("output", None)]) + scoped = create_codemode_tool(runner=runner) + other = create_codemode_tool(name="execute_python", runner=runner) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, other] ) + await scoped.execute(_ctx(harness), {"code": "output(None)"}) + names = set(runner.requests[0].function_names["harness"]) + assert "remember" in names + assert "activate_tools" in names + assert "codemode" not in names + assert "execute_python" not in names + + +@pytest.mark.asyncio +async def test_codemode_utility_collisions_do_not_shadow_tools() -> None: + def upper(value: str) -> str: + return value + + scoped = local_codemode(upper) + conflicting = HarnessTool("upper", lookup.handler) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[conflicting] + ) + with pytest.raises(ValueError, match="utility conflicts with active tool: upper"): + await scoped.execute(_ctx(harness), {"code": "output(upper('hello'))"}) + + +@pytest.mark.asyncio +async def test_inherited_codemode_uses_the_executing_child_harness() -> None: + class Empty(BaseModel): + pass + + @tool() + async def current_agent(context: ToolCallContext, _input: Empty) -> UpperOutput: + return UpperOutput(value=context.harness.agent_id) + + scoped = local_codemode() + parent = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, current_agent] + ) + child = parent._create_child("child-agent", include_history=False) + result = await child._tools["codemode"].execute( + _ctx(child), {"code": "output(current_agent())"} + ) + assert result.value == {"value": "child-agent"} + + +@pytest.mark.asyncio +async def test_convenience_codemode_does_not_fall_back_when_remote_is_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(codemode_module.settings, "sandbox_socket", "/configured.sock") + monkeypatch.setattr(codemode_module.settings, "sandbox_token", None) harness = AgentHarness(model="test", model_client=UnusedModel()) + with pytest.raises(RuntimeError, match="SANDBOX_TOKEN is absent"): + await codemode.execute(_ctx(harness), {"code": "output(None)"}) - await scoped.execute( - ToolCallContext(harness=harness, name=scoped.name), - {"code": "lookup(value='hello'); output('done')"}, + +@pytest.mark.asyncio +async def test_codemode_provides_re_and_math_without_imports() -> None: + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel()) + result = await scoped.execute( + _ctx(harness), + { + "code": ( + "matches = re.findall('[0-9]+', 'a12b34')\n" + "pattern = re.compile('[0-9]+')\n" + "output([matches, math.sqrt(16), math.pi > 3, isinstance(pattern, re.Pattern)])" + ) + }, ) + assert result.value == [["12", "34"], 4.0, True, True] - assert runner.results == [ - ("lookup", {"value": "HELLO", "secret": "internal"}), - ("output", None), - ] + +@pytest.mark.asyncio +async def test_codemode_provides_statistics_itertools_and_decimal_without_imports() -> ( + None +): + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel()) + result = await scoped.execute( + _ctx(harness), + { + "code": ( + "totals = []\n" + "for batch in itertools.batched([1, 2, 3, 4], 2):\n" + " totals += [sum(batch)]\n" + "amount = decimal.Decimal('0.1') + decimal.Decimal('0.2')\n" + "output([totals, statistics.mean(totals), str(amount)])" + ) + }, + ) + assert result.value == [[3, 7], 5, "0.3"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "code", + [ + "import os\noutput(os.getcwd())", + "output(math.__dict__)", + "output(re.enum.global_enum)", + "output(re.search('x', 'x').group())", + "output('x'.upper())", + ], +) +async def test_codemode_module_support_does_not_allow_object_traversal( + code: str, +) -> None: + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel()) + with pytest.raises(RuntimeError, match="Generated code execution failed"): + await scoped.execute(_ctx(harness), {"code": code}) @pytest.mark.asyncio @@ -460,7 +626,7 @@ def wrap_model( [worker_task("upper", value="hello"), worker_task("output", None)] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=wrap_model),), + capabilities=(utility_from_tool(upper, result_adapter=wrap_model),), runner=runner, ) harness = AgentHarness(model="test", model_client=UnusedModel()) @@ -498,7 +664,7 @@ def project( [worker_task("upper", value="hello"), worker_task("output", None)] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=project),), + capabilities=(utility_from_tool(upper, result_adapter=project),), runner=runner, ) harness = AgentHarness(model="test", model_client=UnusedModel()) @@ -523,7 +689,7 @@ def project( runner = RecordingRunner([worker_task("upper", value="hello")]) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=project),), + capabilities=(utility_from_tool(upper, result_adapter=project),), runner=runner, ) harness = AgentHarness(model="test", model_client=UnusedModel()) @@ -544,7 +710,7 @@ def project( runner = RecordingRunner([worker_task("upper", value="hello")]) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=project),), + capabilities=(utility_from_tool(upper, result_adapter=project),), runner=runner, ) harness = AgentHarness(model="test", model_client=UnusedModel()) @@ -621,10 +787,9 @@ async def test_scoped_codemode_rejects_reserved_model_marker_in_output() -> None async def test_scoped_codemode_uses_normal_tool_validation() -> None: invalid_input_runner = RecordingRunner([worker_task("upper", missing="value")]) input_scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper),), runner=invalid_input_runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[upper]) with pytest.raises(RuntimeError, match=r"capability 'upper' failed \(ValueError\)"): await input_scoped.execute( @@ -642,9 +807,11 @@ async def invalid_output( [worker_task("invalid_output", value="hello")] ) output_scoped = create_codemode_tool( - capabilities=(CodeModeCapability(invalid_output),), runner=invalid_output_runner, ) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[invalid_output] + ) with pytest.raises( RuntimeError, match=r"capability 'invalid_output' failed \(ValidationError\)" ): @@ -669,15 +836,9 @@ async def aliased( [worker_task("aliased", "hello"), worker_task("output", None)] ) scoped = create_codemode_tool( - capabilities=( - CodeModeCapability( - aliased, - result_adapter=raw_codemode_result_adapter, - ), - ), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[aliased]) await scoped.execute( ToolCallContext(harness=harness, name=scoped.name), @@ -686,7 +847,7 @@ async def aliased( definition = runner.requests[0].function_names["harness"]["aliased"] assert list(definition.parameters) == ["public"] - assert '"public"' in scoped.description + assert '"public"' not in scoped.description assert runner.results == [ ("aliased", {"value": "HELLO"}), ("output", None), @@ -703,13 +864,12 @@ async def test_scoped_codemode_counts_nested_calls_as_ordinary_usage() -> None: ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper),), runner=runner, ) harness = AgentHarness( model="test", model_client=UnusedModel(), - tools=[scoped], + tools=[scoped, upper], usage_limits=UsageLimits(max_tool_calls=3), ) @@ -747,10 +907,9 @@ async def record_id( ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(record_id),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[record_id]) await scoped.execute( ToolCallContext(harness=harness, name=scoped.name, id="outer-call"), @@ -782,10 +941,13 @@ async def record_id( ], ) def test_scoped_codemode_rejects_reserved_capability_names(name: str) -> None: - reserved = HarnessTool(name, upper.handler) + def reserved(value: str) -> str: + return value + + reserved.__name__ = name with pytest.raises(ValueError, match="reserved"): - create_codemode_tool(capabilities=(CodeModeCapability(reserved),)) + create_codemode_tool(capabilities=[reserved]) @pytest.mark.parametrize( @@ -801,82 +963,83 @@ def test_scoped_codemode_rejects_reserved_capability_names(name: str) -> None: ], ) def test_scoped_codemode_rejects_invalid_capability_names(name: str) -> None: - invalid = HarnessTool(name, upper.handler) + def invalid(value: str) -> str: + return value + + invalid.__name__ = name with pytest.raises(ValueError, match="ASCII public Python identifier"): - create_codemode_tool(capabilities=(CodeModeCapability(invalid),)) + create_codemode_tool(capabilities=[invalid]) def test_scoped_codemode_rejects_duplicate_capability_names() -> None: - duplicate = HarnessTool(upper.name, upper.handler) + def duplicate(value: str) -> str: + return value with pytest.raises(ValueError, match="must be unique"): - create_codemode_tool( - capabilities=(CodeModeCapability(upper), CodeModeCapability(duplicate)) - ) + create_codemode_tool(capabilities=[duplicate, duplicate]) -def test_scoped_codemode_rejects_agent_id_input_field() -> None: +@pytest.mark.asyncio +async def test_codemode_preserves_agent_id_function_arguments() -> None: + def routed(agent_id: str) -> str: + return agent_id + class AgentInput(BaseModel): agent_id: str @tool() - async def routed(_context: ToolCallContext, input_value: AgentInput) -> UpperOutput: - return UpperOutput(value=input_value.agent_id) - - with pytest.raises(ValueError, match="reserved 'agent_id' input field"): - create_codemode_tool(capabilities=(CodeModeCapability(routed),)) + async def routed_tool(_context: ToolCallContext, value: AgentInput) -> UpperOutput: + return UpperOutput(value=value.agent_id) + scoped = local_codemode(routed) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[routed_tool] + ) + result = await scoped.execute( + _ctx(harness), + { + "code": "value = routed(agent_id='child')\noutput(routed_tool(agent_id=value))" + }, + ) + assert result.value == {"value": "child"} -def test_scoped_codemode_requires_immutable_capability_tuple() -> None: - with pytest.raises(TypeError, match="immutable tuple"): - create_codemode_tool(capabilities=[]) # type: ignore[arg-type] +def test_codemode_rejects_tools_as_additional_utilities() -> None: + with pytest.raises(TypeError, match="Python functions, not tools"): + create_codemode_tool(capabilities=[upper]) -def test_inherited_codemode_rejects_non_inherited_capability() -> None: - @tool(inheritance=ToolInheritancePolicy.DO_NOT_INHERIT) - async def parent_only( - _context: ToolCallContext, input_value: UpperInput - ) -> UpperOutput: - return UpperOutput(value=input_value.value) - with pytest.raises(ValueError, match="cannot include.*DO_NOT_INHERIT"): - create_codemode_tool( - capabilities=(CodeModeCapability(parent_only),), - inheritance=ToolInheritancePolicy.INHERIT, - ) +def test_codemode_description_includes_only_utility_signatures() -> None: + def double_string(input_string: str) -> str: + return input_string * 2 + def times_two(input: int) -> int: + return input * 2 -def test_scoped_codemode_description_includes_capability_schema() -> None: - scoped = create_codemode_tool(capabilities=(CodeModeCapability(upper),)) + scoped = create_codemode_tool(capabilities=[double_string, times_two]) custom = create_codemode_tool( - capabilities=(CodeModeCapability(upper),), + capabilities=[double_string], description="Custom orchestration instructions.", ) - assert "Uppercase a value" in scoped.description - assert '"required":["value"]' in scoped.description - assert "Do not use print(); it is unavailable." in scoped.description - assert ( - "helpers such as all, any, min, and max are unavailable" in scoped.description - ) - assert "finish with output(result)" in scoped.description + assert "- double_string(input_string: str) -> str" in scoped.description + assert "- times_two(input: int) -> int" in scoped.description + assert "JSON Schema" not in scoped.description + assert "output(value) exactly once" in scoped.description + assert "do not use print()" in scoped.description + assert "output(x)" in scoped.description assert "Custom orchestration instructions." in custom.description - assert '"required":["value"]' in custom.description - assert "Do not use print(); it is unavailable." in custom.description + assert "- double_string(input_string: str) -> str" in custom.description def test_compatibility_codemode_description_explains_output_contract() -> None: assert "output(value) exactly once" in codemode.description - assert "Do not use print(); it is unavailable." in codemode.description - assert ( - "helpers such as all, any, min, and max are unavailable" in codemode.description - ) - assert "finish with output(result)" in codemode.description + assert "do not use print()" in codemode.description @pytest.mark.asyncio -async def test_scoped_codemode_default_projection_uses_model_context() -> None: +async def test_codemode_returns_structured_json_instead_of_formatted_context() -> None: @tool( context_factory=lambda output: HarnessContextReference( type=HarnessContextType.STRUCTURED, @@ -886,15 +1049,17 @@ async def test_scoped_codemode_default_projection_uses_model_context() -> None: async def lookup(_: AgentHarness, input_value: UpperInput) -> SensitiveOutput: return SensitiveOutput(value=input_value.value.upper(), secret="internal") - scoped = local_codemode(CodeModeCapability(lookup)) - harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[scoped]) + scoped = local_codemode() + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, lookup] + ) result = await scoped.execute( _ctx(harness), {"code": "result = lookup(value='hello')\noutput(result)"}, ) - assert result.value == '{"value":"HELLO"}' + assert result.value == {"value": "HELLO", "secret": "internal"} @pytest.mark.asyncio @@ -903,8 +1068,8 @@ async def test_scoped_codemode_sanitizes_formatted_context_events() -> None: async def lookup(_: AgentHarness, input_value: UpperInput) -> SensitiveOutput: return SensitiveOutput(value=input_value.value, secret="internal-secret") - scoped = local_codemode(CodeModeCapability(lookup)) - harness = AgentHarness(model="test", model_client=UnusedModel()) + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[lookup]) await scoped.execute(_ctx(harness), {"code": "lookup(value='safe'); output(None)"}) @@ -914,8 +1079,8 @@ async def lookup(_: AgentHarness, input_value: UpperInput) -> SensitiveOutput: for event in events if event.type == HarnessEventType.TOOL_COMPLETED and event.payload.get("nested") ) - assert completed.payload["result"]["type"] == "string" - assert completed.payload["result"]["utf8_bytes"] > 0 + assert completed.payload["result"]["type"] == "object" + assert completed.payload["result"]["entries"][1]["value"] == "[REDACTED]" assert "internal-secret" not in str(completed.payload) @@ -924,7 +1089,7 @@ async def test_scoped_codemode_does_not_persist_opaque_string_results() -> None: def project(_tool: HarnessTool, _output: BaseModel) -> str: return "authorization: Bearer projected-secret" - scoped = local_codemode(CodeModeCapability(upper, result_adapter=project)) + scoped = local_codemode(utility_from_tool(upper, result_adapter=project)) harness = AgentHarness(model="test", model_client=UnusedModel()) await scoped.execute(_ctx(harness), {"code": "upper(value='safe'); output(None)"}) @@ -947,7 +1112,7 @@ def project(_tool: HarnessTool, _output: BaseModel) -> dict[str, object]: "items": ["another-secret"], } - scoped = local_codemode(CodeModeCapability(upper, result_adapter=project)) + scoped = local_codemode(utility_from_tool(upper, result_adapter=project)) harness = AgentHarness(model="test", model_client=UnusedModel()) await scoped.execute(_ctx(harness), {"code": "upper(value='safe'); output(None)"}) @@ -982,7 +1147,7 @@ async def test_scoped_codemode_treats_json_looking_strings_as_opaque() -> None: def project(_tool: HarnessTool, _output: BaseModel) -> str: return '{"score":1e9999}' - scoped = local_codemode(CodeModeCapability(upper, result_adapter=project)) + scoped = local_codemode(utility_from_tool(upper, result_adapter=project)) harness = AgentHarness(model="test", model_client=UnusedModel()) await scoped.execute(_ctx(harness), {"code": "upper(value='safe'); output(None)"}) @@ -1116,7 +1281,7 @@ def encode(self, *_args, **_kwargs): async def test_scoped_codemode_rejects_unsafe_projected_results( projection: Any, ) -> None: - scoped = local_codemode(CodeModeCapability(upper, result_adapter=projection)) + scoped = local_codemode(utility_from_tool(upper, result_adapter=projection)) harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[scoped]) with pytest.raises(RuntimeError, match="Generated code execution failed"): @@ -1125,13 +1290,12 @@ async def test_scoped_codemode_rejects_unsafe_projected_results( @pytest.mark.asyncio async def test_scoped_codemode_enforces_source_result_and_output_byte_limits() -> None: - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[upper]) source_limited = local_codemode(limits=CodeModeLimits(max_source_bytes=1)) with pytest.raises(ValueError, match="source exceeds maximum size"): await source_limited.execute(_ctx(harness), {"code": "é"}) result_limited = local_codemode( - CodeModeCapability(upper, result_adapter=value_adapter), limits=CodeModeLimits(max_result_bytes=4), ) with pytest.raises(RuntimeError, match="Generated code execution failed"): @@ -1154,10 +1318,9 @@ async def count(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value) scoped = local_codemode( - CodeModeCapability(count, result_adapter=value_adapter), limits=CodeModeLimits(max_result_bytes=8), ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[count]) code = """ try: count(value='one') @@ -1199,7 +1362,7 @@ async def transport_result( return UpperOutput(value=input_value.value) scoped = local_codemode( - CodeModeCapability( + utility_from_tool( transport_result, result_adapter=lambda _tool, _output: "x" * 127, ), @@ -1257,7 +1420,7 @@ async def transport_result( ) scoped = create_codemode_tool( capabilities=( - CodeModeCapability( + utility_from_tool( transport_result, result_adapter=lambda _tool, _output: "x" * 110, ), @@ -1286,13 +1449,12 @@ async def test_scoped_codemode_counts_nested_calls_once_and_enforces_local_limit None ): scoped = local_codemode( - CodeModeCapability(upper, result_adapter=value_adapter), limits=CodeModeLimits(max_nested_calls=1), ) harness = AgentHarness( model="test", model_client=UnusedModel(), - tools=[scoped], + tools=[scoped, upper], usage_limits=UsageLimits(max_tool_calls=10), ) @@ -1325,10 +1487,9 @@ async def count(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value) scoped = local_codemode( - CodeModeCapability(count, result_adapter=value_adapter), limits=CodeModeLimits(max_nested_calls=1), ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[count]) code = """ count(value='one') try: @@ -1367,11 +1528,12 @@ async def count(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: calls.append(input_value.value) return UpperOutput(value=input_value.value) - scoped = local_codemode(CodeModeCapability(count, result_adapter=value_adapter)) + scoped = local_codemode() harness = AgentHarness( model="test", model_client=UnusedModel(), usage_limits=UsageLimits(max_tool_calls=1), + tools=[count], ) code = """ count(value='one') @@ -1433,10 +1595,11 @@ async def event_target(_: ToolCallContext, input_value: UpperInput) -> UpperOutp ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(event_target, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[event_target] + ) original_emit = harness.emit async def failing_emit( @@ -1492,10 +1655,11 @@ async def event_target( ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(event_target, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[event_target] + ) original_emit = harness.emit async def failing_emit( @@ -1539,10 +1703,11 @@ async def recoverable(_: ToolCallContext, input_value: UpperInput) -> UpperOutpu ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(recoverable, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[recoverable] + ) result = await scoped.execute(_ctx(harness), {"code": "unused"}) @@ -1576,10 +1741,9 @@ async def allowed(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(allowed, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[allowed]) with pytest.raises(ValueError, match="Unknown Code Mode capability: unknown"): await scoped.execute(_ctx(harness), {"code": "unused"}) @@ -1601,7 +1765,7 @@ def project(_tool: HarnessTool, output: BaseModel) -> dict[str, str]: value = SensitiveOutput.model_validate(output) return {"value": value.value, "token": "projected-secret"} - scoped = local_codemode(CodeModeCapability(inspect_value, result_adapter=project)) + scoped = local_codemode(utility_from_tool(inspect_value, result_adapter=project)) harness = AgentHarness( model="test", model_client=UnusedModel(), @@ -1739,8 +1903,10 @@ def test_scoped_codemode_event_context_keeps_booleans_distinct() -> None: @pytest.mark.asyncio async def test_scoped_codemode_emits_projected_completion_event() -> None: - scoped = local_codemode(CodeModeCapability(upper, result_adapter=value_adapter)) - harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[scoped]) + scoped = local_codemode() + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, upper] + ) await harness._execute_tool_calls( [ @@ -1789,7 +1955,7 @@ def project(_tool: HarnessTool, _output: BaseModel) -> dict[str, str]: "password": "sensitive-result-value", } - scoped = local_codemode(CodeModeCapability(inspect_mapping, result_adapter=project)) + scoped = local_codemode(utility_from_tool(inspect_mapping, result_adapter=project)) harness = AgentHarness(model="test", model_client=UnusedModel()) await scoped.execute( @@ -1827,8 +1993,8 @@ async def test_scoped_codemode_records_nested_failure_before_propagating() -> No async def fail(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: raise ValueError("secret failure detail") - scoped = local_codemode(CodeModeCapability(fail)) - harness = AgentHarness(model="test", model_client=UnusedModel()) + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[fail]) with pytest.raises(RuntimeError, match="Generated code execution failed"): await scoped.execute(_ctx(harness), {"code": "fail(value='safe')"}) @@ -1843,7 +2009,7 @@ async def fail(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: assert "secret failure detail" not in str(nested) -def _secret_capability(secret: str) -> CodeModeCapability[Any]: +def _secret_capability(secret: str) -> Callable[..., Any]: @tool(name="secret_value") async def secret_value( _context: ToolCallContext, _input_value: UpperInput @@ -1853,7 +2019,7 @@ async def secret_value( def project(_tool: HarnessTool, _output: BaseModel) -> str: return secret - return CodeModeCapability(secret_value, result_adapter=project) + return utility_from_tool(secret_value, result_adapter=project) async def _assert_reraised_secret_is_redacted( @@ -1945,11 +2111,12 @@ async def wait(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: finally: cancelled.set() - scoped = local_codemode(CodeModeCapability(wait)) + scoped = local_codemode() harness = AgentHarness( model="test", model_client=UnusedModel(), usage_limits=UsageLimits(max_codemode_runtime_seconds=2), + tools=[wait], ) with pytest.raises(RuntimeError, match="timed out"): @@ -1972,8 +2139,8 @@ async def wait(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: finally: cancelled.set() - scoped = local_codemode(CodeModeCapability(wait)) - harness = AgentHarness(model="test", model_client=UnusedModel()) + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[wait]) execution = asyncio.create_task( scoped.execute(_ctx(harness), {"code": "wait(value='x')"}) ) @@ -2066,10 +2233,9 @@ async def test_scoped_codemode_uses_injected_runner_without_sandbox( ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[upper]) result = await scoped.execute( _ctx(harness), @@ -2080,7 +2246,9 @@ async def test_scoped_codemode_uses_injected_runner_without_sandbox( request = runner.requests[0] assert request.code == "upper('hello'); output({'done': True})" assert request.question == "Q?" - assert set(request.function_names["harness"]) == {"upper", "output"} + assert "upper" in request.function_names["harness"] + assert "output" in request.function_names["harness"] + assert "codemode" not in request.function_names["harness"] assert ("upper", {"value": "HELLO"}) in runner.results assert harness.usage.tool_calls == 1 @@ -2191,10 +2359,9 @@ async def count(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value) scoped = local_codemode( - CodeModeCapability(count, result_adapter=value_adapter), limits=CodeModeLimits(max_result_bytes=20, max_cumulative_result_bytes=20), ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[count]) code = """ count(value='hi') try: @@ -2259,10 +2426,9 @@ async def wait(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: await asyncio.Future() scoped = local_codemode( - CodeModeCapability(wait), execution_limiter=CodeModeExecutionLimiter(1), ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[wait]) first = asyncio.create_task( scoped.execute(_ctx(harness), {"code": "wait(value='x')"}) ) @@ -2299,8 +2465,8 @@ async def wait(context: ToolCallContext, _input_value: UpperInput) -> UpperOutpu monkeypatch.setattr("hyperforge.codemode.sandbox.CALLBACK_CANCEL_TIMEOUT", 0.01) limiter = CodeModeExecutionLimiter(1) - scoped = local_codemode(CodeModeCapability(wait), execution_limiter=limiter) - harness = AgentHarness(model="test", model_client=UnusedModel()) + scoped = local_codemode(execution_limiter=limiter) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[wait]) harness._turn_id = "originating-turn" execution = asyncio.create_task( scoped.execute(_ctx(harness), {"code": "wait(value='x')"}) diff --git a/hyperforge/tests/harness_sdk/test_harness.py b/hyperforge/tests/harness_sdk/test_harness.py index dfb9f76f..fc96050c 100644 --- a/hyperforge/tests/harness_sdk/test_harness.py +++ b/hyperforge/tests/harness_sdk/test_harness.py @@ -19,7 +19,6 @@ ModelDelta, NucliaModelClient, ToolCallContext, - ToolInheritancePolicy, UsageLimitExceeded, UsageLimits, codemode, @@ -100,7 +99,7 @@ def test_codemode_can_be_registered_explicitly() -> None: assert "codemode" in {tool.name for tool in harness.iter_tools()} -def test_scoped_codemode_is_not_inherited_by_default() -> None: +def test_codemode_is_inherited_like_other_tools() -> None: async def execute(_context: ToolCallContext, value: ToolInput) -> ToolOutput: return ToolOutput(value=value.value) @@ -114,10 +113,8 @@ async def execute(_context: ToolCallContext, value: ToolInput) -> ToolOutput: child = harness._create_child("child", include_history=False) - assert ordinary.inheritance == ToolInheritancePolicy.INHERIT - assert scoped.inheritance == ToolInheritancePolicy.DO_NOT_INHERIT assert "ordinary" in child._tools - assert "codemode" not in child._tools + assert "codemode" in child._tools def test_turn_loop_clears_pending_tool_result_after_non_empty_response() -> None: From 8b096d565ba007387d724872ba134a9f2ac486b2 Mon Sep 17 00:00:00 2001 From: Nathan Van Gheem <524413+vangheem@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:22:08 -0400 Subject: [PATCH 2/4] Allow public attributes on approved Code Mode types --- docs/harness-sdk.md | 25 ++++++---- hyperforge/src/hyperforge/codemode/worker.py | 16 +++++-- .../hyperforge/harness_sdk/tools/codemode.py | 3 +- hyperforge/tests/harness_sdk/test_codemode.py | 48 ++++++++++++++++++- 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/docs/harness-sdk.md b/docs/harness-sdk.md index d9225f7e..ca01317c 100644 --- a/docs/harness-sdk.md +++ b/docs/harness-sdk.md @@ -301,17 +301,22 @@ built-ins: `abs`, `bool`, `bytes`, `chr`, `complex`, `divmod`, `float`, `hash`, `range`, `repr`, `round`, `slice`, `sorted`, `str`, `sum`, `tuple`, and `zip`. Other helpers, including `all`, `any`, `min`, and `max`, are unavailable. `math`, `re`, `statistics`, `itertools`, and `decimal` are available without -imports. Public module attributes are accessible, including functions, constants, -and classes (for example, -`math.sqrt(4)`, `re.findall(pattern, text)`, or `re.Pattern`). Arbitrary object -attributes, private module attributes, and attributes of unapproved modules are -not. `ALLOWED_GLOBAL_MODULES` in `hyperforge/codemode/modules.py` is the shared -registry for worker globals, attribute access, reserved names, and the tool's -module list. Comprehensions and generator expressions remain supported by -the existing restricted Python interpreter. +imports. You can use their public functions, constants, and classes, as well as +public methods and properties on objects defined by these modules: + +```python +amount = decimal.Decimal('0.1').quantize(decimal.Decimal('0.01')) +match = re.search('(x+)', 'xxx') +output({'amount': str(amount), 'matched': match.group(1)}) +``` + +Imports, private attributes, and attributes on objects from unsupported modules +are unavailable. Submodules are not automatically supported. Comprehensions and +generator expressions are supported. + Convert `decimal.Decimal` results to strings (to preserve precision) or floats -before returning them through `output()`, since the worker protocol carries JSON -values rather than Decimal objects. +before returning them through `output()`, which accepts JSON values rather than +Decimal objects. Authorization, nested/global call limits, projected-result limits, and invalid or repeated `output` attempts remain terminal even if generated code catches the immediate callback error. An output attempt rejected by worker diff --git a/hyperforge/src/hyperforge/codemode/worker.py b/hyperforge/src/hyperforge/codemode/worker.py index 852713f6..1e85d798 100644 --- a/hyperforge/src/hyperforge/codemode/worker.py +++ b/hyperforge/src/hyperforge/codemode/worker.py @@ -25,6 +25,7 @@ guarded_iter_unpack_sequence, guarded_unpack_sequence, safe_builtins, + safer_getattr_raise, ) from hyperforge.definition import FunctionDefinition @@ -50,12 +51,17 @@ def guarded_module_attribute(obj: Any, name: str) -> Any: - """Allow public attributes of registered modules, not arbitrary objects.""" - if not any( - obj is module for module in ALLOWED_GLOBAL_MODULES.values() - ) or name.startswith("_"): + """Allow safe public attributes on approved modules and their types.""" + if name.startswith("_"): raise AttributeError("Object attributes are unavailable") - return getattr(obj, name) + if not any(obj is module for module in ALLOWED_GLOBAL_MODULES.values()): + obj_type = obj if isinstance(obj, type) else type(obj) + approved_modules = { + module.__name__ for module in ALLOWED_GLOBAL_MODULES.values() + } + if obj_type.__module__ not in approved_modules: + raise AttributeError("Object attributes are unavailable") + return safer_getattr_raise(obj, name) def _set_memory_limit(max_memory_bytes: int | None) -> None: diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py index 3b0c2b62..f1acb614 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py @@ -34,7 +34,7 @@ "Execute restricted Python code. Each currently active tool is available as a " "function with the same name and arguments. Tool results are JSON data (dicts/lists). " "Supported: assignments, loops, conditionals, indexing, and builtins including len, " - "range, sum, and sorted. Imports and arbitrary object attributes are unavailable. " + "range, sum, and sorted. Imports and attributes on unapproved objects are unavailable. " "Return the final result by calling output(value) exactly once; do not use print()." ) DEFAULT_MAX_SOURCE_BYTES = 64 * 1024 @@ -808,6 +808,7 @@ def _codemode_description(signatures: list[str], description: str | None = None) "\n\nAvailable Python modules (public attributes only):\n" + "\n".join(f"- {name}" for name in ALLOWED_GLOBAL_MODULES) ) + introduction += "\nPublic attributes are also available on classes and instances defined by these modules." if signatures: introduction += "\n\nAvailable utility functions:\n" + "\n".join(signatures) return introduction + "\n\nExample:\n```python\nx = 1\nx = x * 2\noutput(x)\n```" diff --git a/hyperforge/tests/harness_sdk/test_codemode.py b/hyperforge/tests/harness_sdk/test_codemode.py index bfce0b5b..0e834a3f 100644 --- a/hyperforge/tests/harness_sdk/test_codemode.py +++ b/hyperforge/tests/harness_sdk/test_codemode.py @@ -14,6 +14,7 @@ from hyperforge.codemode import RestrictedPythonTask, WorkerExecutionRequest from hyperforge.codemode import sandbox as sandbox_module +from hyperforge.codemode.worker import guarded_module_attribute from hyperforge.harness_sdk import ( AgentHarness, CodeModeDispatch, @@ -595,6 +596,47 @@ async def test_codemode_provides_statistics_itertools_and_decimal_without_import assert result.value == [[3, 7], 5, "0.3"] +@pytest.mark.asyncio +async def test_codemode_traverses_approved_class_instance_and_property_attributes() -> ( + None +): + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel()) + result = await scoped.execute( + _ctx(harness), + { + "code": ( + "amount = decimal.Decimal.from_float(0.5).quantize(decimal.Decimal('0.01'))\n" + "match = re.search('(x+)', 'xxx')\n" + "pattern = re.compile('x')\n" + "distribution = statistics.NormalDist(mu=2, sigma=1)\n" + "output([str(amount), decimal.getcontext().prec > 0, match.group(1), " + "match.re.pattern, re.Pattern.findall(pattern, 'xx'), " + "distribution.mean, distribution.cdf(2)])" + ) + }, + ) + assert result.value == ["0.50", True, "xxx", "(x+)", ["x", "x"], 2, 0.5] + + +@pytest.mark.parametrize( + "module_name", ["unapproved", "decimal.internal", "decimal_helpers"] +) +def test_attribute_guard_checks_actual_type_module_not_instance_metadata( + module_name: str, +) -> None: + class Unapproved: + public = "blocked" + + Unapproved.__module__ = module_name + instance = Unapproved() + instance.__module__ = "decimal" + with pytest.raises(AttributeError, match="Object attributes are unavailable"): + guarded_module_attribute(instance, "public") + with pytest.raises(AttributeError, match="Object attributes are unavailable"): + guarded_module_attribute(Unapproved, "public") + + @pytest.mark.asyncio @pytest.mark.parametrize( "code", @@ -602,8 +644,12 @@ async def test_codemode_provides_statistics_itertools_and_decimal_without_import "import os\noutput(os.getcwd())", "output(math.__dict__)", "output(re.enum.global_enum)", - "output(re.search('x', 'x').group())", + "output(re.search('x', 'x').__class__)", + "output(re.Pattern.__module__)", + "output(decimal.Decimal('0.1').__class__)", "output('x'.upper())", + "output('x'.format())", + "output('x'.format_map({}))", ], ) async def test_codemode_module_support_does_not_allow_object_traversal( From b26941ae3ba52ab33a96e6c57ed426bd28accfac Mon Sep 17 00:00:00 2001 From: Nathan Van Gheem <524413+vangheem@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:40:14 -0400 Subject: [PATCH 3/4] Describe Code Mode tool results and support safe container operations --- docs/harness-sdk.md | 49 +++- hyperforge/src/hyperforge/codemode/modules.py | 4 + hyperforge/src/hyperforge/codemode/worker.py | 25 +- .../src/hyperforge/harness_sdk/harness.py | 18 +- .../src/hyperforge/harness_sdk/schema.py | 108 +++++++++ .../hyperforge/harness_sdk/tools/codemode.py | 53 ++++- .../src/hyperforge/harness_sdk/tools/utils.py | 3 + hyperforge/tests/harness_sdk/test_codemode.py | 217 +++++++++++++++++- 8 files changed, 443 insertions(+), 34 deletions(-) diff --git a/docs/harness-sdk.md b/docs/harness-sdk.md index ca01317c..eb8b65f8 100644 --- a/docs/harness-sdk.md +++ b/docs/harness-sdk.md @@ -197,10 +197,26 @@ resolves tools against the harness executing it, not a captured parent tool list ## Code Mode -Use `create_codemode_tool()` to orchestrate the agent's currently active tools -with restricted Python. Core tools and active external tools are available -automatically; inactive lazy tools are excluded. The tool set is resolved on -each invocation. Code Mode instances are excluded to prevent recursive execution. +Use `create_codemode_tool()` to orchestrate the agent's currently active external +tools with restricted Python. Application tools are available automatically; +inactive lazy tools and core agent tools (such as memory, tool discovery, and +agent spawning) are excluded. Use core agent tools directly, outside Code Mode. +Code Mode instances are also excluded to prevent recursive execution. + +The Code Mode description includes compact input and return-type signatures for +the available application tools, so generated code knows which result keys it can +use. For example: + +```text +- def list_data_catalog(query: str) -> TypedDict[{"name": str, "data": dict[str, Any]}] +``` + +`TypedDict` describes a JSON object's keys; it is not a class to instantiate in +generated code. Signatures reflect JSON results, and `...` indicates omitted +defaults or abbreviated details. Newly activated lazy tools appear in the +description on the next model call. Tool descriptions and full JSON schemas are +not repeated inside the Code Mode description. Python builtins such as `len`, +`range`, and `sum` remain available. Optional `capabilities` are additional trusted Python utility functions, not `HarnessTool` objects. Both synchronous and asynchronous functions are supported; @@ -246,8 +262,8 @@ agent = AgentHarness( ) ``` -No capabilities are required: `create_codemode_tool()` makes all active agent -tools usable. Active tool calls pass through `HarnessTool.execute()`, preserving +No capabilities are required: `create_codemode_tool()` makes active external +tools usable. Tool calls pass through `HarnessTool.execute()`, preserving JSON Schema and Pydantic input/output validation. Results are the tool's complete JSON-mode output, not its formatted context. **Do not put private fields in tool outputs** just because a context formatter hides them. Remove such fields in the @@ -310,9 +326,26 @@ match = re.search('(x+)', 'xxx') output({'amount': str(amount), 'matched': match.group(1)}) ``` -Imports, private attributes, and attributes on objects from unsupported modules -are unavailable. Submodules are not automatically supported. Comprehensions and +Imports, private attributes, and attributes on unapproved objects are unavailable. +Submodules are not automatically supported. Comprehensions and generator expressions are supported. +Public methods on `list`, `dict`, `str`, and `tuple` are available, including +`list.append()`, `dict.get()`, `dict.items()`, `dict.update()`, and `str.upper()`. +You can also construct these types and assign or delete list/dictionary items: + +```python +result = query_data(sql='SELECT status, COUNT(*) AS count FROM resources.workorder GROUP BY status') +counts = [] +for row in result['data'].get('rows', []): + counts.append(row['count']) +summary = {} +summary['total'] = sum(counts) +output(summary) +``` + +Attribute assignment on modules, classes, and other objects is unavailable. +`str.format()` and `str.format_map()` remain unavailable; use string +concatenation or f-strings instead. Core agent tools remain excluded from Code Mode. Convert `decimal.Decimal` results to strings (to preserve precision) or floats before returning them through `output()`, which accepts JSON values rather than diff --git a/hyperforge/src/hyperforge/codemode/modules.py b/hyperforge/src/hyperforge/codemode/modules.py index d0048c1b..c9646ce8 100644 --- a/hyperforge/src/hyperforge/codemode/modules.py +++ b/hyperforge/src/hyperforge/codemode/modules.py @@ -13,3 +13,7 @@ "itertools": itertools, "decimal": decimal, } + +# Allow useful JSON/container operations without exposing the builtins module. +SAFE_BUILTIN_TYPES = frozenset({list, dict, str, tuple}) +WRITABLE_CONTAINER_TYPES = frozenset({list, dict}) diff --git a/hyperforge/src/hyperforge/codemode/worker.py b/hyperforge/src/hyperforge/codemode/worker.py index 1e85d798..2c9f9f1e 100644 --- a/hyperforge/src/hyperforge/codemode/worker.py +++ b/hyperforge/src/hyperforge/codemode/worker.py @@ -39,7 +39,11 @@ decode_protocol_value, encode_protocol_value, ) -from .modules import ALLOWED_GLOBAL_MODULES +from .modules import ( + ALLOWED_GLOBAL_MODULES, + SAFE_BUILTIN_TYPES, + WRITABLE_CONTAINER_TYPES, +) BLOCKED_EXCEPTIONS = { "BaseException", @@ -51,19 +55,26 @@ def guarded_module_attribute(obj: Any, name: str) -> Any: - """Allow safe public attributes on approved modules and their types.""" + """Allow safe public attributes on approved modules and container types.""" if name.startswith("_"): raise AttributeError("Object attributes are unavailable") if not any(obj is module for module in ALLOWED_GLOBAL_MODULES.values()): obj_type = obj if isinstance(obj, type) else type(obj) - approved_modules = { - module.__name__ for module in ALLOWED_GLOBAL_MODULES.values() - } - if obj_type.__module__ not in approved_modules: + if ( + obj_type not in SAFE_BUILTIN_TYPES + and obj_type.__module__ not in ALLOWED_GLOBAL_MODULES + ): raise AttributeError("Object attributes are unavailable") return safer_getattr_raise(obj, name) +def guarded_container_write(obj: Any) -> Any: + """Allow item assignment/deletion on plain lists and dictionaries only.""" + if type(obj) not in WRITABLE_CONTAINER_TYPES: + raise TypeError("Only list and dict items may be modified") + return obj + + def _set_memory_limit(max_memory_bytes: int | None) -> None: if max_memory_bytes is None or sys.platform == "darwin": return @@ -177,9 +188,11 @@ def _process_question_context_sync( if name not in BLOCKED_EXCEPTIONS }, "sum": sum, + **{value.__name__: value for value in SAFE_BUILTIN_TYPES}, }, "_getitem_": lambda obj, index: obj[index], "_getattr_": guarded_module_attribute, + "_write_": guarded_container_write, "_getiter_": iter, "_inplacevar_": guarded_inplace, "dataclass": dataclass, diff --git a/hyperforge/src/hyperforge/harness_sdk/harness.py b/hyperforge/src/hyperforge/harness_sdk/harness.py index 1f4cc1f4..c1196706 100644 --- a/hyperforge/src/hyperforge/harness_sdk/harness.py +++ b/hyperforge/src/hyperforge/harness_sdk/harness.py @@ -6,7 +6,7 @@ import time import uuid from collections.abc import AsyncIterator, Iterable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Literal, NamedTuple from pydantic import BaseModel @@ -473,11 +473,13 @@ def to_tools( """Expose a legacy agent's published functions as harness tools.""" return published_agent_to_tools(agent, namespace=namespace, manager=manager) - def iter_tools(self, *, include_inactive: bool = False) -> Iterable[HarnessTool]: + def iter_tools( + self, *, include_inactive: bool = False, include_core: bool = True + ) -> Iterable[HarnessTool]: """Iterate over registered tools available to this harness.""" return ( tool - for tool in self._tools.values() + for tool in (self._tools.values() if include_core else self._external_tools) if include_inactive or not tool.lazy_load or tool.name in self._active_lazy_tools @@ -698,6 +700,16 @@ async def llm(self, tools: Iterable[HarnessTool] | None = None) -> AgentResult: ): selected.append(registered) seen.add(registered.name) + selected = [ + replace( + tool, + description=tool.description_factory(self), + description_factory=None, + ) + if tool.description_factory is not None + else tool + for tool in selected + ] self.usage.turns += 1 self._check_limit("max_turns", self.usage.turns) result = AgentResult(text="") diff --git a/hyperforge/src/hyperforge/harness_sdk/schema.py b/hyperforge/src/hyperforge/harness_sdk/schema.py index 6d4788e5..5bcfd658 100644 --- a/hyperforge/src/hyperforge/harness_sdk/schema.py +++ b/hyperforge/src/hyperforge/harness_sdk/schema.py @@ -1,8 +1,44 @@ from __future__ import annotations +import json from copy import deepcopy +from itertools import islice from typing import Any +from pydantic import BaseModel +from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue +from pydantic_core import core_schema + +_JSON_SCALAR_TYPES = { + "string": "str", + "integer": "int", + "number": "float", + "boolean": "bool", + "null": "None", +} + + +class _ToolOutputSchema(GenerateJsonSchema): + def model_schema(self, schema: core_schema.ModelSchema) -> JsonSchemaValue: + # model_dump() without a by_alias override honors each nested model's + # serialize_by_alias setting, rather than forcing one flag globally. + previous = self.by_alias + self.by_alias = bool( + schema["cls"].model_config.get("serialize_by_alias", False) + ) + try: + return super().model_schema(schema) + finally: + self.by_alias = previous + + +def compact_model_output_type(model: type[BaseModel]) -> str: + return compact_json_type( + model.model_json_schema( + mode="serialization", schema_generator=_ToolOutputSchema + ) + ) + def flatten_json_schema(schema: dict[str, Any]) -> dict[str, Any]: """Inline local schema references and remove definition containers.""" @@ -43,3 +79,75 @@ def walk(value: Any, stack: tuple[str, ...] = ()) -> Any: return {key: walk(item, stack) for key, item in current.items()} return walk(root) + + +def compact_json_type(schema: dict[str, Any]) -> str: + """Describe a JSON value without schema metadata or unbounded expansion.""" + remaining_nodes = 64 + + def render( + value: dict[str, Any], depth: int = 0, refs: frozenset[str] = frozenset() + ) -> str: + nonlocal remaining_nodes + remaining_nodes -= 1 + if remaining_nodes < 0: + return "Any" + if ref := value.get("$ref"): + if ref in refs or not ref.startswith("#/"): + return "Any" + resolved: Any = schema + for part in ref[2:].split("/"): + if not isinstance(resolved, dict): + return "Any" + resolved = resolved.get(part.replace("~1", "/").replace("~0", "~")) + if not isinstance(resolved, dict): + return "Any" + return render(resolved, depth, refs | {ref}) + alternatives = value.get("anyOf") or value.get("oneOf") + if alternatives: + return " | ".join( + dict.fromkeys(render(item, depth, refs) for item in alternatives) + ) + value_type = value.get("type") + if isinstance(value_type, list): + return " | ".join( + dict.fromkeys( + render({**value, "type": item}, depth, refs) for item in value_type + ) + ) + if value_type in _JSON_SCALAR_TYPES: + return _JSON_SCALAR_TYPES[value_type] + if value_type == "array": + if depth >= 3: + return "list[Any]" + items = value.get("items", {}) + item_type = ( + render(items, depth + 1, refs) if isinstance(items, dict) else "Any" + ) + if prefix := value.get("prefixItems"): + item_type = " | ".join( + dict.fromkeys(render(item, depth + 1, refs) for item in prefix) + ) + return f"list[{item_type}]" + if value_type == "object" or "properties" in value: + if depth >= 3: + return "dict[str, Any]" + properties = value.get("properties") + if properties is not None: + fields = [ + f"{json.dumps(name)}: {render(field, depth + 1, refs)}" + for name, field in islice(properties.items(), 12) + ] + if len(properties) > 12 or value.get("additionalProperties"): + fields.append("...") + return "TypedDict[{" + ", ".join(fields) + "}]" + additional = value.get("additionalProperties") + item_type = ( + render(additional, depth + 1, refs) + if isinstance(additional, dict) + else "Any" + ) + return f"dict[str, {item_type}]" + return "Any" + + return render(schema) diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py index f1acb614..f67a5ea1 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py @@ -25,16 +25,20 @@ from hyperforge.definition import FunctionDefinition from ..models import HarnessEventType +from ..schema import compact_json_type, compact_model_output_type from ..usage import UsageLimitExceeded from . import HarnessTool, ToolCallContext CODEMODE_TOOL_NAME = "codemode" OUTPUT_FUNCTION_NAME = "output" CODEMODE_USAGE_GUIDANCE = ( - "Execute restricted Python code. Each currently active tool is available as a " - "function with the same name and arguments. Tool results are JSON data (dicts/lists). " - "Supported: assignments, loops, conditionals, indexing, and builtins including len, " + "Execute restricted Python code. Active external tools listed below are available as " + "functions with the same names and arguments. Core agent tools are unavailable. " + "Tool results are JSON data (dicts/lists); TypedDict describes their keys. " + "Supported: assignments, loops, conditionals, indexing, comprehensions, and builtins including len, " "range, sum, and sorted. Imports and attributes on unapproved objects are unavailable. " + "Public list, dict, str, and tuple methods and list/dict item assignment are supported. " + "Private attributes and str.format/format_map are unavailable. " "Return the final result by calling output(value) exactly once; do not use print()." ) DEFAULT_MAX_SOURCE_BYTES = 64 * 1024 @@ -245,10 +249,7 @@ async def execute( "Remote Code Mode execution is required but SANDBOX_TOKEN is absent" ) functions = dict(capability_map) - for active_tool in harness.iter_tools(): - # Exclude every Code Mode instance, including custom-named instances. - if getattr(active_tool.handler, "_codemode", False): - continue + for active_tool in _codemode_tools(harness): _validate_function_name(active_tool.name) if active_tool.name in functions: raise ValueError( @@ -273,13 +274,42 @@ async def execute( execute.__name__ = name setattr(execute, "_codemode", True) tool_description = _codemode_description(signatures, description) + + def describe(harness: Any) -> str: + return _codemode_description( + signatures, + description, + tools=[ + _tool_signature(active_tool) for active_tool in _codemode_tools(harness) + ], + ) + return HarnessTool( name=name, handler=execute, description=tool_description, + description_factory=describe, + ) + + +def _codemode_tools(harness: Any) -> Iterable[HarnessTool[Any, Any]]: + return ( + active_tool + for active_tool in harness.iter_tools(include_core=False) + if not getattr(active_tool.handler, "_codemode", False) ) +def _tool_signature(active_tool: HarnessTool[Any, Any]) -> str: + parameters = active_tool.parameters + required = set(parameters.get("required", [])) + arguments = ", ".join( + f"{name}: {compact_json_type(value)}" + ("" if name in required else " = ...") + for name, value in parameters.get("properties", {}).items() + ) + return f"- def {active_tool.name}({arguments}) -> {compact_model_output_type(active_tool.output_model)}" + + def _validate_function_name(name: str) -> None: if ( not name.isascii() @@ -800,7 +830,12 @@ def _nested_event_context(harness: Any) -> dict[str, Any]: return {"execution_context": context} if context else {} -def _codemode_description(signatures: list[str], description: str | None = None) -> str: +def _codemode_description( + signatures: list[str], + description: str | None = None, + *, + tools: list[str] | None = None, +) -> str: introduction = CODEMODE_USAGE_GUIDANCE if description: introduction = f"{description}\n\n{introduction}" @@ -809,6 +844,8 @@ def _codemode_description(signatures: list[str], description: str | None = None) + "\n".join(f"- {name}" for name in ALLOWED_GLOBAL_MODULES) ) introduction += "\nPublic attributes are also available on classes and instances defined by these modules." + if tools: + introduction += "\n\nAvailable tool functions:\n" + "\n".join(tools) if signatures: introduction += "\n\nAvailable utility functions:\n" + "\n".join(signatures) return introduction + "\n\nExample:\n```python\nx = 1\nx = x * 2\noutput(x)\n```" diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/utils.py b/hyperforge/src/hyperforge/harness_sdk/tools/utils.py index 0551b52f..94f105e3 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/utils.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/utils.py @@ -87,6 +87,7 @@ class HarnessTool[InputT: BaseModel, OutputT: BaseModel]: parameters_schema: dict[str, Any] | None = None context_factory: ContextFactory[OutputT] | None = None lazy_load: bool = False + description_factory: Callable[[AgentHarness], str] | None = None input_model: type[InputT] = field(init=False) output_model: type[OutputT] = field(init=False) _parameters: dict[str, Any] = field(init=False, repr=False) @@ -160,6 +161,7 @@ def tool( parameters_schema: dict[str, Any] | None = None, context_factory: ContextFactory[Any] | None = None, lazy_load: bool = False, + description_factory: Callable[[AgentHarness], str] | None = None, ) -> Callable[[ToolHandler[Any, Any]], HarnessTool[Any, Any]]: """Create a harness tool from an annotated async handler.""" @@ -172,6 +174,7 @@ def decorate(handler: ToolHandler[Any, Any]) -> HarnessTool[Any, Any]: parameters_schema=parameters_schema, context_factory=context_factory, lazy_load=lazy_load, + description_factory=description_factory, ) return decorate diff --git a/hyperforge/tests/harness_sdk/test_codemode.py b/hyperforge/tests/harness_sdk/test_codemode.py index 0e834a3f..757add86 100644 --- a/hyperforge/tests/harness_sdk/test_codemode.py +++ b/hyperforge/tests/harness_sdk/test_codemode.py @@ -10,7 +10,7 @@ from typing import Any, Callable import pytest -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, RootModel, field_serializer from hyperforge.codemode import RestrictedPythonTask, WorkerExecutionRequest from hyperforge.codemode import sandbox as sandbox_module @@ -33,6 +33,7 @@ create_codemode_tool, tool, ) +from hyperforge.harness_sdk.schema import compact_model_output_type from hyperforge.harness_sdk.tools import codemode as codemode_module @@ -496,9 +497,7 @@ async def test_codemode_resolves_lazy_tools_for_each_invocation() -> None: @pytest.mark.asyncio -async def test_codemode_excludes_other_codemode_instances_and_includes_core_tools() -> ( - None -): +async def test_codemode_excludes_other_codemode_instances_and_core_tools() -> None: runner = RecordingRunner([worker_task("output", None)]) scoped = create_codemode_tool(runner=runner) other = create_codemode_tool(name="execute_python", runner=runner) @@ -507,10 +506,151 @@ async def test_codemode_excludes_other_codemode_instances_and_includes_core_tool ) await scoped.execute(_ctx(harness), {"code": "output(None)"}) names = set(runner.requests[0].function_names["harness"]) - assert "remember" in names - assert "activate_tools" in names - assert "codemode" not in names - assert "execute_python" not in names + assert names == {"output"} + + +@pytest.mark.asyncio +async def test_codemode_rejects_core_tool_callbacks() -> None: + runner = CatchingRunner( + [worker_task("remember", text="blocked"), worker_task("output", None)] + ) + scoped = create_codemode_tool(runner=runner) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[scoped]) + assert "remember" in {tool.name for tool in harness.iter_tools()} + with pytest.raises(ValueError, match="Unknown Code Mode capability: remember"): + await scoped.execute( + _ctx(harness), {"code": "remember(text='blocked'); output(None)"} + ) + assert harness.usage.tool_calls == 0 + assert runner.errors == ["Unknown Code Mode capability: remember"] * 2 + + +@pytest.mark.asyncio +async def test_model_receives_compact_active_tool_input_and_output_signatures() -> None: + class CatalogInput(BaseModel): + query: str + limit: int = 10 + + class CatalogOutput(BaseModel): + name: str + data: dict[str, Any] + + @tool( + description="A long catalog description that should not be repeated in Code Mode" + ) + async def list_data_catalog( + _context: ToolCallContext, input_value: CatalogInput + ) -> CatalogOutput: + return CatalogOutput(name=input_value.query, data={"count": input_value.limit}) + + class CapturingModel(UnusedModel): + def __init__(self) -> None: + self.descriptions: list[str] = [] + + async def stream(self, **kwargs: Any) -> AsyncIterator[ModelDelta]: + self.descriptions.append( + next( + item.description + for item in kwargs["tools"] + if item.name == "codemode" + ) + ) + yield ModelDelta(text="done") + + model = CapturingModel() + scoped = local_codemode() + lazy = HarnessTool("lazy_upper", upper.handler, lazy_load=True) + harness = AgentHarness( + model="test", model_client=model, tools=[scoped, list_data_catalog, lazy] + ) + await harness.llm() + description = model.descriptions[-1] + assert ( + '- def list_data_catalog(query: str, limit: int = ...) -> TypedDict[{"name": str, "data": dict[str, Any]}]' + in description + ) + assert "A long catalog description" not in description + assert '"properties"' not in description + assert "- def remember(" not in description + assert "- def lazy_upper(" not in description + + await harness.activate_tools([lazy.name]) + await harness.llm() + assert ( + '- def lazy_upper(value: str) -> TypedDict[{"value": str}]' + in model.descriptions[-1] + ) + assert "- def lazy_upper(" not in model.descriptions[0] + assert "- def list_data_catalog(" not in scoped.description + + result = await scoped.execute( + _ctx(harness), + { + "code": "result = list_data_catalog(query='sales')\noutput(result['data']['count'])" + }, + ) + assert result.value == 10 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("by_alias", [False, True]) +async def test_codemode_signatures_describe_serialized_nested_and_root_outputs( + by_alias: bool, +) -> None: + class Empty(BaseModel): + pass + + class Record(BaseModel): + model_config = {"serialize_by_alias": by_alias} + name: str = Field(serialization_alias="public_name") + score: int | None = None + + @field_serializer("score") + def serialize_score(self, value: int | None) -> str: + return str(value) + + class Records(RootModel[list[Record]]): + pass + + @tool() + async def records(_context: ToolCallContext, _input: Empty) -> Records: + return Records([Record(name="sales", score=2)]) + + class CapturingModel(UnusedModel): + description = "" + + async def stream(self, **kwargs: Any) -> AsyncIterator[ModelDelta]: + self.description = next( + item.description for item in kwargs["tools"] if item.name == "codemode" + ) + yield ModelDelta(text="done") + + model = CapturingModel() + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=model, tools=[scoped, records]) + await harness.llm() + name_key = "public_name" if by_alias else "name" + assert ( + f'- def records() -> list[TypedDict[{{"{name_key}": str, "score": str}}]]' + in model.description + ) + result = await scoped.execute(_ctx(harness), {"code": "output(records())"}) + assert result.value == [{name_key: "sales", "score": "2"}] + + +def test_compact_return_types_handle_unions_and_recursive_models() -> None: + class Node(BaseModel): + name: str + child: "Node | None" = None + values: list[int | bool] + metrics: dict[str, float] + + rendered = compact_model_output_type(Node) + assert '"child": Any | None' in rendered + assert '"values": list[int | bool]' in rendered + assert '"metrics": dict[str, float]' in rendered + assert "$ref" not in rendered + assert len(rendered) < 300 @pytest.mark.asyncio @@ -596,6 +736,60 @@ async def test_codemode_provides_statistics_itertools_and_decimal_without_import assert result.value == [[3, 7], 5, "0.3"] +@pytest.mark.asyncio +async def test_codemode_supports_container_methods_and_item_writes_on_tool_results() -> ( + None +): + class Empty(BaseModel): + pass + + class Rows(BaseModel): + rows: list[dict[str, Any]] + + @tool() + async def get_counts(_context: ToolCallContext, _input: Empty) -> Rows: + return Rows( + rows=[{"status": "completed", "count": 10}, {"status": "open", "count": 4}] + ) + + scoped = local_codemode() + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, get_counts] + ) + result = await scoped.execute( + _ctx(harness), + { + "code": ( + "result = get_counts()\n" + "counts = list()\n" + "by_status = dict()\n" + "for row in result.get('rows', []):\n" + " counts.append(row.get('count', 0))\n" + " by_status[row['status'].upper()] = counts[-1]\n" + "summary = {'temporary': True}\n" + "summary.update({'total': sum(counts)})\n" + "summary['mean'] = statistics.mean(counts)\n" + "summary.setdefault('truncated', False)\n" + "del summary['temporary']\n" + "copied = counts.copy()\n" + "copied[0] = 99\n" + "copied[1:2] = [77, 55]\n" + "copied.pop()\n" + "names = tuple(by_status.keys())\n" + "pairs = [[name, value] for name, value in by_status.items()]\n" + "output({'counts': counts, 'summary': summary, 'copied': copied, 'names': names, 'pairs': pairs})" + ) + }, + ) + assert result.value == { + "counts": [10, 4], + "summary": {"total": 14, "mean": 7, "truncated": False}, + "copied": [99, 77], + "names": ["COMPLETED", "OPEN"], + "pairs": [["COMPLETED", 10], ["OPEN", 4]], + } + + @pytest.mark.asyncio async def test_codemode_traverses_approved_class_instance_and_property_attributes() -> ( None @@ -647,9 +841,14 @@ class Unapproved: "output(re.search('x', 'x').__class__)", "output(re.Pattern.__module__)", "output(decimal.Decimal('0.1').__class__)", - "output('x'.upper())", "output('x'.format())", "output('x'.format_map({}))", + "output(str.format('{0.__class__}', 'x'))", + "output((1).bit_length())", + "output(open('unapproved-file'))", + "math.pi = 0\noutput(math.pi)", + "decimal.getcontext().prec = 2\noutput(decimal.getcontext().prec)", + "str.custom = 'blocked'\noutput(str.custom)", ], ) async def test_codemode_module_support_does_not_allow_object_traversal( From 56c73e3d6e1b0153363d95ff15c2ac68f3546a39 Mon Sep 17 00:00:00 2001 From: Nathan Van Gheem <524413+vangheem@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:38:57 -0400 Subject: [PATCH 4/4] Omit active tool arguments from Code Mode descriptions --- docs/harness-sdk.md | 10 +++++----- .../src/hyperforge/harness_sdk/tools/codemode.py | 10 ++-------- hyperforge/tests/harness_sdk/test_codemode.py | 9 ++++----- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/docs/harness-sdk.md b/docs/harness-sdk.md index eb8b65f8..c7e4ac48 100644 --- a/docs/harness-sdk.md +++ b/docs/harness-sdk.md @@ -203,17 +203,17 @@ inactive lazy tools and core agent tools (such as memory, tool discovery, and agent spawning) are excluded. Use core agent tools directly, outside Code Mode. Code Mode instances are also excluded to prevent recursive execution. -The Code Mode description includes compact input and return-type signatures for -the available application tools, so generated code knows which result keys it can -use. For example: +The Code Mode description includes tool names and compact return types, so +generated code knows which result keys it can use. Arguments are omitted because +they are already described by the tools themselves. For example: ```text -- def list_data_catalog(query: str) -> TypedDict[{"name": str, "data": dict[str, Any]}] +- def list_data_catalog(...) -> TypedDict[{"name": str, "data": dict[str, Any]}] ``` `TypedDict` describes a JSON object's keys; it is not a class to instantiate in generated code. Signatures reflect JSON results, and `...` indicates omitted -defaults or abbreviated details. Newly activated lazy tools appear in the +arguments or abbreviated details. Newly activated lazy tools appear in the description on the next model call. Tool descriptions and full JSON schemas are not repeated inside the Code Mode description. Python builtins such as `len`, `range`, and `sum` remain available. diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py index f67a5ea1..894ce619 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py @@ -25,7 +25,7 @@ from hyperforge.definition import FunctionDefinition from ..models import HarnessEventType -from ..schema import compact_json_type, compact_model_output_type +from ..schema import compact_model_output_type from ..usage import UsageLimitExceeded from . import HarnessTool, ToolCallContext @@ -301,13 +301,7 @@ def _codemode_tools(harness: Any) -> Iterable[HarnessTool[Any, Any]]: def _tool_signature(active_tool: HarnessTool[Any, Any]) -> str: - parameters = active_tool.parameters - required = set(parameters.get("required", [])) - arguments = ", ".join( - f"{name}: {compact_json_type(value)}" + ("" if name in required else " = ...") - for name, value in parameters.get("properties", {}).items() - ) - return f"- def {active_tool.name}({arguments}) -> {compact_model_output_type(active_tool.output_model)}" + return f"- def {active_tool.name}(...) -> {compact_model_output_type(active_tool.output_model)}" def _validate_function_name(name: str) -> None: diff --git a/hyperforge/tests/harness_sdk/test_codemode.py b/hyperforge/tests/harness_sdk/test_codemode.py index 757add86..c8c3dd3b 100644 --- a/hyperforge/tests/harness_sdk/test_codemode.py +++ b/hyperforge/tests/harness_sdk/test_codemode.py @@ -526,7 +526,7 @@ async def test_codemode_rejects_core_tool_callbacks() -> None: @pytest.mark.asyncio -async def test_model_receives_compact_active_tool_input_and_output_signatures() -> None: +async def test_model_receives_compact_active_tool_output_signatures() -> None: class CatalogInput(BaseModel): query: str limit: int = 10 @@ -566,7 +566,7 @@ async def stream(self, **kwargs: Any) -> AsyncIterator[ModelDelta]: await harness.llm() description = model.descriptions[-1] assert ( - '- def list_data_catalog(query: str, limit: int = ...) -> TypedDict[{"name": str, "data": dict[str, Any]}]' + '- def list_data_catalog(...) -> TypedDict[{"name": str, "data": dict[str, Any]}]' in description ) assert "A long catalog description" not in description @@ -577,8 +577,7 @@ async def stream(self, **kwargs: Any) -> AsyncIterator[ModelDelta]: await harness.activate_tools([lazy.name]) await harness.llm() assert ( - '- def lazy_upper(value: str) -> TypedDict[{"value": str}]' - in model.descriptions[-1] + '- def lazy_upper(...) -> TypedDict[{"value": str}]' in model.descriptions[-1] ) assert "- def lazy_upper(" not in model.descriptions[0] assert "- def list_data_catalog(" not in scoped.description @@ -631,7 +630,7 @@ async def stream(self, **kwargs: Any) -> AsyncIterator[ModelDelta]: await harness.llm() name_key = "public_name" if by_alias else "name" assert ( - f'- def records() -> list[TypedDict[{{"{name_key}": str, "score": str}}]]' + f'- def records(...) -> list[TypedDict[{{"{name_key}": str, "score": str}}]]' in model.description ) result = await scoped.execute(_ctx(harness), {"code": "output(records())"})