diff --git a/docs/harness-sdk.md b/docs/harness-sdk.md index 59e16a2..c7e4ac4 100644 --- a/docs/harness-sdk.md +++ b/docs/harness-sdk.md @@ -192,70 +192,58 @@ handlers always receive a validated model instance; they never receive `None`. Tools can attach typed context to their result by setting `context_type` and, if needed, registering a schema and formatter with `register_context()`. -Tools are inherited by spawned sub-agents by default. Set -`inheritance=ToolInheritancePolicy.DO_NOT_INHERIT` on `@tool` or `HarnessTool` -when a tool must remain on the current agent. Scoped Code Mode tools use this -policy by default. +Tools, including Code Mode, are inherited by spawned sub-agents. Code Mode +resolves tools against the harness executing it, not a captured parent tool list. -## Scoped Code Mode +## Code Mode -Use `create_codemode_tool()` when generated Python should orchestrate a small, -explicit capability set without exposing those capabilities as top-level model -tools. The capability list must be an immutable tuple. Scoped Code Mode never -calls `harness.iter_tools()` and never includes core or external tools unless the -caller explicitly passes them. +Use `create_codemode_tool()` to orchestrate the agent's currently active external +tools with restricted Python. Application tools are available automatically; +inactive lazy tools and core agent tools (such as memory, tool discovery, and +agent spawning) are excluded. Use core agent tools directly, outside Code Mode. +Code Mode instances are also excluded to prevent recursive execution. -```python -from pydantic import BaseModel, Field +The Code Mode description includes tool names and compact return types, so +generated code knows which result keys it can use. Arguments are omitted because +they are already described by the tools themselves. For example: + +```text +- def list_data_catalog(...) -> TypedDict[{"name": str, "data": dict[str, Any]}] +``` + +`TypedDict` describes a JSON object's keys; it is not a class to instantiate in +generated code. Signatures reflect JSON results, and `...` indicates omitted +arguments or abbreviated details. Newly activated lazy tools appear in the +description on the next model call. Tool descriptions and full JSON schemas are +not repeated inside the Code Mode description. Python builtins such as `len`, +`range`, and `sum` remain available. + +Optional `capabilities` are additional trusted Python utility functions, not +`HarnessTool` objects. Both synchronous and asynchronous functions are supported; +generated code calls either as an ordinary function, without `await`. Any iterable +is accepted and captured when the Code Mode tool is created. Utilities are not +registered as top-level agent tools. Their Python signatures, rather than JSON +schemas or repeated tool descriptions, are included in the Code Mode description. +```python from hyperforge.harness_sdk import ( AgentHarness, - CodeModeCapability, CodeModeExecutionLimiter, CodeModeLimits, - HarnessTool, - ToolCallContext, create_codemode_tool, - tool, ) -class SearchInput(BaseModel): - query: str = Field(description="Read-only catalog search query") - - -class SearchOutput(BaseModel): - matches: list[dict[str, str]] - internal_cursor: str | None = None - - -@tool(description="Search the approved catalog without modifying it.") -async def search_catalog( - context: ToolCallContext, - input_value: SearchInput, -) -> SearchOutput: - return SearchOutput( - matches=[{"id": "item-1", "title": input_value.query}], - internal_cursor="do-not-expose", - ) +def double_string(input_string: str) -> str: + return input_string * 2 -def project_search_result( - capability: HarnessTool, - output: BaseModel, -) -> dict[str, object]: - del capability - validated = SearchOutput.model_validate(output) - return {"matches": validated.matches} +def times_two(input: int) -> int: + return input * 2 code_mode = create_codemode_tool( - capabilities=( - CodeModeCapability( - search_catalog, - result_adapter=project_search_result, - ), - ), + capabilities=[double_string, times_two], limits=CodeModeLimits( max_source_bytes=64 * 1024, max_result_bytes=256 * 1024, @@ -270,30 +258,31 @@ code_mode = create_codemode_tool( agent = AgentHarness( model="your-model", model_client=model_client, - tools=[code_mode], + tools=[*application_tools, code_mode], ) ``` -Only `code_mode` is registered with the harness in this example. -`search_catalog` is callable from generated code but is not advertised as a -top-level model tool. The Code Mode tool description includes each capability's -description and argument schema so the model can write valid calls. Nested -arguments still pass through `HarnessTool.execute()`, including JSON Schema, -Pydantic input, and output validation. - -Every `CodeModeCapability` has a result adapter. The safe default returns the -tool's formatted model-facing context as a string. Prefer an application adapter, -as above, when generated code needs selected structured fields. Projected values -must contain only JSON worker values; the SDK rejects non-serializable values, -non-finite numbers, Pydantic models at any nesting depth, and the reserved -`__model__` transport key. Use -`raw_codemode_result_adapter` only after explicitly deciding that the complete -JSON-mode Pydantic output is safe for generated code. A raw adapter does not -bypass serialization or result-size validation. +No capabilities are required: `create_codemode_tool()` makes active external +tools usable. Tool calls pass through `HarnessTool.execute()`, preserving +JSON Schema and Pydantic input/output validation. Results are the tool's complete +JSON-mode output, not its formatted context. **Do not put private fields in tool +outputs** just because a context formatter hides them. Remove such fields in the +tool itself if generated code must not see them. + +Utility calls use normal Python argument binding, including defaults, positional- +only, and keyword-only arguments. Type annotations document utilities; they are +not runtime validators. Utilities run on the host and must be trusted, bounded +application functions; the sandbox does not restrict their implementation. +Utility results must contain only JSON worker values. Non-serializable values, +non-finite numbers, nested Pydantic models, and the reserved `__model__` key are +rejected. Tool and utility results share the same serialization and size limits. Capability names must be unique ASCII public, non-keyword Python identifiers and must not conflict with worker names such as `codemode`, `output`, `save`, `question`, -`agent_id`, `dataclass`, `Chunk`, `Context`, `List`, `Any`, or `Dict`. +`agent_id`, `dataclass`, `Chunk`, `Context`, `List`, `Any`, `Dict`, `re`, or `math`. +Utility names must not collide with active tool names; collisions fail rather +than silently shadowing a tool. Function arguments named `agent_id` pass through +unchanged. `CodeModeLimits` applies source, per-call projected-result, cumulative projected-result, final-output, and nested-call limits to each invocation. @@ -327,6 +316,40 @@ built-ins: `abs`, `bool`, `bytes`, `chr`, `complex`, `divmod`, `float`, `hash`, `hex`, `id`, `int`, `isinstance`, `issubclass`, `len`, `oct`, `ord`, `pow`, `range`, `repr`, `round`, `slice`, `sorted`, `str`, `sum`, `tuple`, and `zip`. Other helpers, including `all`, `any`, `min`, and `max`, are unavailable. +`math`, `re`, `statistics`, `itertools`, and `decimal` are available without +imports. You can use their public functions, constants, and classes, as well as +public methods and properties on objects defined by these modules: + +```python +amount = decimal.Decimal('0.1').quantize(decimal.Decimal('0.01')) +match = re.search('(x+)', 'xxx') +output({'amount': str(amount), 'matched': match.group(1)}) +``` + +Imports, private attributes, and attributes on unapproved objects are unavailable. +Submodules are not automatically supported. Comprehensions and +generator expressions are supported. +Public methods on `list`, `dict`, `str`, and `tuple` are available, including +`list.append()`, `dict.get()`, `dict.items()`, `dict.update()`, and `str.upper()`. +You can also construct these types and assign or delete list/dictionary items: + +```python +result = query_data(sql='SELECT status, COUNT(*) AS count FROM resources.workorder GROUP BY status') +counts = [] +for row in result['data'].get('rows', []): + counts.append(row['count']) +summary = {} +summary['total'] = sum(counts) +output(summary) +``` + +Attribute assignment on modules, classes, and other objects is unavailable. +`str.format()` and `str.format_map()` remain unavailable; use string +concatenation or f-strings instead. Core agent tools remain excluded from Code Mode. + +Convert `decimal.Decimal` results to strings (to preserve precision) or floats +before returning them through `output()`, which accepts JSON values rather than +Decimal objects. Authorization, nested/global call limits, projected-result limits, and invalid or repeated `output` attempts remain terminal even if generated code catches the immediate callback error. An output attempt rejected by worker @@ -367,9 +390,14 @@ contract and persists its raw `code` and `question` arguments. Applications must not embed secrets in either field. The nested events described above are the sanitized audit surface for capability calls and results. -The existing exported `codemode` tool remains available for compatibility. It -discovers registered tools and returns raw JSON-mode outputs. New applications -that require capability isolation should use `create_codemode_tool()`. +The exported `codemode` convenience tool uses the same implementation and has +no additional utilities. It retains automatic remote/local selection for +compatibility: remote when a socket is configured, otherwise isolated local +execution (equivalent to `remote_required=None`). +Use the factory's default `remote_required=True` for production fail-closed +remote execution. `CodeModeCapability`, result adapters, and +`ToolInheritancePolicy` have been removed; register tools on the agent and pass +plain Python functions as additional capabilities instead. Remote and isolated-process Code Mode execution move values over the bounded JSON worker protocol. Values crossing those worker boundaries must be JSON diff --git a/hyperforge/src/hyperforge/codemode/modules.py b/hyperforge/src/hyperforge/codemode/modules.py new file mode 100644 index 0000000..c9646ce --- /dev/null +++ b/hyperforge/src/hyperforge/codemode/modules.py @@ -0,0 +1,19 @@ +import decimal +import itertools +import math +import re +import statistics +from types import ModuleType + +# Shared by the worker's globals/attribute guard and the harness tool description. +ALLOWED_GLOBAL_MODULES: dict[str, ModuleType] = { + "re": re, + "math": math, + "statistics": statistics, + "itertools": itertools, + "decimal": decimal, +} + +# Allow useful JSON/container operations without exposing the builtins module. +SAFE_BUILTIN_TYPES = frozenset({list, dict, str, tuple}) +WRITABLE_CONTAINER_TYPES = frozenset({list, dict}) diff --git a/hyperforge/src/hyperforge/codemode/worker.py b/hyperforge/src/hyperforge/codemode/worker.py index 0adda40..2c9f9f1 100644 --- a/hyperforge/src/hyperforge/codemode/worker.py +++ b/hyperforge/src/hyperforge/codemode/worker.py @@ -25,6 +25,7 @@ guarded_iter_unpack_sequence, guarded_unpack_sequence, safe_builtins, + safer_getattr_raise, ) from hyperforge.definition import FunctionDefinition @@ -38,6 +39,11 @@ decode_protocol_value, encode_protocol_value, ) +from .modules import ( + ALLOWED_GLOBAL_MODULES, + SAFE_BUILTIN_TYPES, + WRITABLE_CONTAINER_TYPES, +) BLOCKED_EXCEPTIONS = { "BaseException", @@ -48,6 +54,27 @@ REDACTED_EXECUTION_ERROR = "Generated code execution failed" +def guarded_module_attribute(obj: Any, name: str) -> Any: + """Allow safe public attributes on approved modules and container types.""" + if name.startswith("_"): + raise AttributeError("Object attributes are unavailable") + if not any(obj is module for module in ALLOWED_GLOBAL_MODULES.values()): + obj_type = obj if isinstance(obj, type) else type(obj) + if ( + obj_type not in SAFE_BUILTIN_TYPES + and obj_type.__module__ not in ALLOWED_GLOBAL_MODULES + ): + raise AttributeError("Object attributes are unavailable") + return safer_getattr_raise(obj, name) + + +def guarded_container_write(obj: Any) -> Any: + """Allow item assignment/deletion on plain lists and dictionaries only.""" + if type(obj) not in WRITABLE_CONTAINER_TYPES: + raise TypeError("Only list and dict items may be modified") + return obj + + def _set_memory_limit(max_memory_bytes: int | None) -> None: if max_memory_bytes is None or sys.platform == "darwin": return @@ -118,6 +145,7 @@ def __init__( ): self.pipe = pipe self.functions_agent_id: Dict[str, List[str]] = {} + self._harness_functions = False self.debug = debug self.json_protocol = json_protocol self._redact_errors = False @@ -135,6 +163,7 @@ def _process_question_context_sync( redact_errors: bool = False, ): self._redact_errors = redact_errors + self._harness_functions = redact_errors and set(function_names) == {"harness"} self._output_attempted = False self._output_send_failed = False try: @@ -159,8 +188,11 @@ def _process_question_context_sync( if name not in BLOCKED_EXCEPTIONS }, "sum": sum, + **{value.__name__: value for value in SAFE_BUILTIN_TYPES}, }, "_getitem_": lambda obj, index: obj[index], + "_getattr_": guarded_module_attribute, + "_write_": guarded_container_write, "_getiter_": iter, "_inplacevar_": guarded_inplace, "dataclass": dataclass, @@ -173,6 +205,7 @@ def _process_question_context_sync( "_unpack_sequence_": guarded_unpack_sequence, "save": self.save, "question": question, + **ALLOWED_GLOBAL_MODULES, } ) if self.debug: @@ -240,7 +273,11 @@ def save( def valid_functions_call( self, function_name: str, *args: Any, **kwargs: Any ) -> Any: - agent_id = kwargs.pop("agent_id", None) + # Harness function arguments must pass through unchanged. Legacy published + # agent calls still use agent_id to select the callback destination. + agent_id = ( + "harness" if self._harness_functions else kwargs.pop("agent_id", None) + ) if agent_id is not None: if agent_id not in self.functions_agent_id.get(function_name, []): raise ValueError( diff --git a/hyperforge/src/hyperforge/harness_sdk/__init__.py b/hyperforge/src/hyperforge/harness_sdk/__init__.py index d38cdee..b183783 100644 --- a/hyperforge/src/hyperforge/harness_sdk/__init__.py +++ b/hyperforge/src/hyperforge/harness_sdk/__init__.py @@ -39,20 +39,16 @@ HarnessToolCall, ) from .storage import HarnessStorageProtocol, InMemoryHarnessStorage -from .tools import HarnessTool, ToolCallContext, ToolInheritancePolicy, tool +from .tools import HarnessTool, ToolCallContext, tool from .tools.codemode import ( - CodeModeCapability, CodeModeDispatch, CodeModeExecutionLimiter, CodemodeInput, CodeModeLimits, CodemodeOutput, - CodeModeResultAdapter, CodeModeRunner, codemode, - context_codemode_result_adapter, create_codemode_tool, - raw_codemode_result_adapter, ) from .usage import HarnessUsage, UsageLimitExceeded, UsageLimits @@ -69,11 +65,9 @@ "ChatCompletionToolCallDelta", "ChatCompletionToolCallFunctionDelta", "ChatCompletionUsage", - "CodeModeCapability", "CodeModeDispatch", "CodeModeExecutionLimiter", "CodeModeLimits", - "CodeModeResultAdapter", "CodeModeRunner", "CodemodeInput", "CodemodeOutput", @@ -98,18 +92,15 @@ "NucliaChatCompletionsClient", "NucliaChatCompletionsError", "ReasoningEffort", - "ToolInheritancePolicy", "UsageLimitExceeded", "UsageLimits", "context_schema", "codemode", - "context_codemode_result_adapter", "create_agent", "create_codemode_tool", "format_context", "make_context", "register_context", - "raw_codemode_result_adapter", "tool", "published_agent_to_tools", "preload_published_agent_to_tools", diff --git a/hyperforge/src/hyperforge/harness_sdk/harness.py b/hyperforge/src/hyperforge/harness_sdk/harness.py index f43b66f..c119670 100644 --- a/hyperforge/src/hyperforge/harness_sdk/harness.py +++ b/hyperforge/src/hyperforge/harness_sdk/harness.py @@ -6,7 +6,7 @@ import time import uuid from collections.abc import AsyncIterator, Iterable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Literal, NamedTuple from pydantic import BaseModel @@ -23,7 +23,7 @@ HarnessToolCall, ) from .storage import HarnessStorageProtocol, InMemoryHarnessStorage -from .tools import HarnessTool, ToolCallContext, ToolInheritancePolicy +from .tools import HarnessTool, ToolCallContext from .tools.core import DictOutput, SendMessageInput, SpawnAgentInput, create_core_tools from .usage import HarnessUsage, UsageLimitExceeded, UsageLimits @@ -473,11 +473,13 @@ def to_tools( """Expose a legacy agent's published functions as harness tools.""" return published_agent_to_tools(agent, namespace=namespace, manager=manager) - def iter_tools(self, *, include_inactive: bool = False) -> Iterable[HarnessTool]: + def iter_tools( + self, *, include_inactive: bool = False, include_core: bool = True + ) -> Iterable[HarnessTool]: """Iterate over registered tools available to this harness.""" return ( tool - for tool in self._tools.values() + for tool in (self._tools.values() if include_core else self._external_tools) if include_inactive or not tool.lazy_load or tool.name in self._active_lazy_tools @@ -698,6 +700,16 @@ async def llm(self, tools: Iterable[HarnessTool] | None = None) -> AgentResult: ): selected.append(registered) seen.add(registered.name) + selected = [ + replace( + tool, + description=tool.description_factory(self), + description_factory=None, + ) + if tool.description_factory is not None + else tool + for tool in selected + ] self.usage.turns += 1 self._check_limit("max_turns", self.usage.turns) result = AgentResult(text="") @@ -1133,11 +1145,7 @@ def _create_child(self, child_id: str, *, include_history: bool) -> AgentHarness model=self._config.model, model_client=self._config.model_client, reasoning_effort=self._config.reasoning_effort, - tools=( - tool - for tool in self._config.tools - if tool.inheritance == ToolInheritancePolicy.INHERIT - ), + tools=self._config.tools, system_prompt=self._config.system_prompt, title=self._config.title, conversation_id=self.conversation_id, diff --git a/hyperforge/src/hyperforge/harness_sdk/schema.py b/hyperforge/src/hyperforge/harness_sdk/schema.py index 6d4788e..5bcfd65 100644 --- a/hyperforge/src/hyperforge/harness_sdk/schema.py +++ b/hyperforge/src/hyperforge/harness_sdk/schema.py @@ -1,8 +1,44 @@ from __future__ import annotations +import json from copy import deepcopy +from itertools import islice from typing import Any +from pydantic import BaseModel +from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue +from pydantic_core import core_schema + +_JSON_SCALAR_TYPES = { + "string": "str", + "integer": "int", + "number": "float", + "boolean": "bool", + "null": "None", +} + + +class _ToolOutputSchema(GenerateJsonSchema): + def model_schema(self, schema: core_schema.ModelSchema) -> JsonSchemaValue: + # model_dump() without a by_alias override honors each nested model's + # serialize_by_alias setting, rather than forcing one flag globally. + previous = self.by_alias + self.by_alias = bool( + schema["cls"].model_config.get("serialize_by_alias", False) + ) + try: + return super().model_schema(schema) + finally: + self.by_alias = previous + + +def compact_model_output_type(model: type[BaseModel]) -> str: + return compact_json_type( + model.model_json_schema( + mode="serialization", schema_generator=_ToolOutputSchema + ) + ) + def flatten_json_schema(schema: dict[str, Any]) -> dict[str, Any]: """Inline local schema references and remove definition containers.""" @@ -43,3 +79,75 @@ def walk(value: Any, stack: tuple[str, ...] = ()) -> Any: return {key: walk(item, stack) for key, item in current.items()} return walk(root) + + +def compact_json_type(schema: dict[str, Any]) -> str: + """Describe a JSON value without schema metadata or unbounded expansion.""" + remaining_nodes = 64 + + def render( + value: dict[str, Any], depth: int = 0, refs: frozenset[str] = frozenset() + ) -> str: + nonlocal remaining_nodes + remaining_nodes -= 1 + if remaining_nodes < 0: + return "Any" + if ref := value.get("$ref"): + if ref in refs or not ref.startswith("#/"): + return "Any" + resolved: Any = schema + for part in ref[2:].split("/"): + if not isinstance(resolved, dict): + return "Any" + resolved = resolved.get(part.replace("~1", "/").replace("~0", "~")) + if not isinstance(resolved, dict): + return "Any" + return render(resolved, depth, refs | {ref}) + alternatives = value.get("anyOf") or value.get("oneOf") + if alternatives: + return " | ".join( + dict.fromkeys(render(item, depth, refs) for item in alternatives) + ) + value_type = value.get("type") + if isinstance(value_type, list): + return " | ".join( + dict.fromkeys( + render({**value, "type": item}, depth, refs) for item in value_type + ) + ) + if value_type in _JSON_SCALAR_TYPES: + return _JSON_SCALAR_TYPES[value_type] + if value_type == "array": + if depth >= 3: + return "list[Any]" + items = value.get("items", {}) + item_type = ( + render(items, depth + 1, refs) if isinstance(items, dict) else "Any" + ) + if prefix := value.get("prefixItems"): + item_type = " | ".join( + dict.fromkeys(render(item, depth + 1, refs) for item in prefix) + ) + return f"list[{item_type}]" + if value_type == "object" or "properties" in value: + if depth >= 3: + return "dict[str, Any]" + properties = value.get("properties") + if properties is not None: + fields = [ + f"{json.dumps(name)}: {render(field, depth + 1, refs)}" + for name, field in islice(properties.items(), 12) + ] + if len(properties) > 12 or value.get("additionalProperties"): + fields.append("...") + return "TypedDict[{" + ", ".join(fields) + "}]" + additional = value.get("additionalProperties") + item_type = ( + render(additional, depth + 1, refs) + if isinstance(additional, dict) + else "Any" + ) + return f"dict[str, {item_type}]" + return "Any" + + return render(schema) diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/__init__.py b/hyperforge/src/hyperforge/harness_sdk/tools/__init__.py index 7d0c592..bd281c6 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/__init__.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/__init__.py @@ -3,7 +3,6 @@ HarnessTool, ToolCallContext, ToolHandler, - ToolInheritancePolicy, tool, ) @@ -12,6 +11,5 @@ "HarnessTool", "ToolCallContext", "ToolHandler", - "ToolInheritancePolicy", "tool", ] diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py index 6e943d8..894ce61 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/codemode.py @@ -6,6 +6,7 @@ import threading import time import uuid +from collections.abc import Iterable from dataclasses import dataclass from typing import Any, Awaitable, Callable, NoReturn, Protocol, cast @@ -19,24 +20,26 @@ encode_protocol_value, encode_sandbox_message, ) +from hyperforge.codemode.modules import ALLOWED_GLOBAL_MODULES from hyperforge.codemode.sandbox import MAX_PACKET_BYTES, settings from hyperforge.definition import FunctionDefinition -from ..context import format_context from ..models import HarnessEventType +from ..schema import compact_model_output_type from ..usage import UsageLimitExceeded -from . import HarnessTool, ToolCallContext, ToolInheritancePolicy, tool +from . import HarnessTool, ToolCallContext CODEMODE_TOOL_NAME = "codemode" OUTPUT_FUNCTION_NAME = "output" CODEMODE_USAGE_GUIDANCE = ( - "Return the final result by calling output(value) exactly once. " - "Do not use print(); it is unavailable. " - "Available common built-ins are abs, bool, bytes, chr, complex, divmod, float, " - "hash, hex, id, int, isinstance, issubclass, len, oct, ord, pow, range, repr, " - "round, slice, sorted, str, sum, tuple, and zip; helpers such as all, any, min, " - "and max are unavailable. " - "For example, assign the result to a variable and finish with output(result)." + "Execute restricted Python code. Active external tools listed below are available as " + "functions with the same names and arguments. Core agent tools are unavailable. " + "Tool results are JSON data (dicts/lists); TypedDict describes their keys. " + "Supported: assignments, loops, conditionals, indexing, comprehensions, and builtins including len, " + "range, sum, and sorted. Imports and attributes on unapproved objects are unavailable. " + "Public list, dict, str, and tuple methods and list/dict item assignment are supported. " + "Private attributes and str.format/format_map are unavailable. " + "Return the final result by calling output(value) exactly once; do not use print()." ) DEFAULT_MAX_SOURCE_BYTES = 64 * 1024 DEFAULT_MAX_RESULT_BYTES = 1024 * 1024 @@ -65,6 +68,7 @@ "print", "printed", "question", + *ALLOWED_GLOBAL_MODULES, "save", } ) @@ -91,9 +95,6 @@ _SOURCE_SIZE_CHUNK_CHARS = 4096 _CAPABILITY_EVENT_FAILURE = "Code Mode capability failed" _POLICY_CHECK_FAILURE = "Code Mode policy check failed" -type CodeModeResultAdapter[OutputT: BaseModel] = Callable[ - [HarnessTool[Any, OutputT], OutputT], Any | Awaitable[Any] -] type CodeModeDispatch = Callable[[RestrictedPythonTask], Awaitable[Any]] @@ -110,20 +111,6 @@ class CodemodeOutput(BaseModel): value: Any = None -def context_codemode_result_adapter[OutputT: BaseModel]( - capability: HarnessTool[Any, OutputT], output: OutputT -) -> Any: - """Project a result to the same formatted context shown to the model.""" - return format_context(capability.context(output)) - - -def raw_codemode_result_adapter[OutputT: BaseModel]( - _capability: HarnessTool[Any, OutputT], output: OutputT -) -> Any: - """Explicitly expose the capability's complete JSON-mode output.""" - return output.model_dump(mode="json") - - class CodeModeRunner(Protocol): """Execute a request and route worker callbacks through dispatch. @@ -138,9 +125,10 @@ async def run( @dataclass(frozen=True) -class CodeModeCapability[OutputT: BaseModel]: - tool: HarnessTool[Any, OutputT] - result_adapter: CodeModeResultAdapter[OutputT] = context_codemode_result_adapter +class _CodeModeFunction: + name: str + parameters: dict[str, Any] + execute: Callable[[ToolCallContext, RestrictedPythonTask], Awaitable[Any]] @dataclass(frozen=True) @@ -196,155 +184,61 @@ def release(self) -> None: _process_execution_limiter = CodeModeExecutionLimiter() -@tool( - name=CODEMODE_TOOL_NAME, - description=( - "Execute restricted Python code. Registered tools are available as functions; " - f"{CODEMODE_USAGE_GUIDANCE}" - ), -) -async def codemode( - context: ToolCallContext, input_value: CodemodeInput -) -> CodemodeOutput: - harness = context.harness - tools = { - tool.name: tool - for tool in harness.iter_tools() - if tool.name != CODEMODE_TOOL_NAME - } - result = CodemodeOutput() - - async def dispatch(task: RestrictedPythonTask) -> Any: - if task.function == OUTPUT_FUNCTION_NAME: - if task.args and task.keyword_args: - raise ValueError("output accepts either a positional or keyword value") - if len(task.args) > 1: - raise ValueError("output accepts one value") - result.value = task.args[0] if task.args else task.keyword_args.get("value") - return None - - tool = tools.get(task.function) - if tool is None: - raise ValueError(f"Unknown codemode function: {task.function}") - harness.usage.tool_calls += 1 - harness._check_limit("max_tool_calls", harness.usage.tool_calls) - arguments = _tool_arguments(tool, task.args, task.keyword_args) - output = await tool.execute( - ToolCallContext(harness=harness, name=tool.name), arguments - ) - return output.model_dump(mode="json") - - runner = ( - SandboxRunner.remote(settings.sandbox_socket, dispatch) - if settings.sandbox_socket is not None - else SandboxRunner.isolated_process(dispatch) - ) - definitions = { - name: FunctionDefinition( - name=name, - description=tool.description, - parameters=tool.parameters.get("properties", {}), - ) - for name, tool in tools.items() - } - definitions[OUTPUT_FUNCTION_NAME] = FunctionDefinition( - name=OUTPUT_FUNCTION_NAME, - description="Set the value returned by codemode.", - parameters={"value": {}}, - ) - await runner.run( - WorkerExecutionRequest( - code=input_value.code, - question=input_value.question, - local_vars={}, - global_vars={}, - function_names={"harness": definitions}, - max_runtime_seconds=harness.usage_limits.max_codemode_runtime_seconds, - max_memory_bytes=harness.usage_limits.max_codemode_memory_bytes, - ) - ) - return result - - def create_codemode_tool( *, - capabilities: tuple[CodeModeCapability[Any], ...], + capabilities: Iterable[Callable[..., Any]] = (), limits: CodeModeLimits = CodeModeLimits(), execution_limiter: CodeModeExecutionLimiter = _process_execution_limiter, - remote_required: bool = True, + remote_required: bool | None = True, runner: CodeModeRunner | None = None, - inheritance: ToolInheritancePolicy = ToolInheritancePolicy.DO_NOT_INHERIT, name: str = CODEMODE_TOOL_NAME, description: str | None = None, ) -> HarnessTool[CodemodeInput, CodemodeOutput]: - """Create a Code Mode tool with only the explicitly supplied capabilities. + """Create Code Mode with active agent tools and optional Python utilities. + + Capabilities are trusted host-side sync or async functions, not agent tools. + Only their signatures are advertised; their results must be JSON worker values. ``runner`` defaults to the remote sandbox (or the isolated local process when ``remote_required=False``); inject a ``CodeModeRunner`` for - deterministic tests without any sandbox environment. + deterministic tests without any sandbox environment. ``remote_required=None`` + selects remote execution when a sandbox socket is configured, otherwise local. """ - if not isinstance(capabilities, tuple): - raise TypeError("capabilities must be an immutable tuple") - capability_map: dict[str, CodeModeCapability[Any]] = {} + capability_map: dict[str, _CodeModeFunction] = {} + signatures: list[str] = [] for capability in capabilities: - if not isinstance(capability, CodeModeCapability): - raise TypeError("capabilities must contain CodeModeCapability values") - capability_name = capability.tool.name - if ( - not capability_name.isascii() - or not capability_name.isidentifier() - or keyword.iskeyword(capability_name) - or capability_name.startswith("_") - ): - raise ValueError( - f"Code Mode capability name {capability_name!r} must be an ASCII " - "public Python identifier" - ) - if capability_name in RESERVED_CAPABILITY_NAMES: - raise ValueError( - f"Code Mode capability name {capability_name!r} is reserved" - ) + if not callable(capability) or isinstance(capability, HarnessTool): + raise TypeError("capabilities must contain Python functions, not tools") + capability_name = getattr(capability, "__name__", "") + _validate_function_name(capability_name) if capability_name in capability_map: raise ValueError( f"Code Mode capability names must be unique: {capability_name}" ) - if ( - "agent_id" in capability.tool.input_model.model_fields - or "agent_id" in capability.tool.parameters.get("properties", {}) - ): - raise ValueError( - f"Code Mode capability {capability_name!r} uses the reserved " - "'agent_id' input field" - ) - capability_map[capability_name] = capability - scoped_capabilities = tuple(capabilities) - if inheritance == ToolInheritancePolicy.INHERIT: - non_inherited = sorted( - capability.tool.name - for capability in capabilities - if capability.tool.inheritance == ToolInheritancePolicy.DO_NOT_INHERIT - ) - if non_inherited: - raise ValueError( - "An inherited Code Mode tool cannot include capabilities marked " - f"DO_NOT_INHERIT: {', '.join(non_inherited)}" - ) + signature = inspect.signature(capability) + capability_map[capability_name] = _utility_function(capability, signature) + signatures.append(f"- {capability_name}{signature}") async def execute( context: ToolCallContext, input_value: CodemodeInput ) -> CodemodeOutput: harness = context.harness turn_id = harness.turn_id + requires_remote = ( + remote_required + if remote_required is not None + else settings.sandbox_socket is not None + ) if _utf8_exceeds_limit(input_value.code, limits.max_source_bytes): raise ValueError( "Code Mode source exceeds maximum size: " f"> {limits.max_source_bytes} bytes" ) - socket = settings.sandbox_socket if runner is None and remote_required else None + socket = settings.sandbox_socket if runner is None and requires_remote else None sandbox_token = ( - settings.sandbox_token if runner is None and remote_required else None + settings.sandbox_token if runner is None and requires_remote else None ) - if runner is None and remote_required: + if runner is None and requires_remote: if socket is None: raise RuntimeError( "Remote Code Mode execution is required but SANDBOX_SOCKET " @@ -354,35 +248,108 @@ async def execute( raise RuntimeError( "Remote Code Mode execution is required but SANDBOX_TOKEN is absent" ) + functions = dict(capability_map) + for active_tool in _codemode_tools(harness): + _validate_function_name(active_tool.name) + if active_tool.name in functions: + raise ValueError( + f"Code Mode utility conflicts with active tool: {active_tool.name}" + ) + functions[active_tool.name] = _active_tool_function(active_tool) execution_limiter.acquire() return await _execute_scoped_codemode( harness, input_value, - scoped_capabilities, + functions, limits, socket, sandbox_token, execution_limiter, runner, - remote_required, + requires_remote, parent_call_id=context.id, turn_id=turn_id, ) execute.__name__ = name - tool_description = _scoped_description(scoped_capabilities, description) + setattr(execute, "_codemode", True) + tool_description = _codemode_description(signatures, description) + + def describe(harness: Any) -> str: + return _codemode_description( + signatures, + description, + tools=[ + _tool_signature(active_tool) for active_tool in _codemode_tools(harness) + ], + ) + return HarnessTool( name=name, handler=execute, description=tool_description, - inheritance=inheritance, + description_factory=describe, + ) + + +def _codemode_tools(harness: Any) -> Iterable[HarnessTool[Any, Any]]: + return ( + active_tool + for active_tool in harness.iter_tools(include_core=False) + if not getattr(active_tool.handler, "_codemode", False) + ) + + +def _tool_signature(active_tool: HarnessTool[Any, Any]) -> str: + return f"- def {active_tool.name}(...) -> {compact_model_output_type(active_tool.output_model)}" + + +def _validate_function_name(name: str) -> None: + if ( + not name.isascii() + or not name.isidentifier() + or keyword.iskeyword(name) + or name.startswith("_") + ): + raise ValueError( + f"Code Mode capability name {name!r} must be an ASCII public Python identifier" + ) + if name in RESERVED_CAPABILITY_NAMES: + raise ValueError(f"Code Mode capability name {name!r} is reserved") + + +def _active_tool_function(active_tool: HarnessTool[Any, Any]) -> _CodeModeFunction: + async def execute(context: ToolCallContext, task: RestrictedPythonTask) -> Any: + arguments = _scoped_tool_arguments(active_tool, task.args, task.keyword_args) + output = await active_tool.execute(context, arguments) + return output.model_dump(mode="json") + + return _CodeModeFunction( + active_tool.name, active_tool.parameters.get("properties", {}), execute + ) + + +def _utility_function( + function: Callable[..., Any], signature: inspect.Signature +) -> _CodeModeFunction: + async def execute(_context: ToolCallContext, task: RestrictedPythonTask) -> Any: + bound = signature.bind(*task.args, **task.keyword_args) + result = function(*bound.args, **bound.kwargs) + if inspect.isawaitable(result): + result = await result + return result + + return _CodeModeFunction( + getattr(function, "__name__"), + {name: {} for name in signature.parameters}, + execute, ) async def _execute_scoped_codemode( harness: Any, input_value: CodemodeInput, - capabilities: tuple[CodeModeCapability[Any], ...], + capability_map: dict[str, _CodeModeFunction], limits: CodeModeLimits, socket: str | None, sandbox_token: str | None, @@ -393,7 +360,6 @@ async def _execute_scoped_codemode( parent_call_id: str | None = None, turn_id: str | None = None, ) -> CodemodeOutput: - capability_map = {capability.tool.name: capability for capability in capabilities} result = CodemodeOutput() invocation_state = _InvocationState() output_state = _OutputState(invocation_state) @@ -464,44 +430,36 @@ async def dispatch(task: RestrictedPythonTask) -> Any: { "call": { "id": call_id, - "name": capability.tool.name, - "arguments": _sanitize_task_arguments(capability.tool, task), + "name": capability.name, + "arguments": _sanitize_task_arguments(capability, task), }, **marker, }, ) started = time.perf_counter() try: - arguments = _scoped_tool_arguments( - capability.tool, task.args, task.keyword_args - ) - output = await capability.tool.execute( + projected = await capability.execute( ToolCallContext( harness=harness, - name=capability.tool.name, + name=capability.name, id=call_id, turn_id=turn_id, _emit_failure=event_failure, ), - arguments, + task, ) invocation_state.raise_if_failed() - projected = capability.result_adapter(capability.tool, output) - if inspect.isawaitable(projected): - projected = await projected try: normalized, result_bytes = _normalize_worker_value( projected, limits.max_result_bytes, - f"Code Mode result from {capability.tool.name}", + f"Code Mode result from {capability.name}", ) except (TypeError, ValueError) as exc: invocation_state.latch(exc) raise try: - transport_label = ( - f"Code Mode result transport from {capability.tool.name}" - ) + transport_label = f"Code Mode result transport from {capability.name}" if remote_required: encode_sandbox_message( SandboxMessage.Response(result=normalized), @@ -533,7 +491,7 @@ async def dispatch(task: RestrictedPythonTask) -> Any: HarnessEventType.TOOL_FAILED, { "call_id": call_id, - "tool": capability.tool.name, + "tool": capability.name, "result": {"error": type(exc).__name__}, "duration_ms": round((time.perf_counter() - started) * 1000, 3), **marker, @@ -544,7 +502,7 @@ async def dispatch(task: RestrictedPythonTask) -> Any: if isinstance(exc, _CapabilityEventFailure): raise RuntimeError(_CAPABILITY_EVENT_FAILURE) from None raise RuntimeError( - f"Code Mode capability {capability.tool.name!r} failed " + f"Code Mode capability {capability.name!r} failed " f"({type(exc).__name__})" ) from None else: @@ -552,7 +510,7 @@ async def dispatch(task: RestrictedPythonTask) -> Any: HarnessEventType.TOOL_COMPLETED, { "call_id": call_id, - "tool": capability.tool.name, + "tool": capability.name, "result": sanitized_result, "duration_ms": round((time.perf_counter() - started) * 1000, 3), "result_bytes": result_bytes, @@ -571,12 +529,12 @@ async def dispatch(task: RestrictedPythonTask) -> Any: else SandboxRunner.isolated_process(dispatch) ) definitions = { - capability.tool.name: FunctionDefinition( - name=capability.tool.name, - description=capability.tool.description, - parameters=capability.tool.parameters.get("properties", {}), + capability.name: FunctionDefinition( + name=capability.name, + description="", + parameters=capability.parameters, ) - for capability in capabilities + for capability in capability_map.values() } definitions[OUTPUT_FUNCTION_NAME] = FunctionDefinition( name=OUTPUT_FUNCTION_NAME, @@ -812,10 +770,12 @@ def _utf8_size(value: str) -> int: def _sanitize_task_arguments( - capability: HarnessTool[Any, Any], task: RestrictedPythonTask + capability: _CodeModeFunction, task: RestrictedPythonTask ) -> Any: try: - arguments = _scoped_tool_arguments(capability, task.args, task.keyword_args) + arguments = _function_arguments( + capability.name, capability.parameters, task.args, task.keyword_args + ) except ValueError: return {"invalid": True} return _sanitize_event_value(arguments) @@ -864,60 +824,48 @@ def _nested_event_context(harness: Any) -> dict[str, Any]: return {"execution_context": context} if context else {} -def _scoped_description( - capabilities: tuple[CodeModeCapability[Any], ...], description: str | None = None +def _codemode_description( + signatures: list[str], + description: str | None = None, + *, + tools: list[str] | None = None, ) -> str: - introduction = ( - description - or "Execute restricted Python code using only the scoped capabilities below." + introduction = CODEMODE_USAGE_GUIDANCE + if description: + introduction = f"{description}\n\n{introduction}" + introduction += ( + "\n\nAvailable Python modules (public attributes only):\n" + + "\n".join(f"- {name}" for name in ALLOWED_GLOBAL_MODULES) ) - introduction = f"{introduction} {CODEMODE_USAGE_GUIDANCE}" - if not capabilities: - return introduction - definitions = [] - for capability in capabilities: - schema = json.dumps( - capability.tool.parameters, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ) - definitions.append( - f"{capability.tool.name}: {capability.tool.description or 'No description.'} " - f"Arguments JSON Schema: {schema}" - ) - return f"{introduction}\nCapability functions:\n" + "\n".join(definitions) + introduction += "\nPublic attributes are also available on classes and instances defined by these modules." + if tools: + introduction += "\n\nAvailable tool functions:\n" + "\n".join(tools) + if signatures: + introduction += "\n\nAvailable utility functions:\n" + "\n".join(signatures) + return introduction + "\n\nExample:\n```python\nx = 1\nx = x * 2\noutput(x)\n```" -def _tool_arguments( +def _scoped_tool_arguments( tool: HarnessTool[Any, Any], args: tuple[Any, ...], keyword_args: dict[str, Any], ) -> dict[str, Any]: - if not args: - return keyword_args - names = list(tool.input_model.model_fields) - if len(args) > len(names): - raise ValueError(f"Too many positional arguments for {tool.name}") - arguments = dict(zip(names, args, strict=False)) - duplicates = arguments.keys() & keyword_args.keys() - if duplicates: - duplicate = next(iter(duplicates)) - raise ValueError(f"Multiple values for argument {duplicate!r}") - arguments.update(keyword_args) - return arguments + return _function_arguments( + tool.name, tool.parameters.get("properties", {}), args, keyword_args + ) -def _scoped_tool_arguments( - tool: HarnessTool[Any, Any], +def _function_arguments( + name: str, + parameters: dict[str, Any], args: tuple[Any, ...], keyword_args: dict[str, Any], ) -> dict[str, Any]: if not args: return keyword_args - names = list(tool.parameters.get("properties", {})) + names = list(parameters) if len(args) > len(names): - raise ValueError(f"Too many positional arguments for {tool.name}") + raise ValueError(f"Too many positional arguments for {name}") arguments = dict(zip(names, args, strict=False)) duplicates = arguments.keys() & keyword_args.keys() if duplicates: @@ -927,18 +875,20 @@ def _scoped_tool_arguments( return arguments +# The convenience tool shares the factory's dispatch, limits, and output contract. +# Preserve automatic remote/local selection for the convenience tool; production +# callers should use the factory's fail-closed remote default. +codemode = create_codemode_tool(remote_required=None) + + __all__ = [ - "CodeModeCapability", "CodeModeDispatch", "CodeModeExecutionLimiter", "CodeModeLimits", - "CodeModeResultAdapter", "CodeModeRunner", "CodemodeInput", "CodemodeOutput", "RESERVED_CAPABILITY_NAMES", "codemode", - "context_codemode_result_adapter", "create_codemode_tool", - "raw_codemode_result_adapter", ] diff --git a/hyperforge/src/hyperforge/harness_sdk/tools/utils.py b/hyperforge/src/hyperforge/harness_sdk/tools/utils.py index 9e0b595..94f105e 100644 --- a/hyperforge/src/hyperforge/harness_sdk/tools/utils.py +++ b/hyperforge/src/hyperforge/harness_sdk/tools/utils.py @@ -2,7 +2,6 @@ import inspect from dataclasses import dataclass, field -from enum import StrEnum from typing import TYPE_CHECKING, Any, Awaitable, Callable, cast, get_type_hints from jsonschema import ValidationError as JsonSchemaValidationError @@ -34,11 +33,6 @@ class _CurrentTurn: _CURRENT_TURN = _CurrentTurn() -class ToolInheritancePolicy(StrEnum): - INHERIT = "inherit" - DO_NOT_INHERIT = "do_not_inherit" - - @dataclass(frozen=True) class ToolCallContext: """Execution context passed to a tool handler. @@ -93,7 +87,7 @@ class HarnessTool[InputT: BaseModel, OutputT: BaseModel]: parameters_schema: dict[str, Any] | None = None context_factory: ContextFactory[OutputT] | None = None lazy_load: bool = False - inheritance: ToolInheritancePolicy = ToolInheritancePolicy.INHERIT + description_factory: Callable[[AgentHarness], str] | None = None input_model: type[InputT] = field(init=False) output_model: type[OutputT] = field(init=False) _parameters: dict[str, Any] = field(init=False, repr=False) @@ -167,7 +161,7 @@ def tool( parameters_schema: dict[str, Any] | None = None, context_factory: ContextFactory[Any] | None = None, lazy_load: bool = False, - inheritance: ToolInheritancePolicy = ToolInheritancePolicy.INHERIT, + description_factory: Callable[[AgentHarness], str] | None = None, ) -> Callable[[ToolHandler[Any, Any]], HarnessTool[Any, Any]]: """Create a harness tool from an annotated async handler.""" @@ -180,7 +174,7 @@ def decorate(handler: ToolHandler[Any, Any]) -> HarnessTool[Any, Any]: parameters_schema=parameters_schema, context_factory=context_factory, lazy_load=lazy_load, - inheritance=inheritance, + description_factory=description_factory, ) return decorate @@ -195,6 +189,5 @@ def _is_model_type(value: Any) -> bool: "HarnessTool", "ToolCallContext", "ToolHandler", - "ToolInheritancePolicy", "tool", ] diff --git a/hyperforge/tests/harness_sdk/test_codemode.py b/hyperforge/tests/harness_sdk/test_codemode.py index eb85dce..c8c3dd3 100644 --- a/hyperforge/tests/harness_sdk/test_codemode.py +++ b/hyperforge/tests/harness_sdk/test_codemode.py @@ -1,4 +1,5 @@ import asyncio +import inspect import logging import sys import tempfile @@ -6,16 +7,16 @@ from collections.abc import AsyncIterator from contextvars import ContextVar from pathlib import Path -from typing import Any +from typing import Any, Callable import pytest -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, RootModel, field_serializer from hyperforge.codemode import RestrictedPythonTask, WorkerExecutionRequest from hyperforge.codemode import sandbox as sandbox_module +from hyperforge.codemode.worker import guarded_module_attribute from hyperforge.harness_sdk import ( AgentHarness, - CodeModeCapability, CodeModeDispatch, CodeModeExecutionLimiter, CodemodeInput, @@ -27,13 +28,12 @@ HarnessToolCall, ModelDelta, ToolCallContext, - ToolInheritancePolicy, UsageLimits, codemode, create_codemode_tool, - raw_codemode_result_adapter, tool, ) +from hyperforge.harness_sdk.schema import compact_model_output_type from hyperforge.harness_sdk.tools import codemode as codemode_module @@ -118,7 +118,7 @@ def worker_task(function: str, *args: Any, **kwargs: Any) -> RestrictedPythonTas def local_codemode( - *capabilities: CodeModeCapability[Any], + *capabilities: Callable[..., Any], limits: CodeModeLimits | None = None, execution_limiter: CodeModeExecutionLimiter | None = None, ) -> HarnessTool[Any, Any]: @@ -136,10 +136,34 @@ def local_codemode( ) -def value_adapter( - _tool: HarnessTool[Any, UpperOutput], output: UpperOutput -) -> dict[str, str]: - return {"value": UpperOutput.model_validate(output).value} +def utility_from_tool( + wrapped: HarnessTool, result_adapter: Callable[..., Any] +) -> Callable[..., Any]: + """Reuse result-policy fixtures as plain utilities (no harness context needed).""" + + async def utility(*args: Any, **kwargs: Any) -> Any: + arguments = codemode_module._scoped_tool_arguments(wrapped, args, kwargs) + value = wrapped.input_model.model_validate(arguments) + context = ToolCallContext( + harness=AgentHarness(model="test", model_client=UnusedModel()), + name=wrapped.name, + ) + output = await wrapped.handler(context, value) + return result_adapter(wrapped, output) + + utility.__name__ = wrapped.name + setattr( + utility, + "__signature__", + inspect.Signature( + [ + inspect.Parameter(name, inspect.Parameter.POSITIONAL_OR_KEYWORD) + for name in wrapped.parameters.get("properties", {}) + ] + + [inspect.Parameter("extra", inspect.Parameter.VAR_KEYWORD)] + ), + ) + return utility @pytest.mark.asyncio @@ -181,7 +205,7 @@ async def test_codemode_counts_nested_tool_calls() -> None: usage_limits=UsageLimits(max_tool_calls=1), ) - with pytest.raises(RuntimeError, match="max_tool_calls"): + with pytest.raises(RuntimeError, match="Generated code execution failed"): await codemode.execute( ToolCallContext(harness=harness, name=codemode.name), CodemodeInput(code="upper(value='one')\nupper(value='two')").model_dump(), @@ -196,7 +220,7 @@ async def test_codemode_propagates_tool_validation_errors() -> None: tools=[upper], ) - with pytest.raises(RuntimeError, match="Invalid upper arguments"): + with pytest.raises(RuntimeError, match="Generated code execution failed"): await codemode.execute( ToolCallContext(harness=harness, name=codemode.name), CodemodeInput(code="upper(missing='value')").model_dump(), @@ -254,7 +278,9 @@ async def read_context( request_context.reset(token) assert result.value == "available" - assert call_ids == [None] + assert len(call_ids) == 1 + assert call_ids[0] is not None + assert call_ids[0] != "outer-call" @pytest.mark.asyncio @@ -276,7 +302,7 @@ async def test_codemode_enforces_runtime_limit() -> None: async def test_codemode_blocks_process_control_exceptions() -> None: harness = AgentHarness(model="test", model_client=UnusedModel()) - with pytest.raises(RuntimeError, match="SystemExit.*not defined"): + with pytest.raises(RuntimeError, match="Generated code execution failed"): await codemode.execute( ToolCallContext(harness=harness, name=codemode.name), CodemodeInput(code="raise SystemExit(1)").model_dump(), @@ -300,9 +326,7 @@ async def test_codemode_enforces_memory_limit() -> None: @pytest.mark.asyncio -async def test_scoped_codemode_uses_only_explicit_hidden_capabilities( - monkeypatch: pytest.MonkeyPatch, -) -> None: +async def test_codemode_includes_active_tools_and_additional_utilities() -> None: seen_call_ids: list[str | None] = [] @tool(name="hidden_upper") @@ -316,27 +340,36 @@ async def hidden_upper( async def lower(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value.lower()) + def double_string(input_string: str) -> str: + return input_string * 2 + runner = RecordingRunner( - [worker_task("hidden_upper", "hello"), worker_task("output", "done")] + [ + worker_task("hidden_upper", "hello"), + worker_task("double_string", "hello"), + worker_task("output", "done"), + ] ) scoped = create_codemode_tool( - capabilities=( - CodeModeCapability( - hidden_upper, - result_adapter=raw_codemode_result_adapter, - ), - ), + capabilities=[double_string], runner=runner, ) harness = AgentHarness( model="test", model_client=UnusedModel(), - tools=[scoped, lower], - ) - monkeypatch.setattr( - harness, - "iter_tools", - lambda *args, **kwargs: pytest.fail("scoped Code Mode discovered tools"), + tools=[scoped, lower, hidden_upper], + disabled_core_tools=[ + "remember", + "recall", + "forget", + "spawn_agent", + "send_message", + "wait_agent", + "compact", + "search_tools", + "activate_tools", + "call_tool", + ], ) result = await scoped.execute( @@ -348,15 +381,17 @@ async def lower(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: assert len(seen_call_ids) == 1 assert seen_call_ids[0] is not None assert seen_call_ids[0] != "outer-call" - assert hidden_upper.name not in harness._tools assert set(runner.requests[0].function_names["harness"]) == { "hidden_upper", + "lower", + "double_string", "output", } assert runner.requests[0].code == "hidden_upper('hello'); output('done')" assert runner.requests[0].question == "Why?" assert runner.requests[0].redact_errors is True assert runner.results[0] == ("hidden_upper", {"value": "HELLO"}) + assert runner.results[1] == ("double_string", "hellohello") @pytest.mark.asyncio @@ -373,18 +408,12 @@ async def lower(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value.lower()) scoped = create_codemode_tool( - capabilities=( - CodeModeCapability( - upper, - result_adapter=raw_codemode_result_adapter, - ), - ), remote_required=False, ) harness = AgentHarness( model="test", model_client=UnusedModel(), - tools=[scoped, lower], + tools=[scoped, lower, upper], ) result = await scoped.execute( @@ -395,58 +424,439 @@ async def lower(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: ) assert result.value == "HELLO" - assert upper.name not in harness._tools - with pytest.raises(RuntimeError, match="Generated code execution failed"): - await scoped.execute( - ToolCallContext(harness=harness, name=scoped.name), - {"code": "lower(value='HELLO')"}, - ) + result = await scoped.execute( + _ctx(harness), {"code": "output(lower(value='HELLO'))"} + ) + assert result.value == {"value": "hello"} @pytest.mark.asyncio -async def test_scoped_codemode_default_projection_uses_formatted_context() -> None: +async def test_codemode_active_tool_results_are_raw_json() -> None: runner = RecordingRunner( [worker_task("lookup", value="hello"), worker_task("output", None)] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(lookup),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[lookup]) await scoped.execute( ToolCallContext(harness=harness, name=scoped.name), {"code": "lookup(value='hello'); output('done')"}, ) - assert runner.results == [("lookup", '{"value":"HELLO"}'), ("output", None)] + assert runner.results == [ + ("lookup", {"value": "HELLO", "secret": "internal"}), + ("output", None), + ] @pytest.mark.asyncio -async def test_scoped_codemode_raw_projection_must_be_explicit() -> None: - runner = RecordingRunner( - [worker_task("lookup", value="hello"), worker_task("output", None)] +async def test_codemode_orchestrates_tools_sync_and_async_utilities() -> None: + def double_string(input_string: str, /, *, suffix: str = "!") -> str: + return input_string * 2 + suffix + + async def times_two(input: int = 2) -> int: + await asyncio.sleep(0) + return input * 2 + + utilities = [double_string, times_two] + scoped = create_codemode_tool(capabilities=utilities, remote_required=False) + utilities.clear() # The factory captures the iterable, not the mutable list. + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, upper] ) - scoped = create_codemode_tool( - capabilities=( - CodeModeCapability( - lookup, - result_adapter=raw_codemode_result_adapter, - ), - ), - runner=runner, + result = await scoped.execute( + _ctx(harness), + { + "code": "text = double_string('hi')\ntext = upper(value=text)\noutput([text['value'], times_two(), times_two(input=3)])" + }, + ) + assert result.value == ["HIHI!", 4, 6] + assert harness.usage.tool_calls == 4 + + +@pytest.mark.asyncio +async def test_codemode_resolves_lazy_tools_for_each_invocation() -> None: + lazy = HarnessTool("lazy_upper", upper.handler, lazy_load=True) + scoped = local_codemode() + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, lazy] + ) + + with pytest.raises(RuntimeError, match="Generated code execution failed"): + await scoped.execute( + _ctx(harness), {"code": "output(lazy_upper(value='hello'))"} + ) + + await harness.activate_tools([lazy.name]) + result = await scoped.execute( + _ctx(harness), {"code": "output(lazy_upper(value='hello'))"} + ) + assert result.value == {"value": "HELLO"} + + +@pytest.mark.asyncio +async def test_codemode_excludes_other_codemode_instances_and_core_tools() -> None: + runner = RecordingRunner([worker_task("output", None)]) + scoped = create_codemode_tool(runner=runner) + other = create_codemode_tool(name="execute_python", runner=runner) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, other] + ) + await scoped.execute(_ctx(harness), {"code": "output(None)"}) + names = set(runner.requests[0].function_names["harness"]) + assert names == {"output"} + + +@pytest.mark.asyncio +async def test_codemode_rejects_core_tool_callbacks() -> None: + runner = CatchingRunner( + [worker_task("remember", text="blocked"), worker_task("output", None)] + ) + scoped = create_codemode_tool(runner=runner) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[scoped]) + assert "remember" in {tool.name for tool in harness.iter_tools()} + with pytest.raises(ValueError, match="Unknown Code Mode capability: remember"): + await scoped.execute( + _ctx(harness), {"code": "remember(text='blocked'); output(None)"} + ) + assert harness.usage.tool_calls == 0 + assert runner.errors == ["Unknown Code Mode capability: remember"] * 2 + + +@pytest.mark.asyncio +async def test_model_receives_compact_active_tool_output_signatures() -> None: + class CatalogInput(BaseModel): + query: str + limit: int = 10 + + class CatalogOutput(BaseModel): + name: str + data: dict[str, Any] + + @tool( + description="A long catalog description that should not be repeated in Code Mode" + ) + async def list_data_catalog( + _context: ToolCallContext, input_value: CatalogInput + ) -> CatalogOutput: + return CatalogOutput(name=input_value.query, data={"count": input_value.limit}) + + class CapturingModel(UnusedModel): + def __init__(self) -> None: + self.descriptions: list[str] = [] + + async def stream(self, **kwargs: Any) -> AsyncIterator[ModelDelta]: + self.descriptions.append( + next( + item.description + for item in kwargs["tools"] + if item.name == "codemode" + ) + ) + yield ModelDelta(text="done") + + model = CapturingModel() + scoped = local_codemode() + lazy = HarnessTool("lazy_upper", upper.handler, lazy_load=True) + harness = AgentHarness( + model="test", model_client=model, tools=[scoped, list_data_catalog, lazy] + ) + await harness.llm() + description = model.descriptions[-1] + assert ( + '- def list_data_catalog(...) -> TypedDict[{"name": str, "data": dict[str, Any]}]' + in description + ) + assert "A long catalog description" not in description + assert '"properties"' not in description + assert "- def remember(" not in description + assert "- def lazy_upper(" not in description + + await harness.activate_tools([lazy.name]) + await harness.llm() + assert ( + '- def lazy_upper(...) -> TypedDict[{"value": str}]' in model.descriptions[-1] + ) + assert "- def lazy_upper(" not in model.descriptions[0] + assert "- def list_data_catalog(" not in scoped.description + + result = await scoped.execute( + _ctx(harness), + { + "code": "result = list_data_catalog(query='sales')\noutput(result['data']['count'])" + }, + ) + assert result.value == 10 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("by_alias", [False, True]) +async def test_codemode_signatures_describe_serialized_nested_and_root_outputs( + by_alias: bool, +) -> None: + class Empty(BaseModel): + pass + + class Record(BaseModel): + model_config = {"serialize_by_alias": by_alias} + name: str = Field(serialization_alias="public_name") + score: int | None = None + + @field_serializer("score") + def serialize_score(self, value: int | None) -> str: + return str(value) + + class Records(RootModel[list[Record]]): + pass + + @tool() + async def records(_context: ToolCallContext, _input: Empty) -> Records: + return Records([Record(name="sales", score=2)]) + + class CapturingModel(UnusedModel): + description = "" + + async def stream(self, **kwargs: Any) -> AsyncIterator[ModelDelta]: + self.description = next( + item.description for item in kwargs["tools"] if item.name == "codemode" + ) + yield ModelDelta(text="done") + + model = CapturingModel() + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=model, tools=[scoped, records]) + await harness.llm() + name_key = "public_name" if by_alias else "name" + assert ( + f'- def records(...) -> list[TypedDict[{{"{name_key}": str, "score": str}}]]' + in model.description ) + result = await scoped.execute(_ctx(harness), {"code": "output(records())"}) + assert result.value == [{name_key: "sales", "score": "2"}] + + +def test_compact_return_types_handle_unions_and_recursive_models() -> None: + class Node(BaseModel): + name: str + child: "Node | None" = None + values: list[int | bool] + metrics: dict[str, float] + + rendered = compact_model_output_type(Node) + assert '"child": Any | None' in rendered + assert '"values": list[int | bool]' in rendered + assert '"metrics": dict[str, float]' in rendered + assert "$ref" not in rendered + assert len(rendered) < 300 + + +@pytest.mark.asyncio +async def test_codemode_utility_collisions_do_not_shadow_tools() -> None: + def upper(value: str) -> str: + return value + + scoped = local_codemode(upper) + conflicting = HarnessTool("upper", lookup.handler) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[conflicting] + ) + with pytest.raises(ValueError, match="utility conflicts with active tool: upper"): + await scoped.execute(_ctx(harness), {"code": "output(upper('hello'))"}) + + +@pytest.mark.asyncio +async def test_inherited_codemode_uses_the_executing_child_harness() -> None: + class Empty(BaseModel): + pass + + @tool() + async def current_agent(context: ToolCallContext, _input: Empty) -> UpperOutput: + return UpperOutput(value=context.harness.agent_id) + + scoped = local_codemode() + parent = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, current_agent] + ) + child = parent._create_child("child-agent", include_history=False) + result = await child._tools["codemode"].execute( + _ctx(child), {"code": "output(current_agent())"} + ) + assert result.value == {"value": "child-agent"} + + +@pytest.mark.asyncio +async def test_convenience_codemode_does_not_fall_back_when_remote_is_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(codemode_module.settings, "sandbox_socket", "/configured.sock") + monkeypatch.setattr(codemode_module.settings, "sandbox_token", None) harness = AgentHarness(model="test", model_client=UnusedModel()) + with pytest.raises(RuntimeError, match="SANDBOX_TOKEN is absent"): + await codemode.execute(_ctx(harness), {"code": "output(None)"}) - await scoped.execute( - ToolCallContext(harness=harness, name=scoped.name), - {"code": "lookup(value='hello'); output('done')"}, + +@pytest.mark.asyncio +async def test_codemode_provides_re_and_math_without_imports() -> None: + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel()) + result = await scoped.execute( + _ctx(harness), + { + "code": ( + "matches = re.findall('[0-9]+', 'a12b34')\n" + "pattern = re.compile('[0-9]+')\n" + "output([matches, math.sqrt(16), math.pi > 3, isinstance(pattern, re.Pattern)])" + ) + }, ) + assert result.value == [["12", "34"], 4.0, True, True] - assert runner.results == [ - ("lookup", {"value": "HELLO", "secret": "internal"}), - ("output", None), - ] + +@pytest.mark.asyncio +async def test_codemode_provides_statistics_itertools_and_decimal_without_imports() -> ( + None +): + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel()) + result = await scoped.execute( + _ctx(harness), + { + "code": ( + "totals = []\n" + "for batch in itertools.batched([1, 2, 3, 4], 2):\n" + " totals += [sum(batch)]\n" + "amount = decimal.Decimal('0.1') + decimal.Decimal('0.2')\n" + "output([totals, statistics.mean(totals), str(amount)])" + ) + }, + ) + assert result.value == [[3, 7], 5, "0.3"] + + +@pytest.mark.asyncio +async def test_codemode_supports_container_methods_and_item_writes_on_tool_results() -> ( + None +): + class Empty(BaseModel): + pass + + class Rows(BaseModel): + rows: list[dict[str, Any]] + + @tool() + async def get_counts(_context: ToolCallContext, _input: Empty) -> Rows: + return Rows( + rows=[{"status": "completed", "count": 10}, {"status": "open", "count": 4}] + ) + + scoped = local_codemode() + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, get_counts] + ) + result = await scoped.execute( + _ctx(harness), + { + "code": ( + "result = get_counts()\n" + "counts = list()\n" + "by_status = dict()\n" + "for row in result.get('rows', []):\n" + " counts.append(row.get('count', 0))\n" + " by_status[row['status'].upper()] = counts[-1]\n" + "summary = {'temporary': True}\n" + "summary.update({'total': sum(counts)})\n" + "summary['mean'] = statistics.mean(counts)\n" + "summary.setdefault('truncated', False)\n" + "del summary['temporary']\n" + "copied = counts.copy()\n" + "copied[0] = 99\n" + "copied[1:2] = [77, 55]\n" + "copied.pop()\n" + "names = tuple(by_status.keys())\n" + "pairs = [[name, value] for name, value in by_status.items()]\n" + "output({'counts': counts, 'summary': summary, 'copied': copied, 'names': names, 'pairs': pairs})" + ) + }, + ) + assert result.value == { + "counts": [10, 4], + "summary": {"total": 14, "mean": 7, "truncated": False}, + "copied": [99, 77], + "names": ["COMPLETED", "OPEN"], + "pairs": [["COMPLETED", 10], ["OPEN", 4]], + } + + +@pytest.mark.asyncio +async def test_codemode_traverses_approved_class_instance_and_property_attributes() -> ( + None +): + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel()) + result = await scoped.execute( + _ctx(harness), + { + "code": ( + "amount = decimal.Decimal.from_float(0.5).quantize(decimal.Decimal('0.01'))\n" + "match = re.search('(x+)', 'xxx')\n" + "pattern = re.compile('x')\n" + "distribution = statistics.NormalDist(mu=2, sigma=1)\n" + "output([str(amount), decimal.getcontext().prec > 0, match.group(1), " + "match.re.pattern, re.Pattern.findall(pattern, 'xx'), " + "distribution.mean, distribution.cdf(2)])" + ) + }, + ) + assert result.value == ["0.50", True, "xxx", "(x+)", ["x", "x"], 2, 0.5] + + +@pytest.mark.parametrize( + "module_name", ["unapproved", "decimal.internal", "decimal_helpers"] +) +def test_attribute_guard_checks_actual_type_module_not_instance_metadata( + module_name: str, +) -> None: + class Unapproved: + public = "blocked" + + Unapproved.__module__ = module_name + instance = Unapproved() + instance.__module__ = "decimal" + with pytest.raises(AttributeError, match="Object attributes are unavailable"): + guarded_module_attribute(instance, "public") + with pytest.raises(AttributeError, match="Object attributes are unavailable"): + guarded_module_attribute(Unapproved, "public") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "code", + [ + "import os\noutput(os.getcwd())", + "output(math.__dict__)", + "output(re.enum.global_enum)", + "output(re.search('x', 'x').__class__)", + "output(re.Pattern.__module__)", + "output(decimal.Decimal('0.1').__class__)", + "output('x'.format())", + "output('x'.format_map({}))", + "output(str.format('{0.__class__}', 'x'))", + "output((1).bit_length())", + "output(open('unapproved-file'))", + "math.pi = 0\noutput(math.pi)", + "decimal.getcontext().prec = 2\noutput(decimal.getcontext().prec)", + "str.custom = 'blocked'\noutput(str.custom)", + ], +) +async def test_codemode_module_support_does_not_allow_object_traversal( + code: str, +) -> None: + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel()) + with pytest.raises(RuntimeError, match="Generated code execution failed"): + await scoped.execute(_ctx(harness), {"code": code}) @pytest.mark.asyncio @@ -460,7 +870,7 @@ def wrap_model( [worker_task("upper", value="hello"), worker_task("output", None)] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=wrap_model),), + capabilities=(utility_from_tool(upper, result_adapter=wrap_model),), runner=runner, ) harness = AgentHarness(model="test", model_client=UnusedModel()) @@ -498,7 +908,7 @@ def project( [worker_task("upper", value="hello"), worker_task("output", None)] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=project),), + capabilities=(utility_from_tool(upper, result_adapter=project),), runner=runner, ) harness = AgentHarness(model="test", model_client=UnusedModel()) @@ -523,7 +933,7 @@ def project( runner = RecordingRunner([worker_task("upper", value="hello")]) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=project),), + capabilities=(utility_from_tool(upper, result_adapter=project),), runner=runner, ) harness = AgentHarness(model="test", model_client=UnusedModel()) @@ -544,7 +954,7 @@ def project( runner = RecordingRunner([worker_task("upper", value="hello")]) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=project),), + capabilities=(utility_from_tool(upper, result_adapter=project),), runner=runner, ) harness = AgentHarness(model="test", model_client=UnusedModel()) @@ -621,10 +1031,9 @@ async def test_scoped_codemode_rejects_reserved_model_marker_in_output() -> None async def test_scoped_codemode_uses_normal_tool_validation() -> None: invalid_input_runner = RecordingRunner([worker_task("upper", missing="value")]) input_scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper),), runner=invalid_input_runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[upper]) with pytest.raises(RuntimeError, match=r"capability 'upper' failed \(ValueError\)"): await input_scoped.execute( @@ -642,9 +1051,11 @@ async def invalid_output( [worker_task("invalid_output", value="hello")] ) output_scoped = create_codemode_tool( - capabilities=(CodeModeCapability(invalid_output),), runner=invalid_output_runner, ) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[invalid_output] + ) with pytest.raises( RuntimeError, match=r"capability 'invalid_output' failed \(ValidationError\)" ): @@ -669,15 +1080,9 @@ async def aliased( [worker_task("aliased", "hello"), worker_task("output", None)] ) scoped = create_codemode_tool( - capabilities=( - CodeModeCapability( - aliased, - result_adapter=raw_codemode_result_adapter, - ), - ), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[aliased]) await scoped.execute( ToolCallContext(harness=harness, name=scoped.name), @@ -686,7 +1091,7 @@ async def aliased( definition = runner.requests[0].function_names["harness"]["aliased"] assert list(definition.parameters) == ["public"] - assert '"public"' in scoped.description + assert '"public"' not in scoped.description assert runner.results == [ ("aliased", {"value": "HELLO"}), ("output", None), @@ -703,13 +1108,12 @@ async def test_scoped_codemode_counts_nested_calls_as_ordinary_usage() -> None: ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper),), runner=runner, ) harness = AgentHarness( model="test", model_client=UnusedModel(), - tools=[scoped], + tools=[scoped, upper], usage_limits=UsageLimits(max_tool_calls=3), ) @@ -747,10 +1151,9 @@ async def record_id( ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(record_id),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[record_id]) await scoped.execute( ToolCallContext(harness=harness, name=scoped.name, id="outer-call"), @@ -782,10 +1185,13 @@ async def record_id( ], ) def test_scoped_codemode_rejects_reserved_capability_names(name: str) -> None: - reserved = HarnessTool(name, upper.handler) + def reserved(value: str) -> str: + return value + + reserved.__name__ = name with pytest.raises(ValueError, match="reserved"): - create_codemode_tool(capabilities=(CodeModeCapability(reserved),)) + create_codemode_tool(capabilities=[reserved]) @pytest.mark.parametrize( @@ -801,82 +1207,83 @@ def test_scoped_codemode_rejects_reserved_capability_names(name: str) -> None: ], ) def test_scoped_codemode_rejects_invalid_capability_names(name: str) -> None: - invalid = HarnessTool(name, upper.handler) + def invalid(value: str) -> str: + return value + + invalid.__name__ = name with pytest.raises(ValueError, match="ASCII public Python identifier"): - create_codemode_tool(capabilities=(CodeModeCapability(invalid),)) + create_codemode_tool(capabilities=[invalid]) def test_scoped_codemode_rejects_duplicate_capability_names() -> None: - duplicate = HarnessTool(upper.name, upper.handler) + def duplicate(value: str) -> str: + return value with pytest.raises(ValueError, match="must be unique"): - create_codemode_tool( - capabilities=(CodeModeCapability(upper), CodeModeCapability(duplicate)) - ) + create_codemode_tool(capabilities=[duplicate, duplicate]) -def test_scoped_codemode_rejects_agent_id_input_field() -> None: +@pytest.mark.asyncio +async def test_codemode_preserves_agent_id_function_arguments() -> None: + def routed(agent_id: str) -> str: + return agent_id + class AgentInput(BaseModel): agent_id: str @tool() - async def routed(_context: ToolCallContext, input_value: AgentInput) -> UpperOutput: - return UpperOutput(value=input_value.agent_id) - - with pytest.raises(ValueError, match="reserved 'agent_id' input field"): - create_codemode_tool(capabilities=(CodeModeCapability(routed),)) + async def routed_tool(_context: ToolCallContext, value: AgentInput) -> UpperOutput: + return UpperOutput(value=value.agent_id) + scoped = local_codemode(routed) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[routed_tool] + ) + result = await scoped.execute( + _ctx(harness), + { + "code": "value = routed(agent_id='child')\noutput(routed_tool(agent_id=value))" + }, + ) + assert result.value == {"value": "child"} -def test_scoped_codemode_requires_immutable_capability_tuple() -> None: - with pytest.raises(TypeError, match="immutable tuple"): - create_codemode_tool(capabilities=[]) # type: ignore[arg-type] +def test_codemode_rejects_tools_as_additional_utilities() -> None: + with pytest.raises(TypeError, match="Python functions, not tools"): + create_codemode_tool(capabilities=[upper]) -def test_inherited_codemode_rejects_non_inherited_capability() -> None: - @tool(inheritance=ToolInheritancePolicy.DO_NOT_INHERIT) - async def parent_only( - _context: ToolCallContext, input_value: UpperInput - ) -> UpperOutput: - return UpperOutput(value=input_value.value) - with pytest.raises(ValueError, match="cannot include.*DO_NOT_INHERIT"): - create_codemode_tool( - capabilities=(CodeModeCapability(parent_only),), - inheritance=ToolInheritancePolicy.INHERIT, - ) +def test_codemode_description_includes_only_utility_signatures() -> None: + def double_string(input_string: str) -> str: + return input_string * 2 + def times_two(input: int) -> int: + return input * 2 -def test_scoped_codemode_description_includes_capability_schema() -> None: - scoped = create_codemode_tool(capabilities=(CodeModeCapability(upper),)) + scoped = create_codemode_tool(capabilities=[double_string, times_two]) custom = create_codemode_tool( - capabilities=(CodeModeCapability(upper),), + capabilities=[double_string], description="Custom orchestration instructions.", ) - assert "Uppercase a value" in scoped.description - assert '"required":["value"]' in scoped.description - assert "Do not use print(); it is unavailable." in scoped.description - assert ( - "helpers such as all, any, min, and max are unavailable" in scoped.description - ) - assert "finish with output(result)" in scoped.description + assert "- double_string(input_string: str) -> str" in scoped.description + assert "- times_two(input: int) -> int" in scoped.description + assert "JSON Schema" not in scoped.description + assert "output(value) exactly once" in scoped.description + assert "do not use print()" in scoped.description + assert "output(x)" in scoped.description assert "Custom orchestration instructions." in custom.description - assert '"required":["value"]' in custom.description - assert "Do not use print(); it is unavailable." in custom.description + assert "- double_string(input_string: str) -> str" in custom.description def test_compatibility_codemode_description_explains_output_contract() -> None: assert "output(value) exactly once" in codemode.description - assert "Do not use print(); it is unavailable." in codemode.description - assert ( - "helpers such as all, any, min, and max are unavailable" in codemode.description - ) - assert "finish with output(result)" in codemode.description + assert "do not use print()" in codemode.description @pytest.mark.asyncio -async def test_scoped_codemode_default_projection_uses_model_context() -> None: +async def test_codemode_returns_structured_json_instead_of_formatted_context() -> None: @tool( context_factory=lambda output: HarnessContextReference( type=HarnessContextType.STRUCTURED, @@ -886,15 +1293,17 @@ async def test_scoped_codemode_default_projection_uses_model_context() -> None: async def lookup(_: AgentHarness, input_value: UpperInput) -> SensitiveOutput: return SensitiveOutput(value=input_value.value.upper(), secret="internal") - scoped = local_codemode(CodeModeCapability(lookup)) - harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[scoped]) + scoped = local_codemode() + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, lookup] + ) result = await scoped.execute( _ctx(harness), {"code": "result = lookup(value='hello')\noutput(result)"}, ) - assert result.value == '{"value":"HELLO"}' + assert result.value == {"value": "HELLO", "secret": "internal"} @pytest.mark.asyncio @@ -903,8 +1312,8 @@ async def test_scoped_codemode_sanitizes_formatted_context_events() -> None: async def lookup(_: AgentHarness, input_value: UpperInput) -> SensitiveOutput: return SensitiveOutput(value=input_value.value, secret="internal-secret") - scoped = local_codemode(CodeModeCapability(lookup)) - harness = AgentHarness(model="test", model_client=UnusedModel()) + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[lookup]) await scoped.execute(_ctx(harness), {"code": "lookup(value='safe'); output(None)"}) @@ -914,8 +1323,8 @@ async def lookup(_: AgentHarness, input_value: UpperInput) -> SensitiveOutput: for event in events if event.type == HarnessEventType.TOOL_COMPLETED and event.payload.get("nested") ) - assert completed.payload["result"]["type"] == "string" - assert completed.payload["result"]["utf8_bytes"] > 0 + assert completed.payload["result"]["type"] == "object" + assert completed.payload["result"]["entries"][1]["value"] == "[REDACTED]" assert "internal-secret" not in str(completed.payload) @@ -924,7 +1333,7 @@ async def test_scoped_codemode_does_not_persist_opaque_string_results() -> None: def project(_tool: HarnessTool, _output: BaseModel) -> str: return "authorization: Bearer projected-secret" - scoped = local_codemode(CodeModeCapability(upper, result_adapter=project)) + scoped = local_codemode(utility_from_tool(upper, result_adapter=project)) harness = AgentHarness(model="test", model_client=UnusedModel()) await scoped.execute(_ctx(harness), {"code": "upper(value='safe'); output(None)"}) @@ -947,7 +1356,7 @@ def project(_tool: HarnessTool, _output: BaseModel) -> dict[str, object]: "items": ["another-secret"], } - scoped = local_codemode(CodeModeCapability(upper, result_adapter=project)) + scoped = local_codemode(utility_from_tool(upper, result_adapter=project)) harness = AgentHarness(model="test", model_client=UnusedModel()) await scoped.execute(_ctx(harness), {"code": "upper(value='safe'); output(None)"}) @@ -982,7 +1391,7 @@ async def test_scoped_codemode_treats_json_looking_strings_as_opaque() -> None: def project(_tool: HarnessTool, _output: BaseModel) -> str: return '{"score":1e9999}' - scoped = local_codemode(CodeModeCapability(upper, result_adapter=project)) + scoped = local_codemode(utility_from_tool(upper, result_adapter=project)) harness = AgentHarness(model="test", model_client=UnusedModel()) await scoped.execute(_ctx(harness), {"code": "upper(value='safe'); output(None)"}) @@ -1116,7 +1525,7 @@ def encode(self, *_args, **_kwargs): async def test_scoped_codemode_rejects_unsafe_projected_results( projection: Any, ) -> None: - scoped = local_codemode(CodeModeCapability(upper, result_adapter=projection)) + scoped = local_codemode(utility_from_tool(upper, result_adapter=projection)) harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[scoped]) with pytest.raises(RuntimeError, match="Generated code execution failed"): @@ -1125,13 +1534,12 @@ async def test_scoped_codemode_rejects_unsafe_projected_results( @pytest.mark.asyncio async def test_scoped_codemode_enforces_source_result_and_output_byte_limits() -> None: - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[upper]) source_limited = local_codemode(limits=CodeModeLimits(max_source_bytes=1)) with pytest.raises(ValueError, match="source exceeds maximum size"): await source_limited.execute(_ctx(harness), {"code": "é"}) result_limited = local_codemode( - CodeModeCapability(upper, result_adapter=value_adapter), limits=CodeModeLimits(max_result_bytes=4), ) with pytest.raises(RuntimeError, match="Generated code execution failed"): @@ -1154,10 +1562,9 @@ async def count(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value) scoped = local_codemode( - CodeModeCapability(count, result_adapter=value_adapter), limits=CodeModeLimits(max_result_bytes=8), ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[count]) code = """ try: count(value='one') @@ -1199,7 +1606,7 @@ async def transport_result( return UpperOutput(value=input_value.value) scoped = local_codemode( - CodeModeCapability( + utility_from_tool( transport_result, result_adapter=lambda _tool, _output: "x" * 127, ), @@ -1257,7 +1664,7 @@ async def transport_result( ) scoped = create_codemode_tool( capabilities=( - CodeModeCapability( + utility_from_tool( transport_result, result_adapter=lambda _tool, _output: "x" * 110, ), @@ -1286,13 +1693,12 @@ async def test_scoped_codemode_counts_nested_calls_once_and_enforces_local_limit None ): scoped = local_codemode( - CodeModeCapability(upper, result_adapter=value_adapter), limits=CodeModeLimits(max_nested_calls=1), ) harness = AgentHarness( model="test", model_client=UnusedModel(), - tools=[scoped], + tools=[scoped, upper], usage_limits=UsageLimits(max_tool_calls=10), ) @@ -1325,10 +1731,9 @@ async def count(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value) scoped = local_codemode( - CodeModeCapability(count, result_adapter=value_adapter), limits=CodeModeLimits(max_nested_calls=1), ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[count]) code = """ count(value='one') try: @@ -1367,11 +1772,12 @@ async def count(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: calls.append(input_value.value) return UpperOutput(value=input_value.value) - scoped = local_codemode(CodeModeCapability(count, result_adapter=value_adapter)) + scoped = local_codemode() harness = AgentHarness( model="test", model_client=UnusedModel(), usage_limits=UsageLimits(max_tool_calls=1), + tools=[count], ) code = """ count(value='one') @@ -1433,10 +1839,11 @@ async def event_target(_: ToolCallContext, input_value: UpperInput) -> UpperOutp ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(event_target, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[event_target] + ) original_emit = harness.emit async def failing_emit( @@ -1492,10 +1899,11 @@ async def event_target( ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(event_target, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[event_target] + ) original_emit = harness.emit async def failing_emit( @@ -1539,10 +1947,11 @@ async def recoverable(_: ToolCallContext, input_value: UpperInput) -> UpperOutpu ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(recoverable, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[recoverable] + ) result = await scoped.execute(_ctx(harness), {"code": "unused"}) @@ -1576,10 +1985,9 @@ async def allowed(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(allowed, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[allowed]) with pytest.raises(ValueError, match="Unknown Code Mode capability: unknown"): await scoped.execute(_ctx(harness), {"code": "unused"}) @@ -1601,7 +2009,7 @@ def project(_tool: HarnessTool, output: BaseModel) -> dict[str, str]: value = SensitiveOutput.model_validate(output) return {"value": value.value, "token": "projected-secret"} - scoped = local_codemode(CodeModeCapability(inspect_value, result_adapter=project)) + scoped = local_codemode(utility_from_tool(inspect_value, result_adapter=project)) harness = AgentHarness( model="test", model_client=UnusedModel(), @@ -1739,8 +2147,10 @@ def test_scoped_codemode_event_context_keeps_booleans_distinct() -> None: @pytest.mark.asyncio async def test_scoped_codemode_emits_projected_completion_event() -> None: - scoped = local_codemode(CodeModeCapability(upper, result_adapter=value_adapter)) - harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[scoped]) + scoped = local_codemode() + harness = AgentHarness( + model="test", model_client=UnusedModel(), tools=[scoped, upper] + ) await harness._execute_tool_calls( [ @@ -1789,7 +2199,7 @@ def project(_tool: HarnessTool, _output: BaseModel) -> dict[str, str]: "password": "sensitive-result-value", } - scoped = local_codemode(CodeModeCapability(inspect_mapping, result_adapter=project)) + scoped = local_codemode(utility_from_tool(inspect_mapping, result_adapter=project)) harness = AgentHarness(model="test", model_client=UnusedModel()) await scoped.execute( @@ -1827,8 +2237,8 @@ async def test_scoped_codemode_records_nested_failure_before_propagating() -> No async def fail(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: raise ValueError("secret failure detail") - scoped = local_codemode(CodeModeCapability(fail)) - harness = AgentHarness(model="test", model_client=UnusedModel()) + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[fail]) with pytest.raises(RuntimeError, match="Generated code execution failed"): await scoped.execute(_ctx(harness), {"code": "fail(value='safe')"}) @@ -1843,7 +2253,7 @@ async def fail(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: assert "secret failure detail" not in str(nested) -def _secret_capability(secret: str) -> CodeModeCapability[Any]: +def _secret_capability(secret: str) -> Callable[..., Any]: @tool(name="secret_value") async def secret_value( _context: ToolCallContext, _input_value: UpperInput @@ -1853,7 +2263,7 @@ async def secret_value( def project(_tool: HarnessTool, _output: BaseModel) -> str: return secret - return CodeModeCapability(secret_value, result_adapter=project) + return utility_from_tool(secret_value, result_adapter=project) async def _assert_reraised_secret_is_redacted( @@ -1945,11 +2355,12 @@ async def wait(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: finally: cancelled.set() - scoped = local_codemode(CodeModeCapability(wait)) + scoped = local_codemode() harness = AgentHarness( model="test", model_client=UnusedModel(), usage_limits=UsageLimits(max_codemode_runtime_seconds=2), + tools=[wait], ) with pytest.raises(RuntimeError, match="timed out"): @@ -1972,8 +2383,8 @@ async def wait(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: finally: cancelled.set() - scoped = local_codemode(CodeModeCapability(wait)) - harness = AgentHarness(model="test", model_client=UnusedModel()) + scoped = local_codemode() + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[wait]) execution = asyncio.create_task( scoped.execute(_ctx(harness), {"code": "wait(value='x')"}) ) @@ -2066,10 +2477,9 @@ async def test_scoped_codemode_uses_injected_runner_without_sandbox( ] ) scoped = create_codemode_tool( - capabilities=(CodeModeCapability(upper, result_adapter=value_adapter),), runner=runner, ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[upper]) result = await scoped.execute( _ctx(harness), @@ -2080,7 +2490,9 @@ async def test_scoped_codemode_uses_injected_runner_without_sandbox( request = runner.requests[0] assert request.code == "upper('hello'); output({'done': True})" assert request.question == "Q?" - assert set(request.function_names["harness"]) == {"upper", "output"} + assert "upper" in request.function_names["harness"] + assert "output" in request.function_names["harness"] + assert "codemode" not in request.function_names["harness"] assert ("upper", {"value": "HELLO"}) in runner.results assert harness.usage.tool_calls == 1 @@ -2191,10 +2603,9 @@ async def count(_: ToolCallContext, input_value: UpperInput) -> UpperOutput: return UpperOutput(value=input_value.value) scoped = local_codemode( - CodeModeCapability(count, result_adapter=value_adapter), limits=CodeModeLimits(max_result_bytes=20, max_cumulative_result_bytes=20), ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[count]) code = """ count(value='hi') try: @@ -2259,10 +2670,9 @@ async def wait(_: AgentHarness, _input_value: UpperInput) -> UpperOutput: await asyncio.Future() scoped = local_codemode( - CodeModeCapability(wait), execution_limiter=CodeModeExecutionLimiter(1), ) - harness = AgentHarness(model="test", model_client=UnusedModel()) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[wait]) first = asyncio.create_task( scoped.execute(_ctx(harness), {"code": "wait(value='x')"}) ) @@ -2299,8 +2709,8 @@ async def wait(context: ToolCallContext, _input_value: UpperInput) -> UpperOutpu monkeypatch.setattr("hyperforge.codemode.sandbox.CALLBACK_CANCEL_TIMEOUT", 0.01) limiter = CodeModeExecutionLimiter(1) - scoped = local_codemode(CodeModeCapability(wait), execution_limiter=limiter) - harness = AgentHarness(model="test", model_client=UnusedModel()) + scoped = local_codemode(execution_limiter=limiter) + harness = AgentHarness(model="test", model_client=UnusedModel(), tools=[wait]) harness._turn_id = "originating-turn" execution = asyncio.create_task( scoped.execute(_ctx(harness), {"code": "wait(value='x')"}) diff --git a/hyperforge/tests/harness_sdk/test_harness.py b/hyperforge/tests/harness_sdk/test_harness.py index dfb9f76..fc96050 100644 --- a/hyperforge/tests/harness_sdk/test_harness.py +++ b/hyperforge/tests/harness_sdk/test_harness.py @@ -19,7 +19,6 @@ ModelDelta, NucliaModelClient, ToolCallContext, - ToolInheritancePolicy, UsageLimitExceeded, UsageLimits, codemode, @@ -100,7 +99,7 @@ def test_codemode_can_be_registered_explicitly() -> None: assert "codemode" in {tool.name for tool in harness.iter_tools()} -def test_scoped_codemode_is_not_inherited_by_default() -> None: +def test_codemode_is_inherited_like_other_tools() -> None: async def execute(_context: ToolCallContext, value: ToolInput) -> ToolOutput: return ToolOutput(value=value.value) @@ -114,10 +113,8 @@ async def execute(_context: ToolCallContext, value: ToolInput) -> ToolOutput: child = harness._create_child("child", include_history=False) - assert ordinary.inheritance == ToolInheritancePolicy.INHERIT - assert scoped.inheritance == ToolInheritancePolicy.DO_NOT_INHERIT assert "ordinary" in child._tools - assert "codemode" not in child._tools + assert "codemode" in child._tools def test_turn_loop_clears_pending_tool_result_after_non_empty_response() -> None: