From 34a2a155d21b5a8bd286a86b18967c242d7871a8 Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Thu, 11 Jun 2026 02:27:23 -0700 Subject: [PATCH 01/19] Add graph review app for UnresolvedArenaEnvGraphSpec Adds a Streamlit-based live editor for reviewing and editing UnresolvedArenaEnvGraphSpec YAMLs. The review tool renders: - A mermaid.js spatial graph of initial-state constraints with anchor highlighting, task-constraint dashed edges, and objectReference dotted edges - A task table (index, kind, description, params) - A node card grid with type badges and per-node YAML stanzas The YAML editor auto-renders when the spec is valid and changed. Node cards display a two-letter placeholder thumbnail by default. Also adds YAML round-trip serialization to ArenaEnvGraphSpec and moves assert_unique_ids/assert_references_exist to __post_init__ so validation fires on every load-from-yaml/dict path. Signed-off-by: Qian Lin --- .../review_app.py | 275 +++++++++++ .../review_graph.py | 444 ++++++++++++++++++ .../environments/arena_env_graph_spec.py | 12 + setup.py | 6 + 4 files changed, 737 insertions(+) create mode 100644 isaaclab_arena/agentic_environment_generation/review_app.py create mode 100644 isaaclab_arena/agentic_environment_generation/review_graph.py diff --git a/isaaclab_arena/agentic_environment_generation/review_app.py b/isaaclab_arena/agentic_environment_generation/review_app.py new file mode 100644 index 000000000..1a156190a --- /dev/null +++ b/isaaclab_arena/agentic_environment_generation/review_app.py @@ -0,0 +1,275 @@ +# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Streamlit-backed live editor for ArenaEnvInitialGraphSpec YAMLs. + +Wraps :func:`isaaclab_arena.agentic_environment_generation.review_graph.render_html_for_spec` +in a two-pane Streamlit page so the user can edit the ``ArenaEnvInitialGraphSpec`` +YAML directly in the browser and see the visualization update automatically. + +Launch (always via the wrapper in review_graph.py — handles streamlit flags): + /isaac-sim/python.sh -m isaaclab_arena.agentic_environment_generation.review_graph \\ + --yaml path/to/spec.yaml + +Design: + * Left pane — ``streamlit-ace`` YAML editor + validation badge + Save button. + Validation runs on every rerun (i.e. after each editor blur). When the YAML + is valid and has changed since the last render, the visualization updates + automatically — no button click required. + * Right pane — sandboxed iframe with the rendered review HTML. +""" + +from __future__ import annotations + +import argparse +import traceback +import yaml +from dataclasses import dataclass +from pathlib import Path + +import streamlit as st + +from isaaclab_arena.agentic_environment_generation.review_graph import render_html_for_spec +from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec + +# Visualization iframe height. Tuned so the graph + tasks + node grid all +# fit without an outer Streamlit scrollbar swallowing the inner one. +_IFRAME_HEIGHT_PX = 1100 + + +# --------------------------------------------------------------------------- +# Args + session-state init +# --------------------------------------------------------------------------- + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--yaml", + type=Path, + required=True, + help="Path to the ArenaEnvInitialGraphSpec YAML to open in the editor.", + ) + return parser.parse_args() + + +@dataclass +class _ValidationResult: + spec: ArenaEnvInitialGraphSpec | None + error: str | None + + @property + def is_valid(self) -> bool: + return self.spec is not None + + +def _validate_yaml_text(text: str) -> _ValidationResult: + try: + raw = yaml.safe_load(text) + spec = ArenaEnvInitialGraphSpec.model_validate(raw) + return _ValidationResult(spec=spec, error=None) + except Exception: + return _ValidationResult(spec=None, error=traceback.format_exc()) + + +def _initialize_state(yaml_path: Path) -> None: + """Seed ``st.session_state`` from disk exactly once per session. + + We key off ``_yaml_path`` so that if the user passes a different YAML on + a Streamlit reload (rare — usually the same), we reset cleanly. + """ + if st.session_state.get("_yaml_path") == str(yaml_path): + return + + original_text = yaml_path.read_text(encoding="utf-8") + + st.session_state["_yaml_path"] = str(yaml_path) + st.session_state["original_text"] = original_text + st.session_state["edited_text"] = original_text + # The text whose render is currently displayed. Starts == original so the + # first paint shows the on-disk file (and "Regenerate" is correctly + # disabled until the user edits something). + st.session_state["last_rendered_text"] = original_text + st.session_state["save_path"] = str(yaml_path) + + initial = _validate_yaml_text(original_text) + if not initial.is_valid: + # Defensive: if the on-disk file is already broken we still want to + # show *something*, but we won't pre-render it. The user fixes the + # YAML in the editor, then hits Regenerate. + st.session_state["rendered_html"] = _BROKEN_PLACEHOLDER_HTML + else: + st.session_state["rendered_html"] = render_html_for_spec(initial.spec) + + +# Tiny standalone HTML used when the on-disk YAML is itself invalid. +_BROKEN_PLACEHOLDER_HTML = """ +

No visualization yet — fix the YAML errors to auto-render.

+""" + + +# --------------------------------------------------------------------------- +# UI panels +# --------------------------------------------------------------------------- + + +def _render_validation_badge(validation: _ValidationResult) -> None: + """Show a green tick + summary, or a red cross + the raw exception text.""" + if validation.is_valid: + spec = validation.spec + st.success( + f"Valid spec — {spec.env_name} · {len(spec.nodes)} nodes · " + f"{len(spec.tasks)} tasks · initial state: {spec.initial_state_spec.id}", + icon="✅", + ) + else: + st.error(f"Invalid YAML\n\n```\n{validation.error}\n```", icon="🛑") + + +def _render_save_button(validation: _ValidationResult) -> None: + """Render the Save button. Disabled while the YAML is invalid.""" + can_save = validation.is_valid + save_path_str = st.session_state["save_path"] + + if st.button( + f"Save to {Path(save_path_str).name}", + disabled=not can_save, + use_container_width=True, + help=f"Writes the editor contents to {save_path_str}. Disabled while YAML is invalid.", + ): + try: + Path(save_path_str).write_text(st.session_state["edited_text"], encoding="utf-8") + # Update "original" so future comparisons are against the saved file. + st.session_state["original_text"] = st.session_state["edited_text"] + st.toast(f"Saved → {save_path_str}", icon="💾") + except OSError as exc: + st.error(f"Save failed: {exc}", icon="🛑") + + with st.expander("Change save location", expanded=False): + new_path = st.text_input( + "Save path", + value=save_path_str, + key="save_path_input", + help="Defaults to the YAML file passed via --yaml.", + ) + if new_path and new_path != save_path_str: + st.session_state["save_path"] = new_path + + +def _render_editor_panel(yaml_path: Path) -> _ValidationResult: + """Left pane. Returns the validation result for the current editor text. + + Returning the validation result (rather than stashing it in session_state) + keeps the data flow inside one render pass and avoids a stale-state class + of bug where the badge and the buttons disagree. + """ + # Lazy import so the module is importable from environments that don't + # have streamlit-ace installed yet (we surface a clean error message + # rather than ImportError at module load). + try: + from streamlit_ace import st_ace # noqa: PLC0415 + except ImportError as exc: + # See review_graph._serve_live_editor for why --user --ignore-installed + # is required inside the isaaclab_arena container. + st.error( + "`streamlit-ace` is not installed. Inside the isaaclab_arena container run:\n" + "`python -m pip install --user --ignore-installed streamlit-ace`\n\n" + f"Underlying error: {exc}", + icon="🛑", + ) + st.stop() + + st.subheader("YAML editor") + st.caption(f"Source: `{yaml_path}`") + + # ``auto_update=False`` commits on blur / Ctrl+Enter rather than on every + # keystroke, showing an "Apply" button in the editor toolbar. + new_text = st_ace( + value=st.session_state["edited_text"], + language="yaml", + theme="monokai", + keybinding="vscode", + font_size=13, + tab_size=2, + show_gutter=True, + show_print_margin=False, + wrap=False, + auto_update=False, + min_lines=30, + # Key bound to the YAML path so swapping --yaml between sessions + # forces ace to remount with the new content. + key=f"ace_editor::{yaml_path}", + ) + if new_text is not None: + st.session_state["edited_text"] = new_text + + validation = _validate_yaml_text(st.session_state["edited_text"]) + _render_validation_badge(validation) + + # Auto-render whenever the YAML is valid and has changed since the last + # render. This runs before the right pane is drawn, so the updated HTML + # is already in session_state when the iframe is mounted — no extra rerun + # needed. + edited_since_render = st.session_state["edited_text"] != st.session_state["last_rendered_text"] + if validation.is_valid and edited_since_render: + with st.spinner("Rendering visualization…"): + st.session_state["rendered_html"] = render_html_for_spec(validation.spec) + st.session_state["last_rendered_text"] = st.session_state["edited_text"] + st.toast("Visualization updated.", icon="🔄") + + _render_save_button(validation) + return validation + + +def _render_visualization_panel() -> None: + """Right pane — iframe-mount the cached rendered HTML.""" + st.subheader("Visualization") + st.caption("Updates automatically when the YAML is valid.") + + # ``st.components.v1.html`` wraps the payload in a sandboxed iframe, which + # is what we want — the mermaid CDN script and the static CSS stay + # isolated from Streamlit's own DOM. + st.components.v1.html( + st.session_state["rendered_html"], + height=_IFRAME_HEIGHT_PX, + scrolling=True, + ) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + st.set_page_config( + page_title="ArenaEnvInitialGraphSpec live editor", + layout="wide", + initial_sidebar_state="collapsed", + ) + + args = _parse_args() + yaml_path = args.yaml.resolve() + if not yaml_path.exists(): + st.error(f"YAML file not found: {yaml_path}", icon="🛑") + st.stop() + + _initialize_state(yaml_path) + + st.markdown("### ArenaEnvInitialGraphSpec live editor") + left, right = st.columns([2, 3], gap="large") + with left: + _render_editor_panel(yaml_path) + with right: + _render_visualization_panel() + + +# Streamlit invokes the script top-level on every rerun, so we run main() +# unconditionally. The standard ``if __name__ == "__main__"`` guard would +# also work under ``streamlit run`` but is unnecessary — this module is only +# ever loaded as the Streamlit entrypoint. +main() diff --git a/isaaclab_arena/agentic_environment_generation/review_graph.py b/isaaclab_arena/agentic_environment_generation/review_graph.py new file mode 100644 index 000000000..0bf2ecbda --- /dev/null +++ b/isaaclab_arena/agentic_environment_generation/review_graph.py @@ -0,0 +1,444 @@ +# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""ArenaEnvInitialGraphSpec review tool — Streamlit live editor. + +The CLI is a thin launcher: it boots the Streamlit app in ``review_app.py``. + +The tool accepts an ``ArenaEnvInitialGraphSpec`` YAML as input. The user +edits the spec directly inside the Streamlit editor and the preview updates +in real time. + +Three panels (dark dashboard style) inside the embedded view: + * Top-left — graph diagram (mermaid.js, CDN-loaded) of the initial-state + spatial constraints. Anchor nodes are highlighted; constraints without + a reference (is_anchor / position_limits / at_pose / ...) are listed below + the graph rather than rendered as self-loops. + * Bottom-left — task table (index, kind, description, params). + * Right — node card grid: type badge, asset name, and the per-node YAML + stanza. + +Usage: + /isaac-sim/python.sh -m isaaclab_arena.agentic_environment_generation.review_graph \\ + --yaml isaaclab_arena/tests/test_data/pick_and_place_maple_table_init_env_graph.yaml + + # Custom port: + /isaac-sim/python.sh -m isaaclab_arena.agentic_environment_generation.review_graph \\ + --yaml --port 8600 + +Public API used by ``review_app.py``: + * :func:`render_html_for_spec` — full HTML payload for a spec. +""" + +from __future__ import annotations + +import argparse +import html as html_lib +import os +import re +import subprocess +import sys +import yaml +from pathlib import Path + +from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec +from isaaclab_arena.environments.arena_env_graph_types import ArenaEnvGraphNodeSpec, ArenaEnvGraphStateSpec + + +def main() -> None: + """CLI entry point — argparse parses the user's flags, then we hand off + to Streamlit. The actual interactive UI lives in ``review_app.py``. + """ + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--yaml", + type=Path, + required=True, + help="Path to an ArenaEnvInitialGraphSpec YAML file. The Streamlit app will open it for live editing.", + ) + parser.add_argument( + "--port", + type=int, + default=8501, + help="Streamlit server port (default: 8501).", + ) + args = parser.parse_args() + _serve_live_editor(args.yaml, port=args.port) + + +def _serve_live_editor(yaml_path: Path, port: int = 8501) -> None: + """Spawn ``streamlit run review_app.py -- --yaml `` and wait. + + We resolve ``review_app.py`` next to this file rather than going through + ``-m`` so Streamlit picks the path up cleanly (``streamlit run`` doesn't + accept module dotted-paths). + """ + app_path = Path(__file__).with_name("review_app.py") + if not app_path.exists(): + raise FileNotFoundError(f"Streamlit app not found at {app_path} — installation is incomplete.") + + cmd = [ + sys.executable, + "-m", + "streamlit", + "run", + str(app_path), + "--server.port", + str(port), + # Skip the email prompt the first time Streamlit runs in a fresh + # container — the live editor is a developer tool, not a hosted + # service, and an interactive prompt would block automation. + "--browser.gatherUsageStats", + "false", + # File watcher is a footgun here: Kit's ``SimulationApp`` boot is + # tens of seconds; we don't want Streamlit silently rerunning the + # script (and reissuing the cached_resource init) every time we + # save a source file during development. The user can still hit "R" + # in the browser to force a rerun if they want. + "--server.fileWatcherType", + "none", + "--", + "--yaml", + str(yaml_path.resolve()), + ] + + # Inherit env so the Streamlit subprocess sees PYTHONPATH / isaac-sim + # site-packages exactly the same way we do. + print(f"[review_graph] launching Streamlit live editor: {' '.join(cmd)}", file=sys.stderr) + try: + subprocess.run(cmd, env=os.environ.copy(), check=True) + except FileNotFoundError as exc: + # The plain ``pip install streamlit`` fails inside the isaaclab_arena + # container because streamlit≥1.30 needs uvicorn>=0.30 but Kit ships + # a bundled uvicorn==0.29 under a read-only /isaac-sim/extscache path. + # ``--user --ignore-installed`` sidesteps the rollback by writing + # everything to ~/.local (which is earlier on sys.path than extscache). + raise SystemExit( + "Streamlit is not installed. Inside the isaaclab_arena container run:\n" + " python -m pip install --user --ignore-installed streamlit streamlit-ace" + ) from exc + except KeyboardInterrupt: + # Normal exit path — user hit Ctrl-C in the terminal. + pass + + +# --------------------------------------------------------------------------- +# Public API consumed by review_app.py +# --------------------------------------------------------------------------- + + +def render_html_for_spec(spec: ArenaEnvInitialGraphSpec) -> str: + """Render the review HTML for ``spec`` with placeholder node thumbnails. + + Thin public alias of :func:`_render_html` so external entry points don't + have to reach into a private name. + """ + return _render_html(spec) + + +# --------------------------------------------------------------------------- +# Top-level HTML +# --------------------------------------------------------------------------- + + +def _render_html(spec: ArenaEnvInitialGraphSpec) -> str: + initial_state = spec.initial_state_spec + return f""" + + + +{html_lib.escape(spec.env_name)} — graph review + + + + +
+

{html_lib.escape(spec.env_name)}

+

{len(spec.nodes)} nodes · {len(spec.tasks)} tasks · initial state: {html_lib.escape(initial_state.id)}

+
+
+
+

Spatial graph (initial state: {html_lib.escape(initial_state.id)})

+
{_render_mermaid(spec, initial_state)}
+ {_render_unary_constraints(initial_state)} +
+
+

Tasks

+ {_render_tasks_table(spec)} +
+
+

Nodes

+
{_render_node_cards(spec)}
+
+
+ + + +""" + + +# --------------------------------------------------------------------------- +# Mermaid graph rendering +# --------------------------------------------------------------------------- + + +def _render_mermaid(spec: ArenaEnvInitialGraphSpec, state: ArenaEnvGraphStateSpec) -> str: + """Emit a left-to-right mermaid graph of spatial and task constraints. + + Binary spatial constraints (reference is set) are drawn as solid edges: + subject -->|kind| reference + + Unary spatial constraints (no reference) are omitted from the graph and + listed below it by :func:`_render_unary_constraints` so their params are + visible. + + Task constraints with a child are drawn as dashed edges: + parent -.->|type| child + + object_reference nodes are drawn with a dotted edge to their parent node: + ref_node -. ref .-> parent_node + """ + lines = ["graph LR"] + + anchor_ids: set[str] = set() + edge_nodes: set[str] = set() + + # --- Spatial constraints (binary only) --- + for c in state.spatial_constraints: + kind = c.kind + if kind == "is_anchor": + anchor_ids.add(c.subject) + if c.reference is not None: + lines.append( + f" {_mermaid_id(c.subject)}[{_mermaid_label(c.subject)}]" + f" -->|{kind}| " + f"{_mermaid_id(c.reference)}[{_mermaid_label(c.reference)}]" + ) + edge_nodes.add(c.subject) + edge_nodes.add(c.reference) + + # --- Task constraints (dashed edges, binary only) --- + for tc in state.task_constraints: + if tc.child is not None: + lines.append( + f" {_mermaid_id(tc.parent)}[{_mermaid_label(tc.parent)}]" + f" -.->|{_mermaid_label(tc.type.value)}| " + f"{_mermaid_id(tc.child)}[{_mermaid_label(tc.child)}]" + ) + edge_nodes.add(tc.parent) + edge_nodes.add(tc.child) + + # Include every node from the spec so disconnected ones still appear. + for node in spec.nodes: + if node.id not in edge_nodes: + lines.append(f" {_mermaid_id(node.id)}[{_mermaid_label(node.id)}]") + + # --- object_reference → parent edges (dotted, structural) --- + # Use bare node IDs (no label re-declaration) — all nodes are already + # declared above either in constraint edges or in the disconnected-node block. + nodes_by_id = spec.nodes_by_id + for node in spec.nodes: + if node.type.value == "object_reference" and node.parent is not None: + if node.parent in nodes_by_id: + lines.append(f" {_mermaid_id(node.id)} -.->|ref| {_mermaid_id(node.parent)}") + + # Anchor highlight. + for anchor_id in anchor_ids: + lines.append(f" style {_mermaid_id(anchor_id)} fill:#3a7d44,color:#fff,stroke:#7fd17f,stroke-width:2px") + + # Color nodes by type for quick visual scanning. + type_palette = { + "background": ("#3a4f7a", "#7aa0d8"), + "embodiment": ("#7a3a3a", "#d87a7a"), + "object": ("#7a6b3a", "#d8c47a"), + "object_reference": ("#6b3a7a", "#c47ad8"), + "lighting": ("#3a7a7a", "#7ad8d8"), + } + for node in spec.nodes: + if node.id in anchor_ids: + continue # anchor style wins + fill, stroke = type_palette.get(node.type.value, ("#3a3d44", "#888")) + lines.append(f" style {_mermaid_id(node.id)} fill:{fill},color:#fff,stroke:{stroke}") + + return "\n".join(lines) + + +_MERMAID_ID_SAFE = re.compile(r"[^A-Za-z0-9_]") + + +def _mermaid_id(s: str) -> str: + """Mermaid node identifiers must be alphanumeric / underscore.""" + return _MERMAID_ID_SAFE.sub("_", s) + + +def _mermaid_label(s: str) -> str: + """Escape mermaid-significant characters inside node labels.""" + return s.replace('"', """).replace("|", "|") + + +def _render_unary_constraints(state: ArenaEnvGraphStateSpec) -> str: + """List constraints without a reference below the graph (anchors, position_limits, ...).""" + rows = [] + for c in state.spatial_constraints: + if c.reference is not None: + continue + params = ( + f' {html_lib.escape(yaml.safe_dump(c.params, default_flow_style=True).rstrip())}' + if c.params + else "" + ) + rows.append( + f'
  • {html_lib.escape(c.kind)}' + f" on {html_lib.escape(c.subject)}{params}
  • " + ) + if not rows: + return "" + return ( + f'
    Unary constraints ({len(rows)})' + f'
      {"".join(rows)}
    ' + ) + + +# --------------------------------------------------------------------------- +# Tasks panel +# --------------------------------------------------------------------------- + + +def _render_tasks_table(spec: ArenaEnvInitialGraphSpec) -> str: + if not spec.tasks: + return "

    No tasks defined.

    " + rows = [] + for i, t in enumerate(spec.tasks): + params_str = yaml.safe_dump(t.params, sort_keys=False).rstrip() if t.params else "(empty)" + desc = html_lib.escape(t.description or "") + rows.append( + "" + f"{i}" + f'{html_lib.escape(t.kind)}' + f"{desc}" + f"
    {html_lib.escape(params_str)}
    " + "" + ) + return ( + "" + "" + f"{''.join(rows)}" + "
    #kinddescriptionparams
    " + ) + + +# --------------------------------------------------------------------------- +# Node cards +# --------------------------------------------------------------------------- + + +def _render_node_cards(spec: ArenaEnvInitialGraphSpec) -> str: + return "\n".join(_render_one_node_card(node) for node in spec.nodes) + + +def _render_one_node_card(node: ArenaEnvGraphNodeSpec) -> str: + node_dict = node.model_dump(mode="json", exclude_none=True) + node_yaml = yaml.safe_dump(node_dict, sort_keys=False).rstrip() + thumb = _render_node_thumbnail(node) + return f"""
    + {thumb} +
    +
    {html_lib.escape(node.id)}
    + {html_lib.escape(node.type.value)} +
    +
    {html_lib.escape(node_yaml)}
    +
    """ + + +def _render_node_thumbnail(node: ArenaEnvGraphNodeSpec) -> str: + """Per-node placeholder thumbnail — two-letter initial.""" + initial = (node.name[:2] if node.name else "?").upper() + return f"""
    + {html_lib.escape(initial)} + {html_lib.escape(node.name)} +
    """ + + +# --------------------------------------------------------------------------- +# Styling +# --------------------------------------------------------------------------- + +_CSS = """ +:root { + --bg: #15181d; + --bg-elev: #1d2128; + --bg-elev2: #262b34; + --border: #2f343d; + --fg: #e4e6eb; + --fg-muted: #8a9099; + --accent: #7fd17f; +} +* { box-sizing: border-box; } +body { margin: 0; padding: 24px; font: 14px/1.5 -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: var(--bg); color: var(--fg); } +header { margin-bottom: 16px; } +header h1 { margin: 0; font-size: 28px; font-weight: 700; } +header .sub { margin: 4px 0 0; color: var(--fg-muted); font-size: 13px; } +main { display: grid; grid-template-columns: 1fr 1fr; grid-template-rows: auto auto; + grid-template-areas: "graph nodes" "tasks nodes"; gap: 16px; } +.graph-panel { grid-area: graph; } +.tasks-panel { grid-area: tasks; } +.nodes-panel { grid-area: nodes; } +.panel { background: var(--bg-elev); border: 1px solid var(--border); border-radius: 8px; padding: 16px; } +.panel h2 { margin: 0 0 12px; font-size: 16px; font-weight: 600; letter-spacing: 0.02em; } +.panel h2 .muted { color: var(--fg-muted); font-weight: 400; font-size: 13px; } +code { font-family: ui-monospace, 'SF Mono', Menlo, monospace; font-size: 12px; + background: var(--bg-elev2); padding: 1px 6px; border-radius: 4px; } +pre { font-family: ui-monospace, 'SF Mono', Menlo, monospace; font-size: 12px; + background: var(--bg-elev2); padding: 10px 12px; border-radius: 6px; margin: 0; + white-space: pre-wrap; word-break: break-word; } +.muted { color: var(--fg-muted); } +.badge { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 11px; + font-weight: 600; letter-spacing: 0.03em; background: var(--bg-elev2); color: var(--fg); } +.badge.type-background { background: #3a4f7a; } +.badge.type-embodiment { background: #7a3a3a; } +.badge.type-object { background: #7a6b3a; } +.badge.type-object_reference { background: #6b3a7a; } +.badge.type-lighting { background: #3a7a7a; } +.badge.type-is_anchor { background: #3a7d44; } +.badge.type-position_limits, .badge.type-at_pose, .badge.type-at_position { background: #6b3a7a; } +.badge.type-task { background: #2f343d; border: 1px solid #4a5; color: var(--accent); } +.mermaid { background: var(--bg-elev2); padding: 8px; border-radius: 6px; min-height: 220px; + display: flex; align-items: center; justify-content: center; } +.unary { margin-top: 12px; } +.unary summary { cursor: pointer; color: var(--fg-muted); font-size: 13px; padding: 4px 0; } +.unary ul { margin: 8px 0 0; padding-left: 20px; list-style: disc; color: var(--fg); } +.unary li { padding: 3px 0; } +table.tasks { width: 100%; border-collapse: collapse; } +table.tasks th, table.tasks td { text-align: left; padding: 8px 10px; border-bottom: 1px solid var(--border); + vertical-align: top; font-size: 12px; } +table.tasks th { color: var(--fg-muted); font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; } +table.tasks pre { padding: 6px 8px; font-size: 11px; } +.node-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 12px; } +.node-card { background: var(--bg-elev2); border: 1px solid var(--border); border-radius: 8px; + padding: 12px; display: flex; flex-direction: column; gap: 10px; } +.node-card .thumb { aspect-ratio: 1 / 1; background: linear-gradient(135deg, #2a2f37, #1c2026); + border-radius: 6px; display: flex; flex-direction: column; + align-items: center; justify-content: center; color: var(--fg-muted); + position: relative; overflow: hidden; } +.node-card .thumb-rendered { background: #0e1115; } +.node-card .thumb-rendered img { width: 100%; height: 100%; object-fit: contain; display: block; } +.node-card .thumb-rendered .thumb-name { position: absolute; bottom: 0; left: 0; right: 0; + padding: 4px 6px; background: rgba(15, 17, 21, 0.78); + color: var(--fg); margin: 0; } +.thumb-initial { font-size: 36px; font-weight: 700; color: var(--fg); opacity: 0.6; + font-family: ui-monospace, monospace; } +.thumb-name { font-size: 10px; margin-top: 6px; padding: 0 8px; text-align: center; word-break: break-word; } +.node-meta { display: flex; align-items: center; justify-content: space-between; gap: 8px; } +.node-id { font-family: ui-monospace, monospace; font-size: 13px; font-weight: 600; word-break: break-all; } +.node-yaml { font-size: 11px; } +""" + + +if __name__ == "__main__": + main() diff --git a/isaaclab_arena/environments/arena_env_graph_spec.py b/isaaclab_arena/environments/arena_env_graph_spec.py index 0dc3d6db6..c11c09656 100644 --- a/isaaclab_arena/environments/arena_env_graph_spec.py +++ b/isaaclab_arena/environments/arena_env_graph_spec.py @@ -83,6 +83,18 @@ def write_yaml(self, path: str | Path) -> None: with Path(path).open("w", encoding="utf-8") as f: yaml.safe_dump(self.to_dict(), f, sort_keys=False) + def to_yaml(self, path: str | Path) -> Path: + """Write this spec to ``path`` as YAML. Creates parent dirs as needed. + + Returns the resolved :class:`Path` written. Symmetric with + :meth:`from_yaml`. + """ + out_path = Path(path) + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(self.to_dict(), f, sort_keys=False) + return out_path + @property def nodes_by_id(self) -> dict[str, ArenaEnvGraphNodeSpec]: return {node.id: node for node in self.nodes} diff --git a/setup.py b/setup.py index 582f669ec..4f86c948b 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,11 @@ "tenacity", ] +ENV_REVIEW_DEPS = [ + "streamlit>=1.30", + "streamlit-ace>=0.1.1", +] + setup( name="isaaclab_arena", version=ISAACLAB_ARENA_VERSION_NUMBER, @@ -46,6 +51,7 @@ install_requires=RUNTIME_DEPS, extras_require={ "dev": DEV_DEPS, + "env-review": ENV_REVIEW_DEPS, }, zip_safe=False, ) From 080749130dc5e8093db56c80da19d99571449fd8 Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 01:34:43 -0700 Subject: [PATCH 02/19] fix copyright years --- isaaclab_arena/agentic_environment_generation/review_app.py | 2 +- isaaclab_arena/agentic_environment_generation/review_graph.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/isaaclab_arena/agentic_environment_generation/review_app.py b/isaaclab_arena/agentic_environment_generation/review_app.py index 1a156190a..1e51132d7 100644 --- a/isaaclab_arena/agentic_environment_generation/review_app.py +++ b/isaaclab_arena/agentic_environment_generation/review_app.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). # All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/isaaclab_arena/agentic_environment_generation/review_graph.py b/isaaclab_arena/agentic_environment_generation/review_graph.py index 0bf2ecbda..edee396ac 100644 --- a/isaaclab_arena/agentic_environment_generation/review_graph.py +++ b/isaaclab_arena/agentic_environment_generation/review_graph.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). # All rights reserved. # # SPDX-License-Identifier: Apache-2.0 From 07c9d7f737f5431c5ef504fa5e50d8fabeaebff8 Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 01:35:22 -0700 Subject: [PATCH 03/19] Move folder --- .../agentic_environment_generation/review_app.py | 0 .../agentic_environment_generation/review_graph.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {isaaclab_arena => isaaclab_arena_examples}/agentic_environment_generation/review_app.py (100%) rename {isaaclab_arena => isaaclab_arena_examples}/agentic_environment_generation/review_graph.py (100%) diff --git a/isaaclab_arena/agentic_environment_generation/review_app.py b/isaaclab_arena_examples/agentic_environment_generation/review_app.py similarity index 100% rename from isaaclab_arena/agentic_environment_generation/review_app.py rename to isaaclab_arena_examples/agentic_environment_generation/review_app.py diff --git a/isaaclab_arena/agentic_environment_generation/review_graph.py b/isaaclab_arena_examples/agentic_environment_generation/review_graph.py similarity index 100% rename from isaaclab_arena/agentic_environment_generation/review_graph.py rename to isaaclab_arena_examples/agentic_environment_generation/review_graph.py From f84bb1c3882c16a0b3b639f15c7a58c9c19437db Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 01:37:28 -0700 Subject: [PATCH 04/19] Simplify setup --- setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.py b/setup.py index 4f86c948b..c411f629b 100644 --- a/setup.py +++ b/setup.py @@ -26,9 +26,6 @@ "jupyter", "debugpy", "tenacity", -] - -ENV_REVIEW_DEPS = [ "streamlit>=1.30", "streamlit-ace>=0.1.1", ] @@ -51,7 +48,6 @@ install_requires=RUNTIME_DEPS, extras_require={ "dev": DEV_DEPS, - "env-review": ENV_REVIEW_DEPS, }, zip_safe=False, ) From 19ab3dfd6cb28cbab106d59233e5e6f8ec3c0dcf Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 02:32:54 -0700 Subject: [PATCH 05/19] Refactor graph review tool into review_gui package. Replace monolithic review_graph.py and review_app.py with server, streamlit_ui, and a render subpackage; node thumbnails remain placeholders until SimApp lands. Signed-off-by: Qian Lin --- .../review_graph.py | 444 ------------------ .../review_gui/__init__.py | 6 + .../review_gui/render/__init__.py | 6 + .../review_gui/render/html_document.py | 54 +++ .../review_gui/render/mermaid_graph.py | 96 ++++ .../review_gui/render/panels.py | 80 ++++ .../review_gui/render/styles.py | 75 +++ .../review_gui/render/thumbnails.py | 20 + .../review_gui/server.py | 85 ++++ .../streamlit_ui.py} | 151 ++---- 10 files changed, 466 insertions(+), 551 deletions(-) delete mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_graph.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/__init__.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/render/__init__.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/render/html_document.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/render/mermaid_graph.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/render/styles.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/render/thumbnails.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/server.py rename isaaclab_arena_examples/agentic_environment_generation/{review_app.py => review_gui/streamlit_ui.py} (53%) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_graph.py b/isaaclab_arena_examples/agentic_environment_generation/review_graph.py deleted file mode 100644 index edee396ac..000000000 --- a/isaaclab_arena_examples/agentic_environment_generation/review_graph.py +++ /dev/null @@ -1,444 +0,0 @@ -# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: Apache-2.0 - -"""ArenaEnvInitialGraphSpec review tool — Streamlit live editor. - -The CLI is a thin launcher: it boots the Streamlit app in ``review_app.py``. - -The tool accepts an ``ArenaEnvInitialGraphSpec`` YAML as input. The user -edits the spec directly inside the Streamlit editor and the preview updates -in real time. - -Three panels (dark dashboard style) inside the embedded view: - * Top-left — graph diagram (mermaid.js, CDN-loaded) of the initial-state - spatial constraints. Anchor nodes are highlighted; constraints without - a reference (is_anchor / position_limits / at_pose / ...) are listed below - the graph rather than rendered as self-loops. - * Bottom-left — task table (index, kind, description, params). - * Right — node card grid: type badge, asset name, and the per-node YAML - stanza. - -Usage: - /isaac-sim/python.sh -m isaaclab_arena.agentic_environment_generation.review_graph \\ - --yaml isaaclab_arena/tests/test_data/pick_and_place_maple_table_init_env_graph.yaml - - # Custom port: - /isaac-sim/python.sh -m isaaclab_arena.agentic_environment_generation.review_graph \\ - --yaml --port 8600 - -Public API used by ``review_app.py``: - * :func:`render_html_for_spec` — full HTML payload for a spec. -""" - -from __future__ import annotations - -import argparse -import html as html_lib -import os -import re -import subprocess -import sys -import yaml -from pathlib import Path - -from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec -from isaaclab_arena.environments.arena_env_graph_types import ArenaEnvGraphNodeSpec, ArenaEnvGraphStateSpec - - -def main() -> None: - """CLI entry point — argparse parses the user's flags, then we hand off - to Streamlit. The actual interactive UI lives in ``review_app.py``. - """ - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument( - "--yaml", - type=Path, - required=True, - help="Path to an ArenaEnvInitialGraphSpec YAML file. The Streamlit app will open it for live editing.", - ) - parser.add_argument( - "--port", - type=int, - default=8501, - help="Streamlit server port (default: 8501).", - ) - args = parser.parse_args() - _serve_live_editor(args.yaml, port=args.port) - - -def _serve_live_editor(yaml_path: Path, port: int = 8501) -> None: - """Spawn ``streamlit run review_app.py -- --yaml `` and wait. - - We resolve ``review_app.py`` next to this file rather than going through - ``-m`` so Streamlit picks the path up cleanly (``streamlit run`` doesn't - accept module dotted-paths). - """ - app_path = Path(__file__).with_name("review_app.py") - if not app_path.exists(): - raise FileNotFoundError(f"Streamlit app not found at {app_path} — installation is incomplete.") - - cmd = [ - sys.executable, - "-m", - "streamlit", - "run", - str(app_path), - "--server.port", - str(port), - # Skip the email prompt the first time Streamlit runs in a fresh - # container — the live editor is a developer tool, not a hosted - # service, and an interactive prompt would block automation. - "--browser.gatherUsageStats", - "false", - # File watcher is a footgun here: Kit's ``SimulationApp`` boot is - # tens of seconds; we don't want Streamlit silently rerunning the - # script (and reissuing the cached_resource init) every time we - # save a source file during development. The user can still hit "R" - # in the browser to force a rerun if they want. - "--server.fileWatcherType", - "none", - "--", - "--yaml", - str(yaml_path.resolve()), - ] - - # Inherit env so the Streamlit subprocess sees PYTHONPATH / isaac-sim - # site-packages exactly the same way we do. - print(f"[review_graph] launching Streamlit live editor: {' '.join(cmd)}", file=sys.stderr) - try: - subprocess.run(cmd, env=os.environ.copy(), check=True) - except FileNotFoundError as exc: - # The plain ``pip install streamlit`` fails inside the isaaclab_arena - # container because streamlit≥1.30 needs uvicorn>=0.30 but Kit ships - # a bundled uvicorn==0.29 under a read-only /isaac-sim/extscache path. - # ``--user --ignore-installed`` sidesteps the rollback by writing - # everything to ~/.local (which is earlier on sys.path than extscache). - raise SystemExit( - "Streamlit is not installed. Inside the isaaclab_arena container run:\n" - " python -m pip install --user --ignore-installed streamlit streamlit-ace" - ) from exc - except KeyboardInterrupt: - # Normal exit path — user hit Ctrl-C in the terminal. - pass - - -# --------------------------------------------------------------------------- -# Public API consumed by review_app.py -# --------------------------------------------------------------------------- - - -def render_html_for_spec(spec: ArenaEnvInitialGraphSpec) -> str: - """Render the review HTML for ``spec`` with placeholder node thumbnails. - - Thin public alias of :func:`_render_html` so external entry points don't - have to reach into a private name. - """ - return _render_html(spec) - - -# --------------------------------------------------------------------------- -# Top-level HTML -# --------------------------------------------------------------------------- - - -def _render_html(spec: ArenaEnvInitialGraphSpec) -> str: - initial_state = spec.initial_state_spec - return f""" - - - -{html_lib.escape(spec.env_name)} — graph review - - - - -
    -

    {html_lib.escape(spec.env_name)}

    -

    {len(spec.nodes)} nodes · {len(spec.tasks)} tasks · initial state: {html_lib.escape(initial_state.id)}

    -
    -
    -
    -

    Spatial graph (initial state: {html_lib.escape(initial_state.id)})

    -
    {_render_mermaid(spec, initial_state)}
    - {_render_unary_constraints(initial_state)} -
    -
    -

    Tasks

    - {_render_tasks_table(spec)} -
    -
    -

    Nodes

    -
    {_render_node_cards(spec)}
    -
    -
    - - - -""" - - -# --------------------------------------------------------------------------- -# Mermaid graph rendering -# --------------------------------------------------------------------------- - - -def _render_mermaid(spec: ArenaEnvInitialGraphSpec, state: ArenaEnvGraphStateSpec) -> str: - """Emit a left-to-right mermaid graph of spatial and task constraints. - - Binary spatial constraints (reference is set) are drawn as solid edges: - subject -->|kind| reference - - Unary spatial constraints (no reference) are omitted from the graph and - listed below it by :func:`_render_unary_constraints` so their params are - visible. - - Task constraints with a child are drawn as dashed edges: - parent -.->|type| child - - object_reference nodes are drawn with a dotted edge to their parent node: - ref_node -. ref .-> parent_node - """ - lines = ["graph LR"] - - anchor_ids: set[str] = set() - edge_nodes: set[str] = set() - - # --- Spatial constraints (binary only) --- - for c in state.spatial_constraints: - kind = c.kind - if kind == "is_anchor": - anchor_ids.add(c.subject) - if c.reference is not None: - lines.append( - f" {_mermaid_id(c.subject)}[{_mermaid_label(c.subject)}]" - f" -->|{kind}| " - f"{_mermaid_id(c.reference)}[{_mermaid_label(c.reference)}]" - ) - edge_nodes.add(c.subject) - edge_nodes.add(c.reference) - - # --- Task constraints (dashed edges, binary only) --- - for tc in state.task_constraints: - if tc.child is not None: - lines.append( - f" {_mermaid_id(tc.parent)}[{_mermaid_label(tc.parent)}]" - f" -.->|{_mermaid_label(tc.type.value)}| " - f"{_mermaid_id(tc.child)}[{_mermaid_label(tc.child)}]" - ) - edge_nodes.add(tc.parent) - edge_nodes.add(tc.child) - - # Include every node from the spec so disconnected ones still appear. - for node in spec.nodes: - if node.id not in edge_nodes: - lines.append(f" {_mermaid_id(node.id)}[{_mermaid_label(node.id)}]") - - # --- object_reference → parent edges (dotted, structural) --- - # Use bare node IDs (no label re-declaration) — all nodes are already - # declared above either in constraint edges or in the disconnected-node block. - nodes_by_id = spec.nodes_by_id - for node in spec.nodes: - if node.type.value == "object_reference" and node.parent is not None: - if node.parent in nodes_by_id: - lines.append(f" {_mermaid_id(node.id)} -.->|ref| {_mermaid_id(node.parent)}") - - # Anchor highlight. - for anchor_id in anchor_ids: - lines.append(f" style {_mermaid_id(anchor_id)} fill:#3a7d44,color:#fff,stroke:#7fd17f,stroke-width:2px") - - # Color nodes by type for quick visual scanning. - type_palette = { - "background": ("#3a4f7a", "#7aa0d8"), - "embodiment": ("#7a3a3a", "#d87a7a"), - "object": ("#7a6b3a", "#d8c47a"), - "object_reference": ("#6b3a7a", "#c47ad8"), - "lighting": ("#3a7a7a", "#7ad8d8"), - } - for node in spec.nodes: - if node.id in anchor_ids: - continue # anchor style wins - fill, stroke = type_palette.get(node.type.value, ("#3a3d44", "#888")) - lines.append(f" style {_mermaid_id(node.id)} fill:{fill},color:#fff,stroke:{stroke}") - - return "\n".join(lines) - - -_MERMAID_ID_SAFE = re.compile(r"[^A-Za-z0-9_]") - - -def _mermaid_id(s: str) -> str: - """Mermaid node identifiers must be alphanumeric / underscore.""" - return _MERMAID_ID_SAFE.sub("_", s) - - -def _mermaid_label(s: str) -> str: - """Escape mermaid-significant characters inside node labels.""" - return s.replace('"', """).replace("|", "|") - - -def _render_unary_constraints(state: ArenaEnvGraphStateSpec) -> str: - """List constraints without a reference below the graph (anchors, position_limits, ...).""" - rows = [] - for c in state.spatial_constraints: - if c.reference is not None: - continue - params = ( - f' {html_lib.escape(yaml.safe_dump(c.params, default_flow_style=True).rstrip())}' - if c.params - else "" - ) - rows.append( - f'
  • {html_lib.escape(c.kind)}' - f" on {html_lib.escape(c.subject)}{params}
  • " - ) - if not rows: - return "" - return ( - f'
    Unary constraints ({len(rows)})' - f'
      {"".join(rows)}
    ' - ) - - -# --------------------------------------------------------------------------- -# Tasks panel -# --------------------------------------------------------------------------- - - -def _render_tasks_table(spec: ArenaEnvInitialGraphSpec) -> str: - if not spec.tasks: - return "

    No tasks defined.

    " - rows = [] - for i, t in enumerate(spec.tasks): - params_str = yaml.safe_dump(t.params, sort_keys=False).rstrip() if t.params else "(empty)" - desc = html_lib.escape(t.description or "") - rows.append( - "" - f"{i}" - f'{html_lib.escape(t.kind)}' - f"{desc}" - f"
    {html_lib.escape(params_str)}
    " - "" - ) - return ( - "" - "" - f"{''.join(rows)}" - "
    #kinddescriptionparams
    " - ) - - -# --------------------------------------------------------------------------- -# Node cards -# --------------------------------------------------------------------------- - - -def _render_node_cards(spec: ArenaEnvInitialGraphSpec) -> str: - return "\n".join(_render_one_node_card(node) for node in spec.nodes) - - -def _render_one_node_card(node: ArenaEnvGraphNodeSpec) -> str: - node_dict = node.model_dump(mode="json", exclude_none=True) - node_yaml = yaml.safe_dump(node_dict, sort_keys=False).rstrip() - thumb = _render_node_thumbnail(node) - return f"""
    - {thumb} -
    -
    {html_lib.escape(node.id)}
    - {html_lib.escape(node.type.value)} -
    -
    {html_lib.escape(node_yaml)}
    -
    """ - - -def _render_node_thumbnail(node: ArenaEnvGraphNodeSpec) -> str: - """Per-node placeholder thumbnail — two-letter initial.""" - initial = (node.name[:2] if node.name else "?").upper() - return f"""
    - {html_lib.escape(initial)} - {html_lib.escape(node.name)} -
    """ - - -# --------------------------------------------------------------------------- -# Styling -# --------------------------------------------------------------------------- - -_CSS = """ -:root { - --bg: #15181d; - --bg-elev: #1d2128; - --bg-elev2: #262b34; - --border: #2f343d; - --fg: #e4e6eb; - --fg-muted: #8a9099; - --accent: #7fd17f; -} -* { box-sizing: border-box; } -body { margin: 0; padding: 24px; font: 14px/1.5 -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; - background: var(--bg); color: var(--fg); } -header { margin-bottom: 16px; } -header h1 { margin: 0; font-size: 28px; font-weight: 700; } -header .sub { margin: 4px 0 0; color: var(--fg-muted); font-size: 13px; } -main { display: grid; grid-template-columns: 1fr 1fr; grid-template-rows: auto auto; - grid-template-areas: "graph nodes" "tasks nodes"; gap: 16px; } -.graph-panel { grid-area: graph; } -.tasks-panel { grid-area: tasks; } -.nodes-panel { grid-area: nodes; } -.panel { background: var(--bg-elev); border: 1px solid var(--border); border-radius: 8px; padding: 16px; } -.panel h2 { margin: 0 0 12px; font-size: 16px; font-weight: 600; letter-spacing: 0.02em; } -.panel h2 .muted { color: var(--fg-muted); font-weight: 400; font-size: 13px; } -code { font-family: ui-monospace, 'SF Mono', Menlo, monospace; font-size: 12px; - background: var(--bg-elev2); padding: 1px 6px; border-radius: 4px; } -pre { font-family: ui-monospace, 'SF Mono', Menlo, monospace; font-size: 12px; - background: var(--bg-elev2); padding: 10px 12px; border-radius: 6px; margin: 0; - white-space: pre-wrap; word-break: break-word; } -.muted { color: var(--fg-muted); } -.badge { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 11px; - font-weight: 600; letter-spacing: 0.03em; background: var(--bg-elev2); color: var(--fg); } -.badge.type-background { background: #3a4f7a; } -.badge.type-embodiment { background: #7a3a3a; } -.badge.type-object { background: #7a6b3a; } -.badge.type-object_reference { background: #6b3a7a; } -.badge.type-lighting { background: #3a7a7a; } -.badge.type-is_anchor { background: #3a7d44; } -.badge.type-position_limits, .badge.type-at_pose, .badge.type-at_position { background: #6b3a7a; } -.badge.type-task { background: #2f343d; border: 1px solid #4a5; color: var(--accent); } -.mermaid { background: var(--bg-elev2); padding: 8px; border-radius: 6px; min-height: 220px; - display: flex; align-items: center; justify-content: center; } -.unary { margin-top: 12px; } -.unary summary { cursor: pointer; color: var(--fg-muted); font-size: 13px; padding: 4px 0; } -.unary ul { margin: 8px 0 0; padding-left: 20px; list-style: disc; color: var(--fg); } -.unary li { padding: 3px 0; } -table.tasks { width: 100%; border-collapse: collapse; } -table.tasks th, table.tasks td { text-align: left; padding: 8px 10px; border-bottom: 1px solid var(--border); - vertical-align: top; font-size: 12px; } -table.tasks th { color: var(--fg-muted); font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; } -table.tasks pre { padding: 6px 8px; font-size: 11px; } -.node-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 12px; } -.node-card { background: var(--bg-elev2); border: 1px solid var(--border); border-radius: 8px; - padding: 12px; display: flex; flex-direction: column; gap: 10px; } -.node-card .thumb { aspect-ratio: 1 / 1; background: linear-gradient(135deg, #2a2f37, #1c2026); - border-radius: 6px; display: flex; flex-direction: column; - align-items: center; justify-content: center; color: var(--fg-muted); - position: relative; overflow: hidden; } -.node-card .thumb-rendered { background: #0e1115; } -.node-card .thumb-rendered img { width: 100%; height: 100%; object-fit: contain; display: block; } -.node-card .thumb-rendered .thumb-name { position: absolute; bottom: 0; left: 0; right: 0; - padding: 4px 6px; background: rgba(15, 17, 21, 0.78); - color: var(--fg); margin: 0; } -.thumb-initial { font-size: 36px; font-weight: 700; color: var(--fg); opacity: 0.6; - font-family: ui-monospace, monospace; } -.thumb-name { font-size: 10px; margin-top: 6px; padding: 0 8px; text-align: center; word-break: break-word; } -.node-meta { display: flex; align-items: center; justify-content: space-between; gap: 8px; } -.node-id { font-family: ui-monospace, monospace; font-size: 13px; font-weight: 600; word-break: break-all; } -.node-yaml { font-size: 11px; } -""" - - -if __name__ == "__main__": - main() diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/__init__.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/__init__.py new file mode 100644 index 000000000..821e672b4 --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Live editor and HTML dashboard for :class:`~isaaclab_arena.environments.arena_env_graph_spec.ArenaEnvInitialGraphSpec`.""" diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/__init__.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/__init__.py new file mode 100644 index 000000000..2fd363882 --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""HTML rendering backend for the initial-graph review dashboard.""" diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/html_document.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/html_document.py new file mode 100644 index 000000000..d83c6b275 --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/html_document.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import html as html_lib + +from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec +from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.mermaid_graph import render_mermaid_graph +from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.panels import ( + render_node_cards, + render_tasks_table, + render_unary_constraints, +) +from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.styles import DASHBOARD_CSS + + +def render_html_for_spec(spec: ArenaEnvInitialGraphSpec) -> str: + """Render the self-contained review HTML dashboard for ``spec``.""" + initial_state = spec.initial_state_spec + return f""" + + + +{html_lib.escape(spec.env_name)} — graph review + + + + +
    +

    {html_lib.escape(spec.env_name)}

    +

    {len(spec.nodes)} nodes · {len(spec.tasks)} tasks · initial state: {html_lib.escape(initial_state.id)}

    +
    +
    +
    +

    Spatial graph (initial state: {html_lib.escape(initial_state.id)})

    +
    {render_mermaid_graph(spec, initial_state)}
    + {render_unary_constraints(initial_state)} +
    +
    +

    Tasks

    + {render_tasks_table(spec)} +
    +
    +

    Nodes

    +
    {render_node_cards(spec)}
    +
    +
    + + + +""" diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/mermaid_graph.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/mermaid_graph.py new file mode 100644 index 000000000..21b57ce48 --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/mermaid_graph.py @@ -0,0 +1,96 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import re + +from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec +from isaaclab_arena.environments.arena_env_graph_types import ArenaEnvGraphStateSpec + +_MERMAID_ID_SAFE = re.compile(r"[^A-Za-z0-9_]") + + +def render_mermaid_graph(spec: ArenaEnvInitialGraphSpec, state: ArenaEnvGraphStateSpec) -> str: + """Emit a left-to-right mermaid graph of spatial and task constraints. + + Binary spatial constraints (reference is set) are drawn as solid edges: + subject -->|kind| reference + + Unary spatial constraints (no reference) are omitted from the graph and + listed below it by :func:`render_unary_constraints` so their params are + visible. + + Task constraints with a child are drawn as dashed edges: + parent -.->|type| child + + object_reference nodes are drawn with a dotted edge to their parent node: + ref_node -. ref .-> parent_node + """ + lines = ["graph LR"] + + anchor_ids: set[str] = set() + edge_nodes: set[str] = set() + + for constraint in state.spatial_constraints: + kind = constraint.kind + if kind == "is_anchor": + anchor_ids.add(constraint.subject) + if constraint.reference is not None: + lines.append( + f" {_mermaid_id(constraint.subject)}[{_mermaid_label(constraint.subject)}]" + f" -->|{kind}| " + f"{_mermaid_id(constraint.reference)}[{_mermaid_label(constraint.reference)}]" + ) + edge_nodes.add(constraint.subject) + edge_nodes.add(constraint.reference) + + for task_constraint in state.task_constraints: + if task_constraint.child is not None: + lines.append( + f" {_mermaid_id(task_constraint.parent)}[{_mermaid_label(task_constraint.parent)}]" + f" -.->|{_mermaid_label(task_constraint.type.value)}| " + f"{_mermaid_id(task_constraint.child)}[{_mermaid_label(task_constraint.child)}]" + ) + edge_nodes.add(task_constraint.parent) + edge_nodes.add(task_constraint.child) + + for node in spec.nodes: + if node.id not in edge_nodes: + lines.append(f" {_mermaid_id(node.id)}[{_mermaid_label(node.id)}]") + + nodes_by_id = spec.nodes_by_id + for node in spec.nodes: + if node.type.value == "object_reference" and node.parent is not None: + if node.parent in nodes_by_id: + lines.append(f" {_mermaid_id(node.id)} -.->|ref| {_mermaid_id(node.parent)}") + + for anchor_id in anchor_ids: + lines.append(f" style {_mermaid_id(anchor_id)} fill:#3a7d44,color:#fff,stroke:#7fd17f,stroke-width:2px") + + type_palette = { + "background": ("#3a4f7a", "#7aa0d8"), + "embodiment": ("#7a3a3a", "#d87a7a"), + "object": ("#7a6b3a", "#d8c47a"), + "object_reference": ("#6b3a7a", "#c47ad8"), + "lighting": ("#3a7a7a", "#7ad8d8"), + } + for node in spec.nodes: + if node.id in anchor_ids: + continue + fill, stroke = type_palette.get(node.type.value, ("#3a3d44", "#888")) + lines.append(f" style {_mermaid_id(node.id)} fill:{fill},color:#fff,stroke:{stroke}") + + return "\n".join(lines) + + +def _mermaid_id(value: str) -> str: + """Mermaid node identifiers must be alphanumeric / underscore.""" + return _MERMAID_ID_SAFE.sub("_", value) + + +def _mermaid_label(value: str) -> str: + """Escape mermaid-significant characters inside node labels.""" + return value.replace('"', """).replace("|", "|") diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py new file mode 100644 index 000000000..901b4c331 --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py @@ -0,0 +1,80 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import html as html_lib +import yaml + +from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec +from isaaclab_arena.environments.arena_env_graph_types import ArenaEnvGraphNodeSpec, ArenaEnvGraphStateSpec +from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.thumbnails import ( + render_placeholder_thumbnail, +) + + +def render_unary_constraints(state: ArenaEnvGraphStateSpec) -> str: + """List constraints without a reference below the graph (anchors, position_limits, ...).""" + rows = [] + for constraint in state.spatial_constraints: + if constraint.reference is not None: + continue + params = ( + " {html_lib.escape(yaml.safe_dump(constraint.params, default_flow_style=True).rstrip())}' + if constraint.params + else "" + ) + rows.append( + f'
  • {html_lib.escape(constraint.kind)}' + f" on {html_lib.escape(constraint.subject)}{params}
  • " + ) + if not rows: + return "" + return ( + f'
    Unary constraints ({len(rows)})' + f'
      {"".join(rows)}
    ' + ) + + +def render_tasks_table(spec: ArenaEnvInitialGraphSpec) -> str: + if not spec.tasks: + return "

    No tasks defined.

    " + rows = [] + for index, task in enumerate(spec.tasks): + params_str = yaml.safe_dump(task.params, sort_keys=False).rstrip() if task.params else "(empty)" + description = html_lib.escape(task.description or "") + rows.append( + "" + f"{index}" + f'{html_lib.escape(task.kind)}' + f"{description}" + f"
    {html_lib.escape(params_str)}
    " + "" + ) + return ( + "" + "" + f"{''.join(rows)}" + "
    #kinddescriptionparams
    " + ) + + +def render_node_cards(spec: ArenaEnvInitialGraphSpec) -> str: + return "\n".join(render_node_card(node) for node in spec.nodes) + + +def render_node_card(node: ArenaEnvGraphNodeSpec) -> str: + node_dict = node.model_dump(mode="json", exclude_none=True) + node_yaml = yaml.safe_dump(node_dict, sort_keys=False).rstrip() + thumb = render_placeholder_thumbnail(node) + return f"""
    + {thumb} +
    +
    {html_lib.escape(node.id)}
    + {html_lib.escape(node.type.value)} +
    +
    {html_lib.escape(node_yaml)}
    +
    """ diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/styles.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/styles.py new file mode 100644 index 000000000..279ad349d --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/styles.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +DASHBOARD_CSS = """ +:root { + --bg: #15181d; + --bg-elev: #1d2128; + --bg-elev2: #262b34; + --border: #2f343d; + --fg: #e4e6eb; + --fg-muted: #8a9099; + --accent: #7fd17f; +} +* { box-sizing: border-box; } +body { margin: 0; padding: 24px; font: 14px/1.5 -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: var(--bg); color: var(--fg); } +header { margin-bottom: 16px; } +header h1 { margin: 0; font-size: 28px; font-weight: 700; } +header .sub { margin: 4px 0 0; color: var(--fg-muted); font-size: 13px; } +main { display: grid; grid-template-columns: 1fr 1fr; grid-template-rows: auto auto; + grid-template-areas: "graph nodes" "tasks nodes"; gap: 16px; } +.graph-panel { grid-area: graph; } +.tasks-panel { grid-area: tasks; } +.nodes-panel { grid-area: nodes; } +.panel { background: var(--bg-elev); border: 1px solid var(--border); border-radius: 8px; padding: 16px; } +.panel h2 { margin: 0 0 12px; font-size: 16px; font-weight: 600; letter-spacing: 0.02em; } +.panel h2 .muted { color: var(--fg-muted); font-weight: 400; font-size: 13px; } +code { font-family: ui-monospace, 'SF Mono', Menlo, monospace; font-size: 12px; + background: var(--bg-elev2); padding: 1px 6px; border-radius: 4px; } +pre { font-family: ui-monospace, 'SF Mono', Menlo, monospace; font-size: 12px; + background: var(--bg-elev2); padding: 10px 12px; border-radius: 6px; margin: 0; + white-space: pre-wrap; word-break: break-word; } +.muted { color: var(--fg-muted); } +.badge { display: inline-block; padding: 2px 8px; border-radius: 999px; font-size: 11px; + font-weight: 600; letter-spacing: 0.03em; background: var(--bg-elev2); color: var(--fg); } +.badge.type-background { background: #3a4f7a; } +.badge.type-embodiment { background: #7a3a3a; } +.badge.type-object { background: #7a6b3a; } +.badge.type-object_reference { background: #6b3a7a; } +.badge.type-lighting { background: #3a7a7a; } +.badge.type-is_anchor { background: #3a7d44; } +.badge.type-position_limits, .badge.type-at_pose, .badge.type-at_position { background: #6b3a7a; } +.badge.type-task { background: #2f343d; border: 1px solid #4a5; color: var(--accent); } +.mermaid { background: var(--bg-elev2); padding: 8px; border-radius: 6px; min-height: 220px; + display: flex; align-items: center; justify-content: center; } +.unary { margin-top: 12px; } +.unary summary { cursor: pointer; color: var(--fg-muted); font-size: 13px; padding: 4px 0; } +.unary ul { margin: 8px 0 0; padding-left: 20px; list-style: disc; color: var(--fg); } +.unary li { padding: 3px 0; } +table.tasks { width: 100%; border-collapse: collapse; } +table.tasks th, table.tasks td { text-align: left; padding: 8px 10px; border-bottom: 1px solid var(--border); + vertical-align: top; font-size: 12px; } +table.tasks th { color: var(--fg-muted); font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; } +table.tasks pre { padding: 6px 8px; font-size: 11px; } +.node-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 12px; } +.node-card { background: var(--bg-elev2); border: 1px solid var(--border); border-radius: 8px; + padding: 12px; display: flex; flex-direction: column; gap: 10px; } +.node-card .thumb { aspect-ratio: 1 / 1; background: linear-gradient(135deg, #2a2f37, #1c2026); + border-radius: 6px; display: flex; flex-direction: column; + align-items: center; justify-content: center; color: var(--fg-muted); + position: relative; overflow: hidden; } +.node-card .thumb-rendered { background: #0e1115; } +.node-card .thumb-rendered img { width: 100%; height: 100%; object-fit: contain; display: block; } +.node-card .thumb-rendered .thumb-name { position: absolute; bottom: 0; left: 0; right: 0; + padding: 4px 6px; background: rgba(15, 17, 21, 0.78); + color: var(--fg); margin: 0; } +.thumb-initial { font-size: 36px; font-weight: 700; color: var(--fg); opacity: 0.6; + font-family: ui-monospace, monospace; } +.thumb-name { font-size: 10px; margin-top: 6px; padding: 0 8px; text-align: center; word-break: break-word; } +.node-meta { display: flex; align-items: center; justify-content: space-between; gap: 8px; } +.node-id { font-family: ui-monospace, monospace; font-size: 13px; font-weight: 600; word-break: break-all; } +.node-yaml { font-size: 11px; } +""" diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/thumbnails.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/thumbnails.py new file mode 100644 index 000000000..f2742ee8f --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/thumbnails.py @@ -0,0 +1,20 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import html as html_lib + +from isaaclab_arena.environments.arena_env_graph_types import ArenaEnvGraphNodeSpec + + +# TODO(qianl): Replace placeholder thumbnails with sim-rendered snapshots . +def render_placeholder_thumbnail(node: ArenaEnvGraphNodeSpec) -> str: + """Per-node placeholder thumbnail — two-letter initial.""" + initial = (node.name[:2] if node.name else "?").upper() + return f"""
    + {html_lib.escape(initial)} + {html_lib.escape(node.name)} +
    """ diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/server.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/server.py new file mode 100644 index 000000000..4212cf6a7 --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/server.py @@ -0,0 +1,85 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""CLI launcher for the ArenaEnvInitialGraphSpec live editor. + +Spawns Streamlit with :mod:`~isaaclab_arena_examples.agentic_environment_generation.review_gui.streamlit_ui`. + +Usage: + /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server \\ + --yaml isaaclab_arena/tests/test_data/pick_and_place_maple_table_init_env_graph.yaml + + # Custom port: + /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server \\ + --yaml --port 8600 +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--yaml", + type=Path, + required=True, + help="Path to an ArenaEnvInitialGraphSpec YAML file.", + ) + parser.add_argument( + "--port", + type=int, + default=8501, + help="Streamlit server port (default: 8501).", + ) + args = parser.parse_args() + serve_live_editor(args.yaml, port=args.port) + + +def serve_live_editor(yaml_path: Path, port: int = 8501) -> None: + """Spawn ``streamlit run streamlit_ui.py -- --yaml `` and wait.""" + app_path = Path(__file__).with_name("streamlit_ui.py") + if not app_path.exists(): + raise FileNotFoundError(f"Streamlit app not found at {app_path} — installation is incomplete.") + + cmd = [ + sys.executable, + "-m", + "streamlit", + "run", + str(app_path), + "--server.port", + str(port), + "--browser.gatherUsageStats", + "false", + "--server.fileWatcherType", + "none", + "--", + "--yaml", + str(yaml_path.resolve()), + ] + + print(f"[review_gui] launching Streamlit live editor: {' '.join(cmd)}", file=sys.stderr) + try: + subprocess.run(cmd, env=os.environ.copy(), check=True) + except FileNotFoundError as exc: + raise SystemExit( + "Streamlit is not installed. Inside the isaaclab_arena container run:\n" + " python -m pip install --user --ignore-installed streamlit streamlit-ace" + ) from exc + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_app.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py similarity index 53% rename from isaaclab_arena_examples/agentic_environment_generation/review_app.py rename to isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 1e51132d7..c234d14f3 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_app.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -3,22 +3,12 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Streamlit-backed live editor for ArenaEnvInitialGraphSpec YAMLs. - -Wraps :func:`isaaclab_arena.agentic_environment_generation.review_graph.render_html_for_spec` -in a two-pane Streamlit page so the user can edit the ``ArenaEnvInitialGraphSpec`` -YAML directly in the browser and see the visualization update automatically. - -Launch (always via the wrapper in review_graph.py — handles streamlit flags): - /isaac-sim/python.sh -m isaaclab_arena.agentic_environment_generation.review_graph \\ - --yaml path/to/spec.yaml - -Design: - * Left pane — ``streamlit-ace`` YAML editor + validation badge + Save button. - Validation runs on every rerun (i.e. after each editor blur). When the YAML - is valid and has changed since the last render, the visualization updates - automatically — no button click required. - * Right pane — sandboxed iframe with the rendered review HTML. +"""Streamlit UI for the initial-graph live editor. + +Launch via :mod:`~isaaclab_arena_examples.agentic_environment_generation.review_gui.server`: + + /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server \\ + --yaml isaaclab_arena/tests/test_data/pick_and_place_maple_table_init_env_graph.yaml """ from __future__ import annotations @@ -31,32 +21,24 @@ import streamlit as st -from isaaclab_arena.agentic_environment_generation.review_graph import render_html_for_spec from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec +from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.html_document import render_html_for_spec # Visualization iframe height. Tuned so the graph + tasks + node grid all # fit without an outer Streamlit scrollbar swallowing the inner one. _IFRAME_HEIGHT_PX = 1100 - -# --------------------------------------------------------------------------- -# Args + session-state init -# --------------------------------------------------------------------------- - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--yaml", - type=Path, - required=True, - help="Path to the ArenaEnvInitialGraphSpec YAML to open in the editor.", - ) - return parser.parse_args() +_BROKEN_PLACEHOLDER_HTML = """ +

    No visualization yet — fix the YAML errors to auto-render.

    +""" @dataclass -class _ValidationResult: +class ValidationResult: + """Outcome of parsing and validating YAML text as an initial graph spec.""" + spec: ArenaEnvInitialGraphSpec | None error: str | None @@ -65,21 +47,29 @@ def is_valid(self) -> bool: return self.spec is not None -def _validate_yaml_text(text: str) -> _ValidationResult: +def validate_yaml_text(text: str) -> ValidationResult: + """Parse ``text`` as YAML and validate it as an :class:`ArenaEnvInitialGraphSpec`.""" try: raw = yaml.safe_load(text) spec = ArenaEnvInitialGraphSpec.model_validate(raw) - return _ValidationResult(spec=spec, error=None) + return ValidationResult(spec=spec, error=None) except Exception: - return _ValidationResult(spec=None, error=traceback.format_exc()) + return ValidationResult(spec=None, error=traceback.format_exc()) + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--yaml", + type=Path, + required=True, + help="Path to the ArenaEnvInitialGraphSpec YAML to open in the editor.", + ) + return parser.parse_args() -def _initialize_state(yaml_path: Path) -> None: - """Seed ``st.session_state`` from disk exactly once per session. - We key off ``_yaml_path`` so that if the user passes a different YAML on - a Streamlit reload (rare — usually the same), we reset cleanly. - """ +def initialize_state(yaml_path: Path) -> None: + """Seed ``st.session_state`` from disk exactly once per session.""" if st.session_state.get("_yaml_path") == str(yaml_path): return @@ -88,37 +78,17 @@ def _initialize_state(yaml_path: Path) -> None: st.session_state["_yaml_path"] = str(yaml_path) st.session_state["original_text"] = original_text st.session_state["edited_text"] = original_text - # The text whose render is currently displayed. Starts == original so the - # first paint shows the on-disk file (and "Regenerate" is correctly - # disabled until the user edits something). st.session_state["last_rendered_text"] = original_text st.session_state["save_path"] = str(yaml_path) - initial = _validate_yaml_text(original_text) + initial = validate_yaml_text(original_text) if not initial.is_valid: - # Defensive: if the on-disk file is already broken we still want to - # show *something*, but we won't pre-render it. The user fixes the - # YAML in the editor, then hits Regenerate. st.session_state["rendered_html"] = _BROKEN_PLACEHOLDER_HTML else: st.session_state["rendered_html"] = render_html_for_spec(initial.spec) -# Tiny standalone HTML used when the on-disk YAML is itself invalid. -_BROKEN_PLACEHOLDER_HTML = """ -

    No visualization yet — fix the YAML errors to auto-render.

    -""" - - -# --------------------------------------------------------------------------- -# UI panels -# --------------------------------------------------------------------------- - - -def _render_validation_badge(validation: _ValidationResult) -> None: - """Show a green tick + summary, or a red cross + the raw exception text.""" +def render_validation_badge(validation: ValidationResult) -> None: if validation.is_valid: spec = validation.spec st.success( @@ -130,8 +100,7 @@ def _render_validation_badge(validation: _ValidationResult) -> None: st.error(f"Invalid YAML\n\n```\n{validation.error}\n```", icon="🛑") -def _render_save_button(validation: _ValidationResult) -> None: - """Render the Save button. Disabled while the YAML is invalid.""" +def render_save_button(validation: ValidationResult) -> None: can_save = validation.is_valid save_path_str = st.session_state["save_path"] @@ -143,7 +112,6 @@ def _render_save_button(validation: _ValidationResult) -> None: ): try: Path(save_path_str).write_text(st.session_state["edited_text"], encoding="utf-8") - # Update "original" so future comparisons are against the saved file. st.session_state["original_text"] = st.session_state["edited_text"] st.toast(f"Saved → {save_path_str}", icon="💾") except OSError as exc: @@ -160,21 +128,10 @@ def _render_save_button(validation: _ValidationResult) -> None: st.session_state["save_path"] = new_path -def _render_editor_panel(yaml_path: Path) -> _ValidationResult: - """Left pane. Returns the validation result for the current editor text. - - Returning the validation result (rather than stashing it in session_state) - keeps the data flow inside one render pass and avoids a stale-state class - of bug where the badge and the buttons disagree. - """ - # Lazy import so the module is importable from environments that don't - # have streamlit-ace installed yet (we surface a clean error message - # rather than ImportError at module load). +def render_editor_panel(yaml_path: Path) -> ValidationResult: try: from streamlit_ace import st_ace # noqa: PLC0415 except ImportError as exc: - # See review_graph._serve_live_editor for why --user --ignore-installed - # is required inside the isaaclab_arena container. st.error( "`streamlit-ace` is not installed. Inside the isaaclab_arena container run:\n" "`python -m pip install --user --ignore-installed streamlit-ace`\n\n" @@ -186,8 +143,6 @@ def _render_editor_panel(yaml_path: Path) -> _ValidationResult: st.subheader("YAML editor") st.caption(f"Source: `{yaml_path}`") - # ``auto_update=False`` commits on blur / Ctrl+Enter rather than on every - # keystroke, showing an "Apply" button in the editor toolbar. new_text = st_ace( value=st.session_state["edited_text"], language="yaml", @@ -200,20 +155,14 @@ def _render_editor_panel(yaml_path: Path) -> _ValidationResult: wrap=False, auto_update=False, min_lines=30, - # Key bound to the YAML path so swapping --yaml between sessions - # forces ace to remount with the new content. key=f"ace_editor::{yaml_path}", ) if new_text is not None: st.session_state["edited_text"] = new_text - validation = _validate_yaml_text(st.session_state["edited_text"]) - _render_validation_badge(validation) + validation = validate_yaml_text(st.session_state["edited_text"]) + render_validation_badge(validation) - # Auto-render whenever the YAML is valid and has changed since the last - # render. This runs before the right pane is drawn, so the updated HTML - # is already in session_state when the iframe is mounted — no extra rerun - # needed. edited_since_render = st.session_state["edited_text"] != st.session_state["last_rendered_text"] if validation.is_valid and edited_since_render: with st.spinner("Rendering visualization…"): @@ -221,18 +170,14 @@ def _render_editor_panel(yaml_path: Path) -> _ValidationResult: st.session_state["last_rendered_text"] = st.session_state["edited_text"] st.toast("Visualization updated.", icon="🔄") - _render_save_button(validation) + render_save_button(validation) return validation -def _render_visualization_panel() -> None: - """Right pane — iframe-mount the cached rendered HTML.""" +def render_visualization_panel() -> None: st.subheader("Visualization") st.caption("Updates automatically when the YAML is valid.") - # ``st.components.v1.html`` wraps the payload in a sandboxed iframe, which - # is what we want — the mermaid CDN script and the static CSS stay - # isolated from Streamlit's own DOM. st.components.v1.html( st.session_state["rendered_html"], height=_IFRAME_HEIGHT_PX, @@ -240,11 +185,6 @@ def _render_visualization_panel() -> None: ) -# --------------------------------------------------------------------------- -# Entry point -# --------------------------------------------------------------------------- - - def main() -> None: st.set_page_config( page_title="ArenaEnvInitialGraphSpec live editor", @@ -252,24 +192,21 @@ def main() -> None: initial_sidebar_state="collapsed", ) - args = _parse_args() + args = parse_args() yaml_path = args.yaml.resolve() if not yaml_path.exists(): st.error(f"YAML file not found: {yaml_path}", icon="🛑") st.stop() - _initialize_state(yaml_path) + initialize_state(yaml_path) st.markdown("### ArenaEnvInitialGraphSpec live editor") left, right = st.columns([2, 3], gap="large") with left: - _render_editor_panel(yaml_path) + render_editor_panel(yaml_path) with right: - _render_visualization_panel() + render_visualization_panel() -# Streamlit invokes the script top-level on every rerun, so we run main() -# unconditionally. The standard ``if __name__ == "__main__"`` guard would -# also work under ``streamlit run`` but is unnecessary — this module is only -# ever loaded as the Streamlit entrypoint. +# Streamlit invokes the script top-level on every rerun. main() From 8d5ecabffca185d38e66f3adac68764afb12f51d Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 03:34:24 -0700 Subject: [PATCH 06/19] Rename review_gui render module to dashboard. Replace html_document.render_html_for_spec with dashboard.render_dashboard_html. Signed-off-by: Qian Lin --- .../review_gui/render/{html_document.py => dashboard.py} | 4 ++-- .../review_gui/streamlit_ui.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename isaaclab_arena_examples/agentic_environment_generation/review_gui/render/{html_document.py => dashboard.py} (93%) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/html_document.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py similarity index 93% rename from isaaclab_arena_examples/agentic_environment_generation/review_gui/render/html_document.py rename to isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py index d83c6b275..a4318c3d3 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/html_document.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py @@ -17,8 +17,8 @@ from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.styles import DASHBOARD_CSS -def render_html_for_spec(spec: ArenaEnvInitialGraphSpec) -> str: - """Render the self-contained review HTML dashboard for ``spec``.""" +def render_dashboard_html(spec: ArenaEnvInitialGraphSpec) -> str: + """Render the self-contained review dashboard HTML for ``spec``.""" initial_state = spec.initial_state_spec return f""" diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index c234d14f3..cbbab13ee 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -22,7 +22,7 @@ import streamlit as st from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec -from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.html_document import render_html_for_spec +from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.dashboard import render_dashboard_html # Visualization iframe height. Tuned so the graph + tasks + node grid all # fit without an outer Streamlit scrollbar swallowing the inner one. @@ -85,7 +85,7 @@ def initialize_state(yaml_path: Path) -> None: if not initial.is_valid: st.session_state["rendered_html"] = _BROKEN_PLACEHOLDER_HTML else: - st.session_state["rendered_html"] = render_html_for_spec(initial.spec) + st.session_state["rendered_html"] = render_dashboard_html(initial.spec) def render_validation_badge(validation: ValidationResult) -> None: @@ -166,7 +166,7 @@ def render_editor_panel(yaml_path: Path) -> ValidationResult: edited_since_render = st.session_state["edited_text"] != st.session_state["last_rendered_text"] if validation.is_valid and edited_since_render: with st.spinner("Rendering visualization…"): - st.session_state["rendered_html"] = render_html_for_spec(validation.spec) + st.session_state["rendered_html"] = render_dashboard_html(validation.spec) st.session_state["last_rendered_text"] = st.session_state["edited_text"] st.toast("Visualization updated.", icon="🔄") From 2989b7e33a32625a7023eb2e0166bef10cf2b331 Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 08:26:46 -0700 Subject: [PATCH 07/19] Add SimApp sidecar for fast registry-backed YAML validation in review GUI. Route TaskRegistry and relation checks through a persistent Kit subprocess so Streamlit re-validation stays fast after the first boot, and rebuild validated specs locally via skip_registry context. Signed-off-by: Qian Lin --- .../environments/arena_env_graph_spec.py | 6 +- .../environments/arena_env_graph_types.py | 10 +- .../review_gui/simapp_sidecar.py | 175 ++++++++++++++++++ .../review_gui/simapp_sidecar_client.py | 156 ++++++++++++++++ .../review_gui/streamlit_ui.py | 82 +++++++- 5 files changed, 418 insertions(+), 11 deletions(-) create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py diff --git a/isaaclab_arena/environments/arena_env_graph_spec.py b/isaaclab_arena/environments/arena_env_graph_spec.py index c11c09656..6176357f9 100644 --- a/isaaclab_arena/environments/arena_env_graph_spec.py +++ b/isaaclab_arena/environments/arena_env_graph_spec.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Self -from pydantic import BaseModel, Field, SerializeAsAny, field_validator, model_validator +from pydantic import BaseModel, Field, SerializeAsAny, ValidationInfo, field_validator, model_validator from isaaclab_arena.assets.registries import TaskRegistry from isaaclab_arena.environments.arena_env_graph_types import ( @@ -170,8 +170,10 @@ class ArenaEnvInitialGraphSpec(ArenaEnvGraphSpecBase): initial_state_spec: ArenaEnvGraphStateSpec @model_validator(mode="after") - def validate(self) -> Self: + def validate(self, info: ValidationInfo) -> Self: """Check unique IDs, constraint references, and spatial constraint shapes.""" + if info.context and info.context.get("skip_registry"): + return self assert_unique_ids(self.nodes, [], [self.initial_state_spec]) assert_constraint_references(self.nodes, [self.initial_state_spec]) assert_spatial_constraint_shapes([self.initial_state_spec]) diff --git a/isaaclab_arena/environments/arena_env_graph_types.py b/isaaclab_arena/environments/arena_env_graph_types.py index 740f6a642..38e121175 100644 --- a/isaaclab_arena/environments/arena_env_graph_types.py +++ b/isaaclab_arena/environments/arena_env_graph_types.py @@ -16,7 +16,7 @@ from enum import Enum from typing import Any -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, ValidationInfo, field_validator, model_validator from isaaclab_arena.assets.object_type import ObjectType from isaaclab_arena.assets.registries import ObjectRelationLibraryRegistry, TaskRegistry @@ -115,7 +115,9 @@ class TaskSpec(BaseModel): @field_validator("kind") @classmethod - def _validate_registered_task_type(cls, value: str) -> str: + def _validate_registered_task_type(cls, value: str, info: ValidationInfo) -> str: + if info.context and info.context.get("skip_registry"): + return value registry = TaskRegistry() assert registry.is_registered(value), f"Unknown task kind '{value}'" return value @@ -169,7 +171,9 @@ class SpatialRelationSpec(BaseModel): ) @model_validator(mode="after") - def _validate_kind_and_arity(self) -> SpatialRelationSpec: + def _validate_kind_and_arity(self, info: ValidationInfo) -> SpatialRelationSpec: + if info.context and info.context.get("skip_registry"): + return self registry = ObjectRelationLibraryRegistry() assert registry.is_registered(self.kind), f"Unknown relation kind '{self.kind}'" relation_cls = registry.get_object_relation_by_name(self.kind) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py new file mode 100644 index 000000000..ee107cc75 --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py @@ -0,0 +1,175 @@ +# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Long-lived ``SimulationApp`` host process for the live review editor. + +Boots Kit's ``SimulationApp`` once on *its own* main thread and serves +spec-validation requests over a newline-delimited JSON-RPC pipe on +stdin/stdout. The parent (``streamlit_ui.py`` running inside Streamlit) spawns +exactly one of these and reuses it for the entire server lifetime via +:class:`isaaclab_arena_examples.agentic_environment_generation.review_gui.simapp_sidecar_client.SimAppSidecar`. + +Why a sidecar and not an in-process ``SimulationApp``: + +* ``signal.signal`` only works in the main thread. Streamlit's + ``ScriptRunner`` runs the script in a worker thread, so SimApp's signal + setup raises ``ValueError("signal only works in main thread …")``. +* Registry imports transitively load ``pxr``, which must happen only after + ``SimulationApp`` starts. A dedicated process keeps that ordering safe. + +Protocol (newline-delimited JSON over stdin/stdout): + + Ready handshake (sent by sidecar on boot before reading any request): + {"ready": true} # SimApp boot succeeded + {"ready": false, "error": "..."} # boot failed; sidecar exits + + Requests: + {"cmd": "ping"} + → {"ok": true} + + {"cmd": "validate_spec", "yaml_text": "..."} + → {"ok": true, "spec_dict": {...}} + (full :class:`ArenaEnvInitialGraphSpec` validation including registry + lookups — runs in the sidecar where registries are already warm) + + {"cmd": "shutdown"} + → {"ok": true} # sidecar exits cleanly after replying + + Parent EOF on stdin (parent process died) triggers the same graceful + shutdown as the explicit "shutdown" cmd. + +stdout multiplexing: + +Kit writes a lot to stdout (warnings, replicator startup, etc.) and that +would corrupt the JSON channel the parent reads. We dup the original +stdout fd before touching Kit, then redirect Kit's stdout to stderr — +JSON replies go out through the saved fd; everything else from Kit +appears on the user's terminal via inherited stderr. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import signal +import sys +import traceback +import yaml +from typing import Any + +# --------------------------------------------------------------------------- +# stdout multiplexing setup — run BEFORE importing anything that might +# touch Kit or print to stdout, otherwise Kit's chatter pollutes the JSON +# channel and the parent crashes on the first bad json.loads. +# --------------------------------------------------------------------------- + +_JSON_FD = os.dup(1) # save real stdout for our JSON channel +os.dup2(2, 1) +sys.stdout = sys.stderr + + +def _send(payload: dict[str, Any]) -> None: + """Write one JSON line to the parent on the saved stdout fd.""" + data = (json.dumps(payload) + "\n").encode("utf-8") + os.write(_JSON_FD, data) + + +def _launch_simulation_app(): + """Boot Isaac Sim's ``SimulationApp`` for registry-backed validation.""" + try: + from isaaclab_arena.utils.isaaclab_utils.simulation_app import get_app_launcher # noqa: PLC0415 + + sim_args = argparse.Namespace(headless=True, enable_cameras=False, hide_ui=True, livestream=-1) + return get_app_launcher(sim_args).app + except Exception as exc: + print(f"[simapp_sidecar] SimulationApp launch failed: {exc}", file=sys.stderr) + return None + + +def _install_signal_handlers() -> None: + def _exit(signum, _frame): + raise SystemExit(0) + + signal.signal(signal.SIGTERM, _exit) + signal.signal(signal.SIGINT, _exit) + + +def _serve() -> int: + """Boot SimApp, hand-shake with the parent, then service validation requests.""" + _install_signal_handlers() + + app = _launch_simulation_app() + if app is None: + _send({"ready": False, "error": "SimulationApp launch returned None"}) + return 1 + + from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec # noqa: PLC0415 + + _send({"ready": True}) + + try: + for raw_line in sys.stdin: + line = raw_line.strip() + if not line: + continue + try: + req = json.loads(line) + except json.JSONDecodeError as exc: + _send({"ok": False, "error": f"bad json: {exc}"}) + continue + + cmd = req.get("cmd") + if cmd == "shutdown": + _send({"ok": True}) + return 0 + + if cmd == "ping": + _send({"ok": True}) + continue + + if cmd == "validate_spec": + _send(_handle_validate_spec(req, ArenaEnvInitialGraphSpec)) + continue + + _send({"ok": False, "error": f"unknown cmd: {cmd!r}"}) + + return 0 + finally: + with contextlib.suppress(Exception): + app.close() + + +def _handle_validate_spec(req: dict[str, Any], spec_cls) -> dict[str, Any]: + """Parse and fully validate YAML as an initial graph spec.""" + yaml_text = req.get("yaml_text") + if not isinstance(yaml_text, str): + return {"ok": False, "error": "validate_spec requires string 'yaml_text'"} + + try: + raw = yaml.safe_load(yaml_text) + except Exception as exc: + return {"ok": False, "error": f"yaml parse failed: {exc}", "traceback": traceback.format_exc()} + + if raw is None: + return {"ok": False, "error": "YAML is empty"} + if not isinstance(raw, dict): + return {"ok": False, "error": f"expected mapping, got {type(raw).__name__}"} + + try: + spec = spec_cls.model_validate(raw) + except Exception as exc: + return {"ok": False, "error": f"spec validation failed: {exc}", "traceback": traceback.format_exc()} + + return {"ok": True, "spec_dict": spec.to_dict()} + + +def main() -> int: + return _serve() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py new file mode 100644 index 000000000..06ec72547 --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py @@ -0,0 +1,156 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""JSON-RPC client for the persistent :mod:`isaaclab_arena_examples.agentic_environment_generation.review_gui.simapp_sidecar` process.""" + +from __future__ import annotations + +import contextlib +import json +import os +import subprocess +import sys +import threading +from typing import Any + + +class SimAppSidecarError(RuntimeError): + """Raised when the SimApp sidecar process can't fulfil a request.""" + + +class SimAppSidecar: + """Long-lived Kit/SimApp host process exposed as a validation service. + + See ``simapp_sidecar.py`` for the protocol. Cached via ``@st.cache_resource`` + in the Streamlit app; pipe access is serialized with an internal lock. + """ + + def __init__(self, *, boot_timeout_s: float = 180.0, shutdown_timeout_s: float = 10.0) -> None: + self._proc: subprocess.Popen | None = None + self._lock = threading.Lock() + self._boot_timeout_s = boot_timeout_s + self._shutdown_timeout_s = shutdown_timeout_s + + def start(self) -> None: + """Spawn the sidecar process and wait for its ``{"ready": true}`` handshake.""" + if self._proc is not None and self._proc.poll() is None: + return + + cmd = [ + sys.executable, + "-m", + "isaaclab_arena_examples.agentic_environment_generation.review_gui.simapp_sidecar", + ] + self._proc = subprocess.Popen( + cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=None, + text=True, + bufsize=1, + env=os.environ.copy(), + ) + + line = self._readline_or_die() + try: + msg = json.loads(line) + except json.JSONDecodeError as exc: + self._terminate() + raise SimAppSidecarError(f"Sidecar emitted non-JSON handshake: {line!r}") from exc + + if not msg.get("ready"): + self._terminate() + raise SimAppSidecarError( + f"Sidecar boot failed: {msg.get('error', 'unknown error')}\n{msg.get('traceback', '')}" + ) + + def is_alive(self) -> bool: + return self._proc is not None and self._proc.poll() is None + + def close(self) -> None: + """Send ``shutdown``, then terminate/kill if the process doesn't exit.""" + proc = self._proc + if proc is None: + return + self._proc = None + + if proc.poll() is None: + with contextlib.suppress(Exception): + proc.stdin.write(json.dumps({"cmd": "shutdown"}) + "\n") + proc.stdin.flush() + try: + proc.wait(timeout=self._shutdown_timeout_s) + except subprocess.TimeoutExpired: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + with contextlib.suppress(Exception): + proc.wait(timeout=5) + + with contextlib.suppress(Exception): + if proc.stdin: + proc.stdin.close() + with contextlib.suppress(Exception): + if proc.stdout: + proc.stdout.close() + + def validate_yaml_text(self, yaml_text: str) -> dict[str, Any]: + """Run full spec validation (including registry lookups) in the sidecar.""" + if not self.is_alive(): + raise SimAppSidecarError("SimApp sidecar is not running — start it first") + + with self._lock: + return self._request({"cmd": "validate_spec", "yaml_text": yaml_text}) + + def ping(self) -> bool: + """Cheap liveness check round-trip — returns True on a healthy reply.""" + if not self.is_alive(): + return False + with self._lock: + try: + response = self._request({"cmd": "ping"}) + except SimAppSidecarError: + return False + return bool(response.get("ok")) + + def _request(self, payload: dict[str, Any]) -> dict[str, Any]: + assert self._proc is not None and self._proc.stdin is not None and self._proc.stdout is not None + line = json.dumps(payload) + "\n" + try: + self._proc.stdin.write(line) + self._proc.stdin.flush() + except BrokenPipeError as exc: + raise SimAppSidecarError("sidecar pipe closed unexpectedly") from exc + + reply_line = self._readline_or_die() + try: + return json.loads(reply_line) + except json.JSONDecodeError as exc: + raise SimAppSidecarError(f"sidecar replied with non-JSON: {reply_line!r}") from exc + + def _readline_or_die(self) -> str: + assert self._proc is not None and self._proc.stdout is not None + line = self._proc.stdout.readline() + if line == "": + exit_code = self._proc.poll() + raise SimAppSidecarError( + f"sidecar exited prematurely (exit code: {exit_code}). " + "See its stderr output above for the underlying cause." + ) + return line + + def _terminate(self) -> None: + if self._proc is None: + return + with contextlib.suppress(Exception): + self._proc.terminate() + try: + self._proc.wait(timeout=5) + except subprocess.TimeoutExpired: + with contextlib.suppress(Exception): + self._proc.kill() + self._proc = None diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index cbbab13ee..7ce2996eb 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -9,25 +9,34 @@ /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server \\ --yaml isaaclab_arena/tests/test_data/pick_and_place_maple_table_init_env_graph.yaml + +Registry lookups (task kinds, relation kinds) run in a persistent SimApp +sidecar so YAML re-validation stays fast after the first ~30s sidecar boot. """ from __future__ import annotations import argparse +import atexit import traceback import yaml from dataclasses import dataclass from pathlib import Path +from typing import Any import streamlit as st from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.dashboard import render_dashboard_html +from isaaclab_arena_examples.agentic_environment_generation.review_gui.simapp_sidecar_client import ( + SimAppSidecar, + SimAppSidecarError, +) -# Visualization iframe height. Tuned so the graph + tasks + node grid all -# fit without an outer Streamlit scrollbar swallowing the inner one. _IFRAME_HEIGHT_PX = 1100 +_SKIP_REGISTRY_CONTEXT: dict[str, Any] = {"skip_registry": True} + _BROKEN_PLACEHOLDER_HTML = """ @@ -35,6 +44,36 @@ """ +@st.cache_resource(show_spinner="Booting Isaac Sim sidecar (≈30s first run, cached afterwards)…") +def _get_simapp_sidecar() -> SimAppSidecar | None: + """Spawn the SimApp sidecar once per Streamlit server process.""" + sidecar = SimAppSidecar() + try: + sidecar.start() + except SimAppSidecarError as exc: + print(f"[review_gui] SimApp sidecar failed to start: {exc}", flush=True) + return None + + atexit.register(sidecar.close) + return sidecar + + +def _ensure_sidecar() -> SimAppSidecar | None: + """Return a healthy sidecar, re-spawning if the cached one died.""" + sidecar = _get_simapp_sidecar() + if sidecar is not None and sidecar.is_alive(): + return sidecar + if sidecar is not None: + sidecar.close() + _get_simapp_sidecar.clear() + return _get_simapp_sidecar() + + +def _spec_from_sidecar_dict(spec_dict: dict[str, Any]) -> ArenaEnvInitialGraphSpec: + """Rebuild a validated spec locally without registry imports.""" + return ArenaEnvInitialGraphSpec.model_validate(spec_dict, context=_SKIP_REGISTRY_CONTEXT) + + @dataclass class ValidationResult: """Outcome of parsing and validating YAML text as an initial graph spec.""" @@ -48,14 +87,46 @@ def is_valid(self) -> bool: def validate_yaml_text(text: str) -> ValidationResult: - """Parse ``text`` as YAML and validate it as an :class:`ArenaEnvInitialGraphSpec`.""" + """Parse YAML and validate via the SimApp sidecar (registry lookups run there).""" try: raw = yaml.safe_load(text) - spec = ArenaEnvInitialGraphSpec.model_validate(raw) - return ValidationResult(spec=spec, error=None) except Exception: return ValidationResult(spec=None, error=traceback.format_exc()) + if raw is None: + return ValidationResult(spec=None, error="YAML is empty") + if not isinstance(raw, dict): + return ValidationResult(spec=None, error=f"Expected mapping, got {type(raw).__name__}") + + sidecar = _ensure_sidecar() + if sidecar is None: + return ValidationResult( + spec=None, + error=( + "SimApp sidecar is unavailable — cannot validate registry entries. " + "Check the terminal where you launched the server." + ), + ) + + try: + response = sidecar.validate_yaml_text(text) + except SimAppSidecarError as exc: + _get_simapp_sidecar.clear() + return ValidationResult(spec=None, error=str(exc)) + + if not response.get("ok"): + err = response.get("error", "validation failed") + tb = response.get("traceback", "") + message = f"{err}\n\n{tb}" if tb else str(err) + return ValidationResult(spec=None, error=message) + + try: + spec = _spec_from_sidecar_dict(response["spec_dict"]) + except Exception: + return ValidationResult(spec=None, error=traceback.format_exc()) + + return ValidationResult(spec=spec, error=None) + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) @@ -208,5 +279,4 @@ def main() -> None: render_visualization_panel() -# Streamlit invokes the script top-level on every rerun. main() From 56b912785b96b85511e681e89c684aaa4b1cea92 Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 08:27:50 -0700 Subject: [PATCH 08/19] Add live USD snapshot rendering to the review GUI sidecar. Render node thumbnails via Kit viewport capture in the sidecar, wire render_spec through the client, and inline PNGs in the dashboard. Signed-off-by: Qian Lin --- .../review_gui/render/dashboard.py | 5 +- .../review_gui/render/panels.py | 13 +- .../review_gui/render/thumbnails.py | 14 +- .../review_gui/simapp_sidecar.py | 86 +++-- .../review_gui/simapp_sidecar_client.py | 38 +- .../review_gui/streamlit_ui.py | 34 +- .../review_gui/thumbnail_render.py | 356 ++++++++++++++++++ 7 files changed, 499 insertions(+), 47 deletions(-) create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/thumbnail_render.py diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py index a4318c3d3..731b3b781 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py @@ -17,9 +17,10 @@ from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.styles import DASHBOARD_CSS -def render_dashboard_html(spec: ArenaEnvInitialGraphSpec) -> str: +def render_dashboard_html(spec: ArenaEnvInitialGraphSpec, thumbnails: dict[str, bytes] | None = None) -> str: """Render the self-contained review dashboard HTML for ``spec``.""" initial_state = spec.initial_state_spec + thumbnails = thumbnails or {} return f""" @@ -45,7 +46,7 @@ def render_dashboard_html(spec: ArenaEnvInitialGraphSpec) -> str:

    Nodes

    -
    {render_node_cards(spec)}
    +
    {render_node_cards(spec, thumbnails)}
    diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py index 901b4c331..1f3675b69 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py @@ -10,9 +10,7 @@ from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec from isaaclab_arena.environments.arena_env_graph_types import ArenaEnvGraphNodeSpec, ArenaEnvGraphStateSpec -from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.thumbnails import ( - render_placeholder_thumbnail, -) +from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.thumbnails import render_node_thumbnail def render_unary_constraints(state: ArenaEnvGraphStateSpec) -> str: @@ -62,14 +60,15 @@ def render_tasks_table(spec: ArenaEnvInitialGraphSpec) -> str: ) -def render_node_cards(spec: ArenaEnvInitialGraphSpec) -> str: - return "\n".join(render_node_card(node) for node in spec.nodes) +def render_node_cards(spec: ArenaEnvInitialGraphSpec, thumbnails: dict[str, bytes] | None = None) -> str: + thumbnails = thumbnails or {} + return "\n".join(render_node_card(node, thumbnails.get(node.id)) for node in spec.nodes) -def render_node_card(node: ArenaEnvGraphNodeSpec) -> str: +def render_node_card(node: ArenaEnvGraphNodeSpec, png_bytes: bytes | None = None) -> str: node_dict = node.model_dump(mode="json", exclude_none=True) node_yaml = yaml.safe_dump(node_dict, sort_keys=False).rstrip() - thumb = render_placeholder_thumbnail(node) + thumb = render_node_thumbnail(node, png_bytes) return f"""
    {thumb}
    diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/thumbnails.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/thumbnails.py index f2742ee8f..4b2a5eb23 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/thumbnails.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/thumbnails.py @@ -5,14 +5,22 @@ from __future__ import annotations +import base64 import html as html_lib from isaaclab_arena.environments.arena_env_graph_types import ArenaEnvGraphNodeSpec -# TODO(qianl): Replace placeholder thumbnails with sim-rendered snapshots . -def render_placeholder_thumbnail(node: ArenaEnvGraphNodeSpec) -> str: - """Per-node placeholder thumbnail — two-letter initial.""" +def render_node_thumbnail(node: ArenaEnvGraphNodeSpec, png_bytes: bytes | None = None) -> str: + """Per-node thumbnail: USD capture if available, else two-letter placeholder.""" + if png_bytes: + b64 = base64.b64encode(png_bytes).decode("ascii") + return ( + '
    ' + f'{html_lib.escape(node.name)} thumbnail' + f'{html_lib.escape(node.name)}' + "
    " + ) initial = (node.name[:2] if node.name else "?").upper() return f"""
    {html_lib.escape(initial)} diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py index ee107cc75..95045d69c 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py @@ -6,9 +6,10 @@ """Long-lived ``SimulationApp`` host process for the live review editor. Boots Kit's ``SimulationApp`` once on *its own* main thread and serves -spec-validation requests over a newline-delimited JSON-RPC pipe on -stdin/stdout. The parent (``streamlit_ui.py`` running inside Streamlit) spawns -exactly one of these and reuses it for the entire server lifetime via +validation and thumbnail-render requests over a newline-delimited JSON-RPC +pipe on stdin/stdout. The parent (``streamlit_ui.py`` running inside +Streamlit) spawns exactly one of these and reuses it for the entire server +lifetime via :class:`isaaclab_arena_examples.agentic_environment_generation.review_gui.simapp_sidecar_client.SimAppSidecar`. Why a sidecar and not an in-process ``SimulationApp``: @@ -16,8 +17,10 @@ * ``signal.signal`` only works in the main thread. Streamlit's ``ScriptRunner`` runs the script in a worker thread, so SimApp's signal setup raises ``ValueError("signal only works in main thread …")``. -* Registry imports transitively load ``pxr``, which must happen only after - ``SimulationApp`` starts. A dedicated process keeps that ordering safe. +* ``omni.usd.UsdContext`` is process-singleton AND can't tolerate cross- + thread driving from Streamlit reruns — driving it from worker threads + triggers ``[Error] [omni.usd] UsdContext busy`` and the open_stage call + fails. A dedicated process with serialized request handling avoids both. Protocol (newline-delimited JSON over stdin/stdout): @@ -34,6 +37,12 @@ (full :class:`ArenaEnvInitialGraphSpec` validation including registry lookups — runs in the sidecar where registries are already warm) + {"cmd": "render_spec", "yaml_text": "..."} + → {"ok": true, "paths": {"node_id": "/abs/path/to.png", ...}, + "errors": [{"node_id": "...", "error": "..."}]} + (paths are absolute filesystem paths on the disk cache. The PNGs + themselves stay on disk — the parent reads them itself.) + {"cmd": "shutdown"} → {"ok": true} # sidecar exits cleanly after replying @@ -51,7 +60,6 @@ from __future__ import annotations -import argparse import contextlib import json import os @@ -59,15 +67,10 @@ import sys import traceback import yaml +from pathlib import Path from typing import Any -# --------------------------------------------------------------------------- -# stdout multiplexing setup — run BEFORE importing anything that might -# touch Kit or print to stdout, otherwise Kit's chatter pollutes the JSON -# channel and the parent crashes on the first bad json.loads. -# --------------------------------------------------------------------------- - -_JSON_FD = os.dup(1) # save real stdout for our JSON channel +_JSON_FD = os.dup(1) os.dup2(2, 1) sys.stdout = sys.stderr @@ -78,18 +81,6 @@ def _send(payload: dict[str, Any]) -> None: os.write(_JSON_FD, data) -def _launch_simulation_app(): - """Boot Isaac Sim's ``SimulationApp`` for registry-backed validation.""" - try: - from isaaclab_arena.utils.isaaclab_utils.simulation_app import get_app_launcher # noqa: PLC0415 - - sim_args = argparse.Namespace(headless=True, enable_cameras=False, hide_ui=True, livestream=-1) - return get_app_launcher(sim_args).app - except Exception as exc: - print(f"[simapp_sidecar] SimulationApp launch failed: {exc}", file=sys.stderr) - return None - - def _install_signal_handlers() -> None: def _exit(signum, _frame): raise SystemExit(0) @@ -99,15 +90,26 @@ def _exit(signum, _frame): def _serve() -> int: - """Boot SimApp, hand-shake with the parent, then service validation requests.""" + """Boot SimApp, hand-shake with the parent, then service requests.""" _install_signal_handlers() + try: + from isaaclab_arena_examples.agentic_environment_generation.review_gui.thumbnail_render import ( # noqa: PLC0415 + _launch_simulation_app, + ) + except Exception as exc: + _send({"ready": False, "error": f"import failed: {exc}", "traceback": traceback.format_exc()}) + return 1 + app = _launch_simulation_app() if app is None: _send({"ready": False, "error": "SimulationApp launch returned None"}) return 1 from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec # noqa: PLC0415 + from isaaclab_arena_examples.agentic_environment_generation.review_gui.thumbnail_render import ( # noqa: PLC0415 + _render_thumbnails_with_app, + ) _send({"ready": True}) @@ -135,6 +137,10 @@ def _serve() -> int: _send(_handle_validate_spec(req, ArenaEnvInitialGraphSpec)) continue + if cmd == "render_spec": + _send(_handle_render_spec(app, req, _render_thumbnails_with_app, ArenaEnvInitialGraphSpec)) + continue + _send({"ok": False, "error": f"unknown cmd: {cmd!r}"}) return 0 @@ -167,6 +173,34 @@ def _handle_validate_spec(req: dict[str, Any], spec_cls) -> dict[str, Any]: return {"ok": True, "spec_dict": spec.to_dict()} +def _handle_render_spec( + app, + req: dict[str, Any], + render_fn, + spec_cls, +) -> dict[str, Any]: + """Parse the spec, run thumbnail rendering, marshal the response.""" + yaml_text = req.get("yaml_text") + if not isinstance(yaml_text, str): + return {"ok": False, "error": "render_spec requires string 'yaml_text'"} + + try: + spec = spec_cls.from_dict(yaml.safe_load(yaml_text)) + except Exception as exc: + return {"ok": False, "error": f"spec parse failed: {exc}", "traceback": traceback.format_exc()} + + try: + paths: dict[str, Path] = render_fn(app, spec) + except Exception as exc: + return {"ok": False, "error": f"render failed: {exc}", "traceback": traceback.format_exc()} + + return { + "ok": True, + "paths": {node_id: str(p) for node_id, p in paths.items()}, + "errors": [], + } + + def main() -> int: return _serve() diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py index 06ec72547..546921e3b 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py @@ -13,19 +13,19 @@ import subprocess import sys import threading +import yaml +from pathlib import Path from typing import Any +from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec + class SimAppSidecarError(RuntimeError): """Raised when the SimApp sidecar process can't fulfil a request.""" class SimAppSidecar: - """Long-lived Kit/SimApp host process exposed as a validation service. - - See ``simapp_sidecar.py`` for the protocol. Cached via ``@st.cache_resource`` - in the Streamlit app; pipe access is serialized with an internal lock. - """ + """Long-lived Kit/SimApp host process exposed as a validation and render service.""" def __init__(self, *, boot_timeout_s: float = 180.0, shutdown_timeout_s: float = 10.0) -> None: self._proc: subprocess.Popen | None = None @@ -106,6 +106,34 @@ def validate_yaml_text(self, yaml_text: str) -> dict[str, Any]: with self._lock: return self._request({"cmd": "validate_spec", "yaml_text": yaml_text}) + def render_spec(self, spec: ArenaEnvInitialGraphSpec) -> dict[str, bytes]: + """Ask the sidecar to render thumbnails for ``spec``.""" + if not self.is_alive(): + raise SimAppSidecarError("SimApp sidecar is not running — start it first") + + yaml_text = yaml.safe_dump(spec.to_dict(), sort_keys=False) + + with self._lock: + response = self._request({"cmd": "render_spec", "yaml_text": yaml_text}) + + if not response.get("ok"): + raise SimAppSidecarError( + f"sidecar render failed: {response.get('error', 'unknown')}\n{response.get('traceback', '')}" + ) + + paths: dict[str, str] = response.get("paths", {}) or {} + results: dict[str, bytes] = {} + for node_id, path_str in paths.items(): + path = Path(path_str) + if path.exists() and path.stat().st_size > 0: + results[node_id] = path.read_bytes() + else: + print( + f"[review_gui] sidecar reported {node_id} -> {path_str} but file is missing.", + file=sys.stderr, + ) + return results + def ping(self) -> bool: """Cheap liveness check round-trip — returns True on a healthy reply.""" if not self.is_alive(): diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 7ce2996eb..7d5b95f89 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -10,8 +10,9 @@ /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server \\ --yaml isaaclab_arena/tests/test_data/pick_and_place_maple_table_init_env_graph.yaml -Registry lookups (task kinds, relation kinds) run in a persistent SimApp -sidecar so YAML re-validation stays fast after the first ~30s sidecar boot. +Registry lookups (task kinds, relation kinds, asset USD paths) run in a +persistent SimApp sidecar so YAML re-validation stays fast after the first +~30s sidecar boot. Thumbnails are rendered live by the same sidecar. """ from __future__ import annotations @@ -74,6 +75,31 @@ def _spec_from_sidecar_dict(spec_dict: dict[str, Any]) -> ArenaEnvInitialGraphSp return ArenaEnvInitialGraphSpec.model_validate(spec_dict, context=_SKIP_REGISTRY_CONTEXT) +def _render_with_thumbnails(spec: ArenaEnvInitialGraphSpec) -> str: + """Render review HTML, asking the sidecar for live USD thumbnails.""" + sidecar = _ensure_sidecar() + if sidecar is None: + st.warning( + "Isaac Sim sidecar is unavailable — showing placeholder thumbnails. " + "Check the terminal where you launched the server for the underlying error.", + icon="⚠️", + ) + return render_dashboard_html(spec) + + try: + thumbnails = sidecar.render_spec(spec) + except SimAppSidecarError as exc: + st.error( + f"Sidecar render failed; showing placeholder thumbnails.\n\n```\n{exc}\n```", + icon="🛑", + ) + with st.spinner("Resetting the SimApp sidecar…"): + _get_simapp_sidecar.clear() + return render_dashboard_html(spec) + + return render_dashboard_html(spec, thumbnails=thumbnails if thumbnails else None) + + @dataclass class ValidationResult: """Outcome of parsing and validating YAML text as an initial graph spec.""" @@ -156,7 +182,7 @@ def initialize_state(yaml_path: Path) -> None: if not initial.is_valid: st.session_state["rendered_html"] = _BROKEN_PLACEHOLDER_HTML else: - st.session_state["rendered_html"] = render_dashboard_html(initial.spec) + st.session_state["rendered_html"] = _render_with_thumbnails(initial.spec) def render_validation_badge(validation: ValidationResult) -> None: @@ -237,7 +263,7 @@ def render_editor_panel(yaml_path: Path) -> ValidationResult: edited_since_render = st.session_state["edited_text"] != st.session_state["last_rendered_text"] if validation.is_valid and edited_since_render: with st.spinner("Rendering visualization…"): - st.session_state["rendered_html"] = render_dashboard_html(validation.spec) + st.session_state["rendered_html"] = _render_with_thumbnails(validation.spec) st.session_state["last_rendered_text"] = st.session_state["edited_text"] st.toast("Visualization updated.", icon="🔄") diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/thumbnail_render.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/thumbnail_render.py new file mode 100644 index 000000000..d4ea06a9b --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/thumbnail_render.py @@ -0,0 +1,356 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""USD viewport thumbnail rendering for the review GUI SimApp sidecar.""" + +from __future__ import annotations + +import argparse +import hashlib +import sys +from pathlib import Path + +from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec + +THUMBNAIL_CACHE_DIR = Path(__file__).resolve().parents[3] / ".cache" / "llm_env_gen_thumbnails" + + +def _render_thumbnails_with_app(app, spec: ArenaEnvInitialGraphSpec) -> dict[str, Path]: + """Resolve each node's USD via ``AssetRegistry``, render cache-misses, return PNG paths. + + ``app`` must already be a booted ``SimulationApp``. The caller owns the + lifecycle (Kit may turn ``app.close()`` into ``os._exit(0)`` — that's why + the sidecar holds the only reference and closes it inside its ``finally``). + + Returns ``{node.id: png_path}`` for nodes whose asset USD could be located + *and* whose PNG exists on disk (either from the persistent cache under + ``THUMBNAIL_CACHE_DIR`` or freshly rendered into the cache by + :func:`_capture_usd_thumbnails`). Missing entries fall through to the + placeholder in :func:`_render_node_thumbnail`, so a partial failure (one + bad asset) never breaks the rest of the page. + + We return ``Path`` rather than ``bytes`` so the sidecar protocol can ship + just the filenames over its stdin/stdout pipe (a few hundred bytes of JSON + instead of multiple MB of base64 PNG data). The parent reads the bytes + itself off the shared filesystem cache. + + Ordering matters: ``SimulationApp`` MUST be launched before any + ``AssetRegistry`` access, because ``ensure_assets_registered()`` imports + isaaclab asset modules which transitively load ``pxr``. ``pxr`` loaded + before ``AppLauncher`` puts Kit's extension manager into an unrecoverable + state ("extension class wrapper for base class ... has not been created + yet"). This is the same root cause we fixed for the pytest suite. + """ + asset_paths = _resolve_node_usd_paths(spec) + if not asset_paths: + print("[thumbnail_render] no asset USD paths resolved; skipping thumbnail rendering.", file=sys.stderr) + return {} + + THUMBNAIL_CACHE_DIR.mkdir(parents=True, exist_ok=True) + + # Split into cache-hits vs to-render. Cache key is sha1(usd_path) so + # the same USD across multiple envs / nodes hits the same PNG. + resolved: dict[str, Path] = {} + to_render: dict[str, tuple[str, Path]] = {} + for node_id, usd_path in asset_paths.items(): + cache_path = THUMBNAIL_CACHE_DIR / f"{_usd_cache_key(usd_path)}.png" + if cache_path.exists() and cache_path.stat().st_size > 0: + resolved[node_id] = cache_path + else: + to_render[node_id] = (usd_path, cache_path) + + if to_render: + print( + f"[thumbnail_render] rendering {len(to_render)} new thumbnail(s) " + f"(reusing {len(resolved)} from cache at {THUMBNAIL_CACHE_DIR})...", + file=sys.stderr, + ) + # ``_capture_usd_thumbnails`` still returns ``{node_id: bytes}``, but + # we only use it as a presence signal here — the same call also wrote + # the PNG to ``cache_path`` as a side effect, which is what we return. + captured = _capture_usd_thumbnails(app, to_render) + for node_id, (_usd_path, cache_path) in to_render.items(): + if node_id in captured and cache_path.exists() and cache_path.stat().st_size > 0: + resolved[node_id] = cache_path + else: + print(f"[thumbnail_render] all {len(resolved)} thumbnail(s) served from cache.", file=sys.stderr) + + return resolved + + +def _launch_simulation_app(): + """Boot Isaac Sim's ``SimulationApp`` for headless viewport capture, or ``None`` on failure. + + Kept as a tiny helper so the call site can lazy-import inside this + function — module-level import of ``simulation_app`` would drag Kit + into every invocation, including ``--help``. + """ + try: + # Lazy-import: keeps the default ``review_graph`` invocation Kit-free. + from isaaclab_arena.utils.isaaclab_utils.simulation_app import get_app_launcher # noqa: PLC0415 + + sim_args = argparse.Namespace(headless=True, enable_cameras=True, hide_ui=True, livestream=-1) + return get_app_launcher(sim_args).app + except Exception as exc: + print(f"[thumbnail_render] SimulationApp launch failed: {exc}", file=sys.stderr) + return None + + +def _resolve_node_usd_paths(spec: ArenaEnvInitialGraphSpec) -> dict[str, str]: + """Map ``node.id → usd_path`` via :class:`AssetRegistry`, skipping unresolvable nodes. + + Tries two lookup strategies in order: + + 1. Class-attribute ``cls.usd_path`` — the convention every ``LibraryObject`` + subclass in ``object_library.py`` follows. No instantiation, cheap. + + 2. ``cls().scene_config.robot.spawn.usd_path`` — the convention every + :class:`EmbodimentBase` subclass uses. Requires instantiating the + embodiment because the Franka embodiments populate ``scene_config.robot`` + inside ``__init__`` rather than as a class default. Embodiment + ``__init__`` is light (no Kit / sim required) — it only constructs + configclass objects. + + This function MUST be called only after ``SimulationApp`` has booted — see + the docstring of :func:`_render_thumbnails_with_app` for why. + """ + try: + from isaaclab_arena.assets.registries import AssetRegistry # noqa: PLC0415 + except Exception as exc: + print(f"[thumbnail_render] AssetRegistry import failed: {exc}", file=sys.stderr) + return {} + + registry = AssetRegistry() + paths: dict[str, str] = {} + for node in spec.nodes: + try: + if not registry.is_registered(node.name): + print(f"[thumbnail_render] {node.id}: asset '{node.name}' not registered, skipping.", file=sys.stderr) + continue + cls = registry.get_asset_by_name(node.name) + usd_path = _extract_usd_path(cls) + if not usd_path: + print(f"[thumbnail_render] {node.id}: '{node.name}' has no usd_path, skipping.", file=sys.stderr) + continue + paths[node.id] = usd_path + except Exception as exc: + print(f"[thumbnail_render] {node.id}: lookup failed for '{node.name}': {exc}", file=sys.stderr) + return paths + + +def _extract_usd_path(cls) -> str | None: + """Return the asset's root USD path, or ``None`` if not extractable. + + See :func:`_resolve_node_usd_paths` for the two strategies tried in order. + """ + # Strategy 1: ``LibraryObject`` convention. + usd_path = getattr(cls, "usd_path", None) + if usd_path: + return usd_path + + # Strategy 2: ``EmbodimentBase`` convention. Walk + # ``instance.scene_config.robot.spawn.usd_path``. We instantiate with no + # args; every embodiment ``__init__`` defaults all parameters. + # NoEmbodiment legitimately has no robot — its instance.scene_config + # exists but ``.robot`` is absent / None, so the getattr chain returns + # None and we silently fall through. + try: + instance = cls() + except Exception: + return None + scene_config = getattr(instance, "scene_config", None) + robot = getattr(scene_config, "robot", None) if scene_config is not None else None + spawn = getattr(robot, "spawn", None) if robot is not None else None + return getattr(spawn, "usd_path", None) if spawn is not None else None + + +def _usd_cache_key(usd_path: str) -> str: + return hashlib.sha1(usd_path.encode("utf-8")).hexdigest()[:16] + + +def _capture_usd_thumbnails(app, to_render: dict[str, tuple[str, Path]]) -> dict[str, bytes]: + """Capture all queued USDs under one already-booted ``SimulationApp``. + + Deduplicates by ``usd_path`` so the same USD shared by multiple nodes is + only rendered once and the bytes are fanned back out. + """ + out: dict[str, bytes] = {} + + path_to_node_ids: dict[str, list[str]] = {} + path_to_cache: dict[str, Path] = {} + for node_id, (usd_path, cache_path) in to_render.items(): + path_to_node_ids.setdefault(usd_path, []).append(node_id) + path_to_cache[usd_path] = cache_path + + for usd_path, node_ids in path_to_node_ids.items(): + cache_path = path_to_cache[usd_path] + try: + png_bytes = _render_one_usd(app, usd_path, cache_path) + except Exception as exc: + print(f"[thumbnail_render] render failed for {usd_path}: {exc}", file=sys.stderr) + continue + if png_bytes: + for node_id in node_ids: + out[node_id] = png_bytes + + return out + + +def _render_one_usd(app, usd_path: str, cache_path: Path) -> bytes | None: + """Open ``usd_path`` directly as the stage, frame the camera, capture PNG. + + Opening the USD as the stage root (rather than ``new_stage`` + reference + wrapper) is what makes viewport capture actually produce a file in + headless mode — Kit's viewport machinery binds to the just-opened stage + cleanly, whereas a referenced sub-stage left the render product empty in + every test we tried. The trade-off is that we lose isolation between + captures (each call replaces the stage), but Kit handles that fine + because we call ``open_stage`` again on the next asset. + """ + import omni.usd # noqa: PLC0415 + from omni.kit.viewport.utility import ( # noqa: PLC0415 + capture_viewport_to_file, + frame_viewport_prims, + get_active_viewport, + ) + from pxr import Sdf # noqa: PLC0415 + + ctx = omni.usd.get_context() + if not ctx.open_stage(usd_path): + print(f"[thumbnail_render] open_stage failed: {usd_path}", file=sys.stderr) + return None + stage = ctx.get_stage() + + # Wait for textures / payloads / Nucleus fetches to settle before framing. + _wait_for_stage_load(app, ctx) + + # Standalone object USDs (avocado, bowl, ...) ship no lights, so a viewport + # capture renders them as a near-black silhouette against the dark skybox + # — that's the "blank thumbnail" symptom. Complete scene USDs (maple table) + # already include their own lighting, so this is a no-op for them. + _ensure_default_lighting(stage) + + # Use the default prim if present, otherwise the pseudo-root, for framing. + target_prim = stage.GetDefaultPrim() + if not target_prim or not target_prim.IsValid(): + target_prim = stage.GetPrimAtPath(Sdf.Path("/")) + + viewport = get_active_viewport() + + # Use Kit's own ``frame_viewport_prims`` (the "F"-key equivalent / ``FramePrimsCommand``) + # so we go through the viewport camera controller. Manually editing the + # ``/OmniverseKit_Persp`` xform op directly worked sometimes but Kit's + # camera controller treats /OmniverseKit_Persp as an internal state and + # silently overrode our edits for small assets — that's why avocado / bowl + # captured as tiny specks even with the right math. Letting Kit do the + # framing is both correct and avoids us re-implementing the math. + framed = frame_viewport_prims(viewport, prims=[str(target_prim.GetPath())]) + if not framed: + print(f"[thumbnail_render] warning: frame_viewport_prims failed for {usd_path}", file=sys.stderr) + + # Settle Hydra after camera change so the captured frame matches the new pose. + for _ in range(30): + app.update() + + cache_path.parent.mkdir(parents=True, exist_ok=True) + capture_obj = capture_viewport_to_file(viewport, str(cache_path)) + + _wait_for_capture(app, capture_obj, cache_path, max_updates=600) + + if cache_path.exists() and cache_path.stat().st_size > 0: + return cache_path.read_bytes() + print(f"[thumbnail_render] capture produced no file: {cache_path}", file=sys.stderr) + return None + + +def _wait_for_stage_load(app, usd_context, max_updates: int = 600) -> None: + """Pump frames until ``usd_context.get_stage_loading_status()`` reports nothing pending. + + Returns after stage load completes or after the budget is exhausted. We + also need a few extra frames after the count goes to zero so material + binding / texture upload finishes — they don't show up in the load count. + """ + settled = 0 + for _ in range(max_updates): + app.update() + try: + _msg, loading_count, loaded_count = usd_context.get_stage_loading_status() + except Exception: + return + if loading_count == 0 and loaded_count == 0: + settled += 1 + if settled > 15: + return + else: + settled = 0 + + +def _wait_for_capture(app, capture_obj, cache_path: Path, max_updates: int = 600) -> None: + """Pump ``app.update()`` until the capture PNG lands on disk (or we time out). + + Kit's capture future is fulfilled inside its async loop during + ``app.update()``, but future completion doesn't always coincide with the + file being flushed — checking the file directly is the most reliable + completion signal. We also keep the future-based fast path so a + successful capture doesn't have to wait for the file system to settle. + """ + if capture_obj is None: + for _ in range(max_updates): + app.update() + return + + future = ( + getattr(capture_obj, "_Capture__future", None) + or getattr(capture_obj, "_RenderCapture__future", None) + or getattr(capture_obj, "future", None) + ) + + for _ in range(max_updates): + app.update() + if cache_path.exists() and cache_path.stat().st_size > 0: + return + if future is not None and future.done(): + # Future is done but file might still be flushing — give it a few frames. + for _ in range(15): + app.update() + if cache_path.exists() and cache_path.stat().st_size > 0: + return + return + + +def _ensure_default_lighting(stage) -> None: + """Add a dome + key distant light if the stage has none. + + Without this, standalone object USDs (which don't ship their own lights) + render as a near-black silhouette. We skip the addition if any + ``UsdLuxLight``-derived prim already exists on the stage to avoid + double-lighting scenes like the maple table that bake in their own rig. + """ + from pxr import Gf, Sdf, UsdGeom, UsdLux # noqa: PLC0415 + + for prim in stage.Traverse(): + if ( + prim.HasAPI(UsdLux.LightAPI) + or prim.IsA(UsdLux.BoundableLightBase) + or prim.IsA(UsdLux.NonboundableLightBase) + ): + return + + # Soft hemispherical fill so the asset is visible from any angle, plus a + # weak directional key for shape definition. Intensities are tuned for + # OmniPBR / RTX defaults; tweak if asset libraries adopt darker materials. + dome = UsdLux.DomeLight.Define(stage, Sdf.Path("/_ReviewDomeLight")) + dome.CreateIntensityAttr(800.0) + dome.CreateColorAttr(Gf.Vec3f(1.0, 1.0, 1.0)) + + key = UsdLux.DistantLight.Define(stage, Sdf.Path("/_ReviewKeyLight")) + key.CreateIntensityAttr(2500.0) + key.CreateAngleAttr(2.0) + # Aim the key roughly from the camera's 3/4 angle so the lit side faces + # the viewport. + key_xformable = UsdGeom.Xformable(key.GetPrim()) + key_xformable.ClearXformOpOrder() + rot = key_xformable.AddRotateXYZOp() + rot.Set(Gf.Vec3f(-45.0, 30.0, 0.0)) From e69503a0445e10c7ae4d9e648882de96f4d314df Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 09:41:13 -0700 Subject: [PATCH 09/19] Add prompt-driven generation to the review GUI and defer pxr on asset import. Wire LLM fetch plus sidecar catalogue/compile into the live editor with optional --yaml, and lazy-load USD helpers so library registration avoids early pxr imports. Signed-off-by: Qian Lin --- .../environment_generation_agent.py | 32 +- isaaclab_arena/assets/object.py | 13 +- isaaclab_arena/assets/object_library.py | 2 +- .../assets/object_spawn_defaults.py | 46 +++ isaaclab_arena/assets/object_utils.py | 65 ++-- .../review_gui/server.py | 15 +- .../review_gui/simapp_sidecar.py | 76 +++++ .../review_gui/simapp_sidecar_client.py | 16 + .../review_gui/streamlit_ui.py | 291 +++++++++++++++++- 9 files changed, 487 insertions(+), 69 deletions(-) create mode 100644 isaaclab_arena/assets/object_spawn_defaults.py diff --git a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py index b496f0cdc..522b44d53 100644 --- a/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py +++ b/isaaclab_arena/agentic_environment_generation/environment_generation_agent.py @@ -250,6 +250,33 @@ def generate_spec( A ``(EnvironmentIntentSpec, raw_response)`` tuple. The raw text is useful for debugging. """ + data, text = self.fetch_intent_from_prompt( + prompt, + asset_catalog=asset_catalog, + relation_catalog=relation_catalog, + task_catalog=task_catalog, + temperature=temperature, + max_tokens=max_tokens, + max_retries=max_retries, + ) + spec = EnvironmentIntentSpec.model_validate(data) + return spec, text + + def fetch_intent_from_prompt( + self, + prompt: str, + asset_catalog: AssetCatalogue | None = None, + relation_catalog: RelationCatalogue | None = None, + task_catalog: TaskCatalogue | None = None, + temperature: float = 0.2, + max_tokens: int = 4096, + max_retries: int = 3, + ) -> tuple[dict[str, Any], str]: + """Call the model and return parsed intent JSON without registry validation. + + Registry-backed :class:`EnvironmentIntentSpec` validation should run in a + SimApp process (e.g. the review GUI sidecar) where registries are warm. + """ asset_catalog = asset_catalog or build_asset_catalogue() relation_catalog = relation_catalog or build_relation_catalogue() task_catalog = task_catalog or build_task_catalogue() @@ -268,7 +295,7 @@ def generate_spec( last_exc: Exception | None = None for attempt in range(1 + max_retries): if attempt > 0: - print(f"[generate_spec] retry {attempt}/{max_retries} after: {last_exc}", flush=True) + print(f"[fetch_intent_from_prompt] retry {attempt}/{max_retries} after: {last_exc}", flush=True) try: resp = self.client.chat.completions.create( @@ -299,8 +326,7 @@ def generate_spec( # (e.g. literal tabs) inside JSON strings — DeepSeek-v4-flash is known # to emit these. data = json.loads(text, strict=False) - spec = EnvironmentIntentSpec.model_validate(data) - return spec, text + return data, text except Exception as exc: last_exc = exc diff --git a/isaaclab_arena/assets/object.py b/isaaclab_arena/assets/object.py index e2a92af36..40e9d6089 100644 --- a/isaaclab_arena/assets/object.py +++ b/isaaclab_arena/assets/object.py @@ -11,12 +11,9 @@ from isaaclab.sim.spawners.spawner_cfg import SpawnerCfg from isaaclab_arena.assets.object_base import ObjectBase, ObjectType -from isaaclab_arena.assets.object_utils import detect_object_type from isaaclab_arena.relations.relations import RelationBase from isaaclab_arena.utils.bounding_box import AxisAlignedBoundingBox, quaternion_to_90_deg_z_quarters from isaaclab_arena.utils.pose import Pose -from isaaclab_arena.utils.usd.rigid_bodies import find_shallowest_rigid_body -from isaaclab_arena.utils.usd_helpers import compute_local_bounding_box_from_usd, has_light, open_stage class Object(ObjectBase): @@ -49,6 +46,8 @@ def __init__( "object_type is None (indicating auto-detect) but usd_path is also None. usd_path is required to detect" " object type" ) + from isaaclab_arena.assets.object_utils import detect_object_type + object_type = detect_object_type(usd_path=usd_path) super().__init__(name=name, prim_path=prim_path, object_type=object_type, **kwargs) self.usd_path = usd_path @@ -69,6 +68,8 @@ def add_relation(self, relation: RelationBase) -> None: def get_bounding_box(self) -> AxisAlignedBoundingBox: """Get local bounding box (relative to object origin).""" + from isaaclab_arena.utils.usd_helpers import compute_local_bounding_box_from_usd + assert self.usd_path is not None if self.bounding_box is None: self.bounding_box = compute_local_bounding_box_from_usd(self.usd_path, self.scale) @@ -88,6 +89,8 @@ def get_world_bounding_box(self) -> AxisAlignedBoundingBox: return local_bbox.rotated_90_around_z(quarters).translated(self.initial_pose.position_xyz) def get_corners(self, pos: torch.Tensor) -> torch.Tensor: + from isaaclab_arena.utils.usd_helpers import compute_local_bounding_box_from_usd + assert self.usd_path is not None if self.bounding_box is None: self.bounding_box = compute_local_bounding_box_from_usd(self.usd_path, self.scale) @@ -107,6 +110,8 @@ def enable_reset_pose(self) -> None: def get_contact_sensor_cfg( self, contact_against_object: ObjectBase | None = None, usd_path: str | None = None ) -> ContactSensorCfg: + from isaaclab_arena.utils.usd.rigid_bodies import find_shallowest_rigid_body + assert self.object_type == ObjectType.RIGID, "Contact sensor is only supported for rigid objects" # We override this function from the parent class because in some assets, the rigid body # is not at the root of the USD file. To be robust to this, we find the shallowest rigid body @@ -177,6 +182,8 @@ def _generate_articulation_cfg(self) -> ArticulationCfg: return self._add_initial_pose_to_cfg(object_cfg) def _generate_base_cfg(self) -> AssetBaseCfg: + from isaaclab_arena.utils.usd_helpers import has_light, open_stage + assert self.object_type == ObjectType.BASE if self.spawner_cfg is None: with open_stage(self.usd_path) as stage: diff --git a/isaaclab_arena/assets/object_library.py b/isaaclab_arena/assets/object_library.py index e7bf8c82a..ad565e2ac 100644 --- a/isaaclab_arena/assets/object_library.py +++ b/isaaclab_arena/assets/object_library.py @@ -22,7 +22,7 @@ from isaaclab_arena.assets.lightwheel_lazy import LightwheelLazyPath from isaaclab_arena.assets.object import Object from isaaclab_arena.assets.object_base import ObjectType -from isaaclab_arena.assets.object_utils import ( +from isaaclab_arena.assets.object_spawn_defaults import ( EMPTY_ARTICULATION_INIT_STATE_CFG, RIGID_BODY_PROPS_HIGH_PRECISION, RIGID_BODY_PROPS_MEDIUM_PRECISION, diff --git a/isaaclab_arena/assets/object_spawn_defaults.py b/isaaclab_arena/assets/object_spawn_defaults.py new file mode 100644 index 000000000..4bf52e3f9 --- /dev/null +++ b/isaaclab_arena/assets/object_spawn_defaults.py @@ -0,0 +1,46 @@ +# Copyright (c) 2025-2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Spawn/physics defaults for library objects (no USD / pxr imports).""" + +import isaaclab.sim as sim_utils +from isaaclab.assets import ArticulationCfg + +# Predefined rigid body property configurations for assembly tasks +# High iteration count for precision tasks (peg/hole insertion) +RIGID_BODY_PROPS_HIGH_PRECISION = sim_utils.RigidBodyPropertiesCfg( + disable_gravity=False, + max_depenetration_velocity=5.0, + linear_damping=0.0, + angular_damping=0.0, + max_linear_velocity=1000.0, + max_angular_velocity=3666.0, + enable_gyroscopic_forces=True, + solver_position_iteration_count=192, + solver_velocity_iteration_count=1, + max_contact_impulse=1e32, +) + +# Standard iteration count for gear mesh tasks +RIGID_BODY_PROPS_MEDIUM_PRECISION = sim_utils.RigidBodyPropertiesCfg( + disable_gravity=False, + max_depenetration_velocity=5.0, + linear_damping=0.0, + angular_damping=0.0, + max_linear_velocity=1000.0, + max_angular_velocity=3666.0, + enable_gyroscopic_forces=True, + solver_position_iteration_count=32, + solver_velocity_iteration_count=32, + max_contact_impulse=1e32, +) + +# Initial state configuration for articulations without joints (e.g., rigid bodies treated as articulations). +# We explicitly set joint_pos and joint_vel to empty dicts to avoid the default pattern {".*": 0.0} in ArticulationCfg.InitialStateCfg, +# which would fail to match when there are no joints in the articulation. +EMPTY_ARTICULATION_INIT_STATE_CFG = ArticulationCfg.InitialStateCfg( + joint_pos={}, + joint_vel={}, +) diff --git a/isaaclab_arena/assets/object_utils.py b/isaaclab_arena/assets/object_utils.py index 0b0d2c289..84445f794 100644 --- a/isaaclab_arena/assets/object_utils.py +++ b/isaaclab_arena/assets/object_utils.py @@ -4,15 +4,28 @@ # SPDX-License-Identifier: Apache-2.0 -import isaaclab.sim as sim_utils -from isaaclab.assets import ArticulationCfg -from pxr import Usd +from typing import TYPE_CHECKING from isaaclab_arena.assets.object_base import ObjectType -from isaaclab_arena.utils.usd_helpers import get_prim_depth, is_articulation_root, is_rigid_body +from isaaclab_arena.assets.object_spawn_defaults import ( + EMPTY_ARTICULATION_INIT_STATE_CFG, + RIGID_BODY_PROPS_HIGH_PRECISION, + RIGID_BODY_PROPS_MEDIUM_PRECISION, +) + +if TYPE_CHECKING: + from pxr import Usd + +# Re-export spawn defaults for backward compatibility. +__all__ = [ + "EMPTY_ARTICULATION_INIT_STATE_CFG", + "RIGID_BODY_PROPS_HIGH_PRECISION", + "RIGID_BODY_PROPS_MEDIUM_PRECISION", + "detect_object_type", +] -def detect_object_type(usd_path: str | None = None, stage: Usd.Stage | None = None) -> ObjectType: +def detect_object_type(usd_path: str | None = None, stage: "Usd.Stage | None" = None) -> ObjectType: """Detect the object type of the asset Goes through the USD tree and detects the object type. The detection is based @@ -28,6 +41,10 @@ def detect_object_type(usd_path: str | None = None, stage: Usd.Stage | None = No Returns: The object type of the asset. """ + from pxr import Usd + + from isaaclab_arena.utils.usd_helpers import get_prim_depth, is_articulation_root, is_rigid_body + assert usd_path is not None or stage is not None, "Either usd_path or stage must be provided" assert usd_path is None or stage is None, "Either usd_path or stage must be provided" if usd_path is not None: @@ -62,41 +79,3 @@ def detect_object_type(usd_path: str | None = None, stage: Usd.Stage | None = No return ObjectType.ARTICULATION else: raise ValueError("This should not happen. There is an unknown USD type in the tree.") - - -# Predefined rigid body property configurations for assembly tasks -# High iteration count for precision tasks (peg/hole insertion) -RIGID_BODY_PROPS_HIGH_PRECISION = sim_utils.RigidBodyPropertiesCfg( - disable_gravity=False, - max_depenetration_velocity=5.0, - linear_damping=0.0, - angular_damping=0.0, - max_linear_velocity=1000.0, - max_angular_velocity=3666.0, - enable_gyroscopic_forces=True, - solver_position_iteration_count=192, - solver_velocity_iteration_count=1, - max_contact_impulse=1e32, -) - -# Standard iteration count for gear mesh tasks -RIGID_BODY_PROPS_MEDIUM_PRECISION = sim_utils.RigidBodyPropertiesCfg( - disable_gravity=False, - max_depenetration_velocity=5.0, - linear_damping=0.0, - angular_damping=0.0, - max_linear_velocity=1000.0, - max_angular_velocity=3666.0, - enable_gyroscopic_forces=True, - solver_position_iteration_count=32, - solver_velocity_iteration_count=32, - max_contact_impulse=1e32, -) - -# Initial state configuration for articulations without joints (e.g., rigid bodies treated as articulations). -# We explicitly set joint_pos and joint_vel to empty dicts to avoid the default pattern {".*": 0.0} in ArticulationCfg.InitialStateCfg, -# which would fail to match when there are no joints in the articulation. -EMPTY_ARTICULATION_INIT_STATE_CFG = ArticulationCfg.InitialStateCfg( - joint_pos={}, - joint_vel={}, -) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/server.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/server.py index 4212cf6a7..92b0fa58f 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/server.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/server.py @@ -11,6 +11,9 @@ /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server \\ --yaml isaaclab_arena/tests/test_data/pick_and_place_maple_table_init_env_graph.yaml + # Prompt-only (empty editor until you generate or paste YAML): + /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server + # Custom port: /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server \\ --yaml --port 8600 @@ -33,8 +36,8 @@ def main() -> None: parser.add_argument( "--yaml", type=Path, - required=True, - help="Path to an ArenaEnvInitialGraphSpec YAML file.", + default=None, + help="Optional ArenaEnvInitialGraphSpec YAML to open in the editor.", ) parser.add_argument( "--port", @@ -46,8 +49,8 @@ def main() -> None: serve_live_editor(args.yaml, port=args.port) -def serve_live_editor(yaml_path: Path, port: int = 8501) -> None: - """Spawn ``streamlit run streamlit_ui.py -- --yaml `` and wait.""" +def serve_live_editor(yaml_path: Path | None, port: int = 8501) -> None: + """Spawn ``streamlit run streamlit_ui.py`` and wait.""" app_path = Path(__file__).with_name("streamlit_ui.py") if not app_path.exists(): raise FileNotFoundError(f"Streamlit app not found at {app_path} — installation is incomplete.") @@ -65,9 +68,9 @@ def serve_live_editor(yaml_path: Path, port: int = 8501) -> None: "--server.fileWatcherType", "none", "--", - "--yaml", - str(yaml_path.resolve()), ] + if yaml_path is not None: + cmd.extend(["--yaml", str(yaml_path.resolve())]) print(f"[review_gui] launching Streamlit live editor: {' '.join(cmd)}", file=sys.stderr) try: diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py index 95045d69c..5d40a8f05 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py @@ -43,6 +43,16 @@ (paths are absolute filesystem paths on the disk cache. The PNGs themselves stay on disk — the parent reads them itself.) + {"cmd": "build_catalogues"} + → {"ok": true, "asset_catalogue": {...}, "relation_catalogue": {...}, + "task_catalogue": {...}} + (registry vocabulary for :meth:`EnvironmentGenerationAgent.fetch_intent_from_prompt`) + + {"cmd": "compile_intent", "intent_dict": {...}} + → {"ok": true, "spec_dict": {...}, "has_resolution_errors": bool, + "trace": [{"stage": "...", "query": "...", ...}]} + (validates :class:`EnvironmentIntentSpec` and compiles to initial graph spec) + {"cmd": "shutdown"} → {"ok": true} # sidecar exits cleanly after replying @@ -141,6 +151,14 @@ def _serve() -> int: _send(_handle_render_spec(app, req, _render_thumbnails_with_app, ArenaEnvInitialGraphSpec)) continue + if cmd == "build_catalogues": + _send(_handle_build_catalogues()) + continue + + if cmd == "compile_intent": + _send(_handle_compile_intent(req)) + continue + _send({"ok": False, "error": f"unknown cmd: {cmd!r}"}) return 0 @@ -173,6 +191,64 @@ def _handle_validate_spec(req: dict[str, Any], spec_cls) -> dict[str, Any]: return {"ok": True, "spec_dict": spec.to_dict()} +def _handle_build_catalogues() -> dict[str, Any]: + """Return asset/relation/task catalogues for the env-generation agent.""" + from dataclasses import asdict # noqa: PLC0415 + + from isaaclab_arena.agentic_environment_generation.environment_generation_agent import ( # noqa: PLC0415 + build_asset_catalogue, + build_relation_catalogue, + build_task_catalogue, + ) + + try: + asset_catalogue = build_asset_catalogue() + relation_catalogue = build_relation_catalogue() + task_catalogue = build_task_catalogue() + except Exception as exc: + return {"ok": False, "error": f"catalogue build failed: {exc}", "traceback": traceback.format_exc()} + + return { + "ok": True, + "asset_catalogue": asdict(asset_catalogue), + "relation_catalogue": { + "relations": [asdict(entry) for entry in relation_catalogue.relations], + }, + "task_catalogue": { + "tasks": [asdict(entry) for entry in task_catalogue.tasks], + }, + } + + +def _handle_compile_intent(req: dict[str, Any]) -> dict[str, Any]: + """Validate an EnvironmentIntentSpec and compile it to an initial graph spec.""" + from dataclasses import asdict # noqa: PLC0415 + + from isaaclab_arena.agentic_environment_generation.environment_intent_spec import ( # noqa: PLC0415 + EnvironmentIntentSpec, + ) + from isaaclab_arena.agentic_environment_generation.intent_compiler import IntentCompiler # noqa: PLC0415 + + intent_dict = req.get("intent_dict") + if not isinstance(intent_dict, dict): + return {"ok": False, "error": "compile_intent requires mapping 'intent_dict'"} + + try: + intent = EnvironmentIntentSpec.model_validate(intent_dict) + compiler = IntentCompiler() + spec = compiler.compile(intent) + except Exception as exc: + return {"ok": False, "error": f"intent compile failed: {exc}", "traceback": traceback.format_exc()} + + return { + "ok": True, + "spec_dict": spec.to_dict(), + "has_resolution_errors": compiler.has_resolution_errors, + "trace": [asdict(event) for event in compiler.trace], + "reasoning": intent.reasoning, + } + + def _handle_render_spec( app, req: dict[str, Any], diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py index 546921e3b..d54a01462 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py @@ -106,6 +106,22 @@ def validate_yaml_text(self, yaml_text: str) -> dict[str, Any]: with self._lock: return self._request({"cmd": "validate_spec", "yaml_text": yaml_text}) + def build_catalogues(self) -> dict[str, Any]: + """Build asset/relation/task catalogues from warm registries in the sidecar.""" + if not self.is_alive(): + raise SimAppSidecarError("SimApp sidecar is not running — start it first") + + with self._lock: + return self._request({"cmd": "build_catalogues"}) + + def compile_intent(self, intent_dict: dict[str, Any]) -> dict[str, Any]: + """Validate and compile an EnvironmentIntentSpec in the sidecar.""" + if not self.is_alive(): + raise SimAppSidecarError("SimApp sidecar is not running — start it first") + + with self._lock: + return self._request({"cmd": "compile_intent", "intent_dict": intent_dict}) + def render_spec(self, spec: ArenaEnvInitialGraphSpec) -> dict[str, bytes]: """Ask the sidecar to render thumbnails for ``spec``.""" if not self.is_alive(): diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 7d5b95f89..9eae8c3c1 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -10,9 +10,15 @@ /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server \\ --yaml isaaclab_arena/tests/test_data/pick_and_place_maple_table_init_env_graph.yaml + # Prompt-only (empty editor until you generate or paste YAML): + /isaac-sim/python.sh -m isaaclab_arena_examples.agentic_environment_generation.review_gui.server + Registry lookups (task kinds, relation kinds, asset USD paths) run in a persistent SimApp sidecar so YAML re-validation stays fast after the first ~30s sidecar boot. Thumbnails are rendered live by the same sidecar. + +Natural-language generation calls the LLM from Streamlit (``NV_API_KEY``) and +compiles the returned intent in the sidecar where registries are warm. """ from __future__ import annotations @@ -27,6 +33,16 @@ import streamlit as st +from isaaclab_arena.agentic_environment_generation.asset_matcher import ASSET_ERROR_STAGES +from isaaclab_arena.agentic_environment_generation.environment_generation_agent import ( + AssetCatalogue, + EnvironmentGenerationAgent, + RelationCatalogue, + RelationCatalogueEntry, + TaskCatalogue, + TaskCatalogueEntry, +) +from isaaclab_arena.agentic_environment_generation.intent_compiler import IntentCompiler from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.dashboard import render_dashboard_html from isaaclab_arena_examples.agentic_environment_generation.review_gui.simapp_sidecar_client import ( @@ -38,12 +54,19 @@ _SKIP_REGISTRY_CONTEXT: dict[str, Any] = {"skip_registry": True} +_DEFAULT_GENERATION_PROMPT = ( + "franka pick up avocado from the maple table and place it into a bowl on the table. " + "there are other veggies on the table as distractor" +) + _BROKEN_PLACEHOLDER_HTML = """

    No visualization yet — fix the YAML errors to auto-render.

    """ +_DEFAULT_SAVE_PATH = "isaaclab_arena_environments/llm_generated/generated_spec.yaml" + @st.cache_resource(show_spinner="Booting Isaac Sim sidecar (≈30s first run, cached afterwards)…") def _get_simapp_sidecar() -> SimAppSidecar | None: @@ -114,6 +137,9 @@ def is_valid(self) -> bool: def validate_yaml_text(text: str) -> ValidationResult: """Parse YAML and validate via the SimApp sidecar (registry lookups run there).""" + if not text.strip(): + return ValidationResult(spec=None, error=None) + try: raw = yaml.safe_load(text) except Exception: @@ -154,25 +180,210 @@ def validate_yaml_text(text: str) -> ValidationResult: return ValidationResult(spec=spec, error=None) +@dataclass +class CatalogueBundle: + """Asset/relation/task vocabulary fetched from the SimApp sidecar.""" + + asset_catalogue: AssetCatalogue + relation_catalogue: RelationCatalogue + task_catalogue: TaskCatalogue + + +def _catalogues_from_sidecar_response(response: dict[str, Any]) -> CatalogueBundle: + asset_raw = response["asset_catalogue"] + relation_raw = response["relation_catalogue"] + task_raw = response["task_catalogue"] + return CatalogueBundle( + asset_catalogue=AssetCatalogue( + embodiments=list(asset_raw["embodiments"]), + backgrounds=list(asset_raw["backgrounds"]), + objects=list(asset_raw["objects"]), + ), + relation_catalogue=RelationCatalogue( + relations=[RelationCatalogueEntry(**entry) for entry in relation_raw["relations"]], + ), + task_catalogue=TaskCatalogue( + tasks=[TaskCatalogueEntry(**entry) for entry in task_raw["tasks"]], + ), + ) + + +def _get_catalogue_bundle() -> CatalogueBundle | None: + """Return cached catalogues built in the sidecar (registry lookups run there).""" + cached = st.session_state.get("catalogue_bundle") + if cached is not None: + return cached + + sidecar = _ensure_sidecar() + if sidecar is None: + return None + + try: + response = sidecar.build_catalogues() + except SimAppSidecarError as exc: + st.session_state["catalogue_error"] = str(exc) + _get_simapp_sidecar.clear() + return None + + if not response.get("ok"): + st.session_state["catalogue_error"] = response.get("error", "catalogue build failed") + return None + + bundle = _catalogues_from_sidecar_response(response) + st.session_state["catalogue_bundle"] = bundle + st.session_state.pop("catalogue_error", None) + return bundle + + +def _get_generation_agent() -> EnvironmentGenerationAgent | None: + """Lazy-init the LLM agent when ``NV_API_KEY`` is available.""" + if st.session_state.get("generation_agent_error"): + return None + agent = st.session_state.get("generation_agent") + if agent is not None: + return agent + try: + agent = EnvironmentGenerationAgent() + except AssertionError as exc: + st.session_state["generation_agent_error"] = str(exc) + return None + except Exception as exc: + st.session_state["generation_agent_error"] = f"{type(exc).__name__}: {exc}" + return None + st.session_state["generation_agent"] = agent + st.session_state.pop("generation_agent_error", None) + return agent + + +def _format_trace_lines(trace: list[dict[str, Any]], *, errors_only: bool = False) -> str: + error_stages = ASSET_ERROR_STAGES | IntentCompiler._ERROR_TRACE_STAGES + lines: list[str] = [] + for event in trace: + stage = event.get("stage", "") + if errors_only and stage not in error_stages: + continue + chosen = event.get("chosen") + chosen_str = chosen if chosen is not None else "" + note = event.get("note") or "" + note_str = f" [{note}]" if note else "" + lines.append(f"{stage:34s} {event.get('query', ''):24s} -> {chosen_str}{note_str}") + return "\n".join(lines) + + +def _apply_generated_yaml(yaml_text: str) -> None: + """Push compiled spec YAML into the editor and force a re-render on the next pass.""" + st.session_state["edited_text"] = yaml_text + st.session_state["last_rendered_text"] = "" + st.session_state["editor_version"] = st.session_state.get("editor_version", 0) + 1 + + +def run_generation_pipeline(prompt: str) -> tuple[bool, str]: + """Call the LLM, compile intent in the sidecar, and load YAML into the editor.""" + prompt = prompt.strip() + if not prompt: + return False, "Enter a prompt describing the environment." + + agent = _get_generation_agent() + if agent is None: + err = st.session_state.get( + "generation_agent_error", + "Set NV_API_KEY in the environment before generating specs.", + ) + return False, err + + catalogues = _get_catalogue_bundle() + if catalogues is None: + err = st.session_state.get( + "catalogue_error", + "SimApp sidecar is unavailable — cannot build asset catalogues.", + ) + return False, err + + try: + intent_data, _raw = agent.fetch_intent_from_prompt( + prompt, + asset_catalog=catalogues.asset_catalogue, + relation_catalog=catalogues.relation_catalogue, + task_catalog=catalogues.task_catalogue, + ) + except Exception: + return False, traceback.format_exc() + + sidecar = _ensure_sidecar() + if sidecar is None: + return False, "SimApp sidecar is unavailable — cannot compile intent." + + try: + response = sidecar.compile_intent(intent_data) + except SimAppSidecarError as exc: + _get_simapp_sidecar.clear() + return False, str(exc) + + if not response.get("ok"): + err = response.get("error", "intent compile failed") + tb = response.get("traceback", "") + message = f"{err}\n\n{tb}" if tb else str(err) + return False, message + + try: + yaml_text = yaml.safe_dump(response["spec_dict"], sort_keys=False) + except Exception: + return False, traceback.format_exc() + + _apply_generated_yaml(yaml_text) + + reasoning = response.get("reasoning", "") + if reasoning: + st.session_state["last_generation_reasoning"] = reasoning + + trace = response.get("trace") or [] + if trace: + st.session_state["last_generation_trace"] = trace + + if response.get("has_resolution_errors"): + error_trace = _format_trace_lines(trace, errors_only=True) + return ( + True, + ( + "Spec generated with resolution warnings — review the trace below and edit the YAML as needed.\n\n" + f"{error_trace}" + ), + ) + + return True, "Spec generated and loaded into the YAML editor." + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--yaml", type=Path, - required=True, - help="Path to the ArenaEnvInitialGraphSpec YAML to open in the editor.", + default=None, + help="Optional path to an ArenaEnvInitialGraphSpec YAML to open in the editor.", ) return parser.parse_args() -def initialize_state(yaml_path: Path) -> None: +def initialize_state(yaml_path: Path | None) -> None: """Seed ``st.session_state`` from disk exactly once per session.""" - if st.session_state.get("_yaml_path") == str(yaml_path): + session_key = str(yaml_path.resolve()) if yaml_path is not None else "" + if st.session_state.get("_yaml_path") == session_key: + return + + st.session_state["_yaml_path"] = session_key + st.session_state.setdefault("generation_prompt", _DEFAULT_GENERATION_PROMPT) + st.session_state.setdefault("editor_version", 0) + + if yaml_path is None: + st.session_state["original_text"] = "" + st.session_state["edited_text"] = "" + st.session_state["last_rendered_text"] = "" + st.session_state["rendered_html"] = "" + st.session_state["save_path"] = _DEFAULT_SAVE_PATH return original_text = yaml_path.read_text(encoding="utf-8") - st.session_state["_yaml_path"] = str(yaml_path) st.session_state["original_text"] = original_text st.session_state["edited_text"] = original_text st.session_state["last_rendered_text"] = original_text @@ -186,6 +397,8 @@ def initialize_state(yaml_path: Path) -> None: def render_validation_badge(validation: ValidationResult) -> None: + if validation.spec is None and validation.error is None: + return if validation.is_valid: spec = validation.spec st.success( @@ -200,15 +413,18 @@ def render_validation_badge(validation: ValidationResult) -> None: def render_save_button(validation: ValidationResult) -> None: can_save = validation.is_valid save_path_str = st.session_state["save_path"] + save_label = f"Save to {Path(save_path_str).name}" if save_path_str else "Save YAML" if st.button( - f"Save to {Path(save_path_str).name}", + save_label, disabled=not can_save, use_container_width=True, help=f"Writes the editor contents to {save_path_str}. Disabled while YAML is invalid.", ): try: - Path(save_path_str).write_text(st.session_state["edited_text"], encoding="utf-8") + out_path = Path(save_path_str) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(st.session_state["edited_text"], encoding="utf-8") st.session_state["original_text"] = st.session_state["edited_text"] st.toast(f"Saved → {save_path_str}", icon="💾") except OSError as exc: @@ -219,13 +435,13 @@ def render_save_button(validation: ValidationResult) -> None: "Save path", value=save_path_str, key="save_path_input", - help="Defaults to the YAML file passed via --yaml.", + help="Defaults to the YAML file passed via --yaml, or a generated-spec path when none was given.", ) if new_path and new_path != save_path_str: st.session_state["save_path"] = new_path -def render_editor_panel(yaml_path: Path) -> ValidationResult: +def render_editor_panel(yaml_path: Path | None) -> ValidationResult: try: from streamlit_ace import st_ace # noqa: PLC0415 except ImportError as exc: @@ -238,8 +454,12 @@ def render_editor_panel(yaml_path: Path) -> ValidationResult: st.stop() st.subheader("YAML editor") - st.caption(f"Source: `{yaml_path}`") + if yaml_path is not None: + st.caption(f"Source: `{yaml_path}`") + else: + st.caption("No file loaded — generate a spec or paste YAML.") + editor_key = str(yaml_path) if yaml_path is not None else "new" new_text = st_ace( value=st.session_state["edited_text"], language="yaml", @@ -252,7 +472,7 @@ def render_editor_panel(yaml_path: Path) -> ValidationResult: wrap=False, auto_update=False, min_lines=30, - key=f"ace_editor::{yaml_path}", + key=f"ace_editor::{editor_key}::{st.session_state.get('editor_version', 0)}", ) if new_text is not None: st.session_state["edited_text"] = new_text @@ -271,8 +491,52 @@ def render_editor_panel(yaml_path: Path) -> ValidationResult: return validation +def render_generation_panel() -> None: + """Prompt input and generate-spec controls (top of the left column).""" + st.subheader("Generate from prompt") + st.caption("Calls the env-generation agent (LLM) then compiles intent in the SimApp sidecar.") + + prompt = st.text_area( + "Prompt", + value=st.session_state.get("generation_prompt", _DEFAULT_GENERATION_PROMPT), + height=120, + placeholder="Describe the robot task, scene, objects, and distractors…", + ) + st.session_state["generation_prompt"] = prompt + + agent_error = st.session_state.get("generation_agent_error") + if agent_error: + st.info(f"LLM agent unavailable: {agent_error}", icon="ℹ️") + + if st.button("Generate & compile", type="primary", use_container_width=True): + with st.spinner("Generating spec (LLM call + sidecar compile)…"): + ok, message = run_generation_pipeline(st.session_state["generation_prompt"]) + if ok: + if "resolution warnings" in message: + st.warning(message, icon="⚠️") + else: + st.success(message, icon="✅") + st.rerun() + else: + st.error(f"Generation failed\n\n```\n{message}\n```", icon="🛑") + + reasoning = st.session_state.get("last_generation_reasoning") + if reasoning: + with st.expander("Agent reasoning (last run)", expanded=False): + st.markdown(reasoning) + + trace = st.session_state.get("last_generation_trace") + if trace: + with st.expander("Resolution trace (last run)", expanded=False): + st.code(_format_trace_lines(trace), language=None) + + def render_visualization_panel() -> None: st.subheader("Visualization") + if not st.session_state.get("last_rendered_text", "").strip(): + st.caption("Generate or enter valid YAML to see the visualization.") + return + st.caption("Updates automatically when the YAML is valid.") st.components.v1.html( @@ -290,8 +554,8 @@ def main() -> None: ) args = parse_args() - yaml_path = args.yaml.resolve() - if not yaml_path.exists(): + yaml_path = args.yaml.resolve() if args.yaml is not None else None + if yaml_path is not None and not yaml_path.exists(): st.error(f"YAML file not found: {yaml_path}", icon="🛑") st.stop() @@ -300,6 +564,7 @@ def main() -> None: st.markdown("### ArenaEnvInitialGraphSpec live editor") left, right = st.columns([2, 3], gap="large") with left: + render_generation_panel() render_editor_panel(yaml_path) with right: render_visualization_panel() From 8aee26c70800649904fb43a8629daea83a19d04e Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 09:47:33 -0700 Subject: [PATCH 10/19] Reorder review GUI dashboard: nodes, graph with unary sidebar, tasks. Stack visualizer sections vertically with three node thumbnails per row and unary constraints beside the Mermaid spatial graph instead of below it. Signed-off-by: Qian Lin --- .../review_gui/render/dashboard.py | 18 ++++++++++----- .../review_gui/render/mermaid_graph.py | 2 +- .../review_gui/render/panels.py | 8 +++---- .../review_gui/render/styles.py | 22 +++++++++---------- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py index 731b3b781..ee08bd731 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/dashboard.py @@ -35,19 +35,25 @@ def render_dashboard_html(spec: ArenaEnvInitialGraphSpec, thumbnails: dict[str,

    {len(spec.nodes)} nodes · {len(spec.tasks)} tasks · initial state: {html_lib.escape(initial_state.id)}

    +
    +

    Nodes

    +
    {render_node_cards(spec, thumbnails)}
    +

    Spatial graph (initial state: {html_lib.escape(initial_state.id)})

    -
    {render_mermaid_graph(spec, initial_state)}
    - {render_unary_constraints(initial_state)} +
    +
    +
    {render_mermaid_graph(spec, initial_state)}
    +
    + +

    Tasks

    {render_tasks_table(spec)}
    -
    -

    Nodes

    -
    {render_node_cards(spec, thumbnails)}
    -
    diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/mermaid_graph.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/mermaid_graph.py index 21b57ce48..7ec9c3fb0 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/mermaid_graph.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/mermaid_graph.py @@ -20,7 +20,7 @@ def render_mermaid_graph(spec: ArenaEnvInitialGraphSpec, state: ArenaEnvGraphSta subject -->|kind| reference Unary spatial constraints (no reference) are omitted from the graph and - listed below it by :func:`render_unary_constraints` so their params are + listed to its right by :func:`render_unary_constraints` so their params are visible. Task constraints with a child are drawn as dashed edges: diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py index 1f3675b69..8eeeae19e 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/panels.py @@ -14,7 +14,7 @@ def render_unary_constraints(state: ArenaEnvGraphStateSpec) -> str: - """List constraints without a reference below the graph (anchors, position_limits, ...).""" + """List constraints without a reference beside the spatial graph.""" rows = [] for constraint in state.spatial_constraints: if constraint.reference is not None: @@ -30,10 +30,10 @@ def render_unary_constraints(state: ArenaEnvGraphStateSpec) -> str: f" on {html_lib.escape(constraint.subject)}{params}" ) if not rows: - return "" + return '

    No unary constraints.

    ' return ( - f'
    Unary constraints ({len(rows)})' - f'
      {"".join(rows)}
    ' + f'

    Unary constraints ({len(rows)})

    ' + f'
      {"".join(rows)}
    ' ) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/styles.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/styles.py index 279ad349d..b2638549c 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/styles.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/render/styles.py @@ -19,14 +19,18 @@ header { margin-bottom: 16px; } header h1 { margin: 0; font-size: 28px; font-weight: 700; } header .sub { margin: 4px 0 0; color: var(--fg-muted); font-size: 13px; } -main { display: grid; grid-template-columns: 1fr 1fr; grid-template-rows: auto auto; - grid-template-areas: "graph nodes" "tasks nodes"; gap: 16px; } -.graph-panel { grid-area: graph; } -.tasks-panel { grid-area: tasks; } -.nodes-panel { grid-area: nodes; } +main { display: flex; flex-direction: column; gap: 16px; } .panel { background: var(--bg-elev); border: 1px solid var(--border); border-radius: 8px; padding: 16px; } .panel h2 { margin: 0 0 12px; font-size: 16px; font-weight: 600; letter-spacing: 0.02em; } .panel h2 .muted { color: var(--fg-muted); font-weight: 400; font-size: 13px; } +.graph-row { display: grid; grid-template-columns: minmax(0, 1fr) minmax(220px, 300px); gap: 16px; align-items: start; } +.graph-mermaid { min-width: 0; } +.graph-unary { background: var(--bg-elev2); border: 1px solid var(--border); border-radius: 6px; padding: 12px; } +.unary-heading { margin: 0 0 10px; font-size: 13px; font-weight: 600; letter-spacing: 0.02em; } +.unary-heading .muted { font-weight: 400; } +.unary-list { margin: 0; padding-left: 18px; list-style: disc; color: var(--fg); } +.unary-list li { padding: 4px 0; font-size: 12px; } +.unary-empty { margin: 0; font-size: 12px; } code { font-family: ui-monospace, 'SF Mono', Menlo, monospace; font-size: 12px; background: var(--bg-elev2); padding: 1px 6px; border-radius: 4px; } pre { font-family: ui-monospace, 'SF Mono', Menlo, monospace; font-size: 12px; @@ -44,17 +48,13 @@ .badge.type-position_limits, .badge.type-at_pose, .badge.type-at_position { background: #6b3a7a; } .badge.type-task { background: #2f343d; border: 1px solid #4a5; color: var(--accent); } .mermaid { background: var(--bg-elev2); padding: 8px; border-radius: 6px; min-height: 220px; - display: flex; align-items: center; justify-content: center; } -.unary { margin-top: 12px; } -.unary summary { cursor: pointer; color: var(--fg-muted); font-size: 13px; padding: 4px 0; } -.unary ul { margin: 8px 0 0; padding-left: 20px; list-style: disc; color: var(--fg); } -.unary li { padding: 3px 0; } + display: flex; align-items: center; justify-content: center; margin: 0; } table.tasks { width: 100%; border-collapse: collapse; } table.tasks th, table.tasks td { text-align: left; padding: 8px 10px; border-bottom: 1px solid var(--border); vertical-align: top; font-size: 12px; } table.tasks th { color: var(--fg-muted); font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; } table.tasks pre { padding: 6px 8px; font-size: 11px; } -.node-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 12px; } +.node-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; } .node-card { background: var(--bg-elev2); border: 1px solid var(--border); border-radius: 8px; padding: 12px; display: flex; flex-direction: column; gap: 10px; } .node-card .thumb { aspect-ratio: 1 / 1; background: linear-gradient(135deg, #2a2f37, #1c2026); From 8bd64e80c7115fec28f254f86c6e57804f433328 Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 10:02:17 -0700 Subject: [PATCH 11/19] Add 16-env sim preview rollout to the review GUI sidecar. Link YAML to arena env, solve relations, run 50 zero-action steps, and show overview captures after reset and rollout in the visualization panel. Signed-off-by: Qian Lin --- .../review_gui/sim_preview.py | 148 ++++++++++++++++++ .../review_gui/simapp_sidecar.py | 25 +++ .../review_gui/simapp_sidecar_client.py | 8 + .../review_gui/streamlit_ui.py | 99 ++++++++++-- 4 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py new file mode 100644 index 000000000..fa8e963fe --- /dev/null +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py @@ -0,0 +1,148 @@ +# Copyright (c) 2026, The Isaac Lab Arena Project Developers (https://github.com/isaac-sim/IsaacLab-Arena/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""16-env relation-solver rollout preview for the review GUI SimApp sidecar.""" + +from __future__ import annotations + +import argparse +import contextlib +import math +import sys +import time +import uuid +from pathlib import Path +from typing import Any + +from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec + +PREVIEW_CACHE_DIR = Path(__file__).resolve().parents[3] / ".cache" / "llm_env_gen_sim_preview" + +NUM_ENVS = 16 +ENV_SPACING_M = 1.5 +NUM_STEPS = 50 + + +def _preview_args() -> argparse.Namespace: + return argparse.Namespace( + num_envs=NUM_ENVS, + env_spacing=ENV_SPACING_M, + device="cuda:0", + disable_fabric=False, + solve_relations=True, + placement_seed=None, + resolve_on_reset=False, + random_yaw_init=False, + mimic=False, + distributed=False, + presets=None, + ) + + +def _overview_camera( + num_envs: int, env_spacing: float +) -> tuple[tuple[float, float, float], tuple[float, float, float]]: + """Return (eye, target) for a diagonal overview of the env grid.""" + cols = int(math.ceil(math.sqrt(num_envs))) + rows = int(math.ceil(num_envs / cols)) + span_x = max((cols - 1) * env_spacing, env_spacing) + span_y = max((rows - 1) * env_spacing, env_spacing) + cx, cy = span_x * 0.5, span_y * 0.5 + radius = max(span_x, span_y) + env_spacing + eye = (cx + radius * 0.2, cy - radius * 1.15, radius * 1.25) + target = (cx, cy, 0.35) + return eye, target + + +def _capture_viewport(app, cache_path: Path) -> bytes | None: + from omni.kit.viewport.utility import capture_viewport_to_file, get_active_viewport # noqa: PLC0415 + + from isaaclab_arena_examples.agentic_environment_generation.review_gui.thumbnail_render import ( # noqa: PLC0415 + _wait_for_capture, + ) + + viewport = get_active_viewport() + cache_path.parent.mkdir(parents=True, exist_ok=True) + capture_obj = capture_viewport_to_file(viewport, str(cache_path)) + for _ in range(10): + app.update() + _wait_for_capture(app, capture_obj, cache_path, max_updates=300) + if cache_path.exists() and cache_path.stat().st_size > 0: + return cache_path.read_bytes() + return None + + +def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: + """Link spec → arena env → relation solver → 50 zero-action steps; capture overview frames.""" + import yaml + + from isaaclab_arena.environments.arena_env_builder import ArenaEnvBuilder + from isaaclab_arena.policy.zero_action_policy import ZeroActionPolicy, ZeroActionPolicyArgs + from isaaclab_arena.utils.isaaclab_utils.simulation_app import reapply_viewer_cfg, teardown_simulation_app + + raw = yaml.safe_load(yaml_text) + if not isinstance(raw, dict): + raise ValueError(f"expected mapping, got {type(raw).__name__}") + + initial_spec = ArenaEnvInitialGraphSpec.model_validate(raw) + graph_spec = initial_spec.link() + arena_env = graph_spec.to_arena_env() + preview_name = f"{arena_env.name}_preview_{uuid.uuid4().hex[:8]}" + arena_env.name = preview_name + + args = _preview_args() + builder = ArenaEnvBuilder(arena_env, args) + policy = ZeroActionPolicy(ZeroActionPolicyArgs()) + + PREVIEW_CACHE_DIR.mkdir(parents=True, exist_ok=True) + stamp = int(time.time() * 1000) + first_path = PREVIEW_CACHE_DIR / f"{preview_name}_{stamp}_first.png" + last_path = PREVIEW_CACHE_DIR / f"{preview_name}_{stamp}_last.png" + + env = None + try: + env = builder.make_registered() + reapply_viewer_cfg(env) + + eye, target = _overview_camera(args.num_envs, args.env_spacing) + env.unwrapped.sim.set_camera_view(eye, target) + for _ in range(15): + app.update() + + obs, _ = env.reset() + for _ in range(10): + app.update() + + if _capture_viewport(app, first_path) is None: + raise RuntimeError("failed to capture first-frame overview screenshot") + + for _ in range(NUM_STEPS): + action = policy.get_action(env, obs) + obs, _, _, _, _ = env.step(action) + + for _ in range(10): + app.update() + + if _capture_viewport(app, last_path) is None: + raise RuntimeError("failed to capture last-frame overview screenshot") + + print( + f"[sim_preview] captured {NUM_ENVS} envs @ {ENV_SPACING_M}m spacing, {NUM_STEPS} zero-action steps", + file=sys.stderr, + ) + return { + "ok": True, + "first_frame": str(first_path), + "last_frame": str(last_path), + "env_name": preview_name, + "num_envs": args.num_envs, + "env_spacing": args.env_spacing, + "num_steps": NUM_STEPS, + } + finally: + if env is not None: + with contextlib.suppress(Exception): + env.close() + teardown_simulation_app(suppress_exceptions=True) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py index 5d40a8f05..fa2179e65 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py @@ -53,6 +53,11 @@ "trace": [{"stage": "...", "query": "...", ...}]} (validates :class:`EnvironmentIntentSpec` and compiles to initial graph spec) + {"cmd": "run_sim_preview", "yaml_text": "..."} + → {"ok": true, "first_frame": "/abs/first.png", "last_frame": "/abs/last.png", + "num_envs": 16, "env_spacing": 1.5, "num_steps": 50} + (link → to_arena_env → relation solver → 50 zero-action steps; overview captures) + {"cmd": "shutdown"} → {"ok": true} # sidecar exits cleanly after replying @@ -159,6 +164,10 @@ def _serve() -> int: _send(_handle_compile_intent(req)) continue + if cmd == "run_sim_preview": + _send(_handle_run_sim_preview(app, req)) + continue + _send({"ok": False, "error": f"unknown cmd: {cmd!r}"}) return 0 @@ -249,6 +258,22 @@ def _handle_compile_intent(req: dict[str, Any]) -> dict[str, Any]: } +def _handle_run_sim_preview(app, req: dict[str, Any]) -> dict[str, Any]: + """Build linked env, solve relations, roll out zero actions, capture overview frames.""" + from isaaclab_arena_examples.agentic_environment_generation.review_gui.sim_preview import ( # noqa: PLC0415 + run_sim_preview, + ) + + yaml_text = req.get("yaml_text") + if not isinstance(yaml_text, str): + return {"ok": False, "error": "run_sim_preview requires string 'yaml_text'"} + + try: + return run_sim_preview(app, yaml_text) + except Exception as exc: + return {"ok": False, "error": f"sim preview failed: {exc}", "traceback": traceback.format_exc()} + + def _handle_render_spec( app, req: dict[str, Any], diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py index d54a01462..b7758ed33 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar_client.py @@ -122,6 +122,14 @@ def compile_intent(self, intent_dict: dict[str, Any]) -> dict[str, Any]: with self._lock: return self._request({"cmd": "compile_intent", "intent_dict": intent_dict}) + def run_sim_preview(self, yaml_text: str) -> dict[str, Any]: + """Link, build env, solve relations, and capture overview rollout frames.""" + if not self.is_alive(): + raise SimAppSidecarError("SimApp sidecar is not running — start it first") + + with self._lock: + return self._request({"cmd": "run_sim_preview", "yaml_text": yaml_text}) + def render_spec(self, spec: ArenaEnvInitialGraphSpec) -> dict[str, bytes]: """Ask the sidecar to render thumbnails for ``spec``.""" if not self.is_alive(): diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 9eae8c3c1..4e3f04b56 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -45,6 +45,11 @@ from isaaclab_arena.agentic_environment_generation.intent_compiler import IntentCompiler from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec from isaaclab_arena_examples.agentic_environment_generation.review_gui.render.dashboard import render_dashboard_html +from isaaclab_arena_examples.agentic_environment_generation.review_gui.sim_preview import ( + ENV_SPACING_M, + NUM_ENVS, + NUM_STEPS, +) from isaaclab_arena_examples.agentic_environment_generation.review_gui.simapp_sidecar_client import ( SimAppSidecar, SimAppSidecarError, @@ -353,6 +358,46 @@ def run_generation_pipeline(prompt: str) -> tuple[bool, str]: return True, "Spec generated and loaded into the YAML editor." +def run_sim_preview_pipeline(yaml_text: str) -> tuple[bool, str]: + """Link, build, solve relations, and capture overview frames in the sidecar.""" + validation = validate_yaml_text(yaml_text) + if not validation.is_valid: + return False, validation.error or "YAML must be valid before running sim preview." + + sidecar = _ensure_sidecar() + if sidecar is None: + return False, "SimApp sidecar is unavailable — cannot run sim preview." + + try: + response = sidecar.run_sim_preview(yaml_text) + except SimAppSidecarError as exc: + _get_simapp_sidecar.clear() + return False, str(exc) + + if not response.get("ok"): + err = response.get("error", "sim preview failed") + tb = response.get("traceback", "") + message = f"{err}\n\n{tb}" if tb else str(err) + return False, message + + try: + first_path = Path(response["first_frame"]) + last_path = Path(response["last_frame"]) + st.session_state["sim_preview_first"] = first_path.read_bytes() + st.session_state["sim_preview_last"] = last_path.read_bytes() + except OSError as exc: + return False, f"Failed to read preview frames: {exc}" + + return ( + True, + ( + f"Sim preview complete — {response.get('num_envs', NUM_ENVS)} envs, " + f"{response.get('env_spacing', ENV_SPACING_M)} m spacing, " + f"{response.get('num_steps', NUM_STEPS)} steps." + ), + ) + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -531,20 +576,54 @@ def render_generation_panel() -> None: st.code(_format_trace_lines(trace), language=None) -def render_visualization_panel() -> None: +def render_visualization_panel(validation: ValidationResult) -> None: st.subheader("Visualization") if not st.session_state.get("last_rendered_text", "").strip(): st.caption("Generate or enter valid YAML to see the visualization.") - return - - st.caption("Updates automatically when the YAML is valid.") + else: + st.caption("Updates automatically when the YAML is valid.") + st.components.v1.html( + st.session_state["rendered_html"], + height=_IFRAME_HEIGHT_PX, + scrolling=True, + ) - st.components.v1.html( - st.session_state["rendered_html"], - height=_IFRAME_HEIGHT_PX, - scrolling=True, + st.divider() + st.subheader("Sim preview") + st.caption( + f"Runs link → to_arena_env → relation solver, then {NUM_STEPS} zero-action steps " + f"with {NUM_ENVS} parallel envs at {ENV_SPACING_M} m spacing. " + "Overview captures are taken after reset and after the rollout." ) + if st.button( + "Run link + relation solver preview", + type="secondary", + use_container_width=True, + disabled=not validation.is_valid, + help="Requires valid YAML and a healthy SimApp sidecar. This may take several minutes.", + ): + with st.spinner( + f"Building env, solving relations, and rolling out {NUM_STEPS} steps ({NUM_ENVS} envs @ {ENV_SPACING_M} m)…" + ): + ok, message = run_sim_preview_pipeline(st.session_state["edited_text"]) + if ok: + st.success(message, icon="✅") + st.rerun() + else: + st.error(f"Sim preview failed\n\n```\n{message}\n```", icon="🛑") + + first_frame = st.session_state.get("sim_preview_first") + last_frame = st.session_state.get("sim_preview_last") + if first_frame and last_frame: + frame_cols = st.columns(2) + with frame_cols[0]: + st.caption("Overview — frame 1 (after reset)") + st.image(first_frame, use_container_width=True) + with frame_cols[1]: + st.caption(f"Overview — frame 2 (after {NUM_STEPS} zero-action steps)") + st.image(last_frame, use_container_width=True) + def main() -> None: st.set_page_config( @@ -565,9 +644,9 @@ def main() -> None: left, right = st.columns([2, 3], gap="large") with left: render_generation_panel() - render_editor_panel(yaml_path) + validation = render_editor_panel(yaml_path) with right: - render_visualization_panel() + render_visualization_panel(validation) main() From ab6f54a84a1d6c16d871ad06bd5619ed6779841e Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 10:09:23 -0700 Subject: [PATCH 12/19] Fix viewer look-at when relation solver sets PosePerEnv poses. Multi-env sim preview failed because get_viewer_cfg_look_at_object only handled Pose and PoseRange; add as_single_pose and use it at call sites. Signed-off-by: Qian Lin --- isaaclab_arena/tasks/lift_object_task.py | 10 +++------- isaaclab_arena/tests/test_pose.py | 21 ++++++++++++++++++++- isaaclab_arena/utils/cameras.py | 5 ++--- isaaclab_arena/utils/pose.py | 12 ++++++++++++ 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/isaaclab_arena/tasks/lift_object_task.py b/isaaclab_arena/tasks/lift_object_task.py index 4c4461fd7..100c6f734 100644 --- a/isaaclab_arena/tasks/lift_object_task.py +++ b/isaaclab_arena/tasks/lift_object_task.py @@ -26,7 +26,7 @@ from isaaclab_arena.tasks.task_base import TaskBase from isaaclab_arena.tasks.terminations import lift_object_il_success, lift_object_rl_success from isaaclab_arena.utils.cameras import get_viewer_cfg_look_at_object -from isaaclab_arena.utils.pose import PoseRange +from isaaclab_arena.utils.pose import as_single_pose @register_task @@ -54,9 +54,7 @@ def __init__( self.background_scene = background_scene # Compute goal position from object's initial pose + delta - initial_pose = lift_object.get_initial_pose() - if isinstance(initial_pose, PoseRange): - initial_pose = initial_pose.get_midpoint() + initial_pose = as_single_pose(lift_object.get_initial_pose()) # Store goal pose for success termination (IL/teleoperation uses fixed goal) self.goal_position_xyz = ( @@ -163,9 +161,7 @@ def __init__( self.minimum_height_to_lift = minimum_height_to_lift # Get object's initial pose to compute absolute target ranges - initial_pose = lift_object.get_initial_pose() - if isinstance(initial_pose, PoseRange): - initial_pose = initial_pose.get_midpoint() + initial_pose = as_single_pose(lift_object.get_initial_pose()) # Compute absolute target ranges from deltas self.target_x_range = ( diff --git a/isaaclab_arena/tests/test_pose.py b/isaaclab_arena/tests/test_pose.py index bba6405b1..e5b0cbe41 100644 --- a/isaaclab_arena/tests/test_pose.py +++ b/isaaclab_arena/tests/test_pose.py @@ -5,7 +5,7 @@ import math -from isaaclab_arena.utils.pose import Pose, PosePerEnv, rotate_quat_by_yaw, wrap_angle_to_pi +from isaaclab_arena.utils.pose import Pose, PosePerEnv, PoseRange, as_single_pose, rotate_quat_by_yaw, wrap_angle_to_pi def _yaw_of(quat_xyzw: tuple[float, float, float, float]) -> float: @@ -40,6 +40,25 @@ def test_pose_composition(): assert T_C_A.rotation_xyzw == (0.0, 0.0, 0.0, 1.0) +def test_as_single_pose(): + pose = Pose(position_xyz=(1.0, 2.0, 3.0)) + assert as_single_pose(pose) is pose + + pose_range = PoseRange( + position_xyz_min=(0.0, 0.0, 0.0), + position_xyz_max=(2.0, 4.0, 6.0), + ) + assert as_single_pose(pose_range).position_xyz == (1.0, 2.0, 3.0) + + pose_per_env = PosePerEnv( + poses=[ + Pose(position_xyz=(1.0, 2.0, 3.0)), + Pose(position_xyz=(4.0, 5.0, 6.0)), + ] + ) + assert as_single_pose(pose_per_env).position_xyz == (1.0, 2.0, 3.0) + + def test_pose_per_env_stores_poses(): """Test that PosePerEnv stores the list of Pose objects correctly.""" poses = [ diff --git a/isaaclab_arena/utils/cameras.py b/isaaclab_arena/utils/cameras.py index 4567bb7e0..ddbb4c9e9 100644 --- a/isaaclab_arena/utils/cameras.py +++ b/isaaclab_arena/utils/cameras.py @@ -18,7 +18,7 @@ from isaaclab_arena.assets.asset import Asset from isaaclab_arena.utils.configclass import make_configclass -from isaaclab_arena.utils.pose import PoseRange +from isaaclab_arena.utils.pose import as_single_pose def make_camera_observation_cfg( @@ -114,8 +114,7 @@ def get_viewer_cfg_look_at_object(lookat_object: Asset, offset: np.ndarray) -> V print(f"{lookat_object.name} has no initial pose set. Using default ViewerCfg.") return ViewerCfg() - if isinstance(initial_pose, PoseRange): - initial_pose = initial_pose.get_midpoint() + initial_pose = as_single_pose(initial_pose) # TODO(cvolk): Add float coercion to Pose.__post_init__ so this conversion is unnecessary. # Ensure we only pass primitive Python floats (not NumPy scalars) into ViewerCfg, diff --git a/isaaclab_arena/utils/pose.py b/isaaclab_arena/utils/pose.py index ecc9289d6..3eb869df4 100644 --- a/isaaclab_arena/utils/pose.py +++ b/isaaclab_arena/utils/pose.py @@ -144,3 +144,15 @@ def get_midpoint(self) -> Pose: position_xyz=tuple(position_xyz.tolist()), rotation_xyzw=tuple(quat.tolist()), ) + + +def as_single_pose(pose: Pose | PoseRange | PosePerEnv) -> Pose: + """Return one representative ``Pose`` for viewer and bbox code that needs a single point. + + ``PosePerEnv`` uses env 0; ``PoseRange`` uses its midpoint. + """ + if isinstance(pose, PosePerEnv): + return pose.poses[0] + if isinstance(pose, PoseRange): + return pose.get_midpoint() + return pose From adab971aa3cfe8ee89db2d34a8acc4e3ae15cb9d Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Mon, 15 Jun 2026 10:17:02 -0700 Subject: [PATCH 13/19] Tune sim preview: 10 steps and default scene viewport camera. Drop the custom grid overview camera and task look-at-object offsets; use Isaac Lab ViewerCfg like policy_runner and shorten the rollout. Signed-off-by: Qian Lin --- .../review_gui/sim_preview.py | 39 ++++++------------- .../review_gui/simapp_sidecar.py | 4 +- .../review_gui/streamlit_ui.py | 6 +-- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py index fa8e963fe..a2b73b2d3 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py @@ -9,20 +9,21 @@ import argparse import contextlib -import math import sys import time import uuid from pathlib import Path from typing import Any +from isaaclab.envs.common import ViewerCfg + from isaaclab_arena.environments.arena_env_graph_spec import ArenaEnvInitialGraphSpec PREVIEW_CACHE_DIR = Path(__file__).resolve().parents[3] / ".cache" / "llm_env_gen_sim_preview" NUM_ENVS = 16 ENV_SPACING_M = 1.5 -NUM_STEPS = 50 +NUM_STEPS = 10 def _preview_args() -> argparse.Namespace: @@ -41,21 +42,6 @@ def _preview_args() -> argparse.Namespace: ) -def _overview_camera( - num_envs: int, env_spacing: float -) -> tuple[tuple[float, float, float], tuple[float, float, float]]: - """Return (eye, target) for a diagonal overview of the env grid.""" - cols = int(math.ceil(math.sqrt(num_envs))) - rows = int(math.ceil(num_envs / cols)) - span_x = max((cols - 1) * env_spacing, env_spacing) - span_y = max((rows - 1) * env_spacing, env_spacing) - cx, cy = span_x * 0.5, span_y * 0.5 - radius = max(span_x, span_y) + env_spacing - eye = (cx + radius * 0.2, cy - radius * 1.15, radius * 1.25) - target = (cx, cy, 0.35) - return eye, target - - def _capture_viewport(app, cache_path: Path) -> bytes | None: from omni.kit.viewport.utility import capture_viewport_to_file, get_active_viewport # noqa: PLC0415 @@ -75,12 +61,12 @@ def _capture_viewport(app, cache_path: Path) -> bytes | None: def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: - """Link spec → arena env → relation solver → 50 zero-action steps; capture overview frames.""" + """Link spec → arena env → relation solver → zero-action steps; capture viewport frames.""" import yaml from isaaclab_arena.environments.arena_env_builder import ArenaEnvBuilder from isaaclab_arena.policy.zero_action_policy import ZeroActionPolicy, ZeroActionPolicyArgs - from isaaclab_arena.utils.isaaclab_utils.simulation_app import reapply_viewer_cfg, teardown_simulation_app + from isaaclab_arena.utils.isaaclab_utils.simulation_app import teardown_simulation_app raw = yaml.safe_load(yaml_text) if not isinstance(raw, dict): @@ -103,20 +89,17 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: env = None try: - env = builder.make_registered() - reapply_viewer_cfg(env) - - eye, target = _overview_camera(args.num_envs, args.env_spacing) - env.unwrapped.sim.set_camera_view(eye, target) - for _ in range(15): - app.update() + # Match policy_runner: default Isaac Lab ViewerCfg (not task look-at-object offsets). + env_cfg = builder.compose_manager_cfg() + env_cfg.viewer = ViewerCfg() + env = builder.make_registered(env_cfg) obs, _ = env.reset() for _ in range(10): app.update() if _capture_viewport(app, first_path) is None: - raise RuntimeError("failed to capture first-frame overview screenshot") + raise RuntimeError("failed to capture first-frame viewport screenshot") for _ in range(NUM_STEPS): action = policy.get_action(env, obs) @@ -126,7 +109,7 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: app.update() if _capture_viewport(app, last_path) is None: - raise RuntimeError("failed to capture last-frame overview screenshot") + raise RuntimeError("failed to capture last-frame viewport screenshot") print( f"[sim_preview] captured {NUM_ENVS} envs @ {ENV_SPACING_M}m spacing, {NUM_STEPS} zero-action steps", diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py index fa2179e65..81d874d04 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py @@ -55,8 +55,8 @@ {"cmd": "run_sim_preview", "yaml_text": "..."} → {"ok": true, "first_frame": "/abs/first.png", "last_frame": "/abs/last.png", - "num_envs": 16, "env_spacing": 1.5, "num_steps": 50} - (link → to_arena_env → relation solver → 50 zero-action steps; overview captures) + "num_envs": 16, "env_spacing": 1.5, "num_steps": 10} + (link → to_arena_env → relation solver → 10 zero-action steps; viewport captures) {"cmd": "shutdown"} → {"ok": true} # sidecar exits cleanly after replying diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 4e3f04b56..4d669d81f 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -593,7 +593,7 @@ def render_visualization_panel(validation: ValidationResult) -> None: st.caption( f"Runs link → to_arena_env → relation solver, then {NUM_STEPS} zero-action steps " f"with {NUM_ENVS} parallel envs at {ENV_SPACING_M} m spacing. " - "Overview captures are taken after reset and after the rollout." + "Viewport captures use the default scene camera (same as policy_runner)." ) if st.button( @@ -618,10 +618,10 @@ def render_visualization_panel(validation: ValidationResult) -> None: if first_frame and last_frame: frame_cols = st.columns(2) with frame_cols[0]: - st.caption("Overview — frame 1 (after reset)") + st.caption("Viewport — frame 1 (after reset)") st.image(first_frame, use_container_width=True) with frame_cols[1]: - st.caption(f"Overview — frame 2 (after {NUM_STEPS} zero-action steps)") + st.caption(f"Viewport — frame 2 (after {NUM_STEPS} zero-action steps)") st.image(last_frame, use_container_width=True) From 91ab7d182392642db70303784f11dd01d88a464b Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Tue, 16 Jun 2026 01:21:23 -0700 Subject: [PATCH 14/19] Fix repeat sim preview by sharing eval-runner env teardown. Extract close_env_and_reset_sim so the sidecar resets SimulationContext, closes the gym env, and clears CUDA cache between preview runs instead of swallowing partial cleanup errors. Signed-off-by: Qian Lin --- isaaclab_arena/evaluation/eval_runner.py | 27 +++++---------- .../utils/isaaclab_utils/simulation_app.py | 34 +++++++++++++++++++ .../review_gui/sim_preview.py | 8 ++--- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/isaaclab_arena/evaluation/eval_runner.py b/isaaclab_arena/evaluation/eval_runner.py index c1512700d..42a82169f 100644 --- a/isaaclab_arena/evaluation/eval_runner.py +++ b/isaaclab_arena/evaluation/eval_runner.py @@ -5,14 +5,12 @@ import argparse import dataclasses -import gc import json import math import os import subprocess import sys import tempfile -import torch import traceback from gymnasium.wrappers import RecordVideo from pathlib import Path @@ -24,7 +22,11 @@ from isaaclab_arena.evaluation.policy_runner import get_policy_cls, rollout_policy from isaaclab_arena.metrics.aggregate_metrics import aggregate_metrics from isaaclab_arena.metrics.metrics_logger import MetricsLogger -from isaaclab_arena.utils.isaaclab_utils.simulation_app import SimulationAppContext, teardown_simulation_app +from isaaclab_arena.utils.isaaclab_utils.simulation_app import ( + SimulationAppContext, + close_env_and_reset_sim, + collect_garbage_and_clear_cuda_cache, +) from isaaclab_arena_environments.cli import get_arena_builder_from_cli, get_isaaclab_arena_environments_cli_parser if TYPE_CHECKING: @@ -111,31 +113,18 @@ def get_policy_from_job(job: Job) -> "PolicyBase": return policy -def _collect_garbage_and_clear_cuda_cache() -> None: - gc.collect() - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - def _close_policy(policy: "PolicyBase | None") -> None: try: if policy is not None: policy.close() finally: - _collect_garbage_and_clear_cuda_cache() + collect_garbage_and_clear_cuda_cache() def _close_env(env) -> None: if env is None: return - try: - teardown_simulation_app(suppress_exceptions=False, make_new_stage=True) - finally: - try: - # cleanup managers, including recorder manager closing hdf5 file - env.close() - finally: - _collect_garbage_and_clear_cuda_cache() + close_env_and_reset_sim(env) def _close_job_resources(policy: "PolicyBase | None", env) -> None: @@ -326,7 +315,7 @@ def main(): finally: policy = None env = None - _collect_garbage_and_clear_cuda_cache() + collect_garbage_and_clear_cuda_cache() # Aggregate the metrics from the different experiments into a single view. if metrics_per_run: diff --git a/isaaclab_arena/utils/isaaclab_utils/simulation_app.py b/isaaclab_arena/utils/isaaclab_utils/simulation_app.py index 02db2a0f3..4593986e5 100644 --- a/isaaclab_arena/utils/isaaclab_utils/simulation_app.py +++ b/isaaclab_arena/utils/isaaclab_utils/simulation_app.py @@ -82,6 +82,40 @@ def teardown_simulation_app(suppress_exceptions: bool = False, make_new_stage: b omni.usd.get_context().new_stage() +def collect_garbage_and_clear_cuda_cache() -> None: + """Run GC and release cached CUDA allocations after a sim env is torn down.""" + import gc + import torch + + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + +def close_env_and_reset_sim( + env=None, + *, + suppress_exceptions: bool = False, + make_new_stage: bool = True, +) -> None: + """Tear down sim state and close a gym env so another can be built in the same SimApp. + + Order matches :func:`isaaclab_arena.evaluation.eval_runner._close_env`: stop + :class:`~isaaclab.sim.SimulationContext`, open a fresh USD stage, then close + env managers and collect GPU garbage. + """ + try: + teardown_simulation_app(suppress_exceptions=suppress_exceptions, make_new_stage=make_new_stage) + finally: + if env is not None: + if suppress_exceptions: + with suppress(Exception): + env.close() + else: + env.close() + collect_garbage_and_clear_cuda_cache() + + def reapply_viewer_cfg(env) -> None: """Re-apply ViewerCfg camera position after visualizers are initialized. diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py index a2b73b2d3..36cebe0cc 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py @@ -8,7 +8,6 @@ from __future__ import annotations import argparse -import contextlib import sys import time import uuid @@ -66,7 +65,7 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: from isaaclab_arena.environments.arena_env_builder import ArenaEnvBuilder from isaaclab_arena.policy.zero_action_policy import ZeroActionPolicy, ZeroActionPolicyArgs - from isaaclab_arena.utils.isaaclab_utils.simulation_app import teardown_simulation_app + from isaaclab_arena.utils.isaaclab_utils.simulation_app import close_env_and_reset_sim raw = yaml.safe_load(yaml_text) if not isinstance(raw, dict): @@ -125,7 +124,4 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: "num_steps": NUM_STEPS, } finally: - if env is not None: - with contextlib.suppress(Exception): - env.close() - teardown_simulation_app(suppress_exceptions=True) + close_env_and_reset_sim(env) From 8d4601d68ab4cc2d07ef5cfa0cc4a0eec22370f0 Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Tue, 16 Jun 2026 02:58:21 -0700 Subject: [PATCH 15/19] Run review GUI SimApp sidecar with Kit visualizer enabled. Boot the persistent sidecar with --viz kit so thumbnails and sim preview render in an interactive Kit viewport instead of headless mode. Signed-off-by: Qian Lin --- .../review_gui/simapp_sidecar.py | 2 +- .../review_gui/thumbnail_render.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py index 81d874d04..17b1f727a 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/simapp_sidecar.py @@ -5,7 +5,7 @@ """Long-lived ``SimulationApp`` host process for the live review editor. -Boots Kit's ``SimulationApp`` once on *its own* main thread and serves +Boots Kit's ``SimulationApp`` once (with ``--viz kit``) on *its own* main thread and serves validation and thumbnail-render requests over a newline-delimited JSON-RPC pipe on stdin/stdout. The parent (``streamlit_ui.py`` running inside Streamlit) spawns exactly one of these and reuses it for the entire server diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/thumbnail_render.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/thumbnail_render.py index d4ea06a9b..f63f17d4e 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/thumbnail_render.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/thumbnail_render.py @@ -80,8 +80,13 @@ def _render_thumbnails_with_app(app, spec: ArenaEnvInitialGraphSpec) -> dict[str return resolved +def _sidecar_launch_args() -> argparse.Namespace: + """AppLauncher args for the review GUI sidecar (Kit UI + viewport capture).""" + return argparse.Namespace(visualizer=["kit"], enable_cameras=True, livestream=-1) + + def _launch_simulation_app(): - """Boot Isaac Sim's ``SimulationApp`` for headless viewport capture, or ``None`` on failure. + """Boot Isaac Sim's ``SimulationApp`` with the Kit visualizer, or ``None`` on failure. Kept as a tiny helper so the call site can lazy-import inside this function — module-level import of ``simulation_app`` would drag Kit @@ -91,8 +96,7 @@ def _launch_simulation_app(): # Lazy-import: keeps the default ``review_graph`` invocation Kit-free. from isaaclab_arena.utils.isaaclab_utils.simulation_app import get_app_launcher # noqa: PLC0415 - sim_args = argparse.Namespace(headless=True, enable_cameras=True, hide_ui=True, livestream=-1) - return get_app_launcher(sim_args).app + return get_app_launcher(_sidecar_launch_args()).app except Exception as exc: print(f"[thumbnail_render] SimulationApp launch failed: {exc}", file=sys.stderr) return None From 973016102b4e82b331cf64ef8a2a63f9b1e3559b Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Tue, 16 Jun 2026 02:58:47 -0700 Subject: [PATCH 16/19] Frame sim preview captures on the full multi-env grid overview. Set a raised world-frame camera via ViewerCfg and re-apply it through the viewport controller before each capture so all 16 clones stay in frame. Signed-off-by: Qian Lin --- .../review_gui/sim_preview.py | 45 ++++++++++++++++--- .../review_gui/streamlit_ui.py | 2 +- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py index 36cebe0cc..97530b8fb 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import math import sys import time import uuid @@ -41,6 +42,39 @@ def _preview_args() -> argparse.Namespace: ) +def _overview_camera( + num_envs: int, env_spacing: float +) -> tuple[tuple[float, float, float], tuple[float, float, float]]: + """Return (eye, lookat) in world frame for a high oblique view of the full env grid.""" + cols = int(math.ceil(math.sqrt(num_envs))) + rows = int(math.ceil(num_envs / cols)) + max_x = max((cols - 1) * env_spacing, 0.0) + max_y = max((rows - 1) * env_spacing, 0.0) + cx, cy = max_x * 0.5, max_y * 0.5 + span = max(max_x, max_y, env_spacing) + # Oblique overview: close enough to fill the frame, still high enough for all clones. + height = span * 1.75 + env_spacing * 2.5 + back = span * 1.4 + env_spacing * 2.0 + side = span * 0.25 + eye = (cx + side, cy - back, height) + target = (cx, cy, 0.75) + return eye, target + + +def _apply_overview_camera(env, app, num_envs: int, env_spacing: float) -> None: + """Point the Kit viewport at the full multi-env grid (world frame).""" + eye, target = _overview_camera(num_envs, env_spacing) + unwrapped = env.unwrapped + vcc = getattr(unwrapped, "viewport_camera_controller", None) + if vcc is not None: + vcc.update_view_to_world() + vcc.update_view_location(eye=list(eye), lookat=list(target)) + else: + unwrapped.sim.set_camera_view(eye, target) + for _ in range(20): + app.update() + + def _capture_viewport(app, cache_path: Path) -> bytes | None: from omni.kit.viewport.utility import capture_viewport_to_file, get_active_viewport # noqa: PLC0415 @@ -88,14 +122,14 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: env = None try: - # Match policy_runner: default Isaac Lab ViewerCfg (not task look-at-object offsets). + eye, target = _overview_camera(args.num_envs, args.env_spacing) env_cfg = builder.compose_manager_cfg() - env_cfg.viewer = ViewerCfg() + # World-frame overview (not task look-at-object) so all env clones are visible. + env_cfg.viewer = ViewerCfg(eye=eye, lookat=target, origin_type="world") env = builder.make_registered(env_cfg) obs, _ = env.reset() - for _ in range(10): - app.update() + _apply_overview_camera(env, app, args.num_envs, args.env_spacing) if _capture_viewport(app, first_path) is None: raise RuntimeError("failed to capture first-frame viewport screenshot") @@ -104,8 +138,7 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: action = policy.get_action(env, obs) obs, _, _, _, _ = env.step(action) - for _ in range(10): - app.update() + _apply_overview_camera(env, app, args.num_envs, args.env_spacing) if _capture_viewport(app, last_path) is None: raise RuntimeError("failed to capture last-frame viewport screenshot") diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index 4d669d81f..d364822b9 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -593,7 +593,7 @@ def render_visualization_panel(validation: ValidationResult) -> None: st.caption( f"Runs link → to_arena_env → relation solver, then {NUM_STEPS} zero-action steps " f"with {NUM_ENVS} parallel envs at {ENV_SPACING_M} m spacing. " - "Viewport captures use the default scene camera (same as policy_runner)." + "Viewport captures use a world-frame overview of the full env grid." ) if st.button( From 5f1bce19389097dc9d314b8362d9c3efafa1b71c Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Tue, 16 Jun 2026 07:21:36 -0700 Subject: [PATCH 17/19] Fix repeat sim preview by closing env before resetting the USD stage. Close gym env managers while SimulationContext is still valid, then new_stage and pump Kit updates so stale cloner prims do not break the next preview build in the sidecar. Signed-off-by: Qian Lin --- .../utils/isaaclab_utils/simulation_app.py | 50 ++++++++++++++----- .../review_gui/sim_preview.py | 10 +++- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/isaaclab_arena/utils/isaaclab_utils/simulation_app.py b/isaaclab_arena/utils/isaaclab_utils/simulation_app.py index 4593986e5..7a4d4f264 100644 --- a/isaaclab_arena/utils/isaaclab_utils/simulation_app.py +++ b/isaaclab_arena/utils/isaaclab_utils/simulation_app.py @@ -97,23 +97,47 @@ def close_env_and_reset_sim( *, suppress_exceptions: bool = False, make_new_stage: bool = True, + app=None, ) -> None: """Tear down sim state and close a gym env so another can be built in the same SimApp. - Order matches :func:`isaaclab_arena.evaluation.eval_runner._close_env`: stop - :class:`~isaaclab.sim.SimulationContext`, open a fresh USD stage, then close - env managers and collect GPU garbage. + Close the gym env first so scene managers release prims while + :class:`~isaaclab.sim.SimulationContext` is still valid, then stop the timeline + and open a fresh USD stage. """ - try: - teardown_simulation_app(suppress_exceptions=suppress_exceptions, make_new_stage=make_new_stage) - finally: - if env is not None: - if suppress_exceptions: - with suppress(Exception): - env.close() - else: - env.close() - collect_garbage_and_clear_cuda_cache() + error_manager = suppress(Exception) if suppress_exceptions else nullcontext() + + with error_manager: + if env is not None and not getattr(env.unwrapped, "_is_closed", True): + env.close() + + # env.close() clears the singleton, but callers may omit env or close may fail partway. + with error_manager: + from isaaclab.sim import SimulationContext + + sim = SimulationContext.instance() + if sim is not None: + sim._disable_app_control_on_stop_handle = True # noqa: SLF001 (intentional private attr) + sim.stop() + sim.clear_instance() + + with error_manager: + import omni.timeline + + omni.timeline.get_timeline_interface().stop() + + if make_new_stage: + with error_manager: + import omni.usd + + omni.usd.get_context().new_stage() + + if app is not None: + with error_manager: + for _ in range(20): + app.update() + + collect_garbage_and_clear_cuda_cache() def reapply_viewer_cfg(env) -> None: diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py index 97530b8fb..005bb7d8b 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py @@ -12,6 +12,7 @@ import sys import time import uuid +from contextlib import suppress from pathlib import Path from typing import Any @@ -95,12 +96,16 @@ def _capture_viewport(app, cache_path: Path) -> bytes | None: def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: """Link spec → arena env → relation solver → zero-action steps; capture viewport frames.""" + import gymnasium as gym import yaml from isaaclab_arena.environments.arena_env_builder import ArenaEnvBuilder from isaaclab_arena.policy.zero_action_policy import ZeroActionPolicy, ZeroActionPolicyArgs from isaaclab_arena.utils.isaaclab_utils.simulation_app import close_env_and_reset_sim + # Drop any stale sim/scene state left from a prior preview in this sidecar. + close_env_and_reset_sim(suppress_exceptions=True, app=app) + raw = yaml.safe_load(yaml_text) if not isinstance(raw, dict): raise ValueError(f"expected mapping, got {type(raw).__name__}") @@ -157,4 +162,7 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: "num_steps": NUM_STEPS, } finally: - close_env_and_reset_sim(env) + close_env_and_reset_sim(env, app=app) + with suppress(Exception): + if preview_name in gym.registry: + del gym.registry[preview_name] From 1af0d55a1497f391983cd9caf710c708ddc699f4 Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Tue, 16 Jun 2026 07:43:16 -0700 Subject: [PATCH 18/19] Log sim preview phase timings before Kit scene spawn. Emit flushed stderr milestones for teardown, spec linking, relation solving, and gym.make so sidecar progress is visible during the silent pre-spawn window. Signed-off-by: Qian Lin --- .../review_gui/sim_preview.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py index 005bb7d8b..aad72af82 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/sim_preview.py @@ -26,6 +26,14 @@ ENV_SPACING_M = 1.5 NUM_STEPS = 10 +# Placement pool size when preview uses resolve_on_reset=False (see ObjectPlacerParams). +_PREVIEW_LAYOUTS_PER_ENV = 5 + + +def _preview_log(started_at: float, message: str) -> None: + elapsed = time.monotonic() - started_at + print(f"[sim_preview] +{elapsed:.1f}s {message}", file=sys.stderr, flush=True) + def _preview_args() -> argparse.Namespace: return argparse.Namespace( @@ -103,8 +111,12 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: from isaaclab_arena.policy.zero_action_policy import ZeroActionPolicy, ZeroActionPolicyArgs from isaaclab_arena.utils.isaaclab_utils.simulation_app import close_env_and_reset_sim + started_at = time.monotonic() + _preview_log(started_at, "run_sim_preview started") + # Drop any stale sim/scene state left from a prior preview in this sidecar. close_env_and_reset_sim(suppress_exceptions=True, app=app) + _preview_log(started_at, "cleared stale sim state") raw = yaml.safe_load(yaml_text) if not isinstance(raw, dict): @@ -115,6 +127,7 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: arena_env = graph_spec.to_arena_env() preview_name = f"{arena_env.name}_preview_{uuid.uuid4().hex[:8]}" arena_env.name = preview_name + _preview_log(started_at, f"linked spec → arena env ({preview_name})") args = _preview_args() builder = ArenaEnvBuilder(arena_env, args) @@ -125,13 +138,24 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: first_path = PREVIEW_CACHE_DIR / f"{preview_name}_{stamp}_first.png" last_path = PREVIEW_CACHE_DIR / f"{preview_name}_{stamp}_last.png" + pool_layouts = args.num_envs * _PREVIEW_LAYOUTS_PER_ENV env = None try: eye, target = _overview_camera(args.num_envs, args.env_spacing) + _preview_log( + started_at, + f"solving spatial relations ({args.num_envs} envs, {pool_layouts} layout pool)…", + ) + t_relations = time.monotonic() env_cfg = builder.compose_manager_cfg() + _preview_log(started_at, f"relation solver finished ({time.monotonic() - t_relations:.1f}s)") + # World-frame overview (not task look-at-object) so all env clones are visible. env_cfg.viewer = ViewerCfg(eye=eye, lookat=target, origin_type="world") + _preview_log(started_at, "spawning sim scene (gym.make)…") + t_spawn = time.monotonic() env = builder.make_registered(env_cfg) + _preview_log(started_at, f"sim scene ready ({time.monotonic() - t_spawn:.1f}s)") obs, _ = env.reset() _apply_overview_camera(env, app, args.num_envs, args.env_spacing) @@ -149,8 +173,10 @@ def run_sim_preview(app, yaml_text: str) -> dict[str, Any]: raise RuntimeError("failed to capture last-frame viewport screenshot") print( - f"[sim_preview] captured {NUM_ENVS} envs @ {ENV_SPACING_M}m spacing, {NUM_STEPS} zero-action steps", + f"[sim_preview] captured {NUM_ENVS} envs @ {ENV_SPACING_M}m spacing, {NUM_STEPS} zero-action steps " + f"(total {time.monotonic() - started_at:.1f}s)", file=sys.stderr, + flush=True, ) return { "ok": True, From ce1d94b6595ad7c00b971968c287de8af7b3011c Mon Sep 17 00:00:00 2001 From: Qian Lin Date: Tue, 16 Jun 2026 07:43:17 -0700 Subject: [PATCH 19/19] Skip redundant sidecar validation when launching sim preview. Reuse the editor panel's ValidationResult on preview button click to avoid a second validate_spec IPC round-trip on the same Streamlit rerun. Signed-off-by: Qian Lin --- .../review_gui/streamlit_ui.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py index d364822b9..0d93e65f7 100644 --- a/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py +++ b/isaaclab_arena_examples/agentic_environment_generation/review_gui/streamlit_ui.py @@ -358,11 +358,16 @@ def run_generation_pipeline(prompt: str) -> tuple[bool, str]: return True, "Spec generated and loaded into the YAML editor." -def run_sim_preview_pipeline(yaml_text: str) -> tuple[bool, str]: - """Link, build, solve relations, and capture overview frames in the sidecar.""" - validation = validate_yaml_text(yaml_text) - if not validation.is_valid: - return False, validation.error or "YAML must be valid before running sim preview." +def run_sim_preview_pipeline(yaml_text: str, *, validation: ValidationResult | None = None) -> tuple[bool, str]: + """Link, build, solve relations, and capture overview frames in the sidecar. + + When ``validation`` is already valid (e.g. from the editor panel on the same + rerun), skip the redundant sidecar ``validate_spec`` round-trip. + """ + if validation is None or not validation.is_valid: + validation = validate_yaml_text(yaml_text) + if not validation.is_valid: + return False, validation.error or "YAML must be valid before running sim preview." sidecar = _ensure_sidecar() if sidecar is None: @@ -606,7 +611,7 @@ def render_visualization_panel(validation: ValidationResult) -> None: with st.spinner( f"Building env, solving relations, and rolling out {NUM_STEPS} steps ({NUM_ENVS} envs @ {ENV_SPACING_M} m)…" ): - ok, message = run_sim_preview_pipeline(st.session_state["edited_text"]) + ok, message = run_sim_preview_pipeline(st.session_state["edited_text"], validation=validation) if ok: st.success(message, icon="✅") st.rerun()