|
| 1 | +"""verl-agent / VAGEN backend for GRPO training. |
| 2 | +
|
| 3 | +This module provides the integration point for training via verl-agent |
| 4 | +(https://github.com/VAGEN), which offers: |
| 5 | + - GiGPO (Generalized Group Relative Policy Optimization) |
| 6 | + - Multi-GPU distributed training via veRL |
| 7 | + - Desktop environment integration via WAADesktopEnv |
| 8 | +
|
| 9 | +The actual training loop is managed by verl-agent's own training script, |
| 10 | +not by our GRPOTrainer. This module builds the VAGEN-compatible config |
| 11 | +from our GRPOConfig and documents how to run training. |
| 12 | +
|
| 13 | +Usage: |
| 14 | + To train with the verl backend, set backend="verl" in GRPOConfig. |
| 15 | + The train_with_verl() function will print instructions and raise |
| 16 | + NotImplementedError until full integration is wired up. |
| 17 | +
|
| 18 | + For now, training with verl-agent should be done via: |
| 19 | + 1. Generate a VAGEN config: train_with_verl(config) |
| 20 | + 2. Run verl-agent's training script with that config |
| 21 | +
|
| 22 | +See also: |
| 23 | + - openadapt-evals/configs/train_waa_vagen.yaml |
| 24 | + - docs/verl_agent_decision.md (if available) |
| 25 | +""" |
| 26 | + |
| 27 | +from __future__ import annotations |
| 28 | + |
| 29 | +import logging |
| 30 | +from typing import Any |
| 31 | + |
| 32 | +from openadapt_ml.training.grpo.config import GRPOConfig |
| 33 | + |
| 34 | +logger = logging.getLogger(__name__) |
| 35 | + |
| 36 | +# Deferred import for openadapt-evals WAADesktopEnv (optional dependency) |
| 37 | +try: |
| 38 | + from openadapt_evals.adapters.verl_env import WAADesktopEnv |
| 39 | +except ImportError: |
| 40 | + WAADesktopEnv = None # type: ignore[assignment, misc] |
| 41 | + |
| 42 | + |
| 43 | +def build_vagen_config(config: GRPOConfig) -> dict[str, Any]: |
| 44 | + """Build a VAGEN-compatible config dict from GRPOConfig. |
| 45 | +
|
| 46 | + Maps our config fields to the structure expected by verl-agent's |
| 47 | + training script. This dict can be serialized to YAML for use with |
| 48 | + VAGEN's CLI. |
| 49 | +
|
| 50 | + Args: |
| 51 | + config: Our GRPO training configuration. |
| 52 | +
|
| 53 | + Returns: |
| 54 | + Dict matching VAGEN's expected config structure. |
| 55 | + """ |
| 56 | + return { |
| 57 | + "model": { |
| 58 | + "name": config.model_name, |
| 59 | + "load_in_4bit": config.load_in_4bit, |
| 60 | + "lora_r": config.lora_r, |
| 61 | + "lora_alpha": config.lora_alpha, |
| 62 | + }, |
| 63 | + "training": { |
| 64 | + "learning_rate": config.learning_rate, |
| 65 | + "num_training_steps": config.num_training_steps, |
| 66 | + "save_every_steps": config.save_every_steps, |
| 67 | + "output_dir": config.output_dir, |
| 68 | + "num_rollouts_per_step": config.num_rollouts_per_step, |
| 69 | + "temperature": config.temperature, |
| 70 | + }, |
| 71 | + "environment": { |
| 72 | + "type": "waa_desktop", |
| 73 | + "server_url": config.server_url, |
| 74 | + "task_ids": config.task_ids, |
| 75 | + "max_steps_per_episode": config.max_steps_per_episode, |
| 76 | + "screen_size": list(config.screen_size), |
| 77 | + "stuck_window": config.stuck_window, |
| 78 | + }, |
| 79 | + } |
| 80 | + |
| 81 | + |
| 82 | +def train_with_verl(config: GRPOConfig) -> None: |
| 83 | + """Entry point for verl-agent backend training. |
| 84 | +
|
| 85 | + Currently a placeholder that documents the integration point. |
| 86 | + The actual training happens via verl-agent's own CLI/training script, |
| 87 | + not through this function. |
| 88 | +
|
| 89 | + Args: |
| 90 | + config: GRPO training configuration with backend="verl". |
| 91 | +
|
| 92 | + Raises: |
| 93 | + NotImplementedError: Always, until full verl-agent integration |
| 94 | + is wired up. The error message includes instructions for |
| 95 | + running training via verl-agent directly. |
| 96 | + """ |
| 97 | + vagen_config = build_vagen_config(config) |
| 98 | + |
| 99 | + if WAADesktopEnv is not None: |
| 100 | + logger.info( |
| 101 | + "WAADesktopEnv is available. verl-agent can use it for " |
| 102 | + "desktop environment interaction." |
| 103 | + ) |
| 104 | + else: |
| 105 | + logger.warning( |
| 106 | + "WAADesktopEnv not found. Install openadapt-evals to enable " |
| 107 | + "desktop environment support: uv add openadapt-evals" |
| 108 | + ) |
| 109 | + |
| 110 | + logger.info("VAGEN config built from GRPOConfig:") |
| 111 | + logger.info(" Model: %s", vagen_config["model"]["name"]) |
| 112 | + logger.info(" Tasks: %s", vagen_config["environment"]["task_ids"]) |
| 113 | + logger.info(" Steps: %d", vagen_config["training"]["num_training_steps"]) |
| 114 | + logger.info("") |
| 115 | + logger.info( |
| 116 | + "To train with verl-agent, use the VAGEN training script with " |
| 117 | + "a config derived from the above. Example:" |
| 118 | + ) |
| 119 | + logger.info(" python -m vagen.train --config configs/train_waa_vagen.yaml") |
| 120 | + |
| 121 | + raise NotImplementedError( |
| 122 | + "verl-agent training requires running via VAGEN's training script. " |
| 123 | + "See docs/verl_agent_decision.md for setup instructions. " |
| 124 | + "Use build_vagen_config() to generate a compatible config dict." |
| 125 | + ) |
0 commit comments