Skip to content

Commit 3672e81

Browse files
Zyysurelyyingyingzhaozyying
Zyysurely
and
yingyingzhaozyying
authored
[fix] Align azd ai agent optimize/eval with the V2Preview optimization API (#8589)
* [fix] Align `azd ai agent optimize`/`eval` with the v2 optimization API * envelope + address comments * fix listing optimization jobs --------- Co-authored-by: zyysurely <yingyingzhao@microsoft.com>
1 parent 2b00014 commit 3672e81

35 files changed

Lines changed: 1401 additions & 534 deletions

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_api_version_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ func TestUpdateDataset_APIVersion(t *testing.T) {
170170
configPath := filepath.Join(dir, "eval.yaml")
171171
cfg := &evalConfig{
172172
Config: opt_eval.Config{
173-
DatasetReference: &evalDatasetRef{
173+
Dataset: &evalDatasetRef{
174174
Name: "test-ds",
175175
Version: "v1",
176176
LocalURI: dataDir,

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate.go‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ func submitEvalJobs(
300300
needEvalGen = flags.regenerateEvaluator
301301
if !needDatasetGen {
302302
evalCfg.DatasetFile = existingCfg.DatasetFile
303-
evalCfg.Config.DatasetReference = existingCfg.Config.DatasetReference
303+
evalCfg.Config.Dataset = existingCfg.Config.Dataset
304304
}
305305
if !needEvalGen {
306306
evalCfg.Evaluators = existingCfg.Evaluators
@@ -313,7 +313,9 @@ func submitEvalJobs(
313313
if err != nil {
314314
return nil, err
315315
}
316-
evalCfg.DatasetFile = datasetPath
316+
evalCfg.Dataset = &opt_eval.DatasetRef{
317+
LocalURI: datasetPath,
318+
}
317319
}
318320
}
319321

@@ -363,16 +365,16 @@ func writeAndPrintEvalResult(
363365
fmt.Println(color.GreenString("\nEval suite created"))
364366
}
365367
fmt.Printf(" Config: %s\n", configPath)
366-
if evalCfg.DatasetFile != "" {
367-
fmt.Printf(" Dataset: %s\n", evalCfg.DatasetFile)
368-
} else if evalCfg.DatasetReference != nil && evalCfg.DatasetReference.Name != "" {
369-
ds := evalCfg.DatasetReference.Name
370-
if evalCfg.DatasetReference.Version != "" {
371-
ds += " (" + evalCfg.DatasetReference.Version + ")"
368+
if localPath := evalCfg.LocalDatasetPath(); localPath != "" {
369+
fmt.Printf(" Dataset: %s\n", localPath)
370+
} else if ref := evalCfg.RemoteDatasetReference(); ref != nil {
371+
ds := ref.Name
372+
if ref.Version != "" {
373+
ds += " (" + ref.Version + ")"
372374
}
373375
fmt.Printf(" Dataset: %s\n", ds)
374376
if resolved.hasProject {
375-
fmt.Printf(" %s\n", eval_api.DatasetArtifactPath(resolved.agentProject, evalCfg.DatasetReference))
377+
fmt.Printf(" %s\n", eval_api.DatasetArtifactPath(resolved.agentProject, ref))
376378
}
377379
}
378380
for _, evaluator := range evalCfg.Evaluators {
@@ -418,9 +420,9 @@ func printEvalPortalLinks(ctx context.Context, resolved *evalResolvedContext, ev
418420
return
419421
}
420422
hasLink := false
421-
if evalCfg.DatasetReference != nil && evalCfg.DatasetReference.Name != "" {
423+
if ref := evalCfg.RemoteDatasetReference(); ref != nil {
422424
fmt.Printf("\n "+color.HiBlackString("Portal:")+"\n Dataset: %s\n",
423-
color.CyanString(prefix.DatasetURL(evalCfg.DatasetReference.Name, evalCfg.DatasetReference.Version)))
425+
color.CyanString(prefix.DatasetURL(ref.Name, ref.Version)))
424426
hasLink = true
425427
}
426428
for _, evaluator := range evalCfg.Evaluators {

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_jobs.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ func pollAndFinalizeJobs(
279279
if dsRef == nil {
280280
return
281281
}
282-
evalCfg.DatasetReference = dsRef
282+
evalCfg.Dataset = dsRef
283283

284284
if resolved.hasProject {
285285
localURI, err := eval_api.DownloadDatasetArtifact(

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_prompts.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020

2121
// promptEvalGenerateOptions runs interactive prompts for eval generate options that
2222
// were not provided via flags: name, instruction, trace days, eval model,
23-
// and max samples.
23+
// and max samples. Max samples is skipped when an existing dataset is supplied.
2424
func promptEvalGenerateOptions(ctx context.Context, resolved *evalResolvedContext, flags *evalGenerateFlags, noPrompt bool) error {
2525
azdClient := resolved.azdClient
2626
if noPrompt {
@@ -158,7 +158,9 @@ func promptEvalGenerateOptions(ctx context.Context, resolved *evalResolvedContex
158158
flags.evalModel = selected
159159
}
160160

161-
if !flags.maxSamplesSet {
161+
// Max samples only applies when generating a dataset. If the user supplied
162+
// an existing dataset (--dataset), there is nothing to generate, so skip it.
163+
if !flags.maxSamplesSet && flags.dataset == "" {
162164
resp, err := azdClient.Prompt().Prompt(ctx, &azdext.PromptRequest{
163165
Options: &azdext.PromptOptions{
164166
Message: "Max samples (between 15 and 1000)",
@@ -193,8 +195,8 @@ func promptRegenerateChoices(
193195

194196
// Ask about dataset.
195197
datasetLabel := existingCfg.DatasetFile
196-
if datasetLabel == "" && existingCfg.DatasetReference != nil {
197-
datasetLabel = existingCfg.DatasetReference.Name
198+
if datasetLabel == "" && existingCfg.Dataset != nil {
199+
datasetLabel = existingCfg.Dataset.Name
198200
}
199201
if datasetLabel != "" {
200202
resp, err := prompt.Confirm(ctx, &azdext.ConfirmRequest{

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_generate_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -408,8 +408,8 @@ func TestBuildOpenAIEvalRequest(t *testing.T) {
408408
Name: "agent-1",
409409
Version: "v1",
410410
},
411-
DatasetReference: &evalDatasetRef{Name: "ds", Version: "v1"},
412-
Evaluators: opt_eval.EvaluatorList{{Name: "builtin.quality"}},
411+
Dataset: &evalDatasetRef{Name: "ds", Version: "v1"},
412+
Evaluators: opt_eval.EvaluatorList{{Name: "builtin.quality"}},
413413
},
414414
Options: &opt_eval.Options{EvalModel: "gpt-4o"},
415415
}

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_helpers.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -467,7 +467,7 @@ func promptDatasetSelection(
467467
value := strings.TrimSpace(resp.Value)
468468
if value == "" {
469469
return "", nil, fmt.Errorf(
470-
"a dataset is required: use --dataset <file-or-name>, or provide dataset_file / dataset_reference " +
470+
"a dataset is required: use --dataset <file-or-name>, or provide a dataset " +
471471
"in your config, or run 'azd ai agent eval generate' to generate one")
472472
}
473473

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_run.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -133,21 +133,21 @@ func runEvalRun(ctx context.Context, flags *evalRunFlags, noPrompt bool) error {
133133
)
134134

135135
// Set source from local dataset file or remote dataset reference.
136-
if evalCfg.DatasetFile != "" {
136+
if localPath := evalCfg.LocalDatasetPath(); localPath != "" {
137137
// Resolve relative paths against the agent project directory so
138-
// eval.yaml files with project-relative dataset_file entries work
138+
// eval.yaml files with project-relative local dataset entries work
139139
// regardless of the caller's working directory.
140-
datasetPath := eval_api.ResolveRelPath(evalCfg.DatasetFile, resolved.agentProject)
140+
datasetPath := eval_api.ResolveRelPath(localPath, resolved.agentProject)
141141
items, err := loadJSONLFile[map[string]any](datasetPath)
142142
if err != nil {
143143
return err
144144
}
145145
dataSource.SetFileContent(items)
146-
} else if evalCfg.DatasetReference != nil {
147-
fileID := buildDatasetFileID(resolved.projectEndpoint, evalCfg.DatasetReference)
146+
} else if ref := evalCfg.RemoteDatasetReference(); ref != nil {
147+
fileID := buildDatasetFileID(resolved.projectEndpoint, ref)
148148
dataSource.SetFileID(fileID)
149149
} else {
150-
return fmt.Errorf("no dataset configured; run 'azd ai agent eval generate' or specify dataset_file / dataset_reference in the eval config")
150+
return fmt.Errorf("no dataset configured; run 'azd ai agent eval generate' or specify a dataset in the eval config")
151151
}
152152

153153
runReq.DataSource = dataSource

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ func TestWriteEvalReviewArtifacts(t *testing.T) {
208208
dir := t.TempDir()
209209

210210
cfg := &evalConfig{}
211-
cfg.DatasetReference = &evalDatasetRef{Name: "test-data", Version: "v1"}
211+
cfg.Dataset = &evalDatasetRef{Name: "test-data", Version: "v1"}
212212
cfg.Evaluators = opt_eval.EvaluatorList{{Name: "quality"}}
213213

214214
err := eval_api.WriteEvalReviewArtifacts(dir, cfg)
@@ -420,8 +420,8 @@ func TestEvalConfigRoundTrip(t *testing.T) {
420420
Kind: agent_yaml.AgentKindHosted,
421421
Version: "v1",
422422
},
423-
DatasetReference: &evalDatasetRef{Name: "ds", Version: "v1"},
424-
Evaluators: opt_eval.EvaluatorList{{Name: "builtin.task_adherence"}},
423+
Dataset: &evalDatasetRef{Name: "ds", Version: "v1"},
424+
Evaluators: opt_eval.EvaluatorList{{Name: "builtin.task_adherence"}},
425425
},
426426
Options: &opt_eval.Options{
427427
EvalModel: "gpt-4o",
@@ -441,8 +441,8 @@ func TestEvalConfigRoundTrip(t *testing.T) {
441441
assert.Equal(t, original.Agent.Version, loaded.Agent.Version)
442442
assert.Equal(t, "gpt-4o", loaded.Options.EvalModel)
443443
assert.Equal(t, original.MaxSamples, loaded.MaxSamples)
444-
require.NotNil(t, loaded.DatasetReference)
445-
assert.Equal(t, "ds", loaded.DatasetReference.Name)
444+
require.NotNil(t, loaded.Dataset)
445+
assert.Equal(t, "ds", loaded.Dataset.Name)
446446
require.Len(t, loaded.Evaluators, 1)
447447
assert.Equal(t, "builtin.task_adherence", loaded.Evaluators[0].Name)
448448
}

‎cli/azd/extensions/azure.ai.agents/internal/cmd/eval_update.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,9 @@ func runEvalUpdate(ctx context.Context, flags *evalUpdateFlags, noPrompt bool) e
7171
}
7272

7373
// Detect what has local changes.
74-
hasDataset := evalCfg.DatasetReference != nil &&
75-
evalCfg.DatasetReference.Name != "" &&
76-
evalCfg.DatasetReference.LocalURI != ""
74+
hasDataset := evalCfg.Dataset != nil &&
75+
evalCfg.Dataset.Name != "" &&
76+
evalCfg.Dataset.LocalURI != ""
7777
hasEvaluators := len(evalCfg.Evaluators.FindByLocalURI()) > 0
7878

7979
// Determine what to update based on flags and interactive prompts.
@@ -85,7 +85,7 @@ func runEvalUpdate(ctx context.Context, flags *evalUpdateFlags, noPrompt bool) e
8585
if hasDataset {
8686
updateDS = confirmUpdate(ctx, resolved, fmt.Sprintf(
8787
"Dataset %s has local changes. Upload new version?",
88-
evalCfg.DatasetReference.Name,
88+
evalCfg.Dataset.Name,
8989
))
9090
}
9191
if hasEvaluators {
@@ -145,7 +145,7 @@ func updateDataset(
145145
evalCfg *evalConfig,
146146
configPath string,
147147
) (int, error) {
148-
ref := evalCfg.DatasetReference
148+
ref := evalCfg.Dataset
149149
if ref == nil || ref.Name == "" || ref.LocalURI == "" {
150150
return 0, nil
151151
}

0 commit comments

Comments
 (0)