diff --git a/README.md b/README.md index 2ec9c14..9d2a384 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,6 @@ fun contentPipeline() = workflow("content-pipeline") { agent("reviewer") { role = "Content Reviewer"; model = Models.GEMINI_3_1_PRO } } - rubrics { rubric("content-quality", "content-quality.md") } - state { input("topic", VarType.STRING) variable("draft", VarType.STRING, "the full written article text") @@ -126,7 +124,7 @@ fun contentPipeline() = workflow("content-pipeline") { agent = "writer" prompt = "Write a short article about {topic}. {recommendation}" writes("draft") - rubric = "content-quality" + rubric = "content-quality.md" onScore { whenScore lessThan 70.0 goto "write" // score too low – retry } diff --git a/docs/developer-guide-core.md b/docs/developer-guide-core.md index 63e222d..d1275f1 100644 --- a/docs/developer-guide-core.md +++ b/docs/developer-guide-core.md @@ -182,7 +182,7 @@ map keyed by node ID. For `StandardNode`s with `writes` declared, it routes the - **Single write**: attempts to parse the response as JSON and extract the declared key; falls back to the full raw text if the key is absent or the output is not valid JSON. - **Multiple writes**: the response is parsed as JSON and each declared key is extracted into context. -**`RubricPostProcessor`** — If a node has a `rubricId`, this processor evaluates the output against the rubric's +**`RubricPostProcessor`** — If a node has a `Rubric`, this processor evaluates the output against the rubric's criteria using the `RubricEngine`. It stores the evaluation result in the state for use by `ScoreTransition` rules. If the evaluation fails and no explicit transition handles the low score, it can trigger an **auto-backtrack** to a prior step, enabling self-correcting loops. On auto-backtrack, sets `state.nodeRedirected = true` — downstream processors @@ -710,8 +710,7 @@ The rubric engine evaluates output quality against defined criteria with score-b | Component | Description | |----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| -| `RubricEngine` | Orchestrates evaluation using repository and evaluator | -| `RubricRepository` | Stores rubric definitions (in-memory by default) | +| `RubricEngine` | Orchestrates evaluation using evaluator | | `RubricEvaluator` | Evaluates output against criteria | | `ScoreExtractingEvaluator` | Reads the `score` engine variable from context; accumulates `recommendation` feedback for failing criteria into `_rubric_criterion_feedback` | | `Rubric` | Immutable definition with pass threshold and weighted criteria | @@ -719,6 +718,8 @@ The rubric engine evaluates output quality against defined criteria with score-b ### How Evaluation Works +Rubrics are parsed at build time and stored directly on the node as typed `Rubric` objects. + `ScoreExtractingEvaluator` reads the `score` engine variable directly from the execution context. The score is extracted automatically by `OutputExtractionPostProcessor` whenever the node has a `ScoreTransition` — no JSON parsing is needed in the evaluator itself. If the score falls below a criterion's minimum and a `recommendation` engine variable is present in context, the text is appended to `_rubric_criterion_feedback`. `RubricPostProcessor` uses that list to assemble a combined backtrack context update for self-correcting loops. @@ -861,7 +862,7 @@ Each injector fires when **either** of two conditions is met: ```mermaid flowchart LR - r(["RubricPromptInjector\n· rubricId != null"]) --> s(["ScoreVariableInjector\n· ScoreTransition or\nconsensus branch"]) + r(["RubricPromptInjector\n· rubric != null"]) --> s(["ScoreVariableInjector\n· ScoreTransition or\nconsensus branch"]) s --> a(["ApprovalVariableInjector\n· ApprovalTransition or\nconsensus branch"]) a --> rec(["RecommendationVariable\nInjector\n· Score/Approval or\nconsensus branch"]) rec --> w(["WritesVariableInjector\n· has writes()"]) diff --git a/docs/developer-guide-server.md b/docs/developer-guide-server.md index 4bc4663..7849150 100644 --- a/docs/developer-guide-server.md +++ b/docs/developer-guide-server.md @@ -970,7 +970,6 @@ Every integration test extends `IntegrationTestBase`, which provides: | `pushAndExecuteWithMcp(workflow, ctx, endpoint)` | Executes with MCP-enabled tenant context | | `registerStub(key, response)` | Programmatic stub registration by node ID or agent ID | | `registerStub(scenario, key, response)` | Scenario-specific stub registration | -| `resolveRubricPath(resourceName)` | Copies classpath rubric to temp file for `RubricParser` | #### Writing an Integration Test @@ -1098,28 +1097,10 @@ Place workflow definitions in `src/test/resources/workflows/`. Use `model: "stub #### Rubric Testing -Pre-register parsed rubrics so the executor skips filesystem path resolution: - -```java -private Rubric parseAndRegisterRubric(String rubricId, String resourceName) { - String rubricPath = resolveRubricPath(resourceName); - Rubric parsed = RubricParser.parse(Path.of(rubricPath)); - - Rubric rubric = Rubric.builder() - .id(rubricId) - .name(parsed.getName()) - .version(parsed.getVersion()) - .type(parsed.getType()) - .passThreshold(parsed.getPassThreshold()) - .criteria(parsed.getCriteria()) - .build(); - - hensuEnvironment.getRubricRepository().save(rubric); - return rubric; -} -``` - -Place rubric markdown files in `src/test/resources/rubrics/`. +Rubrics are parsed at build time and stored directly on workflow nodes. Workflow JSON fixtures +include inline rubric content in the `"rubric"` field, which the deserializer parses into typed +`Rubric` objects at load time. No separate registration step is needed — just call +`loadWorkflow("fixture.json")` and the rubric is ready. #### Repository Tests (Testcontainers) diff --git a/docs/dsl-reference.md b/docs/dsl-reference.md index cfd7a90..6a7a549 100644 --- a/docs/dsl-reference.md +++ b/docs/dsl-reference.md @@ -51,10 +51,6 @@ fun myWorkflow() = workflow("WorkflowName") { // Agent definitions } - rubrics { - // Rubric references (optional) - } - config { // Execution settings (optional) } @@ -78,7 +74,6 @@ fun myWorkflow() = workflow("WorkflowName") { |---------------|--------------------------------------------------------------------------------------------------------------------------| | `state { }` | Optional typed state schema. When declared, enables load-time validation of `writes` and `{variable}` prompt references. | | `agents { }` | Agent definitions (models, roles, temperatures) | -| `rubrics { }` | Rubric file references for quality evaluation | | `config { }` | Workflow execution settings | | `graph { }` | Node graph (required) | @@ -155,7 +150,7 @@ Standard nodes execute an agent with a prompt and transition based on the result node("node-id") { agent = "agent-id" prompt = "Your prompt with {placeholders}" - rubric = "rubric-id" // Optional + rubric = "rubric-id.md" // Optional writes("param1", "param2") // Optional — declare state variables this node produces review(ReviewMode.OPTIONAL) // Optional @@ -169,11 +164,11 @@ node("node-id") { #### Standard Node Properties -| Property | Type | Required | Description | -|-------------|---------|----------|-----------------------------------------| -| `agent` | String? | Yes | ID of the agent to execute | -| `prompt` | String? | Yes | Prompt template or `.md` file reference | -| `rubric` | String? | No | ID of rubric to evaluate output quality | +| Property | Type | Required | Description | +|----------|---------|----------|----------------------------------------------------------------| +| `agent` | String? | Yes | ID of the agent to execute | +| `prompt` | String? | Yes | Inline prompt template or `.md` file reference from `prompts/` | +| `rubric` | String? | No | Inline rubric content or `.md` file reference from `rubrics/` | #### Standard Node Functions @@ -219,13 +214,13 @@ parallel("review-committee") { #### Branch Properties -| Property | Type | Required | Description | -|------------|---------------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `agent` | String | Yes | ID of the agent to execute | -| `prompt` | String? | No | Prompt template or `.md` file reference | -| `rubric` | String? | No | ID of rubric for branch evaluation. When set, the rubric's pass/fail result determines the branch's consensus vote (APPROVE/REJECT), overriding text-based heuristics | -| `weight` | Double | No | Vote weight for `WEIGHTED_VOTE` consensus strategy. Higher values give more influence to the branch score (default: 1.0) | -| `yields()` | vararg String | No | State variable names this branch produces as structured domain output. The agent's JSON response must include these fields; the engine extracts and merges them into workflow state | +| Property | Type | Required | Description | +|------------|---------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `agent` | String | Yes | ID of the agent to execute | +| `prompt` | String? | No | Inline prompt template or `.md` file reference from `prompts/` | +| `rubric` | String? | No | Inline rubric content or `.md` file reference from `rubrics/`. When set, the rubric's pass/fail result determines the branch's consensus vote (APPROVE/REJECT), overriding text-based heuristics | +| `weight` | Double | No | Vote weight for `WEIGHTED_VOTE` consensus strategy. Higher values give more influence to the branch score (default: 1.0) | +| `yields()` | vararg String | No | State variable names this branch produces as structured domain output. The agent's JSON response must include these fields; the engine extracts and merges them into workflow state | #### Rubric-Based Consensus @@ -335,7 +330,7 @@ generic("validate-input") { "required" to true } - rubric = "validation-rubric" // Optional + rubric = "validation-rubric.md" // Optional onSuccess goto "process" onFailure retry 2 otherwise "error" @@ -725,7 +720,7 @@ Because `recommendation` is an engine variable, you reference it as a `{placehol node("score-content") { agent = "reviewer" prompt = "Review the article: {article}\nOutput a score and feedback." - rubric = "content-quality" + rubric = "content-quality.md" onScore { whenScore greaterThanOrEqual 80.0 goto "publish" @@ -746,24 +741,15 @@ node("revise") { ``` ## Rubrics +Rubrics define quality evaluation criteria for node outputs. Rubric files should be placed in the `rubrics/` directory of your working directory. -Rubrics define quality evaluation criteria for node outputs. - -```kotlin -rubrics { - rubric("quality-check", "quality.md") // rubrics/quality.md - rubric("pr-review", "templates/pr.md") // rubrics/templates/pr.md - rubric("docs") { file = "documentation.md" } // Alternative syntax -} -``` - -Reference rubrics in nodes: +To use a rubric, specify the filename in the node's `rubric` property: ```kotlin node("review") { agent = "reviewer" prompt = "Review this code" - rubric = "quality-check" + rubric = "content-quality.md" // References rubrics/content-quality.md onScore { whenScore greaterThanOrEqual 80.0 goto "approve" @@ -1094,10 +1080,6 @@ fun contentPipeline() = workflow("ContentPipeline") { } } - rubrics { - rubric("content-quality", "content-quality.md") - } - graph { start at "research" diff --git a/docs/unified-architecture.md b/docs/unified-architecture.md index ae41928..e679f0c 100644 --- a/docs/unified-architecture.md +++ b/docs/unified-architecture.md @@ -760,7 +760,7 @@ CDI wiring, `WorkflowExecutor`, `TenantContext` — against in-memory repositori profile disables PostgreSQL, Flyway, and the scheduler (no Docker required). All integration tests extend `IntegrationTestBase`, which provides CDI injection, per-test -state cleanup, and helpers (`registerStub`, `pushAndExecute`, `resolveRubricPath`). +state cleanup, and helpers (`registerStub`, `pushAndExecute`). ### Repository Tests (Testcontainers PostgreSQL) diff --git a/hensu-cli/src/main/java/io/hensu/cli/review/ReviewTerminal.java b/hensu-cli/src/main/java/io/hensu/cli/review/ReviewTerminal.java index f929aa2..f1e85ac 100644 --- a/hensu-cli/src/main/java/io/hensu/cli/review/ReviewTerminal.java +++ b/hensu-cli/src/main/java/io/hensu/cli/review/ReviewTerminal.java @@ -190,20 +190,18 @@ private void displayHelp() { private ReviewDecision handleBacktrack(ReviewData data) { List steps = data.historySteps(); - if (steps == null || steps.size() <= 1) { + if (steps == null || steps.isEmpty()) { println(styles.warn("No previous steps available to backtrack to.")); return null; } - List validSteps = steps.subList(0, steps.size() - 1); - println(""); println(styles.boxTopWithLabel(styles.dim("backtrack target"))); println(""); - for (int i = validSteps.size() - 1; i >= 0; i--) { - ReviewData.StepInfo step = validSteps.get(i); - int displayNum = validSteps.size() - i; + for (int i = steps.size() - 1; i >= 0; i--) { + ReviewData.StepInfo step = steps.get(i); + int displayNum = steps.size() - i; boolean ok = "SUCCESS".equals(step.status()); String status = styles.successOrError(ok ? "OK" : "FAIL", ok); println(String.format(" [%d] %s (%s)", displayNum, step.nodeId(), status)); @@ -217,12 +215,12 @@ private ReviewDecision handleBacktrack(ReviewData data) { try { int choice = Integer.parseInt(input); if (choice == 0) return null; - if (choice < 1 || choice > validSteps.size()) { + if (choice < 1 || choice > steps.size()) { println(styles.error("Invalid choice. Please select a number from the list.")); return null; } - ReviewData.StepInfo targetStep = validSteps.get(validSteps.size() - choice); + ReviewData.StepInfo targetStep = steps.get(steps.size() - choice); print("Reason for backtracking (optional): "); String reason = readInput(); diff --git a/hensu-cli/src/main/java/io/hensu/cli/visualizer/TextVisualizationFormat.java b/hensu-cli/src/main/java/io/hensu/cli/visualizer/TextVisualizationFormat.java index 76c741d..e3f7ca5 100644 --- a/hensu-cli/src/main/java/io/hensu/cli/visualizer/TextVisualizationFormat.java +++ b/hensu-cli/src/main/java/io/hensu/cli/visualizer/TextVisualizationFormat.java @@ -120,11 +120,14 @@ private String renderNode(Node node, String nodeId, String indent, AnsiStyles st "agent", styles.bold(standardNode.getAgentId()))); } - if (standardNode.getRubricId() != null) { + if (standardNode.getRubric() != null) { sb.append( String.format( "%s%s %-9s %s%n", - indent, styles.boxMid(), "rubric", standardNode.getRubricId())); + indent, + styles.boxMid(), + "rubric", + standardNode.getRubric().getCriteria().size() + " criteria")); } if (standardNode.getReviewConfig() != null) { sb.append( @@ -271,11 +274,14 @@ private String renderNode(Node node, String nodeId, String indent, AnsiStyles st styles.accent(String.valueOf(genericNode.getConfig().size())) + " entries")); } - if (genericNode.getRubricId() != null) { + if (genericNode.getRubric() != null) { sb.append( String.format( "%s%s %-9s %s%n", - indent, styles.boxMid(), "rubric", genericNode.getRubricId())); + indent, + styles.boxMid(), + "rubric", + genericNode.getRubric().getCriteria().size() + " criteria")); } appendTransitions(sb, indent, genericNode.getTransitionRules(), styles); } diff --git a/hensu-cli/src/test/java/io/hensu/cli/review/CLIReviewHandlerTest.java b/hensu-cli/src/test/java/io/hensu/cli/review/CLIReviewHandlerTest.java index 77fb271..aad47e2 100644 --- a/hensu-cli/src/test/java/io/hensu/cli/review/CLIReviewHandlerTest.java +++ b/hensu-cli/src/test/java/io/hensu/cli/review/CLIReviewHandlerTest.java @@ -144,15 +144,10 @@ void shouldContinueWhenBacktrackDisabled() { @Test void shouldContinueWhenNoPreviousStepsToBacktrackTo() { System.setProperty(CLIReviewHandler.INTERACTIVE_PROPERTY, "true"); - // Only one step in history (the current node itself) — nothing to go back to + // Empty history — current node not yet recorded, nothing to go back to CLIReviewHandler manager = new CLIReviewHandler(new Scanner("B\nA\n"), printStream, false); ExecutionHistory history = new ExecutionHistory(); - history.addStep( - ExecutionStep.builder() - .nodeId("only-step") - .result(NodeResult.success("output", Map.of())) - .build()); ReviewOutcome outcome = manager.requestReview( @@ -182,11 +177,6 @@ void shouldAllowBacktrackToPreviousStep() { .nodeId("step-1") .result(NodeResult.success("output1", Map.of())) .build()); - history.addStep( - ExecutionStep.builder() - .nodeId("step-2") - .result(NodeResult.success("output2", Map.of())) - .build()); ReviewOutcome outcome = manager.requestReview( @@ -217,11 +207,6 @@ void shouldCancelBacktrackOnZero() { .nodeId("step-1") .result(NodeResult.success("output1", Map.of())) .build()); - history.addStep( - ExecutionStep.builder() - .nodeId("step-2") - .result(NodeResult.success("output2", Map.of())) - .build()); ReviewOutcome outcome = manager.requestReview( @@ -249,11 +234,6 @@ void shouldUseDefaultReasonWhenBacktrackReasonBlank() { .nodeId("step-1") .result(NodeResult.success("output1", Map.of())) .build()); - history.addStep( - ExecutionStep.builder() - .nodeId("step-2") - .result(NodeResult.success("output2", Map.of())) - .build()); ReviewOutcome outcome = manager.requestReview( diff --git a/hensu-core/README.md b/hensu-core/README.md index 6b97c2a..5b5e989 100644 --- a/hensu-core/README.md +++ b/hensu-core/README.md @@ -113,10 +113,6 @@ rubrics to determine quality scores and pass/fail status. flowchart LR subgraph engine["RubricEngine"] direction LR - subgraph repo["RubricRepository"] - direction TB - inmem(["InMemoryRubricRepository"]) - end subgraph eval["RubricEvaluator"] direction TB score(["ScoreExtractingEvaluator\n(reads score from ctx)"]) @@ -128,10 +124,8 @@ flowchart LR end style engine fill:#2c2c2e, stroke:#3a3a3c, color:#ebebf5, stroke-width:1px - style repo fill:#3a3a3c, stroke:#48484a, color:#ebebf5, stroke-width:1px style eval fill:#3a3a3c, stroke:#48484a, color:#ebebf5, stroke-width:1px style model fill:#3a3a3c, stroke:#48484a, color:#ebebf5, stroke-width:1px - style inmem fill:#2c2c2e, stroke:#48484a, color:#ebebf5, stroke-width:1px style score fill:#2c2c2e, stroke:#0A84FF, color:#ebebf5, stroke-width:1px style rubric fill:#2c2c2e, stroke:#48484a, color:#ebebf5, stroke-width:1px style criterion fill:#2c2c2e, stroke:#48484a, color:#ebebf5, stroke-width:1px @@ -144,8 +138,7 @@ flowchart LR | Type | Description | |--------------------|-------------------------------------------------------------------| -| `RubricEngine` | Orchestrates evaluation using repository and evaluator | -| `RubricRepository` | Stores rubric definitions (in-memory by default) | +| `RubricEngine` | Orchestrates evaluation using evaluator | | `RubricEvaluator` | Evaluates output against criteria (self-eval or external LLM) | | `Rubric` | Immutable rubric definition with pass threshold and criteria list | | `Criterion` | Single evaluation dimension with weight and minimum score | @@ -296,7 +289,7 @@ hensu-core/src/main/java/io/hensu/core/ │ ├── enricher/ │ │ ├── EngineVariableInjector.java # Single-injector interface │ │ ├── EngineVariablePromptEnricher.java # Composite enricher — runs injector chain before each agent call -│ │ ├── RubricPromptInjector.java # Injects rubric criteria when node.rubricId is set +│ │ ├── RubricPromptInjector.java # Injects rubric criteria when node has a parsed Rubric │ │ ├── ScoreVariableInjector.java # Injects `score` requirement on ScoreTransition nodes or consensus branches │ │ ├── ApprovalVariableInjector.java # Injects `approved` requirement on ApprovalTransition nodes or consensus branches │ │ ├── RecommendationVariableInjector.java # Injects `recommendation` on score/approval nodes or consensus branches @@ -350,8 +343,6 @@ hensu-core/src/main/java/io/hensu/core/ │ └── WorkflowValidator.java # Load-time validator for writes + prompt {variable} refs ├── rubric/ # Quality evaluation engine │ ├── RubricEngine.java # Evaluation orchestrator -│ ├── RubricRepository.java # Rubric storage interface -│ ├── RubricParser.java # Markdown rubric file parser │ ├── model/ # Rubric, Criterion, ScoreCondition, etc. │ └── evaluator/ │ ├── RubricEvaluator.java # Evaluator interface diff --git a/hensu-core/src/main/java/io/hensu/core/execution/enricher/EngineVariablePromptEnricher.java b/hensu-core/src/main/java/io/hensu/core/execution/enricher/EngineVariablePromptEnricher.java index 9f5e3e4..c12ac13 100644 --- a/hensu-core/src/main/java/io/hensu/core/execution/enricher/EngineVariablePromptEnricher.java +++ b/hensu-core/src/main/java/io/hensu/core/execution/enricher/EngineVariablePromptEnricher.java @@ -12,7 +12,7 @@ /// /// ### Default pipeline /// {@link #DEFAULT} runs in this order: -/// 1. {@link RubricPromptInjector} — injects rubric criteria when `node.getRubricId()` is set +/// 1. {@link RubricPromptInjector} — injects rubric criteria when `node.getRubric()` is set /// 2. {@link ScoreVariableInjector} — injects `score` requirement when a /// {@link io.hensu.core.workflow.transition.ScoreTransition} exists or consensus branch /// needs self-scoring diff --git a/hensu-core/src/main/java/io/hensu/core/execution/enricher/RubricPromptInjector.java b/hensu-core/src/main/java/io/hensu/core/execution/enricher/RubricPromptInjector.java index 9d29867..d27dc6f 100644 --- a/hensu-core/src/main/java/io/hensu/core/execution/enricher/RubricPromptInjector.java +++ b/hensu-core/src/main/java/io/hensu/core/execution/enricher/RubricPromptInjector.java @@ -1,22 +1,16 @@ package io.hensu.core.execution.enricher; import io.hensu.core.execution.executor.ExecutionContext; -import io.hensu.core.rubric.RubricEngine; -import io.hensu.core.rubric.RubricParser; import io.hensu.core.rubric.model.Criterion; import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.node.Node; import io.hensu.core.workflow.node.StandardNode; -import java.nio.file.Path; -import java.util.Map; -import java.util.Optional; /// Injects rubric criteria into the node prompt. /// -/// Applied when the node declares a `rubricId`. This injector loads the rubric -/// (from the engine cache or from disk), appends a horizontal rule to create a -/// clear cognitive boundary between the content under review and the evaluator -/// instructions, then appends the criteria list. +/// Applied when the node carries a parsed {@link Rubric}. Appends a horizontal rule +/// to create a clear cognitive boundary between the content under review and the +/// evaluator instructions, then appends the criteria list. /// /// ### Example output appended to prompt /// @@ -47,19 +41,18 @@ /// Virtual Threads. /// /// @see EngineVariableInjector -/// @see io.hensu.core.rubric.model.Rubric +/// @see Rubric public class RubricPromptInjector implements EngineVariableInjector { @Override public String inject(String prompt, Node node, ExecutionContext ctx) { - if (!(node instanceof StandardNode sn) || sn.getRubricId() == null) { + if (!(node instanceof StandardNode sn) || sn.getRubric() == null) { return prompt; } - Rubric rubric = loadRubric(sn.getRubricId(), ctx); - return inject(prompt, rubric); + return buildCriteriaSection(prompt, sn.getRubric()); } - /// Injects rubric criteria directly from a `Rubric` object. + /// Injects rubric criteria directly from a {@link Rubric} object. /// /// @param prompt the base prompt text, not null /// @param rubric the rubric whose criteria are injected, not null @@ -89,21 +82,4 @@ protected String buildCriteriaSection(String prompt, Rubric rubric) { } return sb.toString().stripTrailing(); } - - private Rubric loadRubric(String rubricId, ExecutionContext ctx) { - RubricEngine engine = ctx.getRubricEngine(); - Optional cached = engine.getRubric(rubricId); - if (cached.isPresent()) { - return cached.get(); - } - Map rubricPaths = ctx.getWorkflow().getRubrics(); - String path = rubricPaths.get(rubricId); - if (path == null) { - throw new IllegalStateException( - "No path configured for rubric '" + rubricId + "' in workflow definition"); - } - Rubric rubric = RubricParser.parse(Path.of(path)); - engine.registerRubric(rubric); - return rubric; - } } diff --git a/hensu-core/src/main/java/io/hensu/core/execution/parallel/Branch.java b/hensu-core/src/main/java/io/hensu/core/execution/parallel/Branch.java index 43f0476..410d87f 100644 --- a/hensu-core/src/main/java/io/hensu/core/execution/parallel/Branch.java +++ b/hensu-core/src/main/java/io/hensu/core/execution/parallel/Branch.java @@ -1,6 +1,7 @@ package io.hensu.core.execution.parallel; import io.hensu.core.execution.EngineVariables; +import io.hensu.core.rubric.model.Rubric; import java.util.List; /// Represents a single execution branch within a parallel node. @@ -12,7 +13,7 @@ /// @param id unique identifier for this branch within the parallel node, not null /// @param agentId identifier of the agent to execute this branch, not null /// @param prompt the prompt template to send to the agent, may be null -/// @param rubricId optional rubric identifier for evaluating branch output, may be null +/// @param rubric optional parsed rubric for evaluating branch output, may be null /// @param weight vote weight for weighted consensus strategies (default 1.0), positive /// @param yields ordered list of state variable names this branch produces, never null /// @@ -23,7 +24,7 @@ public record Branch( String id, String agentId, String prompt, - String rubricId, + Rubric rubric, double weight, List yields) { @@ -32,9 +33,9 @@ public record Branch( /// @param id unique identifier for this branch, not null /// @param agentId identifier of the agent to execute this branch, not null /// @param prompt the prompt template, may be null - /// @param rubricId optional rubric identifier, may be null - public Branch(String id, String agentId, String prompt, String rubricId) { - this(id, agentId, prompt, rubricId, 1.0, List.of()); + /// @param rubric optional parsed rubric, may be null + public Branch(String id, String agentId, String prompt, Rubric rubric) { + this(id, agentId, prompt, rubric, 1.0, List.of()); } /// Creates a branch with specified weight and no yields. @@ -42,10 +43,10 @@ public Branch(String id, String agentId, String prompt, String rubricId) { /// @param id unique identifier for this branch, not null /// @param agentId identifier of the agent to execute this branch, not null /// @param prompt the prompt template, may be null - /// @param rubricId optional rubric identifier, may be null + /// @param rubric optional parsed rubric, may be null /// @param weight vote weight, positive - public Branch(String id, String agentId, String prompt, String rubricId, double weight) { - this(id, agentId, prompt, rubricId, weight, List.of()); + public Branch(String id, String agentId, String prompt, Rubric rubric, double weight) { + this(id, agentId, prompt, rubric, weight, List.of()); } /// Canonical constructor – defensively copies yields list and validates diff --git a/hensu-core/src/main/java/io/hensu/core/execution/pipeline/OutputExtractionPostProcessor.java b/hensu-core/src/main/java/io/hensu/core/execution/pipeline/OutputExtractionPostProcessor.java index cbc79f0..dd35dc0 100644 --- a/hensu-core/src/main/java/io/hensu/core/execution/pipeline/OutputExtractionPostProcessor.java +++ b/hensu-core/src/main/java/io/hensu/core/execution/pipeline/OutputExtractionPostProcessor.java @@ -119,7 +119,7 @@ public ProcessorOutcome process(ProcessorContext context) { private List engineVarsFor(StandardNode node) { List vars = new ArrayList<>(); - boolean hasScore = node.getRubricId() != null; + boolean hasScore = node.getRubric() != null; boolean hasApproval = false; for (var rule : node.getTransitionRules()) { if (rule instanceof ScoreTransition) hasScore = true; diff --git a/hensu-core/src/main/java/io/hensu/core/execution/pipeline/RubricPostProcessor.java b/hensu-core/src/main/java/io/hensu/core/execution/pipeline/RubricPostProcessor.java index 52b21a9..3832866 100644 --- a/hensu-core/src/main/java/io/hensu/core/execution/pipeline/RubricPostProcessor.java +++ b/hensu-core/src/main/java/io/hensu/core/execution/pipeline/RubricPostProcessor.java @@ -1,12 +1,11 @@ package io.hensu.core.execution.pipeline; import io.hensu.core.execution.result.AutoBacktrack; -import io.hensu.core.execution.result.ExecutionResult; import io.hensu.core.execution.result.ExecutionStep; import io.hensu.core.rubric.RubricEngine; -import io.hensu.core.rubric.RubricNotFoundException; import io.hensu.core.rubric.evaluator.RubricEvaluation; import io.hensu.core.rubric.evaluator.ScoreExtractingEvaluator; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.state.HensuState; import io.hensu.core.workflow.Workflow; import io.hensu.core.workflow.node.Node; @@ -15,15 +14,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; /// Evaluates rubric quality criteria and triggers auto-backtracking on failure. /// -/// When a node has a `rubricId`, this processor: -/// 1. Evaluates the node output against the rubric (registered by -/// {@link io.hensu.core.execution.enricher.RubricPromptInjector} during prompt enrichment) +/// When a node carries a parsed {@link Rubric}, this processor: +/// 1. Evaluates the node output against the rubric directly (no repository lookup) /// 2. Stores the evaluation in state for {@link ScoreTransition} rules /// 3. If evaluation fails and no user-defined score transition matches, /// determines an auto-backtrack target based on score severity @@ -35,7 +32,6 @@ /// /// ### Contracts /// - **Precondition**: `context.result()` is non-null (post-execution pipeline) -/// - **Precondition**: Rubric is already registered in the engine (done during prompt enrichment) /// - **Postcondition**: Returns empty or terminal result /// - **Side effects**: Mutates `state.rubricEvaluation`, may mutate context map /// and history on auto-backtrack @@ -73,22 +69,13 @@ public String id() { @Override public ProcessorOutcome process(ProcessorContext context) { var node = context.currentNode(); - if (node.getRubricId() == null) { + Rubric rubric = node.getRubric(); + if (rubric == null) { return ProcessorOutcome.CONTINUE; } - RubricEvaluation evaluation; - try { - evaluation = - rubricEngine.evaluate( - node.getRubricId(), context.result(), context.state().getContext()); - } catch (RubricNotFoundException e) { - logger.severe( - "Rubric evaluation failed for node " + node.getId() + ": " + e.getMessage()); - return ProcessorOutcome.terminal( - new ExecutionResult.Failure( - context.state(), new IllegalStateException(e.getMessage(), e))); - } + RubricEvaluation evaluation = + rubricEngine.evaluate(rubric, context.result(), context.state().getContext()); context.state().setRubricEvaluation(evaluation); @@ -224,9 +211,7 @@ private String findEarliestLogicalStep(HensuState state, Workflow workflow) { .filter( step -> { Node node = workflow.getNodes().get(step.getNodeId()); - return node != null - && node.getRubricId() != null - && !node.getRubricId().isEmpty(); + return node != null && node.getRubric() != null; }) .findFirst(); @@ -235,16 +220,15 @@ private String findEarliestLogicalStep(HensuState state, Workflow workflow) { private String findPreviousPhase(String currentNodeId, HensuState state, Workflow workflow) { Node currentNode = workflow.getNodes().get(currentNodeId); - String currentRubric = currentNode != null ? currentNode.getRubricId() : null; + Rubric currentRubric = currentNode != null ? currentNode.getRubric() : null; List steps = state.getHistory().getSteps(); for (int i = steps.size() - 1; i >= 0; i--) { ExecutionStep step = steps.get(i); Node stepNode = workflow.getNodes().get(step.getNodeId()); if (stepNode != null - && stepNode.getRubricId() != null - && !stepNode.getRubricId().isEmpty() - && !Objects.equals(stepNode.getRubricId(), currentRubric)) { + && stepNode.getRubric() != null + && stepNode.getRubric() != currentRubric) { return step.getNodeId(); } } diff --git a/hensu-core/src/main/java/io/hensu/core/rubric/RubricEngine.java b/hensu-core/src/main/java/io/hensu/core/rubric/RubricEngine.java index e56b009..6c5c644 100644 --- a/hensu-core/src/main/java/io/hensu/core/rubric/RubricEngine.java +++ b/hensu-core/src/main/java/io/hensu/core/rubric/RubricEngine.java @@ -13,17 +13,25 @@ /// Quality evaluation engine for rubric-based output assessment. /// -/// Evaluates workflow node outputs against configurable rubrics to determine +/// Evaluates workflow node outputs against parsed {@link Rubric} objects to determine /// quality scores and pass/fail status. Supports weighted criteria evaluation /// and provides detailed per-criterion feedback. /// +/// ### Repository placeholder +/// +/// The {@link RubricRepository} field and its accessor methods ({@link #registerRubric}, +/// {@link #exists}, {@link #getRubric}) are not used by the current execution pipeline. +/// Rubrics are parsed at build time and stored directly on the node — the engine +/// receives them as typed {@link Rubric} arguments via {@link #evaluate}. The repository +/// is retained as a placeholder for a future DB-backed rubric store where rubrics are +/// managed independently of workflow definitions. +/// /// ### Contracts -/// - **Precondition**: Rubric must be registered before evaluation /// - **Postcondition**: Returns complete evaluation with all criteria scores /// - **Invariant**: Rubric scores are normalized to 0-100 scale /// /// @implNote Thread-safe if the underlying repository and evaluator are thread-safe. -/// The engine itself maintains no mutable state. +/// The engine itself maintains no mutable state beyond the repository delegate. /// /// @see RubricEvaluator for criterion evaluation logic /// @see Rubric for rubric definition structure @@ -42,7 +50,7 @@ public RubricEngine(RubricRepository repository, RubricEvaluator evaluator) { this.evaluator = evaluator; } - /// Registers a rubric in the repository for later evaluation. + /// Registers a rubric in the repository for later retrieval. /// /// @apiNote **Side effects**: Modifies the rubric repository /// @@ -67,31 +75,17 @@ public Optional getRubric(String rubricId) { return repository.findById(rubricId); } - /// Evaluates a node result against a registered rubric. + /// Evaluates a node result against the given rubric. /// /// Calculates weighted scores for each criterion and produces an overall /// score normalized to 0-100. The evaluation passes if the score meets /// the rubric's pass threshold. /// - /// @param rubricId identifier of the registered rubric, not null + /// @param rubric the parsed rubric to evaluate against, not null /// @param result node execution result to evaluate, not null /// @param context execution context for evaluation, not null /// @return evaluation result with score and criterion details, never null - /// @throws RubricNotFoundException if rubricId is not registered public RubricEvaluation evaluate( - String rubricId, NodeResult result, Map context) - throws RubricNotFoundException { - - Rubric rubric = - repository - .findById(rubricId) - .orElseThrow( - () -> new RubricNotFoundException("Rubric not found: " + rubricId)); - - return evaluateRubric(rubric, result, context); - } - - private RubricEvaluation evaluateRubric( Rubric rubric, NodeResult result, Map context) { List criterionEvaluations = new ArrayList<>(); diff --git a/hensu-core/src/main/java/io/hensu/core/rubric/RubricParser.java b/hensu-core/src/main/java/io/hensu/core/rubric/RubricParser.java index deb86b8..dda50ed 100644 --- a/hensu-core/src/main/java/io/hensu/core/rubric/RubricParser.java +++ b/hensu-core/src/main/java/io/hensu/core/rubric/RubricParser.java @@ -31,17 +31,15 @@ public static Rubric parse(Path path) { } } - private static Rubric parseContent(String filename, String content) { - ParseState state = new ParseState(filename); + public static Rubric parseContent(String nodeId, String content) { + ParseState state = new ParseState(nodeId); - for (String line : content.split("\n")) { - parseLine(line.trim(), state); - } + content.lines().forEach(line -> parseLine(line.trim(), state)); state.saveCurrentSubcriterion(); state.ensureDefaultCriterion(); - return state.buildRubric(); + return state.buildRubric(content); } private static void parseLine(String line, ParseState state) { @@ -159,8 +157,8 @@ private static class ParseState { final List criteria = new ArrayList<>(); - ParseState(String filename) { - this.id = filename.replace(".md", ""); + ParseState(String nodeId) { + this.id = nodeId.replace(".md", ""); this.name = this.id; } @@ -194,7 +192,7 @@ void ensureDefaultCriterion() { } } - Rubric buildRubric() { + Rubric buildRubric(String rawContent) { return Rubric.builder() .id(id) .name(name) @@ -202,6 +200,7 @@ Rubric buildRubric() { .type(type) .passThreshold(passThreshold) .criteria(criteria) + .rawContent(rawContent) .build(); } } diff --git a/hensu-core/src/main/java/io/hensu/core/rubric/evaluator/ScoreExtractingEvaluator.java b/hensu-core/src/main/java/io/hensu/core/rubric/evaluator/ScoreExtractingEvaluator.java index 77917a4..1248d50 100644 --- a/hensu-core/src/main/java/io/hensu/core/rubric/evaluator/ScoreExtractingEvaluator.java +++ b/hensu-core/src/main/java/io/hensu/core/rubric/evaluator/ScoreExtractingEvaluator.java @@ -56,7 +56,7 @@ public double evaluate(Criterion criterion, NodeResult result, Map criteria; + private final String rawContent; private Rubric(Builder builder) { this.id = Objects.requireNonNull(builder.id, "Rubric ID required"); @@ -35,7 +35,8 @@ private Rubric(Builder builder) { this.version = builder.version; this.type = builder.type; this.passThreshold = builder.passThreshold; - this.criteria = Collections.unmodifiableList(builder.criteria); + this.criteria = builder.criteria; + this.rawContent = builder.rawContent; validate(); } @@ -91,6 +92,17 @@ public List getCriteria() { return criteria; } + /// Returns the original markdown source used to construct this rubric. + /// + /// Preserves author prose, examples, and edge-case guidance that the parser + /// discards when extracting structured fields. Used by prompt injection to + /// present the full rubric to the LLM. + /// + /// @return source markdown, or null for programmatically-constructed rubrics + public String getRawContent() { + return rawContent; + } + /// Creates a new rubric builder. /// /// @return new builder instance, never null @@ -108,6 +120,7 @@ public static final class Builder { private RubricType type = RubricType.STANDARD; private double passThreshold = 70.0; private List criteria = List.of(); + private String rawContent; private Builder() {} @@ -165,6 +178,15 @@ public Builder criteria(List criteria) { return this; } + /// Sets the original markdown source. + /// + /// @param rawContent source markdown, nullable + /// @return this builder for chaining + public Builder rawContent(String rawContent) { + this.rawContent = rawContent; + return this; + } + /// Builds the immutable rubric. /// /// @return new Rubric instance, never null diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/Workflow.java b/hensu-core/src/main/java/io/hensu/core/workflow/Workflow.java index b486ed0..5f83844 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/Workflow.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/Workflow.java @@ -1,9 +1,7 @@ package io.hensu.core.workflow; import io.hensu.core.agent.AgentConfig; -import io.hensu.core.execution.parallel.Branch; import io.hensu.core.workflow.node.Node; -import io.hensu.core.workflow.node.ParallelNode; import io.hensu.core.workflow.state.WorkflowStateSchema; import java.util.Collections; import java.util.Map; @@ -12,12 +10,11 @@ /// Immutable workflow definition representing a complete execution graph. /// /// A workflow is a pure data structure containing agent configurations, -/// rubric mappings, node definitions, and execution metadata. Workflows -/// are constructed via the builder pattern and validated on build. +/// node definitions, and execution metadata. Workflows are constructed +/// via the builder pattern and validated on build. /// /// ### Structure /// - **Agents**: Named agent configurations for node execution -/// - **Rubrics**: Mapping of rubric names to definition IDs /// - **Nodes**: Graph of execution nodes with transitions /// - **Config**: Execution behavior settings (timeouts, retries) /// - **Metadata**: Descriptive information (author, description) @@ -38,7 +35,6 @@ public final class Workflow { private final String id; private final String version; private final Map agents; - private final Map rubrics; private final Map nodes; private final String startNode; private final WorkflowConfig config; @@ -49,7 +45,6 @@ private Workflow(Builder builder) { this.id = Objects.requireNonNull(builder.id, "Workflow ID required"); this.version = builder.version; this.agents = Collections.unmodifiableMap(builder.agents); - this.rubrics = Collections.unmodifiableMap(builder.rubrics); this.nodes = Collections.unmodifiableMap(builder.nodes); this.startNode = Objects.requireNonNull(builder.startNode, "Start node required"); this.config = builder.config; @@ -64,35 +59,6 @@ private void validate() { throw new IllegalStateException( "Start node '" + startNode + "' not found in workflow nodes"); } - - for (Map.Entry entry : nodes.entrySet()) { - String rubricId = entry.getValue().getRubricId(); - if (rubricId != null && !rubricId.isEmpty() && !rubrics.containsKey(rubricId)) { - throw new IllegalStateException( - "Node '" - + entry.getKey() - + "' references rubric '" - + rubricId - + "' which is not declared in workflow rubrics"); - } - - if (entry.getValue() instanceof ParallelNode pn) { - for (Branch branch : pn.getBranches()) { - if (branch.rubricId() != null - && !branch.rubricId().isEmpty() - && !rubrics.containsKey(branch.rubricId())) { - throw new IllegalStateException( - "Branch '" - + branch.id() - + "' in parallel node '" - + entry.getKey() - + "' references rubric '" - + branch.rubricId() - + "' which is not declared in workflow rubrics"); - } - } - } - } } /// Returns the unique workflow identifier. @@ -116,13 +82,6 @@ public Map getAgents() { return agents; } - /// Returns the rubric name to ID mappings. - /// - /// @return unmodifiable map of rubric name to definition ID, never null - public Map getRubrics() { - return rubrics; - } - /// Returns all workflow nodes by ID. /// /// @return unmodifiable map of node ID to node definition, never null @@ -180,7 +139,6 @@ public static final class Builder { private String id; private String version = "1.0.0"; private Map agents = Map.of(); - private Map rubrics = Map.of(); private Map nodes = Map.of(); private String startNode; private WorkflowConfig config; @@ -216,15 +174,6 @@ public Builder agents(Map agents) { return this; } - /// Sets the rubric name mappings. - /// - /// @param rubrics map of rubric name to definition ID, not null - /// @return this builder for chaining - public Builder rubrics(Map rubrics) { - this.rubrics = Map.copyOf(rubrics); - return this; - } - /// Sets the workflow nodes. /// /// @param nodes map of node ID to node definition, not null diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/ActionNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/ActionNode.java index 41216dd..e746431 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/ActionNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/ActionNode.java @@ -1,6 +1,7 @@ package io.hensu.core.workflow.node; import io.hensu.core.execution.action.Action; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.TransitionRule; import java.util.List; import java.util.Objects; @@ -45,7 +46,7 @@ public List getActions() { /// /// @return null, action nodes do not support rubric evaluation @Override - public String getRubricId() { + public Rubric getRubric() { return null; } diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/EndNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/EndNode.java index b5da8a0..57d7f67 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/EndNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/EndNode.java @@ -3,6 +3,7 @@ import io.hensu.core.execution.action.Action; import io.hensu.core.execution.executor.EndNodeExecutor; import io.hensu.core.execution.result.ExitStatus; +import io.hensu.core.rubric.model.Rubric; import java.util.Objects; /// End workflow node representing an end state. @@ -43,7 +44,7 @@ public ExitStatus getStatus() { /// /// @return null, end nodes do not support rubric evaluation @Override - public String getRubricId() { + public Rubric getRubric() { return null; } diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/ForkNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/ForkNode.java index 1369b08..c5ae1d2 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/ForkNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/ForkNode.java @@ -1,5 +1,6 @@ package io.hensu.core.workflow.node; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.TransitionRule; import java.util.List; import java.util.Map; @@ -45,7 +46,7 @@ public NodeType getNodeType() { } @Override - public String getRubricId() { + public Rubric getRubric() { return null; } diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/GenericNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/GenericNode.java index 194c700..951704a 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/GenericNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/GenericNode.java @@ -1,5 +1,6 @@ package io.hensu.core.workflow.node; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.TransitionRule; import java.util.HashMap; import java.util.List; @@ -47,7 +48,7 @@ public class GenericNode extends Node { private final String executorType; private final Map config; private final List transitionRules; - private final String rubricId; + private final Rubric rubric; private GenericNode(Builder builder) { super(builder.id); @@ -55,7 +56,7 @@ private GenericNode(Builder builder) { this.config = Map.copyOf(builder.config); this.transitionRules = builder.transitionRules != null ? List.copyOf(builder.transitionRules) : List.of(); - this.rubricId = builder.rubricId; + this.rubric = builder.rubric; } /// The executor type identifier used to look up the appropriate executor. Multiple nodes can @@ -75,8 +76,8 @@ public List getTransitionRules() { } @Override - public String getRubricId() { - return rubricId; + public Rubric getRubric() { + return rubric; } @Override @@ -93,7 +94,7 @@ public static final class Builder { private String executorType; private Map config = new HashMap<>(); private List transitionRules; - private String rubricId; + private Rubric rubric; private Builder() {} @@ -122,8 +123,8 @@ public Builder transitionRules(List transitionRules) { return this; } - public Builder rubricId(String rubricId) { - this.rubricId = rubricId; + public Builder rubric(Rubric rubric) { + this.rubric = rubric; return this; } diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/JoinNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/JoinNode.java index 6fcd595..2ef92e9 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/JoinNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/JoinNode.java @@ -1,5 +1,6 @@ package io.hensu.core.workflow.node; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.TransitionRule; import java.util.List; @@ -55,7 +56,7 @@ public NodeType getNodeType() { } @Override - public String getRubricId() { + public Rubric getRubric() { return null; } diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/LoopNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/LoopNode.java index 3f616a2..9bc6979 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/LoopNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/LoopNode.java @@ -1,5 +1,6 @@ package io.hensu.core.workflow.node; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.BreakRule; import io.hensu.core.workflow.transition.LoopCondition; @@ -12,7 +13,7 @@ public LoopNode(String id) { } @Override - public String getRubricId() { + public Rubric getRubric() { return null; } diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/Node.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/Node.java index 35e3230..bf84af9 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/Node.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/Node.java @@ -1,5 +1,6 @@ package io.hensu.core.workflow.node; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.TransitionRule; import java.util.List; @@ -41,10 +42,10 @@ public String getId() { return id; } - /// Returns the rubric ID for quality evaluation, if configured. + /// Returns the parsed rubric for quality evaluation, if configured. /// - /// @return rubric identifier, or null if no rubric evaluation - public abstract String getRubricId(); + /// @return parsed rubric object, or null if no rubric evaluation + public abstract Rubric getRubric(); /// Returns the node type for executor dispatch. /// diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/ParallelNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/ParallelNode.java index c22abe2..d73ab00 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/ParallelNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/ParallelNode.java @@ -3,6 +3,7 @@ import io.hensu.core.execution.parallel.Branch; import io.hensu.core.execution.parallel.ConsensusConfig; import io.hensu.core.execution.parallel.ConsensusStrategy; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.TransitionRule; import java.util.ArrayList; import java.util.List; @@ -32,7 +33,7 @@ private ParallelNode(Builder builder) { } @Override - public String getRubricId() { + public Rubric getRubric() { return null; // Parallel nodes don't have a direct rubric; each branch may have one } @@ -85,8 +86,8 @@ public Builder branch(String branchId, String agentId, String prompt) { return this; } - public Builder branch(String branchId, String agentId, String prompt, String rubricId) { - this.branches.add(new Branch(branchId, agentId, prompt, rubricId)); + public Builder branch(String branchId, String agentId, String prompt, Rubric rubric) { + this.branches.add(new Branch(branchId, agentId, prompt, rubric)); return this; } diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/StandardNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/StandardNode.java index 19e8f53..cd64781 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/StandardNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/StandardNode.java @@ -3,6 +3,7 @@ import io.hensu.core.plan.Plan; import io.hensu.core.plan.PlanningConfig; import io.hensu.core.review.ReviewConfig; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.TransitionRule; import java.util.List; import java.util.Objects; @@ -32,7 +33,7 @@ public final class StandardNode extends Node { private final NodeType nodeType = NodeType.STANDARD; private final String agentId; private final String prompt; - private final String rubricId; + private final Rubric rubric; private final ReviewConfig reviewConfig; private final List transitionRules; private final List writes; @@ -47,7 +48,7 @@ private StandardNode(Builder builder) { // agentId and prompt can be null for nodes that use other execution strategies this.agentId = builder.agentId; this.prompt = builder.prompt; - this.rubricId = builder.rubricId; + this.rubric = builder.rubric; this.reviewConfig = builder.reviewConfig; this.transitionRules = List.copyOf(builder.transitionRules); this.writes = builder.writes != null ? List.copyOf(builder.writes) : List.of(); @@ -81,12 +82,12 @@ public String getPrompt() { return prompt; } - /// Returns the rubric ID for quality evaluation. + /// Returns the parsed rubric for quality evaluation. /// - /// @return rubric identifier, or null if no evaluation required + /// @return parsed rubric object, or null if no evaluation required @Override - public String getRubricId() { - return rubricId; + public Rubric getRubric() { + return rubric; } /// Returns the node type for executor dispatch. @@ -162,7 +163,7 @@ public static final class Builder { private String id; private String agentId; private String prompt; - private String rubricId; + private Rubric rubric; private ReviewConfig reviewConfig; private List transitionRules; private List writes; @@ -199,12 +200,12 @@ public Builder prompt(String prompt) { return this; } - /// Sets the rubric for quality evaluation. + /// Sets the parsed rubric for quality evaluation. /// - /// @param rubricId rubric identifier, may be null + /// @param rubric parsed rubric object, may be null /// @return this builder for chaining - public Builder rubricId(String rubricId) { - this.rubricId = rubricId; + public Builder rubric(Rubric rubric) { + this.rubric = rubric; return this; } diff --git a/hensu-core/src/main/java/io/hensu/core/workflow/node/SubWorkflowNode.java b/hensu-core/src/main/java/io/hensu/core/workflow/node/SubWorkflowNode.java index c0531b8..f116204 100644 --- a/hensu-core/src/main/java/io/hensu/core/workflow/node/SubWorkflowNode.java +++ b/hensu-core/src/main/java/io/hensu/core/workflow/node/SubWorkflowNode.java @@ -1,5 +1,6 @@ package io.hensu.core.workflow.node; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.transition.TransitionRule; import java.util.List; import java.util.Map; @@ -98,7 +99,7 @@ public NodeType getNodeType() { /// /// @return null, sub-workflow nodes do not support rubric evaluation @Override - public String getRubricId() { + public Rubric getRubric() { return null; } diff --git a/hensu-core/src/test/java/io/hensu/core/execution/WorkflowExecutorRubricTest.java b/hensu-core/src/test/java/io/hensu/core/execution/WorkflowExecutorRubricTest.java index d804287..eaa584a 100644 --- a/hensu-core/src/test/java/io/hensu/core/execution/WorkflowExecutorRubricTest.java +++ b/hensu-core/src/test/java/io/hensu/core/execution/WorkflowExecutorRubricTest.java @@ -9,7 +9,6 @@ import io.hensu.core.agent.AgentResponse; import io.hensu.core.execution.result.ExecutionResult; import io.hensu.core.execution.result.ExitStatus; -import io.hensu.core.rubric.RubricNotFoundException; import io.hensu.core.rubric.evaluator.RubricEvaluation; import io.hensu.core.rubric.model.ComparisonOperator; import io.hensu.core.rubric.model.Criterion; @@ -32,17 +31,17 @@ class WorkflowExecutorRubricTest extends WorkflowExecutorTestBase { void shouldAutoBacktrackOnMinorFailure() throws Exception { // score 75 (< 80 threshold) on first attempt → auto-backtrack → retry → score 90 → // SUCCESS. - when(rubricEngine.getRubric("quality")).thenReturn(Optional.of(qualityRubric())); - when(rubricEngine.evaluate(eq("quality"), any(), any())) + var rubric = qualityRubric(); + when(rubricEngine.evaluate(eq(rubric), any(), any())) .thenReturn( RubricEvaluation.builder() - .rubricId("quality") + .rubricId(rubric.getId()) .score(75.0) .passed(false) .build()) .thenReturn( RubricEvaluation.builder() - .rubricId("quality") + .rubricId(rubric.getId()) .score(90.0) .passed(true) .build()); @@ -52,13 +51,12 @@ void shouldAutoBacktrackOnMinorFailure() throws Exception { .id("work") .agentId("test-agent") .prompt("Do work") - .rubricId("quality") + .rubric(rubric) .transitionRules(List.of(new SuccessTransition("end"))) .build(); var workflow = WorkflowTest.TestWorkflowBuilder.create("backtrack-minor") .agent(agentCfg()) - .rubric("quality", "test-path") .startNode(work) .node(end("end")) .build(); @@ -79,11 +77,11 @@ void shouldAutoBacktrackOnMinorFailure() throws Exception { void shouldStopBacktrackAfterMaxRetries() throws Exception { // rubric always returns score 75 (minor failure); after 3 backtracks // the auto-retry limit is reached and normal SuccessTransition fires. - when(rubricEngine.getRubric("quality")).thenReturn(Optional.of(qualityRubric())); - when(rubricEngine.evaluate(eq("quality"), any(), any())) + var rubric = qualityRubric(); + when(rubricEngine.evaluate(eq(rubric), any(), any())) .thenReturn( RubricEvaluation.builder() - .rubricId("quality") + .rubricId(rubric.getId()) .score(75.0) .passed(false) .build()); @@ -93,13 +91,12 @@ void shouldStopBacktrackAfterMaxRetries() throws Exception { .id("work") .agentId("test-agent") .prompt("Do work") - .rubricId("quality") + .rubric(rubric) .transitionRules(List.of(new SuccessTransition("end"))) .build(); var workflow = WorkflowTest.TestWorkflowBuilder.create("backtrack-exhaust") .agent(agentCfg()) - .rubric("quality", "test-path") .startNode(work) .node(end("end")) .build(); @@ -121,11 +118,11 @@ void shouldStopBacktrackAfterMaxRetries() throws Exception { void shouldPreferOnScoreOverAutoBacktrack() throws Exception { // score 60 would trigger auto-backtrack, but ScoreTransition LT 70 → "revise" // takes precedence, routing to a FAILURE end node. - when(rubricEngine.getRubric("quality")).thenReturn(Optional.of(qualityRubric())); - when(rubricEngine.evaluate(eq("quality"), any(), any())) + var rubric = qualityRubric(); + when(rubricEngine.evaluate(eq(rubric), any(), any())) .thenReturn( RubricEvaluation.builder() - .rubricId("quality") + .rubricId(rubric.getId()) .score(60.0) .passed(false) .build()); @@ -135,7 +132,7 @@ void shouldPreferOnScoreOverAutoBacktrack() throws Exception { .id("review") .agentId("test-agent") .prompt("Review this") - .rubricId("quality") + .rubric(rubric) .transitionRules( List.of( new ScoreTransition( @@ -150,7 +147,6 @@ void shouldPreferOnScoreOverAutoBacktrack() throws Exception { var workflow = WorkflowTest.TestWorkflowBuilder.create("score-precedence") .agent(agentCfg()) - .rubric("quality", "test-path") .startNode(review) .node(failEnd("revise")) .node(end("success-end")) @@ -170,15 +166,15 @@ void shouldPreferOnScoreOverAutoBacktrack() throws Exception { // — Stale score isolation between nodes ——————————————————————————————— @Test - void shouldNotLeakRubricScoreBetweenNodes() throws RubricNotFoundException { - // node1 evaluates rubric (score 85); node2 has no rubricId but uses ScoreTransition. + void shouldNotLeakRubricScoreBetweenNodes() { + // node1 evaluates rubric (score 85); node2 has no rubric but uses ScoreTransition. // If the score leaked, node2 would route "good". It must NOT — the score must be // cleared between nodes and throw instead. - when(rubricEngine.getRubric("quality")).thenReturn(Optional.of(qualityRubric())); - when(rubricEngine.evaluate(eq("quality"), any(), any())) + var rubric = qualityRubric(); + when(rubricEngine.evaluate(eq(rubric), any(), any())) .thenReturn( RubricEvaluation.builder() - .rubricId("quality") + .rubricId(rubric.getId()) .score(85.0) .passed(true) .build()); @@ -188,7 +184,7 @@ void shouldNotLeakRubricScoreBetweenNodes() throws RubricNotFoundException { .id("node1") .agentId("test-agent") .prompt("Step 1") - .rubricId("quality") + .rubric(rubric) .transitionRules(List.of(new SuccessTransition("node2"))) .build(); var node2 = @@ -196,7 +192,7 @@ void shouldNotLeakRubricScoreBetweenNodes() throws RubricNotFoundException { .id("node2") .agentId("test-agent") .prompt("Step 2") - // no rubricId — score must not bleed from node1 + // no rubric — score must not bleed from node1 .transitionRules( List.of( new ScoreTransition( @@ -210,7 +206,6 @@ void shouldNotLeakRubricScoreBetweenNodes() throws RubricNotFoundException { var workflow = WorkflowTest.TestWorkflowBuilder.create("stale-rubric") .agent(agentCfg()) - .rubric("quality", "test-path") .startNode(node1) .node(node2) .node(end("good")) diff --git a/hensu-core/src/test/java/io/hensu/core/execution/WorkflowExecutorScoreRoutingTest.java b/hensu-core/src/test/java/io/hensu/core/execution/WorkflowExecutorScoreRoutingTest.java index c011173..f306b4d 100644 --- a/hensu-core/src/test/java/io/hensu/core/execution/WorkflowExecutorScoreRoutingTest.java +++ b/hensu-core/src/test/java/io/hensu/core/execution/WorkflowExecutorScoreRoutingTest.java @@ -3,13 +3,11 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import io.hensu.core.agent.AgentResponse; import io.hensu.core.execution.result.ExecutionResult; import io.hensu.core.execution.result.ExitStatus; -import io.hensu.core.rubric.RubricNotFoundException; import io.hensu.core.rubric.evaluator.RubricEvaluation; import io.hensu.core.rubric.model.ComparisonOperator; import io.hensu.core.rubric.model.Criterion; @@ -35,8 +33,7 @@ class WorkflowExecutorScoreRoutingTest extends WorkflowExecutorTestBase { @MethodSource("scoreRoutingCases") void shouldRouteBySimpleScoreThreshold(double score, boolean passed, ExitStatus expected) throws Exception { - when(rubricEngine.getRubric("quality")).thenReturn(Optional.of(qualityRubric())); - when(rubricEngine.evaluate(eq("quality"), any(), any())) + when(rubricEngine.evaluate(any(Rubric.class), any(), any())) .thenReturn( RubricEvaluation.builder() .rubricId("quality") @@ -49,7 +46,7 @@ void shouldRouteBySimpleScoreThreshold(double score, boolean passed, ExitStatus .id("review") .agentId("test-agent") .prompt("Review this") - .rubricId("quality") + .rubric(qualityRubric()) .transitionRules( List.of( new ScoreTransition( @@ -68,7 +65,6 @@ void shouldRouteBySimpleScoreThreshold(double score, boolean passed, ExitStatus var workflow = WorkflowTest.TestWorkflowBuilder.create("score-route") .agent(agentCfg()) - .rubric("quality", "test-path") .startNode(review) .node(end("excellent")) .node(failEnd("poor")) @@ -95,8 +91,7 @@ static Stream scoreRoutingCases() { void shouldRouteByScoreRange() throws Exception { // Score 75 matches RANGE 70..89 → routes to "good". // Tests the RANGE operator — not covered by the simple threshold cases above. - when(rubricEngine.getRubric("quality")).thenReturn(Optional.of(qualityRubric())); - when(rubricEngine.evaluate(eq("quality"), any(), any())) + when(rubricEngine.evaluate(any(Rubric.class), any(), any())) .thenReturn( RubricEvaluation.builder() .rubricId("quality") @@ -109,7 +104,7 @@ void shouldRouteByScoreRange() throws Exception { .id("review") .agentId("test-agent") .prompt("Review this") - .rubricId("quality") + .rubric(qualityRubric()) .transitionRules( List.of( new ScoreTransition( @@ -133,7 +128,6 @@ void shouldRouteByScoreRange() throws Exception { var workflow = WorkflowTest.TestWorkflowBuilder.create("score-range") .agent(agentCfg()) - .rubric("quality", "test-path") .startNode(review) .node(end("excellent")) .node(end("good")) @@ -154,8 +148,7 @@ void shouldRouteByScoreRange() throws Exception { @Test void shouldAutoApproveWhenRubricPasses() throws Exception { // rubric.passed=true with a single GTE-80 condition that is satisfied → SUCCESS. - when(rubricEngine.getRubric("quality")).thenReturn(Optional.of(qualityRubric())); - when(rubricEngine.evaluate(eq("quality"), any(), any())) + when(rubricEngine.evaluate(any(Rubric.class), any(), any())) .thenReturn( RubricEvaluation.builder() .rubricId("quality") @@ -168,7 +161,7 @@ void shouldAutoApproveWhenRubricPasses() throws Exception { .id("review") .agentId("test-agent") .prompt("Review this") - .rubricId("quality") + .rubric(qualityRubric()) .transitionRules( List.of( new ScoreTransition( @@ -183,7 +176,6 @@ void shouldAutoApproveWhenRubricPasses() throws Exception { var workflow = WorkflowTest.TestWorkflowBuilder.create("score-pass") .agent(agentCfg()) - .rubric("quality", "test-path") .startNode(review) .node(end("end")) .build(); @@ -200,11 +192,10 @@ void shouldAutoApproveWhenRubricPasses() throws Exception { } @Test - void shouldThrowWhenNoScoreConditionMatches() throws RubricNotFoundException { + void shouldThrowWhenNoScoreConditionMatches() { // Score 85 with a single GTE-90 condition → no match → IllegalStateException. // rubric.passed=true prevents auto-backtrack from kicking in. - when(rubricEngine.getRubric("quality")).thenReturn(Optional.of(qualityRubric())); - when(rubricEngine.evaluate(eq("quality"), any(), any())) + when(rubricEngine.evaluate(any(Rubric.class), any(), any())) .thenReturn( RubricEvaluation.builder() .rubricId("quality") @@ -217,7 +208,7 @@ void shouldThrowWhenNoScoreConditionMatches() throws RubricNotFoundException { .id("review") .agentId("test-agent") .prompt("Review this") - .rubricId("quality") + .rubric(qualityRubric()) .transitionRules( List.of( new ScoreTransition( @@ -231,7 +222,6 @@ void shouldThrowWhenNoScoreConditionMatches() throws RubricNotFoundException { var workflow = WorkflowTest.TestWorkflowBuilder.create("score-nomatch") .agent(agentCfg()) - .rubric("quality", "test-path") .startNode(review) .node(end("excellent")) .build(); @@ -245,8 +235,6 @@ void shouldThrowWhenNoScoreConditionMatches() throws RubricNotFoundException { .hasMessageContaining("No valid transition"); } - // — Helpers —————————————————————————————————————————————————————————————— - private static Rubric qualityRubric() { return Rubric.builder() .id("quality") diff --git a/hensu-core/src/test/java/io/hensu/core/execution/enricher/EngineVariablePromptEnricherTest.java b/hensu-core/src/test/java/io/hensu/core/execution/enricher/EngineVariablePromptEnricherTest.java index d14dc2c..b509d11 100644 --- a/hensu-core/src/test/java/io/hensu/core/execution/enricher/EngineVariablePromptEnricherTest.java +++ b/hensu-core/src/test/java/io/hensu/core/execution/enricher/EngineVariablePromptEnricherTest.java @@ -3,7 +3,6 @@ import static org.assertj.core.api.Assertions.assertThat; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -21,7 +20,7 @@ void shouldChainInjectors() { (prompt, _, _) -> prompt + "[A]", (prompt, _, _) -> prompt + "[B]")); - String result = enricher.enrich("base", minimalNode(), ctx(Map.of(), null, null)); + String result = enricher.enrich("base", minimalNode(), ctx(null, null)); assertThat(result).isEqualTo("base[A][B]"); } diff --git a/hensu-core/src/test/java/io/hensu/core/execution/enricher/EnricherTestBase.java b/hensu-core/src/test/java/io/hensu/core/execution/enricher/EnricherTestBase.java index c5bf5c7..98e3f8c 100644 --- a/hensu-core/src/test/java/io/hensu/core/execution/enricher/EnricherTestBase.java +++ b/hensu-core/src/test/java/io/hensu/core/execution/enricher/EnricherTestBase.java @@ -38,19 +38,13 @@ protected StandardNode minimalNode() { .build(); } - /// Builds an {@link ExecutionContext} with configurable rubric paths and state schema. + /// Builds an {@link ExecutionContext} with configurable state schema and rubric engine. /// - /// @param rubricPaths rubric ID to file path map; pass empty map for no rubric paths /// @param schema optional state schema; pass null for no schema /// @param engine optional rubric engine; pass null if not needed - protected ExecutionContext ctx( - Map rubricPaths, WorkflowStateSchema schema, RubricEngine engine) { + protected ExecutionContext ctx(WorkflowStateSchema schema, RubricEngine engine) { var workflowBuilder = - Workflow.builder() - .id("wf") - .startNode("node") - .nodes(Map.of("node", minimalNode())) - .rubrics(rubricPaths); + Workflow.builder().id("wf").startNode("node").nodes(Map.of("node", minimalNode())); if (schema != null) { workflowBuilder.stateSchema(schema); } diff --git a/hensu-core/src/test/java/io/hensu/core/execution/enricher/RubricPromptInjectorTest.java b/hensu-core/src/test/java/io/hensu/core/execution/enricher/RubricPromptInjectorTest.java index d035396..7a0443b 100644 --- a/hensu-core/src/test/java/io/hensu/core/execution/enricher/RubricPromptInjectorTest.java +++ b/hensu-core/src/test/java/io/hensu/core/execution/enricher/RubricPromptInjectorTest.java @@ -1,18 +1,16 @@ package io.hensu.core.execution.enricher; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import io.hensu.core.execution.result.ExitStatus; -import io.hensu.core.rubric.InMemoryRubricRepository; -import io.hensu.core.rubric.RubricEngine; +import io.hensu.core.rubric.RubricParser; import io.hensu.core.rubric.model.Criterion; import io.hensu.core.rubric.model.Rubric; +import io.hensu.core.workflow.WorkflowTest; import io.hensu.core.workflow.node.EndNode; import io.hensu.core.workflow.node.StandardNode; import io.hensu.core.workflow.transition.SuccessTransition; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -22,12 +20,10 @@ class RubricPromptInjectorTest extends EnricherTestBase { private RubricPromptInjector injector; - private RubricEngine rubricEngine; @BeforeEach void setUp() { injector = new RubricPromptInjector(); - rubricEngine = new RubricEngine(new InMemoryRubricRepository(), (_, _, _) -> 0.0); } @Nested @@ -39,20 +35,20 @@ class SkipConditions { void shouldSkipNonStandardNode() { var end = EndNode.builder().id("end").status(ExitStatus.SUCCESS).build(); - assertThat(injector.inject("base prompt", end, ctx(Map.of(), null, rubricEngine))) + assertThat(injector.inject("base prompt", end, ctx(null, null))) .isEqualTo("base prompt"); } @Test - @DisplayName("returns prompt unchanged when rubricId is null") - void shouldSkipWhenNoRubricId() { + @DisplayName("returns prompt unchanged when rubric is null") + void shouldSkipWhenNoRubric() { var node = StandardNode.builder() .id("node") .transitionRules(List.of(new SuccessTransition("next"))) .build(); - assertThat(injector.inject("base prompt", node, ctx(Map.of(), null, rubricEngine))) + assertThat(injector.inject("base prompt", node, ctx(null, null))) .isEqualTo("base prompt"); } } @@ -61,38 +57,21 @@ void shouldSkipWhenNoRubricId() { @DisplayName("rubric loading") class RubricLoading { - @Test - @DisplayName("uses engine cache — does not attempt disk read when rubric is registered") - void shouldUseCachedRubric() { - rubricEngine.registerRubric(rubric("q", "Quality", null)); - var node = - StandardNode.builder() - .id("node") - .rubricId("q") - .transitionRules(List.of(new SuccessTransition("next"))) - .build(); - // empty rubric path map — disk attempt would throw - assertThat(injector.inject("prompt", node, ctx(Map.of(), null, rubricEngine))) - .contains("Score the content above"); - } + private static final String RUBRIC_CONTENT = + WorkflowTest.TestWorkflowBuilder.RUBRIC_CONTENT; @Test - @DisplayName( - "throws IllegalStateException with rubricId in message when path not configured") - void shouldThrowWhenPathMissing() { + @DisplayName("parses rubric content from node and injects criteria") + void shouldParseRubricContent() { var node = StandardNode.builder() .id("node") - .rubricId("unknown-rubric") + .rubric(RubricParser.parseContent("node", RUBRIC_CONTENT)) .transitionRules(List.of(new SuccessTransition("next"))) .build(); - assertThatThrownBy( - () -> - injector.inject( - "prompt", node, ctx(Map.of(), null, rubricEngine))) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("unknown-rubric"); + assertThat(injector.inject("prompt", node, ctx(null, null))) + .contains("Score the content above"); } } @@ -103,7 +82,7 @@ class CriteriaFormatting { @Test @DisplayName("formats criterion with description as `**name** — description?`") void shouldFormatCriterionWithDescription() { - String result = injector.inject("base", rubric("r", "Structure", "Is it organized")); + String result = injector.inject("base", rubric("Structure", "Is it organized")); assertThat(result).contains("- **Structure** — Is it organized?"); } @@ -111,7 +90,7 @@ void shouldFormatCriterionWithDescription() { @Test @DisplayName("omits em-dash when criterion description is blank") void shouldOmitDashWhenDescriptionBlank() { - String result = injector.inject("base", rubric("r", "Clarity", "")); + String result = injector.inject("base", rubric("Clarity", "")); assertThat(result).contains("- **Clarity**"); assertThat(result).doesNotContain(" — "); @@ -144,11 +123,11 @@ void shouldOmitDashWhenDescriptionNull() { // — Helpers —————————————————————————————————————————————————————————————— - private Rubric rubric(String id, String criterionName, String description) { + private Rubric rubric(String criterionName, String description) { Criterion.Builder cb = Criterion.builder().id("c1").name(criterionName); if (description != null) { cb.description(description); } - return Rubric.builder().id(id).name("Test").criteria(List.of(cb.build())).build(); + return Rubric.builder().id("r").name("Test").criteria(List.of(cb.build())).build(); } } diff --git a/hensu-core/src/test/java/io/hensu/core/execution/enricher/WritesVariableInjectorTest.java b/hensu-core/src/test/java/io/hensu/core/execution/enricher/WritesVariableInjectorTest.java index a18060c..6029b9e 100644 --- a/hensu-core/src/test/java/io/hensu/core/execution/enricher/WritesVariableInjectorTest.java +++ b/hensu-core/src/test/java/io/hensu/core/execution/enricher/WritesVariableInjectorTest.java @@ -10,7 +10,6 @@ import io.hensu.core.workflow.state.WorkflowStateSchema; import io.hensu.core.workflow.transition.SuccessTransition; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -35,7 +34,7 @@ class SkipConditions { void shouldSkipNonStandardNode() { var end = EndNode.builder().id("end").status(ExitStatus.SUCCESS).build(); - assertThat(injector.inject("base", end, ctx(Map.of(), null, null))).isEqualTo("base"); + assertThat(injector.inject("base", end, ctx(null, null))).isEqualTo("base"); } @Test @@ -48,7 +47,7 @@ void shouldSkipWhenWritesEmpty() { .transitionRules(List.of(new SuccessTransition("next"))) .build(); - assertThat(injector.inject("base", node, ctx(Map.of(), null, null))).isEqualTo("base"); + assertThat(injector.inject("base", node, ctx(null, null))).isEqualTo("base"); } } @@ -59,7 +58,7 @@ class FieldInjection { @Test @DisplayName("emits field name only when workflow has no state schema") void shouldEmitNameOnlyWithoutSchema() { - String result = injector.inject("base", nodeWithWrites(), ctx(Map.of(), null, null)); + String result = injector.inject("base", nodeWithWrites(), ctx(null, null)); assertThat(result).contains("\"article\""); assertThat(result).doesNotContain(" — "); @@ -77,7 +76,7 @@ void shouldAppendDescriptionFromSchema() { false, "the full article text"))); - String result = injector.inject("base", nodeWithWrites(), ctx(Map.of(), schema, null)); + String result = injector.inject("base", nodeWithWrites(), ctx(schema, null)); assertThat(result).contains("\"article\" — the full article text"); } @@ -93,7 +92,7 @@ void shouldEmitNameOnlyWhenDescriptionNull() { new StateVariableDeclaration( "article", VarType.STRING, false))); - String result = injector.inject("base", nodeWithWrites(), ctx(Map.of(), schema, null)); + String result = injector.inject("base", nodeWithWrites(), ctx(schema, null)); assertThat(result).contains("\"article\""); assertThat(result).doesNotContain(" — "); @@ -110,7 +109,7 @@ void shouldEmitNameOnlyWhenFieldNotInSchema() { new StateVariableDeclaration( "summary", VarType.STRING, false))); - String result = injector.inject("base", nodeWithWrites(), ctx(Map.of(), schema, null)); + String result = injector.inject("base", nodeWithWrites(), ctx(schema, null)); assertThat(result).contains("\"article\""); assertThat(result).doesNotContain(" — "); diff --git a/hensu-core/src/test/java/io/hensu/core/execution/pipeline/RubricPostProcessorTest.java b/hensu-core/src/test/java/io/hensu/core/execution/pipeline/RubricPostProcessorTest.java index b756ef8..ab82d72 100644 --- a/hensu-core/src/test/java/io/hensu/core/execution/pipeline/RubricPostProcessorTest.java +++ b/hensu-core/src/test/java/io/hensu/core/execution/pipeline/RubricPostProcessorTest.java @@ -3,24 +3,24 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import io.hensu.core.execution.EngineVariables; import io.hensu.core.execution.executor.ExecutionContext; import io.hensu.core.execution.executor.NodeResult; import io.hensu.core.execution.result.ExecutionHistory; -import io.hensu.core.execution.result.ExecutionResult; import io.hensu.core.execution.result.ExecutionStep; import io.hensu.core.rubric.RubricEngine; -import io.hensu.core.rubric.RubricNotFoundException; +import io.hensu.core.rubric.RubricParser; import io.hensu.core.rubric.evaluator.RubricEvaluation; import io.hensu.core.rubric.evaluator.ScoreExtractingEvaluator; import io.hensu.core.rubric.model.ComparisonOperator; import io.hensu.core.rubric.model.DoubleRange; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.rubric.model.ScoreCondition; import io.hensu.core.state.HensuState; import io.hensu.core.workflow.Workflow; +import io.hensu.core.workflow.WorkflowTest; import io.hensu.core.workflow.node.Node; import io.hensu.core.workflow.node.StandardNode; import io.hensu.core.workflow.transition.ScoreTransition; @@ -55,7 +55,7 @@ void setUp() { class SkipConditions { @Test - @DisplayName("returns empty when node has no rubricId") + @DisplayName("returns empty when node has no rubric") void shouldSkipWhenNoRubric() { var ctx = contextWithRubric(null); @@ -71,8 +71,8 @@ class PassingEvaluation { @Test @DisplayName("returns empty when rubric passes") - void shouldContinueOnPass() throws RubricNotFoundException { - var ctx = contextWithRubric("quality"); + void shouldContinueOnPass() { + var ctx = contextWithRubric(RUBRIC_CONTENT); mockRubricEvaluation(90.0, true); var result = processor.process(ctx); @@ -82,8 +82,8 @@ void shouldContinueOnPass() throws RubricNotFoundException { @Test @DisplayName("stores evaluation in state") - void shouldStoreEvaluation() throws RubricNotFoundException { - var ctx = contextWithRubric("quality"); + void shouldStoreEvaluation() { + var ctx = contextWithRubric(RUBRIC_CONTENT); mockRubricEvaluation(90.0, true); processor.process(ctx); @@ -100,7 +100,7 @@ class AutoBacktrack { @Test @DisplayName( "backtracks to earliest rubric node in history on critical failure (score < 30)") - void shouldBacktrackOnCriticalFailure() throws RubricNotFoundException { + void shouldBacktrackOnCriticalFailure() { var ctx = contextWithRubricAndHistory(); mockRubricEvaluation(15.0, false); @@ -114,8 +114,7 @@ void shouldBacktrackOnCriticalFailure() throws RubricNotFoundException { @Test @DisplayName( "backtracks to workflow start node on critical failure when history has no rubric steps") - void shouldBacktrackToStartNodeOnCriticalFailureWithEmptyHistory() - throws RubricNotFoundException { + void shouldBacktrackToStartNodeOnCriticalFailureWithEmptyHistory() { // Node "review" is current; workflow start is "start" — history is empty. // findEarliestLogicalStep must fall back to workflow.getStartNode(). Node startNode = @@ -127,7 +126,7 @@ void shouldBacktrackToStartNodeOnCriticalFailureWithEmptyHistory() Node reviewNode = StandardNode.builder() .id("review") - .rubricId("quality") + .rubric(RubricParser.parseContent("review", RUBRIC_CONTENT)) .transitionRules(List.of(new SuccessTransition("next"))) .build(); @@ -145,7 +144,6 @@ void shouldBacktrackToStartNodeOnCriticalFailureWithEmptyHistory() .id("test-wf") .startNode("start") .nodes(Map.of("start", startNode, "review", reviewNode)) - .rubrics(Map.of("quality", "/tmp/rubric.yaml")) .build(); var execCtx = @@ -168,7 +166,7 @@ void shouldBacktrackToStartNodeOnCriticalFailureWithEmptyHistory() @Test @DisplayName("backtracks to previous rubric-bearing node on moderate failure (score < 60)") - void shouldBacktrackOnModerateFailure() throws RubricNotFoundException { + void shouldBacktrackOnModerateFailure() { var ctx = contextWithRubricAndHistory(); mockRubricEvaluation(45.0, false); @@ -181,7 +179,7 @@ void shouldBacktrackOnModerateFailure() throws RubricNotFoundException { @Test @DisplayName("falls through to retry when no prior rubric phase found on moderate failure") - void shouldRetryWhenNoPhaseFoundOnModerateFailure() throws RubricNotFoundException { + void shouldRetryWhenNoPhaseFoundOnModerateFailure() { // Score 45.0 satisfies < 60 (moderate) AND < 80 (minor). // When findPreviousPhase returns null the moderate block does not return — // execution falls through to the minor retry block. @@ -198,8 +196,8 @@ void shouldRetryWhenNoPhaseFoundOnModerateFailure() throws RubricNotFoundExcepti @Test @DisplayName("retries current node on minor failure (score < 80)") - void shouldRetryOnMinorFailure() throws RubricNotFoundException { - var ctx = contextWithRubric("quality"); + void shouldRetryOnMinorFailure() { + var ctx = contextWithRubric(RUBRIC_CONTENT); mockRubricEvaluation(70.0, false); var result = processor.process(ctx); @@ -211,8 +209,8 @@ void shouldRetryOnMinorFailure() throws RubricNotFoundException { @Test @DisplayName("increments retry counter on minor failure") - void shouldIncrementRetryCounter() throws RubricNotFoundException { - var ctx = contextWithRubric("quality"); + void shouldIncrementRetryCounter() { + var ctx = contextWithRubric(RUBRIC_CONTENT); mockRubricEvaluation(70.0, false); processor.process(ctx); @@ -222,8 +220,8 @@ void shouldIncrementRetryCounter() throws RubricNotFoundException { @Test @DisplayName("does not backtrack and records no history entry after max retry attempts") - void shouldStopAfterMaxRetries() throws RubricNotFoundException { - var ctx = contextWithRubric("quality"); + void shouldStopAfterMaxRetries() { + var ctx = contextWithRubric(RUBRIC_CONTENT); ctx.state().getContext().put("retry_attempt", 3); mockRubricEvaluation(70.0, false); @@ -235,7 +233,7 @@ void shouldStopAfterMaxRetries() throws RubricNotFoundException { @Test @DisplayName("does not auto-backtrack when a matching ScoreTransition handles the failure") - void shouldNotAutoBacktrackWhenScoreTransitionExists() throws RubricNotFoundException { + void shouldNotAutoBacktrackWhenScoreTransitionExists() { // A ScoreTransition that matches score 45.0 — processor must defer to it. var condition = new ScoreCondition( @@ -243,7 +241,7 @@ void shouldNotAutoBacktrackWhenScoreTransitionExists() throws RubricNotFoundExce var node = StandardNode.builder() .id("current") - .rubricId("quality") + .rubric(RubricParser.parseContent("current", RUBRIC_CONTENT)) .transitionRules(List.of(new ScoreTransition(List.of(condition)))) .build(); @@ -260,8 +258,8 @@ void shouldNotAutoBacktrackWhenScoreTransitionExists() throws RubricNotFoundExce @Test @DisplayName( "injects backtrack_reason and failed_criteria into context on critical failure") - void shouldInjectContextKeysOnCriticalFailure() throws RubricNotFoundException { - var ctx = contextWithRubric("quality"); + void shouldInjectContextKeysOnCriticalFailure() { + var ctx = contextWithRubric(RUBRIC_CONTENT); mockRubricEvaluation(15.0, false); processor.process(ctx); @@ -275,7 +273,7 @@ void shouldInjectContextKeysOnCriticalFailure() throws RubricNotFoundException { @Test @DisplayName( "injects backtrack_reason and improvement_suggestions into context on moderate failure") - void shouldInjectContextKeysOnModerateFailure() throws RubricNotFoundException { + void shouldInjectContextKeysOnModerateFailure() { var ctx = contextWithRubricAndHistory(); mockRubricEvaluation(45.0, false); @@ -292,27 +290,10 @@ void shouldInjectContextKeysOnModerateFailure() throws RubricNotFoundException { @DisplayName("error handling") class ErrorHandling { - @Test - @DisplayName("returns Failure on RubricNotFoundException") - void shouldTerminateOnMissingRubric() throws RubricNotFoundException { - var ctx = contextWithRubric("missing-rubric"); - when(rubricEngine.evaluate(eq("missing-rubric"), any(), any())) - .thenThrow(new RubricNotFoundException("Rubric not found: missing-rubric")); - - var result = processor.process(ctx); - - assertThat(result).isInstanceOf(ProcessorOutcome.Terminal.class); - assertThat(((ProcessorOutcome.Terminal) result).result()) - .isInstanceOf(ExecutionResult.Failure.class); - } - @Test @DisplayName("handles plain String in _rubric_criterion_feedback without throwing") - void shouldHandleStringRecommendationsGracefully() throws RubricNotFoundException { - // Regression: OutputExtractionPostProcessor may store a plain String under this key - // when an agent produces a single-value JSON. The safe instanceof List cast must - // silently skip it — backtrack logic must still execute normally. - var ctx = contextWithRubric("quality"); + void shouldHandleStringRecommendationsGracefully() { + var ctx = contextWithRubric(RUBRIC_CONTENT); ctx.state() .getContext() .put(ScoreExtractingEvaluator.RECOMMENDATIONS_KEY, "just a plain string"); @@ -325,8 +306,8 @@ void shouldHandleStringRecommendationsGracefully() throws RubricNotFoundExceptio // — Helpers ————————————————————————————————————————————————————————————— - private void mockRubricEvaluation(double score, boolean passed) throws RubricNotFoundException { - when(rubricEngine.evaluate(eq("quality"), any(), any())) + private void mockRubricEvaluation(double score, boolean passed) { + when(rubricEngine.evaluate(any(Rubric.class), any(), any())) .thenReturn( RubricEvaluation.builder() .rubricId("quality") @@ -335,13 +316,14 @@ private void mockRubricEvaluation(double score, boolean passed) throws RubricNot .build()); } - private ProcessorContext contextWithRubric(String rubricId) { - Node node = - StandardNode.builder() - .id("node") - .rubricId(rubricId) - .transitionRules(List.of(new SuccessTransition("next"))) - .build(); + private static final String RUBRIC_CONTENT = WorkflowTest.TestWorkflowBuilder.RUBRIC_CONTENT; + + private ProcessorContext contextWithRubric(String rubricContent) { + var builder = StandardNode.builder().id("node"); + if (rubricContent != null) { + builder.rubric(RubricParser.parseContent("node", rubricContent)); + } + Node node = builder.transitionRules(List.of(new SuccessTransition("next"))).build(); var state = new HensuState.Builder() @@ -357,7 +339,6 @@ private ProcessorContext contextWithRubric(String rubricId) { .id("test-wf") .startNode("node") .nodes(Map.of("node", node)) - .rubrics(rubricId != null ? Map.of(rubricId, "/tmp/rubric.yaml") : Map.of()) .build(); var execCtx = @@ -386,7 +367,6 @@ private ProcessorContext contextWithNode(Node node) { .id("test-wf") .startNode(node.getId()) .nodes(Map.of(node.getId(), node)) - .rubrics(Map.of("quality", "/tmp/rubric.yaml")) .build(); var execCtx = @@ -401,18 +381,29 @@ private ProcessorContext contextWithNode(Node node) { /// Builds a context where "previous" node (with a different rubric) precedes the current node. /// Enables testing of `findEarliestLogicalStep` and `findPreviousPhase`. + private static final String OTHER_RUBRIC_CONTENT = + """ + # Rubric: other-rubric + ## Metadata + - pass_threshold: 60 + ### Other + #### Completeness + - points: 10 + - evaluation: Is it complete + """; + private ProcessorContext contextWithRubricAndHistory() { Node previousNode = StandardNode.builder() .id("previous") - .rubricId("other-rubric") + .rubric(RubricParser.parseContent("previous", OTHER_RUBRIC_CONTENT)) .transitionRules(List.of(new SuccessTransition("current"))) .build(); Node currentNode = StandardNode.builder() .id("current") - .rubricId("quality") + .rubric(RubricParser.parseContent("current", RUBRIC_CONTENT)) .transitionRules(List.of(new SuccessTransition("next"))) .build(); @@ -438,12 +429,6 @@ private ProcessorContext contextWithRubricAndHistory() { .id("test-wf") .startNode("previous") .nodes(Map.of("previous", previousNode, "current", currentNode)) - .rubrics( - Map.of( - "quality", - "/tmp/rubric.yaml", - "other-rubric", - "/tmp/other.yaml")) .build(); var execCtx = @@ -468,7 +453,7 @@ private ProcessorContext contextWithHistoryAndNoPriorRubricNode() { Node currentNode = StandardNode.builder() .id("current") - .rubricId("quality") + .rubric(RubricParser.parseContent("current", RUBRIC_CONTENT)) .transitionRules(List.of(new SuccessTransition("next"))) .build(); @@ -494,7 +479,6 @@ private ProcessorContext contextWithHistoryAndNoPriorRubricNode() { .id("test-wf") .startNode("previous") .nodes(Map.of("previous", previousNode, "current", currentNode)) - .rubrics(Map.of("quality", "/tmp/rubric.yaml")) .build(); var execCtx = diff --git a/hensu-core/src/test/java/io/hensu/core/rubric/RubricEngineTest.java b/hensu-core/src/test/java/io/hensu/core/rubric/RubricEngineTest.java index 939817f..4a5a2c2 100644 --- a/hensu-core/src/test/java/io/hensu/core/rubric/RubricEngineTest.java +++ b/hensu-core/src/test/java/io/hensu/core/rubric/RubricEngineTest.java @@ -1,7 +1,6 @@ package io.hensu.core.rubric; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import io.hensu.core.execution.executor.NodeResult; import io.hensu.core.rubric.evaluator.RubricEvaluation; @@ -9,104 +8,91 @@ import io.hensu.core.rubric.model.Rubric; import java.util.HashMap; import java.util.List; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class RubricEngineTest { - private RubricEngine engine; - - @BeforeEach - void setUp() { - engine = new RubricEngine(new InMemoryRubricRepository(), (_, _, _) -> 0.0); - } - @Test - void shouldThrowWhenRubricNotRegistered() { - assertThatThrownBy(() -> engine.evaluate("missing", NodeResult.empty(), new HashMap<>())) - .isInstanceOf(RubricNotFoundException.class); - } + void shouldReturnZeroScoreWhenEvaluatorReturnsZero() { + var engine = engineWithFixedScore(0.0); - @Test - void shouldReturnZeroScoreWhenEvaluatorReturnsZero() throws RubricNotFoundException { - engine = engineWithFixedScore(0.0); - engine.registerRubric(rubric(criterion("c1", 1.0))); - - RubricEvaluation eval = engine.evaluate("r1", NodeResult.empty(), new HashMap<>()); + RubricEvaluation eval = + engine.evaluate(rubric(criterion("c1", 1.0)), NodeResult.empty(), new HashMap<>()); assertThat(eval.getScore()).isZero(); assertThat(eval.isPassed()).isFalse(); } @Test - void shouldReturnPerfectScoreWhenEvaluatorReturnsHundred() throws RubricNotFoundException { - engine = engineWithFixedScore(100.0); - engine.registerRubric(rubric(criterion("c1", 1.0))); + void shouldReturnPerfectScoreWhenEvaluatorReturnsHundred() { + var engine = engineWithFixedScore(100.0); - RubricEvaluation eval = engine.evaluate("r1", NodeResult.empty(), new HashMap<>()); + RubricEvaluation eval = + engine.evaluate(rubric(criterion("c1", 1.0)), NodeResult.empty(), new HashMap<>()); assertThat(eval.getScore()).isEqualTo(100.0); assertThat(eval.isPassed()).isTrue(); } @Test - void shouldApplyWeightsProportionally() throws RubricNotFoundException { + void shouldApplyWeightsProportionally() { // c1: weight=3, score=100.0 → contributes 300.0 // c2: weight=1, score=0.0 → contributes 0.0 // finalScore = 300.0 / 4.0 = 75.0 - RubricEngine mixed = + var engine = new RubricEngine( new InMemoryRubricRepository(), (criterion, _, _) -> criterion.getId().equals("c1") ? 100.0 : 0.0); - mixed.registerRubric(rubric(criterion("c1", 3.0), criterion("c2", 1.0))); - - RubricEvaluation eval = mixed.evaluate("r1", NodeResult.empty(), new HashMap<>()); + RubricEvaluation eval = + engine.evaluate( + rubric(criterion("c1", 3.0), criterion("c2", 1.0)), + NodeResult.empty(), + new HashMap<>()); assertThat(eval.getScore()).isEqualTo(75.0); assertThat(eval.isPassed()).isTrue(); } @Test - void shouldPassAtExactThreshold() throws RubricNotFoundException { - // finalScore == passThreshold must pass (>= not >) - engine = engineWithFixedScore(70.0); - engine.registerRubric(rubric(criterion("c1", 1.0))); + void shouldPassAtExactThreshold() { + var engine = engineWithFixedScore(70.0); - RubricEvaluation eval = engine.evaluate("r1", NodeResult.empty(), new HashMap<>()); + RubricEvaluation eval = + engine.evaluate(rubric(criterion("c1", 1.0)), NodeResult.empty(), new HashMap<>()); assertThat(eval.getScore()).isEqualTo(70.0); assertThat(eval.isPassed()).isTrue(); } @Test - void shouldFailWhenScoreJustBelowThreshold() throws RubricNotFoundException { - engine = engineWithFixedScore(69.9); - engine.registerRubric(rubric(criterion("c1", 1.0))); + void shouldFailWhenScoreJustBelowThreshold() { + var engine = engineWithFixedScore(69.9); - RubricEvaluation eval = engine.evaluate("r1", NodeResult.empty(), new HashMap<>()); + RubricEvaluation eval = + engine.evaluate(rubric(criterion("c1", 1.0)), NodeResult.empty(), new HashMap<>()); assertThat(eval.getScore()).isLessThan(70.0); assertThat(eval.isPassed()).isFalse(); } @Test - void shouldReturnZeroWhenAllCriteriaHaveZeroWeight() throws RubricNotFoundException { - engine = engineWithFixedScore(100.0); - engine.registerRubric(rubric(criterion("c1", 0.0))); + void shouldReturnZeroWhenAllCriteriaHaveZeroWeight() { + var engine = engineWithFixedScore(100.0); - RubricEvaluation eval = engine.evaluate("r1", NodeResult.empty(), new HashMap<>()); + RubricEvaluation eval = + engine.evaluate(rubric(criterion("c1", 0.0)), NodeResult.empty(), new HashMap<>()); assertThat(eval.getScore()).isZero(); assertThat(eval.isPassed()).isFalse(); } @Test - void shouldIncludePerCriterionDetailInEvaluation() throws RubricNotFoundException { - engine = engineWithFixedScore(85.0); - engine.registerRubric(rubric(criterion("c1", 1.0))); + void shouldIncludePerCriterionDetailInEvaluation() { + var engine = engineWithFixedScore(85.0); - RubricEvaluation eval = engine.evaluate("r1", NodeResult.empty(), new HashMap<>()); + RubricEvaluation eval = + engine.evaluate(rubric(criterion("c1", 1.0)), NodeResult.empty(), new HashMap<>()); assertThat(eval.getCriterionEvaluations()).hasSize(1); assertThat(eval.getCriterionEvaluations().getFirst().getCriterionId()).isEqualTo("c1"); diff --git a/hensu-core/src/test/java/io/hensu/core/workflow/InMemoryWorkflowRepositoryTest.java b/hensu-core/src/test/java/io/hensu/core/workflow/InMemoryWorkflowRepositoryTest.java deleted file mode 100644 index e040bb6..0000000 --- a/hensu-core/src/test/java/io/hensu/core/workflow/InMemoryWorkflowRepositoryTest.java +++ /dev/null @@ -1,270 +0,0 @@ -package io.hensu.core.workflow; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import io.hensu.core.agent.AgentConfig; -import io.hensu.core.execution.result.ExitStatus; -import io.hensu.core.workflow.node.EndNode; -import io.hensu.core.workflow.node.Node; -import java.time.Instant; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -class InMemoryWorkflowRepositoryTest { - - private InMemoryWorkflowRepository repository; - - @BeforeEach - void setUp() { - repository = new InMemoryWorkflowRepository(); - } - - private Workflow createWorkflow(String id) { - Map agents = - Map.of( - "agent-1", - AgentConfig.builder() - .id("agent-1") - .role("Test") - .model("test-model") - .build()); - Map nodes = - Map.of("end", EndNode.builder().id("end").status(ExitStatus.SUCCESS).build()); - return Workflow.builder() - .id(id) - .version("1.0.0") - .metadata( - new WorkflowMetadata( - id, "Test workflow", "tester", Instant.now(), List.of())) - .agents(agents) - .nodes(nodes) - .startNode("end") - .build(); - } - - @Nested - class Save { - - @Test - void shouldSaveAndRetrieve() { - Workflow workflow = createWorkflow("wf-1"); - - repository.save("tenant-1", workflow); - - assertThat(repository.findById("tenant-1", "wf-1")).isPresent(); - } - - @Test - void shouldOverwriteExisting() { - Workflow v1 = createWorkflow("wf-1"); - repository.save("tenant-1", v1); - - Workflow v2 = - Workflow.builder() - .id("wf-1") - .version("2.0.0") - .metadata( - new WorkflowMetadata( - "wf-1", "Updated", "tester", Instant.now(), List.of())) - .agents(v1.getAgents()) - .nodes(v1.getNodes()) - .startNode("end") - .build(); - repository.save("tenant-1", v2); - - assertThat(repository.findById("tenant-1", "wf-1")) - .isPresent() - .hasValueSatisfying(wf -> assertThat(wf.getVersion()).isEqualTo("2.0.0")); - } - } - - @Nested - class FindById { - - @Test - void shouldReturnEmptyForUnknownWorkflow() { - assertThat(repository.findById("tenant-1", "nonexistent")).isEmpty(); - } - - @Test - void shouldReturnEmptyForUnknownTenant() { - repository.save("tenant-1", createWorkflow("wf-1")); - assertThat(repository.findById("tenant-2", "wf-1")).isEmpty(); - } - } - - @Nested - class FindAll { - - @Test - void shouldReturnAllForTenant() { - repository.save("tenant-1", createWorkflow("wf-1")); - repository.save("tenant-1", createWorkflow("wf-2")); - - List results = repository.findAll("tenant-1"); - assertThat(results).hasSize(2); - } - - @Test - void shouldReturnEmptyForUnknownTenant() { - assertThat(repository.findAll("unknown")).isEmpty(); - } - } - - @Nested - class Exists { - - @Test - void shouldReturnTrueWhenPresent() { - repository.save("tenant-1", createWorkflow("wf-1")); - assertThat(repository.exists("tenant-1", "wf-1")).isTrue(); - } - - @Test - void shouldReturnFalseWhenAbsent() { - assertThat(repository.exists("tenant-1", "wf-1")).isFalse(); - } - } - - @Nested - class Delete { - - @Test - void shouldReturnFalseForNeverSaved() { - assertThat(repository.delete("tenant-1", "wf-1")).isFalse(); - } - } - - @Nested - class DeleteAllForTenant { - - @Test - void shouldDeleteAllAndReturnCount() { - repository.save("tenant-1", createWorkflow("wf-1")); - repository.save("tenant-1", createWorkflow("wf-2")); - - assertThat(repository.deleteAllForTenant("tenant-1")).isEqualTo(2); - assertThat(repository.findAll("tenant-1")).isEmpty(); - } - - @Test - void shouldReturnZeroForUnknownTenant() { - assertThat(repository.deleteAllForTenant("unknown")).isEqualTo(0); - } - } - - @Nested - class Count { - - @Test - void shouldReturnCorrectCount() { - repository.save("tenant-1", createWorkflow("wf-1")); - repository.save("tenant-1", createWorkflow("wf-2")); - - assertThat(repository.count("tenant-1")).isEqualTo(2); - } - - @Test - void shouldReturnZeroForUnknownTenant() { - assertThat(repository.count("unknown")).isEqualTo(0); - } - } - - @Nested - class SoftDelete { - - @Test - void shouldSoftDeleteAndHideFromQueries() { - repository.save("tenant-1", createWorkflow("wf-1")); - - assertThat(repository.delete("tenant-1", "wf-1")).isTrue(); - - assertThat(repository.findById("tenant-1", "wf-1")).isEmpty(); - assertThat(repository.exists("tenant-1", "wf-1")).isFalse(); - assertThat(repository.count("tenant-1")).isZero(); - assertThat(repository.findAll("tenant-1")).isEmpty(); - } - - @Test - void shouldReactivateOnSave() { - repository.save("tenant-1", createWorkflow("wf-1")); - repository.delete("tenant-1", "wf-1"); - - repository.save("tenant-1", createWorkflow("wf-1")); - - assertThat(repository.findById("tenant-1", "wf-1")).isPresent(); - assertThat(repository.exists("tenant-1", "wf-1")).isTrue(); - assertThat(repository.count("tenant-1")).isEqualTo(1); - } - - @Test - void shouldReturnFalseForAlreadyDeleted() { - repository.save("tenant-1", createWorkflow("wf-1")); - - assertThat(repository.delete("tenant-1", "wf-1")).isTrue(); - assertThat(repository.delete("tenant-1", "wf-1")).isFalse(); - } - - @Test - void deleteAllShouldSoftDeleteAndReturnActiveCount() { - repository.save("tenant-1", createWorkflow("wf-1")); - repository.save("tenant-1", createWorkflow("wf-2")); - repository.delete("tenant-1", "wf-1"); - - // Only wf-2 is active, so deleteAll returns 1 - assertThat(repository.deleteAllForTenant("tenant-1")).isEqualTo(1); - assertThat(repository.count("tenant-1")).isZero(); - assertThat(repository.findAll("tenant-1")).isEmpty(); - } - } - - @Nested - class Clear { - - @Test - void shouldClearAllTenants() { - repository.save("tenant-1", createWorkflow("wf-1")); - repository.save("tenant-2", createWorkflow("wf-2")); - - repository.clear(); - - assertThat(repository.count("tenant-1")).isEqualTo(0); - assertThat(repository.count("tenant-2")).isEqualTo(0); - } - } - - @Nested - class TenantIsolation { - - @Test - void shouldIsolateWorkflowsBetweenTenants() { - repository.save("tenant-1", createWorkflow("wf-1")); - repository.save("tenant-2", createWorkflow("wf-2")); - - assertThat(repository.findAll("tenant-1")).hasSize(1); - assertThat(repository.findAll("tenant-2")).hasSize(1); - assertThat(repository.findById("tenant-1", "wf-2")).isEmpty(); - assertThat(repository.findById("tenant-2", "wf-1")).isEmpty(); - } - } - - @Nested - class NullSafety { - - @Test - void shouldRejectNullTenantId() { - assertThatThrownBy(() -> repository.save(null, createWorkflow("wf-1"))) - .isInstanceOf(NullPointerException.class); - } - - @Test - void shouldRejectNullWorkflow() { - assertThatThrownBy(() -> repository.save("tenant-1", null)) - .isInstanceOf(NullPointerException.class); - } - } -} diff --git a/hensu-core/src/test/java/io/hensu/core/workflow/WorkflowMetadataTest.java b/hensu-core/src/test/java/io/hensu/core/workflow/WorkflowMetadataTest.java deleted file mode 100644 index bb29515..0000000 --- a/hensu-core/src/test/java/io/hensu/core/workflow/WorkflowMetadataTest.java +++ /dev/null @@ -1,120 +0,0 @@ -package io.hensu.core.workflow; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.time.Instant; -import java.util.List; -import org.junit.jupiter.api.Test; - -class WorkflowMetadataTest { - - @Test - void shouldCreateWithAllFields() { - // Given - Instant created = Instant.now(); - List tags = List.of("ai", "workflow", "automation"); - - // When - WorkflowMetadata metadata = - new WorkflowMetadata( - "Content Review Workflow", - "Workflow for reviewing content quality", - "John Doe", - created, - tags); - - // Then - assertThat(metadata.name()).isEqualTo("Content Review Workflow"); - assertThat(metadata.getName()).isEqualTo("Content Review Workflow"); - assertThat(metadata.description()).isEqualTo("Workflow for reviewing content quality"); - assertThat(metadata.author()).isEqualTo("John Doe"); - assertThat(metadata.created()).isEqualTo(created); - assertThat(metadata.tags()).containsExactly("ai", "workflow", "automation"); - } - - @Test - void shouldAllowNullValues() { - // When - WorkflowMetadata metadata = new WorkflowMetadata(null, null, null, null, List.of()); - - // Then - assertThat(metadata.name()).isNull(); - assertThat(metadata.description()).isNull(); - assertThat(metadata.author()).isNull(); - assertThat(metadata.created()).isNull(); - } - - @Test - void shouldReturnCopyOfTags() { - // Given - List originalTags = List.of("tag1", "tag2"); - WorkflowMetadata metadata = - new WorkflowMetadata("name", "desc", "author", Instant.now(), originalTags); - - // When - List returnedTags = metadata.tags(); - - // Then - should return a copy - assertThat(returnedTags).isUnmodifiable(); - assertThat(returnedTags).containsExactly("tag1", "tag2"); - } - - @Test - void shouldSupportEmptyTags() { - // When - WorkflowMetadata metadata = - new WorkflowMetadata("name", "desc", "author", Instant.now(), List.of()); - - // Then - assertThat(metadata.tags()).isEmpty(); - } - - @Test - void shouldBeRecordType() { - // Given - WorkflowMetadata metadata = - new WorkflowMetadata("name", "desc", "author", Instant.now(), List.of()); - - // Then - records have auto-generated equals, hashCode, toString - assertThat(metadata).isInstanceOf(Record.class); - } - - @Test - void shouldImplementEquality() { - // Given - Instant timestamp = Instant.now(); - List tags = List.of("tag"); - - WorkflowMetadata metadata1 = - new WorkflowMetadata("name", "desc", "author", timestamp, tags); - WorkflowMetadata metadata2 = - new WorkflowMetadata("name", "desc", "author", timestamp, tags); - WorkflowMetadata metadata3 = - new WorkflowMetadata("different", "desc", "author", timestamp, tags); - - // Then - assertThat(metadata1).isEqualTo(metadata2); - assertThat(metadata1).isNotEqualTo(metadata3); - assertThat(metadata1.hashCode()).isEqualTo(metadata2.hashCode()); - } - - @Test - void shouldReturnMeaningfulToString() { - // Given - WorkflowMetadata metadata = - new WorkflowMetadata( - "Test Workflow", - "A test description", - "Test Author", - Instant.parse("2024-01-15T10:00:00Z"), - List.of("test")); - - // When - String toString = metadata.toString(); - - // Then - assertThat(toString).contains("Test Workflow"); - assertThat(toString).contains("A test description"); - assertThat(toString).contains("Test Author"); - } -} diff --git a/hensu-core/src/test/java/io/hensu/core/workflow/WorkflowTest.java b/hensu-core/src/test/java/io/hensu/core/workflow/WorkflowTest.java index 1a22939..14ebe84 100644 --- a/hensu-core/src/test/java/io/hensu/core/workflow/WorkflowTest.java +++ b/hensu-core/src/test/java/io/hensu/core/workflow/WorkflowTest.java @@ -1,231 +1,12 @@ package io.hensu.core.workflow; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - import io.hensu.core.agent.AgentConfig; -import io.hensu.core.execution.result.ExitStatus; -import io.hensu.core.workflow.node.EndNode; import io.hensu.core.workflow.node.Node; -import io.hensu.core.workflow.node.StandardNode; -import io.hensu.core.workflow.transition.SuccessTransition; import java.util.HashMap; -import java.util.List; import java.util.Map; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; public class WorkflowTest { - // ------------------------------------------------------------------------- - // Builder — required fields, defaults, validation - // ------------------------------------------------------------------------- - - @Nested - class BuilderTest { - - @Test - void shouldBuildWorkflowWithRequiredFields() { - var workflow = - Workflow.builder() - .id("test-workflow") - .nodes(Map.of("start", endNode("start"))) - .startNode("start") - .build(); - - assertThat(workflow.getId()).isEqualTo("test-workflow"); - assertThat(workflow.getStartNode()).isEqualTo("start"); - assertThat(workflow.getNodes()).containsKey("start"); - assertThat(workflow.getVersion()).isEqualTo("1.0.0"); // default - assertThat(workflow.getAgents()).isEmpty(); - assertThat(workflow.getRubrics()).isEmpty(); - } - - @Test - void shouldBuildWorkflowWithAllOptionalFields() { - // All optional fields are preserved through the builder. - // If a field is silently dropped this test fails. - var agentConfig = agentConfig("writer", "claude-sonnet-4"); - var config = new WorkflowConfig(5000L, true, 1000L, null); - - var workflow = - Workflow.builder() - .id("test-workflow") - .version("2.0.0") - .agents(Map.of("writer", agentConfig)) - .rubrics(Map.of("quality", "rubric-content")) - .nodes(Map.of("start", endNode("start"))) - .startNode("start") - .config(config) - .build(); - - assertThat(workflow.getVersion()).isEqualTo("2.0.0"); - assertThat(workflow.getAgents()).containsKey("writer"); - assertThat(workflow.getRubrics()).containsEntry("quality", "rubric-content"); - assertThat(workflow.getConfig()).isEqualTo(config); - } - - @Test - void shouldThrowWhenIdIsNull() { - assertThatThrownBy( - () -> - Workflow.builder() - .nodes(Map.of("start", endNode("start"))) - .startNode("start") - .build()) - .isInstanceOf(NullPointerException.class) - .hasMessageContaining("Workflow ID required"); - } - - @Test - void shouldThrowWhenStartNodeIsNull() { - assertThatThrownBy( - () -> - Workflow.builder() - .id("test-workflow") - .nodes(Map.of("start", endNode("start"))) - .build()) - .isInstanceOf(NullPointerException.class) - .hasMessageContaining("Start node required"); - } - - @Test - void shouldThrowWhenStartNodeNotInNodes() { - assertThatThrownBy( - () -> - Workflow.builder() - .id("test-workflow") - .nodes(Map.of("start", endNode("start"))) - .startNode("nonexistent") - .build()) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Start node 'nonexistent' not found"); - } - - @Test - void shouldThrowWhenNodeReferencesUndeclaredRubric() { - // Node declares rubricId="quality" but the workflow.rubrics map is empty. - // The validate() method must detect this; otherwise a rubric lookup at - // runtime silently fails or throws an unrelated NPE deep in the engine. - var nodeWithRubric = - StandardNode.builder() - .id("start") - .rubricId("quality") - .transitionRules(List.of(new SuccessTransition("end"))) - .build(); - - assertThatThrownBy( - () -> - Workflow.builder() - .id("test-workflow") - .nodes( - Map.of( - "start", - nodeWithRubric, - "end", - endNode("end"))) - .startNode("start") - .build()) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("quality"); - } - } - - // ------------------------------------------------------------------------- - // Immutability — all collections must be unmodifiable after build() - // ------------------------------------------------------------------------- - - @Nested - class ImmutabilityTest { - - @Test - void shouldMakeAllCollectionsImmutable() { - var workflow = - Workflow.builder() - .id("test-workflow") - .agents(Map.of("writer", agentConfig("writer", "any"))) - .rubrics(Map.of("quality", "content")) - .nodes(Map.of("start", endNode("start"))) - .startNode("start") - .build(); - - assertThat(workflow.getAgents()).isUnmodifiable(); - assertThat(workflow.getRubrics()).isUnmodifiable(); - assertThat(workflow.getNodes()).isUnmodifiable(); - } - } - - // ------------------------------------------------------------------------- - // Equality — workflows are keyed by id + version - // ------------------------------------------------------------------------- - - @Nested - class EqualsAndHashCodeTest { - - @Test - void shouldBeEqualWhenIdAndVersionMatch() { - var w1 = minimal("test-workflow", "1.0.0"); - var w2 = minimal("test-workflow", "1.0.0"); - - assertThat(w1).isEqualTo(w2); - assertThat(w1.hashCode()).isEqualTo(w2.hashCode()); - } - - @Test - void shouldNotBeEqualWhenIdDiffers() { - assertThat(minimal("workflow-a", "1.0.0")).isNotEqualTo(minimal("workflow-b", "1.0.0")); - } - - @Test - void shouldNotBeEqualWhenVersionDiffers() { - assertThat(minimal("test-workflow", "1.0.0")) - .isNotEqualTo(minimal("test-workflow", "2.0.0")); - } - } - - // ------------------------------------------------------------------------- - // TestWorkflowBuilder — shared fixture for processor and executor tests - // ------------------------------------------------------------------------- - - @Nested - class TestWorkflowBuilderTest { - - @Test - void singleNodeFactorySetsNodeAsStart() { - var node = endNode("done"); - var workflow = TestWorkflowBuilder.singleNode(node); - - assertThat(workflow.getStartNode()).isEqualTo("done"); - assertThat(workflow.getNodes()).containsKey("done").hasSize(1); - } - - @Test - void withNodesFactorySetsFirstNodeAsStart() { - var first = standardNode("step", "end"); - var last = endNode("end"); - - var workflow = TestWorkflowBuilder.withNodes(first, last); - - assertThat(workflow.getStartNode()).isEqualTo("step"); - assertThat(workflow.getNodes()).hasSize(2); - } - - @Test - void fluentBuilderPreservesRubricAndAgentEntries() { - var workflow = - TestWorkflowBuilder.create("wf") - .startNode(standardNode("step", "end")) - .node(endNode("end")) - .rubric("quality", "Be concise.") - .agent(agentConfig("writer", "claude-sonnet-4")) - .build(); - - assertThat(workflow.getRubrics()).containsEntry("quality", "Be concise."); - assertThat(workflow.getAgents()).containsKey("writer"); - assertThat(workflow.getStartNode()).isEqualTo("step"); - } - } - // ========================================================================= // Shared fixture — embedded so tests that need Workflow construction have // a zero-boilerplate API without a separate utility class. @@ -234,11 +15,21 @@ void fluentBuilderPreservesRubricAndAgentEntries() { public static final class TestWorkflowBuilder { + public static final String RUBRIC_CONTENT = + """ + # Rubric: quality + ## Metadata + - pass_threshold: 70 + ### Quality + #### Structure + - points: 10 + - evaluation: Is it organized + """; + private String id = "test-wf"; private String startNodeId; private final Map nodes = new HashMap<>(); private final Map agents = new HashMap<>(); - private final Map rubrics = new HashMap<>(); private TestWorkflowBuilder() {} @@ -256,7 +47,7 @@ public static Workflow withNodes(Node... nodes) { return b.build(); } - /// Fluent builder for workflows that need rubrics, agents, or custom id. + /// Fluent builder for workflows that need agents or custom id. public static TestWorkflowBuilder create(String id) { var b = new TestWorkflowBuilder(); b.id = id; @@ -279,47 +70,13 @@ public TestWorkflowBuilder agent(AgentConfig agent) { return this; } - public TestWorkflowBuilder rubric(String name, String content) { - this.rubrics.put(name, content); - return this; - } - public Workflow build() { return Workflow.builder() .id(id) .startNode(startNodeId) .nodes(nodes) .agents(agents) - .rubrics(rubrics) .build(); } } - - // ========================================================================= - // Helpers - // ========================================================================= - - private static Node endNode(String id) { - return EndNode.builder().id(id).status(ExitStatus.SUCCESS).build(); - } - - private static Node standardNode(String id, String next) { - return StandardNode.builder() - .id(id) - .transitionRules(List.of(new SuccessTransition(next))) - .build(); - } - - private static AgentConfig agentConfig(String id, String model) { - return AgentConfig.builder().id(id).role("assistant").model(model).build(); - } - - private static Workflow minimal(String id, String version) { - return Workflow.builder() - .id(id) - .version(version) - .nodes(Map.of("start", endNode("start"))) - .startNode("start") - .build(); - } } diff --git a/hensu-core/src/test/java/io/hensu/core/workflow/transition/BreakRuleTest.java b/hensu-core/src/test/java/io/hensu/core/workflow/transition/BreakRuleTest.java deleted file mode 100644 index 26d86b0..0000000 --- a/hensu-core/src/test/java/io/hensu/core/workflow/transition/BreakRuleTest.java +++ /dev/null @@ -1,71 +0,0 @@ -package io.hensu.core.workflow.transition; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.junit.jupiter.api.Test; - -class BreakRuleTest { - - @Test - void shouldCreateBreakRuleWithConditionAndTarget() { - // Given - LoopCondition condition = LoopCondition.Always.INSTANCE; - - // When - BreakRule rule = new BreakRule(condition, "exit-node"); - - // Then - assertThat(rule.condition()).isEqualTo(condition); - assertThat(rule.getCondition()).isEqualTo(condition); - assertThat(rule.targetNode()).isEqualTo("exit-node"); - assertThat(rule.getTargetNode()).isEqualTo("exit-node"); - } - - @Test - void shouldCreateBreakRuleWithExpressionCondition() { - // Given - LoopCondition.Expression condition = new LoopCondition.Expression("count > 10"); - - // When - BreakRule rule = new BreakRule(condition, "done"); - - // Then - assertThat(rule.condition()).isInstanceOf(LoopCondition.Expression.class); - assertThat(((LoopCondition.Expression) rule.condition()).getExpr()).isEqualTo("count > 10"); - } - - @Test - void shouldAllowNullValues() { - // When - BreakRule rule = new BreakRule(null, null); - - // Then - assertThat(rule.condition()).isNull(); - assertThat(rule.targetNode()).isNull(); - } - - @Test - void shouldImplementEquality() { - // Given - BreakRule rule1 = new BreakRule(LoopCondition.Always.INSTANCE, "exit"); - BreakRule rule2 = new BreakRule(LoopCondition.Always.INSTANCE, "exit"); - BreakRule rule3 = new BreakRule(LoopCondition.Always.INSTANCE, "other"); - - // Then - assertThat(rule1).isEqualTo(rule2); - assertThat(rule1).isNotEqualTo(rule3); - assertThat(rule1.hashCode()).isEqualTo(rule2.hashCode()); - } - - @Test - void shouldReturnMeaningfulToString() { - // Given - BreakRule rule = new BreakRule(LoopCondition.Always.INSTANCE, "exit-node"); - - // When - String toString = rule.toString(); - - // Then - assertThat(toString).contains("exit-node"); - } -} diff --git a/hensu-core/src/test/java/io/hensu/core/workflow/transition/LoopConditionTest.java b/hensu-core/src/test/java/io/hensu/core/workflow/transition/LoopConditionTest.java deleted file mode 100644 index 6714f64..0000000 --- a/hensu-core/src/test/java/io/hensu/core/workflow/transition/LoopConditionTest.java +++ /dev/null @@ -1,96 +0,0 @@ -package io.hensu.core.workflow.transition; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -class LoopConditionTest { - - @Nested - class AlwaysConditionTest { - - @Test - void shouldProvidesSingletonInstance() { - // When - LoopCondition.Always instance1 = LoopCondition.Always.INSTANCE; - LoopCondition.Always instance2 = LoopCondition.Always.INSTANCE; - - // Then - assertThat(instance1).isSameAs(instance2); - } - - @Test - void shouldBeInstanceOfLoopCondition() { - // When - LoopCondition condition = LoopCondition.Always.INSTANCE; - - // Then - assertThat(condition).isInstanceOf(LoopCondition.class); - assertThat(condition).isInstanceOf(LoopCondition.Always.class); - } - } - - @Nested - class ExpressionConditionTest { - - @Test - void shouldCreateWithExpression() { - // When - LoopCondition.Expression condition = new LoopCondition.Expression("count < 5"); - - // Then - assertThat(condition.getExpr()).isEqualTo("count < 5"); - } - - @Test - void shouldBeInstanceOfLoopCondition() { - // When - LoopCondition condition = new LoopCondition.Expression("true"); - - // Then - assertThat(condition).isInstanceOf(LoopCondition.class); - assertThat(condition).isInstanceOf(LoopCondition.Expression.class); - } - - @Test - void shouldStoreComplexExpressions() { - // When - LoopCondition.Expression condition = - new LoopCondition.Expression("score >= threshold && attempts < maxAttempts"); - - // Then - assertThat(condition.getExpr()) - .isEqualTo("score >= threshold && attempts < maxAttempts"); - } - - @Test - void shouldAllowNullExpression() { - // When - LoopCondition.Expression condition = new LoopCondition.Expression(null); - - // Then - assertThat(condition.getExpr()).isNull(); - } - - @Test - void shouldAllowEmptyExpression() { - // When - LoopCondition.Expression condition = new LoopCondition.Expression(""); - - // Then - assertThat(condition.getExpr()).isEmpty(); - } - } - - @Test - void shouldDistinguishBetweenConditionTypes() { - // Given - LoopCondition always = LoopCondition.Always.INSTANCE; - LoopCondition expression = new LoopCondition.Expression("x > 0"); - - // Then - assertThat(always).isNotEqualTo(expression); - assertThat(always.getClass()).isNotEqualTo(expression.getClass()); - } -} diff --git a/hensu-core/src/test/java/io/hensu/core/workflow/transition/RubricFailTransitionTest.java b/hensu-core/src/test/java/io/hensu/core/workflow/transition/RubricFailTransitionTest.java index 5c2f28d..528c5e3 100644 --- a/hensu-core/src/test/java/io/hensu/core/workflow/transition/RubricFailTransitionTest.java +++ b/hensu-core/src/test/java/io/hensu/core/workflow/transition/RubricFailTransitionTest.java @@ -1,130 +1,27 @@ package io.hensu.core.workflow.transition; -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import io.hensu.core.execution.executor.NodeResult; import io.hensu.core.execution.result.ExecutionHistory; -import io.hensu.core.rubric.evaluator.RubricEvaluation; import io.hensu.core.state.HensuState; import java.util.HashMap; import java.util.Map; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class RubricFailTransitionTest { - private HensuState state; - - @BeforeEach - void setUp() { - state = - new HensuState( - new HashMap<>(), "test-workflow", "current-node", new ExecutionHistory()); - } - @Test - void shouldEvaluateUsingProvidedFunction() { - // Given - RubricFailTransition transition = + void shouldThrowWhenStateHasNoRubricEvaluation() { + var transition = new RubricFailTransition( evaluation -> evaluation.isPassed() ? "continue" : "revise"); + var state = + new HensuState( + new HashMap<>(), "test-workflow", "current-node", new ExecutionHistory()); + var result = NodeResult.success("output", Map.of()); - RubricEvaluation failedEval = - RubricEvaluation.builder().rubricId("quality").score(50.0).passed(false).build(); - state.setRubricEvaluation(failedEval); - - NodeResult result = NodeResult.success("output", Map.of()); - - // When - String target = transition.evaluate(state, result); - - // Then - assertThat(target).isEqualTo("revise"); - } - - @Test - void shouldReturnContinueWhenPassed() { - // Given - RubricFailTransition transition = - new RubricFailTransition( - evaluation -> evaluation.isPassed() ? "continue" : "revise"); - - RubricEvaluation passedEval = - RubricEvaluation.builder().rubricId("quality").score(90.0).passed(true).build(); - state.setRubricEvaluation(passedEval); - - NodeResult result = NodeResult.success("output", Map.of()); - - // When - String target = transition.evaluate(state, result); - - // Then - assertThat(target).isEqualTo("continue"); - } - - @Test - void shouldAccessScoreInFunction() { - // Given - RubricFailTransition transition = - new RubricFailTransition( - evaluation -> { - if (evaluation.getScore() >= 80) return "excellent"; - if (evaluation.getScore() >= 60) return "good"; - return "needs-improvement"; - }); - - RubricEvaluation eval = - RubricEvaluation.builder().rubricId("quality").score(75.0).passed(true).build(); - state.setRubricEvaluation(eval); - - NodeResult result = NodeResult.success("output", Map.of()); - - // When - String target = transition.evaluate(state, result); - - // Then - assertThat(target).isEqualTo("good"); - } - - @Test - void shouldAccessRubricIdInFunction() { - // Given - RubricFailTransition transition = - new RubricFailTransition(evaluation -> "handled-" + evaluation.getRubricId()); - - RubricEvaluation eval = - RubricEvaluation.builder() - .rubricId("content-quality") - .score(50.0) - .passed(false) - .build(); - state.setRubricEvaluation(eval); - - NodeResult result = NodeResult.success("output", Map.of()); - - // When - String target = transition.evaluate(state, result); - - // Then - assertThat(target).isEqualTo("handled-content-quality"); - } - - @Test - void shouldExposeFunction() { - // Given - var func = (java.util.function.Function) _ -> "target"; - RubricFailTransition transition = new RubricFailTransition(func); - - // Then - assertThat(transition.function()).isSameAs(func); - } - - @Test - void shouldImplementTransitionRule() { - // Given - RubricFailTransition transition = new RubricFailTransition(_ -> "target"); - - // Then - assertThat(transition).isInstanceOf(TransitionRule.class); + assertThatThrownBy(() -> transition.evaluate(state, result)) + .isInstanceOf(NullPointerException.class); } } diff --git a/hensu-core/src/test/java/io/hensu/core/workflow/validation/SubWorkflowGraphValidatorTest.java b/hensu-core/src/test/java/io/hensu/core/workflow/validation/SubWorkflowGraphValidatorTest.java index 573edad..78e4cab 100644 --- a/hensu-core/src/test/java/io/hensu/core/workflow/validation/SubWorkflowGraphValidatorTest.java +++ b/hensu-core/src/test/java/io/hensu/core/workflow/validation/SubWorkflowGraphValidatorTest.java @@ -60,17 +60,6 @@ void selfCycleIsRejected() { .hasMessageContaining("a"); } - @Test - void twoNodeCycleIsRejected() { - Workflow a = wf("a", List.of("b")); - Workflow b = wf("b", List.of("a")); - assertThatThrownBy(() -> SubWorkflowGraphValidator.validate(List.of(a, b))) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("cycle") - .hasMessageContaining("a") - .hasMessageContaining("b"); - } - @Test void disconnectedCycleIsRejected() { // Root 'main' is clean; an isolated 'x' ↔ 'y' cycle still blocks the load. @@ -135,7 +124,10 @@ void rePushShadowsStoredVersion() { repo.put("a", wf("a", List.of("b"))); repo.put("b", staleB); assertThatThrownBy(() -> SubWorkflowGraphValidator.validate(newB, repo::get)) - .isInstanceOf(IllegalStateException.class); + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("cycle") + .hasMessageContaining("a") + .hasMessageContaining("b"); } @Test diff --git a/hensu-dsl/README.md b/hensu-dsl/README.md index ac97e66..ac9176c 100644 --- a/hensu-dsl/README.md +++ b/hensu-dsl/README.md @@ -46,7 +46,7 @@ fun myWorkflow() = workflow("ContentPipeline") { node("review") { agent = "reviewer" prompt = "Review this article: {article}" - rubric = "content-quality" + rubric = "content-quality.md" writes("article") onScore { diff --git a/hensu-dsl/src/main/kotlin/io/hensu/dsl/WorkingDirectory.kt b/hensu-dsl/src/main/kotlin/io/hensu/dsl/WorkingDirectory.kt index 7a7c0c4..ea2ab19 100644 --- a/hensu-dsl/src/main/kotlin/io/hensu/dsl/WorkingDirectory.kt +++ b/hensu-dsl/src/main/kotlin/io/hensu/dsl/WorkingDirectory.kt @@ -126,20 +126,20 @@ class WorkingDirectory(private val root: Path) { } /** - * Resolves a rubric file path from the rubrics directory. + * Resolves a rubric file and returns its content. * * Automatically appends `.md` extension if not present. Supports relative paths within the * rubrics directory (e.g., `templates/code-quality.md`). * * @param name rubric name or relative path (with or without .md extension), not null - * @return absolute path to the rubric file, never null + * @return content of the rubric file as a string, never null * @throws IllegalArgumentException if the rubric file does not exist */ - fun resolveRubric(name: String): Path { + fun resolveRubric(name: String): String { val fileName = if (name.endsWith(".md")) name else "$name.md" val path = rubricsDir.resolve(fileName) require(path.exists()) { "Rubric not found: $path" } - return path + return path.readText() } /** diff --git a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/GenericNodeBuilder.kt b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/GenericNodeBuilder.kt index 33cc7e6..c360b62 100644 --- a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/GenericNodeBuilder.kt +++ b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/GenericNodeBuilder.kt @@ -1,5 +1,6 @@ package io.hensu.dsl.builders +import io.hensu.core.rubric.RubricParser import io.hensu.core.workflow.node.GenericNode import io.hensu.core.workflow.node.Node @@ -116,7 +117,7 @@ class GenericNodeBuilder(private val id: String) : BaseNodeBuilder, TransitionMa .id(id) .executorType(executorType) .config(configMap) - .rubricId(rubric) + .rubric(rubric?.let { RubricParser.parseContent(id, it) }) .transitionRules(transitionBuilder.build()) .build() } diff --git a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/ParallelNodeBuilder.kt b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/ParallelNodeBuilder.kt index 65e497a..780700f 100644 --- a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/ParallelNodeBuilder.kt +++ b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/ParallelNodeBuilder.kt @@ -4,9 +4,11 @@ import io.hensu.core.execution.EngineVariables import io.hensu.core.execution.parallel.Branch import io.hensu.core.execution.parallel.ConsensusConfig import io.hensu.core.execution.parallel.ConsensusStrategy +import io.hensu.core.rubric.RubricParser import io.hensu.core.workflow.node.ParallelNode import io.hensu.dsl.WorkingDirectory import io.hensu.dsl.extensions.resolveAsPrompt +import io.hensu.dsl.extensions.resolveAsRubric import java.util.logging.Logger /** @@ -220,7 +222,7 @@ class BranchBuilder(private val id: String, private val workingDirectory: Workin id, agent ?: throw IllegalStateException("Branch '$id' must have an agent"), prompt.resolveAsPrompt(workingDirectory) ?: "", - rubric, + rubric.resolveAsRubric(workingDirectory)?.let { RubricParser.parseContent(id, it) }, weight, yieldFields, ) diff --git a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/RubricBuilder.kt b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/RubricBuilder.kt deleted file mode 100644 index 4e5630a..0000000 --- a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/RubricBuilder.kt +++ /dev/null @@ -1,72 +0,0 @@ -package io.hensu.dsl.builders - -import io.hensu.dsl.WorkingDirectory - -/** - * DSL builder for registering rubrics within a workflow. - * - * Rubrics define quality evaluation criteria for agent outputs. Files are resolved from the - * `rubrics/` subdirectory of the working directory. - * - * Example: - * ```kotlin - * rubrics { - * rubric("quality", "code-quality.md") // -> rubrics/code-quality.md - * rubric("pr-review", "templates/pr.md") // -> rubrics/templates/pr.md - * rubric("docs") { file = "documentation.md" } - * } - * ``` - * - * @property rubrics mutable map to collect rubric ID to file path mappings - * @property workingDirectory base directory for rubric file resolution - * @see RubricRefBuilder for alternative registration syntax - */ -@WorkflowDsl -class RubricRegistryBuilder( - private val rubrics: MutableMap, - private val workingDirectory: WorkingDirectory, -) { - /** - * Registers a rubric file with the given ID. - * - * The path is relative to the `rubrics/` directory. - * - * @param id unique identifier for referencing this rubric in nodes, not null - * @param name rubric filename or relative path (with or without .md extension), not null - * @throws IllegalArgumentException if the rubric file does not exist - */ - fun rubric(id: String, name: String) { - val resolvedPath = workingDirectory.resolveRubric(name) - rubrics[id] = resolvedPath.toString() - } - - /** - * Registers a rubric using block syntax. - * - * @param id unique identifier for referencing this rubric in nodes, not null - * @param block configuration block to set the [RubricRefBuilder.file] property - * @throws IllegalArgumentException if the rubric file does not exist - */ - fun rubric(id: String, block: RubricRefBuilder.() -> Unit) { - val builder = RubricRefBuilder() - builder.apply(block) - val resolvedPath = workingDirectory.resolveRubric(builder.file) - rubrics[id] = resolvedPath.toString() - } -} - -/** - * Builder for rubric file reference in block syntax. - * - * Example: - * ```kotlin - * rubric("quality") { - * file = "quality.md" - * } - * ``` - */ -@WorkflowDsl -class RubricRefBuilder { - /** Rubric filename or relative path within the rubrics directory. */ - var file: String = "" -} diff --git a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/StandardNodeBuilder.kt b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/StandardNodeBuilder.kt index e67a388..807fb9a 100644 --- a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/StandardNodeBuilder.kt +++ b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/StandardNodeBuilder.kt @@ -5,9 +5,11 @@ import io.hensu.core.plan.Plan import io.hensu.core.plan.PlanningConfig import io.hensu.core.review.ReviewConfig import io.hensu.core.review.ReviewMode +import io.hensu.core.rubric.RubricParser import io.hensu.core.workflow.node.StandardNode import io.hensu.dsl.WorkingDirectory import io.hensu.dsl.extensions.resolveAsPrompt +import io.hensu.dsl.extensions.resolveAsRubric /** * DSL builder for standard workflow nodes. @@ -216,7 +218,9 @@ class StandardNodeBuilder(private val id: String, private val workingDirectory: .id(id) .agentId(agent) .prompt(prompt.resolveAsPrompt(workingDirectory)) - .rubricId(rubric) + .rubric( + rubric.resolveAsRubric(workingDirectory)?.let { RubricParser.parseContent(id, it) } + ) .reviewConfig(reviewConfig) .transitionRules(transitionBuilder.build()) .writes(writes) diff --git a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/WorkflowBuilder.kt b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/WorkflowBuilder.kt index 422cebe..40ee9ba 100644 --- a/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/WorkflowBuilder.kt +++ b/hensu-dsl/src/main/kotlin/io/hensu/dsl/builders/WorkflowBuilder.kt @@ -18,15 +18,14 @@ import java.time.Instant /** * DSL builder for constructing [Workflow] instances. * - * Provides a fluent API for defining workflow components including agents, rubrics, and the - * execution graph. The builder compiles to an immutable Java [Workflow] instance. + * Provides a fluent API for defining workflow components including agents and the execution graph. + * The builder compiles to an immutable Java [Workflow] instance. * * Usage: * ```kotlin * WorkflowBuilder("my-workflow", workingDir).apply { * description = "My workflow description" * agents { ... } - * rubrics { ... } * graph { ... } * }.build() * ``` @@ -40,8 +39,6 @@ class WorkflowBuilder(private val name: String, private val workingDirectory: Wo var description: String? = null var version: String = "1.0.0" private val agentConfigs = mutableMapOf() - - private val rubricPaths = mutableMapOf() private var graphBuilder: GraphBuilder? = null private var configBuilder: WorkflowConfigBuilder? = null private var stateSchema: WorkflowStateSchema? = null @@ -57,19 +54,6 @@ class WorkflowBuilder(private val name: String, private val workingDirectory: Wo registry.apply(block) } - /** - * Defines rubrics for quality evaluation. - * - * Rubrics are resolved from `workingDirectory/rubrics/`. - * - * @param block rubric registration block - * @see RubricRegistryBuilder for rubric definition syntax - */ - fun rubrics(block: RubricRegistryBuilder.() -> Unit) { - val registry = RubricRegistryBuilder(rubricPaths, workingDirectory) - registry.apply(block) - } - /** * Defines the workflow execution graph. * @@ -129,7 +113,6 @@ class WorkflowBuilder(private val name: String, private val workingDirectory: Wo .id(name.sanitizeId()) .version(version) .agents(agentConfigs) - .rubrics(rubricPaths) .nodes(graph.nodes) .startNode(graph.startNode) .config(config) diff --git a/hensu-dsl/src/main/kotlin/io/hensu/dsl/extensions/DslHelpers.kt b/hensu-dsl/src/main/kotlin/io/hensu/dsl/extensions/DslHelpers.kt index 5d37bf5..3077416 100644 --- a/hensu-dsl/src/main/kotlin/io/hensu/dsl/extensions/DslHelpers.kt +++ b/hensu-dsl/src/main/kotlin/io/hensu/dsl/extensions/DslHelpers.kt @@ -44,6 +44,25 @@ fun String?.resolveAsPrompt(workingDirectory: WorkingDirectory): String? { } } +/** + * Resolves a rubric value from the working directory if it's a file reference. + * + * If the string ends with `.md`, resolves it from the rubrics directory. Otherwise, returns the + * string as-is (inline rubric content). + * + * @param workingDirectory base directory for rubric file resolution + * @return resolved rubric content, or null if receiver is null + * @throws IllegalArgumentException if the rubric file does not exist + */ +fun String?.resolveAsRubric(workingDirectory: WorkingDirectory): String? { + if (this == null) return null + return if (isMarkdownFile()) { + workingDirectory.resolveRubric(this) + } else { + this + } +} + // Duration extensions for readable syntax /** Converts this integer to a [Duration] of seconds. */ @@ -114,10 +133,6 @@ val Workflow.nodeCount: Int val Workflow.agentCount: Int get() = agents.size -/** Returns true if this workflow has any rubrics defined. */ -val Workflow.hasRubrics: Boolean - get() = rubrics.isNotEmpty() - // Node extensions /** Returns true if this node has an agent assigned. */ @@ -126,7 +141,7 @@ val StandardNode.hasAgent: Boolean /** Returns true if this node has a rubric for quality evaluation. */ val StandardNode.hasRubric: Boolean - get() = rubricId != null + get() = rubric != null /** Returns true if this node has human review configured. */ val StandardNode.hasReview: Boolean diff --git a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/GenericNodeBuilderTest.kt b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/GenericNodeBuilderTest.kt index 3cf3e8a..e369ab0 100644 --- a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/GenericNodeBuilderTest.kt +++ b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/GenericNodeBuilderTest.kt @@ -45,7 +45,7 @@ class GenericNodeBuilderTest { val node = builder.build() as GenericNode // Then - assertThat(node.rubricId).isEqualTo("validation-quality") + assertThat(node.rubric?.rawContent).isEqualTo("validation-quality") } @Test diff --git a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/NodeBuilderTest.kt b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/NodeBuilderTest.kt index 7d05f2e..99631d9 100644 --- a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/NodeBuilderTest.kt +++ b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/NodeBuilderTest.kt @@ -53,7 +53,7 @@ class NodeBuilderTest { assertThat(node.id).isEqualTo("test-node") assertThat(node.agentId).isEqualTo("test-agent") assertThat(node.prompt).isEqualTo("Test prompt with {variable}") - assertThat(node.rubricId).isEqualTo("test-rubric") + assertThat(node.rubric?.rawContent).isEqualTo("test-rubric") assertThat(node.transitionRules).hasSize(1) } diff --git a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/ParallelNodeBuilderTest.kt b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/ParallelNodeBuilderTest.kt index a41fa27..e53fc6b 100644 --- a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/ParallelNodeBuilderTest.kt +++ b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/ParallelNodeBuilderTest.kt @@ -98,7 +98,7 @@ class ParallelNodeBuilderTest { val node = builder.build() // Then - assertThat(node.branches[0].rubricId()).isEqualTo("code-quality") + assertThat(node.branches[0].rubric()?.rawContent).isEqualTo("code-quality") } @Test diff --git a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/RubricBuilderTest.kt b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/RubricBuilderTest.kt deleted file mode 100644 index b03c31a..0000000 --- a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/RubricBuilderTest.kt +++ /dev/null @@ -1,196 +0,0 @@ -package io.hensu.dsl.builders - -import io.hensu.dsl.WorkingDirectory -import java.nio.file.Files -import java.nio.file.Path -import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Nested -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.io.TempDir - -class RubricBuilderTest { - @TempDir lateinit var tempDir: Path - - private lateinit var workingDir: WorkingDirectory - - @BeforeEach - fun setUp() { - Files.createDirectories(tempDir.resolve("workflows")) - Files.createDirectories(tempDir.resolve("prompts")) - Files.createDirectories(tempDir.resolve("rubrics")) - Files.createDirectories(tempDir.resolve("rubrics/templates")) - workingDir = WorkingDirectory.of(tempDir) - } - - @Nested - inner class RubricRegistryBuilderTest { - @Test - fun `should register rubric with simple name`() { - // Given - val rubricFile = tempDir.resolve("rubrics/code-quality.md") - Files.writeString(rubricFile, "# Code Quality Rubric") - - val rubrics = mutableMapOf() - val registry = RubricRegistryBuilder(rubrics, workingDir) - - // When - registry.rubric("quality", "code-quality.md") - - // Then - assertThat(rubrics).hasSize(1) - assertThat(rubrics["quality"]).isEqualTo(rubricFile.toString()) - } - - @Test - fun `should register rubric without extension`() { - // Given - val rubricFile = tempDir.resolve("rubrics/quality.md") - Files.writeString(rubricFile, "# Quality Rubric") - - val rubrics = mutableMapOf() - val registry = RubricRegistryBuilder(rubrics, workingDir) - - // When - registry.rubric("quality", "quality") - - // Then - assertThat(rubrics["quality"]).isEqualTo(rubricFile.toString()) - } - - @Test - fun `should register rubric in subdirectory`() { - // Given - val rubricFile = tempDir.resolve("rubrics/templates/pr-quality.md") - Files.writeString(rubricFile, "# PR Quality Rubric") - - val rubrics = mutableMapOf() - val registry = RubricRegistryBuilder(rubrics, workingDir) - - // When - registry.rubric("pr", "templates/pr-quality.md") - - // Then - assertThat(rubrics["pr"]).isEqualTo(rubricFile.toString()) - } - - @Test - fun `should register multiple rubrics`() { - // Given - Files.writeString(tempDir.resolve("rubrics/quality.md"), "# Quality") - Files.writeString(tempDir.resolve("rubrics/security.md"), "# Security") - Files.writeString(tempDir.resolve("rubrics/performance.md"), "# Performance") - - val rubrics = mutableMapOf() - val registry = RubricRegistryBuilder(rubrics, workingDir) - - // When - registry.rubric("quality", "quality.md") - registry.rubric("security", "security.md") - registry.rubric("performance", "performance.md") - - // Then - assertThat(rubrics).hasSize(3) - assertThat(rubrics).containsKeys("quality", "security", "performance") - } - - @Test - fun `should throw when rubric file does not exist`() { - // Given - val rubrics = mutableMapOf() - val registry = RubricRegistryBuilder(rubrics, workingDir) - - // When/Then - assertThatThrownBy { registry.rubric("missing", "non-existent.md") } - .isInstanceOf(IllegalArgumentException::class.java) - .hasMessageContaining("Rubric not found") - } - - @Test - fun `should register rubric using DSL block`() { - // Given - val rubricFile = tempDir.resolve("rubrics/code-quality.md") - Files.writeString(rubricFile, "# Code Quality") - - val rubrics = mutableMapOf() - val registry = RubricRegistryBuilder(rubrics, workingDir) - - // When - registry.rubric("quality") { file = "code-quality.md" } - - // Then - assertThat(rubrics["quality"]).isEqualTo(rubricFile.toString()) - } - - @Test - fun `should register rubric with subdirectory path using DSL block`() { - // Given - val rubricFile = tempDir.resolve("rubrics/templates/api-design.md") - Files.writeString(rubricFile, "# API Design Rubric") - - val rubrics = mutableMapOf() - val registry = RubricRegistryBuilder(rubrics, workingDir) - - // When - registry.rubric("api") { file = "templates/api-design.md" } - - // Then - assertThat(rubrics["api"]).isEqualTo(rubricFile.toString()) - } - - @Test - fun `should overwrite rubric with same id`() { - // Given - Files.writeString(tempDir.resolve("rubrics/first.md"), "# First") - Files.writeString(tempDir.resolve("rubrics/second.md"), "# Second") - - val rubrics = mutableMapOf() - val registry = RubricRegistryBuilder(rubrics, workingDir) - - // When - registry.rubric("quality", "first.md") - registry.rubric("quality", "second.md") - - // Then - assertThat(rubrics).hasSize(1) - assertThat(rubrics["quality"]).contains("second.md") - } - } - - @Nested - inner class RubricRefBuilderTest { - @Test - fun `should have empty file by default`() { - // Given - val builder = RubricRefBuilder() - - // Then - assertThat(builder.file).isEmpty() - } - - @Test - fun `should set file property`() { - // Given - val builder = RubricRefBuilder() - - // When - builder.file = "my-rubric.md" - - // Then - assertThat(builder.file).isEqualTo("my-rubric.md") - } - - @Test - fun `should work with DSL apply block`() { - // Given - val builder = RubricRefBuilder() - - // When - builder.apply { file = "templates/code-review.md" } - - // Then - assertThat(builder.file).isEqualTo("templates/code-review.md") - } - } -} diff --git a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/TransitionBuilderTest.kt b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/TransitionBuilderTest.kt index 036e05d..27bd1aa 100644 --- a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/TransitionBuilderTest.kt +++ b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/TransitionBuilderTest.kt @@ -125,15 +125,13 @@ class TransitionBuilderTest { } } - rubrics { rubric("quality", "quality.md") } - graph { start at "step1" node("step1") { agent = "agent1" prompt = "Test" - rubric = "quality" + rubric = "quality.md" onScore { whenScore greaterThanOrEqual 90.0 goto "excellent" @@ -224,15 +222,13 @@ class TransitionBuilderTest { } } - rubrics { rubric("quality", "quality.md") } - graph { start at "step1" node("step1") { agent = "agent1" prompt = "Test" - rubric = "quality" + rubric = "quality.md" // All three types of transitions onSuccess goto "default-success" diff --git a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/WorkflowBuilderTest.kt b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/WorkflowBuilderTest.kt index a659179..2c85e16 100644 --- a/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/WorkflowBuilderTest.kt +++ b/hensu-dsl/src/test/kotlin/io/hensu/dsl/builders/WorkflowBuilderTest.kt @@ -113,25 +113,19 @@ class WorkflowBuilderTest { } @Test - fun `should build workflow with rubrics`() { - // Create rubric files + fun `should build workflow with rubric on node`() { + // Create rubric file Files.writeString(tempDir.resolve("rubrics/quality.md"), "# Quality Rubric") - Files.writeString(tempDir.resolve("rubrics/security.md"), "# Security Rubric") // When val workflow = workflow("WithRubrics", workingDir) { - rubrics { - rubric("quality", "quality.md") - rubric("security", "security.md") - } - graph { start at "step1" node("step1") { agent = "test" - rubric = "quality" + rubric = "quality.md" onSuccess goto "end" } @@ -140,9 +134,8 @@ class WorkflowBuilderTest { } // Then - assertThat(workflow.rubrics).hasSize(2) - assertThat(workflow.rubrics["quality"]).endsWith("rubrics/quality.md") - assertThat(workflow.rubrics["security"]).endsWith("rubrics/security.md") + val step1 = workflow.nodes["step1"] as StandardNode + assertThat(step1.rubric?.rawContent).isEqualTo("# Quality Rubric") } @Test @@ -393,15 +386,13 @@ class WorkflowBuilderTest { } } - rubrics { rubric("quality", "quality.md") } - graph { start at "step1" node("step1") { agent = "agent1" prompt = "Step 1" - rubric = "quality" + rubric = "quality.md" review(ReviewMode.OPTIONAL) @@ -425,7 +416,7 @@ class WorkflowBuilderTest { val step1 = workflow.nodes["step1"] as StandardNode assertThat(step1.agentId).isEqualTo("agent1") assertThat(step1.prompt).isEqualTo("Step 1") - assertThat(step1.rubricId).isEqualTo("quality") + assertThat(step1.rubric?.rawContent).isEqualTo("# Quality Rubric") assertThat(step1.reviewConfig?.mode).isEqualTo(ReviewMode.OPTIONAL) } } diff --git a/hensu-serialization/src/main/java/io/hensu/serialization/NodeDeserializer.java b/hensu-serialization/src/main/java/io/hensu/serialization/NodeDeserializer.java index f69489c..31abe8d 100644 --- a/hensu-serialization/src/main/java/io/hensu/serialization/NodeDeserializer.java +++ b/hensu-serialization/src/main/java/io/hensu/serialization/NodeDeserializer.java @@ -15,6 +15,8 @@ import io.hensu.core.plan.PlanningConfig; import io.hensu.core.review.ReviewConfig; import io.hensu.core.review.ReviewMode; +import io.hensu.core.rubric.RubricParser; +import io.hensu.core.rubric.model.Rubric; import io.hensu.core.workflow.node.*; import io.hensu.core.workflow.transition.TransitionRule; import java.io.IOException; @@ -31,7 +33,7 @@ /// from the `JsonNode` tree to avoid POJO reflection: /// - `ReviewConfig` (mode + two booleans) /// - `ConsensusConfig` (judgeAgentId, strategy, threshold) -/// - `Branch` (id, agentId, prompt, rubricId, weight) +/// - `Branch` (id, agentId, prompt, rubric, weight) /// /// Complex types that contain `Duration` or deeply nested structures delegate to `treeToValue` /// and require reflection registration in `CoreModelNativeConfig` in `hensu-server`: @@ -59,11 +61,11 @@ class NodeDeserializer extends StdDeserializer { /// appropriate subtype builder. /// /// @param p the JSON parser positioned at the start of the node object, not null - /// @param ctxt the deserialization context, not null + /// @param ctx the deserialization context, not null /// @return the constructed `Node`, never null /// @throws IOException if a required field is absent or the `"nodeType"` value is unrecognized @Override - public Node deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + public Node deserialize(JsonParser p, DeserializationContext ctx) throws IOException { ObjectMapper mapper = (ObjectMapper) p.getCodec(); JsonNode root = mapper.readTree(p); @@ -90,7 +92,7 @@ private StandardNode deserializeStandard(ObjectMapper mapper, JsonNode root, Str .id(id) .agentId(textOrNull(root, "agentId")) .prompt(textOrNull(root, "prompt")) - .rubricId(textOrNull(root, "rubricId")) + .rubric(parseRubric(id, textOrNull(root, "rubric"))) .transitionRules( readValue(mapper, root, "transitionRules", TRANSITION_LIST)) .planFailureTarget(textOrNull(root, "planFailureTarget")); @@ -139,7 +141,7 @@ private GenericNode deserializeGeneric(ObjectMapper mapper, JsonNode root, Strin .executorType(root.get("executorType").asText()) .transitionRules( readValue(mapper, root, "transitionRules", TRANSITION_LIST)) - .rubricId(textOrNull(root, "rubricId")); + .rubric(parseRubric(id, textOrNull(root, "rubric"))); if (root.has("config")) { b.config(mapper.convertValue(root.get("config"), OBJECT_MAP)); @@ -243,16 +245,17 @@ private List deserializeBranches(JsonNode array) { b.has("yields") && b.get("yields").isArray() ? readStringList(b.get("yields")) : List.of(); + String branchId = b.get("id").asText(); + String rubricText = + b.has("rubric") && !b.get("rubric").isNull() ? b.get("rubric").asText() : null; branches.add( new Branch( - b.get("id").asText(), + branchId, b.get("agentId").asText(), b.has("prompt") && !b.get("prompt").isNull() ? b.get("prompt").asText() : null, - b.has("rubricId") && !b.get("rubricId").isNull() - ? b.get("rubricId").asText() - : null, + parseRubric(branchId, rubricText), b.has("weight") ? b.get("weight").doubleValue() : 1.0, yields)); } @@ -267,6 +270,10 @@ private List readStringList(JsonNode array) { return result; } + private Rubric parseRubric(String nodeId, String rawContent) { + return rawContent != null ? RubricParser.parseContent(nodeId, rawContent) : null; + } + private String textOrNull(JsonNode root, String field) { return root.has(field) ? root.get(field).asText() : null; } diff --git a/hensu-serialization/src/main/java/io/hensu/serialization/NodeSerializer.java b/hensu-serialization/src/main/java/io/hensu/serialization/NodeSerializer.java index eb76f31..03290ce 100644 --- a/hensu-serialization/src/main/java/io/hensu/serialization/NodeSerializer.java +++ b/hensu-serialization/src/main/java/io/hensu/serialization/NodeSerializer.java @@ -15,11 +15,11 @@ /// ``` /// NodeType Additional fields /// ———————————————+———————————————————————————————————————————————————————————————————————— -/// STANDARD │ agentId, prompt, rubricId, reviewConfig, transitionRules, +/// STANDARD │ agentId, prompt, rubric, reviewConfig, transitionRules, /// │ writes, planningConfig, staticPlan, planFailureTarget /// END │ status /// ACTION │ actions, transitionRules -/// GENERIC │ executorType, config, transitionRules, rubricId +/// GENERIC │ executorType, config, transitionRules, rubric /// PARALLEL │ branches, consensusConfig, transitionRules /// FORK │ targets, targetConfigs, transitionRules, waitForAll /// JOIN │ awaitTargets, mergeStrategy, writes, exports, @@ -72,7 +72,7 @@ private void writeStandardNode(StandardNode n, JsonGenerator gen, SerializerProv throws IOException { writeIfNotNull(gen, "agentId", n.getAgentId()); writeIfNotNull(gen, "prompt", n.getPrompt()); - writeIfNotNull(gen, "rubricId", n.getRubricId()); + writeRubric(gen, n.getRubric()); if (n.getReviewConfig() != null) { gen.writeObjectField("reviewConfig", n.getReviewConfig()); } @@ -106,7 +106,7 @@ private void writeGenericNode(GenericNode n, JsonGenerator gen, SerializerProvid provider.defaultSerializeField("config", n.getConfig(), gen); } provider.defaultSerializeField("transitionRules", n.getTransitionRules(), gen); - writeIfNotNull(gen, "rubricId", n.getRubricId()); + writeRubric(gen, n.getRubric()); } private void writeParallelNode(ParallelNode n, JsonGenerator gen, SerializerProvider provider) @@ -158,6 +158,13 @@ private void writeSubWorkflowNode( provider.defaultSerializeField("transitionRules", n.getTransitionRules(), gen); } + private void writeRubric(JsonGenerator gen, io.hensu.core.rubric.model.Rubric rubric) + throws IOException { + if (rubric != null && rubric.getRawContent() != null) { + gen.writeStringField("rubric", rubric.getRawContent()); + } + } + private void writeIfNotNull(JsonGenerator gen, String field, String value) throws IOException { if (value != null && !value.isEmpty()) { gen.writeStringField(field, value); diff --git a/hensu-serialization/src/test/java/io/hensu/serialization/WorkflowSerializerTest.java b/hensu-serialization/src/test/java/io/hensu/serialization/WorkflowSerializerTest.java index 7ec1aa7..57b9fd5 100644 --- a/hensu-serialization/src/test/java/io/hensu/serialization/WorkflowSerializerTest.java +++ b/hensu-serialization/src/test/java/io/hensu/serialization/WorkflowSerializerTest.java @@ -15,6 +15,7 @@ import io.hensu.core.plan.PlanningConfig; import io.hensu.core.review.ReviewConfig; import io.hensu.core.review.ReviewMode; +import io.hensu.core.rubric.RubricParser; import io.hensu.core.rubric.model.ComparisonOperator; import io.hensu.core.rubric.model.DoubleRange; import io.hensu.core.rubric.model.ScoreCondition; @@ -61,7 +62,7 @@ void roundTrip_standardNode() { StandardNode sn = (StandardNode) node; assertThat(sn.getAgentId()).isEqualTo("writer"); assertThat(sn.getPrompt()).isEqualTo("Write something"); - assertThat(sn.getRubricId()).isEqualTo("quality"); + assertThat(sn.getRubric().getRawContent()).isEqualTo("quality"); assertThat(sn.getWrites()).containsExactly("sentiment", "score"); assertThat(sn.getTransitionRules()).hasSize(2); } @@ -130,7 +131,7 @@ void roundTrip_genericNode() { .executorType("validator") .config(Map.of("minLength", 10, "maxLength", 1000)) .transitionRules(List.of(new SuccessTransition("done"))) - .rubricId("validation-rubric") + .rubric(RubricParser.parseContent("validate", "validation-rubric")) .build(); EndNode end = EndNode.builder().id("done").status(ExitStatus.SUCCESS).build(); @@ -139,7 +140,6 @@ void roundTrip_genericNode() { .id("test") .startNode("validate") .nodes(Map.of("validate", generic, "done", end)) - .rubrics(Map.of("validation-rubric", "test-rubric-path")) .build(); Workflow restored = WorkflowSerializer.fromJson(WorkflowSerializer.toJson(workflow)); @@ -147,7 +147,7 @@ void roundTrip_genericNode() { GenericNode restoredGeneric = (GenericNode) restored.getNodes().get("validate"); assertThat(restoredGeneric.getExecutorType()).isEqualTo("validator"); assertThat(restoredGeneric.getConfig()).containsEntry("minLength", 10); - assertThat(restoredGeneric.getRubricId()).isEqualTo("validation-rubric"); + assertThat(restoredGeneric.getRubric().getRawContent()).isEqualTo("validation-rubric"); } @Test @@ -160,7 +160,7 @@ void roundTrip_parallelNode() { "b2", "reviewer", "prompt2", - "rubric1", + RubricParser.parseContent("b2", "rubric1"), 2.0, List.of("api_schema", "confidence"))) .consensus( @@ -174,7 +174,6 @@ void roundTrip_parallelNode() { .id("test") .startNode("parallel") .nodes(Map.of("parallel", parallel, "done", end)) - .rubrics(Map.of("rubric1", "test-rubric-path")) .build(); Workflow restored = WorkflowSerializer.fromJson(WorkflowSerializer.toJson(workflow)); @@ -416,7 +415,7 @@ void roundTrip_planStep_toolCallWithArguments() { @Test void roundTrip_planStep_synthesize() { // Synthesize with null agentId (as stored before executor enrichment) - // and a ToolCall step — verifies multi-step plan order and action type dispatch. + // and a ToolCall step — verifies multistep plan order and action type dispatch. StandardNode start = StandardNode.builder() .id("start") @@ -609,7 +608,6 @@ private Workflow buildStandardWorkflow() { .role("writer") .model("claude-sonnet-4") .build())) - .rubrics(Map.of("quality", "rubric-001")) .nodes( Map.of( "start", @@ -624,7 +622,7 @@ private StandardNode standardNode() { .id("start") .agentId("writer") .prompt("Write something") - .rubricId("quality") + .rubric(RubricParser.parseContent("start", "quality")) .writes(List.of("sentiment", "score")) .transitionRules( List.of(new SuccessTransition("done"), new FailureTransition(3, "done"))) @@ -639,12 +637,6 @@ private Workflow buildWorkflowWith(Node node) { ? Map.of("start", node) : Map.of("start", node, "done", end); - var builder = Workflow.builder().id("test").startNode("start").nodes(nodes); - - if (node.getRubricId() != null && !node.getRubricId().isEmpty()) { - builder.rubrics(Map.of(node.getRubricId(), "test-rubric-path")); - } - - return builder.build(); + return Workflow.builder().id("test").startNode("start").nodes(nodes).build(); } } diff --git a/hensu-server/src/main/java/io/hensu/server/validation/ValidWorkflowValidator.java b/hensu-server/src/main/java/io/hensu/server/validation/ValidWorkflowValidator.java index 80b38f0..25ede5b 100644 --- a/hensu-server/src/main/java/io/hensu/server/validation/ValidWorkflowValidator.java +++ b/hensu-server/src/main/java/io/hensu/server/validation/ValidWorkflowValidator.java @@ -25,7 +25,7 @@ /// @see ValidWorkflow /// @see ValidId for the identifier pattern definition /// @see InputValidator for shared validation predicates -/// @see LogSanitizer for defense-in-depth at log call sites +/// @see io.hensu.core.util.LogSanitizer for defense-in-depth at log call sites public class ValidWorkflowValidator implements ConstraintValidator { @Override @@ -42,7 +42,6 @@ public boolean isValid(Workflow workflow, ConstraintValidatorContext ctx) { validateAgents(errors, workflow.getAgents()); validateNodes(errors, workflow.getNodes()); - validateRubrics(errors, workflow.getRubrics()); var meta = workflow.getMetadata(); if (meta != null) { @@ -102,7 +101,9 @@ private static void validateNode(List errors, String key, Node node) { case StandardNode sn -> { optionalSafeId(errors, p + ".agentId", sn.getAgentId()); rejectDangerousChars(errors, p + ".prompt", sn.getPrompt()); - optionalSafeId(errors, p + ".rubricId", sn.getRubricId()); + if (sn.getRubric() != null) { + rejectDangerousChars(errors, p + ".rubric", sn.getRubric().getRawContent()); + } optionalSafeId(errors, p + ".planFailureTarget", sn.getPlanFailureTarget()); sn.getWrites().forEach(w -> requireSafeId(errors, p + ".writes[]", w)); } @@ -131,7 +132,9 @@ private static void validateNode(List errors, String key, Node node) { } case GenericNode gn -> { requireSafeId(errors, p + ".executorType", gn.getExecutorType()); - optionalSafeId(errors, p + ".rubricId", gn.getRubricId()); + if (gn.getRubric() != null) { + rejectDangerousChars(errors, p + ".rubric", gn.getRubric().getRawContent()); + } } case EndNode _ -> { /* enum status only — no string fields */ @@ -146,18 +149,10 @@ private static void validateBranch(List errors, String prefix, Branch br requireSafeId(errors, prefix + ".branch.id", branch.id()); requireSafeId(errors, prefix + ".branch.agentId", branch.agentId()); rejectDangerousChars(errors, prefix + ".branch.prompt", branch.prompt()); - optionalSafeId(errors, prefix + ".branch.rubricId", branch.rubricId()); - } - - // ———————————————— Rubrics ———————————————— - - private static void validateRubrics(List errors, Map rubrics) { - if (rubrics == null) return; - rubrics.forEach( - (key, value) -> { - requireSafeId(errors, "rubrics[key]", key); - rejectDangerousChars(errors, "rubrics." + key, value); - }); + if (branch.rubric() != null) { + rejectDangerousChars( + errors, prefix + ".branch.rubric", branch.rubric().getRawContent()); + } } // ———————————————— Metadata ———————————————— diff --git a/hensu-server/src/test/java/io/hensu/server/integration/IntegrationTestBase.java b/hensu-server/src/test/java/io/hensu/server/integration/IntegrationTestBase.java index e003bf6..284b27e 100644 --- a/hensu-server/src/test/java/io/hensu/server/integration/IntegrationTestBase.java +++ b/hensu-server/src/test/java/io/hensu/server/integration/IntegrationTestBase.java @@ -2,8 +2,6 @@ import static org.assertj.core.api.Assertions.assertThat; -import io.hensu.core.HensuEnvironment; -import io.hensu.core.agent.AgentRegistry; import io.hensu.core.agent.stub.StubResponseRegistry; import io.hensu.core.state.HensuSnapshot; import io.hensu.core.state.WorkflowStateRepository; @@ -21,8 +19,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.Duration; import java.util.Map; import java.util.Optional; @@ -73,8 +69,6 @@ abstract class IntegrationTestBase { @Inject WorkflowRepository workflowRepository; @Inject WorkflowStateRepository workflowStateRepository; @Inject WorkflowService workflowService; - @Inject AgentRegistry agentRegistry; - @Inject HensuEnvironment hensuEnvironment; /// Resets all mutable test state before each test method. /// @@ -203,27 +197,6 @@ void registerStub(String scenario, String key, String response) { StubResponseRegistry.getInstance().registerResponse(scenario, key, response); } - /// Resolves a classpath rubric resource to a filesystem path. - /// - /// {@link io.hensu.core.rubric.RubricParser#parse} requires real filesystem - /// paths, so this method copies `/rubrics/{resourceName}` to a temporary file - /// and returns its absolute path. - /// - /// @param resourceName rubric file name (e.g. `quality-high.md`), not null - /// @return absolute filesystem path to the temporary copy, never null - /// @throws RuntimeException if the resource cannot be found or copied - String resolveRubricPath(String resourceName) { - try { - String content = loadClasspathResource("/rubrics/" + resourceName); - Path tempFile = Files.createTempFile("rubric-", "-" + resourceName); - Files.writeString(tempFile, content); - tempFile.toFile().deleteOnExit(); - return tempFile.toAbsolutePath().toString(); - } catch (IOException e) { - throw new RuntimeException("Failed to resolve rubric: " + resourceName, e); - } - } - /// Loads a classpath resource as a UTF-8 string. /// /// @param path absolute classpath path (e.g. `/workflows/basic.json`), not null diff --git a/hensu-server/src/test/java/io/hensu/server/integration/RubricEvaluationIntegrationTest.java b/hensu-server/src/test/java/io/hensu/server/integration/RubricEvaluationIntegrationTest.java index f96d45f..16f8fe2 100644 --- a/hensu-server/src/test/java/io/hensu/server/integration/RubricEvaluationIntegrationTest.java +++ b/hensu-server/src/test/java/io/hensu/server/integration/RubricEvaluationIntegrationTest.java @@ -3,13 +3,10 @@ import static org.assertj.core.api.Assertions.assertThat; import io.hensu.core.execution.result.BacktrackEvent; -import io.hensu.core.rubric.RubricParser; -import io.hensu.core.rubric.model.Rubric; import io.hensu.core.state.HensuSnapshot; import io.hensu.core.workflow.Workflow; import io.hensu.server.workflow.ExecutionStartResult; import io.quarkus.test.junit.QuarkusTest; -import java.nio.file.Path; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; @@ -17,17 +14,9 @@ /// Integration tests for rubric-based quality evaluation during workflow execution. /// /// Covers rubric pass/fail evaluation and automatic backtracking triggered by -/// rubric score thresholds. Rubrics are pre-registered in the repository before -/// execution so that `WorkflowExecutor.registerRubricIfAbsent()` finds them -/// already present, avoiding filesystem path resolution in tests. -/// -/// ### Rubric Pre-Registration Strategy -/// The workflow JSON fixtures map `rubricId` to a definition identifier -/// (e.g., `"quality" -> "quality-high"`). At runtime, the executor would treat -/// the definition value as a filesystem path. In tests, we resolve the classpath -/// rubric file to a temp path, parse it with [RubricParser], and register the -/// resulting [Rubric] under the `rubricId` key used by the workflow. This lets -/// `registerRubricIfAbsent()` skip file parsing entirely. +/// rubric score thresholds. Workflow JSON fixtures carry inline rubric content +/// on the node, which the deserializer parses into typed [Rubric] objects at +/// load time — no separate registration step needed. /// /// ### Score Normalization /// The [ScoreExtractingEvaluator][io.hensu.core.rubric.evaluator.ScoreExtractingEvaluator] @@ -59,7 +48,6 @@ class RubricEvaluationIntegrationTest extends IntegrationTestBase { /// directly to the end node. @Test void shouldCompleteWhenRubricPasses() { - parseAndRegisterRubric("quality-high.md"); Workflow workflow = loadWorkflow("rubric-evaluation-pass.json"); registerStub( @@ -97,7 +85,6 @@ void shouldCompleteWhenRubricPasses() { /// execution history, confirming the retry mechanism was triggered. @Test void shouldRetryAndCompleteOnMinorRubricFailure() { - parseAndRegisterRubric("quality-low.md"); Workflow workflow = loadWorkflow("rubric-backtrack-critical.json"); registerStub("research", "Research findings about quantum computing."); @@ -126,30 +113,4 @@ void shouldRetryAndCompleteOnMinorRubricFailure() { assertThat(backtracks.getFirst().getFrom()).isEqualTo("draft"); assertThat(backtracks.getFirst().getTo()).isEqualTo("draft"); } - - /// Parses a rubric from the classpath and registers it in the rubric - /// repository under the specified `rubricId`. - /// - /// The parsed rubric's internal ID (derived from the temp file name) - /// is replaced by building a new rubric with the desired `rubricId`, - /// preserving all criteria and thresholds from the original file. - /// - /// @param resourceName rubric file under `/rubrics/` (e.g. `"quality-high.md"`), not null - private void parseAndRegisterRubric(String resourceName) { - String rubricPath = resolveRubricPath(resourceName); - Rubric parsed = RubricParser.parse(Path.of(rubricPath)); - - // Rebuild with the rubricId the workflow expects - Rubric rubric = - Rubric.builder() - .id("quality") - .name(parsed.getName()) - .version(parsed.getVersion()) - .type(parsed.getType()) - .passThreshold(parsed.getPassThreshold()) - .criteria(parsed.getCriteria()) - .build(); - - hensuEnvironment.getRubricRepository().save(rubric); - } } diff --git a/hensu-server/src/test/java/io/hensu/server/validation/ValidWorkflowValidatorTest.java b/hensu-server/src/test/java/io/hensu/server/validation/ValidWorkflowValidatorTest.java index 45d17ef..fe31fee 100644 --- a/hensu-server/src/test/java/io/hensu/server/validation/ValidWorkflowValidatorTest.java +++ b/hensu-server/src/test/java/io/hensu/server/validation/ValidWorkflowValidatorTest.java @@ -5,6 +5,7 @@ import io.hensu.core.agent.AgentConfig; import io.hensu.core.execution.result.ExitStatus; +import io.hensu.core.rubric.RubricParser; import io.hensu.core.workflow.Workflow; import io.hensu.core.workflow.node.EndNode; import io.hensu.core.workflow.node.StandardNode; @@ -56,11 +57,11 @@ private Workflow workflowWithAgent(String agentKey, AgentConfig agent) { .build(); } - private Workflow workflowWithStandardNode(String nodeKey, StandardNode node) { + private Workflow workflowWithStandardNode(StandardNode node) { return Workflow.builder() .id("wf-1") .version("1.0.0") - .startNode(nodeKey) + .startNode("step1") .agents( Map.of( "agent-1", @@ -71,7 +72,7 @@ private Workflow workflowWithStandardNode(String nodeKey, StandardNode node) { .build())) .nodes( Map.of( - nodeKey, + "step1", node, "end", EndNode.builder().id("end").status(ExitStatus.SUCCESS).build())) @@ -108,7 +109,7 @@ void shouldRejectIdWithNewline() { .build(); assertThat(validator.isValid(wf, ctx)).isFalse(); - verify(ctx).buildConstraintViolationWithTemplate(contains("id")); + verify(ctx).buildConstraintViolationWithTemplate(contains()); } @Test @@ -255,7 +256,7 @@ void shouldRejectStandardNodeWithInvalidAgentId() { .transitionRules(List.of(new SuccessTransition("end"))) .build(); - var wf = workflowWithStandardNode("step1", node); + var wf = workflowWithStandardNode(node); assertThat(validator.isValid(wf, ctx)).isFalse(); } @@ -270,66 +271,26 @@ void shouldAcceptStandardNodeWithValidPromptContainingNewlines() { .transitionRules(List.of(new SuccessTransition("end"))) .build(); - var wf = workflowWithStandardNode("step1", node); + var wf = workflowWithStandardNode(node); assertThat(validator.isValid(wf, ctx)).isTrue(); } + } + + @Nested + class RubricValidation { @Test - void shouldRejectStandardNodeWithNullByteInPrompt() { + void shouldRejectNodeRubricWithNullByte() { var node = StandardNode.builder() .id("step1") .agentId("agent-1") - .prompt("Normal prompt\u0000hidden instructions") + .rubric(RubricParser.parseContent("step1", "Rate from 1-5\u0000hidden")) .transitionRules(List.of(new SuccessTransition("end"))) .build(); - var wf = workflowWithStandardNode("step1", node); - - assertThat(validator.isValid(wf, ctx)).isFalse(); - } - } - - @Nested - class RubricValidation { - - @Test - void shouldRejectRubricKeyWithSpecialChars() { - var wf = - Workflow.builder() - .id("wf-1") - .version("1.0.0") - .startNode("start") - .nodes( - Map.of( - "start", - EndNode.builder() - .id("start") - .status(ExitStatus.SUCCESS) - .build())) - .rubrics(Map.of("rubric