Remove TestController that was used for evaluation and add analysis prompt variants

Author: ZiyaSelim

src/main/java/de/unistuttgart/iste/meitrex/assignment_service/service/uml_assignment/UmlEvaluationService.java

@@ -1,6 +1,5 @@
 package de.unistuttgart.iste.meitrex.assignment_service.service.uml_assignment;
 
-import de.unistuttgart.iste.meitrex.assignment_service.controller.UmlTestController;
 import de.unistuttgart.iste.meitrex.assignment_service.exception.AiEvaluationException;
 import de.unistuttgart.iste.meitrex.assignment_service.persistence.entity.umlExercise.UmlEvaluationJobEntity;
 import de.unistuttgart.iste.meitrex.assignment_service.persistence.entity.umlExercise.UmlFeedbackEntity;
@@ -161,94 +160,6 @@ private UmlFeedbackResponse performGrading(
         return response;
     }
 
-    /**
-     * Executes the evaluation pipeline without saving to the database.
-     * Returns granular timings for benchmarking LLM configurations.
-     */
-    public UmlTestController.TestEvalResponse dryRunEvaluation(
-            final String studentModel,
-            final String tutorModel,
-            final String gradingRules,
-            final int maxPoints,
-            final double requiredPercentage,
-            final String analysisModelOverride,
-            final String gradingModelOverride,
-            final String promptMode,
-            final String taskDescription
-    ) {
-        long startTotal = System.currentTimeMillis();
-
-        // 1. Run Analysis with Timer
-        long startAnalysis = System.currentTimeMillis();
-        Map<String, String> analysisArgs = new HashMap<>();
-        analysisArgs.put("studentModel", studentModel);
-
-        UmlAnalysisResponse fallbackAnalysis = new UmlAnalysisResponse(
-                Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), false, "Analysis failed."
-        );
-
-        String prompt;
-        if ("TASK_BASED".equals(promptMode)) {
-            prompt = "analysis_task.md";
-            analysisArgs.put("taskDescription", taskDescription);
-            analysisArgs.put("tutorModel", tutorModel);
-        } else if ("PURE_TASK".equals(promptMode)) {
-            prompt = "uml_analysis_pure_task.md";
-            analysisArgs.put("taskDescription", taskDescription);
-        } else {
-            prompt = TEMPLATE_ANALYSIS;
-            analysisArgs.put("tutorModel", tutorModel);
-        }
-
-        UmlAnalysisResponse analysis = ollamaClient.startQuery(
-                UmlAnalysisResponse.class, prompt, analysisArgs, fallbackAnalysis, analysisModelOverride);
-
-        long endAnalysis = System.currentTimeMillis();
-        double analysisDuration = (endAnalysis - startAnalysis) / 1000.0;
-
-        if ("Analysis failed.".equals(analysis.analysisSummary())) {
-            throw new RuntimeException("Analysis LLM failed during dry run.");
-        }
-
-        // 2. Run Grading with Timer
-        long startGrading = System.currentTimeMillis();
-        String effectiveRules = (gradingRules != null && !gradingRules.isBlank()) ? gradingRules : "Standard UML grading.";
-        Map<String, String> gradingArgs = Map.of(
-                "maxPoints", String.valueOf(maxPoints),
-                "passingThreshold", String.valueOf(requiredPercentage * maxPoints),
-                "showSolution", "true",
-                "gradingRules", effectiveRules,
-                "isValid", String.valueOf(analysis.isSemanticallyValid()),
-                "correctElements", formatListForPrompt(analysis.correctElements()),
-                "semanticErrors", formatListForPrompt(analysis.semanticErrors()),
-                "missingElements", formatListForPrompt(analysis.missingElements())
-        );
-
-        UmlFeedbackResponse fallbackGrading = new UmlFeedbackResponse("Grading unavailable.", 0);
-
-        UmlFeedbackResponse grading = ollamaClient.startQuery(
-                UmlFeedbackResponse.class, TEMPLATE_GRADING, gradingArgs, fallbackGrading, gradingModelOverride);
-
-        long endGrading = System.currentTimeMillis();
-        double gradingDuration = (endGrading - startGrading) / 1000.0;
-
-        if ("Grading unavailable.".equals(grading.feedbackText())) {
-            throw new RuntimeException("Grading LLM failed during dry run.");
-        }
-
-        long endTotal = System.currentTimeMillis();
-        double totalDuration = (endTotal - startTotal) / 1000.0;
-
-        // Return the combined payload
-        return new UmlTestController.TestEvalResponse(
-                grading.points(),
-                grading.feedbackText(),
-                analysisDuration,
-                gradingDuration,
-                totalDuration
-        );
-    }
-
     /**
      * Safely formats a list of strings into a single semicolon-separated string.
      * <p>

src/main/resources/prompt_templates/analysis_task.md

@@ -0,0 +1,41 @@
+### ROLE
+You are a Requirements-Driven Software Architecture Evaluator.
+You are evaluating a student's UML class diagram based *strictly on the original assignment task*. A Tutor Reference Solution is provided, but it is only *one possible valid solution*, not the absolute truth.
+
+### DATA TO ANALYZE
+- **Original Assignment Task:**
+---
+{{taskDescription}}
+---
+- **Tutor Reference Solution (Use as a complexity hint only):**
+---
+{{tutorModel}}
+---
+- **Student Submission:**
+---
+{{studentModel}}
+---
+
+### EVALUATION STRATEGY
+1. Read the **Assignment Task** to understand the required entities, attributes, and relationships.
+2. Analyze the **Student Submission**. Does it fulfill the core requirements of the task?
+3. If the student diverges from the Tutor Reference but still logically satisfies the Assignment Task (e.g., using a List of Enum values instead of a dedicated Rating class), mark it as **CORRECT**.
+4. Do NOT penalize for minor syntax variations (`int` vs `Integer`) or slightly different but logical association names.
+
+### CATEGORIZATION STRICTNESS
+- **correctElements:** List elements that successfully fulfill a requirement from the Assignment Task.
+- **missingElements:** List elements explicitly requested by the Assignment Task that the student failed to include.
+- **semanticErrors:** List elements that violate UML logic or explicitly contradict the Assignment Task.
+
+### CRITICAL OUTPUT RULES
+- Ignore visual layout coordinates.
+- Output ONLY valid JSON matching the exact schema below.
+
+**Output Format (JSON):**
+{
+"correctElements": ["Elements fulfilling the task requirements."],
+"semanticErrors": ["Logical violations or contradictions of the task."],
+"missingElements": ["Elements required by the task that are missing."],
+"isSemanticallyValid": <boolean>,
+"analysisSummary": "A concise summary of how well the student met the task requirements."
+}

src/main/resources/prompt_templates/uml_analysis_pure_task.md

@@ -0,0 +1,37 @@
+### ROLE
+You are an Expert Requirements Engineer and Software Architecture Evaluator.
+Your task is to evaluate a student's UML class diagram based STRICTLY and ONLY on the original assignment task. You do not have a reference solution. You must deduce the correctness entirely from domain logic and the provided requirements.
+
+### DATA TO ANALYZE
+- **Original Assignment Task:**
+---
+{{taskDescription}}
+---
+- **Student Submission:**
+---
+{{studentModel}}
+---
+
+### EVALUATION STRATEGY
+1. Read the **Assignment Task** and extract every explicit requirement (entities, attributes, and relationships).
+2. Analyze the **Student Submission**. Check if every requirement from the task is fulfilled.
+3. Because there is no reference solution, you must be tolerant of different architectural choices. If a student uses an Enum, a Class, or an Interface in a way that logically solves the domain problem described in the task, mark it as CORRECT.
+4. Only penalize elements if they directly contradict the task description or violate standard UML logic.
+
+### CATEGORIZATION STRICTNESS
+- **correctElements:** List elements that successfully fulfill a requirement from the Assignment Task.
+- **missingElements:** List elements explicitly requested by the Assignment Task that the student failed to include.
+- **semanticErrors:** List elements that violate UML logic or explicitly contradict the Assignment Task.
+
+### CRITICAL OUTPUT RULES
+- Ignore visual layout coordinates (pos, vdist, layout, etc.).
+- Output ONLY valid JSON matching the exact schema below.
+
+**Output Format (JSON):**
+{
+"correctElements": ["Elements fulfilling the task requirements."],
+"semanticErrors": ["Logical violations or contradictions of the task."],
+"missingElements": ["Elements required by the task that are missing."],
+"isSemanticallyValid": <boolean>,
+"analysisSummary": "A concise summary of how well the student met the task requirements based ONLY on the text prompt."
+}