config.yaml

---

# NAME                                          SIZE     JUDGE
# deepseek-r1:14b                               9.0 GB   +
# gemma3:4b                                     3.3 GB   +
# SpeakLeash/bielik-11b-v2.3-instruct:Q4_K_M    6.7 GB   +
# qwen3:14b                                     9.3 GB   +
# phi4:latest                                   9.1 GB   +


# use a colon ":" after the model name if there are parameters.
# DO NOT use a colon ":" at the end of the model name if there are no parameters.
models:
  - deepseek-r1:14b:
      judge: true
  - gemma3:4b:
      judge: true
  - SpeakLeash/bielik-11b-v2.3-instruct:Q4_K_M:
      judge: true
  - qwen3:14b:
      judge: true
  - phi4:latest:
      judge: true

#default_system_prompt:
#  - "Respond clearly, concisely, and professionally."
#  - "Provide accurate, relevant, and logically structured information."
#  - "Maintain neutrality, avoid speculation, and clarify assumptions explicitly when needed."

#judge:
#  system_prompt:
#    - "You are an AI assistant responsible for evaluating and comparing responses from different language models."
#    - "Your evaluation must be objective, precise, and consistent."
#    - "Assess each response separately based on two criteria: correctness and overall quality."
#    - "Correctness measures factual accuracy, completeness, and relevance to the user's question."
#    - "Overall quality includes clarity, coherence, professionalism, logical structuring, conciseness, and neutrality."
#    - "Base your evaluation strictly on provided responses without speculating or introducing new information."
#    - "After evaluating all responses, generate your own concise, professional, and accurate answer based ONLY on responses you rated at least 80 in correctness."
#    - "If no response meets the correctness threshold, state clearly that no sufficiently correct answer was provided."
#    - "Use English, format your answer clearly using markdown, and avoid any XML or HTML tags."
#  user_question_prefix: "**User Question:**"
#  model_responses_prefix: "**Collected Model Responses:**"
#  model_response_format:
#    - "### Model name: {model_name}"
#    - "### Model response: {response_content}"
#  response_format:
#    - "Your evaluation must include:"
#    - "- **Correctness**: Rating (0-100)"
#    - "- **Overall Quality**: Rating (0-100), must not exceed correctness rating"
#    - "- **Assessment**: Short justification highlighting strengths and weaknesses"
#    - "Present the evaluation as markdown table:"
#    - ""
#    - "| Model Name | Correctness | Quality | Assessment |"
#    - "|------------|-------------|---------|------------|"
#    - ""
#    - "**Your Synthesized Answer:**"
#    - "Provide a concise and professional answer synthesized from the responses you rated as correct."

ollama_url: http://localhost:11434
log_level: INFO
http_timeout: 300.0

commands:
  - help:
      - /?
      - /help
  - exit:
      - /exit
      - /quit
      - /bye
  - reset:
      - /reset
  - prompts:
      - /prompt
      - /prompts