-
Notifications
You must be signed in to change notification settings - Fork 7.6k
feat(providers): add ProviderCapabilities dataclass with unified capability detection #6233
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ | |
|
|
||
| from collections import defaultdict | ||
| from collections.abc import Callable | ||
| from dataclasses import dataclass | ||
| from datetime import datetime | ||
| import json | ||
| import logging | ||
|
|
@@ -345,6 +346,48 @@ def _ensure_litellm() -> bool: | |
| ] | ||
|
|
||
|
|
||
| @dataclass | ||
| class ProviderCapabilities: | ||
| """Describes which features a provider supports. | ||
|
|
||
| This aggregates the scattered capability checks (supports_response_schema, | ||
| supports_function_calling, supports_stop_words, etc.) into a single | ||
| structured object. The LLM class can populate this via litellm introspection | ||
| or provider-specific knowledge. | ||
|
|
||
| Attributes: | ||
| supports_response_format: Whether structured output (json_object / json_schema) works. | ||
| supports_tool_calling: Whether function/tool calling works. | ||
| supports_reasoning: Whether reasoning_effort / thinking tokens are forwarded. | ||
| supports_streaming: Whether streaming responses work. | ||
| supports_image_input: Whether the provider accepts image content blocks. | ||
| supports_stop_words: Whether the ``stop`` parameter is honored. | ||
| """ | ||
|
|
||
| supports_response_format: bool = True | ||
| supports_tool_calling: bool = True | ||
| supports_reasoning: bool = False | ||
| supports_streaming: bool = True | ||
| supports_image_input: bool = False | ||
| supports_stop_words: bool = True | ||
|
|
||
|
|
||
| # Providers known to NOT support response_format. This is the fallback when | ||
| # litellm's supports_response_schema() cannot be used (e.g. litellm unavailable). | ||
| # Extended via community contributions as new providers are discovered. | ||
| _PROVIDERS_WITHOUT_RESPONSE_FORMAT: Final[frozenset[str]] = frozenset( | ||
| {"deepseek", "ollama", "ollama_chat", "hosted_vllm"} | ||
| ) | ||
|
|
||
| # Providers known to NOT support tool calling. | ||
| _PROVIDERS_WITHOUT_TOOL_CALLING: Final[frozenset[str]] = frozenset() | ||
|
|
||
| # Providers known to support reasoning/thinking tokens. | ||
| _PROVIDERS_WITH_REASONING: Final[frozenset[str]] = frozenset( | ||
| {"openai", "azure", "azure_openai", "anthropic", "claude", "bedrock", "aws", "cerebras"} | ||
| ) | ||
|
|
||
|
|
||
| class Delta(TypedDict): | ||
| content: str | None | ||
| role: str | None | ||
|
|
@@ -2350,13 +2393,18 @@ def _validate_call_params(self) -> None: | |
| return | ||
|
|
||
| provider = self._get_custom_llm_provider() | ||
| if self.response_format is not None and not supports_response_schema( | ||
| model=self.model, | ||
| custom_llm_provider=provider, | ||
| ): | ||
| caps = self.get_capabilities() | ||
| if self.response_format is not None and not caps.supports_response_format: | ||
| raise ValueError( | ||
| f"The provider '{provider or self.provider}' does not support " | ||
| "response_format. Remove response_format, use result_as_string=True, " | ||
| "or switch to a provider that supports structured output." | ||
| ) | ||
| if self.reasoning_effort is not None and not caps.supports_reasoning: | ||
| raise ValueError( | ||
| f"The model {self.model} does not support response_format for provider '{provider}'. " | ||
| "Please remove response_format or use a supported model." | ||
| f"The provider '{provider or self.provider}' does not support " | ||
| "reasoning_effort. Remove reasoning_effort or switch to a provider " | ||
| "that supports thinking/reasoning tokens." | ||
| ) | ||
|
|
||
| def supports_function_calling(self) -> bool: | ||
|
|
@@ -2400,6 +2448,63 @@ def supports_stop_words(self) -> bool: | |
| logging.error(f"Failed to get supported params: {e!s}") | ||
| return True # Default to True | ||
|
|
||
| def get_capabilities(self) -> ProviderCapabilities: | ||
| """Return the structured capabilities for this provider/model. | ||
|
|
||
| Aggregates individual capability checks into a single | ||
| :class:`ProviderCapabilities` object. Uses litellm introspection | ||
| when litellm is available, falling back to the static provider | ||
| allowlists defined in this module. | ||
|
|
||
| Returns: | ||
| A fully populated ProviderCapabilities instance. | ||
| """ | ||
| provider = self._get_custom_llm_provider() or self.provider or "openai" | ||
|
|
||
| # --- response_format --- | ||
| supports_response_format = True | ||
| if _ensure_litellm() and supports_response_schema is not None: | ||
| try: | ||
| supports_response_format = supports_response_schema( | ||
| model=self.model, custom_llm_provider=provider | ||
| ) | ||
| except Exception: | ||
| supports_response_format = ( | ||
| provider not in _PROVIDERS_WITHOUT_RESPONSE_FORMAT | ||
| ) | ||
| else: | ||
| supports_response_format = ( | ||
| provider not in _PROVIDERS_WITHOUT_RESPONSE_FORMAT | ||
| ) | ||
|
|
||
| # --- tool calling --- | ||
| supports_tool_calling = self.supports_function_calling() | ||
| if not supports_tool_calling and provider not in _PROVIDERS_WITHOUT_TOOL_CALLING: | ||
| # supports_function_calling may return False for unknown providers; | ||
| # fall back to the allowlist | ||
| supports_tool_calling = True | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
|
|
||
| # --- reasoning --- | ||
| supports_reasoning = provider in _PROVIDERS_WITH_REASONING | ||
|
Comment on lines
+2487
to
+2488
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🧩 Analysis chain🏁 Script executed: # Find the file and verify its existence
find . -type f -name "llm.py" | head -20Repository: crewAIInc/crewAI Length of output: 91 🏁 Script executed: # Get line count to ensure lines 2487-2488 exist
wc -l lib/crewai/src/crewai/llm.py 2>/dev/null || echo "File not found in that path"Repository: crewAIInc/crewAI Length of output: 94 🏁 Script executed: # Read the context around lines 2487-2488
sed -n '2480,2500p' lib/crewai/src/crewai/llm.pyRepository: crewAIInc/crewAI Length of output: 896 🏁 Script executed: # Search for _PROVIDERS_WITH_REASONING definition
rg "_PROVIDERS_WITH_REASONING" --type py -B 5 -A 10Repository: crewAIInc/crewAI Length of output: 1980 🏁 Script executed: # Check if litellm module is available and inspect its structure
python3 - <<'PY'
try:
import litellm
import inspect
# Check for supports_reasoning function
if hasattr(litellm, 'utils'):
print("✓ litellm.utils exists")
if hasattr(litellm.utils, 'supports_reasoning'):
print("✓ litellm.utils.supports_reasoning exists")
sig = inspect.signature(litellm.utils.supports_reasoning)
print(f" Signature: {sig}")
else:
print("✗ litellm.utils.supports_reasoning does NOT exist")
print(" Available attributes in litellm.utils:")
attrs = [x for x in dir(litellm.utils) if 'reasoning' in x.lower()]
if attrs:
print(f" Reasoning-related: {attrs}")
else:
print(" No reasoning-related attributes found")
else:
print("✗ litellm.utils does NOT exist")
# Check litellm version and main attributes
print(f"\nlitellm version: {litellm.__version__ if hasattr(litellm, '__version__') else 'unknown'}")
reasoning_attrs = [x for x in dir(litellm) if 'reasoning' in x.lower()]
if reasoning_attrs:
print(f"Reasoning-related in litellm: {reasoning_attrs}")
except ImportError as e:
print(f"Cannot import litellm: {e}")
except Exception as e:
print(f"Error: {e}")
PYRepository: crewAIInc/crewAI Length of output: 109 🌐 Web query:
💡 Result: LiteLLM provides a built-in utility function, litellm.supports_reasoning(model="<model_name>"), to detect whether a specific model supports reasoning capabilities [1][2]. Key details regarding this functionality include: How it Works: - The function returns True if the model is registered in LiteLLM's internal model registry (model_prices_and_context_window.json) as supporting reasoning, and False otherwise [1][2][3]. - This flag is critical for ensuring that reasoning-specific parameters—such as thinking blocks, reasoning effort (e.g., low, high, x-high), or related arguments—are correctly handled [1][4][5][3]. If this capability is missing or set to false, LiteLLM may raise an UnsupportedParamsError or incorrectly route requests [4][3]. Integration in LiteLLM: - Internal Routing: LiteLLM uses this utility internally to gate reasoning features [6][5]. For example, it helps determine whether to route requests to specialized reasoning APIs or to allow specific parameters like reasoning_effort [6][4][3]. - Data Source: The reasoning capability is defined in LiteLLM's model pricing and context window registry [7][8][5]. Maintenance of these entries is handled via internal model registry updates, ensuring that new models (such as new Claude or Mistral versions) are correctly identified as supporting reasoning [4][5][3]. - Additional Capability Flags: Beyond basic reasoning support, LiteLLM tracks more granular capabilities, such as supports_xhigh_reasoning_effort, supports_none_reasoning_effort, and supports_minimal_reasoning_effort, which can be accessed via litellm.get_model_info [7]. Usage Example: import litellm model_name = "anthropic/claude-3-7-sonnet-20250219" if litellm.supports_reasoning(model=model_name): print(f"{model_name} supports reasoning.") else: print(f"{model_name} does NOT support reasoning.") [1][2] Citations:
🌐 Web query:
💡 Result: The function signature for litellm.supports_reasoning is: litellm.supports_reasoning(model: str, custom_llm_provider: Optional[str] = None) -> bool [1][2] This utility function determines whether a specific model supports reasoning capabilities (such as thinking blocks or reasoning effort parameters) [1][3]. Parameters: - model (str): The name or identifier of the model (e.g., 'anthropic/claude-3-7-sonnet-20250219') [1][3]. - custom_llm_provider (Optional[str]): An optional parameter used to explicitly specify the LLM provider (e.g., 'mistral', 'bedrock') [2][4][5]. This is particularly useful when the provider cannot be automatically inferred from the model string or when resolving provider-specific capability configurations [2][4][5]. The function returns a boolean value: True if the model supports reasoning, and False otherwise [1]. LiteLLM manages these capability flags within its internal model cost and capability registry (model_prices_and_context_window.json) [2][5]. Citations:
🏁 Script executed: # Search for litellm imports and usage in the codebase to understand how it's used
rg "import litellm|from litellm" lib/crewai/src/crewai/llm.py -A 3Repository: crewAIInc/crewAI Length of output: 970 🏁 Script executed: # Check the context of self.model and provider to understand what information is available
sed -n '2470,2510p' lib/crewai/src/crewai/llm.pyRepository: crewAIInc/crewAI Length of output: 1626 Make reasoning detection model-aware. The current provider-level allowlist allows Suggested direction # --- reasoning ---
- supports_reasoning = provider in _PROVIDERS_WITH_REASONING
+ supports_reasoning = provider in _PROVIDERS_WITH_REASONING
+ if litellm:
+ try:
+ supports_reasoning = litellm.supports_reasoning(
+ model=self.model, custom_llm_provider=provider
+ )
+ except Exception:
+ supports_reasoning = provider in _PROVIDERS_WITH_REASONING🤖 Prompt for AI Agents |
||
|
|
||
| # --- streaming --- | ||
| supports_streaming = True # virtually all modern providers support streaming | ||
|
|
||
| # --- image input --- | ||
| supports_image_input = self.supports_multimodal() | ||
|
|
||
| # --- stop words --- | ||
| supports_stop_words = self.supports_stop_words() | ||
|
|
||
| return ProviderCapabilities( | ||
| supports_response_format=supports_response_format, | ||
| supports_tool_calling=supports_tool_calling, | ||
| supports_reasoning=supports_reasoning, | ||
| supports_streaming=supports_streaming, | ||
| supports_image_input=supports_image_input, | ||
| supports_stop_words=supports_stop_words, | ||
| ) | ||
|
|
||
| def get_context_window_size(self) -> int: | ||
| """ | ||
| Returns the context window size, using 75% of the maximum to avoid | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.