Skip to content

Commit b05f6d3

Browse files
ChesarsChesars
authored
fix(groq): check supports_response_schema before using json_tool_call workaround (#17438)
Fixes issue where LiteLLM used the json_tool_call workaround for all Groq models, even those that support native json_schema (e.g., gpt-oss-120b, llama-4, kimi-k2). This caused errors when users passed their own tools alongside structured outputs. Changes: - Check `supports_response_schema()` before using the workaround - Only use json_tool_call workaround for models without native support - Add clear error message when using workaround with user-provided tools - Update model config to set `supports_response_schema: false` for models that don't support native json_schema - Add unit tests for structured output handling
1 parent 6d831ff commit b05f6d3

File tree

4 files changed

+209
-50
lines changed

4 files changed

+209
-50
lines changed

‎litellm/llms/groq/chat/transformation.py‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -218,22 +218,47 @@ def map_openai_params(
218218
When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
219219
- You usually want to provide a single tool
220220
- You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool
221-
- Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective.
221+
- Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model's perspective.
222+
223+
Note: This workaround is only for models that don't support native json_schema.
224+
Models like gpt-oss-120b, llama-4, kimi-k2 support native json_schema and should
225+
pass response_format directly to Groq.
226+
See: https://console.groq.com/docs/structured-outputs#supported-models
222227
"""
223228
if json_schema is not None:
224-
_tool_choice = {
225-
"type": "function",
226-
"function": {"name": "json_tool_call"},
227-
}
228-
_tool = self._create_json_tool_call_for_response_format(
229-
json_schema=json_schema,
230-
)
231-
optional_params["tools"] = [_tool]
232-
optional_params["tool_choice"] = _tool_choice
233-
optional_params["json_mode"] = True
234-
non_default_params.pop(
235-
"response_format", None
236-
) # only remove if it's a json_schema - handled via using groq's tool calling params.
229+
# Check if model supports native response_schema
230+
if not litellm.supports_response_schema(
231+
model=model, custom_llm_provider="groq"
232+
):
233+
# Check if user is also passing tools - this combination won't work
234+
# See: https://console.groq.com/docs/structured-outputs
235+
# "Streaming and tool use are not currently supported with Structured Outputs"
236+
if "tools" in non_default_params:
237+
raise litellm.BadRequestError(
238+
message=f"Groq model '{model}' does not support native structured outputs. "
239+
"LiteLLM uses a tool-calling workaround for structured outputs on this model, "
240+
"which is incompatible with user-provided tools. "
241+
"Either use a model that supports native structured outputs "
242+
"(e.g., gpt-oss-120b, llama-4, kimi-k2), or remove the tools parameter. "
243+
"See: https://console.groq.com/docs/structured-outputs#supported-models",
244+
model=model,
245+
llm_provider="groq",
246+
)
247+
# Use workaround only for models without native support
248+
_tool_choice = {
249+
"type": "function",
250+
"function": {"name": "json_tool_call"},
251+
}
252+
_tool = self._create_json_tool_call_for_response_format(
253+
json_schema=json_schema,
254+
)
255+
optional_params["tools"] = [_tool]
256+
optional_params["tool_choice"] = _tool_choice
257+
optional_params["json_mode"] = True
258+
non_default_params.pop(
259+
"response_format", None
260+
) # only remove if it's a json_schema - handled via using groq's tool calling params.
261+
# else: model supports native json_schema, let response_format pass through
237262
optional_params = super().map_openai_params(
238263
non_default_params, optional_params, model, drop_params
239264
)

‎litellm/model_prices_and_context_window_backup.json‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16881,7 +16881,7 @@
1688116881
"output_cost_per_token": 9.9e-07,
1688216882
"supports_function_calling": true,
1688316883
"supports_reasoning": true,
16884-
"supports_response_schema": true,
16884+
"supports_response_schema": false,
1688516885
"supports_tool_choice": true
1688616886
},
1688716887
"groq/distil-whisper-large-v3-en": {
@@ -16900,7 +16900,7 @@
1690016900
"mode": "chat",
1690116901
"output_cost_per_token": 7e-08,
1690216902
"supports_function_calling": true,
16903-
"supports_response_schema": true,
16903+
"supports_response_schema": false,
1690416904
"supports_tool_choice": true
1690516905
},
1690616906
"groq/gemma2-9b-it": {
@@ -16912,7 +16912,7 @@
1691216912
"mode": "chat",
1691316913
"output_cost_per_token": 2e-07,
1691416914
"supports_function_calling": false,
16915-
"supports_response_schema": true,
16915+
"supports_response_schema": false,
1691616916
"supports_tool_choice": false
1691716917
},
1691816918
"groq/llama-3.1-405b-reasoning": {
@@ -16924,7 +16924,7 @@
1692416924
"mode": "chat",
1692516925
"output_cost_per_token": 7.9e-07,
1692616926
"supports_function_calling": true,
16927-
"supports_response_schema": true,
16927+
"supports_response_schema": false,
1692816928
"supports_tool_choice": true
1692916929
},
1693016930
"groq/llama-3.1-70b-versatile": {
@@ -16937,7 +16937,7 @@
1693716937
"mode": "chat",
1693816938
"output_cost_per_token": 7.9e-07,
1693916939
"supports_function_calling": true,
16940-
"supports_response_schema": true,
16940+
"supports_response_schema": false,
1694116941
"supports_tool_choice": true
1694216942
},
1694316943
"groq/llama-3.1-8b-instant": {
@@ -16949,7 +16949,7 @@
1694916949
"mode": "chat",
1695016950
"output_cost_per_token": 8e-08,
1695116951
"supports_function_calling": true,
16952-
"supports_response_schema": true,
16952+
"supports_response_schema": false,
1695316953
"supports_tool_choice": true
1695416954
},
1695516955
"groq/llama-3.2-11b-text-preview": {
@@ -16962,7 +16962,7 @@
1696216962
"mode": "chat",
1696316963
"output_cost_per_token": 1.8e-07,
1696416964
"supports_function_calling": true,
16965-
"supports_response_schema": true,
16965+
"supports_response_schema": false,
1696616966
"supports_tool_choice": true
1696716967
},
1696816968
"groq/llama-3.2-11b-vision-preview": {
@@ -16975,7 +16975,7 @@
1697516975
"mode": "chat",
1697616976
"output_cost_per_token": 1.8e-07,
1697716977
"supports_function_calling": true,
16978-
"supports_response_schema": true,
16978+
"supports_response_schema": false,
1697916979
"supports_tool_choice": true,
1698016980
"supports_vision": true
1698116981
},
@@ -16989,7 +16989,7 @@
1698916989
"mode": "chat",
1699016990
"output_cost_per_token": 4e-08,
1699116991
"supports_function_calling": true,
16992-
"supports_response_schema": true,
16992+
"supports_response_schema": false,
1699316993
"supports_tool_choice": true
1699416994
},
1699516995
"groq/llama-3.2-3b-preview": {
@@ -17002,7 +17002,7 @@
1700217002
"mode": "chat",
1700317003
"output_cost_per_token": 6e-08,
1700417004
"supports_function_calling": true,
17005-
"supports_response_schema": true,
17005+
"supports_response_schema": false,
1700617006
"supports_tool_choice": true
1700717007
},
1700817008
"groq/llama-3.2-90b-text-preview": {
@@ -17015,7 +17015,7 @@
1701517015
"mode": "chat",
1701617016
"output_cost_per_token": 9e-07,
1701717017
"supports_function_calling": true,
17018-
"supports_response_schema": true,
17018+
"supports_response_schema": false,
1701917019
"supports_tool_choice": true
1702017020
},
1702117021
"groq/llama-3.2-90b-vision-preview": {
@@ -17028,7 +17028,7 @@
1702817028
"mode": "chat",
1702917029
"output_cost_per_token": 9e-07,
1703017030
"supports_function_calling": true,
17031-
"supports_response_schema": true,
17031+
"supports_response_schema": false,
1703217032
"supports_tool_choice": true,
1703317033
"supports_vision": true
1703417034
},
@@ -17052,7 +17052,7 @@
1705217052
"mode": "chat",
1705317053
"output_cost_per_token": 7.9e-07,
1705417054
"supports_function_calling": true,
17055-
"supports_response_schema": true,
17055+
"supports_response_schema": false,
1705617056
"supports_tool_choice": true
1705717057
},
1705817058
"groq/llama-guard-3-8b": {
@@ -17073,7 +17073,7 @@
1707317073
"mode": "chat",
1707417074
"output_cost_per_token": 8e-07,
1707517075
"supports_function_calling": true,
17076-
"supports_response_schema": true,
17076+
"supports_response_schema": false,
1707717077
"supports_tool_choice": true
1707817078
},
1707917079
"groq/llama3-groq-70b-8192-tool-use-preview": {
@@ -17086,7 +17086,7 @@
1708617086
"mode": "chat",
1708717087
"output_cost_per_token": 8.9e-07,
1708817088
"supports_function_calling": true,
17089-
"supports_response_schema": true,
17089+
"supports_response_schema": false,
1709017090
"supports_tool_choice": true
1709117091
},
1709217092
"groq/llama3-groq-8b-8192-tool-use-preview": {
@@ -17099,7 +17099,7 @@
1709917099
"mode": "chat",
1710017100
"output_cost_per_token": 1.9e-07,
1710117101
"supports_function_calling": true,
17102-
"supports_response_schema": true,
17102+
"supports_response_schema": false,
1710317103
"supports_tool_choice": true
1710417104
},
1710517105
"groq/meta-llama/llama-4-maverick-17b-128e-instruct": {
@@ -17145,7 +17145,7 @@
1714517145
"mode": "chat",
1714617146
"output_cost_per_token": 2.4e-07,
1714717147
"supports_function_calling": true,
17148-
"supports_response_schema": true,
17148+
"supports_response_schema": false,
1714917149
"supports_tool_choice": true
1715017150
},
1715117151
"groq/moonshotai/kimi-k2-instruct": {
@@ -17221,7 +17221,7 @@
1722117221
"output_cost_per_token": 5.9e-07,
1722217222
"supports_function_calling": true,
1722317223
"supports_reasoning": true,
17224-
"supports_response_schema": true,
17224+
"supports_response_schema": false,
1722517225
"supports_tool_choice": true
1722617226
},
1722717227
"groq/whisper-large-v3": {

‎model_prices_and_context_window.json‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16881,7 +16881,7 @@
1688116881
"output_cost_per_token": 9.9e-07,
1688216882
"supports_function_calling": true,
1688316883
"supports_reasoning": true,
16884-
"supports_response_schema": true,
16884+
"supports_response_schema": false,
1688516885
"supports_tool_choice": true
1688616886
},
1688716887
"groq/distil-whisper-large-v3-en": {
@@ -16900,7 +16900,7 @@
1690016900
"mode": "chat",
1690116901
"output_cost_per_token": 7e-08,
1690216902
"supports_function_calling": true,
16903-
"supports_response_schema": true,
16903+
"supports_response_schema": false,
1690416904
"supports_tool_choice": true
1690516905
},
1690616906
"groq/gemma2-9b-it": {
@@ -16912,7 +16912,7 @@
1691216912
"mode": "chat",
1691316913
"output_cost_per_token": 2e-07,
1691416914
"supports_function_calling": false,
16915-
"supports_response_schema": true,
16915+
"supports_response_schema": false,
1691616916
"supports_tool_choice": false
1691716917
},
1691816918
"groq/llama-3.1-405b-reasoning": {
@@ -16924,7 +16924,7 @@
1692416924
"mode": "chat",
1692516925
"output_cost_per_token": 7.9e-07,
1692616926
"supports_function_calling": true,
16927-
"supports_response_schema": true,
16927+
"supports_response_schema": false,
1692816928
"supports_tool_choice": true
1692916929
},
1693016930
"groq/llama-3.1-70b-versatile": {
@@ -16937,7 +16937,7 @@
1693716937
"mode": "chat",
1693816938
"output_cost_per_token": 7.9e-07,
1693916939
"supports_function_calling": true,
16940-
"supports_response_schema": true,
16940+
"supports_response_schema": false,
1694116941
"supports_tool_choice": true
1694216942
},
1694316943
"groq/llama-3.1-8b-instant": {
@@ -16949,7 +16949,7 @@
1694916949
"mode": "chat",
1695016950
"output_cost_per_token": 8e-08,
1695116951
"supports_function_calling": true,
16952-
"supports_response_schema": true,
16952+
"supports_response_schema": false,
1695316953
"supports_tool_choice": true
1695416954
},
1695516955
"groq/llama-3.2-11b-text-preview": {
@@ -16962,7 +16962,7 @@
1696216962
"mode": "chat",
1696316963
"output_cost_per_token": 1.8e-07,
1696416964
"supports_function_calling": true,
16965-
"supports_response_schema": true,
16965+
"supports_response_schema": false,
1696616966
"supports_tool_choice": true
1696716967
},
1696816968
"groq/llama-3.2-11b-vision-preview": {
@@ -16975,7 +16975,7 @@
1697516975
"mode": "chat",
1697616976
"output_cost_per_token": 1.8e-07,
1697716977
"supports_function_calling": true,
16978-
"supports_response_schema": true,
16978+
"supports_response_schema": false,
1697916979
"supports_tool_choice": true,
1698016980
"supports_vision": true
1698116981
},
@@ -16989,7 +16989,7 @@
1698916989
"mode": "chat",
1699016990
"output_cost_per_token": 4e-08,
1699116991
"supports_function_calling": true,
16992-
"supports_response_schema": true,
16992+
"supports_response_schema": false,
1699316993
"supports_tool_choice": true
1699416994
},
1699516995
"groq/llama-3.2-3b-preview": {
@@ -17002,7 +17002,7 @@
1700217002
"mode": "chat",
1700317003
"output_cost_per_token": 6e-08,
1700417004
"supports_function_calling": true,
17005-
"supports_response_schema": true,
17005+
"supports_response_schema": false,
1700617006
"supports_tool_choice": true
1700717007
},
1700817008
"groq/llama-3.2-90b-text-preview": {
@@ -17015,7 +17015,7 @@
1701517015
"mode": "chat",
1701617016
"output_cost_per_token": 9e-07,
1701717017
"supports_function_calling": true,
17018-
"supports_response_schema": true,
17018+
"supports_response_schema": false,
1701917019
"supports_tool_choice": true
1702017020
},
1702117021
"groq/llama-3.2-90b-vision-preview": {
@@ -17028,7 +17028,7 @@
1702817028
"mode": "chat",
1702917029
"output_cost_per_token": 9e-07,
1703017030
"supports_function_calling": true,
17031-
"supports_response_schema": true,
17031+
"supports_response_schema": false,
1703217032
"supports_tool_choice": true,
1703317033
"supports_vision": true
1703417034
},
@@ -17052,7 +17052,7 @@
1705217052
"mode": "chat",
1705317053
"output_cost_per_token": 7.9e-07,
1705417054
"supports_function_calling": true,
17055-
"supports_response_schema": true,
17055+
"supports_response_schema": false,
1705617056
"supports_tool_choice": true
1705717057
},
1705817058
"groq/llama-guard-3-8b": {
@@ -17073,7 +17073,7 @@
1707317073
"mode": "chat",
1707417074
"output_cost_per_token": 8e-07,
1707517075
"supports_function_calling": true,
17076-
"supports_response_schema": true,
17076+
"supports_response_schema": false,
1707717077
"supports_tool_choice": true
1707817078
},
1707917079
"groq/llama3-groq-70b-8192-tool-use-preview": {
@@ -17086,7 +17086,7 @@
1708617086
"mode": "chat",
1708717087
"output_cost_per_token": 8.9e-07,
1708817088
"supports_function_calling": true,
17089-
"supports_response_schema": true,
17089+
"supports_response_schema": false,
1709017090
"supports_tool_choice": true
1709117091
},
1709217092
"groq/llama3-groq-8b-8192-tool-use-preview": {
@@ -17099,7 +17099,7 @@
1709917099
"mode": "chat",
1710017100
"output_cost_per_token": 1.9e-07,
1710117101
"supports_function_calling": true,
17102-
"supports_response_schema": true,
17102+
"supports_response_schema": false,
1710317103
"supports_tool_choice": true
1710417104
},
1710517105
"groq/meta-llama/llama-4-maverick-17b-128e-instruct": {
@@ -17145,7 +17145,7 @@
1714517145
"mode": "chat",
1714617146
"output_cost_per_token": 2.4e-07,
1714717147
"supports_function_calling": true,
17148-
"supports_response_schema": true,
17148+
"supports_response_schema": false,
1714917149
"supports_tool_choice": true
1715017150
},
1715117151
"groq/moonshotai/kimi-k2-instruct": {
@@ -17221,7 +17221,7 @@
1722117221
"output_cost_per_token": 5.9e-07,
1722217222
"supports_function_calling": true,
1722317223
"supports_reasoning": true,
17224-
"supports_response_schema": true,
17224+
"supports_response_schema": false,
1722517225
"supports_tool_choice": true
1722617226
},
1722717227
"groq/whisper-large-v3": {

0 commit comments

Comments
 (0)