From 4ac81d49aa0bdee57bccd7e0be600a019f553fab Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Wed, 3 Dec 2025 17:35:12 +0000 Subject: [PATCH 01/20] add content filtering exceptions --- pydantic_ai_slim/pydantic_ai/exceptions.py | 25 +++++++ .../pydantic_ai/models/anthropic.py | 13 +++- pydantic_ai_slim/pydantic_ai/models/google.py | 16 +++-- pydantic_ai_slim/pydantic_ai/models/openai.py | 72 ++++++++++++++++++- tests/models/test_anthropic.py | 57 ++++++++++++++- tests/models/test_google.py | 60 +++++++++++++++- tests/models/test_openai.py | 53 ++++++++++++++ 7 files changed, 283 insertions(+), 13 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py index 0b4500502c..1d81a6f319 100644 --- a/pydantic_ai_slim/pydantic_ai/exceptions.py +++ b/pydantic_ai_slim/pydantic_ai/exceptions.py @@ -24,6 +24,9 @@ 'UsageLimitExceeded', 'ModelAPIError', 'ModelHTTPError', + 'ContentFilterError', + 'PromptContentFilterError', + 'ResponseContentFilterError', 'IncompleteToolCall', 'FallbackExceptionGroup', ) @@ -179,6 +182,28 @@ def __init__(self, status_code: int, model_name: str, body: object | None = None super().__init__(model_name=model_name, message=message) +class ContentFilterError(ModelAPIError): + """Raised when content filtering is triggered by the model provider.""" + + +class PromptContentFilterError(ContentFilterError, ModelHTTPError): + """Raised when the prompt triggers a content filter.""" + + def __init__(self, status_code: int, model_name: str, body: object | None = None): + self.status_code = status_code + self.body = body + message = f'Prompt content filtered, status_code: {status_code}, model_name: {model_name}' + ModelAPIError.__init__(self, model_name, message) + + +class ResponseContentFilterError(ContentFilterError): + """Raised when the generated response triggers a content filter.""" + + def __init__(self, message: str, model_name: str, body: object | None = None): + self.body = body + super().__init__(model_name, message) + + class FallbackExceptionGroup(ExceptionGroup[Any]): """A group of exceptions that can be raised when all fallback models fail.""" diff --git a/pydantic_ai_slim/pydantic_ai/models/anthropic.py b/pydantic_ai_slim/pydantic_ai/models/anthropic.py index b23da276e2..627e67392f 100644 --- a/pydantic_ai_slim/pydantic_ai/models/anthropic.py +++ b/pydantic_ai_slim/pydantic_ai/models/anthropic.py @@ -14,7 +14,7 @@ from .._run_context import RunContext from .._utils import guard_tool_call_id as _guard_tool_call_id from ..builtin_tools import CodeExecutionTool, MCPServerTool, MemoryTool, WebFetchTool, WebSearchTool -from ..exceptions import ModelAPIError, UserError +from ..exceptions import ModelAPIError, ResponseContentFilterError, UserError from ..messages import ( BinaryContent, BuiltinToolCallPart, @@ -526,6 +526,12 @@ def _process_response(self, response: BetaMessage) -> ModelResponse: if raw_finish_reason := response.stop_reason: # pragma: no branch provider_details = {'finish_reason': raw_finish_reason} finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) + if finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {response.model}', + model_name=response.model, + body=response.model_dump(), + ) return ModelResponse( parts=items, @@ -1241,6 +1247,11 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if raw_finish_reason := event.delta.stop_reason: # pragma: no branch self.provider_details = {'finish_reason': raw_finish_reason} self.finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) + if self.finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {self.model_name}', + model_name=self.model_name, + ) elif isinstance(event, BetaRawContentBlockStopEvent): # pragma: no branch if isinstance(current_block, BetaMCPToolUseBlock): diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py index 89290ea3ce..5c3625efe1 100644 --- a/pydantic_ai_slim/pydantic_ai/models/google.py +++ b/pydantic_ai_slim/pydantic_ai/models/google.py @@ -14,7 +14,7 @@ from .._output import OutputObjectDefinition from .._run_context import RunContext from ..builtin_tools import CodeExecutionTool, ImageGenerationTool, WebFetchTool, WebSearchTool -from ..exceptions import ModelAPIError, ModelHTTPError, UserError +from ..exceptions import ModelAPIError, ModelHTTPError, ResponseContentFilterError, UserError from ..messages import ( BinaryContent, BuiltinToolCallPart, @@ -495,8 +495,10 @@ def _process_response(self, response: GenerateContentResponse) -> ModelResponse: if candidate.content is None or candidate.content.parts is None: if finish_reason == 'content_filter' and raw_finish_reason: - raise UnexpectedModelBehavior( - f'Content filter {raw_finish_reason.value!r} triggered', response.model_dump_json() + raise ResponseContentFilterError( + f'Content filter {raw_finish_reason.value!r} triggered', + model_name=response.model_version or self._model_name, + body=response.model_dump_json(), ) parts = [] # pragma: no cover else: @@ -697,9 +699,11 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: yield self._parts_manager.handle_part(vendor_part_id=uuid4(), part=web_fetch_return) if candidate.content is None or candidate.content.parts is None: - if self.finish_reason == 'content_filter' and raw_finish_reason: # pragma: no cover - raise UnexpectedModelBehavior( - f'Content filter {raw_finish_reason.value!r} triggered', chunk.model_dump_json() + if self.finish_reason == 'content_filter' and raw_finish_reason: + raise ResponseContentFilterError( + f'Content filter {raw_finish_reason.value!r} triggered', + model_name=self.model_name, + body=chunk.model_dump_json(), ) else: # pragma: no cover continue diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index 3c5c184a76..44a2bb7c0b 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -20,7 +20,7 @@ from .._thinking_part import split_content_into_text_and_thinking from .._utils import guard_tool_call_id as _guard_tool_call_id, now_utc as _now_utc, number_to_datetime from ..builtin_tools import CodeExecutionTool, ImageGenerationTool, MCPServerTool, WebSearchTool -from ..exceptions import UserError +from ..exceptions import PromptContentFilterError, ResponseContentFilterError, UserError from ..messages import ( AudioUrl, BinaryContent, @@ -552,7 +552,27 @@ async def _completions_create( ) except APIStatusError as e: if (status_code := e.status_code) >= 400: - raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e + # Handle Azure Prompt Filter + err_body: Any = e.body + + if status_code == 400 and isinstance(err_body, dict): + err_dict = cast(dict[str, Any], err_body) + error = err_dict.get('error') + + if isinstance(error, dict): + error_dict = cast(dict[str, Any], error) + if error_dict.get('code') == 'content_filter': + raise PromptContentFilterError( + status_code=status_code, + model_name=self.model_name, + body=cast(dict[str, Any], err_body), + ) from e + + raise ModelHTTPError( + status_code=status_code, + model_name=self.model_name, + body=cast(dict[str, Any], err_body), + ) from e raise # pragma: lax no cover except APIConnectionError as e: raise ModelAPIError(model_name=self.model_name, message=e.message) from e @@ -598,6 +618,14 @@ def _process_response(self, response: chat.ChatCompletion | str) -> ModelRespons raise UnexpectedModelBehavior(f'Invalid response from {self.system} chat completions endpoint: {e}') from e choice = response.choices[0] + + if choice.finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {response.model}', + model_name=response.model, + body=response.model_dump(), + ) + items: list[ModelResponsePart] = [] if thinking_parts := self._process_thinking(choice.message): @@ -1234,6 +1262,12 @@ def _process_response( # noqa: C901 finish_reason: FinishReason | None = None provider_details: dict[str, Any] | None = None raw_finish_reason = details.reason if (details := response.incomplete_details) else response.status + if raw_finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {response.model}', + model_name=response.model, + body=response.model_dump(), + ) if raw_finish_reason: provider_details = {'finish_reason': raw_finish_reason} finish_reason = _RESPONSES_FINISH_REASON_MAP.get(raw_finish_reason) @@ -1390,7 +1424,27 @@ async def _responses_create( # noqa: C901 ) except APIStatusError as e: if (status_code := e.status_code) >= 400: - raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e + # Handle Azure Prompt Filter + err_body: Any = e.body + + if status_code == 400 and isinstance(err_body, dict): + err_dict = cast(dict[str, Any], err_body) + error = err_dict.get('error') + + if isinstance(error, dict): + error_dict = cast(dict[str, Any], error) + if error_dict.get('code') == 'content_filter': + raise PromptContentFilterError( + status_code=status_code, + model_name=self.model_name, + body=cast(dict[str, Any], err_body), + ) from e + + raise ModelHTTPError( + status_code=status_code, + model_name=self.model_name, + body=cast(dict[str, Any], err_body), + ) from e raise # pragma: lax no cover except APIConnectionError as e: raise ModelAPIError(model_name=self.model_name, message=e.message) from e @@ -1875,6 +1929,11 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: continue if raw_finish_reason := choice.finish_reason: + if raw_finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {self.model_name}', + model_name=self.model_name, + ) self.finish_reason = self._map_finish_reason(raw_finish_reason) if provider_details := self._map_provider_details(chunk): @@ -2020,6 +2079,13 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: raw_finish_reason = ( details.reason if (details := chunk.response.incomplete_details) else chunk.response.status ) + + if raw_finish_reason == 'content_filter': + raise ResponseContentFilterError( + f'Content filter triggered for model {self.model_name}', + model_name=self.model_name, + ) + if raw_finish_reason: # pragma: no branch self.provider_details = {'finish_reason': raw_finish_reason} self.finish_reason = _RESPONSES_FINISH_REASON_MAP.get(raw_finish_reason) diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index ad65735d38..2bc99c12cc 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -46,7 +46,7 @@ UserPromptPart, ) from pydantic_ai.builtin_tools import CodeExecutionTool, MCPServerTool, MemoryTool, WebFetchTool, WebSearchTool -from pydantic_ai.exceptions import UserError +from pydantic_ai.exceptions import ResponseContentFilterError, UserError from pydantic_ai.messages import ( BuiltinToolCallEvent, # pyright: ignore[reportDeprecated] BuiltinToolResultEvent, # pyright: ignore[reportDeprecated] @@ -7858,3 +7858,58 @@ async def test_anthropic_cache_messages_real_api(allow_model_requests: None, ant assert usage2.cache_read_tokens > 0 assert usage2.cache_write_tokens > 0 assert usage2.output_tokens > 0 + + +async def test_anthropic_response_filter_error_sync(allow_model_requests: None): + c = completion_message( + [BetaTextBlock(text='partial', type='text')], + usage=BetaUsage(input_tokens=5, output_tokens=10), + ) + # 'refusal' maps to 'content_filter' in _FINISH_REASON_MAP + c.stop_reason = 'refusal' + + mock_client = MockAnthropic.create_mock(c) + m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ResponseContentFilterError) as exc_info: + await agent.run('hello') + + # The mock completion_message uses this model name hardcoded + assert exc_info.value.model_name == 'claude-3-5-haiku-123' + assert 'Content filter triggered' in str(exc_info.value) + + +async def test_anthropic_response_filter_error_stream(allow_model_requests: None): + stream = [ + BetaRawMessageStartEvent( + type='message_start', + message=BetaMessage( + id='msg_123', + model='claude-3-5-haiku-123', + role='assistant', + type='message', + content=[], + stop_reason=None, + usage=BetaUsage(input_tokens=20, output_tokens=0), + ), + ), + BetaRawMessageDeltaEvent( + type='message_delta', + delta=Delta(stop_reason='refusal'), # maps to content_filter + usage=BetaMessageDeltaUsage(input_tokens=20, output_tokens=5), + ), + BetaRawMessageStopEvent(type='message_stop'), + ] + + mock_client = MockAnthropic.create_stream_mock([stream]) + m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ResponseContentFilterError) as exc_info: + async with agent.run_stream('hello') as result: + async for _ in result.stream_text(): + pass + + assert exc_info.value.model_name == 'claude-3-5-haiku-123' + assert 'Content filter triggered' in str(exc_info.value) diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 3ef8cd5dda..3e40f822b5 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -51,7 +51,13 @@ WebFetchTool, WebSearchTool, ) -from pydantic_ai.exceptions import ModelAPIError, ModelHTTPError, ModelRetry, UnexpectedModelBehavior, UserError +from pydantic_ai.exceptions import ( + ModelAPIError, + ModelHTTPError, + ModelRetry, + ResponseContentFilterError, + UserError, +) from pydantic_ai.messages import ( BuiltinToolCallEvent, # pyright: ignore[reportDeprecated] BuiltinToolResultEvent, # pyright: ignore[reportDeprecated] @@ -67,6 +73,7 @@ with try_import() as imports_successful: from google.genai import errors from google.genai.types import ( + Candidate, FinishReason as GoogleFinishReason, GenerateContentResponse, GenerateContentResponseUsageMetadata, @@ -982,7 +989,8 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p ) agent = Agent(m, instructions='You hate the world!', model_settings=settings) - with pytest.raises(UnexpectedModelBehavior, match="Content filter 'SAFETY' triggered"): + # Changed expected exception from UnexpectedModelBehavior to ResponseContentFilterError + with pytest.raises(ResponseContentFilterError, match="Content filter 'SAFETY' triggered"): await agent.run('Tell me a joke about a Brazilians.') @@ -4425,3 +4433,51 @@ def test_google_missing_tool_call_thought_signature(): ], } ) + + +async def test_google_response_filter_error_sync( + allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture +): + model = GoogleModel('gemini-1.5-flash', provider=google_provider) + + candidate = Candidate( + finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None + ) + + response = GenerateContentResponse(candidates=[candidate], model_version='gemini-1.5-flash') + mocker.patch.object(model.client.aio.models, 'generate_content', return_value=response) + + agent = Agent(model=model) + + with pytest.raises(ResponseContentFilterError) as exc_info: + await agent.run('bad content') + + assert exc_info.value.model_name == 'gemini-1.5-flash' + assert 'Content filter' in str(exc_info.value) + + +async def test_google_response_filter_error_stream( + allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture +): + model = GoogleModel('gemini-1.5-flash', provider=google_provider) + + candidate = Candidate( + finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None + ) + + chunk = GenerateContentResponse(candidates=[candidate], model_version='gemini-1.5-flash') + + async def stream_iterator(): + yield chunk + + mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) + + agent = Agent(model=model) + + with pytest.raises(ResponseContentFilterError) as exc_info: + async with agent.run_stream('bad content') as result: + async for _ in result.stream_text(): + pass + + assert exc_info.value.model_name == 'gemini-1.5-flash' + assert 'Content filter' in str(exc_info.value) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index ed68edd94f..e82f0ce96e 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -38,6 +38,7 @@ ) from pydantic_ai._json_schema import InlineDefsJsonSchemaTransformer from pydantic_ai.builtin_tools import WebSearchTool +from pydantic_ai.exceptions import PromptContentFilterError, ResponseContentFilterError from pydantic_ai.models import ModelRequestParameters from pydantic_ai.output import NativeOutput, PromptedOutput, TextOutput, ToolOutput from pydantic_ai.profiles.openai import OpenAIModelProfile, openai_model_profile @@ -3296,3 +3297,55 @@ async def test_openai_reasoning_in_thinking_tags(allow_model_requests: None): """, } ) + + +def test_azure_prompt_filter_error(allow_model_requests: None) -> None: + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'content filter', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), + body={'error': {'code': 'content_filter', 'message': 'The content was filtered.'}}, + ) + ) + m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + with pytest.raises(PromptContentFilterError) as exc_info: + agent.run_sync('bad prompt') + + assert exc_info.value.status_code == 400 + assert exc_info.value.model_name == 'gpt-4o' + assert 'Prompt content filtered' in str(exc_info.value) + + +async def test_openai_response_filter_error_sync(allow_model_requests: None): + c = completion_message( + ChatCompletionMessage(content='partial', role='assistant'), + ) + # Simulate content filter finish reason + c.choices[0].finish_reason = 'content_filter' + + mock_client = MockOpenAI.create_mock(c) + m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ResponseContentFilterError) as exc_info: + await agent.run('hello') + + # assertion: matches the mock's default model name + assert exc_info.value.model_name == 'gpt-4o-123' + assert 'Content filter triggered' in str(exc_info.value) + + +async def test_openai_response_filter_error_stream(allow_model_requests: None): + stream = [text_chunk('hello'), text_chunk('', finish_reason='content_filter')] + mock_client = MockOpenAI.create_mock_stream(stream) + m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ResponseContentFilterError) as exc_info: + async with agent.run_stream('hello') as result: + async for _ in result.stream_text(): + pass + + assert exc_info.value.model_name == 'gpt-4o-123' + assert 'Content filter triggered' in str(exc_info.value) From 404a833661e2ee0b9cc66ebe7832eb797036ae7f Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 5 Dec 2025 09:23:18 +0000 Subject: [PATCH 02/20] resolve issues --- pydantic_ai_slim/pydantic_ai/exceptions.py | 20 +++--- .../pydantic_ai/models/anthropic.py | 2 - pydantic_ai_slim/pydantic_ai/models/google.py | 10 +-- pydantic_ai_slim/pydantic_ai/models/openai.py | 71 +++++++------------ tests/models/test_anthropic.py | 12 +--- tests/models/test_google.py | 43 ++++++----- tests/models/test_openai.py | 30 ++++---- 7 files changed, 79 insertions(+), 109 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py index 1d81a6f319..d9d1745d4d 100644 --- a/pydantic_ai_slim/pydantic_ai/exceptions.py +++ b/pydantic_ai_slim/pydantic_ai/exceptions.py @@ -182,26 +182,28 @@ def __init__(self, status_code: int, model_name: str, body: object | None = None super().__init__(model_name=model_name, message=message) -class ContentFilterError(ModelAPIError): +class ContentFilterError(ModelHTTPError): """Raised when content filtering is triggered by the model provider.""" + def __init__(self, message: str, status_code: int, model_name: str, body: object | None = None): + super().__init__(status_code, model_name, body) + self.message = message + -class PromptContentFilterError(ContentFilterError, ModelHTTPError): +class PromptContentFilterError(ContentFilterError): """Raised when the prompt triggers a content filter.""" def __init__(self, status_code: int, model_name: str, body: object | None = None): - self.status_code = status_code - self.body = body - message = f'Prompt content filtered, status_code: {status_code}, model_name: {model_name}' - ModelAPIError.__init__(self, model_name, message) + message = f"Prompt content filtered by model '{model_name}'" + super().__init__(message, status_code, model_name, body) class ResponseContentFilterError(ContentFilterError): """Raised when the generated response triggers a content filter.""" - def __init__(self, message: str, model_name: str, body: object | None = None): - self.body = body - super().__init__(model_name, message) + def __init__(self, model_name: str, body: object | None = None, status_code: int = 200): + message = f"Response content filtered by model '{model_name}'" + super().__init__(message, status_code, model_name, body) class FallbackExceptionGroup(ExceptionGroup[Any]): diff --git a/pydantic_ai_slim/pydantic_ai/models/anthropic.py b/pydantic_ai_slim/pydantic_ai/models/anthropic.py index 627e67392f..fc57c487c0 100644 --- a/pydantic_ai_slim/pydantic_ai/models/anthropic.py +++ b/pydantic_ai_slim/pydantic_ai/models/anthropic.py @@ -528,7 +528,6 @@ def _process_response(self, response: BetaMessage) -> ModelResponse: finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) if finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {response.model}', model_name=response.model, body=response.model_dump(), ) @@ -1249,7 +1248,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: self.finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) if self.finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {self.model_name}', model_name=self.model_name, ) diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py index 5c3625efe1..532772d5ed 100644 --- a/pydantic_ai_slim/pydantic_ai/models/google.py +++ b/pydantic_ai_slim/pydantic_ai/models/google.py @@ -496,9 +496,7 @@ def _process_response(self, response: GenerateContentResponse) -> ModelResponse: if candidate.content is None or candidate.content.parts is None: if finish_reason == 'content_filter' and raw_finish_reason: raise ResponseContentFilterError( - f'Content filter {raw_finish_reason.value!r} triggered', - model_name=response.model_version or self._model_name, - body=response.model_dump_json(), + model_name=response.model_version or self._model_name, body=response.model_dump_json() ) parts = [] # pragma: no cover else: @@ -700,11 +698,7 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if candidate.content is None or candidate.content.parts is None: if self.finish_reason == 'content_filter' and raw_finish_reason: - raise ResponseContentFilterError( - f'Content filter {raw_finish_reason.value!r} triggered', - model_name=self.model_name, - body=chunk.model_dump_json(), - ) + raise ResponseContentFilterError(model_name=self.model_name, body=chunk.model_dump_json()) else: # pragma: no cover continue diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index 44a2bb7c0b..2b204fe161 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -157,6 +157,24 @@ } +def _check_azure_content_filter(e: APIStatusError, model_name: str) -> None: + """Check if the error is an Azure content filter error and raise PromptContentFilterError if so.""" + if e.status_code == 400: + body_any: Any = e.body + + if isinstance(body_any, dict): + body_dict = cast(dict[str, Any], body_any) + + if (error := body_dict.get('error')) and isinstance(error, dict): + error_dict = cast(dict[str, Any], error) + if error_dict.get('code') == 'content_filter': + raise PromptContentFilterError( + status_code=e.status_code, + model_name=model_name, + body=body_dict, + ) from e + + class OpenAIChatModelSettings(ModelSettings, total=False): """Settings used for an OpenAI model request.""" @@ -552,27 +570,9 @@ async def _completions_create( ) except APIStatusError as e: if (status_code := e.status_code) >= 400: - # Handle Azure Prompt Filter - err_body: Any = e.body - - if status_code == 400 and isinstance(err_body, dict): - err_dict = cast(dict[str, Any], err_body) - error = err_dict.get('error') - - if isinstance(error, dict): - error_dict = cast(dict[str, Any], error) - if error_dict.get('code') == 'content_filter': - raise PromptContentFilterError( - status_code=status_code, - model_name=self.model_name, - body=cast(dict[str, Any], err_body), - ) from e - - raise ModelHTTPError( - status_code=status_code, - model_name=self.model_name, - body=cast(dict[str, Any], err_body), - ) from e + _check_azure_content_filter(e, self.model_name) + + raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover except APIConnectionError as e: raise ModelAPIError(model_name=self.model_name, message=e.message) from e @@ -621,7 +621,6 @@ def _process_response(self, response: chat.ChatCompletion | str) -> ModelRespons if choice.finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {response.model}', model_name=response.model, body=response.model_dump(), ) @@ -1264,7 +1263,6 @@ def _process_response( # noqa: C901 raw_finish_reason = details.reason if (details := response.incomplete_details) else response.status if raw_finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {response.model}', model_name=response.model, body=response.model_dump(), ) @@ -1424,27 +1422,10 @@ async def _responses_create( # noqa: C901 ) except APIStatusError as e: if (status_code := e.status_code) >= 400: - # Handle Azure Prompt Filter - err_body: Any = e.body - - if status_code == 400 and isinstance(err_body, dict): - err_dict = cast(dict[str, Any], err_body) - error = err_dict.get('error') - - if isinstance(error, dict): - error_dict = cast(dict[str, Any], error) - if error_dict.get('code') == 'content_filter': - raise PromptContentFilterError( - status_code=status_code, - model_name=self.model_name, - body=cast(dict[str, Any], err_body), - ) from e - - raise ModelHTTPError( - status_code=status_code, - model_name=self.model_name, - body=cast(dict[str, Any], err_body), - ) from e + _check_azure_content_filter(e, self.model_name) + + # Reverted cast + raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover except APIConnectionError as e: raise ModelAPIError(model_name=self.model_name, message=e.message) from e @@ -1931,7 +1912,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if raw_finish_reason := choice.finish_reason: if raw_finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {self.model_name}', model_name=self.model_name, ) self.finish_reason = self._map_finish_reason(raw_finish_reason) @@ -2082,7 +2062,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if raw_finish_reason == 'content_filter': raise ResponseContentFilterError( - f'Content filter triggered for model {self.model_name}', model_name=self.model_name, ) diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index 2bc99c12cc..54a04cf4d3 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -7872,13 +7872,10 @@ async def test_anthropic_response_filter_error_sync(allow_model_requests: None): m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError) as exc_info: + # Mock uses hardcoded 'claude-3-5-haiku-123' + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): await agent.run('hello') - # The mock completion_message uses this model name hardcoded - assert exc_info.value.model_name == 'claude-3-5-haiku-123' - assert 'Content filter triggered' in str(exc_info.value) - async def test_anthropic_response_filter_error_stream(allow_model_requests: None): stream = [ @@ -7906,10 +7903,7 @@ async def test_anthropic_response_filter_error_stream(allow_model_requests: None m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError) as exc_info: + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass - - assert exc_info.value.model_name == 'claude-3-5-haiku-123' - assert 'Content filter triggered' in str(exc_info.value) diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 3e40f822b5..7b3f8e3199 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -73,7 +73,6 @@ with try_import() as imports_successful: from google.genai import errors from google.genai.types import ( - Candidate, FinishReason as GoogleFinishReason, GenerateContentResponse, GenerateContentResponseUsageMetadata, @@ -990,7 +989,7 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p agent = Agent(m, instructions='You hate the world!', model_settings=settings) # Changed expected exception from UnexpectedModelBehavior to ResponseContentFilterError - with pytest.raises(ResponseContentFilterError, match="Content filter 'SAFETY' triggered"): + with pytest.raises(ResponseContentFilterError, match="Response content filtered by model 'gemini-1.5-flash'"): await agent.run('Tell me a joke about a Brazilians.') @@ -4438,46 +4437,54 @@ def test_google_missing_tool_call_thought_signature(): async def test_google_response_filter_error_sync( allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture ): - model = GoogleModel('gemini-1.5-flash', provider=google_provider) + model_name = 'gemini-2.5-flash' + model = GoogleModel(model_name, provider=google_provider) - candidate = Candidate( + # Create a Candidate mock with the specific failure condition + candidate_mock = mocker.Mock( finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None ) - response = GenerateContentResponse(candidates=[candidate], model_version='gemini-1.5-flash') - mocker.patch.object(model.client.aio.models, 'generate_content', return_value=response) + # Create the Response mock containing the candidate + response_mock = mocker.Mock(candidates=[candidate_mock], model_version=model_name, usage_metadata=None) + + response_mock.model_dump_json.return_value = '{"mock": "json"}' + + # Patch the client + mocker.patch.object(model.client.aio.models, 'generate_content', return_value=response_mock) agent = Agent(model=model) - with pytest.raises(ResponseContentFilterError) as exc_info: + # Verify the exception is raised + with pytest.raises(ResponseContentFilterError, match=f"Response content filtered by model '{model_name}'"): await agent.run('bad content') - assert exc_info.value.model_name == 'gemini-1.5-flash' - assert 'Content filter' in str(exc_info.value) - async def test_google_response_filter_error_stream( allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture ): - model = GoogleModel('gemini-1.5-flash', provider=google_provider) + model_name = 'gemini-2.5-flash' + model = GoogleModel(model_name, provider=google_provider) - candidate = Candidate( + # Create Candidate mock + candidate_mock = mocker.Mock( finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None ) - chunk = GenerateContentResponse(candidates=[candidate], model_version='gemini-1.5-flash') + # Create Chunk mock + chunk_mock = mocker.Mock( + candidates=[candidate_mock], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() + ) + chunk_mock.model_dump_json.return_value = '{"mock": "json"}' async def stream_iterator(): - yield chunk + yield chunk_mock mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) agent = Agent(model=model) - with pytest.raises(ResponseContentFilterError) as exc_info: + with pytest.raises(ResponseContentFilterError, match=f"Response content filtered by model '{model_name}'"): async with agent.run_stream('bad content') as result: async for _ in result.stream_text(): pass - - assert exc_info.value.model_name == 'gemini-1.5-flash' - assert 'Content filter' in str(exc_info.value) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index e82f0ce96e..ca41ff19ad 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3307,14 +3307,13 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: body={'error': {'code': 'content_filter', 'message': 'The content was filtered.'}}, ) ) - m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(PromptContentFilterError) as exc_info: - agent.run_sync('bad prompt') - assert exc_info.value.status_code == 400 - assert exc_info.value.model_name == 'gpt-4o' - assert 'Prompt content filtered' in str(exc_info.value) + # Asserting the full error message structure via match + with pytest.raises(PromptContentFilterError, match=r"Prompt content filtered by model 'gpt-5-mini'"): + agent.run_sync('bad prompt') async def test_openai_response_filter_error_sync(allow_model_requests: None): @@ -3325,27 +3324,24 @@ async def test_openai_response_filter_error_sync(allow_model_requests: None): c.choices[0].finish_reason = 'content_filter' mock_client = MockOpenAI.create_mock(c) - m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError) as exc_info: + # The mock message uses 'gpt-4o-123' by default + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-4o-123'"): await agent.run('hello') - # assertion: matches the mock's default model name - assert exc_info.value.model_name == 'gpt-4o-123' - assert 'Content filter triggered' in str(exc_info.value) - async def test_openai_response_filter_error_stream(allow_model_requests: None): stream = [text_chunk('hello'), text_chunk('', finish_reason='content_filter')] mock_client = MockOpenAI.create_mock_stream(stream) - m = OpenAIChatModel('gpt-4o', provider=OpenAIProvider(openai_client=mock_client)) + + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError) as exc_info: + # The mock chunks use 'gpt-4o-123' by default + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-4o-123'"): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass - - assert exc_info.value.model_name == 'gpt-4o-123' - assert 'Content filter triggered' in str(exc_info.value) From 3771c799afbdeaf769741ed9fd71bc19fbf122dc Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 5 Dec 2025 10:00:05 +0000 Subject: [PATCH 03/20] add docs --- docs/models/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/models/overview.md b/docs/models/overview.md index a568a20161..131caa5679 100644 --- a/docs/models/overview.md +++ b/docs/models/overview.md @@ -229,7 +229,7 @@ contains all the exceptions encountered during the `run` execution. By default, the `FallbackModel` only moves on to the next model if the current model raises a [`ModelAPIError`][pydantic_ai.exceptions.ModelAPIError], which includes -[`ModelHTTPError`][pydantic_ai.exceptions.ModelHTTPError]. You can customize this behavior by +[`ModelHTTPError`][pydantic_ai.exceptions.ModelHTTPError] and [`ContentFilterError`][pydantic_ai.exceptions.ContentFilterError]. You can customize this behavior by passing a custom `fallback_on` argument to the `FallbackModel` constructor. !!! note From 70bcb7455a97cf8753f3d451682a3c0f400727a1 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 5 Dec 2025 10:29:40 +0000 Subject: [PATCH 04/20] test coverage --- tests/models/test_openai.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index ca41ff19ad..85be0d1961 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3299,6 +3299,23 @@ async def test_openai_reasoning_in_thinking_tags(allow_model_requests: None): ) +def test_openai_generic_400_error(allow_model_requests: None) -> None: + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'bad request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), + body={'error': {'code': 'invalid_parameter', 'message': 'Invalid param.'}}, + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + assert not isinstance(exc_info.value, PromptContentFilterError) + assert exc_info.value.status_code == 400 + + def test_azure_prompt_filter_error(allow_model_requests: None) -> None: mock_client = MockOpenAI.create_mock( APIStatusError( From e861a508a20064c2950693393a44ed2bdfb3cf8c Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 5 Dec 2025 11:04:47 +0000 Subject: [PATCH 05/20] improve tests --- tests/models/test_anthropic.py | 5 ++--- tests/models/test_openai.py | 11 +++++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index 926d15d314..fbecd3dd88 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -7912,10 +7912,9 @@ async def test_anthropic_response_filter_error_sync(allow_model_requests: None): c.stop_reason = 'refusal' mock_client = MockAnthropic.create_mock(c) - m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) + m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - # Mock uses hardcoded 'claude-3-5-haiku-123' with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): await agent.run('hello') @@ -7943,7 +7942,7 @@ async def test_anthropic_response_filter_error_stream(allow_model_requests: None ] mock_client = MockAnthropic.create_stream_mock([stream]) - m = AnthropicModel('claude-haiku-4-5', provider=AnthropicProvider(anthropic_client=mock_client)) + m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 85be0d1961..829c52f645 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3339,26 +3339,29 @@ async def test_openai_response_filter_error_sync(allow_model_requests: None): ) # Simulate content filter finish reason c.choices[0].finish_reason = 'content_filter' + c.model = 'gpt-5-mini' mock_client = MockOpenAI.create_mock(c) m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - # The mock message uses 'gpt-4o-123' by default - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-4o-123'"): + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-5-mini'"): await agent.run('hello') async def test_openai_response_filter_error_stream(allow_model_requests: None): stream = [text_chunk('hello'), text_chunk('', finish_reason='content_filter')] + + for chunk in stream: + chunk.model = 'gpt-5-mini' + mock_client = MockOpenAI.create_mock_stream(stream) m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - # The mock chunks use 'gpt-4o-123' by default - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-4o-123'"): + with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-5-mini'"): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass From 51506b2a6a52e486d8bf46762d8a50dd575644bc Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 6 Dec 2025 05:13:17 +0000 Subject: [PATCH 06/20] resolve issues --- pydantic_ai_slim/pydantic_ai/exceptions.py | 4 +- pydantic_ai_slim/pydantic_ai/models/openai.py | 1 - tests/models/test_anthropic.py | 10 +- tests/models/test_google.py | 15 ++- tests/models/test_openai.py | 108 ++++++++++++++---- 5 files changed, 109 insertions(+), 29 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py index d9d1745d4d..c91c967c46 100644 --- a/pydantic_ai_slim/pydantic_ai/exceptions.py +++ b/pydantic_ai_slim/pydantic_ai/exceptions.py @@ -194,7 +194,7 @@ class PromptContentFilterError(ContentFilterError): """Raised when the prompt triggers a content filter.""" def __init__(self, status_code: int, model_name: str, body: object | None = None): - message = f"Prompt content filtered by model '{model_name}'" + message = f"Model '{model_name}' content filter was triggered by the user's prompt" super().__init__(message, status_code, model_name, body) @@ -202,7 +202,7 @@ class ResponseContentFilterError(ContentFilterError): """Raised when the generated response triggers a content filter.""" def __init__(self, model_name: str, body: object | None = None, status_code: int = 200): - message = f"Response content filtered by model '{model_name}'" + message = f"Model '{model_name}' triggered its content filter while generating a response" super().__init__(message, status_code, model_name, body) diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index d0b3c82d93..f12d93a1ab 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -1432,7 +1432,6 @@ async def _responses_create( # noqa: C901 if (status_code := e.status_code) >= 400: _check_azure_content_filter(e, self.model_name) - # Reverted cast raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover except APIConnectionError as e: diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index fbecd3dd88..ca900ab62a 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -7915,7 +7915,10 @@ async def test_anthropic_response_filter_error_sync(allow_model_requests: None): m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): + with pytest.raises( + ResponseContentFilterError, + match=r"Model 'claude-3-5-haiku-123' triggered its content filter while generating a response", + ): await agent.run('hello') @@ -7945,7 +7948,10 @@ async def test_anthropic_response_filter_error_stream(allow_model_requests: None m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'claude-3-5-haiku-123'"): + with pytest.raises( + ResponseContentFilterError, + match=r"Model 'claude-3-5-haiku-123' triggered its content filter while generating a response", + ): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 7b3f8e3199..38ec043c20 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -989,7 +989,10 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p agent = Agent(m, instructions='You hate the world!', model_settings=settings) # Changed expected exception from UnexpectedModelBehavior to ResponseContentFilterError - with pytest.raises(ResponseContentFilterError, match="Response content filtered by model 'gemini-1.5-flash'"): + with pytest.raises( + ResponseContentFilterError, + match="Model 'gemini-1.5-flash' triggered its content filter while generating a response", + ): await agent.run('Tell me a joke about a Brazilians.') @@ -4456,7 +4459,10 @@ async def test_google_response_filter_error_sync( agent = Agent(model=model) # Verify the exception is raised - with pytest.raises(ResponseContentFilterError, match=f"Response content filtered by model '{model_name}'"): + with pytest.raises( + ResponseContentFilterError, + match="Model 'gemini-2.5-flash' triggered its content filter while generating a response", + ): await agent.run('bad content') @@ -4484,7 +4490,10 @@ async def stream_iterator(): agent = Agent(model=model) - with pytest.raises(ResponseContentFilterError, match=f"Response content filtered by model '{model_name}'"): + with pytest.raises( + ResponseContentFilterError, + match="Model 'gemini-2.5-flash' triggered its content filter while generating a response", + ): async with agent.run_stream('bad content') as result: async for _ in result.stream_text(): pass diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 829c52f645..8f49509e78 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -11,6 +11,7 @@ import pytest from inline_snapshot import snapshot from pydantic import AnyUrl, BaseModel, ConfigDict, Discriminator, Field, Tag +from pytest_mock import MockerFixture from typing_extensions import NotRequired, TypedDict from pydantic_ai import ( @@ -72,6 +73,7 @@ from openai.types.chat.chat_completion_message_tool_call import Function from openai.types.chat.chat_completion_token_logprob import ChatCompletionTokenLogprob from openai.types.completion_usage import CompletionUsage, PromptTokensDetails + from openai.types.responses import Response, ResponseCompletedEvent, ResponseCreatedEvent from pydantic_ai.models.google import GoogleModel from pydantic_ai.models.openai import ( @@ -3299,23 +3301,6 @@ async def test_openai_reasoning_in_thinking_tags(allow_model_requests: None): ) -def test_openai_generic_400_error(allow_model_requests: None) -> None: - mock_client = MockOpenAI.create_mock( - APIStatusError( - 'bad request', - response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), - body={'error': {'code': 'invalid_parameter', 'message': 'Invalid param.'}}, - ) - ) - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) - agent = Agent(m) - with pytest.raises(ModelHTTPError) as exc_info: - agent.run_sync('hello') - - assert not isinstance(exc_info.value, PromptContentFilterError) - assert exc_info.value.status_code == 400 - - def test_azure_prompt_filter_error(allow_model_requests: None) -> None: mock_client = MockOpenAI.create_mock( APIStatusError( @@ -3328,8 +3313,9 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - # Asserting the full error message structure via match - with pytest.raises(PromptContentFilterError, match=r"Prompt content filtered by model 'gpt-5-mini'"): + with pytest.raises( + PromptContentFilterError, match=r"Model 'gpt-5-mini' content filter was triggered by the user's prompt" + ): agent.run_sync('bad prompt') @@ -3346,7 +3332,9 @@ async def test_openai_response_filter_error_sync(allow_model_requests: None): m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-5-mini'"): + with pytest.raises( + ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" + ): await agent.run('hello') @@ -3361,7 +3349,85 @@ async def test_openai_response_filter_error_stream(allow_model_requests: None): m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ResponseContentFilterError, match=r"Response content filtered by model 'gpt-5-mini'"): + with pytest.raises( + ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" + ): + async with agent.run_stream('hello') as result: + async for _ in result.stream_text(): + pass + + +def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None: + """Test ResponsesModel (Azure) prompt filter.""" + mock_client = MockOpenAIResponses.create_mock( + APIStatusError( + 'content filter', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), + body={'error': {'code': 'content_filter', 'message': 'The content was filtered.'}}, + ) + ) + m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises( + PromptContentFilterError, match=r"Model 'gpt-5-mini' content filter was triggered by the user's prompt" + ): + agent.run_sync('bad prompt') + + +async def test_responses_response_filter_error_sync(allow_model_requests: None, mocker: MockerFixture): + """Test ResponsesModel sync response filter.""" + mock_openai_client = mocker.Mock() + m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_openai_client)) + agent = Agent(m) + + mock_response = mocker.Mock() + mock_response.model = 'gpt-5-mini' + mock_response.model_dump.return_value = {} + mock_response.created_at = 1234567890 + mock_response.id = 'resp_123' + mock_response.incomplete_details.reason = 'content_filter' + mock_response.output = [] + + mock_openai_client.responses.create = mocker.AsyncMock(return_value=mock_response) + + with pytest.raises( + ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" + ): + await agent.run('hello') + + +async def test_responses_response_filter_error_stream(allow_model_requests: None, mocker: MockerFixture): + """Test ResponsesModel stream response filter.""" + mock_openai_client = mocker.Mock() + m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_openai_client)) + agent = Agent(m) + + resp = Response( + id='resp_123', + model='gpt-5-mini', + created_at=1234567890, + object='response', + status='incomplete', + incomplete_details=cast(Any, {'reason': 'content_filter'}), + output=[], + parallel_tool_calls=False, + tool_choice='none', + tools=[], + ) + + stream = [ + ResponseCreatedEvent(response=resp, type='response.created', sequence_number=0), + ResponseCompletedEvent(response=resp, type='response.completed', sequence_number=1), + ] + + mock_client = MockOpenAIResponses.create_mock_stream(stream) + m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises( + ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" + ): async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass From b67d216bc2a3dd12b5d1d13c09436226c3495df0 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 6 Dec 2025 05:43:48 +0000 Subject: [PATCH 07/20] test coverage --- tests/models/test_openai.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 8f49509e78..dd9c7556b5 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3431,3 +3431,42 @@ async def test_responses_response_filter_error_stream(allow_model_requests: None async with agent.run_stream('hello') as result: async for _ in result.stream_text(): pass + + +def test_openai_400_non_content_filter(allow_model_requests: None) -> None: + """Test a standard 400 error (not content filter) raises ModelHTTPError.""" + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'Bad Request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), + body={'error': {'code': 'invalid_parameter', 'message': 'Invalid parameter'}}, + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + assert exc_info.value.status_code == 400 + + assert not isinstance(exc_info.value, PromptContentFilterError) + + +def test_openai_400_non_dict_body(allow_model_requests: None) -> None: + """Test a 400 error with a non-dict body raises ModelHTTPError.""" + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'Bad Request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), + body='Raw string body', + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + assert exc_info.value.status_code == 400 + assert not isinstance(exc_info.value, PromptContentFilterError) From 4ac16089d9aff6b34f6edc5dc567812449b53af6 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 6 Dec 2025 06:06:38 +0000 Subject: [PATCH 08/20] fix ci fails --- pydantic_ai_slim/pydantic_ai/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_ai_slim/pydantic_ai/_utils.py b/pydantic_ai_slim/pydantic_ai/_utils.py index 97230bb046..d0fc819e73 100644 --- a/pydantic_ai_slim/pydantic_ai/_utils.py +++ b/pydantic_ai_slim/pydantic_ai/_utils.py @@ -215,7 +215,7 @@ async def async_iter_groups() -> AsyncIterator[list[T]]: try: yield async_iter_groups() - finally: # pragma: no cover + finally: # after iteration if a tasks still exists, cancel it, this will only happen if an error occurred if task: task.cancel('Cancelling due to error in iterator') From 264f9e6213cd83db2c5080be7be5690b2cbf2a3f Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Thu, 11 Dec 2025 17:30:43 +0000 Subject: [PATCH 09/20] make consistent for all models --- pydantic_ai_slim/pydantic_ai/_agent_graph.py | 3 + pydantic_ai_slim/pydantic_ai/exceptions.py | 30 +--- .../pydantic_ai/models/anthropic.py | 11 +- pydantic_ai_slim/pydantic_ai/models/google.py | 11 +- pydantic_ai_slim/pydantic_ai/models/openai.py | 65 ++++---- tests/models/test_anthropic.py | 56 +------ tests/models/test_google.py | 69 +------- tests/models/test_model_function.py | 21 +++ tests/models/test_openai.py | 148 +----------------- 9 files changed, 70 insertions(+), 344 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/_agent_graph.py b/pydantic_ai_slim/pydantic_ai/_agent_graph.py index 043c27c4f5..924747f678 100644 --- a/pydantic_ai_slim/pydantic_ai/_agent_graph.py +++ b/pydantic_ai_slim/pydantic_ai/_agent_graph.py @@ -534,6 +534,9 @@ def _finish_handling( ctx: GraphRunContext[GraphAgentState, GraphAgentDeps[DepsT, NodeRunEndT]], response: _messages.ModelResponse, ) -> CallToolsNode[DepsT, NodeRunEndT]: + if response.finish_reason == 'content_filter': + raise exceptions.ContentFilterError(f'Content filter triggered for model {response.model_name}') + response.run_id = response.run_id or ctx.state.run_id # Update usage ctx.state.usage.incr(response.usage) diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py index c91c967c46..a9cabb8b74 100644 --- a/pydantic_ai_slim/pydantic_ai/exceptions.py +++ b/pydantic_ai_slim/pydantic_ai/exceptions.py @@ -25,8 +25,6 @@ 'ModelAPIError', 'ModelHTTPError', 'ContentFilterError', - 'PromptContentFilterError', - 'ResponseContentFilterError', 'IncompleteToolCall', 'FallbackExceptionGroup', ) @@ -155,6 +153,10 @@ def __str__(self) -> str: return self.message +class ContentFilterError(UnexpectedModelBehavior): + """Raised when content filtering is triggered by the model provider.""" + + class ModelAPIError(AgentRunError): """Raised when a model provider API request fails.""" @@ -182,30 +184,6 @@ def __init__(self, status_code: int, model_name: str, body: object | None = None super().__init__(model_name=model_name, message=message) -class ContentFilterError(ModelHTTPError): - """Raised when content filtering is triggered by the model provider.""" - - def __init__(self, message: str, status_code: int, model_name: str, body: object | None = None): - super().__init__(status_code, model_name, body) - self.message = message - - -class PromptContentFilterError(ContentFilterError): - """Raised when the prompt triggers a content filter.""" - - def __init__(self, status_code: int, model_name: str, body: object | None = None): - message = f"Model '{model_name}' content filter was triggered by the user's prompt" - super().__init__(message, status_code, model_name, body) - - -class ResponseContentFilterError(ContentFilterError): - """Raised when the generated response triggers a content filter.""" - - def __init__(self, model_name: str, body: object | None = None, status_code: int = 200): - message = f"Model '{model_name}' triggered its content filter while generating a response" - super().__init__(message, status_code, model_name, body) - - class FallbackExceptionGroup(ExceptionGroup[Any]): """A group of exceptions that can be raised when all fallback models fail.""" diff --git a/pydantic_ai_slim/pydantic_ai/models/anthropic.py b/pydantic_ai_slim/pydantic_ai/models/anthropic.py index 07ca07b9e3..28395f56bd 100644 --- a/pydantic_ai_slim/pydantic_ai/models/anthropic.py +++ b/pydantic_ai_slim/pydantic_ai/models/anthropic.py @@ -14,7 +14,7 @@ from .._run_context import RunContext from .._utils import guard_tool_call_id as _guard_tool_call_id from ..builtin_tools import CodeExecutionTool, MCPServerTool, MemoryTool, WebFetchTool, WebSearchTool -from ..exceptions import ModelAPIError, ResponseContentFilterError, UserError +from ..exceptions import ModelAPIError, UserError from ..messages import ( BinaryContent, BuiltinToolCallPart, @@ -526,11 +526,6 @@ def _process_response(self, response: BetaMessage) -> ModelResponse: if raw_finish_reason := response.stop_reason: # pragma: no branch provider_details = {'finish_reason': raw_finish_reason} finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) - if finish_reason == 'content_filter': - raise ResponseContentFilterError( - model_name=response.model, - body=response.model_dump(), - ) return ModelResponse( parts=items, @@ -1248,10 +1243,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: if raw_finish_reason := event.delta.stop_reason: # pragma: no branch self.provider_details = {'finish_reason': raw_finish_reason} self.finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) - if self.finish_reason == 'content_filter': - raise ResponseContentFilterError( - model_name=self.model_name, - ) elif isinstance(event, BetaRawContentBlockStopEvent): # pragma: no branch if isinstance(current_block, BetaMCPToolUseBlock): diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py index 5228064ddb..0b993b4c33 100644 --- a/pydantic_ai_slim/pydantic_ai/models/google.py +++ b/pydantic_ai_slim/pydantic_ai/models/google.py @@ -14,7 +14,7 @@ from .._output import OutputObjectDefinition from .._run_context import RunContext from ..builtin_tools import CodeExecutionTool, ImageGenerationTool, WebFetchTool, WebSearchTool -from ..exceptions import ModelAPIError, ModelHTTPError, ResponseContentFilterError, UserError +from ..exceptions import ModelAPIError, ModelHTTPError, UserError from ..messages import ( BinaryContent, BuiltinToolCallPart, @@ -494,10 +494,6 @@ def _process_response(self, response: GenerateContentResponse) -> ModelResponse: finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) if candidate.content is None or candidate.content.parts is None: - if finish_reason == 'content_filter' and raw_finish_reason: - raise ResponseContentFilterError( - model_name=response.model_version or self._model_name, body=response.model_dump_json() - ) parts = [] # pragma: no cover else: parts = candidate.content.parts or [] @@ -697,10 +693,7 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: yield self._parts_manager.handle_part(vendor_part_id=uuid4(), part=web_fetch_return) if candidate.content is None or candidate.content.parts is None: - if self.finish_reason == 'content_filter' and raw_finish_reason: - raise ResponseContentFilterError(model_name=self.model_name, body=chunk.model_dump_json()) - else: # pragma: no cover - continue + continue parts = candidate.content.parts if not parts: diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index f12d93a1ab..8ee3de75a0 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -20,7 +20,7 @@ from .._thinking_part import split_content_into_text_and_thinking from .._utils import guard_tool_call_id as _guard_tool_call_id, now_utc as _now_utc, number_to_datetime from ..builtin_tools import CodeExecutionTool, ImageGenerationTool, MCPServerTool, WebSearchTool -from ..exceptions import PromptContentFilterError, ResponseContentFilterError, UserError +from ..exceptions import UserError from ..messages import ( AudioUrl, BinaryContent, @@ -160,8 +160,8 @@ } -def _check_azure_content_filter(e: APIStatusError, model_name: str) -> None: - """Check if the error is an Azure content filter error and raise PromptContentFilterError if so.""" +def _check_azure_content_filter(e: APIStatusError) -> bool: + """Check if the error is an Azure content filter error.""" if e.status_code == 400: body_any: Any = e.body @@ -170,12 +170,8 @@ def _check_azure_content_filter(e: APIStatusError, model_name: str) -> None: if (error := body_dict.get('error')) and isinstance(error, dict): error_dict = cast(dict[str, Any], error) - if error_dict.get('code') == 'content_filter': - raise PromptContentFilterError( - status_code=e.status_code, - model_name=model_name, - body=body_dict, - ) from e + return error_dict.get('code') == 'content_filter' + return False class OpenAIChatModelSettings(ModelSettings, total=False): @@ -572,9 +568,21 @@ async def _completions_create( extra_body=model_settings.get('extra_body'), ) except APIStatusError as e: + if _check_azure_content_filter(e): + return chat.ChatCompletion( + id='content_filter', + choices=[ + chat.chat_completion.Choice( + finish_reason='content_filter', + index=0, + message=chat.ChatCompletionMessage(content='', role='assistant'), + ) + ], + created=0, + model=self.model_name, + object='chat.completion', + ) if (status_code := e.status_code) >= 400: - _check_azure_content_filter(e, self.model_name) - raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover except APIConnectionError as e: @@ -622,12 +630,6 @@ def _process_response(self, response: chat.ChatCompletion | str) -> ModelRespons choice = response.choices[0] - if choice.finish_reason == 'content_filter': - raise ResponseContentFilterError( - model_name=response.model, - body=response.model_dump(), - ) - items: list[ModelResponsePart] = [] if thinking_parts := self._process_thinking(choice.message): @@ -1269,11 +1271,6 @@ def _process_response( # noqa: C901 finish_reason: FinishReason | None = None provider_details: dict[str, Any] | None = None raw_finish_reason = details.reason if (details := response.incomplete_details) else response.status - if raw_finish_reason == 'content_filter': - raise ResponseContentFilterError( - model_name=response.model, - body=response.model_dump(), - ) if raw_finish_reason: provider_details = {'finish_reason': raw_finish_reason} finish_reason = _RESPONSES_FINISH_REASON_MAP.get(raw_finish_reason) @@ -1429,9 +1426,20 @@ async def _responses_create( # noqa: C901 extra_body=model_settings.get('extra_body'), ) except APIStatusError as e: + if _check_azure_content_filter(e): + return responses.Response( + id='content_filter', + model=self.model_name, + created_at=0, + object='response', + status='incomplete', + incomplete_details={'reason': 'content_filter'}, # type: ignore + output=[], + parallel_tool_calls=False, + tool_choice='auto', + tools=[], + ) if (status_code := e.status_code) >= 400: - _check_azure_content_filter(e, self.model_name) - raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover except APIConnectionError as e: @@ -1937,10 +1945,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: continue if raw_finish_reason := choice.finish_reason: - if raw_finish_reason == 'content_filter': - raise ResponseContentFilterError( - model_name=self.model_name, - ) self.finish_reason = self._map_finish_reason(raw_finish_reason) if provider_details := self._map_provider_details(chunk): @@ -2086,11 +2090,6 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: details.reason if (details := chunk.response.incomplete_details) else chunk.response.status ) - if raw_finish_reason == 'content_filter': - raise ResponseContentFilterError( - model_name=self.model_name, - ) - if raw_finish_reason: # pragma: no branch self.provider_details = {'finish_reason': raw_finish_reason} self.finish_reason = _RESPONSES_FINISH_REASON_MAP.get(raw_finish_reason) diff --git a/tests/models/test_anthropic.py b/tests/models/test_anthropic.py index ca900ab62a..2df96b0278 100644 --- a/tests/models/test_anthropic.py +++ b/tests/models/test_anthropic.py @@ -46,7 +46,7 @@ UserPromptPart, ) from pydantic_ai.builtin_tools import CodeExecutionTool, MCPServerTool, MemoryTool, WebFetchTool, WebSearchTool -from pydantic_ai.exceptions import ResponseContentFilterError, UserError +from pydantic_ai.exceptions import UserError from pydantic_ai.messages import ( BuiltinToolCallEvent, # pyright: ignore[reportDeprecated] BuiltinToolResultEvent, # pyright: ignore[reportDeprecated] @@ -7901,57 +7901,3 @@ async def test_anthropic_cache_messages_real_api(allow_model_requests: None, ant assert usage2.cache_read_tokens > 0 assert usage2.cache_write_tokens > 0 assert usage2.output_tokens > 0 - - -async def test_anthropic_response_filter_error_sync(allow_model_requests: None): - c = completion_message( - [BetaTextBlock(text='partial', type='text')], - usage=BetaUsage(input_tokens=5, output_tokens=10), - ) - # 'refusal' maps to 'content_filter' in _FINISH_REASON_MAP - c.stop_reason = 'refusal' - - mock_client = MockAnthropic.create_mock(c) - m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) - agent = Agent(m) - - with pytest.raises( - ResponseContentFilterError, - match=r"Model 'claude-3-5-haiku-123' triggered its content filter while generating a response", - ): - await agent.run('hello') - - -async def test_anthropic_response_filter_error_stream(allow_model_requests: None): - stream = [ - BetaRawMessageStartEvent( - type='message_start', - message=BetaMessage( - id='msg_123', - model='claude-3-5-haiku-123', - role='assistant', - type='message', - content=[], - stop_reason=None, - usage=BetaUsage(input_tokens=20, output_tokens=0), - ), - ), - BetaRawMessageDeltaEvent( - type='message_delta', - delta=Delta(stop_reason='refusal'), # maps to content_filter - usage=BetaMessageDeltaUsage(input_tokens=20, output_tokens=5), - ), - BetaRawMessageStopEvent(type='message_stop'), - ] - - mock_client = MockAnthropic.create_stream_mock([stream]) - m = AnthropicModel('claude-3-5-haiku-123', provider=AnthropicProvider(anthropic_client=mock_client)) - agent = Agent(m) - - with pytest.raises( - ResponseContentFilterError, - match=r"Model 'claude-3-5-haiku-123' triggered its content filter while generating a response", - ): - async with agent.run_stream('hello') as result: - async for _ in result.stream_text(): - pass diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 38ec043c20..5194aa639e 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -52,10 +52,10 @@ WebSearchTool, ) from pydantic_ai.exceptions import ( + ContentFilterError, ModelAPIError, ModelHTTPError, ModelRetry, - ResponseContentFilterError, UserError, ) from pydantic_ai.messages import ( @@ -988,10 +988,9 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p ) agent = Agent(m, instructions='You hate the world!', model_settings=settings) - # Changed expected exception from UnexpectedModelBehavior to ResponseContentFilterError with pytest.raises( - ResponseContentFilterError, - match="Model 'gemini-1.5-flash' triggered its content filter while generating a response", + ContentFilterError, + match='Content filter triggered for model gemini-1.5-flash', ): await agent.run('Tell me a joke about a Brazilians.') @@ -4435,65 +4434,3 @@ def test_google_missing_tool_call_thought_signature(): ], } ) - - -async def test_google_response_filter_error_sync( - allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture -): - model_name = 'gemini-2.5-flash' - model = GoogleModel(model_name, provider=google_provider) - - # Create a Candidate mock with the specific failure condition - candidate_mock = mocker.Mock( - finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None - ) - - # Create the Response mock containing the candidate - response_mock = mocker.Mock(candidates=[candidate_mock], model_version=model_name, usage_metadata=None) - - response_mock.model_dump_json.return_value = '{"mock": "json"}' - - # Patch the client - mocker.patch.object(model.client.aio.models, 'generate_content', return_value=response_mock) - - agent = Agent(model=model) - - # Verify the exception is raised - with pytest.raises( - ResponseContentFilterError, - match="Model 'gemini-2.5-flash' triggered its content filter while generating a response", - ): - await agent.run('bad content') - - -async def test_google_response_filter_error_stream( - allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture -): - model_name = 'gemini-2.5-flash' - model = GoogleModel(model_name, provider=google_provider) - - # Create Candidate mock - candidate_mock = mocker.Mock( - finish_reason=GoogleFinishReason.SAFETY, content=None, grounding_metadata=None, url_context_metadata=None - ) - - # Create Chunk mock - chunk_mock = mocker.Mock( - candidates=[candidate_mock], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() - ) - chunk_mock.model_dump_json.return_value = '{"mock": "json"}' - - async def stream_iterator(): - yield chunk_mock - - mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) - - agent = Agent(model=model) - - with pytest.raises( - ResponseContentFilterError, - match="Model 'gemini-2.5-flash' triggered its content filter while generating a response", - ): - async with agent.run_stream('bad content') as result: - async for _ in result.stream_text(): - pass diff --git a/tests/models/test_model_function.py b/tests/models/test_model_function.py index 196b140454..48476dcccb 100644 --- a/tests/models/test_model_function.py +++ b/tests/models/test_model_function.py @@ -22,6 +22,7 @@ ToolReturnPart, UserPromptPart, ) +from pydantic_ai.exceptions import ContentFilterError from pydantic_ai.models.function import AgentInfo, DeltaToolCall, DeltaToolCalls, FunctionModel from pydantic_ai.models.test import TestModel from pydantic_ai.result import RunUsage @@ -538,3 +539,23 @@ async def test_return_empty(): with pytest.raises(ValueError, match='Stream function must return at least one item'): async with agent.run_stream(''): pass + + +async def test_central_content_filter_handling(): + """ + Test that the agent graph correctly raises ContentFilterError + when ANY model returns finish_reason='content_filter'. + """ + + async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + # Simulate a model response that was cut off by a content filter + return ModelResponse( + parts=[TextPart('Partially generated content...')], model_name='test-model', finish_reason='content_filter' + ) + + model = FunctionModel(function=filtered_response, model_name='test-model') + agent = Agent(model) + + # The exception should be raised by _agent_graph.py + with pytest.raises(ContentFilterError, match='Content filter triggered for model test-model'): + await agent.run('Trigger filter') diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index dd9c7556b5..8d00194336 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -11,7 +11,6 @@ import pytest from inline_snapshot import snapshot from pydantic import AnyUrl, BaseModel, ConfigDict, Discriminator, Field, Tag -from pytest_mock import MockerFixture from typing_extensions import NotRequired, TypedDict from pydantic_ai import ( @@ -39,7 +38,7 @@ ) from pydantic_ai._json_schema import InlineDefsJsonSchemaTransformer from pydantic_ai.builtin_tools import WebSearchTool -from pydantic_ai.exceptions import PromptContentFilterError, ResponseContentFilterError +from pydantic_ai.exceptions import ContentFilterError from pydantic_ai.models import ModelRequestParameters from pydantic_ai.output import NativeOutput, PromptedOutput, TextOutput, ToolOutput from pydantic_ai.profiles.openai import OpenAIModelProfile, openai_model_profile @@ -73,7 +72,6 @@ from openai.types.chat.chat_completion_message_tool_call import Function from openai.types.chat.chat_completion_token_logprob import ChatCompletionTokenLogprob from openai.types.completion_usage import CompletionUsage, PromptTokensDetails - from openai.types.responses import Response, ResponseCompletedEvent, ResponseCreatedEvent from pydantic_ai.models.google import GoogleModel from pydantic_ai.models.openai import ( @@ -3313,52 +3311,11 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises( - PromptContentFilterError, match=r"Model 'gpt-5-mini' content filter was triggered by the user's prompt" - ): + with pytest.raises(ContentFilterError, match=r'Content filter triggered for model gpt-5-mini'): agent.run_sync('bad prompt') -async def test_openai_response_filter_error_sync(allow_model_requests: None): - c = completion_message( - ChatCompletionMessage(content='partial', role='assistant'), - ) - # Simulate content filter finish reason - c.choices[0].finish_reason = 'content_filter' - c.model = 'gpt-5-mini' - - mock_client = MockOpenAI.create_mock(c) - - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) - agent = Agent(m) - - with pytest.raises( - ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" - ): - await agent.run('hello') - - -async def test_openai_response_filter_error_stream(allow_model_requests: None): - stream = [text_chunk('hello'), text_chunk('', finish_reason='content_filter')] - - for chunk in stream: - chunk.model = 'gpt-5-mini' - - mock_client = MockOpenAI.create_mock_stream(stream) - - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) - agent = Agent(m) - - with pytest.raises( - ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" - ): - async with agent.run_stream('hello') as result: - async for _ in result.stream_text(): - pass - - def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None: - """Test ResponsesModel (Azure) prompt filter.""" mock_client = MockOpenAIResponses.create_mock( APIStatusError( 'content filter', @@ -3369,104 +3326,5 @@ def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises( - PromptContentFilterError, match=r"Model 'gpt-5-mini' content filter was triggered by the user's prompt" - ): + with pytest.raises(ContentFilterError, match=r'Content filter triggered for model gpt-5-mini'): agent.run_sync('bad prompt') - - -async def test_responses_response_filter_error_sync(allow_model_requests: None, mocker: MockerFixture): - """Test ResponsesModel sync response filter.""" - mock_openai_client = mocker.Mock() - m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_openai_client)) - agent = Agent(m) - - mock_response = mocker.Mock() - mock_response.model = 'gpt-5-mini' - mock_response.model_dump.return_value = {} - mock_response.created_at = 1234567890 - mock_response.id = 'resp_123' - mock_response.incomplete_details.reason = 'content_filter' - mock_response.output = [] - - mock_openai_client.responses.create = mocker.AsyncMock(return_value=mock_response) - - with pytest.raises( - ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" - ): - await agent.run('hello') - - -async def test_responses_response_filter_error_stream(allow_model_requests: None, mocker: MockerFixture): - """Test ResponsesModel stream response filter.""" - mock_openai_client = mocker.Mock() - m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_openai_client)) - agent = Agent(m) - - resp = Response( - id='resp_123', - model='gpt-5-mini', - created_at=1234567890, - object='response', - status='incomplete', - incomplete_details=cast(Any, {'reason': 'content_filter'}), - output=[], - parallel_tool_calls=False, - tool_choice='none', - tools=[], - ) - - stream = [ - ResponseCreatedEvent(response=resp, type='response.created', sequence_number=0), - ResponseCompletedEvent(response=resp, type='response.completed', sequence_number=1), - ] - - mock_client = MockOpenAIResponses.create_mock_stream(stream) - m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) - agent = Agent(m) - - with pytest.raises( - ResponseContentFilterError, match=r"Model 'gpt-5-mini' triggered its content filter while generating a response" - ): - async with agent.run_stream('hello') as result: - async for _ in result.stream_text(): - pass - - -def test_openai_400_non_content_filter(allow_model_requests: None) -> None: - """Test a standard 400 error (not content filter) raises ModelHTTPError.""" - mock_client = MockOpenAI.create_mock( - APIStatusError( - 'Bad Request', - response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), - body={'error': {'code': 'invalid_parameter', 'message': 'Invalid parameter'}}, - ) - ) - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) - agent = Agent(m) - - with pytest.raises(ModelHTTPError) as exc_info: - agent.run_sync('hello') - - assert exc_info.value.status_code == 400 - - assert not isinstance(exc_info.value, PromptContentFilterError) - - -def test_openai_400_non_dict_body(allow_model_requests: None) -> None: - """Test a 400 error with a non-dict body raises ModelHTTPError.""" - mock_client = MockOpenAI.create_mock( - APIStatusError( - 'Bad Request', - response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), - body='Raw string body', - ) - ) - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) - agent = Agent(m) - - with pytest.raises(ModelHTTPError) as exc_info: - agent.run_sync('hello') - - assert exc_info.value.status_code == 400 - assert not isinstance(exc_info.value, PromptContentFilterError) From a0a85cb6a720057cf5be985c747b45f861509223 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 13 Dec 2025 15:02:21 +0000 Subject: [PATCH 10/20] improvements --- docs/models/overview.md | 2 +- pydantic_ai_slim/pydantic_ai/_agent_graph.py | 9 ++++-- pydantic_ai_slim/pydantic_ai/models/openai.py | 2 +- tests/models/test_model_function.py | 28 +++++++++++++--- tests/models/test_openai.py | 32 +++++++++++++++++++ 5 files changed, 64 insertions(+), 9 deletions(-) diff --git a/docs/models/overview.md b/docs/models/overview.md index d6618790f2..b18885d6d5 100644 --- a/docs/models/overview.md +++ b/docs/models/overview.md @@ -229,7 +229,7 @@ contains all the exceptions encountered during the `run` execution. By default, the `FallbackModel` only moves on to the next model if the current model raises a [`ModelAPIError`][pydantic_ai.exceptions.ModelAPIError], which includes -[`ModelHTTPError`][pydantic_ai.exceptions.ModelHTTPError] and [`ContentFilterError`][pydantic_ai.exceptions.ContentFilterError]. You can customize this behavior by +[`ModelHTTPError`][pydantic_ai.exceptions.ModelHTTPError]. You can customize this behavior by passing a custom `fallback_on` argument to the `FallbackModel` constructor. !!! note diff --git a/pydantic_ai_slim/pydantic_ai/_agent_graph.py b/pydantic_ai_slim/pydantic_ai/_agent_graph.py index 924747f678..12a3050792 100644 --- a/pydantic_ai_slim/pydantic_ai/_agent_graph.py +++ b/pydantic_ai_slim/pydantic_ai/_agent_graph.py @@ -534,9 +534,6 @@ def _finish_handling( ctx: GraphRunContext[GraphAgentState, GraphAgentDeps[DepsT, NodeRunEndT]], response: _messages.ModelResponse, ) -> CallToolsNode[DepsT, NodeRunEndT]: - if response.finish_reason == 'content_filter': - raise exceptions.ContentFilterError(f'Content filter triggered for model {response.model_name}') - response.run_id = response.run_id or ctx.state.run_id # Update usage ctx.state.usage.incr(response.usage) @@ -611,6 +608,12 @@ async def _run_stream() -> AsyncIterator[_messages.HandleResponseEvent]: # noqa f'Model token limit ({max_tokens or "provider default"}) exceeded before any response was generated. Increase the `max_tokens` model setting, or simplify the prompt to result in a shorter response that will fit within the limit.' ) + # Check for content filter on empty response + if self.model_response.finish_reason == 'content_filter': + raise exceptions.ContentFilterError( + f'Content filter triggered for model {self.model_response.model_name}' + ) + # we got an empty response. # this sometimes happens with anthropic (and perhaps other models) # when the model has already returned text along side tool calls diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index a3fd86f22f..92ec23f809 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -605,7 +605,7 @@ async def _completions_create( chat.chat_completion.Choice( finish_reason='content_filter', index=0, - message=chat.ChatCompletionMessage(content='', role='assistant'), + message=chat.ChatCompletionMessage(content=None, role='assistant'), ) ], created=0, diff --git a/tests/models/test_model_function.py b/tests/models/test_model_function.py index 48476dcccb..6401a61424 100644 --- a/tests/models/test_model_function.py +++ b/tests/models/test_model_function.py @@ -544,18 +544,38 @@ async def test_return_empty(): async def test_central_content_filter_handling(): """ Test that the agent graph correctly raises ContentFilterError - when ANY model returns finish_reason='content_filter'. + when a model returns finish_reason='content_filter' AND empty content. """ async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: - # Simulate a model response that was cut off by a content filter + # Simulate a model response that was blocked completely return ModelResponse( - parts=[TextPart('Partially generated content...')], model_name='test-model', finish_reason='content_filter' + parts=[], # Empty parts triggers the exception + model_name='test-model', + finish_reason='content_filter', ) model = FunctionModel(function=filtered_response, model_name='test-model') agent = Agent(model) - # The exception should be raised by _agent_graph.py with pytest.raises(ContentFilterError, match='Content filter triggered for model test-model'): await agent.run('Trigger filter') + + +async def test_central_content_filter_with_partial_content(): + """ + Test that the agent graph returns partial content (does not raise exception) + even if finish_reason='content_filter', provided parts are not empty. + """ + + async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + return ModelResponse( + parts=[TextPart('Partially generated content...')], model_name='test-model', finish_reason='content_filter' + ) + + model = FunctionModel(function=filtered_response, model_name='test-model') + agent = Agent(model) + + # Should NOT raise ContentFilterError + result = await agent.run('Trigger filter') + assert result.output == 'Partially generated content...' diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 6a389c6d53..86ac81e5da 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3357,3 +3357,35 @@ def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None with pytest.raises(ContentFilterError, match=r'Content filter triggered for model gpt-5-mini'): agent.run_sync('bad prompt') + + +async def test_openai_response_filter_error_sync(allow_model_requests: None): + """Test that ContentFilterError is raised when response is empty and finish_reason is content_filter.""" + c = completion_message( + ChatCompletionMessage(content=None, role='assistant'), + ) + c.choices[0].finish_reason = 'content_filter' + c.model = 'gpt-5-mini' + + mock_client = MockOpenAI.create_mock(c) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ContentFilterError, match=r'Content filter triggered for model gpt-5-mini'): + await agent.run('hello') + + +async def test_openai_response_filter_with_partial_content(allow_model_requests: None): + """Test that NO exception is raised if content is returned, even if finish_reason is content_filter.""" + c = completion_message( + ChatCompletionMessage(content='Partial content', role='assistant'), + ) + c.choices[0].finish_reason = 'content_filter' + + mock_client = MockOpenAI.create_mock(c) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + # Should NOT raise ContentFilterError + result = await agent.run('hello') + assert result.output == 'Partial content' From bba451d2fd5b2c336a376ae49c0cdab44f286e14 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 13 Dec 2025 16:32:08 +0000 Subject: [PATCH 11/20] test coverage --- pydantic_ai_slim/pydantic_ai/_utils.py | 2 +- tests/models/test_google.py | 54 ++++++++++++++++++++++++++ tests/models/test_openai.py | 38 ++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/pydantic_ai_slim/pydantic_ai/_utils.py b/pydantic_ai_slim/pydantic_ai/_utils.py index d0fc819e73..97230bb046 100644 --- a/pydantic_ai_slim/pydantic_ai/_utils.py +++ b/pydantic_ai_slim/pydantic_ai/_utils.py @@ -215,7 +215,7 @@ async def async_iter_groups() -> AsyncIterator[list[T]]: try: yield async_iter_groups() - finally: + finally: # pragma: no cover # after iteration if a tasks still exists, cancel it, this will only happen if an error occurred if task: task.cancel('Cancelling due to error in iterator') diff --git a/tests/models/test_google.py b/tests/models/test_google.py index ef6bbb9d9e..d4204bd6ae 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -4619,3 +4619,57 @@ def get_country() -> str: ), ] ) + + +async def test_google_stream_empty_chunk( + allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture +): + """Test that empty chunks in the stream are ignored (coverage for continue).""" + model_name = 'gemini-2.5-flash' + model = GoogleModel(model_name, provider=google_provider) + + # Chunk with NO content + empty_candidate = mocker.Mock(finish_reason=None, content=None) + empty_candidate.grounding_metadata = None + empty_candidate.url_context_metadata = None + + chunk_empty = mocker.Mock( + candidates=[empty_candidate], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() + ) + chunk_empty.model_dump_json.return_value = '{}' + + # Chunk WITH content (valid) + part_mock = mocker.Mock( + text='Hello', + thought=False, + function_call=None, + inline_data=None, + executable_code=None, + code_execution_result=None, + ) + part_mock.thought_signature = None + + valid_candidate = mocker.Mock( + finish_reason=GoogleFinishReason.STOP, + content=mocker.Mock(parts=[part_mock]), + grounding_metadata=None, + url_context_metadata=None, + ) + + chunk_valid = mocker.Mock( + candidates=[valid_candidate], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() + ) + chunk_valid.model_dump_json.return_value = '{"content": "Hello"}' + + async def stream_iterator(): + yield chunk_empty + yield chunk_valid + + mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) + + agent = Agent(model=model) + + async with agent.run_stream('hello') as result: + output = await result.get_output() + + assert output == 'Hello' diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 86ac81e5da..43ef8fd2b5 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3389,3 +3389,41 @@ async def test_openai_response_filter_with_partial_content(allow_model_requests: # Should NOT raise ContentFilterError result = await agent.run('hello') assert result.output == 'Partial content' + + +def test_openai_400_non_content_filter(allow_model_requests: None) -> None: + """Test a 400 error that is NOT a content filter (different code).""" + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'Bad Request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), + body={'error': {'code': 'invalid_parameter', 'message': 'Invalid param.'}}, + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + # Should be ModelHTTPError, NOT ContentFilterError + assert not isinstance(exc_info.value, ContentFilterError) + assert exc_info.value.status_code == 400 + + +def test_openai_400_non_dict_body(allow_model_requests: None) -> None: + """Test a 400 error where the body is not a dictionary.""" + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'Bad Request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), + body='Raw string body', + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + agent = Agent(m) + + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + assert exc_info.value.status_code == 400 From d9693256a388c768ef3f7114d57db7d482cc5660 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 13 Dec 2025 16:56:21 +0000 Subject: [PATCH 12/20] test coverage --- pydantic_ai_slim/pydantic_ai/models/google.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py index e4b66c220c..f4b72a25b8 100644 --- a/pydantic_ai_slim/pydantic_ai/models/google.py +++ b/pydantic_ai_slim/pydantic_ai/models/google.py @@ -503,7 +503,7 @@ def _process_response(self, response: GenerateContentResponse) -> ModelResponse: finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) if candidate.content is None or candidate.content.parts is None: - parts = [] # pragma: no cover + parts = [] else: parts = candidate.content.parts or [] From 7561b637d4a3600480ff319643d8a30fcf7be1ba Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Thu, 18 Dec 2025 10:53:41 +0000 Subject: [PATCH 13/20] improvements --- pydantic_ai_slim/pydantic_ai/_agent_graph.py | 10 ++++++- pydantic_ai_slim/pydantic_ai/exceptions.py | 2 +- pydantic_ai_slim/pydantic_ai/models/google.py | 3 ++ pydantic_ai_slim/pydantic_ai/models/openai.py | 29 ++++++++++++------- tests/models/test_google.py | 2 +- tests/models/test_model_function.py | 2 +- tests/models/test_openai.py | 20 ++++++------- 7 files changed, 43 insertions(+), 25 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/_agent_graph.py b/pydantic_ai_slim/pydantic_ai/_agent_graph.py index 21b063d394..6e62683ebf 100644 --- a/pydantic_ai_slim/pydantic_ai/_agent_graph.py +++ b/pydantic_ai_slim/pydantic_ai/_agent_graph.py @@ -605,8 +605,16 @@ async def _run_stream() -> AsyncIterator[_messages.HandleResponseEvent]: # noqa # Check for content filter on empty response if self.model_response.finish_reason == 'content_filter': + details = self.model_response.provider_details or {} + reason = details.get('finish_reason', 'content_filter') + + try: + body = _messages.ModelMessagesTypeAdapter.dump_json([self.model_response]).decode() + except Exception: + body = str(self.model_response) + raise exceptions.ContentFilterError( - f'Content filter triggered for model {self.model_response.model_name}' + f"Content filter triggered. Finish reason: '{reason}'", body=body ) # we got an empty response. diff --git a/pydantic_ai_slim/pydantic_ai/exceptions.py b/pydantic_ai_slim/pydantic_ai/exceptions.py index a9cabb8b74..20df96978c 100644 --- a/pydantic_ai_slim/pydantic_ai/exceptions.py +++ b/pydantic_ai_slim/pydantic_ai/exceptions.py @@ -154,7 +154,7 @@ def __str__(self) -> str: class ContentFilterError(UnexpectedModelBehavior): - """Raised when content filtering is triggered by the model provider.""" + """Raised when content filtering is triggered by the model provider resulting in an empty response.""" class ModelAPIError(AgentRunError): diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py index f4b72a25b8..c9577225cb 100644 --- a/pydantic_ai_slim/pydantic_ai/models/google.py +++ b/pydantic_ai_slim/pydantic_ai/models/google.py @@ -500,6 +500,9 @@ def _process_response(self, response: GenerateContentResponse) -> ModelResponse: raw_finish_reason = candidate.finish_reason if raw_finish_reason: # pragma: no branch vendor_details = {'finish_reason': raw_finish_reason.value} + # Add safety ratings to provider details + if candidate.safety_ratings: + vendor_details['safety_ratings'] = [r.model_dump(by_alias=True) for r in candidate.safety_ratings] finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) if candidate.content is None or candidate.content.parts is None: diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index 4513d90a90..104feed018 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -176,18 +176,17 @@ def _resolve_openai_image_generation_size( return mapped_size -def _check_azure_content_filter(e: APIStatusError) -> bool: - """Check if the error is an Azure content filter error.""" - if e.status_code == 400: +def _check_azure_content_filter(e: APIStatusError, system: str) -> dict[str, Any] | None: + """Check if the error is an Azure content filter error. Returns inner error dict if match.""" + if system == 'azure' and e.status_code == 400: body_any: Any = e.body - if isinstance(body_any, dict): body_dict = cast(dict[str, Any], body_any) - if (error := body_dict.get('error')) and isinstance(error, dict): error_dict = cast(dict[str, Any], error) - return error_dict.get('code') == 'content_filter' - return False + if error_dict.get('code') == 'content_filter': + return error_dict + return None class OpenAIChatModelSettings(ModelSettings, total=False): @@ -598,8 +597,8 @@ async def _completions_create( extra_body=model_settings.get('extra_body'), ) except APIStatusError as e: - if _check_azure_content_filter(e): - return chat.ChatCompletion( + if azure_error := _check_azure_content_filter(e, self.system): + resp = chat.ChatCompletion( id='content_filter', choices=[ chat.chat_completion.Choice( @@ -612,6 +611,10 @@ async def _completions_create( model=self.model_name, object='chat.completion', ) + # Attach inner error details for _process_provider_details to pick up + if inner_error := azure_error.get('innererror'): + setattr(resp, 'azure_content_filter_result', inner_error) + return resp if (status_code := e.status_code) >= 400: raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover @@ -630,7 +633,11 @@ def _process_provider_details(self, response: chat.ChatCompletion) -> dict[str, This method may be overridden by subclasses of `OpenAIChatModel` to apply custom mappings. """ - return _map_provider_details(response.choices[0]) + details = _map_provider_details(response.choices[0]) + + if hasattr(response, 'azure_content_filter_result'): + details['content_filter_result'] = getattr(response, 'azure_content_filter_result') + return details def _process_response(self, response: chat.ChatCompletion | str) -> ModelResponse: """Process a non-streamed response, and prepare a message to return.""" @@ -1460,7 +1467,7 @@ async def _responses_create( # noqa: C901 extra_body=model_settings.get('extra_body'), ) except APIStatusError as e: - if _check_azure_content_filter(e): + if _check_azure_content_filter(e, self.system): return responses.Response( id='content_filter', model=self.model_name, diff --git a/tests/models/test_google.py b/tests/models/test_google.py index d4204bd6ae..810eae7e55 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -1002,7 +1002,7 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p with pytest.raises( ContentFilterError, - match='Content filter triggered for model gemini-1.5-flash', + match="Content filter triggered. Finish reason: 'SAFETY'", ): await agent.run('Tell me a joke about a Brazilians.') diff --git a/tests/models/test_model_function.py b/tests/models/test_model_function.py index 6401a61424..0d6fa3143b 100644 --- a/tests/models/test_model_function.py +++ b/tests/models/test_model_function.py @@ -558,7 +558,7 @@ async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> Mo model = FunctionModel(function=filtered_response, model_name='test-model') agent = Agent(model) - with pytest.raises(ContentFilterError, match='Content filter triggered for model test-model'): + with pytest.raises(ContentFilterError, match="Content filter triggered. Finish reason: 'content_filter'"): await agent.run('Trigger filter') diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 43ef8fd2b5..70f002ea73 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -58,7 +58,7 @@ ) with try_import() as imports_successful: - from openai import APIConnectionError, APIStatusError, AsyncOpenAI + from openai import APIConnectionError, APIStatusError, AsyncAzureOpenAI, AsyncOpenAI from openai.types import chat from openai.types.chat.chat_completion import ChoiceLogprobs from openai.types.chat.chat_completion_chunk import ( @@ -82,6 +82,7 @@ OpenAISystemPromptRole, ) from pydantic_ai.profiles.openai import OpenAIJsonSchemaTransformer + from pydantic_ai.providers.azure import AzureProvider from pydantic_ai.providers.cerebras import CerebrasProvider from pydantic_ai.providers.google import GoogleProvider from pydantic_ai.providers.ollama import OllamaProvider @@ -3337,10 +3338,10 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: ) ) - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + m = OpenAIChatModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) agent = Agent(m) - with pytest.raises(ContentFilterError, match=r'Content filter triggered for model gpt-5-mini'): + with pytest.raises(ContentFilterError, match=r"Content filter triggered. Finish reason: 'content_filter'"): agent.run_sync('bad prompt') @@ -3352,15 +3353,15 @@ def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None body={'error': {'code': 'content_filter', 'message': 'The content was filtered.'}}, ) ) - m = OpenAIResponsesModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + + m = OpenAIResponsesModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) agent = Agent(m) - with pytest.raises(ContentFilterError, match=r'Content filter triggered for model gpt-5-mini'): + with pytest.raises(ContentFilterError, match=r"Content filter triggered. Finish reason: 'content_filter'"): agent.run_sync('bad prompt') async def test_openai_response_filter_error_sync(allow_model_requests: None): - """Test that ContentFilterError is raised when response is empty and finish_reason is content_filter.""" c = completion_message( ChatCompletionMessage(content=None, role='assistant'), ) @@ -3371,14 +3372,14 @@ async def test_openai_response_filter_error_sync(allow_model_requests: None): m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - with pytest.raises(ContentFilterError, match=r'Content filter triggered for model gpt-5-mini'): + with pytest.raises(ContentFilterError, match=r"Content filter triggered. Finish reason: 'content_filter'"): await agent.run('hello') async def test_openai_response_filter_with_partial_content(allow_model_requests: None): """Test that NO exception is raised if content is returned, even if finish_reason is content_filter.""" c = completion_message( - ChatCompletionMessage(content='Partial content', role='assistant'), + ChatCompletionMessage(content='Partial', role='assistant'), ) c.choices[0].finish_reason = 'content_filter' @@ -3386,9 +3387,8 @@ async def test_openai_response_filter_with_partial_content(allow_model_requests: m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) agent = Agent(m) - # Should NOT raise ContentFilterError result = await agent.run('hello') - assert result.output == 'Partial content' + assert result.output == 'Partial' def test_openai_400_non_content_filter(allow_model_requests: None) -> None: From 66daa8726d0b1cda48aa49d55b644a691232dcc5 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Thu, 18 Dec 2025 10:58:14 +0000 Subject: [PATCH 14/20] remove unwanted tests --- tests/models/test_google.py | 54 ------------------------------------- 1 file changed, 54 deletions(-) diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 810eae7e55..04cf9dafb1 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -4619,57 +4619,3 @@ def get_country() -> str: ), ] ) - - -async def test_google_stream_empty_chunk( - allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture -): - """Test that empty chunks in the stream are ignored (coverage for continue).""" - model_name = 'gemini-2.5-flash' - model = GoogleModel(model_name, provider=google_provider) - - # Chunk with NO content - empty_candidate = mocker.Mock(finish_reason=None, content=None) - empty_candidate.grounding_metadata = None - empty_candidate.url_context_metadata = None - - chunk_empty = mocker.Mock( - candidates=[empty_candidate], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() - ) - chunk_empty.model_dump_json.return_value = '{}' - - # Chunk WITH content (valid) - part_mock = mocker.Mock( - text='Hello', - thought=False, - function_call=None, - inline_data=None, - executable_code=None, - code_execution_result=None, - ) - part_mock.thought_signature = None - - valid_candidate = mocker.Mock( - finish_reason=GoogleFinishReason.STOP, - content=mocker.Mock(parts=[part_mock]), - grounding_metadata=None, - url_context_metadata=None, - ) - - chunk_valid = mocker.Mock( - candidates=[valid_candidate], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() - ) - chunk_valid.model_dump_json.return_value = '{"content": "Hello"}' - - async def stream_iterator(): - yield chunk_empty - yield chunk_valid - - mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) - - agent = Agent(model=model) - - async with agent.run_stream('hello') as result: - output = await result.get_output() - - assert output == 'Hello' From 6ad2c14a3479b66a55c45b8c103ff2c744f08d69 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Thu, 18 Dec 2025 11:43:05 +0000 Subject: [PATCH 15/20] fix format errors --- tests/models/test_openai.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index f8548ab2b1..7f0a49ca49 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -39,9 +39,8 @@ UserPromptPart, ) from pydantic_ai._json_schema import InlineDefsJsonSchemaTransformer -from pydantic_ai.builtin_tools import WebSearchTool -from pydantic_ai.exceptions import ContentFilterError from pydantic_ai.builtin_tools import ImageGenerationTool, WebSearchTool +from pydantic_ai.exceptions import ContentFilterError from pydantic_ai.models import ModelRequestParameters from pydantic_ai.output import NativeOutput, PromptedOutput, TextOutput, ToolOutput from pydantic_ai.profiles.openai import OpenAIModelProfile, openai_model_profile @@ -3661,6 +3660,8 @@ def test_openai_400_non_dict_body(allow_model_requests: None) -> None: agent.run_sync('hello') assert exc_info.value.status_code == 400 + + async def test_openai_chat_instructions_after_system_prompts(allow_model_requests: None): """Test that instructions are inserted after all system prompts in mapped messages.""" mock_client = MockOpenAI.create_mock(completion_message(ChatCompletionMessage(content='ok', role='assistant'))) From 3e37ae649c52c4bfbd9b759694eabf685d397116 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Thu, 18 Dec 2025 13:39:11 +0000 Subject: [PATCH 16/20] test coverage --- tests/models/test_google.py | 55 +++++++++++++++++++++++++++++ tests/models/test_model_function.py | 32 +++++++++++++++++ tests/models/test_openai.py | 20 +++++++++-- 3 files changed, 105 insertions(+), 2 deletions(-) diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 760486164e..83cac7c668 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -5029,3 +5029,58 @@ async def test_google_system_prompts_and_instructions_ordering(google_provider: } ) assert contents == snapshot([{'role': 'user', 'parts': [{'text': 'Hello'}]}]) + + +async def test_google_stream_empty_chunk( + allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture +): + """Test that empty chunks in the stream are ignored.""" + model_name = 'gemini-2.5-flash' + model = GoogleModel(model_name, provider=google_provider) + + # Chunk with NO content + empty_candidate = mocker.Mock(finish_reason=None, content=None) + empty_candidate.grounding_metadata = None + empty_candidate.url_context_metadata = None + + chunk_empty = mocker.Mock( + candidates=[empty_candidate], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() + ) + chunk_empty.model_dump_json.return_value = '{}' + + # Chunk WITH content (valid) + part_mock = mocker.Mock( + text='Hello', + thought=False, + function_call=None, + inline_data=None, + executable_code=None, + code_execution_result=None, + ) + part_mock.thought_signature = None + + valid_candidate = mocker.Mock( + finish_reason=GoogleFinishReason.STOP, + content=mocker.Mock(parts=[part_mock]), + grounding_metadata=None, + url_context_metadata=None, + ) + + chunk_valid = mocker.Mock( + candidates=[valid_candidate], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() + ) + chunk_valid.model_dump_json.return_value = '{"content": "Hello"}' + + async def stream_iterator(): + yield chunk_empty + yield chunk_valid + + mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) + + agent = Agent(model=model) + + # Use run_stream to hit the mocked method + async with agent.run_stream('hello') as result: + output = await result.get_output() + + assert output == 'Hello' diff --git a/tests/models/test_model_function.py b/tests/models/test_model_function.py index 0d6fa3143b..f40bd1473b 100644 --- a/tests/models/test_model_function.py +++ b/tests/models/test_model_function.py @@ -579,3 +579,35 @@ async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> Mo # Should NOT raise ContentFilterError result = await agent.run('Trigger filter') assert result.output == 'Partially generated content...' + + +async def test_content_filter_serialization_fallback(): + """ + Test fallback to str() when ModelResponse cannot be serialized to JSON. + Covers the except block in _agent_graph.py. + """ + + class UnserializableObj: + def __repr__(self): + return '' + + # Create a response that will fail JSON serialization + response = ModelResponse( + parts=[], + model_name='test-model', + finish_reason='content_filter', + # Inject unserializable object into provider_details to break dumping + provider_details={'bad_data': UnserializableObj()}, + ) + + async def bad_response_model(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + return response + + model = FunctionModel(function=bad_response_model, model_name='test-model') + agent = Agent(model) + + with pytest.raises(ContentFilterError) as exc_info: + await agent.run('trigger') + + # Verify it fell back to str() representation + assert '' in str(exc_info.value.body) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 7f0a49ca49..c8a45f26f9 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3563,19 +3563,35 @@ async def test_openai_reasoning_in_thinking_tags(allow_model_requests: None): def test_azure_prompt_filter_error(allow_model_requests: None) -> None: + body = { + 'error': { + 'code': 'content_filter', + 'message': 'The content was filtered.', + 'innererror': { + 'code': 'ResponsibleAIPolicyViolation', + 'content_filter_result': {'hate': {'filtered': True, 'severity': 'high'}}, + }, + } + } + mock_client = MockOpenAI.create_mock( APIStatusError( 'content filter', response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), - body={'error': {'code': 'content_filter', 'message': 'The content was filtered.'}}, + body=body, ) ) m = OpenAIChatModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) agent = Agent(m) - with pytest.raises(ContentFilterError, match=r"Content filter triggered. Finish reason: 'content_filter'"): + try: agent.run_sync('bad prompt') + except ContentFilterError as e: + # Check that the exception was raised + assert "Content filter triggered. Finish reason: 'content_filter'" in str(e) + # Verify body contains the inner error details (serialized) + assert 'ResponsibleAIPolicyViolation' in str(e.body) def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None: From 5522fbbc7c2b6148658a26be95a10184eeb2380f Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Thu, 18 Dec 2025 15:06:29 +0000 Subject: [PATCH 17/20] test coverage --- tests/models/test_openai.py | 64 +++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index c8a45f26f9..39320d876f 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3640,36 +3640,53 @@ async def test_openai_response_filter_with_partial_content(allow_model_requests: assert result.output == 'Partial' -def test_openai_400_non_content_filter(allow_model_requests: None) -> None: - """Test a 400 error that is NOT a content filter (different code).""" +def test_azure_400_non_content_filter(allow_model_requests: None) -> None: + """Test a 400 error from Azure that is NOT a content filter (different code).""" mock_client = MockOpenAI.create_mock( APIStatusError( 'Bad Request', - response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), body={'error': {'code': 'invalid_parameter', 'message': 'Invalid param.'}}, ) ) - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + m = OpenAIChatModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) agent = Agent(m) with pytest.raises(ModelHTTPError) as exc_info: agent.run_sync('hello') - # Should be ModelHTTPError, NOT ContentFilterError - assert not isinstance(exc_info.value, ContentFilterError) assert exc_info.value.status_code == 400 + assert not isinstance(exc_info.value, ContentFilterError) -def test_openai_400_non_dict_body(allow_model_requests: None) -> None: - """Test a 400 error where the body is not a dictionary.""" +def test_azure_400_non_dict_body(allow_model_requests: None) -> None: + """Test a 400 error from Azure where the body is not a dictionary.""" mock_client = MockOpenAI.create_mock( APIStatusError( 'Bad Request', - response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://api.openai.com/v1')), + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), body='Raw string body', ) ) - m = OpenAIChatModel('gpt-5-mini', provider=OpenAIProvider(openai_client=mock_client)) + m = OpenAIChatModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) + agent = Agent(m) + + with pytest.raises(ModelHTTPError) as exc_info: + agent.run_sync('hello') + + assert exc_info.value.status_code == 400 + + +def test_azure_400_malformed_error(allow_model_requests: None) -> None: + """Test a 400 error from Azure where body matches dict but error structure is wrong.""" + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'Bad Request', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), + body={'something_else': 'foo'}, # No 'error' key + ) + ) + m = OpenAIChatModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) agent = Agent(m) with pytest.raises(ModelHTTPError) as exc_info: @@ -3678,6 +3695,33 @@ def test_openai_400_non_dict_body(allow_model_requests: None) -> None: assert exc_info.value.status_code == 400 +def test_azure_prompt_filter_no_inner_error(allow_model_requests: None) -> None: + """Test Azure content filter that lacks the innererror details.""" + # Valid content filter code, but missing 'innererror' + body = { + 'error': { + 'code': 'content_filter', + 'message': 'The content was filtered.', + # No innererror + } + } + + mock_client = MockOpenAI.create_mock( + APIStatusError( + 'content filter', + response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), + body=body, + ) + ) + + m = OpenAIChatModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) + agent = Agent(m) + + # Should still raise ContentFilterError + with pytest.raises(ContentFilterError, match=r"Content filter triggered. Finish reason: 'content_filter'"): + agent.run_sync('bad prompt') + + async def test_openai_chat_instructions_after_system_prompts(allow_model_requests: None): """Test that instructions are inserted after all system prompts in mapped messages.""" mock_client = MockOpenAI.create_mock(completion_message(ChatCompletionMessage(content='ok', role='assistant'))) From c53e55899b3737d56421f09c80029ea5d1d21ff4 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 19 Dec 2025 16:14:37 +0000 Subject: [PATCH 18/20] improvements --- pydantic_ai_slim/pydantic_ai/models/google.py | 6 + pydantic_ai_slim/pydantic_ai/models/openai.py | 118 +++++++++++------- tests/models/test_google.py | 74 ++++++++++- tests/models/test_model_function.py | 73 ----------- tests/models/test_openai.py | 53 ++++++-- tests/test_agent.py | 73 +++++++++++ 6 files changed, 270 insertions(+), 127 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/models/google.py b/pydantic_ai_slim/pydantic_ai/models/google.py index 7ccd278254..4f7cef311c 100644 --- a/pydantic_ai_slim/pydantic_ai/models/google.py +++ b/pydantic_ai_slim/pydantic_ai/models/google.py @@ -716,6 +716,12 @@ async def _get_event_iterator(self) -> AsyncIterator[ModelResponseStreamEvent]: raw_finish_reason = candidate.finish_reason if raw_finish_reason: self.provider_details = {'finish_reason': raw_finish_reason.value} + + if candidate.safety_ratings: + self.provider_details['safety_ratings'] = [ + r.model_dump(by_alias=True) for r in candidate.safety_ratings + ] + self.finish_reason = _FINISH_REASON_MAP.get(raw_finish_reason) # Google streams the grounding metadata (including the web search queries and results) diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index 286a06011e..df4f1f3691 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -11,7 +11,7 @@ from functools import cached_property from typing import Any, Literal, cast, overload -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from pydantic_core import to_json from typing_extensions import assert_never, deprecated @@ -164,6 +164,34 @@ _OPENAI_IMAGE_SIZES: tuple[_OPENAI_IMAGE_SIZE, ...] = _utils.get_args(_OPENAI_IMAGE_SIZE) +class _AzureContentFilterResultDetail(BaseModel): + filtered: bool + severity: str + + +class _AzureContentFilterResult(BaseModel): + hate: _AzureContentFilterResultDetail | None = None + self_harm: _AzureContentFilterResultDetail | None = None + sexual: _AzureContentFilterResultDetail | None = None + violence: _AzureContentFilterResultDetail | None = None + jailbreak: dict[str, Any] | None = None + + +class _AzureInnerError(BaseModel): + code: str + content_filter_result: _AzureContentFilterResult + + +class _AzureError(BaseModel): + code: str + message: str + innererror: _AzureInnerError | None = None + + +class _AzureErrorResponse(BaseModel): + error: _AzureError + + def _resolve_openai_image_generation_size( tool: ImageGenerationTool, ) -> _OPENAI_IMAGE_SIZE: @@ -194,16 +222,32 @@ def _resolve_openai_image_generation_size( return mapped_size -def _check_azure_content_filter(e: APIStatusError, system: str) -> dict[str, Any] | None: - """Check if the error is an Azure content filter error. Returns inner error dict if match.""" - if system == 'azure' and e.status_code == 400: - body_any: Any = e.body - if isinstance(body_any, dict): - body_dict = cast(dict[str, Any], body_any) - if (error := body_dict.get('error')) and isinstance(error, dict): - error_dict = cast(dict[str, Any], error) - if error_dict.get('code') == 'content_filter': - return error_dict +def _check_azure_content_filter(e: APIStatusError, system: str, model_name: str) -> ModelResponse | None: + """Check if the error is an Azure content filter error.""" + body_any: Any = e.body + + if system == 'azure' and e.status_code == 400 and isinstance(body_any, dict): + try: + error_data = _AzureErrorResponse.model_validate(body_any) + + if error_data.error.code == 'content_filter': + provider_details: dict[str, Any] = {'finish_reason': 'content_filter'} + + if error_data.error.innererror: + provider_details['content_filter_result'] = ( + error_data.error.innererror.content_filter_result.model_dump(exclude_none=True) + ) + + return ModelResponse( + parts=[], # Empty parts to trigger content filter error in agent graph + model_name=model_name, + timestamp=_utils.now_utc(), + provider_name=system, + finish_reason='content_filter', + provider_details=provider_details, + ) + except ValidationError: + pass return None @@ -545,6 +589,11 @@ async def request( response = await self._completions_create( messages, False, cast(OpenAIChatModelSettings, model_settings or {}), model_request_parameters ) + + # Handle ModelResponse returned directly (for content filters) + if isinstance(response, ModelResponse): + return response + model_response = self._process_response(response) return model_response @@ -583,7 +632,7 @@ async def _completions_create( stream: Literal[False], model_settings: OpenAIChatModelSettings, model_request_parameters: ModelRequestParameters, - ) -> chat.ChatCompletion: ... + ) -> chat.ChatCompletion | ModelResponse: ... async def _completions_create( self, @@ -591,7 +640,7 @@ async def _completions_create( stream: bool, model_settings: OpenAIChatModelSettings, model_request_parameters: ModelRequestParameters, - ) -> chat.ChatCompletion | AsyncStream[ChatCompletionChunk]: + ) -> chat.ChatCompletion | AsyncStream[ChatCompletionChunk] | ModelResponse: tools = self._get_tools(model_request_parameters) web_search_options = self._get_web_search_options(model_request_parameters) @@ -655,24 +704,8 @@ async def _completions_create( extra_body=model_settings.get('extra_body'), ) except APIStatusError as e: - if azure_error := _check_azure_content_filter(e, self.system): - resp = chat.ChatCompletion( - id='content_filter', - choices=[ - chat.chat_completion.Choice( - finish_reason='content_filter', - index=0, - message=chat.ChatCompletionMessage(content=None, role='assistant'), - ) - ], - created=0, - model=self.model_name, - object='chat.completion', - ) - # Attach inner error details for _process_provider_details to pick up - if inner_error := azure_error.get('innererror'): - setattr(resp, 'azure_content_filter_result', inner_error) - return resp + if model_response := _check_azure_content_filter(e, self.system, self.model_name): + return model_response if (status_code := e.status_code) >= 400: raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover @@ -1253,6 +1286,11 @@ async def request( response = await self._responses_create( messages, False, cast(OpenAIResponsesModelSettings, model_settings or {}), model_request_parameters ) + + # Handle ModelResponse + if isinstance(response, ModelResponse): + return response + return self._process_response(response, model_request_parameters) @asynccontextmanager @@ -1433,7 +1471,7 @@ async def _responses_create( # noqa: C901 stream: bool, model_settings: OpenAIResponsesModelSettings, model_request_parameters: ModelRequestParameters, - ) -> responses.Response | AsyncStream[responses.ResponseStreamEvent]: + ) -> responses.Response | AsyncStream[responses.ResponseStreamEvent] | ModelResponse: tools = ( self._get_builtin_tools(model_request_parameters) + list(model_settings.get('openai_builtin_tools', [])) @@ -1535,19 +1573,9 @@ async def _responses_create( # noqa: C901 extra_body=model_settings.get('extra_body'), ) except APIStatusError as e: - if _check_azure_content_filter(e, self.system): - return responses.Response( - id='content_filter', - model=self.model_name, - created_at=0, - object='response', - status='incomplete', - incomplete_details={'reason': 'content_filter'}, # type: ignore - output=[], - parallel_tool_calls=False, - tool_choice='auto', - tools=[], - ) + if model_response := _check_azure_content_filter(e, self.system, self.model_name): + return model_response + if (status_code := e.status_code) >= 400: raise ModelHTTPError(status_code=status_code, model_name=self.model_name, body=e.body) from e raise # pragma: lax no cover diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 83cac7c668..5e9f5e6e15 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -3,6 +3,7 @@ import asyncio import base64 import datetime +import json import os import re import tempfile @@ -5042,9 +5043,15 @@ async def test_google_stream_empty_chunk( empty_candidate = mocker.Mock(finish_reason=None, content=None) empty_candidate.grounding_metadata = None empty_candidate.url_context_metadata = None + # FIX: Set safety_ratings to None to avoid iteration error + empty_candidate.safety_ratings = None chunk_empty = mocker.Mock( - candidates=[empty_candidate], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() + candidates=[empty_candidate], + model_version=model_name, + usage_metadata=None, + create_time=datetime.datetime.now(), + response_id='resp_empty', ) chunk_empty.model_dump_json.return_value = '{}' @@ -5065,9 +5072,15 @@ async def test_google_stream_empty_chunk( grounding_metadata=None, url_context_metadata=None, ) + # FIX: Set safety_ratings to None + valid_candidate.safety_ratings = None chunk_valid = mocker.Mock( - candidates=[valid_candidate], model_version=model_name, usage_metadata=None, create_time=datetime.datetime.now() + candidates=[valid_candidate], + model_version=model_name, + usage_metadata=None, + create_time=datetime.datetime.now(), + response_id='resp_valid', ) chunk_valid.model_dump_json.return_value = '{"content": "Hello"}' @@ -5084,3 +5097,60 @@ async def stream_iterator(): output = await result.get_output() assert output == 'Hello' + + +async def test_google_stream_safety_filter( + allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture +): + """Test that safety ratings are captured in the exception body when streaming.""" + model_name = 'gemini-2.5-flash' + model = GoogleModel(model_name, provider=google_provider) + + # Mock safety ratings + safety_rating = mocker.Mock(category='HARM_CATEGORY_HATE_SPEECH', probability='HIGH', blocked=True) + # Configure model_dump + safety_rating.model_dump.return_value = { + 'category': 'HARM_CATEGORY_HATE_SPEECH', + 'probability': 'HIGH', + 'blocked': True, + } + + candidate = mocker.Mock( + finish_reason=GoogleFinishReason.SAFETY, + content=None, + safety_ratings=[safety_rating], + # Ensure these are None to avoid iteration errors + grounding_metadata=None, + url_context_metadata=None, + ) + + chunk = mocker.Mock( + candidates=[candidate], + model_version=model_name, + usage_metadata=None, + create_time=datetime.datetime.now(), + response_id='resp_123', # Set string ID + ) + chunk.model_dump_json.return_value = '{"mock": "json"}' + + async def stream_iterator(): + yield chunk + + mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) + + agent = Agent(model=model) + + with pytest.raises(ContentFilterError) as exc_info: + async with agent.run_stream('bad content'): + pass + + # Verify exception message + assert 'Content filter triggered' in str(exc_info.value) + + # Verify safety ratings are present in the body (serialized ModelResponse) + assert exc_info.value.body is not None + body_json = json.loads(exc_info.value.body) + + response_msg = body_json[0] + assert response_msg['provider_details']['finish_reason'] == 'SAFETY' + assert response_msg['provider_details']['safety_ratings'][0]['category'] == 'HARM_CATEGORY_HATE_SPEECH' diff --git a/tests/models/test_model_function.py b/tests/models/test_model_function.py index f40bd1473b..196b140454 100644 --- a/tests/models/test_model_function.py +++ b/tests/models/test_model_function.py @@ -22,7 +22,6 @@ ToolReturnPart, UserPromptPart, ) -from pydantic_ai.exceptions import ContentFilterError from pydantic_ai.models.function import AgentInfo, DeltaToolCall, DeltaToolCalls, FunctionModel from pydantic_ai.models.test import TestModel from pydantic_ai.result import RunUsage @@ -539,75 +538,3 @@ async def test_return_empty(): with pytest.raises(ValueError, match='Stream function must return at least one item'): async with agent.run_stream(''): pass - - -async def test_central_content_filter_handling(): - """ - Test that the agent graph correctly raises ContentFilterError - when a model returns finish_reason='content_filter' AND empty content. - """ - - async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: - # Simulate a model response that was blocked completely - return ModelResponse( - parts=[], # Empty parts triggers the exception - model_name='test-model', - finish_reason='content_filter', - ) - - model = FunctionModel(function=filtered_response, model_name='test-model') - agent = Agent(model) - - with pytest.raises(ContentFilterError, match="Content filter triggered. Finish reason: 'content_filter'"): - await agent.run('Trigger filter') - - -async def test_central_content_filter_with_partial_content(): - """ - Test that the agent graph returns partial content (does not raise exception) - even if finish_reason='content_filter', provided parts are not empty. - """ - - async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: - return ModelResponse( - parts=[TextPart('Partially generated content...')], model_name='test-model', finish_reason='content_filter' - ) - - model = FunctionModel(function=filtered_response, model_name='test-model') - agent = Agent(model) - - # Should NOT raise ContentFilterError - result = await agent.run('Trigger filter') - assert result.output == 'Partially generated content...' - - -async def test_content_filter_serialization_fallback(): - """ - Test fallback to str() when ModelResponse cannot be serialized to JSON. - Covers the except block in _agent_graph.py. - """ - - class UnserializableObj: - def __repr__(self): - return '' - - # Create a response that will fail JSON serialization - response = ModelResponse( - parts=[], - model_name='test-model', - finish_reason='content_filter', - # Inject unserializable object into provider_details to break dumping - provider_details={'bad_data': UnserializableObj()}, - ) - - async def bad_response_model(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: - return response - - model = FunctionModel(function=bad_response_model, model_name='test-model') - agent = Agent(model) - - with pytest.raises(ContentFilterError) as exc_info: - await agent.run('trigger') - - # Verify it fell back to str() representation - assert '' in str(exc_info.value.body) diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 39320d876f..2599f3cb21 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3569,7 +3569,12 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: 'message': 'The content was filtered.', 'innererror': { 'code': 'ResponsibleAIPolicyViolation', - 'content_filter_result': {'hate': {'filtered': True, 'severity': 'high'}}, + 'content_filter_result': { + 'hate': {'filtered': True, 'severity': 'high'}, + 'self_harm': {'filtered': False, 'severity': 'safe'}, + 'sexual': {'filtered': False, 'severity': 'safe'}, + 'violence': {'filtered': False, 'severity': 'medium'}, + }, }, } } @@ -3585,13 +3590,47 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: m = OpenAIChatModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) agent = Agent(m) - try: + with pytest.raises( + ContentFilterError, match=r"Content filter triggered. Finish reason: 'content_filter'" + ) as exc_info: agent.run_sync('bad prompt') - except ContentFilterError as e: - # Check that the exception was raised - assert "Content filter triggered. Finish reason: 'content_filter'" in str(e) - # Verify body contains the inner error details (serialized) - assert 'ResponsibleAIPolicyViolation' in str(e.body) + + assert exc_info.value.body is not None + assert json.loads(exc_info.value.body) == snapshot( + [ + { + 'parts': [], + 'usage': { + 'input_tokens': 0, + 'cache_write_tokens': 0, + 'cache_read_tokens': 0, + 'output_tokens': 0, + 'input_audio_tokens': 0, + 'cache_audio_read_tokens': 0, + 'output_audio_tokens': 0, + 'details': {}, + }, + 'model_name': 'gpt-5-mini', + 'timestamp': IsStr(), + 'kind': 'response', + 'provider_name': 'azure', + 'provider_url': None, + 'provider_details': { + 'finish_reason': 'content_filter', + 'content_filter_result': { + 'hate': {'filtered': True, 'severity': 'high'}, + 'self_harm': {'filtered': False, 'severity': 'safe'}, + 'sexual': {'filtered': False, 'severity': 'safe'}, + 'violence': {'filtered': False, 'severity': 'medium'}, + }, + }, + 'provider_response_id': None, + 'finish_reason': 'content_filter', + 'run_id': IsStr(), + 'metadata': None, + } + ] + ) def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None: diff --git a/tests/test_agent.py b/tests/test_agent.py index 6ce2d91c54..8fed2154dc 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -65,6 +65,7 @@ WebSearchTool, WebSearchUserLocation, ) +from pydantic_ai.exceptions import ContentFilterError from pydantic_ai.models.function import AgentInfo, DeltaToolCall, DeltaToolCalls, FunctionModel from pydantic_ai.models.test import TestModel from pydantic_ai.output import OutputObjectDefinition, StructuredDict, ToolOutput @@ -7165,3 +7166,75 @@ async def test_dynamic_tool_in_run_call(): assert isinstance(tool, WebSearchTool) assert tool.user_location is not None assert tool.user_location.get('city') == 'Berlin' + + +async def test_central_content_filter_handling(): + """ + Test that the agent graph correctly raises ContentFilterError + when a model returns finish_reason='content_filter' AND empty content. + """ + + async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + return ModelResponse( + parts=[], + model_name='test-model', + finish_reason='content_filter', + provider_details={'finish_reason': 'content_filter'}, + ) + + model = FunctionModel(function=filtered_response, model_name='test-model') + agent = Agent(model) + + with pytest.raises(ContentFilterError, match="Content filter triggered. Finish reason: 'content_filter'"): + await agent.run('Trigger filter') + + +async def test_central_content_filter_with_partial_content(): + """ + Test that the agent graph returns partial content (does not raise exception) + even if finish_reason='content_filter', provided parts are not empty. + """ + + async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + return ModelResponse( + parts=[TextPart('Partially generated content...')], model_name='test-model', finish_reason='content_filter' + ) + + model = FunctionModel(function=filtered_response, model_name='test-model') + agent = Agent(model) + + # Should NOT raise ContentFilterError + result = await agent.run('Trigger filter') + assert result.output == 'Partially generated content...' + + +async def test_content_filter_serialization_fallback(): + """ + Test fallback to str() when ModelResponse cannot be serialized to JSON. + Covers the except block in _agent_graph.py. + """ + + class UnserializableObj: + def __repr__(self): + return '' + + # Create a response that will fail JSON serialization + response = ModelResponse( + parts=[], + model_name='test-model', + finish_reason='content_filter', + # Inject unserializable object into provider_details to break dumping + provider_details={'bad_data': UnserializableObj()}, + ) + + async def bad_response_model(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + return response + + model = FunctionModel(function=bad_response_model, model_name='test-model') + agent = Agent(model) + + with pytest.raises(ContentFilterError) as exc_info: + await agent.run('trigger') + + # Verify it fell back to str() representation + assert '' in str(exc_info.value.body) From a315a548b375733270502fc773c5e725738e1d7c Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Fri, 19 Dec 2025 16:19:40 +0000 Subject: [PATCH 19/20] improvements --- tests/models/test_google.py | 67 ------------------------------------- 1 file changed, 67 deletions(-) diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 5e9f5e6e15..9fa7ae521d 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -5032,73 +5032,6 @@ async def test_google_system_prompts_and_instructions_ordering(google_provider: assert contents == snapshot([{'role': 'user', 'parts': [{'text': 'Hello'}]}]) -async def test_google_stream_empty_chunk( - allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture -): - """Test that empty chunks in the stream are ignored.""" - model_name = 'gemini-2.5-flash' - model = GoogleModel(model_name, provider=google_provider) - - # Chunk with NO content - empty_candidate = mocker.Mock(finish_reason=None, content=None) - empty_candidate.grounding_metadata = None - empty_candidate.url_context_metadata = None - # FIX: Set safety_ratings to None to avoid iteration error - empty_candidate.safety_ratings = None - - chunk_empty = mocker.Mock( - candidates=[empty_candidate], - model_version=model_name, - usage_metadata=None, - create_time=datetime.datetime.now(), - response_id='resp_empty', - ) - chunk_empty.model_dump_json.return_value = '{}' - - # Chunk WITH content (valid) - part_mock = mocker.Mock( - text='Hello', - thought=False, - function_call=None, - inline_data=None, - executable_code=None, - code_execution_result=None, - ) - part_mock.thought_signature = None - - valid_candidate = mocker.Mock( - finish_reason=GoogleFinishReason.STOP, - content=mocker.Mock(parts=[part_mock]), - grounding_metadata=None, - url_context_metadata=None, - ) - # FIX: Set safety_ratings to None - valid_candidate.safety_ratings = None - - chunk_valid = mocker.Mock( - candidates=[valid_candidate], - model_version=model_name, - usage_metadata=None, - create_time=datetime.datetime.now(), - response_id='resp_valid', - ) - chunk_valid.model_dump_json.return_value = '{"content": "Hello"}' - - async def stream_iterator(): - yield chunk_empty - yield chunk_valid - - mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) - - agent = Agent(model=model) - - # Use run_stream to hit the mocked method - async with agent.run_stream('hello') as result: - output = await result.get_output() - - assert output == 'Hello' - - async def test_google_stream_safety_filter( allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture ): From 3e15bf4e4684f2854fa8bc3f72cb6dee37c94155 Mon Sep 17 00:00:00 2001 From: AlanPonnachan Date: Sat, 20 Dec 2025 09:18:11 +0000 Subject: [PATCH 20/20] improvements --- pydantic_ai_slim/pydantic_ai/_agent_graph.py | 5 +- pydantic_ai_slim/pydantic_ai/models/openai.py | 13 ++-- tests/models/test_google.py | 73 ++++--------------- tests/models/test_openai.py | 34 ++------- tests/test_agent.py | 32 -------- 5 files changed, 28 insertions(+), 129 deletions(-) diff --git a/pydantic_ai_slim/pydantic_ai/_agent_graph.py b/pydantic_ai_slim/pydantic_ai/_agent_graph.py index 6e62683ebf..ced731eb36 100644 --- a/pydantic_ai_slim/pydantic_ai/_agent_graph.py +++ b/pydantic_ai_slim/pydantic_ai/_agent_graph.py @@ -608,10 +608,7 @@ async def _run_stream() -> AsyncIterator[_messages.HandleResponseEvent]: # noqa details = self.model_response.provider_details or {} reason = details.get('finish_reason', 'content_filter') - try: - body = _messages.ModelMessagesTypeAdapter.dump_json([self.model_response]).decode() - except Exception: - body = str(self.model_response) + body = _messages.ModelMessagesTypeAdapter.dump_json([self.model_response]).decode() raise exceptions.ContentFilterError( f"Content filter triggered. Finish reason: '{reason}'", body=body diff --git a/pydantic_ai_slim/pydantic_ai/models/openai.py b/pydantic_ai_slim/pydantic_ai/models/openai.py index df4f1f3691..e729e91325 100644 --- a/pydantic_ai_slim/pydantic_ai/models/openai.py +++ b/pydantic_ai_slim/pydantic_ai/models/openai.py @@ -166,7 +166,8 @@ class _AzureContentFilterResultDetail(BaseModel): filtered: bool - severity: str + severity: str | None = None + detected: bool | None = None class _AzureContentFilterResult(BaseModel): @@ -174,7 +175,8 @@ class _AzureContentFilterResult(BaseModel): self_harm: _AzureContentFilterResultDetail | None = None sexual: _AzureContentFilterResultDetail | None = None violence: _AzureContentFilterResultDetail | None = None - jailbreak: dict[str, Any] | None = None + jailbreak: _AzureContentFilterResultDetail | None = None + profanity: _AzureContentFilterResultDetail | None = None class _AzureInnerError(BaseModel): @@ -224,6 +226,7 @@ def _resolve_openai_image_generation_size( def _check_azure_content_filter(e: APIStatusError, system: str, model_name: str) -> ModelResponse | None: """Check if the error is an Azure content filter error.""" + # Assign to Any to avoid 'dict[Unknown, Unknown]' inference in strict mode body_any: Any = e.body if system == 'azure' and e.status_code == 400 and isinstance(body_any, dict): @@ -724,11 +727,7 @@ def _process_provider_details(self, response: chat.ChatCompletion) -> dict[str, This method may be overridden by subclasses of `OpenAIChatModel` to apply custom mappings. """ - details = _map_provider_details(response.choices[0]) - - if hasattr(response, 'azure_content_filter_result'): - details['content_filter_result'] = getattr(response, 'azure_content_filter_result') - return details + return _map_provider_details(response.choices[0]) def _process_response(self, response: chat.ChatCompletion | str) -> ModelResponse: """Process a non-streamed response, and prepare a message to return.""" diff --git a/tests/models/test_google.py b/tests/models/test_google.py index 9fa7ae521d..5cfdc5ad21 100644 --- a/tests/models/test_google.py +++ b/tests/models/test_google.py @@ -1008,9 +1008,23 @@ async def test_google_model_safety_settings(allow_model_requests: None, google_p with pytest.raises( ContentFilterError, match="Content filter triggered. Finish reason: 'SAFETY'", - ): + ) as exc_info: await agent.run('Tell me a joke about a Brazilians.') + # Verify that we captured the safety settings in the exception body + assert exc_info.value.body is not None + body_json = json.loads(exc_info.value.body) + assert len(body_json) == 1 + response_msg = body_json[0] + + assert response_msg['finish_reason'] == 'content_filter' + details = response_msg['provider_details'] + assert details['finish_reason'] == 'SAFETY' + assert len(details['safety_ratings']) > 0 + # The first rating should reflect the blocking + assert details['safety_ratings'][0]['category'] == 'HARM_CATEGORY_HATE_SPEECH' + assert details['safety_ratings'][0]['blocked'] is True + async def test_google_model_web_search_tool(allow_model_requests: None, google_provider: GoogleProvider): m = GoogleModel('gemini-2.5-pro', provider=google_provider) @@ -5030,60 +5044,3 @@ async def test_google_system_prompts_and_instructions_ordering(google_provider: } ) assert contents == snapshot([{'role': 'user', 'parts': [{'text': 'Hello'}]}]) - - -async def test_google_stream_safety_filter( - allow_model_requests: None, google_provider: GoogleProvider, mocker: MockerFixture -): - """Test that safety ratings are captured in the exception body when streaming.""" - model_name = 'gemini-2.5-flash' - model = GoogleModel(model_name, provider=google_provider) - - # Mock safety ratings - safety_rating = mocker.Mock(category='HARM_CATEGORY_HATE_SPEECH', probability='HIGH', blocked=True) - # Configure model_dump - safety_rating.model_dump.return_value = { - 'category': 'HARM_CATEGORY_HATE_SPEECH', - 'probability': 'HIGH', - 'blocked': True, - } - - candidate = mocker.Mock( - finish_reason=GoogleFinishReason.SAFETY, - content=None, - safety_ratings=[safety_rating], - # Ensure these are None to avoid iteration errors - grounding_metadata=None, - url_context_metadata=None, - ) - - chunk = mocker.Mock( - candidates=[candidate], - model_version=model_name, - usage_metadata=None, - create_time=datetime.datetime.now(), - response_id='resp_123', # Set string ID - ) - chunk.model_dump_json.return_value = '{"mock": "json"}' - - async def stream_iterator(): - yield chunk - - mocker.patch.object(model.client.aio.models, 'generate_content_stream', return_value=stream_iterator()) - - agent = Agent(model=model) - - with pytest.raises(ContentFilterError) as exc_info: - async with agent.run_stream('bad content'): - pass - - # Verify exception message - assert 'Content filter triggered' in str(exc_info.value) - - # Verify safety ratings are present in the body (serialized ModelResponse) - assert exc_info.value.body is not None - body_json = json.loads(exc_info.value.body) - - response_msg = body_json[0] - assert response_msg['provider_details']['finish_reason'] == 'SAFETY' - assert response_msg['provider_details']['safety_ratings'][0]['category'] == 'HARM_CATEGORY_HATE_SPEECH' diff --git a/tests/models/test_openai.py b/tests/models/test_openai.py index 2599f3cb21..7228435db0 100644 --- a/tests/models/test_openai.py +++ b/tests/models/test_openai.py @@ -3574,6 +3574,8 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'medium'}, + 'jailbreak': {'filtered': False, 'detected': False}, + 'profanity': {'filtered': False, 'detected': True}, # Added profanity }, }, } @@ -3596,6 +3598,7 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: agent.run_sync('bad prompt') assert exc_info.value.body is not None + assert json.loads(exc_info.value.body) == snapshot( [ { @@ -3622,6 +3625,8 @@ def test_azure_prompt_filter_error(allow_model_requests: None) -> None: 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'medium'}, + 'jailbreak': {'filtered': False, 'detected': False}, + 'profanity': {'filtered': False, 'detected': True}, }, }, 'provider_response_id': None, @@ -3649,7 +3654,7 @@ def test_responses_azure_prompt_filter_error(allow_model_requests: None) -> None agent.run_sync('bad prompt') -async def test_openai_response_filter_error_sync(allow_model_requests: None): +async def test_openai_response_filter_error(allow_model_requests: None): c = completion_message( ChatCompletionMessage(content=None, role='assistant'), ) @@ -3734,33 +3739,6 @@ def test_azure_400_malformed_error(allow_model_requests: None) -> None: assert exc_info.value.status_code == 400 -def test_azure_prompt_filter_no_inner_error(allow_model_requests: None) -> None: - """Test Azure content filter that lacks the innererror details.""" - # Valid content filter code, but missing 'innererror' - body = { - 'error': { - 'code': 'content_filter', - 'message': 'The content was filtered.', - # No innererror - } - } - - mock_client = MockOpenAI.create_mock( - APIStatusError( - 'content filter', - response=httpx.Response(status_code=400, request=httpx.Request('POST', 'https://example.com/v1')), - body=body, - ) - ) - - m = OpenAIChatModel('gpt-5-mini', provider=AzureProvider(openai_client=cast(AsyncAzureOpenAI, mock_client))) - agent = Agent(m) - - # Should still raise ContentFilterError - with pytest.raises(ContentFilterError, match=r"Content filter triggered. Finish reason: 'content_filter'"): - agent.run_sync('bad prompt') - - async def test_openai_chat_instructions_after_system_prompts(allow_model_requests: None): """Test that instructions are inserted after all system prompts in mapped messages.""" mock_client = MockOpenAI.create_mock(completion_message(ChatCompletionMessage(content='ok', role='assistant'))) diff --git a/tests/test_agent.py b/tests/test_agent.py index 8fed2154dc..c3bdae376f 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -7206,35 +7206,3 @@ async def filtered_response(messages: list[ModelMessage], info: AgentInfo) -> Mo # Should NOT raise ContentFilterError result = await agent.run('Trigger filter') assert result.output == 'Partially generated content...' - - -async def test_content_filter_serialization_fallback(): - """ - Test fallback to str() when ModelResponse cannot be serialized to JSON. - Covers the except block in _agent_graph.py. - """ - - class UnserializableObj: - def __repr__(self): - return '' - - # Create a response that will fail JSON serialization - response = ModelResponse( - parts=[], - model_name='test-model', - finish_reason='content_filter', - # Inject unserializable object into provider_details to break dumping - provider_details={'bad_data': UnserializableObj()}, - ) - - async def bad_response_model(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: - return response - - model = FunctionModel(function=bad_response_model, model_name='test-model') - agent = Agent(model) - - with pytest.raises(ContentFilterError) as exc_info: - await agent.run('trigger') - - # Verify it fell back to str() representation - assert '' in str(exc_info.value.body)