From 9f3ebddb2375c0c494a61fb81591693c93797812 Mon Sep 17 00:00:00 2001 From: Vojtech Rinik Date: Fri, 5 Dec 2025 08:51:30 +0100 Subject: [PATCH 1/6] Add structured output support for Anthropic provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements structured outputs using Anthropic's structured-outputs-2025-11-13 beta API. Changes: - Add structured output capability detection for Claude 4+ models - Implement output_format parameter with json_schema type in chat payload - Add anthropic-beta header handling to append structured-outputs beta version - Add comprehensive specs for structured output functionality - Refactor model version detection helpers (claude3_or_newer, claude4_or_newer) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- lib/ruby_llm/providers/anthropic.rb | 19 ++++++ .../providers/anthropic/capabilities.rb | 23 +++++-- lib/ruby_llm/providers/anthropic/chat.rb | 14 ++++- .../ruby_llm/providers/anthropic/chat_spec.rb | 62 +++++++++++++++++++ 4 files changed, 111 insertions(+), 7 deletions(-) diff --git a/lib/ruby_llm/providers/anthropic.rb b/lib/ruby_llm/providers/anthropic.rb index cd7d38055..29e18d008 100644 --- a/lib/ruby_llm/providers/anthropic.rb +++ b/lib/ruby_llm/providers/anthropic.rb @@ -11,6 +11,8 @@ class Anthropic < Provider include Anthropic::Streaming include Anthropic::Tools + STRUCTURED_OUTPUTS_BETA = 'structured-outputs-2025-11-13' + def api_base 'https://api.anthropic.com' end @@ -22,6 +24,23 @@ def headers } end + def complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, &block) # rubocop:disable Metrics/ParameterLists + headers = add_structured_output_beta_header(headers) if schema + super + end + + private + + def add_structured_output_beta_header(headers) + existing_beta = headers['anthropic-beta'] + new_beta = if existing_beta + "#{existing_beta},#{STRUCTURED_OUTPUTS_BETA}" + else + STRUCTURED_OUTPUTS_BETA + end + headers.merge('anthropic-beta' => new_beta) + end + class << self def capabilities Anthropic::Capabilities diff --git a/lib/ruby_llm/providers/anthropic/capabilities.rb b/lib/ruby_llm/providers/anthropic/capabilities.rb index 710b55386..5f0699608 100644 --- a/lib/ruby_llm/providers/anthropic/capabilities.rb +++ b/lib/ruby_llm/providers/anthropic/capabilities.rb @@ -31,11 +31,25 @@ def supports_vision?(model_id) end def supports_functions?(model_id) - model_id.match?(/claude-3/) + claude3_or_newer?(model_id) end def supports_json_mode?(model_id) - model_id.match?(/claude-3/) + claude3_or_newer?(model_id) + end + + def supports_structured_output?(model_id) + # Structured outputs supported on Claude 4+ (Sonnet 4.5, Opus 4.1, etc.) + # https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs + claude4_or_newer?(model_id) + end + + def claude3_or_newer?(model_id) + model_id.match?(/claude-[34]|claude-(sonnet|opus|haiku)-[4-9]/) + end + + def claude4_or_newer?(model_id) + model_id.match?(/claude-[4-9]|claude-(sonnet|opus|haiku)-[4-9]/) end def supports_extended_thinking?(model_id) @@ -92,12 +106,13 @@ def modalities_for(model_id) def capabilities_for(model_id) capabilities = ['streaming'] - if model_id.match?(/claude-3/) + if claude3_or_newer?(model_id) capabilities << 'function_calling' capabilities << 'batch' end - capabilities << 'reasoning' if model_id.match?(/claude-3-7|-4/) + capabilities << 'structured_output' if supports_structured_output?(model_id) + capabilities << 'reasoning' if model_id.match?(/claude-3-7|claude-[4-9]|claude-(sonnet|opus)-[4-9]/) capabilities << 'citations' if model_id.match?(/claude-3\.5|claude-3-7/) capabilities end diff --git a/lib/ruby_llm/providers/anthropic/chat.rb b/lib/ruby_llm/providers/anthropic/chat.rb index cbc9f1161..11db616b8 100644 --- a/lib/ruby_llm/providers/anthropic/chat.rb +++ b/lib/ruby_llm/providers/anthropic/chat.rb @@ -11,12 +11,12 @@ def completion_url '/v1/messages' end - def render_payload(messages, tools:, temperature:, model:, stream: false, schema: nil) # rubocop:disable Metrics/ParameterLists,Lint/UnusedMethodArgument + def render_payload(messages, tools:, temperature:, model:, stream: false, schema: nil) # rubocop:disable Metrics/ParameterLists system_messages, chat_messages = separate_messages(messages) system_content = build_system_content(system_messages) build_base_payload(chat_messages, model, stream).tap do |payload| - add_optional_fields(payload, system_content:, tools:, temperature:) + add_optional_fields(payload, system_content:, tools:, temperature:, schema:) end end @@ -54,10 +54,18 @@ def build_base_payload(chat_messages, model, stream) } end - def add_optional_fields(payload, system_content:, tools:, temperature:) + def add_optional_fields(payload, system_content:, tools:, temperature:, schema: nil) payload[:tools] = tools.values.map { |t| Tools.function_for(t) } if tools.any? payload[:system] = system_content unless system_content.empty? payload[:temperature] = temperature unless temperature.nil? + add_output_format(payload, schema) if schema + end + + def add_output_format(payload, schema) + payload[:output_format] = { + type: 'json_schema', + schema: schema + } end def parse_completion_response(response) diff --git a/spec/ruby_llm/providers/anthropic/chat_spec.rb b/spec/ruby_llm/providers/anthropic/chat_spec.rb index 5a3038602..5a9442426 100644 --- a/spec/ruby_llm/providers/anthropic/chat_spec.rb +++ b/spec/ruby_llm/providers/anthropic/chat_spec.rb @@ -2,6 +2,27 @@ require 'spec_helper' +RSpec.describe RubyLLM::Providers::Anthropic do + include_context 'with configured RubyLLM' + + describe '#complete with structured outputs' do + let(:provider) { described_class.new(RubyLLM.config) } + + describe '#add_structured_output_beta_header' do + it 'adds beta header when schema is provided' do + headers = provider.send(:add_structured_output_beta_header, {}) + expect(headers['anthropic-beta']).to eq('structured-outputs-2025-11-13') + end + + it 'appends to existing beta header' do + existing_headers = { 'anthropic-beta' => 'existing-beta' } + headers = provider.send(:add_structured_output_beta_header, existing_headers) + expect(headers['anthropic-beta']).to eq('existing-beta,structured-outputs-2025-11-13') + end + end + end +end + RSpec.describe RubyLLM::Providers::Anthropic::Chat do describe '.render_payload' do let(:model) { instance_double(RubyLLM::Model::Info, id: 'claude-sonnet-4-5', max_tokens: nil) } @@ -27,6 +48,47 @@ expect(payload[:system]).to eq(system_raw.value) expect(payload[:messages].first[:content]).to eq([{ type: 'text', text: 'Hello there' }]) end + + it 'includes output_format when schema is provided' do + user_message = RubyLLM::Message.new(role: :user, content: 'Hello') + schema = { + type: 'object', + properties: { + name: { type: 'string' } + }, + required: ['name'], + additionalProperties: false + } + + payload = described_class.render_payload( + [user_message], + tools: {}, + temperature: nil, + model: model, + stream: false, + schema: schema + ) + + expect(payload[:output_format]).to eq({ + type: 'json_schema', + schema: schema + }) + end + + it 'does not include output_format when schema is nil' do + user_message = RubyLLM::Message.new(role: :user, content: 'Hello') + + payload = described_class.render_payload( + [user_message], + tools: {}, + temperature: nil, + model: model, + stream: false, + schema: nil + ) + + expect(payload).not_to have_key(:output_format) + end end describe '.parse_completion_response' do From 5ddd881bf4e4154fc90b6c63d798997241f2e7ba Mon Sep 17 00:00:00 2001 From: Vojtech Rinik Date: Fri, 5 Dec 2025 09:37:56 +0100 Subject: [PATCH 2/6] Extract SchemaValidator class from Anthropic chat module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move schema validation logic into dedicated class to reduce complexity in the Chat module and improve separation of concerns. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- lib/ruby_llm/providers/anthropic/chat.rb | 1 + .../providers/anthropic/schema_validator.rb | 71 +++++++++++ .../anthropic/schema_validator_spec.rb | 120 ++++++++++++++++++ 3 files changed, 192 insertions(+) create mode 100644 lib/ruby_llm/providers/anthropic/schema_validator.rb create mode 100644 spec/ruby_llm/providers/anthropic/schema_validator_spec.rb diff --git a/lib/ruby_llm/providers/anthropic/chat.rb b/lib/ruby_llm/providers/anthropic/chat.rb index 11db616b8..f2b41721a 100644 --- a/lib/ruby_llm/providers/anthropic/chat.rb +++ b/lib/ruby_llm/providers/anthropic/chat.rb @@ -62,6 +62,7 @@ def add_optional_fields(payload, system_content:, tools:, temperature:, schema: end def add_output_format(payload, schema) + SchemaValidator.new(schema).validate! payload[:output_format] = { type: 'json_schema', schema: schema diff --git a/lib/ruby_llm/providers/anthropic/schema_validator.rb b/lib/ruby_llm/providers/anthropic/schema_validator.rb new file mode 100644 index 000000000..abc57fece --- /dev/null +++ b/lib/ruby_llm/providers/anthropic/schema_validator.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class Anthropic + # Validates JSON schemas for Anthropic structured outputs + class SchemaValidator + def initialize(schema) + @schema = schema + end + + def validate! + validate_node(@schema, 'schema') + end + + private + + def validate_node(schema, path) + return unless schema.is_a?(Hash) + + validate_object_schema(schema, path) + validate_properties(schema, path) + validate_items(schema, path) + validate_combinators(schema, path) + end + + def validate_object_schema(schema, path) + return unless value(schema, :type) == 'object' && value(schema, :additionalProperties) != false + + raise ArgumentError, + "#{path}: Object schemas must set 'additionalProperties' to false for Anthropic structured outputs." + end + + def validate_properties(schema, path) + properties = value(schema, :properties) + return unless properties.is_a?(Hash) + + properties.each do |key, prop_schema| + validate_node(prop_schema, "#{path}.properties.#{key}") + end + end + + def validate_items(schema, path) + items = value(schema, :items) + validate_node(items, "#{path}.items") if items + end + + def validate_combinators(schema, path) + %i[anyOf oneOf allOf].each do |keyword| + schemas = value(schema, keyword) + next unless schemas.is_a?(Array) + + schemas.each_with_index do |sub_schema, index| + validate_node(sub_schema, "#{path}.#{keyword}[#{index}]") + end + end + end + + def value(schema, key) + return unless schema.is_a?(Hash) + + symbol_key = key.is_a?(Symbol) ? key : key.to_sym + return schema[symbol_key] if schema.key?(symbol_key) + + string_key = key.to_s + schema[string_key] if schema.key?(string_key) + end + end + end + end +end diff --git a/spec/ruby_llm/providers/anthropic/schema_validator_spec.rb b/spec/ruby_llm/providers/anthropic/schema_validator_spec.rb new file mode 100644 index 000000000..2d980f8f4 --- /dev/null +++ b/spec/ruby_llm/providers/anthropic/schema_validator_spec.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe RubyLLM::Providers::Anthropic::SchemaValidator do + describe '#validate!' do + subject(:validate!) { described_class.new(schema).validate! } + + context 'with valid schemas' do + let(:schema) { { type: 'object', properties: { name: { type: 'string' } }, additionalProperties: false } } + + it 'accepts schema with additionalProperties: false' do + expect { validate! }.not_to raise_error + end + end + + context 'with string keys' do + let(:schema) do + { + 'type' => 'object', + 'properties' => { 'name' => { 'type' => 'string' } }, + 'additionalProperties' => false + } + end + + it 'accepts schema with string keys' do + expect { validate! }.not_to raise_error + end + end + + context 'with missing additionalProperties' do + let(:schema) { { type: 'object', properties: { name: { type: 'string' } } } } + + it 'rejects schema' do + expect { validate! }.to raise_error(ArgumentError, /additionalProperties.*to false/) + end + end + + context 'with nested object missing additionalProperties' do + let(:schema) do + { + type: 'object', + properties: { user: { type: 'object', properties: { name: { type: 'string' } } } }, + additionalProperties: false + } + end + + it 'rejects schema and reports path' do + expect { validate! }.to raise_error(ArgumentError, /schema\.properties\.user/) + end + end + + context 'with array items containing objects' do + let(:schema) do + { + type: 'object', + properties: { + users: { + type: 'array', + items: { type: 'object', properties: { name: { type: 'string' } } } + } + }, + additionalProperties: false + } + end + + it 'rejects invalid nested array items' do + expect { validate! }.to raise_error(ArgumentError, /schema\.properties\.users\.items/) + end + end + + context 'with valid array items' do + let(:schema) do + { + type: 'object', + properties: { + users: { + type: 'array', + items: { type: 'object', properties: { name: { type: 'string' } }, additionalProperties: false } + } + }, + additionalProperties: false + } + end + + it 'accepts valid nested array items' do + expect { validate! }.not_to raise_error + end + end + + context 'with anyOf containing invalid object' do + let(:schema) do + { + type: 'object', + properties: { + value: { + anyOf: [ + { type: 'string' }, + { type: 'object', properties: { id: { type: 'integer' } } } + ] + } + }, + additionalProperties: false + } + end + + it 'rejects invalid object in anyOf' do + expect { validate! }.to raise_error(ArgumentError, /schema\.properties\.value\.anyOf\[1\]/) + end + end + + context 'with non-object types' do + let(:schema) { { type: 'string' } } + + it 'accepts non-object schemas without additionalProperties' do + expect { validate! }.not_to raise_error + end + end + end +end From cdea7947b00e7da16b4f1173bdc07bf72bcfee24 Mon Sep 17 00:00:00 2001 From: Vojtech Rinik Date: Fri, 5 Dec 2025 09:46:56 +0100 Subject: [PATCH 3/6] adding a comment to schema validator --- .../providers/anthropic/schema_validator.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/ruby_llm/providers/anthropic/schema_validator.rb b/lib/ruby_llm/providers/anthropic/schema_validator.rb index abc57fece..35774bbb9 100644 --- a/lib/ruby_llm/providers/anthropic/schema_validator.rb +++ b/lib/ruby_llm/providers/anthropic/schema_validator.rb @@ -3,7 +3,20 @@ module RubyLLM module Providers class Anthropic - # Validates JSON schemas for Anthropic structured outputs + # Validates JSON schemas for Anthropic structured outputs. + # + # Anthropic requires all object schemas to have `additionalProperties: false`. + # This constraint ensures the model produces deterministic, well-defined JSON + # without unexpected extra fields. The API will reject schemas that don't meet + # this requirement. + # + # This validator recursively checks all objects in the schema, including: + # - Top-level object schemas + # - Nested objects in `properties` + # - Objects in array `items` + # - Objects in `anyOf`, `oneOf`, and `allOf` combinators + # + # @see https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs class SchemaValidator def initialize(schema) @schema = schema From 2f0f6c3aed80be8c4860c863e35c8099d7f10f64 Mon Sep 17 00:00:00 2001 From: Vojtech Rinik Date: Fri, 5 Dec 2025 19:13:59 +0100 Subject: [PATCH 4/6] Disable RSpec/MultipleDescribes cop for Anthropic chat spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec file legitimately tests two separate classes (Provider and Chat module) that are closely related but have distinct responsibilities. Disabling this cop for this file is the appropriate solution. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- spec/ruby_llm/providers/anthropic/chat_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/ruby_llm/providers/anthropic/chat_spec.rb b/spec/ruby_llm/providers/anthropic/chat_spec.rb index 5a9442426..9f1b25caa 100644 --- a/spec/ruby_llm/providers/anthropic/chat_spec.rb +++ b/spec/ruby_llm/providers/anthropic/chat_spec.rb @@ -2,6 +2,7 @@ require 'spec_helper' +# rubocop:disable RSpec/MultipleDescribes RSpec.describe RubyLLM::Providers::Anthropic do include_context 'with configured RubyLLM' @@ -115,3 +116,4 @@ end end end +# rubocop:enable RSpec/MultipleDescribes From f853ac67e18cb7d08610ee5a458893139aeed6f1 Mon Sep 17 00:00:00 2001 From: Vojtech Rinik Date: Mon, 8 Dec 2025 15:19:09 +0100 Subject: [PATCH 5/6] vcr for anthropic structured output --- lib/ruby_llm/models.json | 3 +- ...n_schema_and_returns_structured_output.yml | 84 ++++++++ ...oving_schema_with_nil_mid-conversation.yml | 180 ++++++++++++++++++ spec/ruby_llm/chat_schema_spec.rb | 5 +- spec/support/models_to_test.rb | 1 + 5 files changed, 269 insertions(+), 4 deletions(-) create mode 100644 spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_accepts_a_json_schema_and_returns_structured_output.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_allows_removing_schema_with_nil_mid-conversation.yml diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index 2e6033e17..355250e4e 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -174,7 +174,8 @@ ] }, "capabilities": [ - "function_calling" + "function_calling", + "structured_output" ], "pricing": { "text_tokens": { diff --git a/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_accepts_a_json_schema_and_returns_structured_output.yml b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_accepts_a_json_schema_and_returns_structured_output.yml new file mode 100644 index 000000000..e1314725a --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_accepts_a_json_schema_and_returns_structured_output.yml @@ -0,0 +1,84 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-haiku-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named John who is 30 years old"}]}],"stream":false,"max_tokens":64000,"output_format":{"type":"json_schema","schema":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}}}' + headers: + Anthropic-Beta: + - structured-outputs-2025-11-13 + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:13:56 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:13:56Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:13:56Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:13:55Z' + Retry-After: + - '4' + Anthropic-Ratelimit-Tokens-Limit: + - '4800000' + Anthropic-Ratelimit-Tokens-Remaining: + - '4800000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:13:56Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '1276' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01Pqutax4swVnZUJ1oeUD86c","type":"message","role":"assistant","content":[{"type":"text","text":"{\"name\":\"John\",\"age\":30}"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":181,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:13:56 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_allows_removing_schema_with_nil_mid-conversation.yml b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_allows_removing_schema_with_nil_mid-conversation.yml new file mode 100644 index 000000000..7317c10a7 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_allows_removing_schema_with_nil_mid-conversation.yml @@ -0,0 +1,180 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-haiku-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named Bob"}]}],"stream":false,"max_tokens":64000,"output_format":{"type":"json_schema","schema":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}}}' + headers: + Anthropic-Beta: + - structured-outputs-2025-11-13 + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:13:57 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:13:57Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:13:57Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:13:56Z' + Retry-After: + - '3' + Anthropic-Ratelimit-Tokens-Limit: + - '4800000' + Anthropic-Ratelimit-Tokens-Remaining: + - '4800000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:13:57Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '1075' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01GPp3TzjpRktd5iCu9sFUAz","type":"message","role":"assistant","content":[{"type":"text","text":"{\"name\":\"Bob\",\"age\":30}"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":174,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:13:57 GMT +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-haiku-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named Bob"}]},{"role":"assistant","content":[{"type":"text","text":"{\"name\":\"Bob\",\"age\":30}"}]},{"role":"user","content":[{"type":"text","text":"Now + just tell me about Ruby"}]}],"stream":false,"max_tokens":64000}' + headers: + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:14:00 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:13:58Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:14:00Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:13:58Z' + Retry-After: + - '2' + Anthropic-Ratelimit-Tokens-Limit: + - '4800000' + Anthropic-Ratelimit-Tokens-Remaining: + - '4800000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:13:58Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '2593' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01Kj3FFsWWTK5VGSTy5r35FJ","type":"message","role":"assistant","content":[{"type":"text","text":"Ruby + is a dynamic, interpreted programming language known for its simplicity and + productivity. Here are some key points about it:\n\n**Key Characteristics:**\n- + **Syntax**: Clean, readable, and beginner-friendly\n- **Type System**: Dynamically + typed\n- **Paradigm**: Object-oriented with functional programming features\n- + **Philosophy**: \"Principle of Least Surprise\" - the language behaves in + ways that minimize confusion\n\n**Common Uses:**\n- Web development (especially + with Ruby on Rails framework)\n- Scripting and automation\n- Data processing\n- + DevOps tools\n\n**Popular Frameworks:**\n- **Ruby on Rails**: The most popular + web framework\n- Sinatra: Lightweight web framework\n- Jekyll: Static site + generator\n\n**Strengths:**\n- Rapid development and prototyping\n- Large, + active community\n- Extensive libraries (gems)\n- Great for startups and MVPs\n\n**Weaknesses:**\n- + Slower execution speed compared to compiled languages\n- Higher memory consumption\n- + Less suitable for performance-critical applications\n\nRuby was created by + Yukihiro Matsumoto in 1995 and remains popular for web development, though + its market share has shifted somewhat with the rise of JavaScript/Node.js + and Python."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":33,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":277,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:14:00 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/chat_schema_spec.rb b/spec/ruby_llm/chat_schema_spec.rb index 2ba4d62c4..bf0dd4bc2 100644 --- a/spec/ruby_llm/chat_schema_spec.rb +++ b/spec/ruby_llm/chat_schema_spec.rb @@ -18,9 +18,8 @@ } end - # Test OpenAI-compatible providers that support structured output - # Note: Only test models that have json_schema support, not just json_object - CHAT_MODELS.select { |model_info| %i[openai].include?(model_info[:provider]) }.each do |model_info| + # Test providers that support JSON Schema structured output + CHAT_MODELS.select { |m| %i[openai anthropic].include?(m[:provider]) }.each do |model_info| model = model_info[:model] provider = model_info[:provider] diff --git a/spec/support/models_to_test.rb b/spec/support/models_to_test.rb index 63abad8a6..da94141f8 100644 --- a/spec/support/models_to_test.rb +++ b/spec/support/models_to_test.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true CHAT_MODELS = [ + { provider: :anthropic, model: 'claude-haiku-4-5' }, { provider: :openrouter, model: 'claude-haiku-4-5' }, { provider: :bedrock, model: 'claude-3-5-haiku' }, { provider: :deepseek, model: 'deepseek-chat' }, From ec98fd6511020ae4254381220c11cd98c9dd284e Mon Sep 17 00:00:00 2001 From: Vojtech Rinik Date: Mon, 8 Dec 2025 16:06:27 +0100 Subject: [PATCH 6/6] testing schema for haiku4.5, sonnet4.5 --- lib/ruby_llm/models.json | 3 +- ...n_schema_and_returns_structured_output.yml | 84 ++++++++ ...oving_schema_with_nil_mid-conversation.yml | 180 ++++++++++++++++++ spec/ruby_llm/chat_schema_spec.rb | 8 +- spec/support/models_to_test.rb | 8 +- 5 files changed, 278 insertions(+), 5 deletions(-) create mode 100644 spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_accepts_a_json_schema_and_returns_structured_output.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_allows_removing_schema_with_nil_mid-conversation.yml diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index 355250e4e..c276cfc37 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -373,7 +373,8 @@ ] }, "capabilities": [ - "function_calling" + "function_calling", + "structured_output" ], "pricing": { "text_tokens": { diff --git a/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_accepts_a_json_schema_and_returns_structured_output.yml b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_accepts_a_json_schema_and_returns_structured_output.yml new file mode 100644 index 000000000..51b079d17 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_accepts_a_json_schema_and_returns_structured_output.yml @@ -0,0 +1,84 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-sonnet-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named John who is 30 years old"}]}],"stream":false,"max_tokens":64000,"output_format":{"type":"json_schema","schema":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}}}' + headers: + Anthropic-Beta: + - structured-outputs-2025-11-13 + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:58:47 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:58:46Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:58:47Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:58:44Z' + Retry-After: + - '14' + Anthropic-Ratelimit-Tokens-Limit: + - '2400000' + Anthropic-Ratelimit-Tokens-Remaining: + - '2400000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:58:46Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '3130' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-sonnet-4-5-20250929","id":"msg_01VWfH4YoXJw74skHGn3YVkK","type":"message","role":"assistant","content":[{"type":"text","text":"{\"name\":\"John\",\"age\":30}"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":181,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:58:47 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_allows_removing_schema_with_nil_mid-conversation.yml b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_allows_removing_schema_with_nil_mid-conversation.yml new file mode 100644 index 000000000..244b9a198 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_allows_removing_schema_with_nil_mid-conversation.yml @@ -0,0 +1,180 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-sonnet-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named Bob"}]}],"stream":false,"max_tokens":64000,"output_format":{"type":"json_schema","schema":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}}}' + headers: + Anthropic-Beta: + - structured-outputs-2025-11-13 + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:51:16 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:51:15Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:51:16Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:51:14Z' + Retry-After: + - '45' + Anthropic-Ratelimit-Tokens-Limit: + - '2400000' + Anthropic-Ratelimit-Tokens-Remaining: + - '2400000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:51:15Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '2538' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-sonnet-4-5-20250929","id":"msg_01AyRyrgMZ8dMcUTxK1yra1V","type":"message","role":"assistant","content":[{"type":"text","text":"{\"name\":\"Bob\",\"age\":30}"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":174,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:51:16 GMT +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-sonnet-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named Bob"}]},{"role":"assistant","content":[{"type":"text","text":"{\"name\":\"Bob\",\"age\":30}"}]},{"role":"user","content":[{"type":"text","text":"Now + just tell me about Ruby"}]}],"stream":false,"max_tokens":64000}' + headers: + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:51:24 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:51:18Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:51:23Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:51:17Z' + Retry-After: + - '46' + Anthropic-Ratelimit-Tokens-Limit: + - '2400000' + Anthropic-Ratelimit-Tokens-Remaining: + - '2400000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:51:18Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '7064' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-sonnet-4-5-20250929","id":"msg_016LdxZzPJJ94dNFVghPm4vQ","type":"message","role":"assistant","content":[{"type":"text","text":"Ruby + is a dynamic, open-source programming language with a focus on simplicity + and productivity. It was created by Yukihiro Matsumoto (often called \"Matz\") + in the mid-1990s in Japan.\n\nKey features of Ruby include:\n\n- **Object-oriented**: + Everything in Ruby is an object, including primitive data types\n- **Elegant + syntax**: Designed to be natural to read and easy to write, following the + principle of least surprise\n- **Dynamic typing**: Variables don''t need explicit + type declarations\n- **Flexible**: Supports multiple programming paradigms + including procedural, object-oriented, and functional programming\n- **Rich + standard library**: Comes with extensive built-in functionality\n\nRuby became + particularly popular due to **Ruby on Rails**, a web application framework + that revolutionized web development with its \"convention over configuration\" + philosophy and rapid development capabilities.\n\nCommon uses:\n- Web development + (especially with Rails)\n- Scripting and automation\n- DevOps tools (like + Chef and Puppet)\n- Prototyping\n\nRuby emphasizes developer happiness and + readable code, making it a favorite among many programmers despite not being + the fastest language in terms of execution speed."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":33,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":257,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:51:23 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/chat_schema_spec.rb b/spec/ruby_llm/chat_schema_spec.rb index bf0dd4bc2..4e9db41f6 100644 --- a/spec/ruby_llm/chat_schema_spec.rb +++ b/spec/ruby_llm/chat_schema_spec.rb @@ -19,7 +19,7 @@ end # Test providers that support JSON Schema structured output - CHAT_MODELS.select { |m| %i[openai anthropic].include?(m[:provider]) }.each do |model_info| + CHAT_SCHEMA_MODELS.reject { _1[:provider] == :gemini }.each do |model_info| model = model_info[:model] provider = model_info[:provider] @@ -27,7 +27,9 @@ let(:chat) { RubyLLM.chat(model: model, provider: provider) } it 'accepts a JSON schema and returns structured output' do - skip 'Model does not support structured output' unless chat.model.structured_output? + # All models listed here should support structured output and the + # metadata should confirm that + raise 'Model returns false for structured_output?' unless chat.model.structured_output? response = chat .with_schema(person_schema) @@ -66,7 +68,7 @@ end # Test Gemini provider separately due to different schema format - CHAT_MODELS.select { |model_info| model_info[:provider] == :gemini }.each do |model_info| + CHAT_SCHEMA_MODELS.select { _1[:provider] == :gemini }.each do |model_info| model = model_info[:model] provider = model_info[:provider] diff --git a/spec/support/models_to_test.rb b/spec/support/models_to_test.rb index da94141f8..1f7623fe9 100644 --- a/spec/support/models_to_test.rb +++ b/spec/support/models_to_test.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true CHAT_MODELS = [ - { provider: :anthropic, model: 'claude-haiku-4-5' }, { provider: :openrouter, model: 'claude-haiku-4-5' }, { provider: :bedrock, model: 'claude-3-5-haiku' }, { provider: :deepseek, model: 'deepseek-chat' }, @@ -15,6 +14,13 @@ { provider: :vertexai, model: 'gemini-2.5-flash' } ].freeze +CHAT_SCHEMA_MODELS = [ + { provider: :anthropic, model: 'claude-haiku-4-5' }, + { provider: :anthropic, model: 'claude-sonnet-4-5' }, + { provider: :gemini, model: 'gemini-2.5-flash' }, + { provider: :openai, model: 'gpt-4.1-nano' } +].freeze + PDF_MODELS = [ { provider: :anthropic, model: 'claude-haiku-4-5' }, { provider: :bedrock, model: 'claude-3-7-sonnet' },