diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index 2e6033e17..c276cfc37 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -174,7 +174,8 @@ ] }, "capabilities": [ - "function_calling" + "function_calling", + "structured_output" ], "pricing": { "text_tokens": { @@ -372,7 +373,8 @@ ] }, "capabilities": [ - "function_calling" + "function_calling", + "structured_output" ], "pricing": { "text_tokens": { diff --git a/lib/ruby_llm/providers/anthropic.rb b/lib/ruby_llm/providers/anthropic.rb index cd7d38055..29e18d008 100644 --- a/lib/ruby_llm/providers/anthropic.rb +++ b/lib/ruby_llm/providers/anthropic.rb @@ -11,6 +11,8 @@ class Anthropic < Provider include Anthropic::Streaming include Anthropic::Tools + STRUCTURED_OUTPUTS_BETA = 'structured-outputs-2025-11-13' + def api_base 'https://api.anthropic.com' end @@ -22,6 +24,23 @@ def headers } end + def complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, &block) # rubocop:disable Metrics/ParameterLists + headers = add_structured_output_beta_header(headers) if schema + super + end + + private + + def add_structured_output_beta_header(headers) + existing_beta = headers['anthropic-beta'] + new_beta = if existing_beta + "#{existing_beta},#{STRUCTURED_OUTPUTS_BETA}" + else + STRUCTURED_OUTPUTS_BETA + end + headers.merge('anthropic-beta' => new_beta) + end + class << self def capabilities Anthropic::Capabilities diff --git a/lib/ruby_llm/providers/anthropic/capabilities.rb b/lib/ruby_llm/providers/anthropic/capabilities.rb index 710b55386..5f0699608 100644 --- a/lib/ruby_llm/providers/anthropic/capabilities.rb +++ b/lib/ruby_llm/providers/anthropic/capabilities.rb @@ -31,11 +31,25 @@ def supports_vision?(model_id) end def supports_functions?(model_id) - model_id.match?(/claude-3/) + claude3_or_newer?(model_id) end def supports_json_mode?(model_id) - model_id.match?(/claude-3/) + claude3_or_newer?(model_id) + end + + def supports_structured_output?(model_id) + # Structured outputs supported on Claude 4+ (Sonnet 4.5, Opus 4.1, etc.) + # https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs + claude4_or_newer?(model_id) + end + + def claude3_or_newer?(model_id) + model_id.match?(/claude-[34]|claude-(sonnet|opus|haiku)-[4-9]/) + end + + def claude4_or_newer?(model_id) + model_id.match?(/claude-[4-9]|claude-(sonnet|opus|haiku)-[4-9]/) end def supports_extended_thinking?(model_id) @@ -92,12 +106,13 @@ def modalities_for(model_id) def capabilities_for(model_id) capabilities = ['streaming'] - if model_id.match?(/claude-3/) + if claude3_or_newer?(model_id) capabilities << 'function_calling' capabilities << 'batch' end - capabilities << 'reasoning' if model_id.match?(/claude-3-7|-4/) + capabilities << 'structured_output' if supports_structured_output?(model_id) + capabilities << 'reasoning' if model_id.match?(/claude-3-7|claude-[4-9]|claude-(sonnet|opus)-[4-9]/) capabilities << 'citations' if model_id.match?(/claude-3\.5|claude-3-7/) capabilities end diff --git a/lib/ruby_llm/providers/anthropic/chat.rb b/lib/ruby_llm/providers/anthropic/chat.rb index cbc9f1161..f2b41721a 100644 --- a/lib/ruby_llm/providers/anthropic/chat.rb +++ b/lib/ruby_llm/providers/anthropic/chat.rb @@ -11,12 +11,12 @@ def completion_url '/v1/messages' end - def render_payload(messages, tools:, temperature:, model:, stream: false, schema: nil) # rubocop:disable Metrics/ParameterLists,Lint/UnusedMethodArgument + def render_payload(messages, tools:, temperature:, model:, stream: false, schema: nil) # rubocop:disable Metrics/ParameterLists system_messages, chat_messages = separate_messages(messages) system_content = build_system_content(system_messages) build_base_payload(chat_messages, model, stream).tap do |payload| - add_optional_fields(payload, system_content:, tools:, temperature:) + add_optional_fields(payload, system_content:, tools:, temperature:, schema:) end end @@ -54,10 +54,19 @@ def build_base_payload(chat_messages, model, stream) } end - def add_optional_fields(payload, system_content:, tools:, temperature:) + def add_optional_fields(payload, system_content:, tools:, temperature:, schema: nil) payload[:tools] = tools.values.map { |t| Tools.function_for(t) } if tools.any? payload[:system] = system_content unless system_content.empty? payload[:temperature] = temperature unless temperature.nil? + add_output_format(payload, schema) if schema + end + + def add_output_format(payload, schema) + SchemaValidator.new(schema).validate! + payload[:output_format] = { + type: 'json_schema', + schema: schema + } end def parse_completion_response(response) diff --git a/lib/ruby_llm/providers/anthropic/schema_validator.rb b/lib/ruby_llm/providers/anthropic/schema_validator.rb new file mode 100644 index 000000000..35774bbb9 --- /dev/null +++ b/lib/ruby_llm/providers/anthropic/schema_validator.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class Anthropic + # Validates JSON schemas for Anthropic structured outputs. + # + # Anthropic requires all object schemas to have `additionalProperties: false`. + # This constraint ensures the model produces deterministic, well-defined JSON + # without unexpected extra fields. The API will reject schemas that don't meet + # this requirement. + # + # This validator recursively checks all objects in the schema, including: + # - Top-level object schemas + # - Nested objects in `properties` + # - Objects in array `items` + # - Objects in `anyOf`, `oneOf`, and `allOf` combinators + # + # @see https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs + class SchemaValidator + def initialize(schema) + @schema = schema + end + + def validate! + validate_node(@schema, 'schema') + end + + private + + def validate_node(schema, path) + return unless schema.is_a?(Hash) + + validate_object_schema(schema, path) + validate_properties(schema, path) + validate_items(schema, path) + validate_combinators(schema, path) + end + + def validate_object_schema(schema, path) + return unless value(schema, :type) == 'object' && value(schema, :additionalProperties) != false + + raise ArgumentError, + "#{path}: Object schemas must set 'additionalProperties' to false for Anthropic structured outputs." + end + + def validate_properties(schema, path) + properties = value(schema, :properties) + return unless properties.is_a?(Hash) + + properties.each do |key, prop_schema| + validate_node(prop_schema, "#{path}.properties.#{key}") + end + end + + def validate_items(schema, path) + items = value(schema, :items) + validate_node(items, "#{path}.items") if items + end + + def validate_combinators(schema, path) + %i[anyOf oneOf allOf].each do |keyword| + schemas = value(schema, keyword) + next unless schemas.is_a?(Array) + + schemas.each_with_index do |sub_schema, index| + validate_node(sub_schema, "#{path}.#{keyword}[#{index}]") + end + end + end + + def value(schema, key) + return unless schema.is_a?(Hash) + + symbol_key = key.is_a?(Symbol) ? key : key.to_sym + return schema[symbol_key] if schema.key?(symbol_key) + + string_key = key.to_s + schema[string_key] if schema.key?(string_key) + end + end + end + end +end diff --git a/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_accepts_a_json_schema_and_returns_structured_output.yml b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_accepts_a_json_schema_and_returns_structured_output.yml new file mode 100644 index 000000000..e1314725a --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_accepts_a_json_schema_and_returns_structured_output.yml @@ -0,0 +1,84 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-haiku-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named John who is 30 years old"}]}],"stream":false,"max_tokens":64000,"output_format":{"type":"json_schema","schema":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}}}' + headers: + Anthropic-Beta: + - structured-outputs-2025-11-13 + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:13:56 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:13:56Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:13:56Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:13:55Z' + Retry-After: + - '4' + Anthropic-Ratelimit-Tokens-Limit: + - '4800000' + Anthropic-Ratelimit-Tokens-Remaining: + - '4800000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:13:56Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '1276' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01Pqutax4swVnZUJ1oeUD86c","type":"message","role":"assistant","content":[{"type":"text","text":"{\"name\":\"John\",\"age\":30}"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":181,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:13:56 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_allows_removing_schema_with_nil_mid-conversation.yml b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_allows_removing_schema_with_nil_mid-conversation.yml new file mode 100644 index 000000000..7317c10a7 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-haiku-4-5_allows_removing_schema_with_nil_mid-conversation.yml @@ -0,0 +1,180 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-haiku-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named Bob"}]}],"stream":false,"max_tokens":64000,"output_format":{"type":"json_schema","schema":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}}}' + headers: + Anthropic-Beta: + - structured-outputs-2025-11-13 + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:13:57 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:13:57Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:13:57Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:13:56Z' + Retry-After: + - '3' + Anthropic-Ratelimit-Tokens-Limit: + - '4800000' + Anthropic-Ratelimit-Tokens-Remaining: + - '4800000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:13:57Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '1075' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01GPp3TzjpRktd5iCu9sFUAz","type":"message","role":"assistant","content":[{"type":"text","text":"{\"name\":\"Bob\",\"age\":30}"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":174,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:13:57 GMT +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-haiku-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named Bob"}]},{"role":"assistant","content":[{"type":"text","text":"{\"name\":\"Bob\",\"age\":30}"}]},{"role":"user","content":[{"type":"text","text":"Now + just tell me about Ruby"}]}],"stream":false,"max_tokens":64000}' + headers: + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:14:00 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '4000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:13:58Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '800000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:14:00Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:13:58Z' + Retry-After: + - '2' + Anthropic-Ratelimit-Tokens-Limit: + - '4800000' + Anthropic-Ratelimit-Tokens-Remaining: + - '4800000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:13:58Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '2593' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_01Kj3FFsWWTK5VGSTy5r35FJ","type":"message","role":"assistant","content":[{"type":"text","text":"Ruby + is a dynamic, interpreted programming language known for its simplicity and + productivity. Here are some key points about it:\n\n**Key Characteristics:**\n- + **Syntax**: Clean, readable, and beginner-friendly\n- **Type System**: Dynamically + typed\n- **Paradigm**: Object-oriented with functional programming features\n- + **Philosophy**: \"Principle of Least Surprise\" - the language behaves in + ways that minimize confusion\n\n**Common Uses:**\n- Web development (especially + with Ruby on Rails framework)\n- Scripting and automation\n- Data processing\n- + DevOps tools\n\n**Popular Frameworks:**\n- **Ruby on Rails**: The most popular + web framework\n- Sinatra: Lightweight web framework\n- Jekyll: Static site + generator\n\n**Strengths:**\n- Rapid development and prototyping\n- Large, + active community\n- Extensive libraries (gems)\n- Great for startups and MVPs\n\n**Weaknesses:**\n- + Slower execution speed compared to compiled languages\n- Higher memory consumption\n- + Less suitable for performance-critical applications\n\nRuby was created by + Yukihiro Matsumoto in 1995 and remains popular for web development, though + its market share has shifted somewhat with the rise of JavaScript/Node.js + and Python."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":33,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":277,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:14:00 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_accepts_a_json_schema_and_returns_structured_output.yml b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_accepts_a_json_schema_and_returns_structured_output.yml new file mode 100644 index 000000000..51b079d17 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_accepts_a_json_schema_and_returns_structured_output.yml @@ -0,0 +1,84 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-sonnet-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named John who is 30 years old"}]}],"stream":false,"max_tokens":64000,"output_format":{"type":"json_schema","schema":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}}}' + headers: + Anthropic-Beta: + - structured-outputs-2025-11-13 + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:58:47 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:58:46Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:58:47Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:58:44Z' + Retry-After: + - '14' + Anthropic-Ratelimit-Tokens-Limit: + - '2400000' + Anthropic-Ratelimit-Tokens-Remaining: + - '2400000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:58:46Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '3130' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-sonnet-4-5-20250929","id":"msg_01VWfH4YoXJw74skHGn3YVkK","type":"message","role":"assistant","content":[{"type":"text","text":"{\"name\":\"John\",\"age\":30}"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":181,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:58:47 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_allows_removing_schema_with_nil_mid-conversation.yml b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_allows_removing_schema_with_nil_mid-conversation.yml new file mode 100644 index 000000000..244b9a198 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_with_schema_with_anthropic_claude-sonnet-4-5_allows_removing_schema_with_nil_mid-conversation.yml @@ -0,0 +1,180 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-sonnet-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named Bob"}]}],"stream":false,"max_tokens":64000,"output_format":{"type":"json_schema","schema":{"type":"object","properties":{"name":{"type":"string"},"age":{"type":"integer"}},"required":["name","age"],"additionalProperties":false}}}' + headers: + Anthropic-Beta: + - structured-outputs-2025-11-13 + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:51:16 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:51:15Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:51:16Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:51:14Z' + Retry-After: + - '45' + Anthropic-Ratelimit-Tokens-Limit: + - '2400000' + Anthropic-Ratelimit-Tokens-Remaining: + - '2400000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:51:15Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '2538' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-sonnet-4-5-20250929","id":"msg_01AyRyrgMZ8dMcUTxK1yra1V","type":"message","role":"assistant","content":[{"type":"text","text":"{\"name\":\"Bob\",\"age\":30}"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":174,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":12,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:51:16 GMT +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-sonnet-4-5","messages":[{"role":"user","content":[{"type":"text","text":"Generate + a person named Bob"}]},{"role":"assistant","content":[{"type":"text","text":"{\"name\":\"Bob\",\"age\":30}"}]},{"role":"user","content":[{"type":"text","text":"Now + just tell me about Ruby"}]}],"stream":false,"max_tokens":64000}' + headers: + User-Agent: + - Faraday v2.14.0 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 08 Dec 2025 14:51:24 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Input-Tokens-Limit: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '2000000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-12-08T14:51:18Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '400000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-12-08T14:51:23Z' + Anthropic-Ratelimit-Requests-Limit: + - '4000' + Anthropic-Ratelimit-Requests-Remaining: + - '3999' + Anthropic-Ratelimit-Requests-Reset: + - '2025-12-08T14:51:17Z' + Retry-After: + - '46' + Anthropic-Ratelimit-Tokens-Limit: + - '2400000' + Anthropic-Ratelimit-Tokens-Remaining: + - '2400000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-12-08T14:51:18Z' + Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Anthropic-Organization-Id: + - "" + X-Envoy-Upstream-Service-Time: + - '7064' + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"model":"claude-sonnet-4-5-20250929","id":"msg_016LdxZzPJJ94dNFVghPm4vQ","type":"message","role":"assistant","content":[{"type":"text","text":"Ruby + is a dynamic, open-source programming language with a focus on simplicity + and productivity. It was created by Yukihiro Matsumoto (often called \"Matz\") + in the mid-1990s in Japan.\n\nKey features of Ruby include:\n\n- **Object-oriented**: + Everything in Ruby is an object, including primitive data types\n- **Elegant + syntax**: Designed to be natural to read and easy to write, following the + principle of least surprise\n- **Dynamic typing**: Variables don''t need explicit + type declarations\n- **Flexible**: Supports multiple programming paradigms + including procedural, object-oriented, and functional programming\n- **Rich + standard library**: Comes with extensive built-in functionality\n\nRuby became + particularly popular due to **Ruby on Rails**, a web application framework + that revolutionized web development with its \"convention over configuration\" + philosophy and rapid development capabilities.\n\nCommon uses:\n- Web development + (especially with Rails)\n- Scripting and automation\n- DevOps tools (like + Chef and Puppet)\n- Prototyping\n\nRuby emphasizes developer happiness and + readable code, making it a favorite among many programmers despite not being + the fastest language in terms of execution speed."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":33,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":257,"service_tier":"standard"}}' + recorded_at: Mon, 08 Dec 2025 14:51:23 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/chat_schema_spec.rb b/spec/ruby_llm/chat_schema_spec.rb index 2ba4d62c4..4e9db41f6 100644 --- a/spec/ruby_llm/chat_schema_spec.rb +++ b/spec/ruby_llm/chat_schema_spec.rb @@ -18,9 +18,8 @@ } end - # Test OpenAI-compatible providers that support structured output - # Note: Only test models that have json_schema support, not just json_object - CHAT_MODELS.select { |model_info| %i[openai].include?(model_info[:provider]) }.each do |model_info| + # Test providers that support JSON Schema structured output + CHAT_SCHEMA_MODELS.reject { _1[:provider] == :gemini }.each do |model_info| model = model_info[:model] provider = model_info[:provider] @@ -28,7 +27,9 @@ let(:chat) { RubyLLM.chat(model: model, provider: provider) } it 'accepts a JSON schema and returns structured output' do - skip 'Model does not support structured output' unless chat.model.structured_output? + # All models listed here should support structured output and the + # metadata should confirm that + raise 'Model returns false for structured_output?' unless chat.model.structured_output? response = chat .with_schema(person_schema) @@ -67,7 +68,7 @@ end # Test Gemini provider separately due to different schema format - CHAT_MODELS.select { |model_info| model_info[:provider] == :gemini }.each do |model_info| + CHAT_SCHEMA_MODELS.select { _1[:provider] == :gemini }.each do |model_info| model = model_info[:model] provider = model_info[:provider] diff --git a/spec/ruby_llm/providers/anthropic/chat_spec.rb b/spec/ruby_llm/providers/anthropic/chat_spec.rb index 5a3038602..9f1b25caa 100644 --- a/spec/ruby_llm/providers/anthropic/chat_spec.rb +++ b/spec/ruby_llm/providers/anthropic/chat_spec.rb @@ -2,6 +2,28 @@ require 'spec_helper' +# rubocop:disable RSpec/MultipleDescribes +RSpec.describe RubyLLM::Providers::Anthropic do + include_context 'with configured RubyLLM' + + describe '#complete with structured outputs' do + let(:provider) { described_class.new(RubyLLM.config) } + + describe '#add_structured_output_beta_header' do + it 'adds beta header when schema is provided' do + headers = provider.send(:add_structured_output_beta_header, {}) + expect(headers['anthropic-beta']).to eq('structured-outputs-2025-11-13') + end + + it 'appends to existing beta header' do + existing_headers = { 'anthropic-beta' => 'existing-beta' } + headers = provider.send(:add_structured_output_beta_header, existing_headers) + expect(headers['anthropic-beta']).to eq('existing-beta,structured-outputs-2025-11-13') + end + end + end +end + RSpec.describe RubyLLM::Providers::Anthropic::Chat do describe '.render_payload' do let(:model) { instance_double(RubyLLM::Model::Info, id: 'claude-sonnet-4-5', max_tokens: nil) } @@ -27,6 +49,47 @@ expect(payload[:system]).to eq(system_raw.value) expect(payload[:messages].first[:content]).to eq([{ type: 'text', text: 'Hello there' }]) end + + it 'includes output_format when schema is provided' do + user_message = RubyLLM::Message.new(role: :user, content: 'Hello') + schema = { + type: 'object', + properties: { + name: { type: 'string' } + }, + required: ['name'], + additionalProperties: false + } + + payload = described_class.render_payload( + [user_message], + tools: {}, + temperature: nil, + model: model, + stream: false, + schema: schema + ) + + expect(payload[:output_format]).to eq({ + type: 'json_schema', + schema: schema + }) + end + + it 'does not include output_format when schema is nil' do + user_message = RubyLLM::Message.new(role: :user, content: 'Hello') + + payload = described_class.render_payload( + [user_message], + tools: {}, + temperature: nil, + model: model, + stream: false, + schema: nil + ) + + expect(payload).not_to have_key(:output_format) + end end describe '.parse_completion_response' do @@ -53,3 +116,4 @@ end end end +# rubocop:enable RSpec/MultipleDescribes diff --git a/spec/ruby_llm/providers/anthropic/schema_validator_spec.rb b/spec/ruby_llm/providers/anthropic/schema_validator_spec.rb new file mode 100644 index 000000000..2d980f8f4 --- /dev/null +++ b/spec/ruby_llm/providers/anthropic/schema_validator_spec.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe RubyLLM::Providers::Anthropic::SchemaValidator do + describe '#validate!' do + subject(:validate!) { described_class.new(schema).validate! } + + context 'with valid schemas' do + let(:schema) { { type: 'object', properties: { name: { type: 'string' } }, additionalProperties: false } } + + it 'accepts schema with additionalProperties: false' do + expect { validate! }.not_to raise_error + end + end + + context 'with string keys' do + let(:schema) do + { + 'type' => 'object', + 'properties' => { 'name' => { 'type' => 'string' } }, + 'additionalProperties' => false + } + end + + it 'accepts schema with string keys' do + expect { validate! }.not_to raise_error + end + end + + context 'with missing additionalProperties' do + let(:schema) { { type: 'object', properties: { name: { type: 'string' } } } } + + it 'rejects schema' do + expect { validate! }.to raise_error(ArgumentError, /additionalProperties.*to false/) + end + end + + context 'with nested object missing additionalProperties' do + let(:schema) do + { + type: 'object', + properties: { user: { type: 'object', properties: { name: { type: 'string' } } } }, + additionalProperties: false + } + end + + it 'rejects schema and reports path' do + expect { validate! }.to raise_error(ArgumentError, /schema\.properties\.user/) + end + end + + context 'with array items containing objects' do + let(:schema) do + { + type: 'object', + properties: { + users: { + type: 'array', + items: { type: 'object', properties: { name: { type: 'string' } } } + } + }, + additionalProperties: false + } + end + + it 'rejects invalid nested array items' do + expect { validate! }.to raise_error(ArgumentError, /schema\.properties\.users\.items/) + end + end + + context 'with valid array items' do + let(:schema) do + { + type: 'object', + properties: { + users: { + type: 'array', + items: { type: 'object', properties: { name: { type: 'string' } }, additionalProperties: false } + } + }, + additionalProperties: false + } + end + + it 'accepts valid nested array items' do + expect { validate! }.not_to raise_error + end + end + + context 'with anyOf containing invalid object' do + let(:schema) do + { + type: 'object', + properties: { + value: { + anyOf: [ + { type: 'string' }, + { type: 'object', properties: { id: { type: 'integer' } } } + ] + } + }, + additionalProperties: false + } + end + + it 'rejects invalid object in anyOf' do + expect { validate! }.to raise_error(ArgumentError, /schema\.properties\.value\.anyOf\[1\]/) + end + end + + context 'with non-object types' do + let(:schema) { { type: 'string' } } + + it 'accepts non-object schemas without additionalProperties' do + expect { validate! }.not_to raise_error + end + end + end +end diff --git a/spec/support/models_to_test.rb b/spec/support/models_to_test.rb index 63abad8a6..1f7623fe9 100644 --- a/spec/support/models_to_test.rb +++ b/spec/support/models_to_test.rb @@ -14,6 +14,13 @@ { provider: :vertexai, model: 'gemini-2.5-flash' } ].freeze +CHAT_SCHEMA_MODELS = [ + { provider: :anthropic, model: 'claude-haiku-4-5' }, + { provider: :anthropic, model: 'claude-sonnet-4-5' }, + { provider: :gemini, model: 'gemini-2.5-flash' }, + { provider: :openai, model: 'gpt-4.1-nano' } +].freeze + PDF_MODELS = [ { provider: :anthropic, model: 'claude-haiku-4-5' }, { provider: :bedrock, model: 'claude-3-7-sonnet' },