diff --git a/lib/braintrust/api.rb b/lib/braintrust/api.rb index 824c362..a556dea 100644 --- a/lib/braintrust/api.rb +++ b/lib/braintrust/api.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "api/datasets" +require_relative "api/experiments" require_relative "api/functions" module Braintrust @@ -20,6 +21,12 @@ def datasets @datasets ||= API::Datasets.new(self) end + # Access to experiments API + # @return [API::Experiments] + def experiments + @experiments ||= API::Experiments.new(self) + end + # Access to functions API # @return [API::Functions] def functions diff --git a/lib/braintrust/api/experiments.rb b/lib/braintrust/api/experiments.rb new file mode 100644 index 0000000..2028c05 --- /dev/null +++ b/lib/braintrust/api/experiments.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require "net/http" +require "json" +require "uri" +require_relative "../logger" + +module Braintrust + class API + # Experiments API namespace + # Provides methods for fetching experiment comparison data + class Experiments + def initialize(api) + @api = api + @state = api.state + end + + # Fetch experiment comparison data + # GET /experiment-comparison2 + # Returns score and metric summaries with comparison to baseline experiment + # @param experiment_id [String] Current experiment ID + # @param base_experiment_id [String, nil] Baseline experiment ID to compare against. + # If nil, API will auto-select based on experiment metadata or most recent experiment. + # @return [Hash] Response with scores, metrics, and comparison info + # - "scores" [Hash] Score summaries keyed by name + # - "metrics" [Hash] Metric summaries keyed by name + # - "comparisonExperimentName" [String, nil] Name of baseline experiment + # - "comparisonExperimentId" [String, nil] ID of baseline experiment + def comparison(experiment_id:, base_experiment_id: nil) + params = {"experiment_id" => experiment_id} + params["base_experiment_id"] = base_experiment_id if base_experiment_id + + http_get("/experiment-comparison2", params) + end + + private + + # Core HTTP request method with logging + # @param method [Symbol] :get or :post + # @param path [String] API path + # @param params [Hash] Query params (for GET) + # @param payload [Hash, nil] JSON payload (for POST) + # @return [Hash] Parsed JSON response + def http_request(method, path, params: {}, payload: nil) + # Build URI - use api_url for this endpoint + base = @state.api_url || "https://api.braintrust.dev" + uri = URI("#{base}#{path}") + uri.query = URI.encode_www_form(params) unless params.empty? + + # Create request + request = case method + when :get + Net::HTTP::Get.new(uri) + when :post + req = Net::HTTP::Post.new(uri) + req["Content-Type"] = "application/json" + req.body = JSON.dump(payload) if payload + req + else + raise ArgumentError, "Unsupported HTTP method: #{method}" + end + + # Use Bearer token format for API endpoints + request["Authorization"] = "Bearer #{@state.api_key}" + + # Execute request with timing + start_time = Time.now + Log.debug("[API] #{method.upcase} #{uri}") + + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = (uri.scheme == "https") + response = http.request(request) + + duration_ms = ((Time.now - start_time) * 1000).round(2) + Log.debug("[API] #{method.upcase} #{uri} -> #{response.code} (#{duration_ms}ms, #{response.body.bytesize} bytes)") + + # Handle response + unless response.is_a?(Net::HTTPSuccess) + Log.debug("[API] Error response body: #{response.body}") + raise Error, "HTTP #{response.code} for #{method.upcase} #{uri}: #{response.body}" + end + + JSON.parse(response.body) + end + + # HTTP GET with query params - returns parsed JSON + def http_get(path, params = {}) + http_request(:get, path, params: params) + end + end + end +end diff --git a/lib/braintrust/eval.rb b/lib/braintrust/eval.rb index cc14076..1edbb5c 100644 --- a/lib/braintrust/eval.rb +++ b/lib/braintrust/eval.rb @@ -2,6 +2,8 @@ require_relative "eval/scorer" require_relative "eval/runner" +require_relative "eval/summary" +require_relative "eval/result" require_relative "internal/experiments" require "opentelemetry/sdk" @@ -199,13 +201,15 @@ def scorer(name, callable = nil, &block) # @param metadata [Hash] Optional experiment metadata # @param update [Boolean] If true, allow reusing existing experiment (default: false) # @param quiet [Boolean] If true, suppress result output (default: false) + # @param send_logs [Boolean] If true (default), create experiment on server and send span data. + # If false, run evaluation locally without sending data to Braintrust. # @param state [State, nil] Braintrust state (defaults to global state) # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider (defaults to global) # @return [Result] def run(project:, experiment:, task:, scorers:, cases: nil, dataset: nil, parallelism: 1, tags: nil, metadata: nil, update: false, quiet: false, - state: nil, tracer_provider: nil) + send_logs: true, state: nil, tracer_provider: nil) # Validate required parameters validate_params!(project: project, experiment: experiment, cases: cases, dataset: dataset, task: task, scorers: scorers) @@ -223,17 +227,48 @@ def run(project:, experiment:, task:, scorers:, cases = resolve_dataset(dataset, project, state) end + if send_logs + run_with_server( + project: project, experiment: experiment, task: task, scorers: scorers, + cases: cases, parallelism: parallelism, tags: tags, metadata: metadata, + update: update, quiet: quiet, state: state, tracer_provider: tracer_provider + ) + else + run_local( + project: project, experiment: experiment, task: task, scorers: scorers, + cases: cases, parallelism: parallelism, quiet: quiet, state: state + ) + end + end + + private + + # Print result summary to stdout + # @param result [Result] The evaluation result + def print_result(result) + puts result.to_pretty + end + + # Run evaluation with server integration (send_logs: true) + # Creates experiment, sends span data, fetches comparison summary + def run_with_server(project:, experiment:, task:, scorers:, cases:, + parallelism:, tags:, metadata:, update:, quiet:, state:, tracer_provider:) + require_relative "api" + # Register project and experiment via API - result = Internal::Experiments.get_or_create( + reg_result = Internal::Experiments.get_or_create( experiment, project, state: state, tags: tags, metadata: metadata, update: update ) - experiment_id = result[:experiment_id] - project_id = result[:project_id] - project_name = result[:project_name] + experiment_id = reg_result[:experiment_id] + project_id = reg_result[:project_id] + project_name = reg_result[:project_name] + + # Generate permalink + permalink = "#{state.app_url}/app/#{state.org_name}/object?object_type=experiment&object_id=#{experiment_id}" - # Instantiate Runner and run evaluation + # Instantiate Runner and run evaluation with tracing runner = Runner.new( experiment_id: experiment_id, experiment_name: experiment, @@ -244,20 +279,166 @@ def run(project:, experiment:, task:, scorers:, state: state, tracer_provider: tracer_provider ) - result = runner.run(cases, parallelism: parallelism) - # Print result summary unless quiet + start_time = Time.now + run_result = runner.run(cases, parallelism: parallelism) + duration = Time.now - start_time + + # Flush spans before fetching comparison data from API + # This ensures the server has received all span data before we query for results + actual_tracer_provider = tracer_provider || OpenTelemetry.tracer_provider + actual_tracer_provider.force_flush if actual_tracer_provider.respond_to?(:force_flush) + + # Fetch comparison summary from API + summary = fetch_comparison_summary( + experiment_id: experiment_id, + experiment_name: experiment, + project_name: project_name, + permalink: permalink, + duration: duration, + errors: run_result.errors, + raw_scores: run_result.scores, + state: state + ) + + # Create result with summary + result = Result.new( + experiment_id: experiment_id, + experiment_name: experiment, + project_id: project_id, + project_name: project_name, + permalink: permalink, + errors: run_result.errors, + duration: duration, + scores: run_result.scores, + summary: summary + ) + print_result(result) unless quiet + result + end + # Run evaluation locally without server (send_logs: false) + # No experiment created, no span data sent, local summary only + def run_local(project:, experiment:, task:, scorers:, cases:, + parallelism:, quiet:, state:) + # Create a no-op tracer provider that doesn't send data + no_op_tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new + + # Instantiate Runner with no-op tracer (no data sent) + runner = Runner.new( + experiment_id: nil, + experiment_name: experiment, + project_id: nil, + project_name: project, + task: task, + scorers: scorers, + state: state, + tracer_provider: no_op_tracer_provider + ) + + start_time = Time.now + run_result = runner.run(cases, parallelism: parallelism) + duration = Time.now - start_time + + # Build local summary from raw scores + summary = ExperimentSummary.from_raw_scores( + run_result.scores, + { + project_name: project, + experiment_name: experiment, + experiment_id: nil, + experiment_url: nil, + duration: duration, + error_count: run_result.errors.length, + errors: run_result.errors + } + ) + + # Create result with local summary + result = Result.new( + experiment_id: nil, + experiment_name: experiment, + project_id: nil, + project_name: project, + permalink: nil, + errors: run_result.errors, + duration: duration, + scores: run_result.scores, + summary: summary + ) + + print_result(result) unless quiet result end - private + # Fetch comparison summary from API, falling back to local on failure or empty response + def fetch_comparison_summary(experiment_id:, experiment_name:, project_name:, + permalink:, duration:, errors:, raw_scores:, state:) + api = API.new(state: state) + local_metadata = { + project_name: project_name, + experiment_name: experiment_name, + experiment_id: experiment_id, + experiment_url: permalink, + duration: duration, + error_count: errors.length, + errors: errors + } - # Print result summary to stdout - # @param result [Result] The evaluation result - def print_result(result) - puts result.to_pretty + begin + api_response = api.experiments.comparison(experiment_id: experiment_id) + + # If API returned empty scores, fall back to local data + if api_response["scores"].nil? || api_response["scores"].empty? + Log.debug("API returned no scores, using local summary") + return ExperimentSummary.from_raw_scores(raw_scores, local_metadata) + end + + build_server_summary(api_response, local_metadata) + rescue => e + Log.warn("Failed to fetch comparison summary: #{e.message}. Falling back to local summary.") + ExperimentSummary.from_raw_scores(raw_scores, local_metadata) + end + end + + # Build ExperimentSummary from API response + def build_server_summary(api_response, metadata) + # Transform API scores into ScoreSummary objects + scores = (api_response["scores"] || {}).map do |name, data| + [name, ScoreSummary.new( + name: name, + score: data["score"], + diff: data["diff"], + improvements: data["improvements"], + regressions: data["regressions"] + )] + end.to_h + + # Transform API metrics into MetricSummary objects + metrics = (api_response["metrics"] || {}).map do |name, data| + [name, MetricSummary.new( + name: name, + metric: data["metric"], + unit: data["unit"] || "", + diff: data["diff"] + )] + end.to_h + + # Build comparison info if present + comparison = if api_response["comparisonExperimentName"] + ComparisonInfo.new( + baseline_experiment_id: api_response["comparisonExperimentId"], + baseline_experiment_name: api_response["comparisonExperimentName"] + ) + end + + ExperimentSummary.new( + scores: scores, + metrics: metrics.empty? ? nil : metrics, + comparison: comparison, + **metadata + ) end # Validate required parameters diff --git a/lib/braintrust/eval/formatter.rb b/lib/braintrust/eval/formatter.rb index d1cfa5f..efafdab 100644 --- a/lib/braintrust/eval/formatter.rb +++ b/lib/braintrust/eval/formatter.rb @@ -30,6 +30,15 @@ module Formatter # Maximum length for error messages before truncation MAX_ERROR_LENGTH = 150 + # Column widths for comparison table + COLUMN_WIDTHS = { + name: 22, + value: 12, + change: 10, + improvements: 14, + regressions: 12 + }.freeze + class << self # Format an experiment summary for CLI output # @param summary [ExperimentSummary] The experiment summary @@ -39,24 +48,41 @@ def format_experiment_summary(summary) lines = [] - # Metadata section - lines << format_metadata_row("Project", summary.project_name) - lines << format_metadata_row("Experiment", summary.experiment_name) - lines << format_metadata_row("ID", summary.experiment_id) - lines << format_metadata_row("Duration", format_duration(summary.duration)) - lines << format_metadata_row("Errors", summary.error_count.to_s) + # Comparison header (if comparing) + if summary.comparison&.baseline_experiment_name + lines << format_comparison_header( + summary.comparison.baseline_experiment_name, + summary.experiment_name + ) + lines << "" + end - # Scores section (if any) + # Scores section if summary.scores&.any? - lines << "" lines << colorize("Scores", :white) - # Calculate max scorer name length for alignment - max_name_len = summary.scores.values.map { |s| s.name.length }.max || 0 - name_width = [max_name_len + 2, 20].max # +2 for "◯ " prefix + if has_comparison_data?(summary.scores) + lines << format_scores_table_header + summary.scores.each_value do |score| + lines << format_comparison_score_row(score) + end + else + # Simple format without comparison columns + max_name_len = summary.scores.values.map { |s| s.name.length }.max || 0 + name_width = [max_name_len + 2, 20].max + summary.scores.each_value do |score| + lines << format_simple_score_row(score, name_width) + end + end + end - summary.scores.each_value do |score| - lines << format_score_row(score, name_width) + # Metrics section (only if present - comparison mode) + if summary.metrics&.any? + lines << "" + lines << colorize("Metrics", :white) + lines << format_metrics_table_header + summary.metrics.each_value do |metric| + lines << format_metric_row(metric) end end @@ -64,7 +90,6 @@ def format_experiment_summary(summary) if summary.errors&.any? lines << "" lines << colorize("Errors", :white) - summary.errors.each do |error| lines << format_error_row(error) end @@ -79,22 +104,151 @@ def format_experiment_summary(summary) wrap_in_box(lines, "Experiment summary") end - # Format a metadata row (label: value) - # @param label [String] Row label - # @param value [String] Row value + # Format comparison header line + # @param baseline [String] Baseline experiment name + # @param comparison [String] Comparison (current) experiment name + # @return [String] Formatted header + def format_comparison_header(baseline, comparison) + "#{baseline} (baseline) #{colorize("←", :gray)} #{comparison} (comparison)" + end + + # Format table header for scores section (5 columns) + # @return [String] Header row + def format_scores_table_header + name = pad_cell("Name", COLUMN_WIDTHS[:name], :left) + value = pad_cell("Value", COLUMN_WIDTHS[:value], :right) + change = pad_cell("Change", COLUMN_WIDTHS[:change], :right) + improvements = pad_cell("Improvements", COLUMN_WIDTHS[:improvements], :right) + regressions = pad_cell("Regressions", COLUMN_WIDTHS[:regressions], :right) + colorize("#{name}#{value}#{change}#{improvements}#{regressions}", :dim) + end + + # Format table header for metrics section (3 columns) + # @return [String] Header row + def format_metrics_table_header + name = pad_cell("Name", COLUMN_WIDTHS[:name], :left) + value = pad_cell("Value", COLUMN_WIDTHS[:value], :right) + change = pad_cell("Change", COLUMN_WIDTHS[:change], :right) + colorize("#{name}#{value}#{change}", :dim) + end + + # Format a score row with all comparison columns (5 columns) + # @param score [ScoreSummary] The score summary + # @return [String] Formatted row + def format_comparison_score_row(score) + name = "#{colorize("◯", :blue)} #{score.name}" + value = format_score_value(score.score) + change = format_change(score.diff) + improvements = format_count(score.improvements, :green) + regressions = format_count(score.regressions, :red) + + pad_cell(name, COLUMN_WIDTHS[:name], :left) + + pad_cell(value, COLUMN_WIDTHS[:value], :right) + + pad_cell(change, COLUMN_WIDTHS[:change], :right) + + pad_cell(improvements, COLUMN_WIDTHS[:improvements], :right) + + pad_cell(regressions, COLUMN_WIDTHS[:regressions], :right) + end + + # Format a simple score row without comparison columns + # @param score [ScoreSummary] The score summary + # @param name_width [Integer] Width for the name column # @return [String] Formatted row - def format_metadata_row(label, value) - "#{colorize(label + ":", :dim)} #{value}" + def format_simple_score_row(score, name_width = 20) + name = "#{colorize("◯", :blue)} #{score.name}" + value = format_score_value(score.score) + pad_cell(name, name_width, :left) + " " + pad_cell(value, 10, :right) + end + + # Format a metric row (3 columns) + # @param metric [MetricSummary] The metric summary + # @return [String] Formatted row + def format_metric_row(metric) + name = "#{colorize("◯", :magenta)} #{metric.name}" + value = format_metric_value(metric.metric, metric.unit) + change = format_change(metric.diff) + + pad_cell(name, COLUMN_WIDTHS[:name], :left) + + pad_cell(value, COLUMN_WIDTHS[:value], :right) + + pad_cell(change, COLUMN_WIDTHS[:change], :right) end - # Format duration for display - # @param duration [Float] Duration in seconds - # @return [String] Formatted duration (e.g., "1.2345s" or "123ms") - def format_duration(duration) - if duration < 1 - "#{(duration * 1000).round(0)}ms" + # Format a score value as percentage + # @param score [Float] Score value (0.0 to 1.0) + # @return [String] Formatted percentage + def format_score_value(score) + return colorize("-", :gray) if score.nil? + colorize("#{(score * 100).round(2)}%", :white) + end + + # Format a metric value with unit + # @param value [Float] Metric value + # @param unit [String] Unit suffix + # @return [String] Formatted value with unit + def format_metric_value(value, unit) + return colorize("-", :gray) if value.nil? + + # Format based on magnitude and type + formatted = if value == value.to_i + value.to_i.to_s + elsif value < 0.01 + value.round(4).to_s + else + value.round(2).to_s + end + + colorize("#{formatted}#{unit}", :white) + end + + # Format a change value (diff) + # @param diff [Float, nil] Difference value (as ratio, e.g., 0.05 = +5%) + # @return [String] Formatted change with color + def format_change(diff) + return colorize("-", :gray) if diff.nil? + + # diff is already a ratio, convert to percentage + percentage = (diff * 100).round(2) + sign = (percentage >= 0) ? "+" : "" + formatted = "#{sign}#{percentage}%" + + if percentage > 0 + colorize(formatted, :green) + elsif percentage < 0 + colorize(formatted, :red) else - "#{duration.round(4)}s" + colorize(formatted, :gray) + end + end + + # Format an improvement/regression count + # @param count [Integer, nil] Count value + # @param color [Symbol] Color to apply if non-zero + # @return [String] Formatted count + def format_count(count, color = nil) + return colorize("-", :gray) if count.nil? || count == 0 + + if color + colorize(count.to_s, :dim, color) + else + count.to_s + end + end + + # Check if any scores have comparison data + # @param scores [Hash] Scores to check + # @return [Boolean] True if any score has diff data + def has_comparison_data?(scores) + scores&.values&.any? { |s| !s.diff.nil? } + end + + # Format a duration value for display + # Shows milliseconds for < 1 second, seconds otherwise + # @param seconds [Float] Duration in seconds + # @return [String] Formatted duration (e.g., "500ms" or "1.2345s") + def format_duration(seconds) + if seconds < 1 + "#{(seconds * 1000).round}ms" + else + "#{seconds.round(4)}s" end end @@ -125,16 +279,6 @@ def colorize(text, *styles) "#{codes}#{text}#{COLORS[:reset]}" end - # Format a score row for display - # @param score [ScorerStats] The scorer statistics - # @param name_width [Integer] Width for the name column - # @return [String] Formatted row - def format_score_row(score, name_width = 20) - name = "#{colorize("◯", :blue)} #{score.name}" - value = colorize("#{(score.score_mean * 100).round(2)}%", :white) - pad_cell(name, name_width, :left) + " " + pad_cell(value, 10, :right) - end - # Pad a cell to a given width, accounting for ANSI codes # @param text [String] Cell text (may contain ANSI codes) # @param width [Integer] Target width @@ -156,8 +300,6 @@ def pad_cell(text, width, align) # @param text [String] Text that may contain escape sequences # @return [Integer] Visible character count def visible_text_length(text) - # Strip ANSI color codes: \e[...m - # Strip OSC 8 hyperlinks: \e]8;;...\e\\ (the URL part is invisible) text .gsub(/\e\[[0-9;]*m/, "") # ANSI color codes .gsub(/\e\]8;;[^\e]*\e\\/, "") # OSC 8 hyperlink sequences diff --git a/lib/braintrust/eval/result.rb b/lib/braintrust/eval/result.rb index c18af30..b552075 100644 --- a/lib/braintrust/eval/result.rb +++ b/lib/braintrust/eval/result.rb @@ -6,22 +6,23 @@ module Braintrust module Eval # Result represents the outcome of an evaluation run - # Contains experiment metadata, errors, timing information, and raw score data + # Contains experiment metadata, errors, timing information, and summary data class Result attr_reader :experiment_id, :experiment_name, :project_id, :project_name, - :permalink, :errors, :duration, :scores + :permalink, :errors, :duration, :scores, :summary # Create a new result - # @param experiment_id [String] The experiment ID + # @param experiment_id [String, nil] The experiment ID (nil for local-only mode) # @param experiment_name [String] The experiment name - # @param project_id [String] The project ID + # @param project_id [String, nil] The project ID (nil for local-only mode) # @param project_name [String] The project name - # @param permalink [String] Link to view the experiment in Braintrust UI + # @param permalink [String, nil] Link to view the experiment in Braintrust UI # @param errors [Array] List of errors that occurred # @param duration [Float] Duration in seconds # @param scores [Hash, nil] Raw score data { scorer_name => Array } + # @param summary [ExperimentSummary, nil] Pre-computed summary (if nil, computed lazily) def initialize(experiment_id:, experiment_name:, project_id:, project_name:, - permalink:, errors:, duration:, scores: nil) + permalink:, errors:, duration:, scores: nil, summary: nil) @experiment_id = experiment_id @experiment_name = experiment_name @project_id = project_id @@ -30,6 +31,7 @@ def initialize(experiment_id:, experiment_name:, project_id:, project_name:, @errors = errors @duration = duration @scores = scores + @summary = summary || build_summary_without_comparison end # Check if the evaluation was successful (no errors) @@ -44,23 +46,18 @@ def failed? !success? end - # Get the experiment summary (lazily computed) - # @return [ExperimentSummary] Summary view model for Formatter - def summary - @summary ||= build_summary - end - # Format the result as a human-readable string (Go SDK format) # @return [String] def to_s - [ + lines = [ "Experiment: #{experiment_name}", - "Project: #{project_name}", - "ID: #{experiment_id}", - "Link: #{permalink}", - "Duration: #{duration.round(4)}s", - "Errors: #{errors.length}" - ].join("\n") + "Project: #{project_name}" + ] + lines << "ID: #{experiment_id}" if experiment_id + lines << "Link: #{permalink}" if permalink + lines << "Duration: #{duration.round(4)}s" + lines << "Errors: #{errors.length}" + lines.join("\n") end # Format the result as a pretty CLI output with box drawing and colors @@ -69,40 +66,22 @@ def to_pretty Formatter.format_experiment_summary(summary) end - # Get statistics for all scorers (lazily computed from scores) - # @return [Hash] Scorer stats keyed by scorer name - def scorer_stats - @scorer_stats ||= build_scorer_stats - end - private - # Build scorer statistics from raw score data - # @return [Hash] Scorer stats keyed by scorer name - def build_scorer_stats - return {} if scores.nil? || scores.empty? - - stats = {} - scores.each do |name, score_values| - next if score_values.empty? - mean = score_values.sum.to_f / score_values.size - stats[name] = ScorerStats.new(name: name, score_mean: mean) - end - stats - end - - # Build experiment summary view model - # @return [ExperimentSummary] Summary with all data for Formatter - def build_summary - ExperimentSummary.new( - project_name: project_name, - experiment_name: experiment_name, - experiment_id: experiment_id, - experiment_url: permalink, - scores: scorer_stats, - duration: duration, - error_count: errors.length, - errors: errors + # Build summary from raw scores when comparison data is unavailable + # @return [ExperimentSummary] + def build_summary_without_comparison + ExperimentSummary.from_raw_scores( + scores || {}, + { + project_name: project_name, + experiment_name: experiment_name, + experiment_id: experiment_id, + experiment_url: permalink, + duration: duration, + error_count: errors.length, + errors: errors + } ) end end diff --git a/lib/braintrust/eval/summary.rb b/lib/braintrust/eval/summary.rb index 2f3639c..d591594 100644 --- a/lib/braintrust/eval/summary.rb +++ b/lib/braintrust/eval/summary.rb @@ -2,31 +2,92 @@ module Braintrust module Eval - # Aggregated statistics for a single scorer across test cases - # @attr name [String] Scorer name - # @attr score_mean [Float] Mean score (0.0 to 1.0) - ScorerStats = Struct.new(:name, :score_mean, keyword_init: true) + # Score summary - unified for both local and comparison modes + # For local mode: diff/improvements/regressions are nil + # @attr name [String] Score name + # @attr score [Float] Average score (0.0 to 1.0) + # @attr diff [Float, nil] Difference vs baseline (percentage, e.g., 0.05 = +5%) + # @attr improvements [Integer, nil] Count of improved test cases + # @attr regressions [Integer, nil] Count of regressed test cases + class ScoreSummary + attr_reader :name, :score, :diff, :improvements, :regressions + + def initialize(name:, score:, diff: nil, improvements: nil, regressions: nil) + @name = name + @score = score + @diff = diff + @improvements = improvements + @regressions = regressions + end + + # Build from raw score values (computes mean) + # @param name [String] Score name + # @param values [Array] Raw score values + # @return [ScoreSummary] + def self.from_values(name, values) + return new(name: name, score: 0.0) if values.empty? + mean = values.sum.to_f / values.size + new(name: name, score: mean) + end + end + + # Metric summary for server-computed metrics (duration, tokens, etc.) + # @attr name [String] Metric name + # @attr metric [Float] Metric value + # @attr unit [String] Unit label (e.g., "ms", "$", "tok") + # @attr diff [Float, nil] Difference vs baseline (percentage) + MetricSummary = Struct.new(:name, :metric, :unit, :diff, keyword_init: true) + + # Comparison metadata for the summary header + # @attr baseline_experiment_id [String, nil] Baseline experiment ID + # @attr baseline_experiment_name [String, nil] Baseline experiment name + ComparisonInfo = Struct.new(:baseline_experiment_id, :baseline_experiment_name, keyword_init: true) # Summary of results from an Experiment # Typically used to generate experiment output # @attr project_name [String] Project name # @attr experiment_name [String] Experiment name - # @attr experiment_id [String] Experiment ID - # @attr experiment_url [String] URL to view experiment in Braintrust UI - # @attr scores [Hash] Scorer stats keyed by scorer name + # @attr experiment_id [String, nil] Experiment ID (nil for local-only mode) + # @attr experiment_url [String, nil] URL to view experiment in Braintrust UI + # @attr scores [Hash] Score summaries keyed by scorer name + # @attr metrics [Hash, nil] Metric summaries (nil for local mode) + # @attr comparison [ComparisonInfo, nil] Comparison metadata (nil for local mode) # @attr duration [Float] Duration in seconds # @attr error_count [Integer] Number of errors # @attr errors [Array] Error messages with locations - ExperimentSummary = Struct.new( - :project_name, - :experiment_name, - :experiment_id, - :experiment_url, - :scores, - :duration, - :error_count, - :errors, - keyword_init: true - ) + class ExperimentSummary + attr_reader :project_name, :experiment_name, :experiment_id, :experiment_url, + :scores, :metrics, :comparison, :duration, :error_count, :errors + + def initialize(project_name:, experiment_name:, experiment_id:, experiment_url:, + scores:, duration:, error_count:, errors:, metrics: nil, comparison: nil) + @project_name = project_name + @experiment_name = experiment_name + @experiment_id = experiment_id + @experiment_url = experiment_url + @scores = scores + @metrics = metrics + @comparison = comparison + @duration = duration + @error_count = error_count + @errors = errors + end + + # Build from raw score values (local mode, no comparison) + # @param raw_scores [Hash>] Raw score values + # @param metadata [Hash] Experiment metadata (project_name, experiment_name, duration, errors, etc.) + # @return [ExperimentSummary] + def self.from_raw_scores(raw_scores, metadata) + scores = (raw_scores || {}).map do |name, values| + [name.to_s, ScoreSummary.from_values(name.to_s, values)] + end.to_h + new( + scores: scores, + metrics: nil, + comparison: nil, + **metadata + ) + end + end end end diff --git a/test/braintrust/api/experiments_test.rb b/test/braintrust/api/experiments_test.rb new file mode 100644 index 0000000..0a5ec12 --- /dev/null +++ b/test/braintrust/api/experiments_test.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require "test_helper" + +class Braintrust::API::ExperimentsTest < Minitest::Test + # Note: These tests require recorded VCR cassettes with valid experiment IDs. + # The experiments endpoint is exercised during the eval_test.rb integration tests. + # These unit tests are placeholders for future direct API testing. + + def test_comparison_method_exists + # Test that the comparison method exists with correct signature + # This is a simple unit test that doesn't require API access + api = Braintrust::API + experiments_class = api::Experiments + + assert experiments_class.instance_methods.include?(:comparison) + end + + def test_experiments_accessor_on_api + skip "Requires API key" unless ENV["BRAINTRUST_API_KEY"] + + # Test that the experiments accessor returns an Experiments instance + VCR.use_cassette("api/init") do + state = get_integration_test_state(enable_tracing: false) + api = Braintrust::API.new(state: state) + + assert_instance_of Braintrust::API::Experiments, api.experiments + end + end +end diff --git a/test/braintrust/eval/formatter_test.rb b/test/braintrust/eval/formatter_test.rb index 05dcd89..3c1f331 100644 --- a/test/braintrust/eval/formatter_test.rb +++ b/test/braintrust/eval/formatter_test.rb @@ -57,10 +57,10 @@ def test_terminal_link_osc8_when_tty end end - def test_format_score_row - # Test formatting a score row - score = Braintrust::Eval::ScorerStats.new(name: "exact_match", score_mean: 0.923) - result = Braintrust::Eval::Formatter.format_score_row(score) + def test_format_simple_score_row + # Test formatting a simple score row (no comparison data) + score = Braintrust::Eval::ScoreSummary.new(name: "exact_match", score: 0.923) + result = Braintrust::Eval::Formatter.format_simple_score_row(score) assert_match(/exact_match/, result) assert_match(/92\.3%/, result) @@ -75,8 +75,8 @@ def test_format_experiment_summary_without_tty experiment_id: "exp-123", experiment_url: "https://braintrust.dev/exp/123", scores: { - "accuracy" => Braintrust::Eval::ScorerStats.new(name: "accuracy", score_mean: 0.85), - "relevance" => Braintrust::Eval::ScorerStats.new(name: "relevance", score_mean: 0.92) + "accuracy" => Braintrust::Eval::ScoreSummary.new(name: "accuracy", score: 0.85), + "relevance" => Braintrust::Eval::ScoreSummary.new(name: "relevance", score: 0.92) }, duration: 1.2345, error_count: 0, @@ -89,13 +89,6 @@ def test_format_experiment_summary_without_tty assert_match(/╭─ Experiment summary/, result) assert_match(/╰─+╯/, result) - # Check metadata - assert_match(/Experiment:.*my-experiment/, result) - assert_match(/Project:.*my-project/, result) - assert_match(/ID:.*exp-123/, result) - assert_match(/Duration:.*1\.2345s/, result) # >= 1s shows as seconds - assert_match(/Errors:.*0/, result) - # Check scores section assert_match(/Scores/, result) assert_match(/accuracy/, result) @@ -110,7 +103,7 @@ def test_format_experiment_summary_without_tty end def test_format_experiment_summary_with_errors - # Test formatting a summary with errors (also tests millisecond duration) + # Test formatting a summary with errors stub_tty(false) do summary = Braintrust::Eval::ExperimentSummary.new( project_name: "my-project", @@ -118,7 +111,7 @@ def test_format_experiment_summary_with_errors experiment_id: "exp-123", experiment_url: "https://braintrust.dev/exp/123", scores: {}, - duration: 0.5, # < 1s, should display as 500ms + duration: 0.5, error_count: 2, errors: [ "Task failed for input 'bad': division by zero", @@ -128,10 +121,6 @@ def test_format_experiment_summary_with_errors result = Braintrust::Eval::Formatter.format_experiment_summary(summary) - # Check metadata shows error count and millisecond duration - assert_match(/Duration:.*500ms/, result) - assert_match(/Errors:.*2/, result) - # Check errors section assert_match(/Errors/, result) # Section header assert_match(/Task failed for input 'bad'/, result) @@ -147,11 +136,27 @@ def test_format_experiment_summary_empty assert_equal "", result end - def test_format_metadata_row - # Test metadata row formatting + def test_format_change_positive + # Test change formatting for positive values + stub_tty(false) do + result = Braintrust::Eval::Formatter.format_change(0.05) # +5% + assert_equal "+5.0%", result + end + end + + def test_format_change_negative + # Test change formatting for negative values + stub_tty(false) do + result = Braintrust::Eval::Formatter.format_change(-0.1) # -10% + assert_equal "-10.0%", result + end + end + + def test_format_change_nil + # Test change formatting for nil values stub_tty(false) do - result = Braintrust::Eval::Formatter.format_metadata_row("Label", "value") - assert_equal "Label: value", result + result = Braintrust::Eval::Formatter.format_change(nil) + assert_equal "-", result end end @@ -229,6 +234,191 @@ def test_wrap_in_box_without_tty end end + # Comparison formatting tests + def test_format_comparison_header + stub_tty(false) do + result = Braintrust::Eval::Formatter.format_comparison_header("baseline-exp", "current-exp") + assert_match(/baseline-exp/, result) + assert_match(/current-exp/, result) + assert_match(/baseline/, result) + assert_match(/comparison/, result) + end + end + + def test_format_comparison_score_row + stub_tty(false) do + score = Braintrust::Eval::ScoreSummary.new( + name: "accuracy", + score: 0.85, + diff: 0.05, + improvements: 3, + regressions: 1 + ) + + result = Braintrust::Eval::Formatter.format_comparison_score_row(score) + + assert_match(/accuracy/, result) + assert_match(/85\.0%/, result) + assert_match(/\+5\.0%/, result) + assert_match(/3/, result) + assert_match(/1/, result) + end + end + + def test_format_metric_row + stub_tty(false) do + metric = Braintrust::Eval::MetricSummary.new( + name: "duration", + metric: 1.25, + unit: "s", + diff: -0.1 + ) + + result = Braintrust::Eval::Formatter.format_metric_row(metric) + + assert_match(/duration/, result) + assert_match(/1\.25s/, result) + assert_match(/-10\.0%/, result) + end + end + + def test_format_score_value + stub_tty(false) do + # Normal value + assert_equal "85.0%", Braintrust::Eval::Formatter.format_score_value(0.85) + # Nil value + assert_equal "-", Braintrust::Eval::Formatter.format_score_value(nil) + # 100% + assert_equal "100.0%", Braintrust::Eval::Formatter.format_score_value(1.0) + # 0% + assert_equal "0.0%", Braintrust::Eval::Formatter.format_score_value(0.0) + end + end + + def test_format_metric_value + stub_tty(false) do + # Normal decimal value + assert_equal "1.25s", Braintrust::Eval::Formatter.format_metric_value(1.25, "s") + # Integer value + assert_equal "150tok", Braintrust::Eval::Formatter.format_metric_value(150.0, "tok") + # Small value + assert_equal "0.0012ms", Braintrust::Eval::Formatter.format_metric_value(0.0012, "ms") + # Nil value + assert_equal "-", Braintrust::Eval::Formatter.format_metric_value(nil, "s") + end + end + + def test_format_count + stub_tty(false) do + # Normal count + assert_equal "5", Braintrust::Eval::Formatter.format_count(5) + # Zero returns dash + assert_equal "-", Braintrust::Eval::Formatter.format_count(0) + # Nil returns dash + assert_equal "-", Braintrust::Eval::Formatter.format_count(nil) + end + end + + def test_has_comparison_data + # With diff data + scores_with_diff = { + "test" => Braintrust::Eval::ScoreSummary.new(name: "test", score: 0.5, diff: 0.1) + } + assert Braintrust::Eval::Formatter.has_comparison_data?(scores_with_diff) + + # Without diff data + scores_without_diff = { + "test" => Braintrust::Eval::ScoreSummary.new(name: "test", score: 0.5) + } + refute Braintrust::Eval::Formatter.has_comparison_data?(scores_without_diff) + + # Empty scores + refute Braintrust::Eval::Formatter.has_comparison_data?({}) + + # Nil scores + refute Braintrust::Eval::Formatter.has_comparison_data?(nil) + end + + def test_format_experiment_summary_with_comparison + stub_tty(false) do + comparison = Braintrust::Eval::ComparisonInfo.new( + baseline_experiment_id: "base-123", + baseline_experiment_name: "baseline-exp" + ) + scores = { + "accuracy" => Braintrust::Eval::ScoreSummary.new( + name: "accuracy", + score: 0.9, + diff: 0.05, + improvements: 2, + regressions: 1 + ) + } + metrics = { + "duration" => Braintrust::Eval::MetricSummary.new( + name: "duration", + metric: 1.5, + unit: "s", + diff: -0.1 + ) + } + + summary = Braintrust::Eval::ExperimentSummary.new( + project_name: "project", + experiment_name: "experiment", + experiment_id: "exp-789", + experiment_url: "https://example.com", + scores: scores, + metrics: metrics, + comparison: comparison, + duration: 10.0, + error_count: 0, + errors: [] + ) + + result = Braintrust::Eval::Formatter.format_experiment_summary(summary) + + # Check comparison header + assert_match(/baseline-exp.*baseline.*experiment.*comparison/i, result) + + # Check scores section with all columns + assert_match(/Scores/, result) + assert_match(/accuracy/, result) + assert_match(/90\.0%/, result) + assert_match(/\+5\.0%/, result) + + # Check metrics section + assert_match(/Metrics/, result) + assert_match(/duration/, result) + assert_match(/1\.5s/, result) + assert_match(/-10\.0%/, result) + end + end + + def test_format_scores_table_header + stub_tty(false) do + header = Braintrust::Eval::Formatter.format_scores_table_header + + assert_match(/Name/, header) + assert_match(/Value/, header) + assert_match(/Change/, header) + assert_match(/Improvements/, header) + assert_match(/Regressions/, header) + end + end + + def test_format_metrics_table_header + stub_tty(false) do + header = Braintrust::Eval::Formatter.format_metrics_table_header + + assert_match(/Name/, header) + assert_match(/Value/, header) + assert_match(/Change/, header) + refute_match(/Improvements/, header) + refute_match(/Regressions/, header) + end + end + private # Helper to stub $stdout.tty? for testing diff --git a/test/braintrust/eval/functions_test.rb b/test/braintrust/eval/functions_test.rb index cef3e91..a1a6f50 100644 --- a/test/braintrust/eval/functions_test.rb +++ b/test/braintrust/eval/functions_test.rb @@ -11,14 +11,14 @@ def setup end def get_test_state_and_api - state = get_integration_test_state - api = Braintrust::API.new(state: state) - [state, api] + rig = get_integration_test_state_with_memory_exporter + api = Braintrust::API.new(state: rig[:state]) + [rig, api] end def test_functions_task_returns_callable VCR.use_cassette("eval_functions/task_callable") do - state, api = get_test_state_and_api + rig, api = get_test_state_and_api # This test verifies that Functions.task returns a callable object # The callable should accept an input and invoke the remote function function_slug = "test-ruby-sdk-task-callable" @@ -46,7 +46,7 @@ def test_functions_task_returns_callable task = Braintrust::Eval::Functions.task( project: @project_name, slug: function_slug, - state: state + state: rig[:state] ) # Should be callable @@ -56,7 +56,7 @@ def test_functions_task_returns_callable def test_functions_task_invokes_remote VCR.use_cassette("eval_functions/task_invoke") do - state, api = get_test_state_and_api + rig, api = get_test_state_and_api # This test verifies that calling the task actually invokes the remote function function_slug = "test-ruby-sdk-task-invoke" @@ -83,7 +83,7 @@ def test_functions_task_invokes_remote task = Braintrust::Eval::Functions.task( project: @project_name, slug: function_slug, - state: state + state: rig[:state] ) result = task.call("world") @@ -96,7 +96,7 @@ def test_functions_task_invokes_remote def test_functions_scorer_returns_scorer VCR.use_cassette("eval_functions/scorer") do - state, api = get_test_state_and_api + rig, api = get_test_state_and_api # This test verifies that Functions.scorer returns a Scorer object function_slug = "test-ruby-sdk-scorer" @@ -124,7 +124,7 @@ def test_functions_scorer_returns_scorer scorer = Braintrust::Eval::Functions.scorer( project: @project_name, slug: function_slug, - state: state + state: rig[:state] ) # Should be a Scorer instance @@ -135,7 +135,7 @@ def test_functions_scorer_returns_scorer def test_use_remote_task_in_eval_run VCR.use_cassette("eval_functions/eval_run") do - state, api = get_test_state_and_api + rig, api = get_test_state_and_api # This test verifies that remote tasks can be used in Eval.run # This is the main use case: calling server-side prompts in evals function_slug = "test-ruby-sdk-eval-task" @@ -163,7 +163,7 @@ def test_use_remote_task_in_eval_run task = Braintrust::Eval::Functions.task( project: @project_name, slug: function_slug, - state: state + state: rig[:state] ) # Use in Eval.run with a simple exact match scorer @@ -181,7 +181,8 @@ def test_use_remote_task_in_eval_run output.to_s.include?(expected) ? 1.0 : 0.0 end ], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -193,7 +194,7 @@ def test_use_remote_task_in_eval_run def test_use_remote_scorer_in_eval_run VCR.use_cassette("eval_functions/remote_scorer") do - state, api = get_test_state_and_api + rig, api = get_test_state_and_api # This test verifies that remote scorers can be used in Eval.run # This tests the "online scorer" functionality function_slug = "test-ruby-sdk-eval-scorer" @@ -230,7 +231,7 @@ def test_use_remote_scorer_in_eval_run scorer = Braintrust::Eval::Functions.scorer( project: @project_name, slug: function_slug, - state: state + state: rig[:state] ) # Simple task that uppercases @@ -246,7 +247,8 @@ def test_use_remote_scorer_in_eval_run ], task: task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) diff --git a/test/braintrust/eval/result_test.rb b/test/braintrust/eval/result_test.rb index 40d4cfd..cee8960 100644 --- a/test/braintrust/eval/result_test.rb +++ b/test/braintrust/eval/result_test.rb @@ -97,8 +97,8 @@ def test_result_requires_all_fields end end - def test_scorer_stats_computes_mean - # Test that scorer_stats computes mean from raw scores + def test_summary_scores_computes_mean + # Test that summary scores computes mean from raw scores result = Braintrust::Eval::Result.new( experiment_id: "exp_123", experiment_name: "my-experiment", @@ -113,17 +113,17 @@ def test_scorer_stats_computes_mean } ) - stats = result.scorer_stats + scores = result.summary.scores - assert_equal 2, stats.size - assert_equal "exact_match", stats["exact_match"].name - assert_equal 0.75, stats["exact_match"].score_mean - assert_equal "relevance", stats["relevance"].name - assert_in_delta 0.8, stats["relevance"].score_mean, 0.001 + assert_equal 2, scores.size + assert_equal "exact_match", scores["exact_match"].name + assert_equal 0.75, scores["exact_match"].score + assert_equal "relevance", scores["relevance"].name + assert_in_delta 0.8, scores["relevance"].score, 0.001 end - def test_scorer_stats_empty_when_no_scores - # Test scorer_stats returns empty hash when no score data + def test_summary_scores_empty_when_no_scores + # Test summary.scores returns empty hash when no score data result = Braintrust::Eval::Result.new( experiment_id: "exp_123", experiment_name: "my-experiment", @@ -134,7 +134,7 @@ def test_scorer_stats_empty_when_no_scores duration: 1.5 ) - assert_equal({}, result.scorer_stats) + assert_equal({}, result.summary.scores) end def test_summary_builds_from_scores @@ -160,7 +160,7 @@ def test_summary_builds_from_scores assert_equal "exp_123", summary.experiment_id assert_equal "https://braintrust.dev/link", summary.experiment_url assert_equal 1, summary.scores.size - assert_in_delta 0.6667, summary.scores["exact_match"].score_mean, 0.001 + assert_in_delta 0.6667, summary.scores["exact_match"].score, 0.001 assert_equal 1.5, summary.duration assert_equal 0, summary.error_count assert_equal [], summary.errors @@ -229,7 +229,7 @@ def test_to_pretty_with_scores end def test_to_pretty_without_scores - # Test to_pretty still works when no score data (shows metadata only) + # Test to_pretty still works when no score data (shows link only) result = Braintrust::Eval::Result.new( experiment_id: "exp_123", experiment_name: "my-experiment", @@ -243,11 +243,7 @@ def test_to_pretty_without_scores output = result.to_pretty assert_match(/Experiment summary/, output) - assert_match(/Experiment:.*my-experiment/, output) - assert_match(/Project:.*my-project/, output) - assert_match(/ID:.*exp_123/, output) - assert_match(/Duration:.*1\.5s/, output) # >= 1s shows as seconds - assert_match(/Errors:.*0/, output) + assert_match(/View results for my-experiment/, output) # Should not have Scores section refute_match(/Scores/, output) end @@ -266,7 +262,6 @@ def test_to_pretty_with_errors output = result.to_pretty - assert_match(/Errors:.*1/, output) assert_match(/Errors/, output) # Errors section header assert_match(/Task failed for input 'bad'/, output) end diff --git a/test/braintrust/eval/summary_test.rb b/test/braintrust/eval/summary_test.rb new file mode 100644 index 0000000..7bcba4f --- /dev/null +++ b/test/braintrust/eval/summary_test.rb @@ -0,0 +1,231 @@ +# frozen_string_literal: true + +require "test_helper" +require "braintrust/eval/summary" + +class Braintrust::Eval::SummaryTest < Minitest::Test + # ScoreSummary tests + def test_score_summary_struct_fields + score = Braintrust::Eval::ScoreSummary.new( + name: "accuracy", + score: 0.85, + diff: 0.05, + improvements: 3, + regressions: 1 + ) + + assert_equal "accuracy", score.name + assert_equal 0.85, score.score + assert_equal 0.05, score.diff + assert_equal 3, score.improvements + assert_equal 1, score.regressions + end + + def test_score_summary_from_values_computes_mean + score = Braintrust::Eval::ScoreSummary.from_values("test", [1.0, 0.5, 0.8]) + + assert_equal "test", score.name + assert_in_delta 0.7667, score.score, 0.001 + assert_nil score.diff + assert_nil score.improvements + assert_nil score.regressions + end + + def test_score_summary_from_values_handles_empty_array + score = Braintrust::Eval::ScoreSummary.from_values("test", []) + + assert_equal "test", score.name + assert_equal 0.0, score.score + end + + def test_score_summary_from_values_with_single_value + score = Braintrust::Eval::ScoreSummary.from_values("test", [0.9]) + + assert_equal "test", score.name + assert_equal 0.9, score.score + end + + def test_score_summary_optional_fields_nil_by_default + score = Braintrust::Eval::ScoreSummary.new(name: "test", score: 0.5) + + assert_equal "test", score.name + assert_equal 0.5, score.score + assert_nil score.diff + assert_nil score.improvements + assert_nil score.regressions + end + + # MetricSummary tests + def test_metric_summary_struct_fields + metric = Braintrust::Eval::MetricSummary.new( + name: "duration", + metric: 1.25, + unit: "s", + diff: -0.05 + ) + + assert_equal "duration", metric.name + assert_equal 1.25, metric.metric + assert_equal "s", metric.unit + assert_equal(-0.05, metric.diff) + end + + def test_metric_summary_with_nil_diff + metric = Braintrust::Eval::MetricSummary.new( + name: "tokens", + metric: 150, + unit: "tok" + ) + + assert_equal "tokens", metric.name + assert_equal 150, metric.metric + assert_equal "tok", metric.unit + assert_nil metric.diff + end + + # ComparisonInfo tests + def test_comparison_info_struct_fields + comparison = Braintrust::Eval::ComparisonInfo.new( + baseline_experiment_id: "base-123", + baseline_experiment_name: "baseline-exp" + ) + + assert_equal "base-123", comparison.baseline_experiment_id + assert_equal "baseline-exp", comparison.baseline_experiment_name + end + + # ExperimentSummary tests + def test_experiment_summary_struct_fields + summary = Braintrust::Eval::ExperimentSummary.new( + project_name: "my-project", + experiment_name: "my-experiment", + experiment_id: "exp-123", + experiment_url: "https://example.com/exp/123", + scores: {}, + metrics: {}, + comparison: nil, + duration: 1.5, + error_count: 0, + errors: [] + ) + + assert_equal "my-project", summary.project_name + assert_equal "my-experiment", summary.experiment_name + assert_equal "exp-123", summary.experiment_id + assert_equal "https://example.com/exp/123", summary.experiment_url + assert_equal({}, summary.scores) + assert_equal({}, summary.metrics) + assert_nil summary.comparison + assert_equal 1.5, summary.duration + assert_equal 0, summary.error_count + assert_equal [], summary.errors + end + + def test_experiment_summary_from_scores + raw_scores = { + "accuracy" => [0.9, 0.8, 0.85], + "relevance" => [1.0, 0.5] + } + metadata = { + project_name: "test-project", + experiment_name: "test-exp", + experiment_id: "exp-456", + experiment_url: "https://example.com", + duration: 2.0, + error_count: 0, + errors: [] + } + + summary = Braintrust::Eval::ExperimentSummary.from_raw_scores(raw_scores, metadata) + + assert_equal "test-project", summary.project_name + assert_equal "test-exp", summary.experiment_name + assert_equal 2, summary.scores.size + assert_in_delta 0.85, summary.scores["accuracy"].score, 0.001 + assert_in_delta 0.75, summary.scores["relevance"].score, 0.001 + assert_nil summary.metrics + assert_nil summary.comparison + end + + def test_experiment_summary_from_scores_with_symbol_keys + raw_scores = { + accuracy: [0.9, 0.8], + relevance: [1.0] + } + metadata = { + project_name: "test", + experiment_name: "test", + experiment_id: nil, + experiment_url: nil, + duration: 1.0, + error_count: 0, + errors: [] + } + + summary = Braintrust::Eval::ExperimentSummary.from_raw_scores(raw_scores, metadata) + + # Keys should be converted to strings + assert_equal 2, summary.scores.size + assert summary.scores.key?("accuracy") + assert summary.scores.key?("relevance") + end + + def test_experiment_summary_from_scores_with_nil_scores + metadata = { + project_name: "test", + experiment_name: "test", + experiment_id: nil, + experiment_url: nil, + duration: 1.0, + error_count: 0, + errors: [] + } + + summary = Braintrust::Eval::ExperimentSummary.from_raw_scores(nil, metadata) + + assert_equal({}, summary.scores) + end + + def test_experiment_summary_with_comparison + comparison = Braintrust::Eval::ComparisonInfo.new( + baseline_experiment_id: "base-123", + baseline_experiment_name: "baseline" + ) + scores = { + "accuracy" => Braintrust::Eval::ScoreSummary.new( + name: "accuracy", + score: 0.9, + diff: 0.05, + improvements: 2, + regressions: 0 + ) + } + metrics = { + "duration" => Braintrust::Eval::MetricSummary.new( + name: "duration", + metric: 1.5, + unit: "s", + diff: -0.1 + ) + } + + summary = Braintrust::Eval::ExperimentSummary.new( + project_name: "project", + experiment_name: "experiment", + experiment_id: "exp-789", + experiment_url: "https://example.com", + scores: scores, + metrics: metrics, + comparison: comparison, + duration: 10.0, + error_count: 0, + errors: [] + ) + + assert_equal "baseline", summary.comparison.baseline_experiment_name + assert_equal 1, summary.scores.size + assert_equal 1, summary.metrics.size + assert_equal 0.05, summary.scores["accuracy"].diff + assert_equal(-0.1, summary.metrics["duration"].diff) + end +end diff --git a/test/braintrust/eval_test.rb b/test/braintrust/eval_test.rb index 3896464..06c9189 100644 --- a/test/braintrust/eval_test.rb +++ b/test/braintrust/eval_test.rb @@ -16,7 +16,7 @@ def test_eval_scorer_helper def test_eval_run_basic VCR.use_cassette("eval/run_basic") do - state = get_integration_test_state + rig = get_integration_test_state_with_memory_exporter task = ->(input) { input.upcase } scorer = Braintrust::Eval.scorer("exact") do |input, expected, output| @@ -32,7 +32,8 @@ def test_eval_run_basic ], task: task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -45,7 +46,7 @@ def test_eval_run_basic def test_eval_run_with_task_error VCR.use_cassette("eval/run_task_error") do - state = get_integration_test_state + rig = get_integration_test_state_with_memory_exporter task = ->(input) { raise "Task failed!" if input == "bad" @@ -65,7 +66,8 @@ def test_eval_run_with_task_error ], task: task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -77,7 +79,7 @@ def test_eval_run_with_task_error def test_eval_run_with_scorer_error VCR.use_cassette("eval/run_scorer_error") do - state = get_integration_test_state + rig = get_integration_test_state_with_memory_exporter task = ->(input) { input.upcase } @@ -95,7 +97,8 @@ def test_eval_run_with_scorer_error ], task: task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -149,7 +152,7 @@ def test_eval_scorer_error_records_exception_event def test_eval_run_with_multiple_scorers VCR.use_cassette("eval/run_multiple_scorers") do - state = get_integration_test_state + rig = get_integration_test_state_with_memory_exporter task = ->(input) { input.upcase } @@ -169,7 +172,8 @@ def test_eval_run_with_multiple_scorers ], task: task, scorers: [scorer1, scorer2], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -179,7 +183,7 @@ def test_eval_run_with_multiple_scorers def test_eval_run_with_callable_task VCR.use_cassette("eval/run_callable_task") do - state = get_integration_test_state + rig = get_integration_test_state_with_memory_exporter callable_task = Class.new do def call(input) @@ -199,7 +203,8 @@ def call(input) ], task: callable_task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -238,7 +243,7 @@ def test_eval_run_validates_task_callable def test_eval_run_with_method_scorer VCR.use_cassette("eval/run_method_scorer") do - state = get_integration_test_state + rig = get_integration_test_state_with_memory_exporter task = ->(input) { input.upcase } # Use a lambda instead of nested method @@ -252,7 +257,8 @@ def test_eval_run_with_method_scorer ], task: task, scorers: [test_method_scorer], # Pass lambda directly - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -304,11 +310,8 @@ def test_eval_task_error_records_exception_on_task_span def test_eval_run_with_tracing VCR.use_cassette("eval/run_with_tracing") do - # Set up test rig for capturing spans (includes Braintrust processor) - rig = setup_otel_test_rig - - # Initialize and login - state = get_integration_test_state + # Use eval test state which gives us both logged-in state and in-memory tracing + rig = get_integration_test_state_with_memory_exporter task = ->(input) { input.upcase } scorer = Braintrust::Eval.scorer("exact") { |i, e, o| (o == e) ? 1.0 : 0.0 } @@ -319,15 +322,17 @@ def test_eval_run_with_tracing cases: [{input: "hello", expected: "HELLO"}], task: task, scorers: [scorer], - state: state, - tracer_provider: rig.tracer_provider, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) assert result.success? # Verify spans were created - spans = rig.drain + # force_flush is already called by Eval.run, so we just need to get the finished spans + rig[:tracer_provider].force_flush + spans = rig[:exporter].finished_spans # Should have: 1 eval span, 1 task span, 1 score span assert_equal 3, spans.length @@ -367,8 +372,8 @@ def test_eval_run_with_tracing # Test dataset integration: dataset as string (same project as experiment) def test_eval_run_with_dataset_string VCR.use_cassette("eval/dataset_string") do - state = get_integration_test_state - api = Braintrust::API.new(state: state) + rig = get_integration_test_state_with_memory_exporter + api = Braintrust::API.new(state: rig[:state]) # Create a test dataset with records project_name = "ruby-sdk-test" @@ -403,7 +408,8 @@ def test_eval_run_with_dataset_string dataset: dataset_name, # String - should fetch from same project task: task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -417,8 +423,8 @@ def test_eval_run_with_dataset_string # Test dataset integration: dataset as hash with name + project def test_eval_run_with_dataset_hash_name_project VCR.use_cassette("eval/dataset_hash_name_project") do - state = get_integration_test_state - api = Braintrust::API.new(state: state) + rig = get_integration_test_state_with_memory_exporter + api = Braintrust::API.new(state: rig[:state]) # Create a test dataset project_name = "ruby-sdk-test" @@ -446,7 +452,8 @@ def test_eval_run_with_dataset_hash_name_project dataset: {name: dataset_name, project: project_name}, task: task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -457,8 +464,8 @@ def test_eval_run_with_dataset_hash_name_project # Test dataset integration: dataset as hash with id def test_eval_run_with_dataset_hash_id VCR.use_cassette("eval/dataset_hash_id") do - state = get_integration_test_state - api = Braintrust::API.new(state: state) + rig = get_integration_test_state_with_memory_exporter + api = Braintrust::API.new(state: rig[:state]) # Create a test dataset project_name = "ruby-sdk-test" @@ -486,7 +493,8 @@ def test_eval_run_with_dataset_hash_id dataset: {id: dataset_id}, # By ID only task: task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -497,8 +505,8 @@ def test_eval_run_with_dataset_hash_id # Test dataset integration: dataset with limit option def test_eval_run_with_dataset_limit VCR.use_cassette("eval/dataset_limit") do - state = get_integration_test_state - api = Braintrust::API.new(state: state) + rig = get_integration_test_state_with_memory_exporter + api = Braintrust::API.new(state: rig[:state]) # Create a test dataset with multiple records project_name = "ruby-sdk-test" @@ -537,7 +545,8 @@ def test_eval_run_with_dataset_limit dataset: {name: dataset_name, project: project_name, limit: 2}, task: task, scorers: [scorer], - state: state, + state: rig[:state], + tracer_provider: rig[:tracer_provider], quiet: true ) @@ -548,27 +557,26 @@ def test_eval_run_with_dataset_limit # Test dataset integration: error when both dataset and cases provided def test_eval_run_with_both_dataset_and_cases_errors - VCR.use_cassette("eval/run_both_dataset_and_cases_error") do - state = get_integration_test_state - - task = ->(input) { input.upcase } - scorer = Braintrust::Eval.scorer("exact") { |i, e, o| (o == e) ? 1.0 : 0.0 } + # This test doesn't actually need VCR since it fails before any API calls + state = get_unit_test_state - # Try to provide both dataset and cases - should raise error - error = assert_raises(ArgumentError) do - Braintrust::Eval.run( - project: "ruby-sdk-test", - experiment: "test-error", - dataset: "some-dataset", - cases: [{input: "test"}], - task: task, - scorers: [scorer], - state: state - ) - end + task = ->(input) { input.upcase } + scorer = Braintrust::Eval.scorer("exact") { |i, e, o| (o == e) ? 1.0 : 0.0 } - assert_match(/mutually exclusive/i, error.message) + # Try to provide both dataset and cases - should raise error + error = assert_raises(ArgumentError) do + Braintrust::Eval.run( + project: "ruby-sdk-test", + experiment: "test-error", + dataset: "some-dataset", + cases: [{input: "test"}], + task: task, + scorers: [scorer], + state: state + ) end + + assert_match(/mutually exclusive/i, error.message) end # ============================================ @@ -750,4 +758,41 @@ def test_eval_run_invalid_parallelism_falls_back_to_sequential assert result.success? assert_equal %w[a b], order end + + # ============================================ + # force_flush verification tests + # ============================================ + + def test_eval_run_calls_force_flush_before_fetching_comparison + VCR.use_cassette("eval/run_basic") do + rig = get_integration_test_state_with_memory_exporter + tracer_provider = rig[:tracer_provider] + + # Track whether force_flush was called + force_flush_called = false + original_force_flush = tracer_provider.method(:force_flush) + + # Use stub to wrap the original method and verify it's called + tracer_provider.stub(:force_flush, -> { + force_flush_called = true + original_force_flush.call + }) do + task = ->(input) { input.upcase } + scorer = Braintrust::Eval.scorer("exact") { |i, e, o| (o == e) ? 1.0 : 0.0 } + + Braintrust::Eval.run( + project: "ruby-sdk-test", + experiment: "test-ruby-sdk-basic", + cases: [{input: "hello", expected: "HELLO"}], + task: task, + scorers: [scorer], + state: rig[:state], + tracer_provider: tracer_provider, + quiet: true + ) + end + + assert force_flush_called, "Eval.run should call force_flush on tracer_provider before fetching comparison" + end + end end diff --git a/test/fixtures/vcr_cassettes/api/init.yml b/test/fixtures/vcr_cassettes/api/init.yml new file mode 100644 index 0000000..e640191 --- /dev/null +++ b/test/fixtures/vcr_cassettes/api/init.yml @@ -0,0 +1,83 @@ +--- +http_interactions: +- request: + method: post + uri: https://www.braintrust.dev/api/apikey/login + body: + encoding: UTF-8 + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - www.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - "*" + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '395' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MzM2Y2Q2YTAtNTIyYS00ZTk1LTlkYTQtNjY2MWUyOWE5ZjJl'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 11 Dec 2025 18:42:27 GMT + Etag: + - '"12n7ok4b5phaz"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - "/api/apikey/login" + X-Nonce: + - MzM2Y2Q2YTAtNTIyYS00ZTk1LTlkYTQtNjY2MWUyOWE5ZjJl + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - iad1::iad1::k6pms-1765478547168-26fe4b7559bd + body: + encoding: UTF-8 + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:42:27 GMT +recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/dataset_hash_id.yml b/test/fixtures/vcr_cassettes/eval/dataset_hash_id.yml index 55950f0..4bf26e9 100644 --- a/test/fixtures/vcr_cassettes/eval/dataset_hash_id.yml +++ b/test/fixtures/vcr_cassettes/eval/dataset_hash_id.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-ZDU3OWQwMTAtYzQzZC00N2YxLWFjNzctNDkxMzkyYmM5YTEx'' *.js.stripe.com + ''nonce-NjE5MjkyOTYtZjMyYy00NmZhLTliYjctYzJmMjIyZjYyYzEz'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:31 GMT + - Thu, 11 Dec 2025 18:52:02 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,21 +71,21 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - ZDU3OWQwMTAtYzQzZC00N2YxLWFjNzctNDkxMzkyYmM5YTEx + - NjE5MjkyOTYtZjMyYy00NmZhLTliYjctYzJmMjIyZjYyYzEz X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::vgjfx-1761319769584-2ffc8b24e6a3 + - iad1::iad1::rrnvl-1765479122704-46651ca8a633 body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:31 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:52:02 GMT - request: method: post uri: https://www.braintrust.dev/api/dataset/register body: encoding: UTF-8 - string: '{"dataset_name":"test-ruby-sdk-dataset-id","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","project_name":"ruby-sdk-test"}' + string: '{"dataset_name":"test-ruby-sdk-dataset-id","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_name":"ruby-sdk-test"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -109,21 +110,22 @@ http_interactions: - '551' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-ZmZmMDdjMjgtNmNkZS00MTM4LTg0MDAtYmNmNGVmZjU2NjQ0'' *.js.stripe.com + ''nonce-MWQyZGI2NjgtMWIzZi00MGVjLWI3MTItZjE4YzAwMzUxMTk4'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:32 GMT + - Thu, 11 Dec 2025 18:52:02 GMT Etag: - - '"phszpfole9fb"' + - '"qcf9ey5drzfb"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -142,18 +144,18 @@ http_interactions: X-Matched-Path: - "/api/dataset/register" X-Nonce: - - ZmZmMDdjMjgtNmNkZS00MTM4LTg0MDAtYmNmNGVmZjU2NjQ0 + - MWQyZGI2NjgtMWIzZi00MGVjLWI3MTItZjE4YzAwMzUxMTk4 X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::p9gw7-1761319771936-769b9ea74a18 + - iad1::iad1::hmwpf-1765479122846-ababeaa0ee74 body: encoding: UTF-8 - string: '{"project":{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null},"dataset":{"id":"f6040259-2f57-463c-ad11-5f0976480ae0","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-dataset-id","description":null,"created":"2025-10-24T14:47:37.853Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null},"found_existing":true}' - recorded_at: Fri, 24 Oct 2025 15:29:32 GMT + string: '{"project":{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null},"dataset":{"id":"02c8546e-38f8-46c1-8b09-df9c08854341","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-id","description":null,"created":"2025-12-11T18:38:01.718Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null},"found_existing":true}' + recorded_at: Thu, 11 Dec 2025 18:52:02 GMT - request: method: post - uri: https://api.braintrust.dev/v1/dataset/f6040259-2f57-463c-ad11-5f0976480ae0/insert + uri: https://staging-api.braintrust.dev/v1/dataset/02c8546e-38f8-46c1-8b09-df9c08854341/insert body: encoding: UTF-8 string: '{"events":[{"input":"test","expected":"TEST"}]}' @@ -165,7 +167,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -182,43 +184,43 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:32 GMT + - Thu, 11 Dec 2025 18:52:03 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 3800ff4e-76c3-4db9-ba76-b1dfac848b52 + - df0f0065-51b0-458f-bcfd-74cced11a2f4 X-Bt-Internal-Trace-Id: - - 68fb9b5c000000004e00408c3245ae99 + - 693b12d3000000006bc905a9b4330aaf X-Amz-Apigw-Id: - - S9U2eEN_IAMEURw= + - Vb_hBFC4oAMEdOQ= Vary: - Origin, Accept-Encoding Etag: - - W/"34-kpTdz8lS672FvIsIVFHZ8Z75loI" + - W/"34-8JSSpoW7pcILZAQfop1UIyGR6/U" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5c-001111f77272d9132d153b73;Parent=46b22aeb9d81f6e3;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12d2-40df3ad046f9aecd1251f81f;Parent=01d3102cc6f6688f;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 f8debc28b6c73eb3dc7540e2ac2f0e18.cloudfront.net (CloudFront), 1.1 f25b89e7ef738cb8bb7e28e041d8fe54.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 06b7f1d77ee52f2e46654588c01fafb0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - QEXg_Jf2pT3XKOlRi2HB04ROe093kcGpWh_Fbaitn7ngDXjZvFR_Jg== + - CFrug_dypMrUJzc_F8iCB5plikTD255ua-Xvxh_HEAcWlJK1g9VG2Q== body: encoding: ASCII-8BIT - string: '{"row_ids":["5dcd0c01-13dd-4bd6-842f-5478c963d4d2"]}' - recorded_at: Fri, 24 Oct 2025 15:29:32 GMT + string: '{"row_ids":["eef9aa47-378d-4752-9f8c-72f8021ee763"]}' + recorded_at: Thu, 11 Dec 2025 18:52:03 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"f6040259-2f57-463c-ad11-5f0976480ae0"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"02c8546e-38f8-46c1-8b09-df9c08854341"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -227,7 +229,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -240,54 +242,51 @@ http_interactions: Content-Type: - application/json Content-Length: - - '3185' + - '2332' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:32 GMT + - Thu, 11 Dec 2025 18:52:03 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - a6e021fa-d559-4505-ba77-0ffbc96bcff5 + - 839fbc96-c832-49be-8899-45e4b5af1a78 X-Bt-Internal-Trace-Id: - - 68fb9b5c0000000052c56f9dddfb5706 + - 693b12d30000000065e73d826870341c X-Amz-Apigw-Id: - - S9U2hFzOoAMEHzA= + - Vb_hIHLcIAMEOPg= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5c-0cbcfbc5668735d17956d33b;Parent=668dcfa397621f76;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12d3-581dd96f052f35bf5a5a2d85;Parent=1192c13247dd7b2d;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Cursor: - - aPuRis2rAAA + - aTsPiuPtAAA Via: - - 1.1 f5527f719bbc0d2932043daaeff80252.cloudfront.net (CloudFront), 1.1 d50d90bbddca57e02d6288d86c88470a.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 67c9dfb38affc3a08786f92219ccc792.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - o7_aLL_9GHCnKNur2O73gmvcChz1LMldLQwPxhsXQ4HgZNr0wr9Jig== + - lhYl9xyrZM5DIwNgJ8dVgjdUxsM56uf2tSohuQ7OM6lqML9_RfyVHA== body: encoding: ASCII-8BIT string: | - {"_pagination_key":"p07564810819080880128","_xact_id":"1000196022105546841","created":"2025-10-24T15:29:32.318Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"5dcd0c01-13dd-4bd6-842f-5478c963d4d2","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"cbe220a1-4f3c-4be0-a910-26bf87f11f59","span_id":"cbe220a1-4f3c-4be0-a910-26bf87f11f59","tags":null} - {"_pagination_key":"p07564802288482189312","_xact_id":"1000196021975380235","created":"2025-10-24T14:56:25.536Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"9346985b-ac19-47e4-81cf-8aa0f0f0176a","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"3816c372-b36e-4e05-88a6-5e86254050bb","span_id":"3816c372-b36e-4e05-88a6-5e86254050bb","tags":null} - {"_pagination_key":"p07564801958911475712","_xact_id":"1000196021970351385","created":"2025-10-24T14:55:09.030Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"91594e1e-bca7-4d82-a9a9-2d294aea954d","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"102467dc-10f4-4a17-94ba-8358587bfe80","span_id":"102467dc-10f4-4a17-94ba-8358587bfe80","tags":null} - {"_pagination_key":"p07564801559978508288","_xact_id":"1000196021964264151","created":"2025-10-24T14:53:36.427Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"210f2cb4-bf0a-4ae3-b7fd-53b70a5bdc4b","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"83d7412d-8dd2-41c7-9895-8a5719c86e75","span_id":"83d7412d-8dd2-41c7-9895-8a5719c86e75","tags":null} - {"_pagination_key":"p07564800887563288576","_xact_id":"1000196021954003909","created":"2025-10-24T14:50:59.550Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"e9c2225b-59ba-4796-98dc-61219ab4828c","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"9b630e57-d51b-4cf0-b316-19c8fb089013","span_id":"9b630e57-d51b-4cf0-b316-19c8fb089013","tags":null} - {"_pagination_key":"p07564800709715558400","_xact_id":"1000196021951290168","created":"2025-10-24T14:50:17.976Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"e5527e26-fe2f-4517-b8ca-0fd138d980be","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"53b1f92e-c259-4559-9e56-e31c194f922c","span_id":"53b1f92e-c259-4559-9e56-e31c194f922c","tags":null} - {"_pagination_key":"p07564800024440930304","_xact_id":"1000196021940833707","created":"2025-10-24T14:47:37.921Z","dataset_id":"f6040259-2f57-463c-ad11-5f0976480ae0","expected":"TEST","id":"32e3891f-69fa-4d89-be8b-b9c691f5fe2f","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"e5e96bf1-209e-4d96-bb90-e39dd5bab4f0","span_id":"e5e96bf1-209e-4d96-bb90-e39dd5bab4f0","tags":null} - recorded_at: Fri, 24 Oct 2025 15:29:32 GMT + {"_pagination_key":"p07582675098489389056","_xact_id":"1000196294692818089","audit_data":[{"_xact_id":"1000196294692818089","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:52:03.010Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","id":"eef9aa47-378d-4752-9f8c-72f8021ee763","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"3be90d25-5d85-4eaf-b573-c4c41a9c0f8a","span_id":"3be90d25-5d85-4eaf-b573-c4c41a9c0f8a","tags":null} + {"_pagination_key":"p07582672561744052224","_xact_id":"1000196294654110427","audit_data":[{"_xact_id":"1000196294654110427","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:13.699Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","id":"2787844c-196e-4446-b5a7-1d255bf2cc15","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"07f805a3-e68b-4990-a5a9-367ea21d3b28","span_id":"07f805a3-e68b-4990-a5a9-367ea21d3b28","tags":null} + {"_pagination_key":"p07582671935010111488","_xact_id":"1000196294644547226","audit_data":[{"_xact_id":"1000196294644547226","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:47.354Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","id":"77a419b5-c01a-45b5-b224-9a2843c1c3fa","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"a62f2dd8-6e60-46bc-904d-be55e49c18bf","span_id":"a62f2dd8-6e60-46bc-904d-be55e49c18bf","tags":null} + {"_pagination_key":"p07582671486812225536","_xact_id":"1000196294637708269","audit_data":[{"_xact_id":"1000196294637708269","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:01.811Z","dataset_id":"02c8546e-38f8-46c1-8b09-df9c08854341","expected":"TEST","id":"a2832a5b-129e-454c-8d0d-6d2483948616","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"af66dac9-542f-4ea8-9b52-922234f4d3f0","span_id":"af66dac9-542f-4ea8-9b52-922234f4d3f0","tags":null} + recorded_at: Thu, 11 Dec 2025 18:52:03 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"f6040259-2f57-463c-ad11-5f0976480ae0"}]},"select":[{"op":"star"}],"limit":1000,"cursor":"aPuRis2rAAA"},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"02c8546e-38f8-46c1-8b09-df9c08854341"}]},"select":[{"op":"star"}],"limit":1000,"cursor":"aTsPiuPtAAA"},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -296,7 +295,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -313,38 +312,38 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:33 GMT + - Thu, 11 Dec 2025 18:52:04 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 17a8f9b4-31e9-483b-a158-de887df9f9fc + - 051702d6-61e8-4398-8185-0e4ac13b07ff X-Bt-Internal-Trace-Id: - - 68fb9b5c0000000032a041840c2be844 + - 693b12d4000000000f6ace58fa9ed1f0 X-Amz-Apigw-Id: - - S9U2lEkkIAMEiqg= + - Vb_hMGfYoAMEBtw= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5c-46cca62608748eca550bdd58;Parent=4993f4968cf0f988;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12d4-3b623580397e1dc97eb44fcc;Parent=2c6c8373ec2719f9;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 bf8d7cb6fca5d51158e1109ca40fe242.cloudfront.net (CloudFront), 1.1 8cdf4e2d4f4070992665477c4dbca0c0.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 97713e58966a50f0173f1cdb4e67aea0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - lvvQgSq6gldMnrfwR4nOxSSEk6x2Aj48ljAzzhRH7pDYgrs3huxboA== + - TxUrF98ermOEIsrylg9AowQfIA-IZXSEKnbAqqa10oyYtM_JsmHfeQ== body: encoding: ASCII-8BIT string: '' - recorded_at: Fri, 24 Oct 2025 15:29:33 GMT + recorded_at: Thu, 11 Dec 2025 18:52:04 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -356,7 +355,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -373,45 +372,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:33 GMT + - Thu, 11 Dec 2025 18:52:04 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 607955de-d262-443d-a6d3-bf72dd3c11a0 + - 41b4e85a-316f-47ca-8d97-4216322f47d5 X-Bt-Internal-Trace-Id: - - 68fb9b5d000000007de0590e1ad67207 + - 693b12d40000000063659c8f902c794d X-Amz-Apigw-Id: - - S9U2pGd1oAMEU7A= + - Vb_hNHJ6oAMEGOQ= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5d-6e4c1c777d42427d634ccead;Parent=528fc2cd1cab4cda;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12d4-5ce797e155af8443337c8fc1;Parent=1ed97a24f9c6da8b;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 bd3e3884ce6fe1fd36336541cce9ec7e.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 9733ed47722484c4e79e66f9a01f7260.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - Fk3lrm3cdhGym9QkfoNZSUkK1c_9bTBCxrBnqaX8OMatSGuoeuy30A== + - 1QyMfb4P66rBP_u-tKWXJiiMQya-BbuTaKChMOXrEgj5pRI8wDrKmQ== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:33 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:52:04 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-id","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-id","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -420,7 +419,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -433,39 +432,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '391' + - '393' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:33 GMT + - Thu, 11 Dec 2025 18:52:04 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 4b6699da-8ccc-4559-83a8-b78adff6c860 + - a849e798-3127-4e03-9bfd-8463691614d2 X-Bt-Internal-Trace-Id: - - 68fb9b5d000000006c2affe40afd57a8 + - 693b12d40000000036c3c1a03b1d73c2 X-Amz-Apigw-Id: - - S9U2rEHEoAMEjeg= + - Vb_hPEAPIAMED3A= Vary: - Origin, Accept-Encoding Etag: - - W/"187-vzidsy1hbvmgrjVQ0RAXiEKH2Rc" + - W/"189-WF+6dM24f2v2isgh7xwCcdaZyKA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b5d-27b77f47569a33c617bb6986;Parent=453497dffe353ea7;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12d4-67146a0f1dbe589f58b75f87;Parent=5658392e0b62064d;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 8a9cdb228e33f8d52a4b42c56ca26590.cloudfront.net (CloudFront), 1.1 76d4de5b65bdf749a3f97445d1b9f4d2.cloudfront.net + - 1.1 77512d19cb7ad3590ec138f5d5b2dd86.cloudfront.net (CloudFront), 1.1 06b7f1d77ee52f2e46654588c01fafb0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - RTikKucW-hlXS7I0JEkE89suRnArte11DMJxtX6vABjgKyXeQB7vVQ== + - cDGejbvUENfbrWVGUo7ShNsVl1DwosDtc3JHd8h6tSdH47_US6sUvw== body: encoding: ASCII-8BIT - string: '{"id":"6e338119-5266-4979-abcf-9930e165bd3b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-id-975ec196","description":null,"created":"2025-10-24T15:29:33.607Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:33 GMT + string: '{"id":"99910465-b769-4856-b3ab-a75ed0c074d9","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-id-abd38611","description":null,"created":"2025-12-11T18:52:04.419Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:52:04 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=99910465-b769-4856-b3ab-a75ed0c074d9 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:52:05 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 4d4a1085-f711-4b1b-93fa-2cf8999ce769 + X-Bt-Internal-Trace-Id: + - 693b12d400000000117d45ae930f2c9a + X-Amz-Apigw-Id: + - Vb_hRH0xIAMEvCg= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b12d4-337b40ab38b983910f5da279;Parent=0f0acb89a21977ba;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 77512d19cb7ad3590ec138f5d5b2dd86.cloudfront.net (CloudFront), 1.1 5104f1902e86dd6fe91fb7c451bbdefe.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - yNJiC8YSdpFQ-QH6KSV17Wi0ooTqJplnRnAHRi5tcv53VWVGbUyxPg== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:52:05 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/dataset_hash_name_project.yml b/test/fixtures/vcr_cassettes/eval/dataset_hash_name_project.yml index 7b7fe44..e9d1dab 100644 --- a/test/fixtures/vcr_cassettes/eval/dataset_hash_name_project.yml +++ b/test/fixtures/vcr_cassettes/eval/dataset_hash_name_project.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-YzM1MmM5ZWEtMjBkNS00OTVlLWEwZmUtMzJjOTExOWVhYTk5'' *.js.stripe.com + ''nonce-ODViYWNjMWUtNTliMS00YTJmLThjOGQtZDBlYTRiMGI4OTE0'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:21 GMT + - Thu, 11 Dec 2025 18:52:12 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,21 +71,21 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - YzM1MmM5ZWEtMjBkNS00OTVlLWEwZmUtMzJjOTExOWVhYTk5 + - ODViYWNjMWUtNTliMS00YTJmLThjOGQtZDBlYTRiMGI4OTE0 X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::lzcff-1761319761796-a72711017b55 + - iad1::iad1::tjx7g-1765479132544-a4a8482beed9 body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:21 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:52:12 GMT - request: method: post uri: https://www.braintrust.dev/api/dataset/register body: encoding: UTF-8 - string: '{"dataset_name":"test-ruby-sdk-dataset-hash","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","project_name":"ruby-sdk-test"}' + string: '{"dataset_name":"test-ruby-sdk-dataset-hash","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_name":"ruby-sdk-test"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -109,21 +110,22 @@ http_interactions: - '553' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-OTQ0OWQ2OTgtOWQzNi00MGM5LThlNzAtZGRjZDdlOWI2ZTA2'' *.js.stripe.com + ''nonce-MTZlZWZmY2ItMzA4MS00MjVlLTg1YzktYTZjZTVlNWI5NzVh'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:21 GMT + - Thu, 11 Dec 2025 18:52:12 GMT Etag: - - '"bixmo10ea2fd"' + - '"zn0wf0p4difd"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -142,18 +144,18 @@ http_interactions: X-Matched-Path: - "/api/dataset/register" X-Nonce: - - OTQ0OWQ2OTgtOWQzNi00MGM5LThlNzAtZGRjZDdlOWI2ZTA2 + - MTZlZWZmY2ItMzA4MS00MjVlLTg1YzktYTZjZTVlNWI5NzVh X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::xn6ps-1761319761907-e3fc138d7f34 + - iad1::iad1::nqq6q-1765479132670-ed28fcab4d99 body: encoding: UTF-8 - string: '{"project":{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null},"dataset":{"id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-dataset-hash","description":null,"created":"2025-10-24T14:47:32.248Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null},"found_existing":true}' - recorded_at: Fri, 24 Oct 2025 15:29:22 GMT + string: '{"project":{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null},"dataset":{"id":"c6c16b01-8b0a-4831-8c0c-6338462c663e","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-hash","description":null,"created":"2025-12-11T18:38:13.127Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null},"found_existing":true}' + recorded_at: Thu, 11 Dec 2025 18:52:12 GMT - request: method: post - uri: https://api.braintrust.dev/v1/dataset/437c6648-948f-4fbb-a2ae-b9fce616f04b/insert + uri: https://staging-api.braintrust.dev/v1/dataset/c6c16b01-8b0a-4831-8c0c-6338462c663e/insert body: encoding: UTF-8 string: '{"events":[{"input":"test","expected":"TEST"}]}' @@ -165,7 +167,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -182,40 +184,40 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:22 GMT + - Thu, 11 Dec 2025 18:52:13 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - c508cd82-44f9-49e5-9fb3-cae21c93f758 + - 8706569d-ffa5-40b4-ac86-e954ab1d144d X-Bt-Internal-Trace-Id: - - 68fb9b5200000000630da34caddf2615 + - 693b12dc00000000386e04bd0d8826e9 X-Amz-Apigw-Id: - - S9U04EG8IAMEngA= + - Vb_ikFjkIAMELng= Vary: - Origin, Accept-Encoding Etag: - - W/"34-yaM4yXvRIXuz8SGv91qmSY8O7jI" + - W/"34-hRbpQT90PBZSSZrTxzYs4MxTBYs" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b52-38e13af93e583e8a6b168c58;Parent=4cd98d3c928b319a;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12dc-3b3c92093f3cb3326e0d50af;Parent=1389c916a30a6bd6;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 d8e93128b8c3fa45992684bc1f50eeb8.cloudfront.net (CloudFront), 1.1 6c9d752a06683b70c3a48f07f80d43b0.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 92a8583818732665d61a5f5d4edba97a.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - HK5xlOExF9rjjxWlavU7AfpnsnftZIyhkxdeDVi5BjL89JHWbyrUtQ== + - POZ3lAdbelhekAsDHZY9CCtTKbOCvEglNZIiTKQWgolo-76tpfK1KQ== body: encoding: ASCII-8BIT - string: '{"row_ids":["0f511326-3913-4f4d-8c57-b72138757906"]}' - recorded_at: Fri, 24 Oct 2025 15:29:22 GMT + string: '{"row_ids":["fb8ec781-15ab-42e3-901e-996842e78dab"]}' + recorded_at: Thu, 11 Dec 2025 18:52:13 GMT - request: method: get - uri: https://api.braintrust.dev/v1/dataset?dataset_name=test-ruby-sdk-dataset-hash&project_name=ruby-sdk-test + uri: https://staging-api.braintrust.dev/v1/dataset?dataset_name=test-ruby-sdk-dataset-hash&project_name=ruby-sdk-test body: encoding: US-ASCII string: '' @@ -227,7 +229,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Authorization: - Bearer response: @@ -242,43 +244,43 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:22 GMT + - Thu, 11 Dec 2025 18:52:13 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - b92523e8-6cf4-4f82-b9fc-894847fd7bf2 + - a9e73bc0-3729-4658-ab77-d2178bdc4cc4 X-Bt-Internal-Trace-Id: - - 68fb9b520000000027f3b9d6acc4fbf5 + - 693b12dd0000000014c27d9e31913b8f X-Amz-Apigw-Id: - - S9U06GaAIAMEsig= + - Vb_inHyRIAMEIzw= Vary: - Origin, Accept-Encoding Etag: - - W/"11e-etr0K7lqT/s5w0XMFHlzAQSUlos" + - W/"11e-BHp0obpkBYOr7euBGhB/ltoT2VU" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b52-563464734927969113d1dec9;Parent=099b64531c9c106a;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12dd-2532310f1f59390733de467a;Parent=75656e407af8ee4a;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 5af2699243b550d789ef9dce0b522ed2.cloudfront.net (CloudFront), 1.1 8ca36406fe3aa11c1641e5bc917c8a74.cloudfront.net + - 1.1 77512d19cb7ad3590ec138f5d5b2dd86.cloudfront.net (CloudFront), 1.1 fc34781ab7aa403dba42cfccdb88981a.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - tMm9tF47lv8Kw5OHiU5Pp1CnbhPMywxjArEpHl2dZyS3XJvlTRmOxA== + - IaZ2t9dWhzrn0xqZZ0oYZ6yqDHZNlMmEzW076ryiswh8ldaIXXcEtQ== body: encoding: ASCII-8BIT - string: '{"objects":[{"id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-dataset-hash","description":null,"created":"2025-10-24T14:47:32.248Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null}]}' - recorded_at: Fri, 24 Oct 2025 15:29:22 GMT + string: '{"objects":[{"id":"c6c16b01-8b0a-4831-8c0c-6338462c663e","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-hash","description":null,"created":"2025-12-11T18:38:13.127Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null}]}' + recorded_at: Thu, 11 Dec 2025 18:52:13 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"437c6648-948f-4fbb-a2ae-b9fce616f04b"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"c6c16b01-8b0a-4831-8c0c-6338462c663e"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -287,7 +289,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -300,54 +302,51 @@ http_interactions: Content-Type: - application/json Content-Length: - - '3185' + - '2332' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:26 GMT + - Thu, 11 Dec 2025 18:52:13 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - aa02e415-72b4-44dd-9a9c-2c47ba110912 + - 4549aaec-5be9-4cd9-af49-7ae1338b0bfe X-Bt-Internal-Trace-Id: - - 68fb9b520000000054657a36c848e050 + - 693b12dd00000000425ff7ae51c9b10a X-Amz-Apigw-Id: - - S9U08HkbIAMEudw= + - Vb_ipGbGIAMEF-Q= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b52-102080064e89440e0a40061c;Parent=405cbdb8b9ed622c;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12dd-2b9189dd18ba4582419174dc;Parent=5e744d30d82eb06c;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Cursor: - - aPuRhMB+AAA + - aTsPlQp1AAA Via: - - 1.1 9750f5ee94b45ad0faba87b3fac2aad6.cloudfront.net (CloudFront), 1.1 d50d90bbddca57e02d6288d86c88470a.cloudfront.net + - 1.1 350c00f07ce2a077574ee5c2ad078d3e.cloudfront.net (CloudFront), 1.1 97713e58966a50f0173f1cdb4e67aea0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - FxNA4vVVBD8sEL_ql_L5rYpF_WivqJWf1XKeXqn8LpbaQUdt-UF4vg== + - They8Of5uOsF5C5UtEvE5Zr-2sbejTHlnWxfgAEVgdGUbeof3IqJVg== body: encoding: ASCII-8BIT string: | - {"_pagination_key":"p07564810775701094400","_xact_id":"1000196022104884918","created":"2025-10-24T15:29:22.059Z","dataset_id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","expected":"TEST","id":"0f511326-3913-4f4d-8c57-b72138757906","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"b2d1984c-1b5b-4381-a865-8f697c24e35e","span_id":"b2d1984c-1b5b-4381-a865-8f697c24e35e","tags":null} - {"_pagination_key":"p07564802271149948928","_xact_id":"1000196021975115766","created":"2025-10-24T14:56:21.835Z","dataset_id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","expected":"TEST","id":"a0e188c6-06ab-4243-a891-13cf39658838","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"6ea8e88a-8c11-4ad1-b7fe-82db33fab21a","span_id":"6ea8e88a-8c11-4ad1-b7fe-82db33fab21a","tags":null} - {"_pagination_key":"p07564801941572354048","_xact_id":"1000196021970086811","created":"2025-10-24T14:55:04.946Z","dataset_id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","expected":"TEST","id":"b1bd97b7-8eea-4178-a655-78686009a34f","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"5938a4ef-3c48-496b-9319-27be046e0bb6","span_id":"5938a4ef-3c48-496b-9319-27be046e0bb6","tags":null} - {"_pagination_key":"p07564801568653836288","_xact_id":"1000196021964396526","created":"2025-10-24T14:53:38.775Z","dataset_id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","expected":"TEST","id":"2f580925-5904-4ecc-ad6e-9d9f1c7ae279","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"c5341054-1e08-4b20-b193-fbf0b647165e","span_id":"c5341054-1e08-4b20-b193-fbf0b647165e","tags":null} - {"_pagination_key":"p07564800904936161280","_xact_id":"1000196021954268998","created":"2025-10-24T14:51:03.734Z","dataset_id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","expected":"TEST","id":"f8c9c1f6-aecb-42d3-bf55-61200a831c3b","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"fb16c2bb-8179-4ee8-8c8b-697b9baee22b","span_id":"fb16c2bb-8179-4ee8-8c8b-697b9baee22b","tags":null} - {"_pagination_key":"p07564800748768264192","_xact_id":"1000196021951886065","created":"2025-10-24T14:50:27.168Z","dataset_id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","expected":"TEST","id":"6cd04d88-c492-412a-a334-1a960eb6630e","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"e0b67914-201a-431e-88b2-cc8f467e1be9","span_id":"e0b67914-201a-431e-88b2-cc8f467e1be9","tags":null} - {"_pagination_key":"p07564799998450073600","_xact_id":"1000196021940437118","created":"2025-10-24T14:47:32.416Z","dataset_id":"437c6648-948f-4fbb-a2ae-b9fce616f04b","expected":"TEST","id":"3c0fe7f9-5900-4d6c-b8a5-ee451f023063","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"663eca2e-391b-485d-b993-d00d56c0a612","span_id":"663eca2e-391b-485d-b993-d00d56c0a612","tags":null} - recorded_at: Fri, 24 Oct 2025 15:29:26 GMT + {"_pagination_key":"p07582675137692434432","_xact_id":"1000196294693416280","audit_data":[{"_xact_id":"1000196294693416280","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:52:12.922Z","dataset_id":"c6c16b01-8b0a-4831-8c0c-6338462c663e","expected":"TEST","id":"fb8ec781-15ab-42e3-901e-996842e78dab","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"af2c9d90-b20b-4709-817f-2dac6ebeaaba","span_id":"af2c9d90-b20b-4709-817f-2dac6ebeaaba","tags":null} + {"_pagination_key":"p07582672552986738688","_xact_id":"1000196294653976801","audit_data":[{"_xact_id":"1000196294653976801","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:11.510Z","dataset_id":"c6c16b01-8b0a-4831-8c0c-6338462c663e","expected":"TEST","id":"71a3ca5e-3006-4d7c-bede-0c30fbecca6f","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"555d3035-97c3-49b5-9559-bda18f999639","span_id":"555d3035-97c3-49b5-9559-bda18f999639","tags":null} + {"_pagination_key":"p07582671982931083264","_xact_id":"1000196294645278442","audit_data":[{"_xact_id":"1000196294645278442","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:58.529Z","dataset_id":"c6c16b01-8b0a-4831-8c0c-6338462c663e","expected":"TEST","id":"bc63978c-90ec-496c-b0b3-73cb77256315","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"42a94b74-4ae8-4ec0-94a1-f59ccf664d0f","span_id":"42a94b74-4ae8-4ec0-94a1-f59ccf664d0f","tags":null} + {"_pagination_key":"p07582671530408345600","_xact_id":"1000196294638373493","audit_data":[{"_xact_id":"1000196294638373493","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:13.218Z","dataset_id":"c6c16b01-8b0a-4831-8c0c-6338462c663e","expected":"TEST","id":"f813fbc9-05fb-46b6-a66d-a812120add0f","input":"test","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"738341e3-e5e8-46f6-9aa1-44fd8724e936","span_id":"738341e3-e5e8-46f6-9aa1-44fd8724e936","tags":null} + recorded_at: Thu, 11 Dec 2025 18:52:13 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"437c6648-948f-4fbb-a2ae-b9fce616f04b"}]},"select":[{"op":"star"}],"limit":1000,"cursor":"aPuRhMB+AAA"},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"c6c16b01-8b0a-4831-8c0c-6338462c663e"}]},"select":[{"op":"star"}],"limit":1000,"cursor":"aTsPlQp1AAA"},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -356,7 +355,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -373,38 +372,38 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:26 GMT + - Thu, 11 Dec 2025 18:52:13 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 0eac9b87-962a-461a-9444-3d195fa4f419 + - cfb4058d-2db4-4807-b99b-1172936d21f4 X-Bt-Internal-Trace-Id: - - 68fb9b5600000000380f42464c44308b + - 693b12dd000000003b8ee6e514cfcb08 X-Amz-Apigw-Id: - - S9U1kFOBoAMEc5w= + - Vb_itEBvIAMEs4Q= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b56-2d03821a6efe04503632a879;Parent=39de2916a29b2535;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12dd-05160a7c0b7d064329f26ca2;Parent=4d47493c17fa9bc5;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 52143757d25f4b31ebf04bc09765f6c0.cloudfront.net (CloudFront), 1.1 d3a2886dcf7e47b1e56baf628c69dc24.cloudfront.net + - 1.1 350c00f07ce2a077574ee5c2ad078d3e.cloudfront.net (CloudFront), 1.1 a14a267d508606220614716764dbbd3e.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - OejrOX9mscMdCI5v982xIIWmM5vRKMmjQdOdXXgbYz0Jr2a13oGr6A== + - vGjM6Q8tyhpglFOVsMnW11tKQr4DZVbD2s7bKGtkIsCf_m6hZibufg== body: encoding: ASCII-8BIT string: '' - recorded_at: Fri, 24 Oct 2025 15:29:26 GMT + recorded_at: Thu, 11 Dec 2025 18:52:13 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -416,7 +415,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -433,45 +432,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:26 GMT + - Thu, 11 Dec 2025 18:52:14 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 52494500-f645-41b8-a9ad-97d0ebf30b5c + - 66051d28-2695-4353-923f-c70ee6909506 X-Bt-Internal-Trace-Id: - - 68fb9b56000000002a1a4a5602c7ae05 + - 693b12dd000000002a7054e4684f8104 X-Amz-Apigw-Id: - - S9U1nEaYIAMEbbg= + - Vb_ivHEdIAMEXvA= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b56-6086488836751f045fcc8c11;Parent=5be33725beec3226;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12dd-1b7fe4bd4ae95e0e63fdc82a;Parent=625fe529224fb96e;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 e80aeefdda01afc3c41fc332ff42e7ac.cloudfront.net (CloudFront), 1.1 cc4ee60e87dead01c9e2d4b985af043e.cloudfront.net + - 1.1 350c00f07ce2a077574ee5c2ad078d3e.cloudfront.net (CloudFront), 1.1 faa1f1cd9e8aec6c42fd30b6d46e49f4.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - insio6UL96qsSoHumY8GK03w3gV7HRrA7Z6PsRGnXr3zbAbZ6g1Mwg== + - SaRsWU9rGE9obbzEV3C88emk1mfGAPnP-7xuII9rd9Li-268DLueGg== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:26 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:52:13 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-hash","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-hash","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -480,7 +479,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -493,39 +492,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '393' + - '395' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:26 GMT + - Thu, 11 Dec 2025 18:52:14 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 62a0c8bb-9472-479e-91bd-27206cb7319e + - 829844df-0702-487b-ba73-7f9c6d16d64b X-Bt-Internal-Trace-Id: - - 68fb9b56000000004fd8f3a32e078cdd + - 693b12de00000000026ea97be6fc542a X-Amz-Apigw-Id: - - S9U1oEE9IAMEK5w= + - Vb_iwFnjoAMETzA= Vary: - Origin, Accept-Encoding Etag: - - W/"189-2gNNMPof7iUUyv6guL2KlU8WidU" + - W/"18b-FoPrkOv56GiinRX5Pm5LcsfSBs4" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b56-51153f31228b1f061646a09f;Parent=76457dd59f665b8c;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12de-71439a0b06c583403a2c6729;Parent=1107698f156ee5c2;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 e80aeefdda01afc3c41fc332ff42e7ac.cloudfront.net (CloudFront), 1.1 06d42d2d80190e168b9494192458b51a.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 734dd528086f4273ac466f3c1bd45b14.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - 3hsfGo5xc31qPYtXwUexTBEQSuZmAwAK3dvs-9IE59sEfgZOOxJfCg== + - hAXUiqj-SJPMMMqmEumDl54c6gTkRmOcFMKrFQWn7FgFur4tBcN-Bg== body: encoding: ASCII-8BIT - string: '{"id":"fd5a2dae-a8d4-4763-b326-61f96bc94069","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-hash-c1a35648","description":null,"created":"2025-10-24T15:29:26.958Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:26 GMT + string: '{"id":"7ef85b28-b1c4-4904-bd24-1ed9790d606e","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-hash-46f4c1c0","description":null,"created":"2025-12-11T18:52:14.176Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:52:14 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=7ef85b28-b1c4-4904-bd24-1ed9790d606e + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:52:14 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - af77246b-4e26-41d5-984e-94511caca30f + X-Bt-Internal-Trace-Id: + - 693b12de00000000248f052bc894374b + X-Amz-Apigw-Id: + - Vb_iyGK8oAMEtMA= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b12de-7bb33ddf49bbd1f075d625b4;Parent=19cdd945a4512465;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 77512d19cb7ad3590ec138f5d5b2dd86.cloudfront.net (CloudFront), 1.1 92a8583818732665d61a5f5d4edba97a.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - Zg-SalBUt24N4Wq99TT3O_W_dlHrlIfAOzx6sW7hytMId-8bUiLPgQ== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:52:14 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/dataset_limit.yml b/test/fixtures/vcr_cassettes/eval/dataset_limit.yml index cdfb588..0c8062f 100644 --- a/test/fixtures/vcr_cassettes/eval/dataset_limit.yml +++ b/test/fixtures/vcr_cassettes/eval/dataset_limit.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-NDlkMGU4NjQtMGVjNy00M2MwLTg5M2MtZmZjMDI4MDZlZTU1'' *.js.stripe.com + ''nonce-ZGRhODk2MjctNzIxMC00NjE4LTlmYzItMTIxMjUwMTE2YTRl'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:20 GMT + - Thu, 11 Dec 2025 19:09:19 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,21 +71,21 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - NDlkMGU4NjQtMGVjNy00M2MwLTg5M2MtZmZjMDI4MDZlZTU1 + - ZGRhODk2MjctNzIxMC00NjE4LTlmYzItMTIxMjUwMTE2YTRl X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::cwc74-1761319760305-e2ed33aa9410 + - iad1::iad1::8vb6j-1765480159423-1516270052f1 body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:20 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 19:09:19 GMT - request: method: post uri: https://www.braintrust.dev/api/dataset/register body: encoding: UTF-8 - string: '{"dataset_name":"test-ruby-sdk-dataset-limit","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","project_name":"ruby-sdk-test"}' + string: '{"dataset_name":"test-ruby-sdk-dataset-limit","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_name":"ruby-sdk-test"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -109,21 +110,22 @@ http_interactions: - '554' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-NDI4MzQ3MWYtNDI5Yy00MmUxLTlkNzQtNDEwYjQ2ZmZiOTZh'' *.js.stripe.com + ''nonce-ZDdjMDIyYjUtZTFmNS00Y2Y0LTk5MzktNTczNDY4YTY4Zjlk'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:20 GMT + - Thu, 11 Dec 2025 19:09:19 GMT Etag: - - '"j7hn1snxn4fe"' + - '"10a7g5jf8uife"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -142,18 +144,18 @@ http_interactions: X-Matched-Path: - "/api/dataset/register" X-Nonce: - - NDI4MzQ3MWYtNDI5Yy00MmUxLTlkNzQtNDEwYjQ2ZmZiOTZh + - ZDdjMDIyYjUtZTFmNS00Y2Y0LTk5MzktNTczNDY4YTY4Zjlk X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::bkdgs-1761319760414-46ecb0f9acee + - iad1::iad1::n9s5l-1765480159870-81e1dc349668 body: encoding: UTF-8 - string: '{"project":{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null},"dataset":{"id":"971ac27f-ef12-488c-a28a-fd37c6b13019","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-dataset-limit","description":null,"created":"2025-10-24T14:47:35.778Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null},"found_existing":true}' - recorded_at: Fri, 24 Oct 2025 15:29:20 GMT + string: '{"project":{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null},"dataset":{"id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-limit","description":null,"created":"2025-12-11T18:38:08.788Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null},"found_existing":true}' + recorded_at: Thu, 11 Dec 2025 19:09:19 GMT - request: method: post - uri: https://api.braintrust.dev/v1/dataset/971ac27f-ef12-488c-a28a-fd37c6b13019/insert + uri: https://staging-api.braintrust.dev/v1/dataset/f15434ad-cc1a-4ea4-9c8c-e8a32582cb67/insert body: encoding: UTF-8 string: '{"events":[{"input":"one","expected":"ONE"},{"input":"two","expected":"TWO"},{"input":"three","expected":"THREE"},{"input":"four","expected":"FOUR"},{"input":"five","expected":"FIVE"}]}' @@ -165,7 +167,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -182,40 +184,40 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:20 GMT + - Thu, 11 Dec 2025 19:09:20 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 41481734-711e-4269-aefc-d07697f37679 + - bd1c667b-10b8-4009-aca2-e07f7f90e1da X-Bt-Internal-Trace-Id: - - 68fb9b50000000006702c56074e660cd + - 693b16e00000000011f2f433a94caaa6 X-Amz-Apigw-Id: - - S9U0pH8AoAMEagw= + - VcCDEEC3oAMEn3Q= Vary: - Origin, Accept-Encoding Etag: - - W/"d0-7Xa5utR8e4hZsK8fums6Kv9NaZQ" + - W/"d0-Wzb4UzA6hjUUi+1cHM1PntwKX0E" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b50-27e87e7415522d666fd94193;Parent=45c7752d9e897939;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b16e0-3c506089000ad3ad23d3a25e;Parent=414b42e662951d42;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 98bc8180e0431e8f05afc9802305f1d2.cloudfront.net (CloudFront), 1.1 8fd21502425077e617fde7325b45e112.cloudfront.net + - 1.1 08bb3c305487b3a7b5b4360d422af708.cloudfront.net (CloudFront), 1.1 92a8583818732665d61a5f5d4edba97a.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - FQk29PFicWU7WCQoBrrDSUjTSf_QzMw1wp-31YTH7DWOkKHD_19RDg== + - O208TdvTdDif4kY8jH2GLKcBBGfVYx8d5rQmEVMc3v6xNP8koZTzcw== body: encoding: ASCII-8BIT - string: '{"row_ids":["03f5aad1-9247-4a54-9b51-3ea7de0a6d47","390bffc7-471e-4aa0-aa57-3655fccae7d7","4ae1a6cb-69bc-490c-b557-871ceb68390d","5f734bb5-8cbd-4dba-977e-beab55ad93ab","28079c9d-0c01-411a-a72e-fb05ecaaffd9"]}' - recorded_at: Fri, 24 Oct 2025 15:29:20 GMT + string: '{"row_ids":["a38ed3e1-c0fe-459a-ab63-4b96ab5d15fe","0ebde395-c4d2-44ca-92fa-e571957ea22b","8dd8f7c5-6926-4867-bbc2-c03579fafb31","fbbbc66d-13a8-4c18-ad3f-13fe95db2ec0","106c4844-4783-49ae-bbee-faaaa06ff803"]}' + recorded_at: Thu, 11 Dec 2025 19:09:20 GMT - request: method: get - uri: https://api.braintrust.dev/v1/dataset?dataset_name=test-ruby-sdk-dataset-limit&project_name=ruby-sdk-test + uri: https://staging-api.braintrust.dev/v1/dataset?dataset_name=test-ruby-sdk-dataset-limit&project_name=ruby-sdk-test body: encoding: US-ASCII string: '' @@ -227,7 +229,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Authorization: - Bearer response: @@ -242,43 +244,43 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:20 GMT + - Thu, 11 Dec 2025 19:09:21 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - a583df59-34aa-4806-99d1-dbc3de049076 + - 04b49db2-8a98-4176-9915-e756c00509cb X-Bt-Internal-Trace-Id: - - 68fb9b500000000059d19bec11c3c28e + - 693b16e1000000001022e783d760722e X-Amz-Apigw-Id: - - S9U0rGxvoAMET4A= + - VcCDWG6VIAMEBXA= Vary: - Origin, Accept-Encoding Etag: - - W/"11f-huufLqMkt8ToOqqGPpxgUG6MwLM" + - W/"11f-1xQSKe+NbQQ3FGn574jf3noysgw" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b50-4cada314368db21c26edf2b2;Parent=1c87473c806a1eda;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b16e1-68c73f414542468a080da24d;Parent=6160f8c381d81a54;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 f8debc28b6c73eb3dc7540e2ac2f0e18.cloudfront.net (CloudFront), 1.1 cc4ee60e87dead01c9e2d4b985af043e.cloudfront.net + - 1.1 14c5b6fdfe1b1584391cee9b27bdcf4a.cloudfront.net (CloudFront), 1.1 d0868637a3730c2afc20b3628b047a9a.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - i3qZoN5zYGIslexWlnTfvp4FBBCgo8dvN09_arRl1WPuVBcZPIEHMA== + - 6SKYvFkB_bnwdg-TRdNvmyfk8X32opNgUvgcrFPTIER1uUHdeV1VIg== body: encoding: ASCII-8BIT - string: '{"objects":[{"id":"971ac27f-ef12-488c-a28a-fd37c6b13019","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-dataset-limit","description":null,"created":"2025-10-24T14:47:35.778Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null}]}' - recorded_at: Fri, 24 Oct 2025 15:29:20 GMT + string: '{"objects":[{"id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-limit","description":null,"created":"2025-12-11T18:38:08.788Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null}]}' + recorded_at: Thu, 11 Dec 2025 19:09:21 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"971ac27f-ef12-488c-a28a-fd37c6b13019"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -287,7 +289,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -300,79 +302,64 @@ http_interactions: Content-Type: - application/json Content-Length: - - '15911' + - '11652' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:21 GMT + - Thu, 11 Dec 2025 19:09:22 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - b951c898-de9d-46e5-b85e-3dee3f1624cc + - b5c0763c-82ee-4481-8a98-c43a087f2a73 X-Bt-Internal-Trace-Id: - - 68fb9b500000000033e3c6662ae1cdf4 + - 693b16e2000000000844e804ede42693 X-Amz-Apigw-Id: - - S9U0tHY1oAMEbbg= + - VcCDZGYRoAMEH9A= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b50-7f5d39730fb40cd73f043ba9;Parent=65340106ed8b6cb4;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b16e2-6692614d70e568592eec5fc7;Parent=0f6a41290960cfaf;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Cursor: - - aPuRiMjoAAA + - aTsPkftuAAA Via: - - 1.1 57a5349e40888d521545fc9b83f270a4.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net + - 1.1 04eb98a9e0ea7d312d38391a3e694d2e.cloudfront.net (CloudFront), 1.1 a97e58ffc6f61ef3eab965cbc26fc6f0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - 0N8C8dD75XgwN13znW6SyBlX2a-ik4JYDFk4nu_1xr_MqL-sSVgfCA== + - whNHRM0MlFoDVafekbwgY3KYblrB8imuw9Pc3WudwVeZARESOs68SA== body: encoding: ASCII-8BIT string: | - {"_pagination_key":"p07564810767049031684","_xact_id":"1000196022104752898","created":"2025-10-24T15:29:20.543Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FIVE","id":"28079c9d-0c01-411a-a72e-fb05ecaaffd9","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"bb79ad2d-ae5c-45b8-b554-932d7bf9665e","span_id":"bb79ad2d-ae5c-45b8-b554-932d7bf9665e","tags":null} - {"_pagination_key":"p07564810767049031683","_xact_id":"1000196022104752898","created":"2025-10-24T15:29:20.543Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FOUR","id":"5f734bb5-8cbd-4dba-977e-beab55ad93ab","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"f95b53d1-897f-4fa0-9a56-9c86e5db1471","span_id":"f95b53d1-897f-4fa0-9a56-9c86e5db1471","tags":null} - {"_pagination_key":"p07564810767049031682","_xact_id":"1000196022104752898","created":"2025-10-24T15:29:20.543Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"THREE","id":"4ae1a6cb-69bc-490c-b557-871ceb68390d","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"ac072cfb-039c-4e15-8252-484b93278de9","span_id":"ac072cfb-039c-4e15-8252-484b93278de9","tags":null} - {"_pagination_key":"p07564810767049031681","_xact_id":"1000196022104752898","created":"2025-10-24T15:29:20.543Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"TWO","id":"390bffc7-471e-4aa0-aa57-3655fccae7d7","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"99dae2ee-442f-4a7f-a899-4c56fcb1f689","span_id":"99dae2ee-442f-4a7f-a899-4c56fcb1f689","tags":null} - {"_pagination_key":"p07564810767049031680","_xact_id":"1000196022104752898","created":"2025-10-24T15:29:20.543Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"ONE","id":"03f5aad1-9247-4a54-9b51-3ea7de0a6d47","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"b619f600-0e13-4831-9ad6-87892e8384d5","span_id":"b619f600-0e13-4831-9ad6-87892e8384d5","tags":null} - {"_pagination_key":"p07564802249427976196","_xact_id":"1000196021974784315","created":"2025-10-24T14:56:14.129Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FIVE","id":"930f8396-5704-4b83-a563-84e559f8adf3","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"c136520a-9501-43d6-a923-c3034c15bc4a","span_id":"c136520a-9501-43d6-a923-c3034c15bc4a","tags":null} - {"_pagination_key":"p07564802249427976195","_xact_id":"1000196021974784315","created":"2025-10-24T14:56:14.129Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FOUR","id":"2582254b-af6a-4996-b7ec-042eeab7344e","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"73f84b71-379a-44f9-b7b1-21facbfd5b10","span_id":"73f84b71-379a-44f9-b7b1-21facbfd5b10","tags":null} - {"_pagination_key":"p07564802249427976194","_xact_id":"1000196021974784315","created":"2025-10-24T14:56:14.129Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"THREE","id":"7e032196-72fb-423c-8a43-cefcbc94527e","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"da6a70b5-b5a6-46bc-9759-a74656bf5b4a","span_id":"da6a70b5-b5a6-46bc-9759-a74656bf5b4a","tags":null} - {"_pagination_key":"p07564802249427976193","_xact_id":"1000196021974784315","created":"2025-10-24T14:56:14.129Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"TWO","id":"7fdf7250-d842-40ee-8eab-a9f648eca6d3","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"e43e73ec-28f5-41b2-bbf3-a056a0f6ecb6","span_id":"e43e73ec-28f5-41b2-bbf3-a056a0f6ecb6","tags":null} - {"_pagination_key":"p07564802249427976192","_xact_id":"1000196021974784315","created":"2025-10-24T14:56:14.129Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"ONE","id":"25f51b54-d155-4115-8ae0-928383e1f741","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"6bd0ee6a-75f8-4c68-8464-71c13bf79cc5","span_id":"6bd0ee6a-75f8-4c68-8464-71c13bf79cc5","tags":null} - {"_pagination_key":"p07564801950266294276","_xact_id":"1000196021970219470","created":"2025-10-24T14:55:07.653Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FIVE","id":"4f0635da-127b-4b52-bc0a-5ae2ebe2a761","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"9bd60509-ae3a-4742-a752-a9767d3f13f6","span_id":"9bd60509-ae3a-4742-a752-a9767d3f13f6","tags":null} - {"_pagination_key":"p07564801950266294275","_xact_id":"1000196021970219470","created":"2025-10-24T14:55:07.653Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FOUR","id":"e2e07c59-091a-45fe-9516-8d935e99e984","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"e9c0e181-5d31-4e8e-a598-6c2de6bece60","span_id":"e9c0e181-5d31-4e8e-a598-6c2de6bece60","tags":null} - {"_pagination_key":"p07564801950266294274","_xact_id":"1000196021970219470","created":"2025-10-24T14:55:07.653Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"THREE","id":"2a552e95-f045-4cbc-867e-11966c6b687c","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"afb7c6a5-556f-4ccb-80f3-bf8b6415a215","span_id":"afb7c6a5-556f-4ccb-80f3-bf8b6415a215","tags":null} - {"_pagination_key":"p07564801950266294273","_xact_id":"1000196021970219470","created":"2025-10-24T14:55:07.653Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"TWO","id":"bb9a0f94-e208-4e04-8ff4-1c4e6896018e","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"04c95e68-a88a-4363-8d9a-5f741160f735","span_id":"04c95e68-a88a-4363-8d9a-5f741160f735","tags":null} - {"_pagination_key":"p07564801950266294272","_xact_id":"1000196021970219470","created":"2025-10-24T14:55:07.653Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"ONE","id":"816fda75-b7bf-4a0e-89a1-075a45f24ea6","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"663c8567-d02a-4d39-9df0-8067f0836a07","span_id":"663c8567-d02a-4d39-9df0-8067f0836a07","tags":null} - {"_pagination_key":"p07564801546970202116","_xact_id":"1000196021964065660","created":"2025-10-24T14:53:33.340Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FIVE","id":"56d848dd-116e-4df0-a3fc-5d1028f6b995","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"19ee8ad1-e620-4d72-87db-b2613276f308","span_id":"19ee8ad1-e620-4d72-87db-b2613276f308","tags":null} - {"_pagination_key":"p07564801546970202115","_xact_id":"1000196021964065660","created":"2025-10-24T14:53:33.340Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FOUR","id":"e9053176-785b-4906-9f84-f6743ddb64d5","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"effff2ca-2d0d-440f-95a1-66f2afde35aa","span_id":"effff2ca-2d0d-440f-95a1-66f2afde35aa","tags":null} - {"_pagination_key":"p07564801546970202114","_xact_id":"1000196021964065660","created":"2025-10-24T14:53:33.340Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"THREE","id":"e0fbf77d-6c3d-46e8-8623-1ce923af8378","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"8ed9b350-6653-4406-a50e-73e62938762b","span_id":"8ed9b350-6653-4406-a50e-73e62938762b","tags":null} - {"_pagination_key":"p07564801546970202113","_xact_id":"1000196021964065660","created":"2025-10-24T14:53:33.340Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"TWO","id":"6d04c38c-a548-4d09-b26e-860d038de9fa","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"4d1a0136-93f1-46fb-8c57-da9d1ace4ed4","span_id":"4d1a0136-93f1-46fb-8c57-da9d1ace4ed4","tags":null} - {"_pagination_key":"p07564801546970202112","_xact_id":"1000196021964065660","created":"2025-10-24T14:53:33.340Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"ONE","id":"979d1b89-073c-4cce-a6e0-631b2166b87b","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"84812c07-fe11-4bd4-ac7e-b4c5b1ac8dc7","span_id":"84812c07-fe11-4bd4-ac7e-b4c5b1ac8dc7","tags":null} - {"_pagination_key":"p07564800896235929604","_xact_id":"1000196021954136243","created":"2025-10-24T14:51:01.212Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FIVE","id":"35d29f22-a1ff-486d-a1f1-80af728b1a41","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"4194f09c-fdff-4a7c-a5c9-a85c1355386c","span_id":"4194f09c-fdff-4a7c-a5c9-a85c1355386c","tags":null} - {"_pagination_key":"p07564800896235929603","_xact_id":"1000196021954136243","created":"2025-10-24T14:51:01.211Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FOUR","id":"884ee16d-4218-429d-a046-ee950f720ba9","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"d5c01630-273f-4c52-943d-d9d345c5c338","span_id":"d5c01630-273f-4c52-943d-d9d345c5c338","tags":null} - {"_pagination_key":"p07564800896235929602","_xact_id":"1000196021954136243","created":"2025-10-24T14:51:01.211Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"THREE","id":"76835cda-c36e-4c03-a817-2327d7498eaf","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"6bb8ce45-ecb8-457e-b787-72d81cec2db2","span_id":"6bb8ce45-ecb8-457e-b787-72d81cec2db2","tags":null} - {"_pagination_key":"p07564800896235929601","_xact_id":"1000196021954136243","created":"2025-10-24T14:51:01.211Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"TWO","id":"9de7f6d2-98bc-4445-b4b5-eb3d55f2eb7e","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"e7f7057c-669a-4304-8c52-3abaa4b8da4a","span_id":"e7f7057c-669a-4304-8c52-3abaa4b8da4a","tags":null} - {"_pagination_key":"p07564800896235929600","_xact_id":"1000196021954136243","created":"2025-10-24T14:51:01.211Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"ONE","id":"901c91ba-b5b5-48bb-a3c5-a5617933f37e","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"f470b4f1-3d6d-4b87-9271-ba1bee1f8dd0","span_id":"f470b4f1-3d6d-4b87-9271-ba1bee1f8dd0","tags":null} - {"_pagination_key":"p07564800714058956804","_xact_id":"1000196021951356443","created":"2025-10-24T14:50:19.156Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FIVE","id":"fc895c97-5c1b-42ae-805f-ea9525206fa8","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"944073d2-d63b-4fc5-8ecb-583ef4895a3e","span_id":"944073d2-d63b-4fc5-8ecb-583ef4895a3e","tags":null} - {"_pagination_key":"p07564800714058956803","_xact_id":"1000196021951356443","created":"2025-10-24T14:50:19.156Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FOUR","id":"f6b97ab7-3901-4dd3-bf5e-f4a97cacd6f6","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"3d3fd002-ce9a-4388-86e7-b1dbb593a280","span_id":"3d3fd002-ce9a-4388-86e7-b1dbb593a280","tags":null} - {"_pagination_key":"p07564800714058956802","_xact_id":"1000196021951356443","created":"2025-10-24T14:50:19.155Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"THREE","id":"c40bd964-74b2-44fb-86ab-8ef46a647142","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"85b49716-44d3-46b5-93a0-af30f4398390","span_id":"85b49716-44d3-46b5-93a0-af30f4398390","tags":null} - {"_pagination_key":"p07564800714058956801","_xact_id":"1000196021951356443","created":"2025-10-24T14:50:19.155Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"TWO","id":"cd0779e5-c420-4fa9-be4c-1e31d87ff810","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"e7bc89bc-7765-4c68-84d1-fca697b0aa41","span_id":"e7bc89bc-7765-4c68-84d1-fca697b0aa41","tags":null} - {"_pagination_key":"p07564800714058956800","_xact_id":"1000196021951356443","created":"2025-10-24T14:50:19.155Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"ONE","id":"9ad0e8bb-0786-4253-aafc-4fd8225fc30c","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"f26b2a26-6d96-494d-ac5f-adb603c3134c","span_id":"f26b2a26-6d96-494d-ac5f-adb603c3134c","tags":null} - {"_pagination_key":"p07564800015771107332","_xact_id":"1000196021940701416","created":"2025-10-24T14:47:35.851Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FIVE","id":"b318dbb7-298b-4d93-8ca2-1c18f3d67143","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"6e0c99d2-cc56-427e-89e1-d43d3796050b","span_id":"6e0c99d2-cc56-427e-89e1-d43d3796050b","tags":null} - {"_pagination_key":"p07564800015771107331","_xact_id":"1000196021940701416","created":"2025-10-24T14:47:35.851Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"FOUR","id":"b15cdb4a-a92d-4b2f-8610-c204acb2bb43","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"8a11f796-eca1-4f48-b35c-4ac4895a1891","span_id":"8a11f796-eca1-4f48-b35c-4ac4895a1891","tags":null} - {"_pagination_key":"p07564800015771107330","_xact_id":"1000196021940701416","created":"2025-10-24T14:47:35.850Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"THREE","id":"4b3c2d69-a5c7-4c7e-9fd9-b412df938f5f","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"33cec0e4-9ea3-4f74-9884-fada7fccac36","span_id":"33cec0e4-9ea3-4f74-9884-fada7fccac36","tags":null} - {"_pagination_key":"p07564800015771107329","_xact_id":"1000196021940701416","created":"2025-10-24T14:47:35.850Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"TWO","id":"e3cd33f9-f285-4b20-964d-1fce43aee05f","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"f68e2b00-74da-4661-9d71-22cdb4e020ff","span_id":"f68e2b00-74da-4661-9d71-22cdb4e020ff","tags":null} - {"_pagination_key":"p07564800015771107328","_xact_id":"1000196021940701416","created":"2025-10-24T14:47:35.850Z","dataset_id":"971ac27f-ef12-488c-a28a-fd37c6b13019","expected":"ONE","id":"31d27741-1b4e-4881-8bac-43d7ae358e40","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"9e706fd1-0e7d-49f3-a4fb-98359b5035f0","span_id":"9e706fd1-0e7d-49f3-a4fb-98359b5035f0","tags":null} - recorded_at: Fri, 24 Oct 2025 15:29:21 GMT + {"_pagination_key":"p07582679549860249604","_xact_id":"1000196294760740618","audit_data":[{"_xact_id":"1000196294760740618","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T19:09:20.091Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"FIVE","id":"106c4844-4783-49ae-bbee-faaaa06ff803","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"e07136b3-eac7-43f7-a4e4-1b37ca6dca10","span_id":"e07136b3-eac7-43f7-a4e4-1b37ca6dca10","tags":null} + {"_pagination_key":"p07582679549860249603","_xact_id":"1000196294760740618","audit_data":[{"_xact_id":"1000196294760740618","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T19:09:20.091Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"FOUR","id":"fbbbc66d-13a8-4c18-ad3f-13fe95db2ec0","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"e059b9bd-0cf5-4ceb-92fd-cff82ce34663","span_id":"e059b9bd-0cf5-4ceb-92fd-cff82ce34663","tags":null} + {"_pagination_key":"p07582679549860249602","_xact_id":"1000196294760740618","audit_data":[{"_xact_id":"1000196294760740618","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T19:09:20.090Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"THREE","id":"8dd8f7c5-6926-4867-bbc2-c03579fafb31","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"c82db177-e1fe-4b89-8a11-21df4e896430","span_id":"c82db177-e1fe-4b89-8a11-21df4e896430","tags":null} + {"_pagination_key":"p07582679549860249601","_xact_id":"1000196294760740618","audit_data":[{"_xact_id":"1000196294760740618","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T19:09:20.090Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"TWO","id":"0ebde395-c4d2-44ca-92fa-e571957ea22b","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"55abd2ff-c3c3-4066-a1d3-b2aa6245ad1c","span_id":"55abd2ff-c3c3-4066-a1d3-b2aa6245ad1c","tags":null} + {"_pagination_key":"p07582679549860249600","_xact_id":"1000196294760740618","audit_data":[{"_xact_id":"1000196294760740618","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T19:09:20.090Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"ONE","id":"a38ed3e1-c0fe-459a-ab63-4b96ab5d15fe","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"30f920f8-bef7-4766-8a78-fbece463e7e9","span_id":"30f920f8-bef7-4766-8a78-fbece463e7e9","tags":null} + {"_pagination_key":"p07582672513725497348","_xact_id":"1000196294653377722","audit_data":[{"_xact_id":"1000196294653377722","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:01.267Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"FIVE","id":"defb5419-f17d-425f-bbcb-bd0f7c2bbd76","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"4315fca6-18df-4bf7-b56b-67684e3f54a6","span_id":"4315fca6-18df-4bf7-b56b-67684e3f54a6","tags":null} + {"_pagination_key":"p07582672513725497347","_xact_id":"1000196294653377722","audit_data":[{"_xact_id":"1000196294653377722","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:01.267Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"FOUR","id":"1b3bfe56-4e2d-4843-a4f8-88c5705abf7c","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"e121a492-12bf-4c1c-9a3e-45229eaddc5d","span_id":"e121a492-12bf-4c1c-9a3e-45229eaddc5d","tags":null} + {"_pagination_key":"p07582672513725497346","_xact_id":"1000196294653377722","audit_data":[{"_xact_id":"1000196294653377722","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:01.267Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"THREE","id":"edf9962a-59e7-44ae-be3f-de580a76eaa4","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"22a1c4a3-7ab5-46d0-969d-455496c6a822","span_id":"22a1c4a3-7ab5-46d0-969d-455496c6a822","tags":null} + {"_pagination_key":"p07582672513725497345","_xact_id":"1000196294653377722","audit_data":[{"_xact_id":"1000196294653377722","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:01.267Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"TWO","id":"5a6fc88d-ca18-4645-ad6c-15764416b050","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"e3cdce92-7672-4a41-94a2-1354b2df3fa3","span_id":"e3cdce92-7672-4a41-94a2-1354b2df3fa3","tags":null} + {"_pagination_key":"p07582672513725497344","_xact_id":"1000196294653377722","audit_data":[{"_xact_id":"1000196294653377722","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:01.267Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"ONE","id":"13151f6e-0b8b-4b2e-a2a0-70f4c259e940","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"a67d7e80-dd99-4476-bac1-845e778df346","span_id":"a67d7e80-dd99-4476-bac1-845e778df346","tags":null} + {"_pagination_key":"p07582671952438886404","_xact_id":"1000196294644813168","audit_data":[{"_xact_id":"1000196294644813168","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:51.085Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"FIVE","id":"09f9320d-c8c8-42ed-af57-c47b3c054713","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"11dd5633-17ca-465e-a237-f662a2298f5d","span_id":"11dd5633-17ca-465e-a237-f662a2298f5d","tags":null} + {"_pagination_key":"p07582671952438886403","_xact_id":"1000196294644813168","audit_data":[{"_xact_id":"1000196294644813168","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:51.085Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"FOUR","id":"4ab8551e-969a-4c29-bc29-fc5f983d5ad6","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"31c1809c-3f0e-459b-97a0-7bd02fa3f797","span_id":"31c1809c-3f0e-459b-97a0-7bd02fa3f797","tags":null} + {"_pagination_key":"p07582671952438886402","_xact_id":"1000196294644813168","audit_data":[{"_xact_id":"1000196294644813168","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:51.085Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"THREE","id":"311d9743-7041-4401-b511-312bbadc4f3a","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"3a4f539d-dbea-448f-8b2d-9fafb97ef0b0","span_id":"3a4f539d-dbea-448f-8b2d-9fafb97ef0b0","tags":null} + {"_pagination_key":"p07582671952438886401","_xact_id":"1000196294644813168","audit_data":[{"_xact_id":"1000196294644813168","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:51.085Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"TWO","id":"60fc5781-f5ca-48dc-b4f4-7ca94b18c4d7","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"91a241d2-4ba6-4298-8b41-998456e116d0","span_id":"91a241d2-4ba6-4298-8b41-998456e116d0","tags":null} + {"_pagination_key":"p07582671952438886400","_xact_id":"1000196294644813168","audit_data":[{"_xact_id":"1000196294644813168","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:51.085Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"ONE","id":"2e52e855-3f62-4e6d-beae-6208a9a2bfc7","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"36468df2-b471-47d0-b793-016f0e0c69d7","span_id":"36468df2-b471-47d0-b793-016f0e0c69d7","tags":null} + {"_pagination_key":"p07582671517271326724","_xact_id":"1000196294638173038","audit_data":[{"_xact_id":"1000196294638173038","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:08.910Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"FIVE","id":"474ad4e0-e710-412f-ad1e-40f5a38db2b4","input":"five","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"d0bc3fd6-f054-45b0-9564-139bc4c62713","span_id":"d0bc3fd6-f054-45b0-9564-139bc4c62713","tags":null} + {"_pagination_key":"p07582671517271326723","_xact_id":"1000196294638173038","audit_data":[{"_xact_id":"1000196294638173038","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:08.909Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"FOUR","id":"456c61e4-b56c-4bae-824a-7bddcf82a90b","input":"four","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"456d86ad-c774-491c-aec1-116a54b88ecd","span_id":"456d86ad-c774-491c-aec1-116a54b88ecd","tags":null} + {"_pagination_key":"p07582671517271326722","_xact_id":"1000196294638173038","audit_data":[{"_xact_id":"1000196294638173038","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:08.909Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"THREE","id":"8d9ba888-7a2e-4f5f-a708-416ac2f01a39","input":"three","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"517f8042-73fb-4c30-a5bf-3fb121cc4c53","span_id":"517f8042-73fb-4c30-a5bf-3fb121cc4c53","tags":null} + {"_pagination_key":"p07582671517271326721","_xact_id":"1000196294638173038","audit_data":[{"_xact_id":"1000196294638173038","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:08.909Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"TWO","id":"089408a6-62ba-4a85-a164-a2dcacf0f408","input":"two","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"4b02456a-edfb-4db8-b986-5ddd421cc10e","span_id":"4b02456a-edfb-4db8-b986-5ddd421cc10e","tags":null} + {"_pagination_key":"p07582671517271326720","_xact_id":"1000196294638173038","audit_data":[{"_xact_id":"1000196294638173038","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:08.909Z","dataset_id":"f15434ad-cc1a-4ea4-9c8c-e8a32582cb67","expected":"ONE","id":"d303a06e-9365-426f-a9c2-aeb5fa1352e3","input":"one","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"e712cf95-a30d-4e12-ba10-4093c0ae281e","span_id":"e712cf95-a30d-4e12-ba10-4093c0ae281e","tags":null} + recorded_at: Thu, 11 Dec 2025 19:09:22 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -384,7 +371,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -401,45 +388,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:21 GMT + - Thu, 11 Dec 2025 19:09:23 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - a29e3419-1385-41cd-ac28-4754aa402c03 + - 432506fc-77a8-4815-89a8-1900ede60287 X-Bt-Internal-Trace-Id: - - 68fb9b51000000001e473fcbf07624e9 + - 693b16e200000000178dfe3073327e20 X-Amz-Apigw-Id: - - S9U0xHHVoAMEK5w= + - VcCDgGwqIAMEZwQ= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b51-34f4d9da34be5b4a36afd5ec;Parent=4183f89d3ab122f3;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b16e2-4b3419e32c6b31365e3b786e;Parent=245e27c410c4215b;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 bf8d7cb6fca5d51158e1109ca40fe242.cloudfront.net (CloudFront), 1.1 f391dfb0806f29cccc5f1df3e1ae836e.cloudfront.net + - 1.1 04eb98a9e0ea7d312d38391a3e694d2e.cloudfront.net (CloudFront), 1.1 c5b4420a76f7dc44d5e569e0747ac050.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - RQXpqeeoZYpK2wZVQ9U5YeoSqdCZq6YvE_A2KORe0GuoVJZ1iILtVA== + - KqOVcGpcn6rKOsTsgYtkGwv1BT12FZUwQsOy_apy8EkbvgU96ltKTQ== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:21 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 19:09:22 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-limit","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-limit","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -448,7 +435,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -461,39 +448,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '394' + - '396' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:21 GMT + - Thu, 11 Dec 2025 19:09:23 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 4cff1dc3-e9bd-4f36-9845-6dd4a90efacf + - ab15c101-a21c-4e1a-8b22-6882fe4cc1f5 X-Bt-Internal-Trace-Id: - - 68fb9b51000000000b8815ca9ad4e40f + - 693b16e3000000005aa8b4a801295af5 X-Amz-Apigw-Id: - - S9U0zG29IAMEJSQ= + - VcCDjHTqIAMEkvQ= Vary: - Origin, Accept-Encoding Etag: - - W/"18a-5FHFzDBSN1dijrHxF+cr7kwvay8" + - W/"18c-fAXxtX9uvbmz7bEPNnlJg/LsdLo" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b51-1c6a7a4a5439df9d20e7f184;Parent=7b7ac6e7841e9caf;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b16e3-112877112f39d8f24672f039;Parent=24fe02d1457582bd;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 98bc8180e0431e8f05afc9802305f1d2.cloudfront.net (CloudFront), 1.1 33aae203c47fd9e0f18a8f3f6d37fbfc.cloudfront.net + - 1.1 04eb98a9e0ea7d312d38391a3e694d2e.cloudfront.net (CloudFront), 1.1 faa1f1cd9e8aec6c42fd30b6d46e49f4.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - JDkF6tr7dazVytBWdL35iM8WTTA_ezvE0G0XXgNPTsbOREk20rcTDg== + - X2VCeEjFI0DA9f9VlCmIlYJUR-cN3SDkGzCniN_BiP8eGX8eiapNHw== body: encoding: ASCII-8BIT - string: '{"id":"d52c067a-1284-43da-8810-f9f08c8d6390","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-limit-54a67883","description":null,"created":"2025-10-24T15:29:21.709Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:21 GMT + string: '{"id":"12b5ad76-f00e-4c47-8919-738bfacdff50","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-limit-7b7efb44","description":null,"created":"2025-12-11T19:09:23.184Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 19:09:23 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=12b5ad76-f00e-4c47-8919-738bfacdff50 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 19:09:24 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 3467c30d-d29d-4bdc-be86-6045cdaa6efd + X-Bt-Internal-Trace-Id: + - 693b16e4000000004456ea171262e786 + X-Amz-Apigw-Id: + - VcCDvECwIAMEaIw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b16e4-36ecf52612e5221039ca6dbd;Parent=5aa36bce04c8d5b6;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 08bb3c305487b3a7b5b4360d422af708.cloudfront.net (CloudFront), 1.1 9f575b6f0217c33f22e6da5419974422.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - SPO4et8z-tIaAd765NNroffV0-2hI1GahPY8TUkwHjXxxPFskqMIvQ== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 19:09:24 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/dataset_string.yml b/test/fixtures/vcr_cassettes/eval/dataset_string.yml index 9e51821..4012a22 100644 --- a/test/fixtures/vcr_cassettes/eval/dataset_string.yml +++ b/test/fixtures/vcr_cassettes/eval/dataset_string.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-MjJkMjQxMTAtOGY0NC00NjlmLWIwYjItZTgzMmM2NDgxNWMw'' *.js.stripe.com + ''nonce-Mjk1M2JkOTktYzdlNi00MmRjLWE2ZDAtMjYwNGQ3MGY2NTBk'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:17 GMT + - Thu, 11 Dec 2025 18:51:47 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,21 +71,21 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - MjJkMjQxMTAtOGY0NC00NjlmLWIwYjItZTgzMmM2NDgxNWMw + - Mjk1M2JkOTktYzdlNi00MmRjLWE2ZDAtMjYwNGQ3MGY2NTBk X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::l5hxc-1761319757879-4e915b56401c + - iad1::iad1::xw2vz-1765479107253-a778c2779638 body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:17 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:51:47 GMT - request: method: post uri: https://www.braintrust.dev/api/dataset/register body: encoding: UTF-8 - string: '{"dataset_name":"test-ruby-sdk-dataset-string","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","project_name":"ruby-sdk-test","description":"Test + string: '{"dataset_name":"test-ruby-sdk-dataset-string","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","project_name":"ruby-sdk-test","description":"Test dataset for eval integration"}' headers: Accept-Encoding: @@ -110,21 +111,22 @@ http_interactions: - '586' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-MGQ1YTYyMzgtMTU5YS00ZWZmLWE3ODEtMjg0MmZmMDg5ZmI2'' *.js.stripe.com + ''nonce-MGFlZGJiNzUtMzUzNy00NzkzLWE1ZTYtYzQwNWY3ZDkzYjI1'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:18 GMT + - Thu, 11 Dec 2025 18:51:47 GMT Etag: - - '"a4x4kpmslpga"' + - '"3cecse9d58ga"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -143,19 +145,19 @@ http_interactions: X-Matched-Path: - "/api/dataset/register" X-Nonce: - - MGQ1YTYyMzgtMTU5YS00ZWZmLWE3ODEtMjg0MmZmMDg5ZmI2 + - MGFlZGJiNzUtMzUzNy00NzkzLWE1ZTYtYzQwNWY3ZDkzYjI1 X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::hq6rh-1761319757997-ac94099633c1 + - iad1::iad1::2x2z8-1765479107388-069096558db4 body: encoding: UTF-8 - string: '{"project":{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null},"dataset":{"id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-dataset-string","description":"Test - dataset for eval integration","created":"2025-10-24T14:47:39.027Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null},"found_existing":true}' - recorded_at: Fri, 24 Oct 2025 15:29:18 GMT + string: '{"project":{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null},"dataset":{"id":"6371684d-d959-4681-a6ff-f0b43917995d","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-string","description":"Test + dataset for eval integration","created":"2025-12-11T18:38:10.725Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null},"found_existing":true}' + recorded_at: Thu, 11 Dec 2025 18:51:47 GMT - request: method: post - uri: https://api.braintrust.dev/v1/dataset/f096af81-b0c5-4859-b0d1-bc00b3f7498b/insert + uri: https://staging-api.braintrust.dev/v1/dataset/6371684d-d959-4681-a6ff-f0b43917995d/insert body: encoding: UTF-8 string: '{"events":[{"input":"hello","expected":"HELLO"},{"input":"world","expected":"WORLD"}]}' @@ -167,7 +169,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -184,40 +186,40 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:18 GMT + - Thu, 11 Dec 2025 18:51:48 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - f1dc96d8-18a9-4b0f-9467-02ffea7be904 + - 369c7342-d351-4e8e-989f-c4cd66d92298 X-Bt-Internal-Trace-Id: - - 68fb9b4e000000004dd488c457cee9fc + - 693b12c3000000003ccd2871e22796ac X-Amz-Apigw-Id: - - S9U0QHgmIAMEagw= + - Vb_enHndoAMEdYQ= Vary: - Origin, Accept-Encoding Etag: - - W/"5b-x0P9Hh0WNOHwYEqQwJliVm4Ldnw" + - W/"5b-2m7QKX8U4Sn92w3Rk91djUTO9Eg" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4e-2bcec0ca35bdbf08496cdb18;Parent=3c52a0e40f153600;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12c3-659eb1eb25cb36bc468f440d;Parent=5348e0931806ce11;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 82139f26335f87e45d45c08d5208817a.cloudfront.net (CloudFront), 1.1 e42e8491a089e2183879e26e61dae708.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 eea48723a858e610f4ce8d4f4093fc88.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - Qm6GVrfALOOT7gl9rhD7KmFHxMYGnEP10I_8SGCbqFIeoEEXmSzI9w== + - 6h1akP1HUOZqPiPCq6KODajsaFJRepSI4JlPOAlQzXduIpXwFn3lsg== body: encoding: ASCII-8BIT - string: '{"row_ids":["d6e323d1-1f09-46bf-badc-5732448eff41","5c663805-48a0-4349-bf62-b87a36843b0e"]}' - recorded_at: Fri, 24 Oct 2025 15:29:18 GMT + string: '{"row_ids":["08c0aa1f-dbcd-4c48-9262-190cdca8ac4a","1cb0d6fd-0d41-4966-a6db-b40bfa47060b"]}' + recorded_at: Thu, 11 Dec 2025 18:51:48 GMT - request: method: get - uri: https://api.braintrust.dev/v1/dataset?dataset_name=test-ruby-sdk-dataset-string&project_name=ruby-sdk-test + uri: https://staging-api.braintrust.dev/v1/dataset?dataset_name=test-ruby-sdk-dataset-string&project_name=ruby-sdk-test body: encoding: US-ASCII string: '' @@ -229,7 +231,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Authorization: - Bearer response: @@ -244,44 +246,44 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:18 GMT + - Thu, 11 Dec 2025 18:51:48 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - e4454fc4-05ba-41fa-adb4-2a07a33fbaef + - 1d337f06-9657-407e-967e-4513847dbd90 X-Bt-Internal-Trace-Id: - - 68fb9b4e0000000057bcf229b9606369 + - 693b12c4000000004e553f45399aed82 X-Amz-Apigw-Id: - - S9U0VFWvIAMEcMA= + - Vb_euGKHIAMEtPw= Vary: - Origin, Accept-Encoding Etag: - - W/"13f-a0MzO8wjtcLQpHCKiM0ff88CfKo" + - W/"13f-RIfUI4VkZaLf/V1PtD6y/dMQruE" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4e-07e38d632340cb8b57ea3176;Parent=3847c527484ea4fe;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12c4-32794b2b44576c3020defb8a;Parent=4eb7a3d0164abf1c;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 bf8d7cb6fca5d51158e1109ca40fe242.cloudfront.net (CloudFront), 1.1 537c1727cc67e6d2567bb61ae0478182.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 5104f1902e86dd6fe91fb7c451bbdefe.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - gfC4Oin8PV_-9SW2OBO-VYs5QZP0y-ndpPvLYByFFsb-ez04Kxz2iw== + - g0F1e2qgLyackDpzKGMeqOJ32N3xj5gRpR5wcZd3dWNdizKoMT-9sA== body: encoding: ASCII-8BIT - string: '{"objects":[{"id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-dataset-string","description":"Test - dataset for eval integration","created":"2025-10-24T14:47:39.027Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null}]}' - recorded_at: Fri, 24 Oct 2025 15:29:18 GMT + string: '{"objects":[{"id":"6371684d-d959-4681-a6ff-f0b43917995d","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-dataset-string","description":"Test + dataset for eval integration","created":"2025-12-11T18:38:10.725Z","deleted_at":null,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null}]}' + recorded_at: Thu, 11 Dec 2025 18:51:48 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"f096af81-b0c5-4859-b0d1-bc00b3f7498b"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"6371684d-d959-4681-a6ff-f0b43917995d"}]},"select":[{"op":"star"}],"limit":1000},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -290,7 +292,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -303,61 +305,55 @@ http_interactions: Content-Type: - application/json Content-Length: - - '6398' + - '4680' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:19 GMT + - Thu, 11 Dec 2025 18:51:48 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - ddff9c8d-da6e-47da-82fc-0d01131f0a75 + - 9398741f-83a0-4aa6-bbd8-cbbe796e14fd X-Bt-Internal-Trace-Id: - - 68fb9b4e000000003c1cd6cb1c7019c0 + - 693b12c40000000009ef314bcd0849cb X-Amz-Apigw-Id: - - S9U0XHYCoAMEinA= + - Vb_ewGKnoAMElYA= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4e-3d40a5e62e288ca80dfc2ecf;Parent=3ef45dfc7d4f56a4;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12c4-7146ed2247160eb63b866ef1;Parent=6749a2e0494702b8;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Cursor: - - aPuRi9FZAAA + - aTsPkwIhAAA Via: - - 1.1 bf8d7cb6fca5d51158e1109ca40fe242.cloudfront.net (CloudFront), 1.1 8fd21502425077e617fde7325b45e112.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 40185075b4d4551b3f16d587d961063c.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - pU6qtaNM85lCn_hM4fXYypvBwI6xja_cgOcLWQNiX9l5XiwCq2LMHg== + - erb3dK0u3pKpxMnvRns4lypqUrU_wGuYrG5ZhUL0LUfKWcMCLfFoPg== body: encoding: ASCII-8BIT string: | - {"_pagination_key":"p07564810762653270017","_xact_id":"1000196022104685824","created":"2025-10-24T15:29:18.118Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"WORLD","id":"5c663805-48a0-4349-bf62-b87a36843b0e","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"38465bd9-c88e-4b09-a0f7-fb889f4efb03","span_id":"38465bd9-c88e-4b09-a0f7-fb889f4efb03","tags":null} - {"_pagination_key":"p07564810762653270016","_xact_id":"1000196022104685824","created":"2025-10-24T15:29:18.117Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"HELLO","id":"d6e323d1-1f09-46bf-badc-5732448eff41","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"9106f621-9914-4274-943f-a72d586cd20a","span_id":"9106f621-9914-4274-943f-a72d586cd20a","tags":null} - {"_pagination_key":"p07564802301532962817","_xact_id":"1000196021975579374","created":"2025-10-24T14:56:28.676Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"WORLD","id":"81cd63cd-5194-46f9-bd99-2f6be9bac4ec","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"5f71f2c1-462d-4f94-9aa2-6b6ddc1a86ab","span_id":"5f71f2c1-462d-4f94-9aa2-6b6ddc1a86ab","tags":null} - {"_pagination_key":"p07564802301532962816","_xact_id":"1000196021975579374","created":"2025-10-24T14:56:28.676Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"HELLO","id":"be1e5315-fe5f-4a04-bba8-5f5dcc5725b2","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"c06a0594-ce3a-4a98-bc76-0f336537ecef","span_id":"c06a0594-ce3a-4a98-bc76-0f336537ecef","tags":null} - {"_pagination_key":"p07564801932895518721","_xact_id":"1000196021969954413","created":"2025-10-24T14:55:02.890Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"WORLD","id":"3089276e-10d9-4a27-be9e-71c802cc34b6","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"87603660-5645-49dd-80df-e86347b239a5","span_id":"87603660-5645-49dd-80df-e86347b239a5","tags":null} - {"_pagination_key":"p07564801932895518720","_xact_id":"1000196021969954413","created":"2025-10-24T14:55:02.890Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"HELLO","id":"0f28b856-dbab-4c67-98f1-4013384def26","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"466f9b57-7eff-4847-b37b-654925504afa","span_id":"466f9b57-7eff-4847-b37b-654925504afa","tags":null} - {"_pagination_key":"p07564801577295216641","_xact_id":"1000196021964528383","created":"2025-10-24T14:53:40.313Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"WORLD","id":"b0ddf05d-7ea3-4258-8c9a-04d1dabc2502","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"5587975e-e806-4b98-9f22-21a8e5c19261","span_id":"5587975e-e806-4b98-9f22-21a8e5c19261","tags":null} - {"_pagination_key":"p07564801577295216640","_xact_id":"1000196021964528383","created":"2025-10-24T14:53:40.312Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"HELLO","id":"cc21d5d6-1a88-4eb6-877d-321e3215027e","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"ec545b27-906e-4f37-aba8-4e288370192d","span_id":"ec545b27-906e-4f37-aba8-4e288370192d","tags":null} - {"_pagination_key":"p07564800913618960385","_xact_id":"1000196021954401487","created":"2025-10-24T14:51:05.529Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"WORLD","id":"2b22dd8f-ca8a-47b1-95ef-45a9e1b41dd0","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"7d531220-f242-4ce0-bead-11cce8cac7b9","span_id":"7d531220-f242-4ce0-bead-11cce8cac7b9","tags":null} - {"_pagination_key":"p07564800913618960384","_xact_id":"1000196021954401487","created":"2025-10-24T14:51:05.528Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"HELLO","id":"ab80363e-9869-4ae6-91d9-468cf85dc427","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"9e8fbd9f-1349-4ac7-9872-aae27c576778","span_id":"9e8fbd9f-1349-4ac7-9872-aae27c576778","tags":null} - {"_pagination_key":"p07564800757433892865","_xact_id":"1000196021952018292","created":"2025-10-24T14:50:28.718Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"WORLD","id":"83940e1a-e90d-4178-aa6e-c4c04df81a0b","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"0144fec9-1079-4a8d-b37c-340d080fe785","span_id":"0144fec9-1079-4a8d-b37c-340d080fe785","tags":null} - {"_pagination_key":"p07564800757433892864","_xact_id":"1000196021952018292","created":"2025-10-24T14:50:28.718Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"HELLO","id":"d6712e66-9fbd-4950-bd70-d2e2c1212ee1","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"8b66803c-a2d4-481b-8979-2c5ae0d2a746","span_id":"8b66803c-a2d4-481b-8979-2c5ae0d2a746","tags":null} - {"_pagination_key":"p07564800028797632513","_xact_id":"1000196021940900185","created":"2025-10-24T14:47:39.145Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"WORLD","id":"985a1de4-cdef-41b6-b415-e6d515e0d29c","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"a3131d67-3e8e-4686-97f6-0da978274110","span_id":"a3131d67-3e8e-4686-97f6-0da978274110","tags":null} - {"_pagination_key":"p07564800028797632512","_xact_id":"1000196021940900185","created":"2025-10-24T14:47:39.145Z","dataset_id":"f096af81-b0c5-4859-b0d1-bc00b3f7498b","expected":"HELLO","id":"f6b2016b-aaf5-455e-be96-1daa8532980d","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","root_span_id":"c8bc42f2-09c7-4df8-8bb7-465e2054907e","span_id":"c8bc42f2-09c7-4df8-8bb7-465e2054907e","tags":null} - recorded_at: Fri, 24 Oct 2025 15:29:19 GMT + {"_pagination_key":"p07582675028953071617","_xact_id":"1000196294691757049","audit_data":[{"_xact_id":"1000196294691757049","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:51:47.591Z","dataset_id":"6371684d-d959-4681-a6ff-f0b43917995d","expected":"WORLD","id":"1cb0d6fd-0d41-4966-a6db-b40bfa47060b","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"e4bb7fd8-50de-4124-a3c5-931c7ceb86ba","span_id":"e4bb7fd8-50de-4124-a3c5-931c7ceb86ba","tags":null} + {"_pagination_key":"p07582675028953071616","_xact_id":"1000196294691757049","audit_data":[{"_xact_id":"1000196294691757049","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:51:47.590Z","dataset_id":"6371684d-d959-4681-a6ff-f0b43917995d","expected":"HELLO","id":"08c0aa1f-dbcd-4c48-9262-190cdca8ac4a","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"c82f6bd1-f142-4b67-934d-3e2377b2b828","span_id":"c82f6bd1-f142-4b67-934d-3e2377b2b828","tags":null} + {"_pagination_key":"p07582672535480369153","_xact_id":"1000196294653709675","audit_data":[{"_xact_id":"1000196294653709675","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:07.109Z","dataset_id":"6371684d-d959-4681-a6ff-f0b43917995d","expected":"WORLD","id":"c28ad1a0-32b7-46e1-8110-30736a03cf92","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"b3d8b7c9-f192-48cb-bd31-ca648e1e3ebe","span_id":"b3d8b7c9-f192-48cb-bd31-ca648e1e3ebe","tags":null} + {"_pagination_key":"p07582672535480369152","_xact_id":"1000196294653709675","audit_data":[{"_xact_id":"1000196294653709675","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:42:07.109Z","dataset_id":"6371684d-d959-4681-a6ff-f0b43917995d","expected":"HELLO","id":"4ace1bef-af11-4924-b3d7-f5a786dabda4","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"38a4f964-7d23-48d2-9ccb-c585c68d5341","span_id":"38a4f964-7d23-48d2-9ccb-c585c68d5341","tags":null} + {"_pagination_key":"p07582671961176014849","_xact_id":"1000196294644946486","audit_data":[{"_xact_id":"1000196294644946486","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:53.487Z","dataset_id":"6371684d-d959-4681-a6ff-f0b43917995d","expected":"WORLD","id":"389802f0-5390-4766-b8f0-7f4b3e0d7133","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"6ef02790-af17-48cd-894e-cd1b5fb3713e","span_id":"6ef02790-af17-48cd-894e-cd1b5fb3713e","tags":null} + {"_pagination_key":"p07582671961176014848","_xact_id":"1000196294644946486","audit_data":[{"_xact_id":"1000196294644946486","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:39:53.487Z","dataset_id":"6371684d-d959-4681-a6ff-f0b43917995d","expected":"HELLO","id":"1c5c1e6b-520d-4e94-b4ce-0a2f0fb7ffdf","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"cc9ff9ec-c76c-4c7a-bcad-6d4e6fe22687","span_id":"cc9ff9ec-c76c-4c7a-bcad-6d4e6fe22687","tags":null} + {"_pagination_key":"p07582671521678688257","_xact_id":"1000196294638240289","audit_data":[{"_xact_id":"1000196294638240289","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:10.824Z","dataset_id":"6371684d-d959-4681-a6ff-f0b43917995d","expected":"WORLD","id":"e8fe5304-e18f-44a4-8e1f-cf3995fd0df7","input":"world","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"a39d849b-58e7-4dd4-a4fa-b8ae49d927e9","span_id":"a39d849b-58e7-4dd4-a4fa-b8ae49d927e9","tags":null} + {"_pagination_key":"p07582671521678688256","_xact_id":"1000196294638240289","audit_data":[{"_xact_id":"1000196294638240289","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"comments":null,"created":"2025-12-11T18:38:10.823Z","dataset_id":"6371684d-d959-4681-a6ff-f0b43917995d","expected":"HELLO","id":"6fb7e56a-9ded-48e8-85d3-b44c2d7f9710","input":"hello","is_root":true,"metadata":null,"origin":null,"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","root_span_id":"5a516e88-896c-45fc-a2e2-34848ab73344","span_id":"5a516e88-896c-45fc-a2e2-34848ab73344","tags":null} + recorded_at: Thu, 11 Dec 2025 18:51:48 GMT - request: method: post - uri: https://api.braintrust.dev/btql + uri: https://staging-api.braintrust.dev/btql body: encoding: UTF-8 - string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"f096af81-b0c5-4859-b0d1-bc00b3f7498b"}]},"select":[{"op":"star"}],"limit":1000,"cursor":"aPuRi9FZAAA"},"fmt":"jsonl"}' + string: '{"query":{"from":{"op":"function","name":{"op":"ident","name":["dataset"]},"args":[{"op":"literal","value":"6371684d-d959-4681-a6ff-f0b43917995d"}]},"select":[{"op":"star"}],"limit":1000,"cursor":"aTsPkwIhAAA"},"fmt":"jsonl"}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -366,7 +362,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -383,38 +379,38 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:19 GMT + - Thu, 11 Dec 2025 18:51:48 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - c457f1a9-649d-44f1-a8fc-2ea2356abded + - 48f87e42-a278-4bae-867b-f63e780182b8 X-Bt-Internal-Trace-Id: - - 68fb9b4f0000000010ec0ac3b0c27960 + - 693b12c40000000031d3251d4a246aef X-Amz-Apigw-Id: - - S9U0bGDcoAMEfUg= + - Vb_e0EkToAMEUJw= Vary: - Origin Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4f-5fa9b8e004119e97478c9cce;Parent=5f3d45f7eb21ccbf;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12c4-0dc4d4f702a17f0361526945;Parent=4936f11c2c561009;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 0ac640943c2918c03a0350f4e8b083a8.cloudfront.net (CloudFront), 1.1 50670fc09f8465be7ae4adcf6e33ab7a.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 0f73566e099d1b7819d92fde74c3925e.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - HAyhR9mWLK6ZCaMoDBkU74sIypwtYDE48rrZrx3aTTQdR5xv6_kV5Q== + - xaUaRfut-wP7zW25F5AxZ6I82pyWdXsgA5RhRiM31h2S340_U4wyEg== body: encoding: ASCII-8BIT string: '' - recorded_at: Fri, 24 Oct 2025 15:29:19 GMT + recorded_at: Thu, 11 Dec 2025 18:51:48 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -426,7 +422,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -443,45 +439,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:19 GMT + - Thu, 11 Dec 2025 18:51:49 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - d6117b03-66e7-438b-8801-eb6dcbc0dd5d + - 673f3053-6ee6-4bff-a61d-b81aee4c4a4b X-Bt-Internal-Trace-Id: - - 68fb9b4f000000007f63434de00ce06d + - 693b12c5000000004bb0d83a43a9258e X-Amz-Apigw-Id: - - S9U0eGgUIAMEGkw= + - Vb_e2E2IoAMEDCQ= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4f-4e64f44434fe9e0c2a6b3de7;Parent=5962d3a64ed696f1;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12c5-0929bbf40cc470d442554541;Parent=1fe18189e5b2bf61;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 bf8d7cb6fca5d51158e1109ca40fe242.cloudfront.net (CloudFront), 1.1 c4ce298584668e99f320a46c88c4a04a.cloudfront.net + - 1.1 77512d19cb7ad3590ec138f5d5b2dd86.cloudfront.net (CloudFront), 1.1 06b7f1d77ee52f2e46654588c01fafb0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - GesAazZ0PQWbRgChduYdy80sRusSPhz8PfkG12aF2XrcxGFPncGjQg== + - xxCe9tlGNybA-jAmdbsZLm4wjZN_hyuez273YDunoTUf8e3TAJ8I0g== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:19 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:51:49 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-dataset-string","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-dataset-string","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -490,7 +486,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -503,39 +499,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '403' + - '405' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:19 GMT + - Thu, 11 Dec 2025 18:51:49 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - d8615d47-0672-4b10-9e32-24e440c40ec6 + - 8aaf4ee7-b97c-4f17-a087-ed53372af9a2 X-Bt-Internal-Trace-Id: - - 68fb9b4f000000002b36117115cc6caf + - 693b12c50000000040b24a9c4eeefefb X-Amz-Apigw-Id: - - S9U0fELnoAMEBCA= + - Vb_e3FWMoAMEBMA= Vary: - Origin, Accept-Encoding Etag: - - W/"193-Z59/frEBRXDNCnKijGRdxtfNXfc" + - W/"195-ymcPv1rezYbwcUZ5rYE1RiOE+k8" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4f-6389292644f1716c022b877e;Parent=188cf81cd830bc3b;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12c5-16a96caa5ce27b97420b7597;Parent=1ebd94655945e1f6;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 0ac640943c2918c03a0350f4e8b083a8.cloudfront.net (CloudFront), 1.1 76d4de5b65bdf749a3f97445d1b9f4d2.cloudfront.net + - 1.1 350c00f07ce2a077574ee5c2ad078d3e.cloudfront.net (CloudFront), 1.1 168a3079f6c62f465885c1d82b0b1d5c.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - A_pxjIeKEDKRrTm5EuC2B_YvPH5AreESg7HboB8L7yTMdPoT19TuBQ== + - 4s9PSlWE2DGWP1aJfKtqzbt4O4AjqXXbtzQvvt21FZC9WRpoUzKGTg== body: encoding: ASCII-8BIT - string: '{"id":"e0b3fe15-801b-43a7-8c4e-d315e6893b79","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-exp-dataset-string-e44eb59f","description":null,"created":"2025-10-24T15:29:19.744Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:19 GMT + string: '{"id":"2784b73f-1107-4860-92bb-9984b5b0f14d","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-exp-dataset-string-c13d128d","description":null,"created":"2025-12-11T18:51:49.265Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:51:49 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=2784b73f-1107-4860-92bb-9984b5b0f14d + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:51:49 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - cc1ac2aa-99af-406e-8602-926c17de4a33 + X-Bt-Internal-Trace-Id: + - 693b12c5000000001e8b7136abc201ac + X-Amz-Apigw-Id: + - Vb_e5Gk7IAMEYVw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b12c5-27f0c86b01ba45dd55b4caa5;Parent=3f9255462117b7a4;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 350c00f07ce2a077574ee5c2ad078d3e.cloudfront.net (CloudFront), 1.1 b2a97308187ed38f56c38676d5710b7e.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - atPzgf4c1vnZUdbnrkD3NfVeCXuLtwwJUgP27p6FKkLmKT5MRnEmQg== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:51:49 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/run_basic.yml b/test/fixtures/vcr_cassettes/eval/run_basic.yml index 2e44160..93cbb1b 100644 --- a/test/fixtures/vcr_cassettes/eval/run_basic.yml +++ b/test/fixtures/vcr_cassettes/eval/run_basic.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-ODU5NThlZjQtYTc4Mi00NzNjLThhNjUtZGM0MDI3ZmY0ZjU5'' *.js.stripe.com + ''nonce-ZjIxNzFmNzgtNjU5ZC00NGFjLWE0MjEtM2Y5MWI5NWRmZjlj'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:28 GMT + - Thu, 11 Dec 2025 18:48:01 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - ODU5NThlZjQtYTc4Mi00NzNjLThhNjUtZGM0MDI3ZmY0ZjU5 + - ZjIxNzFmNzgtNjU5ZC00NGFjLWE0MjEtM2Y5MWI5NWRmZjlj X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::6wxfx-1761319768580-d594fd959f1b + - iad1::iad1::gbx7c-1765478881579-7cda303d37ab body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:28 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:48:01 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -110,45 +111,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:28 GMT + - Thu, 11 Dec 2025 18:48:01 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - cd8a7e4c-d3ef-4fa0-b138-2e4e5930e4fd + - 07c90f66-bcd7-4c38-9f55-b550fe8ed054 X-Bt-Internal-Trace-Id: - - 68fb9b58000000004ca6a91581398951 + - 693b11e100000000225d22c7d41bee54 X-Amz-Apigw-Id: - - S9U17EeoIAMEONQ= + - Vb-7WF6YIAMEZdQ= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b58-000a3c517f18350e03e42903;Parent=471f0f1b80eeaacd;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b11e1-378a32de324853ae54dae824;Parent=5173bbd4eca1e7c9;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 ad22d4e4410fd07809425488bf6e79be.cloudfront.net (CloudFront), 1.1 3a80316ee74668c7049eb31054a07168.cloudfront.net + - 1.1 f00bb95b8c55d1da29ae14415a8da844.cloudfront.net (CloudFront), 1.1 c5b4420a76f7dc44d5e569e0747ac050.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - kDkai0l1a7Mme8JKHy7lcnfw4hVES24kxJxGeuFqC2Vlj3quuuetUQ== + - dBqYoS2upAc2XqVb_Zx1PNGmEjZJZZ9SLTYUKWiXvxJVjeA9yLtV7w== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:28 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:48:01 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-basic","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-basic","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -157,7 +158,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -170,39 +171,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '390' + - '392' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:28 GMT + - Thu, 11 Dec 2025 18:48:02 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 5f77d605-11b4-4416-9ae5-73957472a435 + - 92902d17-2a5d-489a-95fc-15b8ecbfbf1b X-Bt-Internal-Trace-Id: - - 68fb9b5800000000590a298f49ec6f62 + - 693b11e20000000050a8c81ac06cecf0 X-Amz-Apigw-Id: - - S9U18EhuIAMEevw= + - Vb-7YFgDIAMETUw= Vary: - Origin, Accept-Encoding Etag: - - W/"186-fXNEnjeggAwKUpBVc0djCdNIhgM" + - W/"188-QCOEEZALhJJSyqDl5RSYi9JLMkk" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b58-7c9b2c156f502c9e0b04a7de;Parent=1dba9f7bbfc8676b;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b11e2-4a370d9d7688e78467b5b884;Parent=4e231071155b6b1f;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 6e202b767e6bdee837ba15ada7e3120e.cloudfront.net (CloudFront), 1.1 bd3e3884ce6fe1fd36336541cce9ec7e.cloudfront.net + - 1.1 f106673324d6e3006e921c9e92055438.cloudfront.net (CloudFront), 1.1 06b7f1d77ee52f2e46654588c01fafb0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - jbNsw2uOz9g5mXzittSE4rZKsgcCs4rVHqQa6f1CptSBTuOC3u9cjQ== + - iJa3s4GqAN4_ry7EII3Zmnz7AsoAbQKwTNJWhgVY2j-1OW0B3cwkmg== body: encoding: ASCII-8BIT - string: '{"id":"d16ca3bc-8cf3-4567-ac85-cf27a862f36e","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-basic-1e6875b4","description":null,"created":"2025-10-24T15:29:28.940Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:28 GMT + string: '{"id":"5fde71d5-a520-42f1-97cd-dee3bc7f6d51","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-basic-ddecc4ee","description":null,"created":"2025-12-11T18:48:02.206Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:48:02 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=5fde71d5-a520-42f1-97cd-dee3bc7f6d51 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:48:03 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 3a72ba7e-a6fe-4552-be8e-1f25237b61be + X-Bt-Internal-Trace-Id: + - 693b11e20000000013c8112361ad9438 + X-Amz-Apigw-Id: + - Vb-7bEw5oAMENOA= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b11e2-692f7ac60518f6130ce0ffb4;Parent=541f73977b7cf7ad;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 57cf2ea4206ff643a898087150f0df7a.cloudfront.net (CloudFront), 1.1 8f8f56e20a7e26014a52398627840a50.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 2umwLv-piXvHL6rc9s3o1Fyq8wGKyQ2e-48jCtZQB-9e24h5VDTylA== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:48:03 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/run_callable_task.yml b/test/fixtures/vcr_cassettes/eval/run_callable_task.yml index 17f3d83..caec2bd 100644 --- a/test/fixtures/vcr_cassettes/eval/run_callable_task.yml +++ b/test/fixtures/vcr_cassettes/eval/run_callable_task.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-ZGU3YmZhZWYtMWRmMC00MTE1LWI3Y2EtMzA1ZWY3NTY2YjUy'' *.js.stripe.com + ''nonce-ZGM2NjIyNjktMWFmZS00ZDVmLThiMWEtN2ZjNTg0MTJiZTA3'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:29 GMT + - Thu, 11 Dec 2025 18:50:06 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - ZGU3YmZhZWYtMWRmMC00MTE1LWI3Y2EtMzA1ZWY3NTY2YjUy + - ZGM2NjIyNjktMWFmZS00ZDVmLThiMWEtN2ZjNTg0MTJiZTA3 X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::6pkgq-1761319769134-2e9a164cec63 + - iad1::iad1::b2wlv-1765479006285-0f1f4b5a8049 body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:29 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:50:06 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -110,45 +111,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:29 GMT + - Thu, 11 Dec 2025 18:50:06 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - c11709f6-75bd-4119-b006-f965e83e3c01 + - 73755ea3-7360-478c-88bf-e934346633af X-Bt-Internal-Trace-Id: - - 68fb9b59000000002e098a1019c72e55 + - 693b125e000000005e04bea44ed43b36 X-Amz-Apigw-Id: - - S9U2AHg_oAMEXag= + - Vb_O0FRGoAMEPNA= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b59-158f240638b86e04721a7e8c;Parent=309c603195fdf5fb;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b125e-0797d9e86162bdad6860b382;Parent=2bc09894c69e2ae9;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 98bc8180e0431e8f05afc9802305f1d2.cloudfront.net (CloudFront), 1.1 babb61789095288ef1d09146c15e7f1e.cloudfront.net + - 1.1 57cf2ea4206ff643a898087150f0df7a.cloudfront.net (CloudFront), 1.1 06b7f1d77ee52f2e46654588c01fafb0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - bopXk8V72vtQG8ZTjeKncRfBgAICj-VB5MKHGOqFbZCljKo26GLe-A== + - "-hb548OD1iT70kk6vfqttf8OuPHGWFKe94k4-zOBgL39aJ57o--USA==" body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:29 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:50:06 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-callable-task","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-callable-task","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -157,7 +158,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -170,39 +171,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '398' + - '400' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:29 GMT + - Thu, 11 Dec 2025 18:50:06 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - ce9a5c5a-5796-4d07-ba01-718e8c9a2ba6 + - 293cf2dc-7315-472b-80d7-5c4dd27aed9e X-Bt-Internal-Trace-Id: - - 68fb9b5900000000029f6a0511ecdc20 + - 693b125e0000000054260e89cc6c09b2 X-Amz-Apigw-Id: - - S9U2BHpwIAMEdBg= + - Vb_O2GTcIAMEjIA= Vary: - Origin, Accept-Encoding Etag: - - W/"18e-bzTRqq5W4IIpf6TrhrTUXUPQos8" + - W/"190-U8lyL67SWJ7P6lu07UeaUimMEuo" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b59-77160de9467a1d2259ee9abd;Parent=36242f5c2eed95e2;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b125e-2293089a503e313621b52b01;Parent=46db2b192ef89dff;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 c50e3f7de0b772d07240015272b1aff6.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net + - 1.1 e00a954a79782fdd3cf71c54f8d15496.cloudfront.net (CloudFront), 1.1 645b3b9307ff6780fc1a76b712bc5728.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - DMqM4vWzSn4-iEXYtNEIRxN1Jeqwb6GN6cshIMfKDnO6TevcIfvJIQ== + - 1EHF_e4QvzSBpVIhyEO8axf6NZgPG0XbCNr1qZs3HIMNs0cuHHo46Q== body: encoding: ASCII-8BIT - string: '{"id":"155e36f8-2349-4e28-932b-256c53fdd089","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-callable-task-fe9f841b","description":null,"created":"2025-10-24T15:29:29.504Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:29 GMT + string: '{"id":"8ec22cf6-dfcf-426a-8ef0-c9181b3d11e3","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-callable-task-595ba27c","description":null,"created":"2025-12-11T18:50:06.719Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:50:06 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=8ec22cf6-dfcf-426a-8ef0-c9181b3d11e3 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:50:07 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 2a43d541-57ce-43ce-a803-acd5a297a340 + X-Bt-Internal-Trace-Id: + - 693b125e000000007ec5cb75c382000e + X-Amz-Apigw-Id: + - Vb_O4EzjoAMEV_A= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b125e-74fcc7207c072fe51a91c2d2;Parent=183539f7be475cfe;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 57cf2ea4206ff643a898087150f0df7a.cloudfront.net (CloudFront), 1.1 a97e58ffc6f61ef3eab965cbc26fc6f0.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 6XJ6S-k7EbMYo89kxD2XZ1cOEBa0GnhtpfCZD4fZbF3k4TkEIp6_lg== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:50:07 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/run_method_scorer.yml b/test/fixtures/vcr_cassettes/eval/run_method_scorer.yml index c138a07..d9bdcf5 100644 --- a/test/fixtures/vcr_cassettes/eval/run_method_scorer.yml +++ b/test/fixtures/vcr_cassettes/eval/run_method_scorer.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-MGZhNGMwZTktMDE1Ny00YzI5LTk5NzAtMWY2YTAzOGYyMTlj'' *.js.stripe.com + ''nonce-Yjg5ZjE3NTAtYzAwZC00YTAzLTk5ZjMtNWIwYzVlZjg3MWU5'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:17 GMT + - Thu, 11 Dec 2025 18:50:20 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - MGZhNGMwZTktMDE1Ny00YzI5LTk5NzAtMWY2YTAzOGYyMTlj + - Yjg5ZjE3NTAtYzAwZC00YTAzLTk5ZjMtNWIwYzVlZjg3MWU5 X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::j9trb-1761319757422-46a1f9b7773c + - iad1::iad1::psrxm-1765479020689-b676af01ec5f body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:17 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:50:20 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -110,45 +111,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:17 GMT + - Thu, 11 Dec 2025 18:50:21 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 2aec001d-65e6-482a-a605-ae28d5b17fb9 + - 76ea0269-1145-4815-9ffc-0f9fda3370e6 X-Bt-Internal-Trace-Id: - - 68fb9b4d00000000395bad9c30ed2b54 + - 693b126c000000002c4fbd9bf6064917 X-Amz-Apigw-Id: - - S9U0LGNioAMET4A= + - Vb_RFFXHIAMEM7g= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4d-0fd078b31da06d152aaa79e7;Parent=47a77bae2ec03f5a;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b126c-2df242a063c9b5c3652c3591;Parent=12750168df5b7f65;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 f8b0654d6e6bbf12f54a635de5db7ee4.cloudfront.net (CloudFront), 1.1 babb61789095288ef1d09146c15e7f1e.cloudfront.net + - 1.1 e00a954a79782fdd3cf71c54f8d15496.cloudfront.net (CloudFront), 1.1 67c9dfb38affc3a08786f92219ccc792.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - C-wqj9X9oHbcBWIYgKWOxjvLcw6v5cN6485lWcOmJmVsrlKeiK8Drg== + - IX4NPhT1QFlnlVXBG5qcHyH5pUKSCgxu3vHMoJjVnrO7eFwA3BQdlg== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:17 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:50:20 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-method-scorer","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-method-scorer","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -157,7 +158,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -170,39 +171,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '398' + - '400' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:17 GMT + - Thu, 11 Dec 2025 18:50:21 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 39844976-f877-44c8-ba75-ff3f4ab32114 + - b0dcd268-371b-4f26-b61e-8aacbd8ce2d6 X-Bt-Internal-Trace-Id: - - 68fb9b4d0000000018f2586245d2929a + - 693b126d000000000f007b62174ea4bd X-Amz-Apigw-Id: - - S9U0MG8hIAMEe4g= + - Vb_RHEnGoAMEPNQ= Vary: - Origin, Accept-Encoding Etag: - - W/"18e-s3ydO+uZp8uu5fS7tsnHDojbdAE" + - W/"190-aqRBqF/1oZOp8hNmMPabDLPbzAE" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4d-46ea317852b8bc6f7d1cf708;Parent=59c884cd4e260870;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b126d-6ed2416339f9170d4556af09;Parent=62e025f9c559deec;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 48fa2d8b9525abe889eff7ccc8591f7e.cloudfront.net (CloudFront), 1.1 fdc88b576635a6d1858343ad162c44fc.cloudfront.net + - 1.1 5a917b0edc7c6d2cee02a076aac1e824.cloudfront.net (CloudFront), 1.1 40185075b4d4551b3f16d587d961063c.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - E31FhSdvbg6c2yGs5khJQrCkCRXR4ju-Kbdj8hBBM3qRBnIDBoBnbw== + - _vsNWNUBFZgsgRJWSgh15yecQGCIkJL1yrZOqKaOwNlTXWzNJFgxVA== body: encoding: ASCII-8BIT - string: '{"id":"9b8e0f85-441f-49c7-94d6-9936e467dbef","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-method-scorer-912f9b8f","description":null,"created":"2025-10-24T15:29:17.789Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:17 GMT + string: '{"id":"31f370aa-00e8-4ceb-be80-07ad21273063","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-method-scorer-0d20efe5","description":null,"created":"2025-12-11T18:50:21.218Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:50:21 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=31f370aa-00e8-4ceb-be80-07ad21273063 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:50:21 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 6d3d0d9d-b1ec-49da-a544-f5001b53d724 + X-Bt-Internal-Trace-Id: + - 693b126d0000000070da78fe9c85b61d + X-Amz-Apigw-Id: + - Vb_RJF1JIAMESlw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b126d-423b6be91078860d2128223b;Parent=73ba8b0c93e3ea6c;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 57cf2ea4206ff643a898087150f0df7a.cloudfront.net (CloudFront), 1.1 67c9dfb38affc3a08786f92219ccc792.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - _yb99TLUS44ihdUGjXyfUVjiuglJMjiZHLqtPjXyrHYbxc-iRLNwFw== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:50:21 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/run_multiple_scorers.yml b/test/fixtures/vcr_cassettes/eval/run_multiple_scorers.yml index 038c18a..ac69d76 100644 --- a/test/fixtures/vcr_cassettes/eval/run_multiple_scorers.yml +++ b/test/fixtures/vcr_cassettes/eval/run_multiple_scorers.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-YWI3MDAxNDItOGQ0OS00NDM1LWE5MDYtYTI2MjliY2NlYmNm'' *.js.stripe.com + ''nonce-Njg0NGJjNjUtZDI4Yi00MjBhLWI3MjYtZWNiODk4NGVjZTE3'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:27 GMT + - Thu, 11 Dec 2025 18:51:26 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - YWI3MDAxNDItOGQ0OS00NDM1LWE5MDYtYTI2MjliY2NlYmNm + - Njg0NGJjNjUtZDI4Yi00MjBhLWI3MjYtZWNiODk4NGVjZTE3 X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::nbkx9-1761319767528-d3b417b9bb87 + - iad1::iad1::vfnd4-1765479086226-091b37f8d0f8 body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:27 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:51:26 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -110,45 +111,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:27 GMT + - Thu, 11 Dec 2025 18:51:26 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - dcb41721-f11d-4763-ac7f-91232abee065 + - 43507342-d710-48de-b94f-2b8ab796d08e X-Bt-Internal-Trace-Id: - - 68fb9b570000000036f618013ce92afc + - 693b12ae0000000002309209a7575c44 X-Amz-Apigw-Id: - - S9U1wEHPoAMEERQ= + - Vb_bUH_toAMEDfQ= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b57-55e278855eacff8063291411;Parent=1a567593a2ee747b;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12ae-245b4cfa643be1e84fbec478;Parent=6cd48d75e3c50903;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 bf8d7cb6fca5d51158e1109ca40fe242.cloudfront.net (CloudFront), 1.1 3a80316ee74668c7049eb31054a07168.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 ded721d3aeecfb4f35d39da9e2d34066.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - zY0jwX4n6eIW49F0jK90WCiRw_XDqR_tSUoDLOmD9utnn-g_aZW1Gw== + - YJHAdLd_10PzW9UJEC0uDZFpM2wOBDs3kGYY1T3nhE3vk-rCHIAJZg== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:27 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:51:26 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-multiple-scorers","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-multiple-scorers","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -157,7 +158,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -170,39 +171,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '401' + - '403' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:27 GMT + - Thu, 11 Dec 2025 18:51:26 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - b66d7de4-fc66-4666-98a8-e4f8dfbdb37e + - 947b8e96-a8b0-41b3-bc95-ca47eb259224 X-Bt-Internal-Trace-Id: - - 68fb9b57000000003a1164c03827ae30 + - 693b12ae00000000175d4e1b5bde8dfe X-Amz-Apigw-Id: - - S9U1xGjQIAMEv_A= + - Vb_bWGjPoAMEDCQ= Vary: - Origin, Accept-Encoding Etag: - - W/"191-Jaui3/uUepU3ZoIJvzFp6E70mAc" + - W/"193-Wg+pjp7MwmFul2CBO/k4c+UaGS0" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b57-16ffd5011c7c2c5f7fe7ea0e;Parent=3ac8024ca6ce9dd4;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b12ae-0b51801d7bc2a4b83c0cafce;Parent=37153d786442c0d0;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 f8debc28b6c73eb3dc7540e2ac2f0e18.cloudfront.net (CloudFront), 1.1 d3041c3025b9205db460853b5b9626bc.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 a97e58ffc6f61ef3eab965cbc26fc6f0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - FUqiNz62UyNAgOhIn6HsdMNEZOkaonNbYrui3JlWDxjcl3BUORV-Eg== + - 06Aex6b-mKbFmQS4wMekVYPpO9OBonhnrtv__ym3F8gPhgk7zWo2pQ== body: encoding: ASCII-8BIT - string: '{"id":"1c288ff2-8a84-462e-b7f7-11b71699404b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-multiple-scorers-bdddcb71","description":null,"created":"2025-10-24T15:29:27.894Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:27 GMT + string: '{"id":"35ec174c-b505-4a47-8f5e-eb5e9618dbdf","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-multiple-scorers-8d1830fb","description":null,"created":"2025-12-11T18:51:26.711Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:51:26 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=35ec174c-b505-4a47-8f5e-eb5e9618dbdf + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:51:27 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 66bacda7-9b7f-45db-b2de-2f0e20c4c2f6 + X-Bt-Internal-Trace-Id: + - 693b12ae000000005fb430f9a32503d6 + X-Amz-Apigw-Id: + - Vb_bZGQBIAMEcBw= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b12ae-4f8431f66a6ba4e347b3a067;Parent=6bceaf8990f3405b;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 91b2b204b1d28fa4ef9ff2294e4c25c4.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - mrA0wWVtU7_O2BmqTxB-XPakaPXS3PGYNztWQR1FzmyRmNRkiLZYfw== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:51:27 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/run_scorer_error.yml b/test/fixtures/vcr_cassettes/eval/run_scorer_error.yml index 47d214c..a277d8d 100644 --- a/test/fixtures/vcr_cassettes/eval/run_scorer_error.yml +++ b/test/fixtures/vcr_cassettes/eval/run_scorer_error.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-Y2UwNDBhY2MtMjU0Ny00NTRkLTg0N2ItZGI4OTQ4ZmI2MjM0'' *.js.stripe.com + ''nonce-ZGM5MzA0M2EtMWU0Yy00MjYwLTgyZDUtZjExZTc2NzI4MjJl'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:19 GMT + - Thu, 11 Dec 2025 18:49:42 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - Y2UwNDBhY2MtMjU0Ny00NTRkLTg0N2ItZGI4OTQ4ZmI2MjM0 + - ZGM5MzA0M2EtMWU0Yy00MjYwLTgyZDUtZjExZTc2NzI4MjJl X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::s27qb-1761319759832-b9907970dc2f + - iad1::iad1::b5h6h-1765478982339-fd44170d076c body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:19 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:49:42 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -110,45 +111,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:20 GMT + - Thu, 11 Dec 2025 18:49:42 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 549e4f69-f064-4958-8a16-da32e9b3e219 + - 66d7e164-c9a1-4d7b-a96f-c1add92fe597 X-Bt-Internal-Trace-Id: - - 68fb9b4f0000000010bbfc237014deb3 + - 693b1246000000006af7cd86a48e4354 X-Amz-Apigw-Id: - - S9U0jFYloAMEXMw= + - Vb_LFG5xoAMEYTg= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b4f-701cb3d84d8ec743644ca66f;Parent=1d9928876bfead56;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b1246-232e178748c13b0018639d28;Parent=68be40c6b585a705;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 f8b0654d6e6bbf12f54a635de5db7ee4.cloudfront.net (CloudFront), 1.1 baec235d174153a8f2e92ea724643824.cloudfront.net + - 1.1 e00a954a79782fdd3cf71c54f8d15496.cloudfront.net (CloudFront), 1.1 9f575b6f0217c33f22e6da5419974422.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - "-MLEMjytRWi7x7lX5_eEDPxCEs8oOHaIzQ_PHKmwKakkHHEIljtMuA==" + - rG0fIOoSBAYdWBLpGI2dAWVAG__AiVaImZqqz3GEyTmZDoUtNwf5rw== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:20 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:49:42 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-scorer-error","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-scorer-error","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -157,7 +158,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -170,39 +171,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '397' + - '399' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:20 GMT + - Thu, 11 Dec 2025 18:49:42 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 9e534f11-e794-4bfc-a1a5-e051d558785f + - 17c13e82-5924-4257-bb3f-eef0e713b83a X-Bt-Internal-Trace-Id: - - 68fb9b50000000005d8dd68ecf9d5d44 + - 693b12460000000019e186fe3081e54a X-Amz-Apigw-Id: - - S9U0lGmyIAMEJSQ= + - Vb_LHH9JIAMEZfg= Vary: - Origin, Accept-Encoding Etag: - - W/"18d-uORWhJs0D/7dPuBAH6Qd5quVKGs" + - W/"18f-ZDxs6ytp8d36/zKSHH7+LUmNDpM" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b50-0f501a6a34f55ec4406253cb;Parent=72f28c2a44d9f5b0;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b1246-34a5c2352bd14c1a292b12d5;Parent=55966a1b4584e5ee;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 8a9cdb228e33f8d52a4b42c56ca26590.cloudfront.net (CloudFront), 1.1 d3a2886dcf7e47b1e56baf628c69dc24.cloudfront.net + - 1.1 57cf2ea4206ff643a898087150f0df7a.cloudfront.net (CloudFront), 1.1 bebfdaf3481b8e276dc3fc8a17fefd66.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - h9ASYGHOmyE05t8TJM9oa_gPtK4YGW67ksTwXhDD_hQO6zaUtLB7MA== + - kaXzhc9XNZcGFvv_9aYChKhUZBSCogZm98HEtMfpqD5X0tRflGf7Ug== body: encoding: ASCII-8BIT - string: '{"id":"8a22930d-677b-4d06-9d8d-7375e4a99c69","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-scorer-error-347045bc","description":null,"created":"2025-10-24T15:29:20.232Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:20 GMT + string: '{"id":"02a1b304-cab1-4e07-a805-08d4fd0599e6","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-scorer-error-85ba8b12","description":null,"created":"2025-12-11T18:49:42.902Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:49:42 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=02a1b304-cab1-4e07-a805-08d4fd0599e6 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:49:43 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 57a384c7-103e-47cd-bee4-953bd95b131d + X-Bt-Internal-Trace-Id: + - 693b12470000000047925ee17afe4f27 + X-Amz-Apigw-Id: + - Vb_LKH3EIAMEtqg= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b1247-50dcbb5e170b5a5b19257735;Parent=059aa0c17b0f430f;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 57cf2ea4206ff643a898087150f0df7a.cloudfront.net (CloudFront), 1.1 a97e58ffc6f61ef3eab965cbc26fc6f0.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - dS2uarhwa5tRK6HtUpYiVLPJSbGmgQNhsdNopkQLEfGogiEt681qZQ== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:49:43 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/run_task_error.yml b/test/fixtures/vcr_cassettes/eval/run_task_error.yml index 063773c..06b88d5 100644 --- a/test/fixtures/vcr_cassettes/eval/run_task_error.yml +++ b/test/fixtures/vcr_cassettes/eval/run_task_error.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-YTdiM2I3YWQtNmQ2NS00ODFlLWFkYWEtY2Q4ZmYxZTI5MjY5'' *.js.stripe.com + ''nonce-N2QwNmQ1YWMtYzY1OS00MjFmLTk1OTMtNTYzNTNhZTJkNWUx'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:27 GMT + - Thu, 11 Dec 2025 18:49:15 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - YTdiM2I3YWQtNmQ2NS00ODFlLWFkYWEtY2Q4ZmYxZTI5MjY5 + - N2QwNmQ1YWMtYzY1OS00MjFmLTk1OTMtNTYzNTNhZTJkNWUx X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::c74h2-1761319767039-eae7392ab902 + - iad1::iad1::fmgk8-1765478955817-1ef3b3334e59 body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:27 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:49:15 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -110,45 +111,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:27 GMT + - Thu, 11 Dec 2025 18:49:16 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 458deff8-1676-4c5f-ba52-eb218174b897 + - a9908cab-7c44-4925-a7fd-feecaa536f12 X-Bt-Internal-Trace-Id: - - 68fb9b570000000031641f80ff770a79 + - 693b122c000000006d03267e3bdcdf92 X-Amz-Apigw-Id: - - S9U1rEy6IAMEs1A= + - Vb_G9F3_oAMEvCg= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b57-7c0dacf30e269be541928691;Parent=0762235f4e0124d2;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b122c-10af31783887b31d022dbbd0;Parent=572775df7d2e83aa;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 52143757d25f4b31ebf04bc09765f6c0.cloudfront.net (CloudFront), 1.1 e42e8491a089e2183879e26e61dae708.cloudfront.net + - 1.1 f00bb95b8c55d1da29ae14415a8da844.cloudfront.net (CloudFront), 1.1 8f8f56e20a7e26014a52398627840a50.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - 7DfDSFrrPNMzke_ZUS5qlLR1X_TCJuGVuvg-cF2B6gwdy4B-C2AMCg== + - p7NcYu55mhnxjznJgIsmYuUAOvCYzCbbcTefa1PuunJYZ9kZ_Y9WbQ== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:27 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:49:16 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-task-error","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-task-error","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -157,7 +158,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -170,39 +171,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '395' + - '397' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:49:16 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 72a40b1d-8b9f-40e8-96f6-d9f7c4eda794 + X-Bt-Internal-Trace-Id: + - 693b122c000000002a3ef2dc09d1e8d5 + X-Amz-Apigw-Id: + - Vb_G-EvjoAMEb5A= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"18d-tCdQBMvF0vWSg8MbXHR9HyR6yzw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b122c-0f7e5bac1e3a9a3e42907be9;Parent=44bb470b465de2bd;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 57cf2ea4206ff643a898087150f0df7a.cloudfront.net (CloudFront), 1.1 67c9dfb38affc3a08786f92219ccc792.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 0KE7V53kCaAWV4wx1aDlZAUanblYjJqy7m1g2WSa4wR26YzEGoL6DQ== + body: + encoding: ASCII-8BIT + string: '{"id":"2a5c5549-1c6d-4f3b-9fc8-34bb0f0a4e3b","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-task-error-c7b9d1c6","description":null,"created":"2025-12-11T18:49:16.363Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:49:16 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=2a5c5549-1c6d-4f3b-9fc8-34bb0f0a4e3b + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:27 GMT + - Thu, 11 Dec 2025 18:49:17 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 590d6555-fc0a-43cc-9ca1-7411e86009dc + - 83129eb3-169b-432a-8f1c-327bb8a36f24 X-Bt-Internal-Trace-Id: - - 68fb9b570000000059933bc5639feb98 + - 693b122c000000003f700ec95f5a0e6e X-Amz-Apigw-Id: - - S9U1tFHYoAMEagw= + - Vb_HBEq_oAMEu2A= Vary: - Origin, Accept-Encoding Etag: - - W/"18b-kFbJ2eJgoMyYsFDkztTlt3q/bII" + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b57-6a77760327995c0b2151c1a3;Parent=538e408c42c2d2f1;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b122c-1501a0e340e62abd765464eb;Parent=620126effe373b95;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 5af2699243b550d789ef9dce0b522ed2.cloudfront.net (CloudFront), 1.1 8fd21502425077e617fde7325b45e112.cloudfront.net + - 1.1 e00a954a79782fdd3cf71c54f8d15496.cloudfront.net (CloudFront), 1.1 284ac69616559909913fa0f0502158ea.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - bfaOaXvNwATMQEvNiolyr9ZEkZg8QcL66mKwBle4Ets_qz8nseiz8A== + - '078FpnCHj2G3B-AFNLIlO6KJFiIGAD6VOV5LwFgTRYOCsHaSCaZg9w==' body: encoding: ASCII-8BIT - string: '{"id":"474d637b-5955-47af-beec-acd36770346b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-task-error-9fafffe8","description":null,"created":"2025-10-24T15:29:27.450Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:27 GMT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:49:17 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval/run_with_tracing.yml b/test/fixtures/vcr_cassettes/eval/run_with_tracing.yml index befec65..e22dcb6 100644 --- a/test/fixtures/vcr_cassettes/eval/run_with_tracing.yml +++ b/test/fixtures/vcr_cassettes/eval/run_with_tracing.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-ZjFjNjUyN2YtYzYyMi00YWZmLTg1MDEtZTE3NTJjZWEyYjY5'' *.js.stripe.com + ''nonce-MGY2YjFlNGItNTg5OC00Nzk5LWJhZWYtMWQ3NmQ2MDFmZjZl'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:29:28 GMT + - Thu, 11 Dec 2025 18:48:18 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - ZjFjNjUyN2YtYzYyMi00YWZmLTg1MDEtZTE3NTJjZWEyYjY5 + - MGY2YjFlNGItNTg5OC00Nzk5LWJhZWYtMWQ3NmQ2MDFmZjZl X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::td4ks-1761319767983-9a99f48aebeb + - iad1::iad1::ndz2l-1765478898894-61ab2df61395 body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:29:28 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 18:48:18 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -110,45 +111,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:28 GMT + - Thu, 11 Dec 2025 18:48:19 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 4f3f3e98-7258-4cd3-8f08-f94517e50676 + - 38476a7a-edb1-4f71-a226-27b006d30d8a X-Bt-Internal-Trace-Id: - - 68fb9b58000000002b579666489e7525 + - 693b11f30000000046cb6683f778d08b X-Amz-Apigw-Id: - - S9U12FVHIAMEepA= + - Vb--CHxZoAMENiA= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b58-382e601d62e0952731fa98b1;Parent=253657dc7bff0a50;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b11f3-340bb65c3a072c0241e14d89;Parent=24013bd87c93e428;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 5ba825173b1f7429171e730e7ae12588.cloudfront.net (CloudFront), 1.1 06d42d2d80190e168b9494192458b51a.cloudfront.net + - 1.1 f106673324d6e3006e921c9e92055438.cloudfront.net (CloudFront), 1.1 94ad62d30119f960dbd892be99684e02.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - e7tUpBrm_t8-n6OWDVYnP2wAILwfHYQGHyxmXAKnfZbZV9Kqul9riQ== + - ErM3wTtDbJkzxD7Ssce1X7QAkh8R71u_gDALdwbJ9PcgHN8UM7CVWw== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:29:28 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 18:48:19 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-tracing","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-tracing","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -157,7 +158,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -170,39 +171,99 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '392' + - '394' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:29:28 GMT + - Thu, 11 Dec 2025 18:48:19 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - bd22508d-07c0-48f3-9f41-cfaf3fb4cbed + - 59ee9bbe-b3b2-4ed9-b3e6-1797010132c7 X-Bt-Internal-Trace-Id: - - 68fb9b5800000000391fa7188db86e87 + - 693b11f30000000067373f35df76b8b2 X-Amz-Apigw-Id: - - S9U14Hq6oAMEidw= + - Vb--EH-3IAMER5w= Vary: - Origin, Accept-Encoding Etag: - - W/"188-sMRkPe4xb01orS2grZbTg04ER5M" + - W/"18a-LGpZAdi5eoEgLzm//Tz9TCmUy3I" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb9b58-69cf2f924dc8c3a634bc5e89;Parent=580822d3acaf93e5;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b11f3-08de67c14f7a6b3a42ebbb69;Parent=23f32f97d736d6e1;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 52143757d25f4b31ebf04bc09765f6c0.cloudfront.net (CloudFront), 1.1 a9a7890bd69fb04b587cd51baf217b1c.cloudfront.net + - 1.1 f00bb95b8c55d1da29ae14415a8da844.cloudfront.net (CloudFront), 1.1 06b7f1d77ee52f2e46654588c01fafb0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - DoRwzsuErX1HfI82cRT6e3Bebaz2_wxtTS2AhXDKT-QKDuXaNxNvhg== + - fTc9U5vn1k7dY4_WruSl8JmL0HuUKSeKUjMvDgi2WTXdsdj7d2kfHA== body: encoding: ASCII-8BIT - string: '{"id":"0ba9d958-28f5-4535-a429-e86350addd66","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-tracing-c74d4b1c","description":null,"created":"2025-10-24T15:29:28.509Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:29:28 GMT + string: '{"id":"35c62596-bb92-4779-9753-fc27a4f17f00","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-tracing-fdfd53c5","description":null,"created":"2025-12-11T18:48:19.354Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 18:48:19 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=35c62596-bb92-4779-9753-fc27a4f17f00 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 18:48:19 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - be2b270c-788d-475e-94f3-8875cb1cd2af + X-Bt-Internal-Trace-Id: + - 693b11f3000000005407d02b044e44d6 + X-Amz-Apigw-Id: + - Vb--GGoZoAMEIEA= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b11f3-6f87607a105245522272294c;Parent=64fd091a10390821;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 f106673324d6e3006e921c9e92055438.cloudfront.net (CloudFront), 1.1 5f9847e2035814141303960526e10e26.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - vyNajA109TQjeqC1KpKEjoO7AyZIoI5jEpFBfYpuU-0TeHjibEB8zw== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 18:48:19 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval_functions/eval_run.yml b/test/fixtures/vcr_cassettes/eval_functions/eval_run.yml index 9edcb10..f5bf185 100644 --- a/test/fixtures/vcr_cassettes/eval_functions/eval_run.yml +++ b/test/fixtures/vcr_cassettes/eval_functions/eval_run.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-OWRjNDAyNzktZGQ1Mi00NGRlLWJjNzktYmI4ZTI4YTQ1ZWRi'' *.js.stripe.com + ''nonce-ZDhjYzIxNDgtMDkzYy00YmExLWI2MDUtYTE4NDFkY2Y1YzZj'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:09:37 GMT + - Thu, 11 Dec 2025 19:09:46 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - OWRjNDAyNzktZGQ1Mi00NGRlLWJjNzktYmI4ZTI4YTQ1ZWRi + - ZDhjYzIxNDgtMDkzYy00YmExLWI2MDUtYTE4NDFkY2Y1YzZj X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::sf48l-1761318577764-5a87fbb48444 + - iad1::iad1::fjx8s-1765480186308-f7e408a5435f body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:09:37 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 19:09:46 GMT - request: method: get - uri: https://api.braintrust.dev/v1/project?project_name=ruby-sdk-test + uri: https://staging-api.braintrust.dev/v1/project?project_name=ruby-sdk-test body: encoding: US-ASCII string: '' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Authorization: - Bearer response: @@ -108,43 +109,43 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:38 GMT + - Thu, 11 Dec 2025 19:10:53 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 74dde7a5-cd10-461a-9ea4-88f3fa0708a3 + - 471239fa-87f6-4d3f-9bf7-33191c363d0f X-Bt-Internal-Trace-Id: - - 68fb96b1000000006eb9d6fa28e1b37d + - 693b173d0000000002fdadc9ad21b9c9 X-Amz-Apigw-Id: - - S9R72H1OIAMEsIA= + - VcCRnE41oAMElMQ= Vary: - Origin, Accept-Encoding Etag: - - W/"fa-+YSPhVhMcSODQVu7iSaJULbdOe4" + - W/"fa-HrwmCajpdJfSgika/ZPujG7WfoY" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b1-7f32a636150dd85f075d9af2;Parent=11b4882d4ddc1b09;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b173d-527f9af753cc8ec94baec46e;Parent=483d379a272cca83;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 a1546fc751225809c39b89ba9e8d715c.cloudfront.net (CloudFront), 1.1 3a80316ee74668c7049eb31054a07168.cloudfront.net + - 1.1 77512d19cb7ad3590ec138f5d5b2dd86.cloudfront.net (CloudFront), 1.1 dc6928d732b026211beb221384112f80.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - "-Sdpn_czWT3nP5IdGm_so_WxX-LMuJlTPhfBtDEfMxmgh0FSpliChQ==" + - nZCqecjQJFZm5EPpag4VWhtn7iYFuIxikDHmOZh-2rLxN7rlA5EgXA== body: encoding: ASCII-8BIT - string: '{"objects":[{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}]}' - recorded_at: Fri, 24 Oct 2025 15:09:38 GMT + string: '{"objects":[{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}]}' + recorded_at: Thu, 11 Dec 2025 19:10:53 GMT - request: method: post - uri: https://api.braintrust.dev/v1/function + uri: https://staging-api.braintrust.dev/v1/function body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-eval-task","name":"test-ruby-sdk-eval-task","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Uppercase + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","slug":"test-ruby-sdk-eval-task","name":"test-ruby-sdk-eval-task","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Uppercase this: {{input}}. Return ONLY the uppercase version, nothing else."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0}}}}' headers: Accept-Encoding: @@ -154,7 +155,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -171,41 +172,41 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:38 GMT + - Thu, 11 Dec 2025 19:10:53 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 5c077309-248f-4d59-835d-1598c0f33b08 + - 2cb1beba-88e2-4eec-9370-e16353467eac X-Bt-Internal-Trace-Id: - - 68fb96b2000000006da8150dee0bf7f1 + - 693b173d000000007ad423b5e7d764e2 X-Amz-Apigw-Id: - - S9R74ECkoAMEGkw= + - VcCRqE7coAMEmGQ= Vary: - Origin, Accept-Encoding Etag: - - W/"223-ipBaUnj4py2MsTNCGBYPx3mfuYc" + - W/"223-jtBuXzTS7Jzk3eqbdrz3QXXziqM" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b2-4ecf297222a1e6206fd52320;Parent=641e62f093c52edc;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b173d-50107a3c67186d650ab093a1;Parent=4f3388c2eab60c6b;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 eb2e4893b47f0d155cd51b82c2a8d596.cloudfront.net (CloudFront), 1.1 76d4de5b65bdf749a3f97445d1b9f4d2.cloudfront.net + - 1.1 77512d19cb7ad3590ec138f5d5b2dd86.cloudfront.net (CloudFront), 1.1 67c9dfb38affc3a08786f92219ccc792.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - _kalqPiXC9m4rW_lgrdgekVBwuN9WFOp6X1Rtg4ObJqbM_xR8LhwYw== + - l0lswbg8FjBLZUYIs_JNZoUN8Ak_yUQyEAgaKv_UOeiRtkl3KA01TA== body: encoding: ASCII-8BIT - string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-eval-task","name":"test-ruby-sdk-eval-task","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Uppercase - this: {{input}}. Return ONLY the uppercase version, nothing else."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0}}},"id":"4bf8c762-f38f-45b5-ba62-9fd9994fbb5c","created":"2025-10-24T15:09:38.141Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196022027347036"}' - recorded_at: Fri, 24 Oct 2025 15:09:38 GMT + string: '{"log_id":"p","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","slug":"test-ruby-sdk-eval-task","name":"test-ruby-sdk-eval-task","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Uppercase + this: {{input}}. Return ONLY the uppercase version, nothing else."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0}}},"id":"284f8891-c3fe-41e2-af14-63c827a89462","created":"2025-12-11T19:10:53.606Z","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","_xact_id":"1000196294766839345"}' + recorded_at: Thu, 11 Dec 2025 19:10:53 GMT - request: method: get - uri: https://api.braintrust.dev/v1/function?project_name=ruby-sdk-test&slug=test-ruby-sdk-eval-task + uri: https://staging-api.braintrust.dev/v1/function?project_name=ruby-sdk-test&slug=test-ruby-sdk-eval-task body: encoding: US-ASCII string: '' @@ -217,7 +218,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Authorization: - Bearer response: @@ -232,41 +233,41 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:38 GMT + - Thu, 11 Dec 2025 19:10:57 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - f5e9ba41-8142-4b5a-9e84-87b4e1f66d2c + - 2ee22c71-3308-44a0-bb5f-5998f5b0ffb0 X-Bt-Internal-Trace-Id: - - 68fb96b200000000564a9d54e73a8fde + - 693b17400000000065aba58b5c9add91 X-Amz-Apigw-Id: - - S9R77EuQoAMEPPA= + - VcCSJH1vIAMEDwQ= Vary: - Origin, Accept-Encoding Etag: - - W/"29a-u6LJ9XSYn2ObRgTCbQk86a3esFA" + - W/"29a-KFoTpWeb9GuA0eLM29yFza5GK3Y" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b2-77ea500d2b6d6e0c1d80fecc;Parent=2ef0bf6d10322a64;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b1740-155b07fe0d9f74cb390a7d90;Parent=1cf3e4a26c2267f6;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 58a45bf3f07dfdca95ebcb7935e84994.cloudfront.net (CloudFront), 1.1 fc527c9e0a8a22ce61e1580f7382bcf8.cloudfront.net + - 1.1 7d77965d78b3f4565239009cf6e62356.cloudfront.net (CloudFront), 1.1 91b2b204b1d28fa4ef9ff2294e4c25c4.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - EjirEgRMXQOjPFL8l_W5f83fWd6l4_giXTtiuwPz0c2O-UVeUhaAdA== + - 2XMZtOzniP04G7O-aD7Swwp7tBSXzjhgWs9ovYMqROUfSDKhatTLNw== body: encoding: ASCII-8BIT - string: '{"objects":[{"id":"4bf8c762-f38f-45b5-ba62-9fd9994fbb5c","_xact_id":"1000196022027347036","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-ruby-sdk-eval-task","slug":"test-ruby-sdk-eval-task","description":null,"created":"2025-10-24T15:09:38.141Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Uppercase + string: '{"objects":[{"id":"284f8891-c3fe-41e2-af14-63c827a89462","_xact_id":"1000196294766839345","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","log_id":"p","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"test-ruby-sdk-eval-task","slug":"test-ruby-sdk-eval-task","description":null,"created":"2025-12-11T19:10:53.606Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"user","content":"Uppercase this: {{input}}. Return ONLY the uppercase version, nothing else."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0}}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}]}' - recorded_at: Fri, 24 Oct 2025 15:09:38 GMT + recorded_at: Thu, 11 Dec 2025 19:10:57 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -278,7 +279,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -295,45 +296,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:38 GMT + - Thu, 11 Dec 2025 19:10:57 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 48dca7f8-821f-4182-898f-456112aa85bb + - 2d789562-0cd8-438c-87c7-738779211be4 X-Bt-Internal-Trace-Id: - - 68fb96b200000000214b199f875d93a5 + - 693b17410000000075e6aba5675310bf X-Amz-Apigw-Id: - - S9R8AEcRoAMEo-A= + - VcCSUFn7oAMEBVg= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b2-60003ac34576e0bc62a82fcf;Parent=3b920a3cd0f6e7a4;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b1741-59ac07d47cc9e1a73102d46d;Parent=6f2bd7ce29bc02fe;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 a1546fc751225809c39b89ba9e8d715c.cloudfront.net (CloudFront), 1.1 e42e8491a089e2183879e26e61dae708.cloudfront.net + - 1.1 7d77965d78b3f4565239009cf6e62356.cloudfront.net (CloudFront), 1.1 d0868637a3730c2afc20b3628b047a9a.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - VIrsqNPnuOc2IzWJoL-3T_AiKCw053h0tOUKoC4UImidCiIxE8ns5g== + - wGOebuXgdId5VCpNyU3jL05PAPlT1-LzJcexWpDFd-JDXGTgCTTbzQ== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:09:38 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 19:10:57 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-remote-task-eval","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-remote-task-eval","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -342,7 +343,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -355,44 +356,44 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '401' + - '403' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:39 GMT + - Thu, 11 Dec 2025 19:10:58 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 61808dc4-f443-40fa-be18-88390db9fea5 + - cea0b737-3af5-4bf1-bfc2-e1ce8748d8e8 X-Bt-Internal-Trace-Id: - - 68fb96b30000000015b5068ffe631948 + - 693b1741000000002d6eec051dca2216 X-Amz-Apigw-Id: - - S9R8CHhooAMEsbg= + - VcCSXEZOIAMEvGw= Vary: - Origin, Accept-Encoding Etag: - - W/"191-mfh1k2yDNYiLAYSSOH1eBV7FDIA" + - W/"193-McPgqnPT5TxWHO06duDiMEombdY" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b3-37dd4a58519a779805f0e6ee;Parent=37c412f87a3bca03;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b1741-5e44f5e64e3fccb22a26e418;Parent=4353511895cbe4d0;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 f8b0654d6e6bbf12f54a635de5db7ee4.cloudfront.net (CloudFront), 1.1 f25b89e7ef738cb8bb7e28e041d8fe54.cloudfront.net + - 1.1 3bd704a723998d21cd3d1c7e5aa5b320.cloudfront.net (CloudFront), 1.1 9f575b6f0217c33f22e6da5419974422.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - 8dVN6yx3RjAry-gFWbKmgRlexwIoLu_5j__LBS88UJsVmbK8CYX5LA== + - NkElNj6hP0R29UTVkRckSj_q0KdfwAJ85TK4zniyBMrf5b3yfQ7zig== body: encoding: ASCII-8BIT - string: '{"id":"ebc23a05-56c8-43ad-89eb-75bd21a67ca0","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-remote-task-eval-4e7ed7fc","description":null,"created":"2025-10-24T15:09:39.153Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:09:39 GMT + string: '{"id":"d12c3748-ba5d-4841-b563-bb7d07f3e9f5","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-remote-task-eval-de7b8a06","description":null,"created":"2025-12-11T19:10:58.058Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 19:10:58 GMT - request: method: post - uri: https://api.braintrust.dev/v1/function/4bf8c762-f38f-45b5-ba62-9fd9994fbb5c/invoke + uri: https://staging-api.braintrust.dev/v1/function/284f8891-c3fe-41e2-af14-63c827a89462/invoke body: encoding: UTF-8 string: '{"input":"hello"}' @@ -404,7 +405,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -421,75 +422,62 @@ http_interactions: Connection: - keep-alive Date: - - Fri, 24 Oct 2025 15:09:39 GMT + - Thu, 11 Dec 2025 19:10:59 GMT X-Bt-Used-Endpoint: - - OPENAI_API_KEY + - AZURE_DEFAULT_CREDENTIALS X-Amzn-Requestid: - - 2e514414-51b0-40e5-be21-7b900b371c7c - Cf-Ray: - - 9933bc62ce877000-IAD + - c9a0b4d3-1884-4705-af25-ba3ab00d4f55 + X-Ms-Deployment-Name: + - gpt-4o-mini X-Ratelimit-Remaining-Requests: - - '29999' - Openai-Processing-Ms: - - '148' - Cf-Cache-Status: - - DYNAMIC + - '2497' + X-Ms-Region: + - East US + X-Ms-Rai-Invoked: + - 'true' Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload X-Bt-Cached: - HIT - X-Openai-Proxy-Wasm: - - v0.1 - X-Ratelimit-Reset-Requests: - - 2ms - X-Envoy-Upstream-Service-Time: - - '174' - Openai-Organization: - - braintrust-data X-Request-Id: - - req_f20f6ed75807481da924cf0485ba71b5 - Set-Cookie: - - _cfuvid=4VTCgNGoAPfpvffbl._761TJKO8uQWMOWEDFfJXCE5w-1761248737998-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - 67cdbb06-a307-496e-9fea-b4d5d40fd9c6 Cache-Control: - max-age=604800 + Azureml-Model-Session: + - d20251210231011-628f6c5d8b8c414b + Apim-Request-Id: + - f8b57d87-c027-41df-9512-553c3d34f6cb X-Ratelimit-Limit-Tokens: - - '150000000' + - '250000' X-Content-Type-Options: - nosniff X-Ratelimit-Remaining-Tokens: - - '149999980' + - '249914' X-Bt-Function-Creds-Cached: - - HIT + - MISS X-Amzn-Trace-Id: - - Root=1-68fb96b3-26bfb6e719e36457289f1d07;Parent=34bb38ec4c7db17b;Sampled=0;Lineage=1:8be8f50d:0 + - Root=1-693b1742-05e3565234587a0010c555ec;Parent=61aa8201575b8edf;Sampled=0;Lineage=1:98839c8e:0 X-Bt-Function-Meta-Cached: - MISS - X-Ratelimit-Reset-Tokens: - - 0s - Openai-Version: - - '2020-10-01' X-Ratelimit-Limit-Requests: - - '30000' - Openai-Project: - - proj_wMRY6YpEiASXMxPIIcI9nQRi + - '2500' X-Cache: - Miss from cloudfront Via: - - 1.1 ed016821a44f073856f1ffba399e1728.cloudfront.net (CloudFront) + - 1.1 81d12325eefc0deca593ce76681fa256.cloudfront.net (CloudFront) X-Amz-Cf-Pop: - - JFK50-P2 + - JFK52-P8 X-Amz-Cf-Id: - - ra6p69WiH6YyKpQOHdZVUV7lEiNwlyIGEFbCYinOgUYIWFCL9Asncg== + - "--J2UfWewoIucBLRrJKwguaeJDnOEpsR5VOhcjlsPiW5qCp5O2W-Vg==" Age: - - '69841' + - '1956' body: encoding: UTF-8 string: '"HELLO."' - recorded_at: Fri, 24 Oct 2025 15:09:39 GMT + recorded_at: Thu, 11 Dec 2025 19:10:59 GMT - request: method: post - uri: https://api.braintrust.dev/v1/function/4bf8c762-f38f-45b5-ba62-9fd9994fbb5c/invoke + uri: https://staging-api.braintrust.dev/v1/function/284f8891-c3fe-41e2-af14-63c827a89462/invoke body: encoding: UTF-8 string: '{"input":"world"}' @@ -501,7 +489,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -518,17 +506,17 @@ http_interactions: Connection: - keep-alive Date: - - Fri, 24 Oct 2025 15:09:39 GMT + - Thu, 11 Dec 2025 19:10:59 GMT X-Bt-Used-Endpoint: - OPENAI_API_KEY X-Amzn-Requestid: - - 4d0ef6aa-da8d-467a-9f0d-c46d37bc47e5 + - e64f85d0-2c1e-49f5-b6c6-002fbdda956b Cf-Ray: - - 9939e3cf5ac456fe-IAD + - 9ac7193ec8f3390b-IAD X-Ratelimit-Remaining-Requests: - '29999' Openai-Processing-Ms: - - '326' + - '336' Cf-Cache-Status: - DYNAMIC Strict-Transport-Security: @@ -540,13 +528,13 @@ http_interactions: X-Ratelimit-Reset-Requests: - 2ms X-Envoy-Upstream-Service-Time: - - '448' + - '901' Openai-Organization: - braintrust-data X-Request-Id: - - req_29b8e7e1b70d4d16816e3fd07ffba681 + - req_7d47cc78217f43aaa985652d97135cd6 Set-Cookie: - - _cfuvid=PqY.DNF4YO_xTXnHBU2K5w.5YEQwcpNmpl3tGrpztHc-1761313267614-0.0.1.1-604800000; + - _cfuvid=pt5Tk_mLp9ws7gcYSfWJTsCx6x7kVG_FiVX8FcW5R6c-1765478303505-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Cache-Control: - max-age=604800 @@ -559,7 +547,7 @@ http_interactions: X-Bt-Function-Creds-Cached: - HIT X-Amzn-Trace-Id: - - Root=1-68fb96b3-08cd8f3e4bd316ff058b8cf1;Parent=5fb61802e77644ef;Sampled=0;Lineage=1:8be8f50d:0 + - Root=1-693b1743-098cda2a42123bf1255184f4;Parent=74c68fcb4a5d85c0;Sampled=0;Lineage=1:98839c8e:0 X-Bt-Function-Meta-Cached: - HIT X-Ratelimit-Reset-Tokens: @@ -573,15 +561,75 @@ http_interactions: X-Cache: - Miss from cloudfront Via: - - 1.1 bd3e3884ce6fe1fd36336541cce9ec7e.cloudfront.net (CloudFront) + - 1.1 67c9dfb38affc3a08786f92219ccc792.cloudfront.net (CloudFront) X-Amz-Cf-Pop: - - JFK50-P2 + - JFK52-P8 X-Amz-Cf-Id: - - 6ghLEQg5WgzlzEcwf5s3zPND8cMCYOFYtV7e0jJ1wDhwVTuG9EgPmQ== + - C8-UzLt_nMVB5MQcK931yBpKM8ssX0kf9K5-7AsGubzADl9etUUqNg== Age: - - '5312' + - '1955' body: encoding: UTF-8 string: '"WORLD."' - recorded_at: Fri, 24 Oct 2025 15:09:39 GMT + recorded_at: Thu, 11 Dec 2025 19:10:59 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=d12c3748-ba5d-4841-b563-bb7d07f3e9f5 + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 19:11:00 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - 1d7d3f98-f20e-4fa3-ac90-2b3bcfcaaa9b + X-Bt-Internal-Trace-Id: + - 693b1743000000006c870db53ef6a13e + X-Amz-Apigw-Id: + - VcCSnHHHIAMEJkA= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b1743-2acad55e41a77c1f22c89412;Parent=228c7c41cbfe6613;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 a5aff17032e65d74ef5c955d5493c06e.cloudfront.net (CloudFront), 1.1 0923b90a5b7ec988436ae37e0b8c6774.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - yOQzjPpk0UKgME3AV2OgXReTRQc13V726Zd5RdGJYivFA3CwAIZCEw== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 19:11:00 GMT recorded_with: VCR 6.3.1 diff --git a/test/fixtures/vcr_cassettes/eval_functions/remote_scorer.yml b/test/fixtures/vcr_cassettes/eval_functions/remote_scorer.yml index 1984bfd..c369fb2 100644 --- a/test/fixtures/vcr_cassettes/eval_functions/remote_scorer.yml +++ b/test/fixtures/vcr_cassettes/eval_functions/remote_scorer.yml @@ -34,24 +34,25 @@ http_interactions: Cache-Control: - public, max-age=0, must-revalidate Content-Length: - - '257' + - '395' Content-Security-Policy: - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' - ''nonce-Nzg0OGVlODEtYTlhYi00NjcwLTgzOWEtYjFlYzA3NzZmNzc5'' *.js.stripe.com + ''nonce-OWJmZGRjMDctNmJiYi00Mjk1LWI1YzUtNzUxOGNkYjE2YjY1'' *.js.stripe.com js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev - btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com; - font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com; - object-src ''none''; base-uri ''self''; form-action ''self''; frame-ancestors - ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14; + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; report-to csp-endpoint-0' Content-Type: - application/json; charset=utf-8 Date: - - Fri, 24 Oct 2025 15:09:41 GMT + - Thu, 11 Dec 2025 19:12:09 GMT Etag: - - '"ubzjf1iqqj75"' + - '"12n7ok4b5phaz"' Reporting-Endpoints: - - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=14" + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" Server: - Vercel Strict-Transport-Security: @@ -70,18 +71,18 @@ http_interactions: X-Matched-Path: - "/api/apikey/login" X-Nonce: - - Nzg0OGVlODEtYTlhYi00NjcwLTgzOWEtYjFlYzA3NzZmNzc5 + - OWJmZGRjMDctNmJiYi00Mjk1LWI1YzUtNzUxOGNkYjE2YjY1 X-Vercel-Cache: - MISS X-Vercel-Id: - - iad1::iad1::c2q6t-1761318581076-fe31aa3b3c22 + - iad1::iad1::5qtgc-1765480329667-85cf8d5c8c1b body: encoding: UTF-8 - string: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' - recorded_at: Fri, 24 Oct 2025 15:09:41 GMT + string: '{"org_info":[{"id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"braintrustdata.com","api_url":"https://staging-api.braintrust.dev","git_metadata":{"fields":["commit","branch","tag","author_name","author_email","commit_message","commit_time","dirty"],"collect":"some"},"is_universal_api":true,"proxy_url":"https://staging-api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + recorded_at: Thu, 11 Dec 2025 19:12:09 GMT - request: method: get - uri: https://api.braintrust.dev/v1/project?project_name=ruby-sdk-test + uri: https://staging-api.braintrust.dev/v1/project?project_name=ruby-sdk-test body: encoding: US-ASCII string: '' @@ -93,7 +94,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Authorization: - Bearer response: @@ -108,43 +109,43 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:41 GMT + - Thu, 11 Dec 2025 19:12:09 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 2d3ee385-2945-4e4a-98ad-38415be65bbd + - 6723bf91-0711-450e-9f3f-1b277f3aa7f3 X-Bt-Internal-Trace-Id: - - 68fb96b5000000007c82800b9535fce1 + - 693b178900000000738ef5a42fd81177 X-Amz-Apigw-Id: - - S9R8XE3IoAMEdpg= + - VcCdmEHLIAMEUkg= Vary: - Origin, Accept-Encoding Etag: - - W/"fa-+YSPhVhMcSODQVu7iSaJULbdOe4" + - W/"fa-HrwmCajpdJfSgika/ZPujG7WfoY" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b5-5855c6004fb2fd7e2e8bd0bb;Parent=3eb1bcf9ecae88ac;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b1789-1c8f439836d01b9671cafc97;Parent=4ba11465c8e3f694;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 c50e3f7de0b772d07240015272b1aff6.cloudfront.net (CloudFront), 1.1 92672fff57a11d8cf4f64313a69242d0.cloudfront.net + - 1.1 cfc9f11ee8d72e5bdd45ea3851048d52.cloudfront.net (CloudFront), 1.1 663d1f2ecede39322c8a8836b25e1bcc.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - j5Eg_0e6Gj85ZFKMxhhj3Bhru8sM6g0UEXM7NpVOtk_Xq2nFPpekVA== + - XsQ6lN2JXompIMXxpQsEYuJP91JmtJ_r9Dou0Rget--RBvo3ECwKyA== body: encoding: ASCII-8BIT - string: '{"objects":[{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}]}' - recorded_at: Fri, 24 Oct 2025 15:09:41 GMT + string: '{"objects":[{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}]}' + recorded_at: Thu, 11 Dec 2025 19:12:10 GMT - request: method: post - uri: https://api.braintrust.dev/v1/function + uri: https://staging-api.braintrust.dev/v1/function body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-eval-scorer","name":"test-ruby-sdk-eval-scorer","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","slug":"test-ruby-sdk-eval-scorer","name":"test-ruby-sdk-eval-scorer","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You are a scorer. Evaluate if the output matches the expected value."},{"role":"user","content":"Does ''{{output}}'' match ''{{expected}}''? Answer ''correct'' or ''incorrect''."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0},"parser":{"type":"llm_classifier","use_cot":true,"choice_scores":{"correct":1.0,"incorrect":0.0}}}}}' headers: @@ -155,7 +156,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -172,42 +173,42 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:41 GMT + - Thu, 11 Dec 2025 19:12:10 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - fb5c61e4-c17f-4f12-9f86-be64fef0449f + - a4cda943-fc9c-4cd3-b907-073cbb446d21 X-Bt-Internal-Trace-Id: - - 68fb96b5000000002d21721e4cbd03b2 + - 693b178a000000000c3a65fc19c29000 X-Amz-Apigw-Id: - - S9R8ZG-qoAMER_g= + - VcCdrGkBIAMEFvA= Vary: - Origin, Accept-Encoding Etag: - - W/"2e5-86Sv80KtusxuXg7gb5hHvUsYWFk" + - W/"2e5-JGbx0+Sr9TQHDIsMqp7ZVaqXOqk" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b5-495d51c14dcdf33b3917c83a;Parent=55c0d3e4164e8258;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b178a-1f2759001b4fb8756a1ad428;Parent=3ecb43b231172408;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 f8b0654d6e6bbf12f54a635de5db7ee4.cloudfront.net (CloudFront), 1.1 3a80316ee74668c7049eb31054a07168.cloudfront.net + - 1.1 7ea7b08f7e1d26433f5be7f6d4283d3c.cloudfront.net (CloudFront), 1.1 81d12325eefc0deca593ce76681fa256.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - MKjOUNPFtDLeeqUI0GOkcpGoChAab4yq6O870JAtzpR709tkZ-5_3A== + - Z9z4ANfrnNivXKgnQkS_0lVsCG6XjELDcoVhoKHBwI92QrXDrsTDQw== body: encoding: ASCII-8BIT - string: '{"log_id":"p","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","slug":"test-ruby-sdk-eval-scorer","name":"test-ruby-sdk-eval-scorer","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + string: '{"log_id":"p","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","slug":"test-ruby-sdk-eval-scorer","name":"test-ruby-sdk-eval-scorer","function_data":{"type":"prompt"},"prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You are a scorer. Evaluate if the output matches the expected value."},{"role":"user","content":"Does - ''{{output}}'' match ''{{expected}}''? Answer ''correct'' or ''incorrect''."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0},"parser":{"type":"llm_classifier","use_cot":true,"choice_scores":{"correct":1,"incorrect":0}}}},"id":"d7669acd-3a20-4698-b553-f20cb020108f","created":"2025-10-24T15:09:41.468Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196022027545774"}' - recorded_at: Fri, 24 Oct 2025 15:09:41 GMT + ''{{output}}'' match ''{{expected}}''? Answer ''correct'' or ''incorrect''."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0},"parser":{"type":"llm_classifier","use_cot":true,"choice_scores":{"correct":1,"incorrect":0}}}},"id":"c79dc30e-6eba-4a6d-95ed-e12bebe54d9b","created":"2025-12-11T19:12:10.507Z","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","_xact_id":"1000196294771873884"}' + recorded_at: Thu, 11 Dec 2025 19:12:11 GMT - request: method: get - uri: https://api.braintrust.dev/v1/function?project_name=ruby-sdk-test&slug=test-ruby-sdk-eval-scorer + uri: https://staging-api.braintrust.dev/v1/function?project_name=ruby-sdk-test&slug=test-ruby-sdk-eval-scorer body: encoding: US-ASCII string: '' @@ -219,7 +220,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Authorization: - Bearer response: @@ -234,42 +235,42 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:42 GMT + - Thu, 11 Dec 2025 19:12:11 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 5bd217f7-1b40-4c65-b6bc-647ba883e206 + - d71fdf5d-280a-4921-88a5-0a93a5fa7761 X-Bt-Internal-Trace-Id: - - 68fb96b5000000007d42cc1f908fee91 + - 693b178b00000000696d85f362f9be3c X-Amz-Apigw-Id: - - S9R8dGh0IAMEHVw= + - VcCdzEoNIAMEAlQ= Vary: - Origin, Accept-Encoding Etag: - - W/"35c-7y7laf0B9BI3D8Mtl/VsJFAzrKE" + - W/"35c-+RVu3hfbb49qOZiIlEA2vuRxTk8" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b5-3515106a6691149e50c165ff;Parent=34c7cd58a6baff75;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b178b-487cba856d9b864b2e789106;Parent=128feeb95a4e76c3;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 f8debc28b6c73eb3dc7540e2ac2f0e18.cloudfront.net (CloudFront), 1.1 ed016821a44f073856f1ffba399e1728.cloudfront.net + - 1.1 7ea7b08f7e1d26433f5be7f6d4283d3c.cloudfront.net (CloudFront), 1.1 97713e58966a50f0173f1cdb4e67aea0.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - OQrczI4IsJCtdlY-WZ3g3tGFHx3-AX-M9KibbFeshQZ6OiG8pQjXJA== + - OOsJGIzRl-ji1eMFPNRIfdJNB_rht-fAqMfo7upG8ftSZBiXyhgeIg== body: encoding: ASCII-8BIT - string: '{"objects":[{"id":"d7669acd-3a20-4698-b553-f20cb020108f","_xact_id":"1000196022027545774","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"test-ruby-sdk-eval-scorer","slug":"test-ruby-sdk-eval-scorer","description":null,"created":"2025-10-24T15:09:41.468Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You + string: '{"objects":[{"id":"c79dc30e-6eba-4a6d-95ed-e12bebe54d9b","_xact_id":"1000196294771873884","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","log_id":"p","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"test-ruby-sdk-eval-scorer","slug":"test-ruby-sdk-eval-scorer","description":null,"created":"2025-12-11T19:12:10.507Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You are a scorer. Evaluate if the output matches the expected value."},{"role":"user","content":"Does ''{{output}}'' match ''{{expected}}''? Answer ''correct'' or ''incorrect''."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0},"parser":{"type":"llm_classifier","use_cot":true,"choice_scores":{"correct":1,"incorrect":0}}}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}]}' - recorded_at: Fri, 24 Oct 2025 15:09:42 GMT + recorded_at: Thu, 11 Dec 2025 19:12:11 GMT - request: method: post - uri: https://api.braintrust.dev/v1/project + uri: https://staging-api.braintrust.dev/v1/project body: encoding: UTF-8 string: '{"name":"ruby-sdk-test"}' @@ -281,7 +282,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -298,45 +299,45 @@ http_interactions: Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:42 GMT + - Thu, 11 Dec 2025 19:12:11 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 334a2254-1c4f-43a7-9995-49b70169eac4 + - fcb9c044-4e5b-448c-9ec0-708eb8510c46 X-Bt-Internal-Trace-Id: - - 68fb96b60000000075fabf960df0c1df + - 693b178b0000000015a9f755181ca82c X-Amz-Apigw-Id: - - S9R8hGsroAMEGyg= + - VcCd5GgxIAMEngA= Vary: - Origin, Accept-Encoding Etag: - - W/"ec-ae+104VIsuTGSnPanxA+Dc4IyRE" + - W/"ec-tjqKTzXkSD9WFkzugu+08z52cpA" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b6-365414c82eb2cb4b33f6e030;Parent=05111cceb012466c;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b178b-1b7b4e1d2f104db30f8d6831;Parent=7d934ea57793e65e;Sampled=0;Lineage=1:fc3b4ff1:0 X-Bt-Found-Existing: - 'true' Via: - - 1.1 bf8d7cb6fca5d51158e1109ca40fe242.cloudfront.net (CloudFront), 1.1 e42e8491a089e2183879e26e61dae708.cloudfront.net + - 1.1 7ea7b08f7e1d26433f5be7f6d4283d3c.cloudfront.net (CloudFront), 1.1 a9dc097bbaf22a663c80eb85450d7cce.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - R7DaQGIaCz2Vi0_mDQIc_FBKO1G3xEqVgEcj8un17iIsIckBtGJ7Aw== + - Q_kEoblhPp5QBVwQ8gumfFKswedJuKP9bxDBZXlk9skDH-8BAyP3dw== body: encoding: ASCII-8BIT - string: '{"id":"c532bc50-7094-4bbb-8704-42344c9728b9","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"ruby-sdk-test","created":"2025-10-22T02:53:49.779Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' - recorded_at: Fri, 24 Oct 2025 15:09:42 GMT + string: '{"id":"ac86d18e-af78-4caf-918d-b89ad108ec67","org_id":"5d7c97d7-fef1-4cb7-bda6-7e3756a0ca8e","name":"ruby-sdk-test","created":"2025-10-22T03:32:12.324Z","deleted_at":null,"user_id":"f2ddc4e6-a51a-4a60-9734-9af4ea05c6ef","settings":null}' + recorded_at: Thu, 11 Dec 2025 19:12:11 GMT - request: method: post - uri: https://api.braintrust.dev/v1/experiment + uri: https://staging-api.braintrust.dev/v1/experiment body: encoding: UTF-8 - string: '{"project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-remote-scorer-eval","ensure_new":true}' + string: '{"project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-remote-scorer-eval","ensure_new":true}' headers: Accept-Encoding: - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 @@ -345,7 +346,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -358,44 +359,44 @@ http_interactions: Content-Type: - application/json; charset=utf-8 Content-Length: - - '403' + - '405' Connection: - keep-alive X-Amz-Cf-Pop: - - JFK50-P2 - - JFK50-P5 + - JFK50-P10 + - JFK52-P8 Date: - - Fri, 24 Oct 2025 15:09:42 GMT + - Thu, 11 Dec 2025 19:12:12 GMT Access-Control-Allow-Credentials: - 'true' X-Amzn-Requestid: - - 79d85d72-84b9-423e-80ed-3205fa6ba209 + - 35e07d51-4f0e-42b7-a3f0-f31b5aa491e6 X-Bt-Internal-Trace-Id: - - 68fb96b60000000006e679fe295ad759 + - 693b178b000000001b3424863a65d9e2 X-Amz-Apigw-Id: - - S9R8jEB6oAMEsig= + - VcCd7HxEIAMEvGw= Vary: - Origin, Accept-Encoding Etag: - - W/"193-YAPWC3gzaEVmbQxU7YZmv4IVeOU" + - W/"195-3QAI5G6Px+CzyHASrk0FxUNN560" Access-Control-Expose-Headers: - x-bt-cursor,x-bt-found-existing,x-bt-query-plan X-Amzn-Trace-Id: - - Root=1-68fb96b6-37ab25e219b75d2639f2b114;Parent=1a8bf0d44812091f;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-693b178b-2073517a3e76cfdf797f951d;Parent=05a6ba3d0aa6c4c3;Sampled=0;Lineage=1:fc3b4ff1:0 Via: - - 1.1 53a1f042d35b1ad7e45dd18908041b36.cloudfront.net (CloudFront), 1.1 8ca36406fe3aa11c1641e5bc917c8a74.cloudfront.net + - 1.1 7d77965d78b3f4565239009cf6e62356.cloudfront.net (CloudFront), 1.1 663d1f2ecede39322c8a8836b25e1bcc.cloudfront.net (CloudFront) X-Cache: - Miss from cloudfront X-Amz-Cf-Id: - - 710kAY3fynR-OkQYvgMDJqg_JdgWkkypSirQGKcPxNhjI5KThtClbw== + - WGr6QRJLLRjcWgQFCau5nyPNFLVF-oFjDR6QfrFsweVRpFbApLQkcw== body: encoding: ASCII-8BIT - string: '{"id":"845bf1f9-4c97-4d8d-8dc9-72e4acc8125b","project_id":"c532bc50-7094-4bbb-8704-42344c9728b9","name":"test-ruby-sdk-remote-scorer-eval-a0947db1","description":null,"created":"2025-10-24T15:09:42.408Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' - recorded_at: Fri, 24 Oct 2025 15:09:42 GMT + string: '{"id":"902f477a-efb9-4f5b-a94e-057943d787cc","project_id":"ac86d18e-af78-4caf-918d-b89ad108ec67","name":"test-ruby-sdk-remote-scorer-eval-4fa91e80","description":null,"created":"2025-12-11T19:12:12.072Z","repo_info":null,"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"c755328d-f64a-4737-a984-e83c088cd9f7","metadata":null,"tags":null}' + recorded_at: Thu, 11 Dec 2025 19:12:12 GMT - request: method: post - uri: https://api.braintrust.dev/v1/function/d7669acd-3a20-4698-b553-f20cb020108f/invoke + uri: https://staging-api.braintrust.dev/v1/function/c79dc30e-6eba-4a6d-95ed-e12bebe54d9b/invoke body: encoding: UTF-8 string: '{"input":{"input":"hello","expected":"HELLO","output":"HELLO","metadata":{}}}' @@ -407,7 +408,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -424,75 +425,62 @@ http_interactions: Connection: - keep-alive Date: - - Fri, 24 Oct 2025 15:09:43 GMT + - Thu, 11 Dec 2025 19:12:12 GMT X-Bt-Used-Endpoint: - - OPENAI_API_KEY + - AZURE_DEFAULT_CREDENTIALS X-Amzn-Requestid: - - 8c34da6c-204c-42dc-b993-3f4f409b78db - Cf-Ray: - - 9939e4096dd8083d-IAD + - 88a2ac39-6e3b-46a1-ad85-af8d6140ee4f + X-Ms-Deployment-Name: + - gpt-4o-mini X-Ratelimit-Remaining-Requests: - - '29999' - Openai-Processing-Ms: - - '807' - Cf-Cache-Status: - - DYNAMIC + - '2499' + X-Ms-Region: + - East US + X-Ms-Rai-Invoked: + - 'true' Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload X-Bt-Cached: - HIT - X-Openai-Proxy-Wasm: - - v0.1 - X-Ratelimit-Reset-Requests: - - 2ms - X-Envoy-Upstream-Service-Time: - - '1012' - Openai-Organization: - - braintrust-data X-Request-Id: - - req_183e857a9a414f5eacf144e983805362 - Set-Cookie: - - _cfuvid=NZaCohaxOxkI95Mx4m4OR3zzTPXWjQPiBzBUYdI8OiY-1761313277463-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - 524b6f3e-95ec-4c01-9d6b-eba7ee8b5b78 Cache-Control: - max-age=604800 + Azureml-Model-Session: + - d20251210231011-628f6c5d8b8c414b + Apim-Request-Id: + - 935e7ef9-1be8-4289-9acb-3a49452c55b5 X-Ratelimit-Limit-Tokens: - - '150000000' + - '250000' X-Content-Type-Options: - nosniff X-Ratelimit-Remaining-Tokens: - - '149999965' + - '249966' X-Bt-Function-Creds-Cached: - HIT X-Amzn-Trace-Id: - - Root=1-68fb96b6-7cbf910a24cf9f3e7d1dc3df;Parent=5ad0d8514bf62fbd;Sampled=0;Lineage=1:8be8f50d:0 + - Root=1-693b178c-4da0f37b4c05d37161ba889b;Parent=5f1b546c6abc561b;Sampled=0;Lineage=1:98839c8e:0 X-Bt-Function-Meta-Cached: - MISS - X-Ratelimit-Reset-Tokens: - - 0s - Openai-Version: - - '2020-10-01' X-Ratelimit-Limit-Requests: - - '30000' - Openai-Project: - - proj_wMRY6YpEiASXMxPIIcI9nQRi + - '2500' X-Cache: - Miss from cloudfront Via: - - 1.1 10a23502057a5449ee9e08eab6e9c0d4.cloudfront.net (CloudFront) + - 1.1 94ad62d30119f960dbd892be99684e02.cloudfront.net (CloudFront) X-Amz-Cf-Pop: - - JFK50-P2 + - JFK52-P8 X-Amz-Cf-Id: - - Lf635YZAk3njb6QJTwgiWncBs-qPgLBZoXWPuMrxAtNPeX_UvtGf5A== + - 05VPYCTxzuP7LOhVsK8YCsmKCO2uWtxEFRUb9SepHotqWXCh2T_z1A== Age: - - '5305' + - '2034' body: encoding: UTF-8 string: '"correct"' - recorded_at: Fri, 24 Oct 2025 15:09:43 GMT + recorded_at: Thu, 11 Dec 2025 19:12:12 GMT - request: method: post - uri: https://api.braintrust.dev/v1/function/d7669acd-3a20-4698-b553-f20cb020108f/invoke + uri: https://staging-api.braintrust.dev/v1/function/c79dc30e-6eba-4a6d-95ed-e12bebe54d9b/invoke body: encoding: UTF-8 string: '{"input":{"input":"world","expected":"WORLD","output":"WORLD","metadata":{}}}' @@ -504,7 +492,7 @@ http_interactions: User-Agent: - Ruby Host: - - api.braintrust.dev + - staging-api.braintrust.dev Content-Type: - application/json Authorization: @@ -521,70 +509,117 @@ http_interactions: Connection: - keep-alive Date: - - Fri, 24 Oct 2025 15:09:43 GMT + - Thu, 11 Dec 2025 19:12:12 GMT X-Bt-Used-Endpoint: - - OPENAI_API_KEY + - AZURE_DEFAULT_CREDENTIALS X-Amzn-Requestid: - - 9c00aa95-d935-42a5-8ac1-a7dc600f90f3 - Cf-Ray: - - 9939e4120e29827b-IAD + - c430d6fe-6c08-43e3-a8fd-0d1332f1d07c + X-Ms-Deployment-Name: + - gpt-4o-mini X-Ratelimit-Remaining-Requests: - - '29999' - Openai-Processing-Ms: - - '1292' - Cf-Cache-Status: - - DYNAMIC + - '2498' + X-Ms-Region: + - East US + X-Ms-Rai-Invoked: + - 'true' Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload X-Bt-Cached: - HIT - X-Openai-Proxy-Wasm: - - v0.1 - X-Ratelimit-Reset-Requests: - - 2ms - X-Envoy-Upstream-Service-Time: - - '1356' - Openai-Organization: - - braintrust-data X-Request-Id: - - req_3b4cbed806f74c87a3271f5c6a604e5e - Set-Cookie: - - _cfuvid=bnvjD_AxaE5R4fbzu0ZeunODOvUneRBfGsQ4qqV2PNs-1761313279181-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - 87a051ba-805d-48c9-9f08-65f3b6989fed Cache-Control: - max-age=604800 + Azureml-Model-Session: + - d20251210231011-628f6c5d8b8c414b + Apim-Request-Id: + - 02c95985-2b21-4561-b4d7-e39fcca0de46 X-Ratelimit-Limit-Tokens: - - '150000000' + - '250000' X-Content-Type-Options: - nosniff X-Ratelimit-Remaining-Tokens: - - '149999965' + - '249932' X-Bt-Function-Creds-Cached: - HIT X-Amzn-Trace-Id: - - Root=1-68fb96b7-4a895e6c79fef3070bc35e38;Parent=38a2dbefe6ae1b3c;Sampled=0;Lineage=1:8be8f50d:0 + - Root=1-693b178c-1b9f8b240201b3ef2a754551;Parent=7a32d4cc0567b4ae;Sampled=0;Lineage=1:98839c8e:0 X-Bt-Function-Meta-Cached: - HIT - X-Ratelimit-Reset-Tokens: - - 0s - Openai-Version: - - '2020-10-01' X-Ratelimit-Limit-Requests: - - '30000' - Openai-Project: - - proj_wMRY6YpEiASXMxPIIcI9nQRi + - '2500' X-Cache: - Miss from cloudfront Via: - - 1.1 d4b67fda8355378cec4afc079701f8e6.cloudfront.net (CloudFront) + - 1.1 e313693179ab4339368ba93edb851532.cloudfront.net (CloudFront) X-Amz-Cf-Pop: - - JFK50-P2 + - JFK52-P8 X-Amz-Cf-Id: - - WncpHoCD1o_fpgbl1SYCSIhiFzNaFFEbWaQfnmqOO9P3DAiYiatXfA== + - y5WHHtl4sunQ-zwfqP8SYI9krNN8by3-eeGZc16ys1Y612wmdByeiA== Age: - - '5303' + - '2033' body: encoding: UTF-8 string: '"correct"' - recorded_at: Fri, 24 Oct 2025 15:09:43 GMT + recorded_at: Thu, 11 Dec 2025 19:12:12 GMT +- request: + method: get + uri: https://staging-api.braintrust.dev/experiment-comparison2?experiment_id=902f477a-efb9-4f5b-a94e-057943d787cc + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + User-Agent: + - Ruby + Host: + - staging-api.braintrust.dev + Authorization: + - Bearer + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '26' + Connection: + - keep-alive + X-Amz-Cf-Pop: + - JFK50-P10 + - JFK52-P8 + Date: + - Thu, 11 Dec 2025 19:12:13 GMT + Access-Control-Allow-Credentials: + - 'true' + X-Amzn-Requestid: + - b3d8ac6f-c77e-4f8a-8b5b-9ea8bf38d2be + X-Bt-Internal-Trace-Id: + - 693b178d000000005a3ca3016c2a683a + X-Amz-Apigw-Id: + - VcCeGEY2oAMEdhQ= + Vary: + - Origin, Accept-Encoding + Etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + X-Amzn-Trace-Id: + - Root=1-693b178d-1a37b89555deada239ee7eb5;Parent=0c6904676fcd20a1;Sampled=0;Lineage=1:fc3b4ff1:0 + Via: + - 1.1 7d77965d78b3f4565239009cf6e62356.cloudfront.net (CloudFront), 1.1 1600051e9ac5d9cd9f5329cf4d5a06f8.cloudfront.net + (CloudFront) + X-Cache: + - Miss from cloudfront + X-Amz-Cf-Id: + - 9jWjk_RKq65JaOsOEr8nCN-B_plwCS2sDzp7rPsW730DU8XSH3rR0A== + body: + encoding: ASCII-8BIT + string: '{"scores":{},"metrics":{}}' + recorded_at: Thu, 11 Dec 2025 19:12:13 GMT recorded_with: VCR 6.3.1 diff --git a/test/test_helper.rb b/test/test_helper.rb index 73fcc23..c8998db 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -192,6 +192,33 @@ def get_openai_key def get_anthropic_key ENV["ANTHROPIC_API_KEY"] || "sk-ant-test-key-for-vcr" end + + # Creates a State for integration tests with in-memory exporter + # This performs login (via VCR cassettes) but uses an in-memory span exporter + # instead of OTLP, so force_flush doesn't cause timeouts in tests. + # Use this for tests that need API access AND tracing with fast flush. + # @param options [Hash] Options to pass to Braintrust.init + # @return [Hash] { state:, tracer_provider:, exporter: } + def get_integration_test_state_with_memory_exporter(**options) + require "opentelemetry/sdk" + + # Create in-memory exporter (no HTTP calls, instant flush) + exporter = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new + + # Create a new TracerProvider for this test + tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new + + # Initialize state with in-memory exporter + state = Braintrust.init( + set_global: false, + blocking_login: true, + tracer_provider: tracer_provider, + exporter: exporter, + **options + ) + + {state: state, tracer_provider: tracer_provider, exporter: exporter} + end end # Include helper in all test cases