Initial Support for Remote Evals Devserver

examples/build.gradle

@@ -141,3 +141,18 @@ task runSpringAI(type: JavaExec) {
     suspend = false
   }
 }
+
+
+task runRemoteEval(type: JavaExec) {
+  group = 'Braintrust SDK Examples'
+  description = 'Run the remote eval example'
+  classpath = sourceSets.main.runtimeClasspath
+  mainClass = 'dev.braintrust.examples.RemoteEvalExample'
+  systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel
+  debugOptions {
+    enabled = true
+    port = 5566
+    server = true
+    suspend = false
+  }
+}

examples/src/main/java/dev/braintrust/examples/RemoteEvalExample.java

@@ -0,0 +1,76 @@
+package dev.braintrust.examples;
+
+import com.openai.client.okhttp.OpenAIOkHttpClient;
+import com.openai.models.ChatModel;
+import com.openai.models.chat.completions.ChatCompletionCreateParams;
+import dev.braintrust.Braintrust;
+import dev.braintrust.devserver.Devserver;
+import dev.braintrust.devserver.RemoteEval;
+import dev.braintrust.eval.Scorer;
+import dev.braintrust.instrumentation.openai.BraintrustOpenAI;
+import java.util.List;
+
+/** Simple Dev Server for Remote Evals */
+public class RemoteEvalExample {
+    public static void main(String[] args) throws Exception {
+        var braintrust = Braintrust.get();
+        var openTelemetry = braintrust.openTelemetryCreate();
+        var openAIClient = BraintrustOpenAI.wrapOpenAI(openTelemetry, OpenAIOkHttpClient.fromEnv());
+
+        RemoteEval<String, String> foodTypeEval =
+                RemoteEval.<String, String>builder()
+                        .name("food-type-classifier")
+                        .taskFunction(
+                                food -> {
+                                    var request =
+                                            ChatCompletionCreateParams.builder()
+                                                    .model(ChatModel.GPT_4O_MINI)
+                                                    .addSystemMessage("Return a one word answer")
+                                                    .addUserMessage(
+                                                            "What kind of food is " + food + "?")
+                                                    .maxTokens(50L)
+                                                    .temperature(0.0)
+                                                    .build();
+                                    var response =
+                                            openAIClient.chat().completions().create(request);
+                                    return response.choices()
+                                            .get(0)
+                                            .message()
+                                            .content()
+                                            .orElse("")
+                                            .toLowerCase();
+                                })
+                        .scorers(
+                                List.of(
+                                        Scorer.of("static_scorer", (expected, result) -> 0.7),
+                                        Scorer.of(
+                                                "close_enough_match",
+                                                (expected, result) ->
+                                                        expected.trim()
+                                                                        .equalsIgnoreCase(
+                                                                                result.trim())
+                                                                ? 1.0
+                                                                : 0.0)))
+                        .build();
+
+        Devserver devserver =
+                Devserver.builder()
+                        .config(braintrust.config())
+                        .registerEval(foodTypeEval)
+                        .host("localhost") // set to 0.0.0.0 to bind all interfaces
+                        .port(8301)
+                        .build();
+
+        Runtime.getRuntime()
+                .addShutdownHook(
+                        new Thread(
+                                () -> {
+                                    System.out.println("Shutting down...");
+                                    devserver.stop();
+                                    System.out.flush();
+                                    System.err.flush();
+                                }));
+        System.out.println("Starting Braintrust dev server");
+        devserver.start();
+    }
+}

src/main/java/dev/braintrust/BraintrustUtils.java

@@ -3,6 +3,9 @@
 import dev.braintrust.api.BraintrustApiClient;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import javax.annotation.Nonnull;
 
 public class BraintrustUtils {
@@ -28,7 +31,7 @@ public static URI createProjectURI(
         }
     }
 
-    static Parent parseParent(@Nonnull String parentStr) {
+    public static Parent parseParent(@Nonnull String parentStr) {
         String[] parts = parentStr.split(":");
         if (parts.length != 2) {
             throw new IllegalArgumentException("Invalid parent format: " + parentStr);
@@ -42,4 +45,18 @@ public String toParentValue() {
             return type + ":" + id;
         }
     }
+
+    public static List<String> parseCsv(String csv) {
+        if (csv == null || csv.isBlank()) {
+            return List.of();
+        }
+
+        return Arrays.stream(csv.split("\\s*,\\s*")).toList();
+    }
+
+    public static <T> List<T> append(List<T> list, T value) {
+        List<T> result = new ArrayList<>(list);
+        result.add(value);
+        return result;
+    }
 }

src/main/java/dev/braintrust/api/BraintrustApiClient.java

@@ -26,6 +26,14 @@
  * {@link dev.braintrust.eval.Eval} or {@link dev.braintrust.trace.BraintrustTracing}
  */
 public interface BraintrustApiClient {
+    /**
+     * Attempt Braintrust login
+     *
+     * @return LoginResponse containing organization info
+     * @throws LoginException if login fails due to invalid credentials or network errors
+     */
+    LoginResponse login() throws LoginException;
+
     /** Creates or gets a project by name. */
     Project getOrCreateProject(String projectName);
 
@@ -117,15 +125,16 @@ public Experiment getOrCreateExperiment(CreateExperimentRequest request) {
             }
         }
 
-        private LoginResponse login() {
+        @Override
+        public LoginResponse login() throws LoginException {
             try {
                 return postAsync(
                                 "/api/apikey/login",
                                 new LoginRequest(config.apiKey()),
                                 LoginResponse.class)
                         .get();
             } catch (InterruptedException | ExecutionException e) {
-                throw new RuntimeException(e);
+                throw new LoginException("Failed to login to Braintrust", e);
             }
         }
 
@@ -403,6 +412,12 @@ public InMemoryImpl(
             this.prompts.addAll(prompts);
         }
 
+        @Override
+        public LoginResponse login() {
+            return new LoginResponse(
+                    organizationAndProjectInfos.stream().map(o -> o.orgInfo).toList());
+        }
+
         @Override
         public Project getOrCreateProject(String projectName) {
             // Find existing project by name

src/main/java/dev/braintrust/api/LoginException.java

@@ -0,0 +1,23 @@
+package dev.braintrust.api;
+
+import javax.annotation.Nullable;
+
+/**
+ * Exception thrown when login to Braintrust fails.
+ *
+ * <p>This is a RuntimeException so it doesn't require explicit handling, but callers can catch it
+ * specifically if they want to handle login failures differently from other errors.
+ */
+public class LoginException extends RuntimeException {
+    public LoginException(String message) {
+        super(message);
+    }
+
+    public LoginException(String message, @Nullable Throwable cause) {
+        super(message, cause);
+    }
+
+    public LoginException(Throwable cause) {
+        super(cause);
+    }
+}

src/main/java/dev/braintrust/config/BraintrustConfig.java

@@ -51,6 +51,12 @@ public final class BraintrustConfig extends BaseConfig {
     private final boolean exportSpansInMemoryForUnitTest =
             getConfig("BRAINTRUST_JAVA_EXPORT_SPANS_IN_MEMORY_FOR_UNIT_TEST", false);
 
+    /** CORS origins to allow when running remote eval devserver */
+    private final String devserverCorsOriginWhitelistCsv =
+            getConfig(
+                    "BRAINTRUST_DEVSERVER_CORS_ORIGIN_WHITELIST_CSV",
+                    "https://www.braintrust.dev,https://www.braintrustdata.com,http://localhost:3000");
+
     public static BraintrustConfig fromEnvironment() {
         return of();
     }
@@ -192,8 +198,8 @@ Builder experimentalOtelLogs(boolean value) {
             return this;
         }
 
-        // hiding visibility. only used for testing
-        Builder exportSpansInMemoryForUnitTest(boolean value) {
+        // only used for testing
+        public Builder exportSpansInMemoryForUnitTest(boolean value) {
             envOverrides.put(
                     "BRAINTRUST_JAVA_EXPORT_SPANS_IN_MEMORY_FOR_UNIT_TEST", String.valueOf(value));
             return this;
@@ -209,6 +215,11 @@ public Builder x509TrustManager(X509TrustManager value) {
             return this;
         }
 
+        public Builder devserverCorsOriginWhitelistCsv(String csv) {
+            envOverrides.put("BRAINTRUST_DEVSERVER_CORS_ORIGIN_WHITELIST_CSV", csv);
+            return this;
+        }
+
         public BraintrustConfig build() {
             return new BraintrustConfig(envOverrides, sslContext, x509TrustManager);
         }