Upgrade autoevals to zod 4

evals/package.json

@@ -14,7 +14,7 @@
   "dependencies": {
     "autoevals": "workspace:*",
     "braintrust": "^0.0.140",
-    "zod": "^3.22.4"
+    "zod": "^4.2.1"
   },
   "devDependencies": {
     "@types/node": "^20.10.5",

evals/src/autoevals.eval.ts

@@ -7,7 +7,7 @@ import {
   coqaCaseSchema,
   dataDir,
 } from "./datasets";
-import { z } from "zod";
+import { z } from "zod/v3";
 import {
   AnswerCorrectness,
   ClosedQA,

evals/src/datasets.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v3";
 
 import path from "path";
 

evals/src/sync_datasets.ts

@@ -1,6 +1,6 @@
 import { duckq, getDuckDBConn } from "./duckdb";
 
-import { z } from "zod";
+import { z } from "zod/v3";
 import {
   coqaSchema,
   dataDir,

js/ragas.ts

@@ -5,10 +5,10 @@ import { Scorer, ScorerArgs } from "./score";
 import { DEFAULT_MODEL, LLMArgs } from "./llm";
 import { buildOpenAIClient, extractOpenAIArgs } from "./oai";
 import OpenAI from "openai";
+import { zodFunction } from "openai/helpers/zod";
 import { ListContains } from "./list";
 import { EmbeddingSimilarity } from "./string";
-import { z } from "zod";
-import zodToJsonSchema from "zod-to-json-schema";
+import { z } from "zod/v3";
 import { makePartial, ScorerWithPartial } from "./partial";
 
 type RagasArgs = {
@@ -87,14 +87,11 @@ export const ContextEntityRecall: ScorerWithPartial<
       },
     ],
     tools: [
-      {
-        type: "function",
-        function: {
-          name: "extract_entities",
-          description: "Extract unique entities from a given text",
-          parameters: zodToJsonSchema(entitySchema),
-        },
-      },
+      zodFunction({
+        name: "extract_entities",
+        description: "Extract unique entities from a given text",
+        parameters: entitySchema,
+      }),
     ],
     tool_choice: { type: "function", function: { name: "extract_entities" } },
   });
@@ -167,14 +164,11 @@ export const ContextRelevancy: ScorerWithPartial<string, RagasArgs> =
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "extract_sentences",
-            description: "Extract relevant sentences from a given context",
-            parameters: zodToJsonSchema(relevantSentencesSchema),
-          },
-        },
+        zodFunction({
+          name: "extract_sentences",
+          description: "Extract relevant sentences from a given context",
+          parameters: relevantSentencesSchema,
+        }),
       ],
       tool_choice: {
         type: "function",
@@ -267,13 +261,10 @@ export const ContextRecall: ScorerWithPartial<string, RagasArgs> = makePartial(
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "extract_statements",
-            parameters: zodToJsonSchema(contextRecallSchema),
-          },
-        },
+        zodFunction({
+          name: "extract_statements",
+          parameters: contextRecallSchema,
+        }),
       ],
       tool_choice: {
         type: "function",
@@ -367,15 +358,11 @@ export const ContextPrecision: ScorerWithPartial<string, RagasArgs> =
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "verify",
-            description:
-              "Verify if context was useful in arriving at the answer",
-            parameters: zodToJsonSchema(contextPrecisionSchema),
-          },
-        },
+        zodFunction({
+          name: "verify",
+          description: "Verify if context was useful in arriving at the answer",
+          parameters: contextPrecisionSchema,
+        }),
       ],
       tool_choice: { type: "function", function: { name: "verify" } },
     });
@@ -494,14 +481,11 @@ export const Faithfulness: ScorerWithPartial<string, RagasArgs> = makePartial(
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "extract_statements",
-            description: "Extract statements from an answer given a question",
-            parameters: zodToJsonSchema(extractedStatementsSchema),
-          },
-        },
+        zodFunction({
+          name: "extract_statements",
+          description: "Extract statements from an answer given a question",
+          parameters: extractedStatementsSchema,
+        }),
       ],
       tool_choice: {
         type: "function",
@@ -525,15 +509,12 @@ export const Faithfulness: ScorerWithPartial<string, RagasArgs> = makePartial(
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "judge_statements",
-            description:
-              "Judge whether the statements are faithful to the context",
-            parameters: zodToJsonSchema(statementFaithfulnessSchema),
-          },
-        },
+        zodFunction({
+          name: "judge_statements",
+          description:
+            "Judge whether the statements are faithful to the context",
+          parameters: statementFaithfulnessSchema,
+        }),
       ],
       tool_choice: { type: "function", function: { name: "judge_statements" } },
     });
@@ -635,15 +616,12 @@ export const AnswerRelevancy: ScorerWithPartial<
           },
         ],
         tools: [
-          {
-            type: "function",
-            function: {
-              name: "generate_question",
-              description:
-                "Generate a question for the given answer and identify if the answer is noncommittal",
-              parameters: zodToJsonSchema(questionGenSchema),
-            },
-          },
+          zodFunction({
+            name: "generate_question",
+            description:
+              "Generate a question for the given answer and identify if the answer is noncommittal",
+            parameters: questionGenSchema,
+          }),
         ],
         tool_choice: {
           type: "function",
@@ -810,14 +788,11 @@ export const AnswerCorrectness: ScorerWithPartial<
         },
       ],
       tools: [
-        {
-          type: "function",
-          function: {
-            name: "classify_statements",
-            description: "Classify statements as TP, FP, or FN",
-            parameters: zodToJsonSchema(answerCorrectnessClassificationSchema),
-          },
-        },
+        zodFunction({
+          name: "classify_statements",
+          description: "Classify statements as TP, FP, or FN",
+          parameters: answerCorrectnessClassificationSchema,
+        }),
       ],
       tool_choice: {
         type: "function",

js/templates.ts

@@ -1,4 +1,4 @@
-import { z } from "zod";
+import { z } from "zod/v3";
 import * as yaml from "js-yaml";
 
 import battle from "../templates/battle.yaml";
@@ -13,7 +13,7 @@ import translation from "../templates/translation.yaml";
 
 export const modelGradedSpecSchema = z.object({
   prompt: z.string(),
-  choice_scores: z.record(z.number()),
+  choice_scores: z.record(z.string(), z.number()),
   model: z.string().optional(),
   use_cot: z.boolean().optional(),
   temperature: z.number().optional(),

package.json

@@ -1,6 +1,6 @@
 {
   "name": "autoevals",
-  "version": "0.0.131",
+  "version": "0.1.0",
   "description": "Universal library for evaluating AI models",
   "repository": {
     "type": "git",
@@ -44,7 +44,11 @@
     "typedoc": "^0.25.13",
     "typedoc-plugin-markdown": "^3.17.1",
     "typescript": "^5.9.2",
-    "vitest": "^2.1.9"
+    "vitest": "^2.1.9",
+    "zod": "3.25.67"
+  },
+  "peerDependencies": {
+    "zod": "^3.0.0 || ^4.0.0"
   },
   "dependencies": {
     "ajv": "^8.17.1",
@@ -53,9 +57,8 @@
     "js-yaml": "^4.1.0",
     "linear-sum-assignment": "^1.0.7",
     "mustache": "^4.2.0",
-    "openai": "^6.3.0",
-    "zod": "^3.25.76",
-    "zod-to-json-schema": "^3.24.6"
+    "openai": "^6.7.0",
+    "zod-to-json-schema": "3.25.0"
   },
   "packageManager": "pnpm@10.26.2"
 }