diff --git a/evals/datasets/coqa-closed-qa.json b/evals/datasets/coqa-closed-qa.json index 31af669..a09b9b8 100644 --- a/evals/datasets/coqa-closed-qa.json +++ b/evals/datasets/coqa-closed-qa.json @@ -39,32 +39,10 @@ "no" ], "answer_start": [ - 59, - 18, - 196, - 281, - 428, - 512, - 678, - 647, - 718, - 1035, - 1143, - 965 + 59, 18, 196, 281, 428, 512, 678, 647, 718, 1035, 1143, 965 ], "answer_end": [ - 93, - 80, - 215, - 315, - 490, - 549, - 716, - 676, - 776, - 1097, - 1170, - 1008 + 93, 80, 215, 315, 490, 549, 716, 676, 776, 1097, 1170, 1008 ] } } @@ -109,32 +87,10 @@ "no" ], "answer_start": [ - 59, - 18, - 196, - 281, - 428, - 512, - 678, - 647, - 718, - 1035, - 1143, - 965 + 59, 18, 196, 281, 428, 512, 678, 647, 718, 1035, 1143, 965 ], "answer_end": [ - 93, - 80, - 215, - 315, - 490, - 549, - 716, - 676, - 776, - 1097, - 1170, - 1008 + 93, 80, 215, 315, 490, 549, 716, 676, 776, 1097, 1170, 1008 ] } } @@ -176,32 +132,8 @@ "yes", "unknown" ], - "answer_start": [ - 37, - 167, - 303, - 281, - 552, - 708, - 964, - 910, - -1, - 1056, - -1 - ], - "answer_end": [ - 43, - 175, - 346, - 302, - 577, - 723, - 995, - 935, - -1, - 1173, - -1 - ] + "answer_start": [37, 167, 303, 281, 552, 708, 964, 910, -1, 1056, -1], + "answer_end": [43, 175, 346, 302, 577, 723, 995, 935, -1, 1173, -1] } } }, @@ -242,32 +174,8 @@ "yes", "unknown" ], - "answer_start": [ - 37, - 167, - 303, - 281, - 552, - 708, - 964, - 910, - -1, - 1056, - -1 - ], - "answer_end": [ - 43, - 175, - 346, - 302, - 577, - 723, - 995, - 935, - -1, - 1173, - -1 - ] + "answer_start": [37, 167, 303, 281, 552, 708, 964, 910, -1, 1056, -1], + "answer_end": [43, 175, 346, 302, 577, 723, 995, 935, -1, 1173, -1] } } }, @@ -317,38 +225,12 @@ "\"Thank you\"" ], "answer_start": [ - 19, - 136, - 146, - 169, - 304, - 348, - 214, - 579, - 727, - 423, - 917, - 594, - 1771, - 1667, + 19, 136, 146, 169, 304, 348, 214, 579, 727, 423, 917, 594, 1771, 1667, 1874 ], "answer_end": [ - 115, - 165, - 165, - 185, - 324, - 375, - 302, - 725, - 769, - 544, - 924, - 725, - 1814, - 1769, - 1885 + 115, 165, 165, 185, 324, 375, 302, 725, 769, 544, 924, 725, 1814, + 1769, 1885 ] } } @@ -399,38 +281,12 @@ "\"Thank you\"" ], "answer_start": [ - 19, - 136, - 146, - 169, - 304, - 348, - 214, - 579, - 727, - 423, - 917, - 594, - 1771, - 1667, + 19, 136, 146, 169, 304, 348, 214, 579, 727, 423, 917, 594, 1771, 1667, 1874 ], "answer_end": [ - 115, - 165, - 165, - 185, - 324, - 375, - 302, - 725, - 769, - 544, - 924, - 725, - 1814, - 1769, - 1885 + 115, 165, 165, 185, 324, 375, 302, 725, 769, 544, 924, 725, 1814, + 1769, 1885 ] } } @@ -491,48 +347,12 @@ "A cop" ], "answer_start": [ - 8, - 8, - 8, - 9, - 512, - 856, - 573, - 512, - 512, - 856, - 1041, - 1041, - 1041, - 1217, - 1216, - 1217, - 1217, - 1338, - 8, - 8 + 8, 8, 8, 9, 512, 856, 573, 512, 512, 856, 1041, 1041, 1041, 1217, + 1216, 1217, 1217, 1338, 8, 8 ], "answer_end": [ - 181, - 183, - 66, - 181, - 690, - 1039, - 690, - 662, - 690, - 1039, - 1103, - 1134, - 1165, - 1336, - 1336, - 1336, - 1336, - 1381, - 138, - 65 + 181, 183, 66, 181, 690, 1039, 690, 662, 690, 1039, 1103, 1134, 1165, + 1336, 1336, 1336, 1336, 1381, 138, 65 ] } } @@ -593,48 +413,12 @@ "A cop" ], "answer_start": [ - 8, - 8, - 8, - 9, - 512, - 856, - 573, - 512, - 512, - 856, - 1041, - 1041, - 1041, - 1217, - 1216, - 1217, - 1217, - 1338, - 8, - 8 + 8, 8, 8, 9, 512, 856, 573, 512, 512, 856, 1041, 1041, 1041, 1217, + 1216, 1217, 1217, 1338, 8, 8 ], "answer_end": [ - 181, - 183, - 66, - 181, - 690, - 1039, - 690, - 662, - 690, - 1039, - 1103, - 1134, - 1165, - 1336, - 1336, - 1336, - 1336, - 1381, - 138, - 65 + 181, 183, 66, 181, 690, 1039, 690, 662, 690, 1039, 1103, 1134, 1165, + 1336, 1336, 1336, 1336, 1381, 138, 65 ] } } @@ -691,44 +475,12 @@ "yes" ], "answer_start": [ - 0, - 56, - 190, - 343, - 414, - 565, - 601, - 888, - 906, - 1098, - 1179, - 1265, - 1296, - 1382, - 1436, - 1481, - 1514, - 1546 + 0, 56, 190, 343, 414, 565, 601, 888, 906, 1098, 1179, 1265, 1296, + 1382, 1436, 1481, 1514, 1546 ], "answer_end": [ - 54, - 107, - 341, - 367, - 497, - 600, - 631, - 903, - 965, - 1177, - 1216, - 1291, - 1350, - 1434, - 1454, - 1509, - 1541, - 1576 + 54, 107, 341, 367, 497, 600, 631, 903, 965, 1177, 1216, 1291, 1350, + 1434, 1454, 1509, 1541, 1576 ] } } @@ -785,44 +537,12 @@ "yes" ], "answer_start": [ - 0, - 56, - 190, - 343, - 414, - 565, - 601, - 888, - 906, - 1098, - 1179, - 1265, - 1296, - 1382, - 1436, - 1481, - 1514, - 1546 + 0, 56, 190, 343, 414, 565, 601, 888, 906, 1098, 1179, 1265, 1296, + 1382, 1436, 1481, 1514, 1546 ], "answer_end": [ - 54, - 107, - 341, - 367, - 497, - 600, - 631, - 903, - 965, - 1177, - 1216, - 1291, - 1350, - 1434, - 1454, - 1509, - 1541, - 1576 + 54, 107, 341, 367, 497, 600, 631, 903, 965, 1177, 1216, 1291, 1350, + 1434, 1454, 1509, 1541, 1576 ] } } @@ -869,34 +589,10 @@ "St. George, Tompkinsville, Clifton, and Stapleton" ], "answer_start": [ - 28, - 45, - 80, - 17, - 88, - 322, - 448, - 477, - 600, - 808, - 831, - 894, - 939 + 28, 45, 80, 17, 88, 322, 448, 477, 600, 808, 831, 894, 939 ], "answer_end": [ - 32, - 58, - 88, - 21, - 119, - 356, - 455, - 493, - 620, - 829, - 890, - 906, - 988 + 32, 58, 88, 21, 119, 356, 455, 493, 620, 829, 890, 906, 988 ] } } @@ -943,34 +639,10 @@ "St. George, Tompkinsville, Clifton, and Stapleton" ], "answer_start": [ - 28, - 45, - 80, - 17, - 88, - 322, - 448, - 477, - 600, - 808, - 831, - 894, - 939 + 28, 45, 80, 17, 88, 322, 448, 477, 600, 808, 831, 894, 939 ], "answer_end": [ - 32, - 58, - 88, - 21, - 119, - 356, - 455, - 493, - 620, - 829, - 890, - 906, - 988 + 32, 58, 88, 21, 119, 356, 455, 493, 620, 829, 890, 906, 988 ] } } @@ -1015,32 +687,10 @@ "The flashlight" ], "answer_start": [ - 24, - 71, - 104, - 193, - 396, - 396, - 771, - 864, - 907, - 1641, - 1696, - 1274 + 24, 71, 104, 193, 396, 396, 771, 864, 907, 1641, 1696, 1274 ], "answer_end": [ - 69, - 102, - 130, - 217, - 461, - 461, - 829, - 906, - 1016, - 1677, - 1726, - 1339 + 69, 102, 130, 217, 461, 461, 829, 906, 1016, 1677, 1726, 1339 ] } } @@ -1085,32 +735,10 @@ "The flashlight" ], "answer_start": [ - 24, - 71, - 104, - 193, - 396, - 396, - 771, - 864, - 907, - 1641, - 1696, - 1274 + 24, 71, 104, 193, 396, 396, 771, 864, 907, 1641, 1696, 1274 ], "answer_end": [ - 69, - 102, - 130, - 217, - 461, - 461, - 829, - 906, - 1016, - 1677, - 1726, - 1339 + 69, 102, 130, 217, 461, 461, 829, 906, 1016, 1677, 1726, 1339 ] } } @@ -1171,48 +799,12 @@ "2, Giordano told authorities that he had been snorkeling with Gardner" ], "answer_start": [ - 211, - 184, - 494, - 48, - 260, - 70, - 345, - 435, - 678, - 772, - 839, - 1021, - 1040, - 1154, - 1111, - 1269, - 1407, - 1506, - 993, - 1111 + 211, 184, 494, 48, 260, 70, 345, 435, 678, 772, 839, 1021, 1040, 1154, + 1111, 1269, 1407, 1506, 993, 1111 ], "answer_end": [ - 224, - 197, - 511, - 57, - 271, - 126, - 348, - 437, - 713, - 778, - 863, - 1033, - 1064, - 1164, - 1119, - 1300, - 1452, - 1508, - 1002, - 1177 + 224, 197, 511, 57, 271, 126, 348, 437, 713, 778, 863, 1033, 1064, + 1164, 1119, 1300, 1452, 1508, 1002, 1177 ] } } @@ -1273,48 +865,12 @@ "2, Giordano told authorities that he had been snorkeling with Gardner" ], "answer_start": [ - 211, - 184, - 494, - 48, - 260, - 70, - 345, - 435, - 678, - 772, - 839, - 1021, - 1040, - 1154, - 1111, - 1269, - 1407, - 1506, - 993, - 1111 + 211, 184, 494, 48, 260, 70, 345, 435, 678, 772, 839, 1021, 1040, 1154, + 1111, 1269, 1407, 1506, 993, 1111 ], "answer_end": [ - 224, - 197, - 511, - 57, - 271, - 126, - 348, - 437, - 713, - 778, - 863, - 1033, - 1064, - 1164, - 1119, - 1300, - 1452, - 1508, - 1002, - 1177 + 224, 197, 511, 57, 271, 126, 348, 437, 713, 778, 863, 1033, 1064, + 1164, 1119, 1300, 1452, 1508, 1002, 1177 ] } } @@ -1354,30 +910,8 @@ "yes", "unknown" ], - "answer_start": [ - 159, - 34, - 196, - 224, - 731, - 794, - 1120, - 1190, - 1436, - -1 - ], - "answer_end": [ - 177, - 54, - 222, - 264, - 757, - 805, - 1158, - 1212, - 1448, - -1 - ] + "answer_start": [159, 34, 196, 224, 731, 794, 1120, 1190, 1436, -1], + "answer_end": [177, 54, 222, 264, 757, 805, 1158, 1212, 1448, -1] } } }, @@ -1416,30 +950,8 @@ "yes", "unknown" ], - "answer_start": [ - 159, - 34, - 196, - 224, - 731, - 794, - 1120, - 1190, - 1436, - -1 - ], - "answer_end": [ - 177, - 54, - 222, - 264, - 757, - 805, - 1158, - 1212, - 1448, - -1 - ] + "answer_start": [159, 34, 196, 224, 731, 794, 1120, 1190, 1436, -1], + "answer_end": [177, 54, 222, 264, 757, 805, 1158, 1212, 1448, -1] } } }, @@ -1478,30 +990,8 @@ "Pfc. Andrew Holmes", "Holmes is charged with the premeditated deaths of three civilians" ], - "answer_start": [ - 28, - 28, - 77, - 267, - 363, - 362, - 1229, - 482, - 545, - 662 - ], - "answer_end": [ - 62, - 62, - 181, - 361, - 430, - 481, - 1362, - 541, - 563, - 727 - ] + "answer_start": [28, 28, 77, 267, 363, 362, 1229, 482, 545, 662], + "answer_end": [62, 62, 181, 361, 430, 481, 1362, 541, 563, 727] } } }, @@ -1540,30 +1030,8 @@ "Pfc. Andrew Holmes", "Holmes is charged with the premeditated deaths of three civilians" ], - "answer_start": [ - 28, - 28, - 77, - 267, - 363, - 362, - 1229, - 482, - 545, - 662 - ], - "answer_end": [ - 62, - 62, - 181, - 361, - 430, - 481, - 1362, - 541, - 563, - 727 - ] + "answer_start": [28, 28, 77, 267, 363, 362, 1229, 482, 545, 662], + "answer_end": [62, 62, 181, 361, 430, 481, 1362, 541, 563, 727] } } }, @@ -1602,30 +1070,8 @@ "38", "just that it has bible references and shows him enjoying gos[e; singing with his wife" ], - "answer_start": [ - 330, - 851, - 851, - 945, - 1175, - 753, - 1472, - 504, - 851, - 6 - ], - "answer_end": [ - 383, - 1085, - 971, - 970, - 1253, - 791, - 1511, - 671, - 884, - 132 - ] + "answer_start": [330, 851, 851, 945, 1175, 753, 1472, 504, 851, 6], + "answer_end": [383, 1085, 971, 970, 1253, 791, 1511, 671, 884, 132] } } }, @@ -1664,30 +1110,8 @@ "38", "just that it has bible references and shows him enjoying gos[e; singing with his wife" ], - "answer_start": [ - 330, - 851, - 851, - 945, - 1175, - 753, - 1472, - 504, - 851, - 6 - ], - "answer_end": [ - 383, - 1085, - 971, - 970, - 1253, - 791, - 1511, - 671, - 884, - 132 - ] + "answer_start": [330, 851, 851, 945, 1175, 753, 1472, 504, 851, 6], + "answer_end": [383, 1085, 971, 970, 1253, 791, 1511, 671, 884, 132] } } }, @@ -1741,42 +1165,12 @@ "no" ], "answer_start": [ - 32, - 37, - 571, - 806, - 278, - 841, - 949, - 971, - 995, - 372, - 806, - 841, - 1618, - 1636, - 1579, - 1654, - 1671 + 32, 37, 571, 806, 278, 841, 949, 971, 995, 372, 806, 841, 1618, 1636, + 1579, 1654, 1671 ], "answer_end": [ - 36, - 67, - 575, - 819, - 282, - 862, - 969, - 1020, - 1020, - 380, - 820, - 863, - 1632, - 1651, - 1597, - 1670, - 1746 + 36, 67, 575, 819, 282, 862, 969, 1020, 1020, 380, 820, 863, 1632, + 1651, 1597, 1670, 1746 ] } } @@ -1831,42 +1225,12 @@ "no" ], "answer_start": [ - 32, - 37, - 571, - 806, - 278, - 841, - 949, - 971, - 995, - 372, - 806, - 841, - 1618, - 1636, - 1579, - 1654, - 1671 + 32, 37, 571, 806, 278, 841, 949, 971, 995, 372, 806, 841, 1618, 1636, + 1579, 1654, 1671 ], "answer_end": [ - 36, - 67, - 575, - 819, - 282, - 862, - 969, - 1020, - 1020, - 380, - 820, - 863, - 1632, - 1651, - 1597, - 1670, - 1746 + 36, 67, 575, 819, 282, 862, 969, 1020, 1020, 380, 820, 863, 1632, + 1651, 1597, 1670, 1746 ] } } @@ -1911,32 +1275,10 @@ "at least $500" ], "answer_start": [ - 127, - 759, - 523, - 534, - 534, - 213, - 408, - 456, - 644, - 297, - 48, - 442 + 127, 759, 523, 534, 534, 213, 408, 456, 644, 297, 48, 442 ], "answer_end": [ - 129, - 766, - 526, - 555, - 554, - 220, - 411, - 484, - 649, - 316, - 56, - 455 + 129, 766, 526, 555, 554, 220, 411, 484, 649, 316, 56, 455 ] } } @@ -1981,32 +1323,10 @@ "at least $500" ], "answer_start": [ - 127, - 759, - 523, - 534, - 534, - 213, - 408, - 456, - 644, - 297, - 48, - 442 + 127, 759, 523, 534, 534, 213, 408, 456, 644, 297, 48, 442 ], "answer_end": [ - 129, - 766, - 526, - 555, - 554, - 220, - 411, - 484, - 649, - 316, - 56, - 455 + 129, 766, 526, 555, 554, 220, 411, 484, 649, 316, 56, 455 ] } } @@ -2046,30 +1366,8 @@ "threw a ball into the water", "got very wet" ], - "answer_start": [ - 33, - 76, - 381, - 170, - 749, - 552, - 658, - 897, - 1001, - 1189 - ], - "answer_end": [ - 59, - 91, - 400, - 198, - 763, - 561, - 668, - 933, - 1028, - 1201 - ] + "answer_start": [33, 76, 381, 170, 749, 552, 658, 897, 1001, 1189], + "answer_end": [59, 91, 400, 198, 763, 561, 668, 933, 1028, 1201] } } }, @@ -2108,30 +1406,8 @@ "threw a ball into the water", "got very wet" ], - "answer_start": [ - 33, - 76, - 381, - 170, - 749, - 552, - 658, - 897, - 1001, - 1189 - ], - "answer_end": [ - 59, - 91, - 400, - 198, - 763, - 561, - 668, - 933, - 1028, - 1201 - ] + "answer_start": [33, 76, 381, 170, 749, 552, 658, 897, 1001, 1189], + "answer_end": [59, 91, 400, 198, 763, 561, 668, 933, 1028, 1201] } } }, @@ -2191,48 +1467,12 @@ "No" ], "answer_start": [ - 936, - 906, - 2, - 969, - 1001, - 1017, - 845, - 1129, - 1159, - 1225, - 100, - 181, - 689, - 671, - 2, - 732, - 373, - 215, - 329, - 546 + 936, 906, 2, 969, 1001, 1017, 845, 1129, 1159, 1225, 100, 181, 689, + 671, 2, 732, 373, 215, 329, 546 ], "answer_end": [ - 964, - 933, - 9, - 1016, - 1016, - 1024, - 870, - 1157, - 1222, - 1267, - 112, - 190, - 728, - 683, - 35, - 763, - 474, - 280, - 355, - 634 + 964, 933, 9, 1016, 1016, 1024, 870, 1157, 1222, 1267, 112, 190, 728, + 683, 35, 763, 474, 280, 355, 634 ] } } @@ -2293,48 +1533,12 @@ "No" ], "answer_start": [ - 936, - 906, - 2, - 969, - 1001, - 1017, - 845, - 1129, - 1159, - 1225, - 100, - 181, - 689, - 671, - 2, - 732, - 373, - 215, - 329, - 546 + 936, 906, 2, 969, 1001, 1017, 845, 1129, 1159, 1225, 100, 181, 689, + 671, 2, 732, 373, 215, 329, 546 ], "answer_end": [ - 964, - 933, - 9, - 1016, - 1016, - 1024, - 870, - 1157, - 1222, - 1267, - 112, - 190, - 728, - 683, - 35, - 763, - 474, - 280, - 355, - 634 + 964, 933, 9, 1016, 1016, 1024, 870, 1157, 1222, 1267, 112, 190, 728, + 683, 35, 763, 474, 280, 355, 634 ] } } @@ -2395,48 +1599,12 @@ "unknown" ], "answer_start": [ - 15, - 100, - 186, - 234, - 235, - 451, - 15, - 702, - 992, - 839, - 901, - 711, - 1, - 1039, - 1189, - 1149, - 1039, - 15, - 312, - -1 + 15, 100, 186, 234, 235, 451, 15, 702, 992, 839, 901, 711, 1, 1039, + 1189, 1149, 1039, 15, 312, -1 ], "answer_end": [ - 98, - 184, - 212, - 250, - 295, - 497, - 83, - 882, - 1037, - 900, - 988, - 1074, - 1583, - 1348, - 1197, - 1173, - 1348, - 229, - 450, - -1 + 98, 184, 212, 250, 295, 497, 83, 882, 1037, 900, 988, 1074, 1583, + 1348, 1197, 1173, 1348, 229, 450, -1 ] } } @@ -2497,48 +1665,12 @@ "unknown" ], "answer_start": [ - 15, - 100, - 186, - 234, - 235, - 451, - 15, - 702, - 992, - 839, - 901, - 711, - 1, - 1039, - 1189, - 1149, - 1039, - 15, - 312, - -1 + 15, 100, 186, 234, 235, 451, 15, 702, 992, 839, 901, 711, 1, 1039, + 1189, 1149, 1039, 15, 312, -1 ], "answer_end": [ - 98, - 184, - 212, - 250, - 295, - 497, - 83, - 882, - 1037, - 900, - 988, - 1074, - 1583, - 1348, - 1197, - 1173, - 1348, - 229, - 450, - -1 + 98, 184, 212, 250, 295, 497, 83, 882, 1037, 900, 988, 1074, 1583, + 1348, 1197, 1173, 1348, 229, 450, -1 ] } } @@ -2601,50 +1733,12 @@ "\"Follow me! It's urgent!\"" ], "answer_start": [ - 0, - 0, - 74, - 134, - 588, - 883, - 220, - 389, - 678, - 1211, - 1462, - 1548, - 975, - 659, - 1574, - 1636, - 220, - 292, - 293, - 497, - 758 + 0, 0, 74, 134, 588, 883, 220, 389, 678, 1211, 1462, 1548, 975, 659, + 1574, 1636, 220, 292, 293, 497, 758 ], "answer_end": [ - 37, - 75, - 101, - 219, - 609, - 973, - 292, - 495, - 876, - 1346, - 1572, - 1573, - 1045, - 708, - 1636, - 1774, - 292, - 336, - 377, - 575, - 797 + 37, 75, 101, 219, 609, 973, 292, 495, 876, 1346, 1572, 1573, 1045, + 708, 1636, 1774, 292, 336, 377, 575, 797 ] } } @@ -2707,50 +1801,12 @@ "\"Follow me! It's urgent!\"" ], "answer_start": [ - 0, - 0, - 74, - 134, - 588, - 883, - 220, - 389, - 678, - 1211, - 1462, - 1548, - 975, - 659, - 1574, - 1636, - 220, - 292, - 293, - 497, - 758 + 0, 0, 74, 134, 588, 883, 220, 389, 678, 1211, 1462, 1548, 975, 659, + 1574, 1636, 220, 292, 293, 497, 758 ], "answer_end": [ - 37, - 75, - 101, - 219, - 609, - 973, - 292, - 495, - 876, - 1346, - 1572, - 1573, - 1045, - 708, - 1636, - 1774, - 292, - 336, - 377, - 575, - 797 + 37, 75, 101, 219, 609, 973, 292, 495, 876, 1346, 1572, 1573, 1045, + 708, 1636, 1774, 292, 336, 377, 575, 797 ] } } @@ -2811,48 +1867,12 @@ "no" ], "answer_start": [ - 0, - 47, - 74, - 199, - 284, - 284, - 310, - 402, - 403, - 486, - 362, - 455, - 542, - 577, - 757, - 825, - 164, - 169, - 199, - 262 + 0, 47, 74, 199, 284, 284, 310, 402, 403, 486, 362, 455, 542, 577, 757, + 825, 164, 169, 199, 262 ], "answer_end": [ - 46, - 88, - 104, - 227, - 325, - 325, - 360, - 454, - 454, - 540, - 401, - 481, - 573, - 620, - 796, - 862, - 199, - 199, - 261, - 284 + 46, 88, 104, 227, 325, 325, 360, 454, 454, 540, 401, 481, 573, 620, + 796, 862, 199, 199, 261, 284 ] } } @@ -2913,48 +1933,12 @@ "no" ], "answer_start": [ - 0, - 47, - 74, - 199, - 284, - 284, - 310, - 402, - 403, - 486, - 362, - 455, - 542, - 577, - 757, - 825, - 164, - 169, - 199, - 262 + 0, 47, 74, 199, 284, 284, 310, 402, 403, 486, 362, 455, 542, 577, 757, + 825, 164, 169, 199, 262 ], "answer_end": [ - 46, - 88, - 104, - 227, - 325, - 325, - 360, - 454, - 454, - 540, - 401, - 481, - 573, - 620, - 796, - 862, - 199, - 199, - 261, - 284 + 46, 88, 104, 227, 325, 325, 360, 454, 454, 540, 401, 481, 573, 620, + 796, 862, 199, 199, 261, 284 ] } } @@ -3001,34 +1985,10 @@ "the 7-year-old" ], "answer_start": [ - 19, - 103, - 538, - 527, - 58, - 223, - 917, - 990, - 1065, - 1141, - 834, - 813, - 813 + 19, 103, 538, 527, 58, 223, 917, 990, 1065, 1141, 834, 813, 813 ], "answer_end": [ - 59, - 130, - 565, - 565, - 65, - 227, - 946, - 1061, - 1078, - 1183, - 850, - 828, - 829 + 59, 130, 565, 565, 65, 227, 946, 1061, 1078, 1183, 850, 828, 829 ] } } @@ -3075,34 +2035,10 @@ "the 7-year-old" ], "answer_start": [ - 19, - 103, - 538, - 527, - 58, - 223, - 917, - 990, - 1065, - 1141, - 834, - 813, - 813 + 19, 103, 538, 527, 58, 223, 917, 990, 1065, 1141, 834, 813, 813 ], "answer_end": [ - 59, - 130, - 565, - 565, - 65, - 227, - 946, - 1061, - 1078, - 1183, - 850, - 828, - 829 + 59, 130, 565, 565, 65, 227, 946, 1061, 1078, 1183, 850, 828, 829 ] } } @@ -3149,34 +2085,10 @@ "no" ], "answer_start": [ - 1032, - 1168, - 1197, - -1, - 991, - 1385, - 1361, - 1405, - 144, - 1511, - 434, - 544, - 714 + 1032, 1168, 1197, -1, 991, 1385, 1361, 1405, 144, 1511, 434, 544, 714 ], "answer_end": [ - 1110, - 1195, - 1221, - -1, - 1044, - 1475, - 1476, - 1437, - 180, - 1590, - 505, - 583, - 753 + 1110, 1195, 1221, -1, 1044, 1475, 1476, 1437, 180, 1590, 505, 583, 753 ] } } @@ -3223,34 +2135,10 @@ "no" ], "answer_start": [ - 1032, - 1168, - 1197, - -1, - 991, - 1385, - 1361, - 1405, - 144, - 1511, - 434, - 544, - 714 + 1032, 1168, 1197, -1, 991, 1385, 1361, 1405, 144, 1511, 434, 544, 714 ], "answer_end": [ - 1110, - 1195, - 1221, - -1, - 1044, - 1475, - 1476, - 1437, - 180, - 1590, - 505, - 583, - 753 + 1110, 1195, 1221, -1, 1044, 1475, 1476, 1437, 180, 1590, 505, 583, 753 ] } } diff --git a/evals/datasets/coqa-context-relevancy.json b/evals/datasets/coqa-context-relevancy.json index ac72863..116cd25 100644 --- a/evals/datasets/coqa-context-relevancy.json +++ b/evals/datasets/coqa-context-relevancy.json @@ -38,32 +38,10 @@ "no" ], "answer_start": [ - 59, - 18, - 196, - 281, - 428, - 512, - 678, - 647, - 718, - 1035, - 1143, - 965 + 59, 18, 196, 281, 428, 512, 678, 647, 718, 1035, 1143, 965 ], "answer_end": [ - 93, - 80, - 215, - 315, - 490, - 549, - 716, - 676, - 776, - 1097, - 1170, - 1008 + 93, 80, 215, 315, 490, 549, 716, 676, 776, 1097, 1170, 1008 ] } } @@ -107,32 +85,10 @@ "no" ], "answer_start": [ - 59, - 18, - 196, - 281, - 428, - 512, - 678, - 647, - 718, - 1035, - 1143, - 965 + 59, 18, 196, 281, 428, 512, 678, 647, 718, 1035, 1143, 965 ], "answer_end": [ - 93, - 80, - 215, - 315, - 490, - 549, - 716, - 676, - 776, - 1097, - 1170, - 1008 + 93, 80, 215, 315, 490, 549, 716, 676, 776, 1097, 1170, 1008 ] } } @@ -176,32 +132,10 @@ "no" ], "answer_start": [ - 59, - 18, - 196, - 281, - 428, - 512, - 678, - 647, - 718, - 1035, - 1143, - 965 + 59, 18, 196, 281, 428, 512, 678, 647, 718, 1035, 1143, 965 ], "answer_end": [ - 93, - 80, - 215, - 315, - 490, - 549, - 716, - 676, - 776, - 1097, - 1170, - 1008 + 93, 80, 215, 315, 490, 549, 716, 676, 776, 1097, 1170, 1008 ] } } @@ -242,32 +176,8 @@ "yes", "unknown" ], - "answer_start": [ - 37, - 167, - 303, - 281, - 552, - 708, - 964, - 910, - -1, - 1056, - -1 - ], - "answer_end": [ - 43, - 175, - 346, - 302, - 577, - 723, - 995, - 935, - -1, - 1173, - -1 - ] + "answer_start": [37, 167, 303, 281, 552, 708, 964, 910, -1, 1056, -1], + "answer_end": [43, 175, 346, 302, 577, 723, 995, 935, -1, 1173, -1] } } }, @@ -307,32 +217,8 @@ "yes", "unknown" ], - "answer_start": [ - 37, - 167, - 303, - 281, - 552, - 708, - 964, - 910, - -1, - 1056, - -1 - ], - "answer_end": [ - 43, - 175, - 346, - 302, - 577, - 723, - 995, - 935, - -1, - 1173, - -1 - ] + "answer_start": [37, 167, 303, 281, 552, 708, 964, 910, -1, 1056, -1], + "answer_end": [43, 175, 346, 302, 577, 723, 995, 935, -1, 1173, -1] } } }, @@ -372,32 +258,8 @@ "yes", "unknown" ], - "answer_start": [ - 37, - 167, - 303, - 281, - 552, - 708, - 964, - 910, - -1, - 1056, - -1 - ], - "answer_end": [ - 43, - 175, - 346, - 302, - 577, - 723, - 995, - 935, - -1, - 1173, - -1 - ] + "answer_start": [37, 167, 303, 281, 552, 708, 964, 910, -1, 1056, -1], + "answer_end": [43, 175, 346, 302, 577, 723, 995, 935, -1, 1173, -1] } } }, @@ -446,38 +308,12 @@ "\"Thank you\"" ], "answer_start": [ - 19, - 136, - 146, - 169, - 304, - 348, - 214, - 579, - 727, - 423, - 917, - 594, - 1771, - 1667, + 19, 136, 146, 169, 304, 348, 214, 579, 727, 423, 917, 594, 1771, 1667, 1874 ], "answer_end": [ - 115, - 165, - 165, - 185, - 324, - 375, - 302, - 725, - 769, - 544, - 924, - 725, - 1814, - 1769, - 1885 + 115, 165, 165, 185, 324, 375, 302, 725, 769, 544, 924, 725, 1814, + 1769, 1885 ] } } @@ -527,38 +363,12 @@ "\"Thank you\"" ], "answer_start": [ - 19, - 136, - 146, - 169, - 304, - 348, - 214, - 579, - 727, - 423, - 917, - 594, - 1771, - 1667, + 19, 136, 146, 169, 304, 348, 214, 579, 727, 423, 917, 594, 1771, 1667, 1874 ], "answer_end": [ - 115, - 165, - 165, - 185, - 324, - 375, - 302, - 725, - 769, - 544, - 924, - 725, - 1814, - 1769, - 1885 + 115, 165, 165, 185, 324, 375, 302, 725, 769, 544, 924, 725, 1814, + 1769, 1885 ] } } @@ -608,38 +418,12 @@ "\"Thank you\"" ], "answer_start": [ - 19, - 136, - 146, - 169, - 304, - 348, - 214, - 579, - 727, - 423, - 917, - 594, - 1771, - 1667, + 19, 136, 146, 169, 304, 348, 214, 579, 727, 423, 917, 594, 1771, 1667, 1874 ], "answer_end": [ - 115, - 165, - 165, - 185, - 324, - 375, - 302, - 725, - 769, - 544, - 924, - 725, - 1814, - 1769, - 1885 + 115, 165, 165, 185, 324, 375, 302, 725, 769, 544, 924, 725, 1814, + 1769, 1885 ] } } @@ -699,48 +483,12 @@ "A cop" ], "answer_start": [ - 8, - 8, - 8, - 9, - 512, - 856, - 573, - 512, - 512, - 856, - 1041, - 1041, - 1041, - 1217, - 1216, - 1217, - 1217, - 1338, - 8, - 8 + 8, 8, 8, 9, 512, 856, 573, 512, 512, 856, 1041, 1041, 1041, 1217, + 1216, 1217, 1217, 1338, 8, 8 ], "answer_end": [ - 181, - 183, - 66, - 181, - 690, - 1039, - 690, - 662, - 690, - 1039, - 1103, - 1134, - 1165, - 1336, - 1336, - 1336, - 1336, - 1381, - 138, - 65 + 181, 183, 66, 181, 690, 1039, 690, 662, 690, 1039, 1103, 1134, 1165, + 1336, 1336, 1336, 1336, 1381, 138, 65 ] } } @@ -800,48 +548,12 @@ "A cop" ], "answer_start": [ - 8, - 8, - 8, - 9, - 512, - 856, - 573, - 512, - 512, - 856, - 1041, - 1041, - 1041, - 1217, - 1216, - 1217, - 1217, - 1338, - 8, - 8 + 8, 8, 8, 9, 512, 856, 573, 512, 512, 856, 1041, 1041, 1041, 1217, + 1216, 1217, 1217, 1338, 8, 8 ], "answer_end": [ - 181, - 183, - 66, - 181, - 690, - 1039, - 690, - 662, - 690, - 1039, - 1103, - 1134, - 1165, - 1336, - 1336, - 1336, - 1336, - 1381, - 138, - 65 + 181, 183, 66, 181, 690, 1039, 690, 662, 690, 1039, 1103, 1134, 1165, + 1336, 1336, 1336, 1336, 1381, 138, 65 ] } } @@ -901,48 +613,12 @@ "A cop" ], "answer_start": [ - 8, - 8, - 8, - 9, - 512, - 856, - 573, - 512, - 512, - 856, - 1041, - 1041, - 1041, - 1217, - 1216, - 1217, - 1217, - 1338, - 8, - 8 + 8, 8, 8, 9, 512, 856, 573, 512, 512, 856, 1041, 1041, 1041, 1217, + 1216, 1217, 1217, 1338, 8, 8 ], "answer_end": [ - 181, - 183, - 66, - 181, - 690, - 1039, - 690, - 662, - 690, - 1039, - 1103, - 1134, - 1165, - 1336, - 1336, - 1336, - 1336, - 1381, - 138, - 65 + 181, 183, 66, 181, 690, 1039, 690, 662, 690, 1039, 1103, 1134, 1165, + 1336, 1336, 1336, 1336, 1381, 138, 65 ] } } @@ -998,44 +674,12 @@ "yes" ], "answer_start": [ - 0, - 56, - 190, - 343, - 414, - 565, - 601, - 888, - 906, - 1098, - 1179, - 1265, - 1296, - 1382, - 1436, - 1481, - 1514, - 1546 + 0, 56, 190, 343, 414, 565, 601, 888, 906, 1098, 1179, 1265, 1296, + 1382, 1436, 1481, 1514, 1546 ], "answer_end": [ - 54, - 107, - 341, - 367, - 497, - 600, - 631, - 903, - 965, - 1177, - 1216, - 1291, - 1350, - 1434, - 1454, - 1509, - 1541, - 1576 + 54, 107, 341, 367, 497, 600, 631, 903, 965, 1177, 1216, 1291, 1350, + 1434, 1454, 1509, 1541, 1576 ] } } @@ -1091,44 +735,12 @@ "yes" ], "answer_start": [ - 0, - 56, - 190, - 343, - 414, - 565, - 601, - 888, - 906, - 1098, - 1179, - 1265, - 1296, - 1382, - 1436, - 1481, - 1514, - 1546 + 0, 56, 190, 343, 414, 565, 601, 888, 906, 1098, 1179, 1265, 1296, + 1382, 1436, 1481, 1514, 1546 ], "answer_end": [ - 54, - 107, - 341, - 367, - 497, - 600, - 631, - 903, - 965, - 1177, - 1216, - 1291, - 1350, - 1434, - 1454, - 1509, - 1541, - 1576 + 54, 107, 341, 367, 497, 600, 631, 903, 965, 1177, 1216, 1291, 1350, + 1434, 1454, 1509, 1541, 1576 ] } } @@ -1184,44 +796,12 @@ "yes" ], "answer_start": [ - 0, - 56, - 190, - 343, - 414, - 565, - 601, - 888, - 906, - 1098, - 1179, - 1265, - 1296, - 1382, - 1436, - 1481, - 1514, - 1546 + 0, 56, 190, 343, 414, 565, 601, 888, 906, 1098, 1179, 1265, 1296, + 1382, 1436, 1481, 1514, 1546 ], "answer_end": [ - 54, - 107, - 341, - 367, - 497, - 600, - 631, - 903, - 965, - 1177, - 1216, - 1291, - 1350, - 1434, - 1454, - 1509, - 1541, - 1576 + 54, 107, 341, 367, 497, 600, 631, 903, 965, 1177, 1216, 1291, 1350, + 1434, 1454, 1509, 1541, 1576 ] } } @@ -1267,34 +847,10 @@ "St. George, Tompkinsville, Clifton, and Stapleton" ], "answer_start": [ - 28, - 45, - 80, - 17, - 88, - 322, - 448, - 477, - 600, - 808, - 831, - 894, - 939 + 28, 45, 80, 17, 88, 322, 448, 477, 600, 808, 831, 894, 939 ], "answer_end": [ - 32, - 58, - 88, - 21, - 119, - 356, - 455, - 493, - 620, - 829, - 890, - 906, - 988 + 32, 58, 88, 21, 119, 356, 455, 493, 620, 829, 890, 906, 988 ] } } @@ -1340,34 +896,10 @@ "St. George, Tompkinsville, Clifton, and Stapleton" ], "answer_start": [ - 28, - 45, - 80, - 17, - 88, - 322, - 448, - 477, - 600, - 808, - 831, - 894, - 939 + 28, 45, 80, 17, 88, 322, 448, 477, 600, 808, 831, 894, 939 ], "answer_end": [ - 32, - 58, - 88, - 21, - 119, - 356, - 455, - 493, - 620, - 829, - 890, - 906, - 988 + 32, 58, 88, 21, 119, 356, 455, 493, 620, 829, 890, 906, 988 ] } } @@ -1413,34 +945,10 @@ "St. George, Tompkinsville, Clifton, and Stapleton" ], "answer_start": [ - 28, - 45, - 80, - 17, - 88, - 322, - 448, - 477, - 600, - 808, - 831, - 894, - 939 + 28, 45, 80, 17, 88, 322, 448, 477, 600, 808, 831, 894, 939 ], "answer_end": [ - 32, - 58, - 88, - 21, - 119, - 356, - 455, - 493, - 620, - 829, - 890, - 906, - 988 + 32, 58, 88, 21, 119, 356, 455, 493, 620, 829, 890, 906, 988 ] } } @@ -1484,32 +992,10 @@ "The flashlight" ], "answer_start": [ - 24, - 71, - 104, - 193, - 396, - 396, - 771, - 864, - 907, - 1641, - 1696, - 1274 + 24, 71, 104, 193, 396, 396, 771, 864, 907, 1641, 1696, 1274 ], "answer_end": [ - 69, - 102, - 130, - 217, - 461, - 461, - 829, - 906, - 1016, - 1677, - 1726, - 1339 + 69, 102, 130, 217, 461, 461, 829, 906, 1016, 1677, 1726, 1339 ] } } @@ -1553,32 +1039,10 @@ "The flashlight" ], "answer_start": [ - 24, - 71, - 104, - 193, - 396, - 396, - 771, - 864, - 907, - 1641, - 1696, - 1274 + 24, 71, 104, 193, 396, 396, 771, 864, 907, 1641, 1696, 1274 ], "answer_end": [ - 69, - 102, - 130, - 217, - 461, - 461, - 829, - 906, - 1016, - 1677, - 1726, - 1339 + 69, 102, 130, 217, 461, 461, 829, 906, 1016, 1677, 1726, 1339 ] } } @@ -1622,32 +1086,10 @@ "The flashlight" ], "answer_start": [ - 24, - 71, - 104, - 193, - 396, - 396, - 771, - 864, - 907, - 1641, - 1696, - 1274 + 24, 71, 104, 193, 396, 396, 771, 864, 907, 1641, 1696, 1274 ], "answer_end": [ - 69, - 102, - 130, - 217, - 461, - 461, - 829, - 906, - 1016, - 1677, - 1726, - 1339 + 69, 102, 130, 217, 461, 461, 829, 906, 1016, 1677, 1726, 1339 ] } } @@ -1707,48 +1149,12 @@ "2, Giordano told authorities that he had been snorkeling with Gardner" ], "answer_start": [ - 211, - 184, - 494, - 48, - 260, - 70, - 345, - 435, - 678, - 772, - 839, - 1021, - 1040, - 1154, - 1111, - 1269, - 1407, - 1506, - 993, - 1111 + 211, 184, 494, 48, 260, 70, 345, 435, 678, 772, 839, 1021, 1040, 1154, + 1111, 1269, 1407, 1506, 993, 1111 ], "answer_end": [ - 224, - 197, - 511, - 57, - 271, - 126, - 348, - 437, - 713, - 778, - 863, - 1033, - 1064, - 1164, - 1119, - 1300, - 1452, - 1508, - 1002, - 1177 + 224, 197, 511, 57, 271, 126, 348, 437, 713, 778, 863, 1033, 1064, + 1164, 1119, 1300, 1452, 1508, 1002, 1177 ] } } @@ -1808,48 +1214,12 @@ "2, Giordano told authorities that he had been snorkeling with Gardner" ], "answer_start": [ - 211, - 184, - 494, - 48, - 260, - 70, - 345, - 435, - 678, - 772, - 839, - 1021, - 1040, - 1154, - 1111, - 1269, - 1407, - 1506, - 993, - 1111 + 211, 184, 494, 48, 260, 70, 345, 435, 678, 772, 839, 1021, 1040, 1154, + 1111, 1269, 1407, 1506, 993, 1111 ], "answer_end": [ - 224, - 197, - 511, - 57, - 271, - 126, - 348, - 437, - 713, - 778, - 863, - 1033, - 1064, - 1164, - 1119, - 1300, - 1452, - 1508, - 1002, - 1177 + 224, 197, 511, 57, 271, 126, 348, 437, 713, 778, 863, 1033, 1064, + 1164, 1119, 1300, 1452, 1508, 1002, 1177 ] } } @@ -1909,48 +1279,12 @@ "2, Giordano told authorities that he had been snorkeling with Gardner" ], "answer_start": [ - 211, - 184, - 494, - 48, - 260, - 70, - 345, - 435, - 678, - 772, - 839, - 1021, - 1040, - 1154, - 1111, - 1269, - 1407, - 1506, - 993, - 1111 + 211, 184, 494, 48, 260, 70, 345, 435, 678, 772, 839, 1021, 1040, 1154, + 1111, 1269, 1407, 1506, 993, 1111 ], "answer_end": [ - 224, - 197, - 511, - 57, - 271, - 126, - 348, - 437, - 713, - 778, - 863, - 1033, - 1064, - 1164, - 1119, - 1300, - 1452, - 1508, - 1002, - 1177 + 224, 197, 511, 57, 271, 126, 348, 437, 713, 778, 863, 1033, 1064, + 1164, 1119, 1300, 1452, 1508, 1002, 1177 ] } } @@ -1989,30 +1323,8 @@ "yes", "unknown" ], - "answer_start": [ - 159, - 34, - 196, - 224, - 731, - 794, - 1120, - 1190, - 1436, - -1 - ], - "answer_end": [ - 177, - 54, - 222, - 264, - 757, - 805, - 1158, - 1212, - 1448, - -1 - ] + "answer_start": [159, 34, 196, 224, 731, 794, 1120, 1190, 1436, -1], + "answer_end": [177, 54, 222, 264, 757, 805, 1158, 1212, 1448, -1] } } }, @@ -2050,30 +1362,8 @@ "yes", "unknown" ], - "answer_start": [ - 159, - 34, - 196, - 224, - 731, - 794, - 1120, - 1190, - 1436, - -1 - ], - "answer_end": [ - 177, - 54, - 222, - 264, - 757, - 805, - 1158, - 1212, - 1448, - -1 - ] + "answer_start": [159, 34, 196, 224, 731, 794, 1120, 1190, 1436, -1], + "answer_end": [177, 54, 222, 264, 757, 805, 1158, 1212, 1448, -1] } } }, @@ -2111,30 +1401,8 @@ "yes", "unknown" ], - "answer_start": [ - 159, - 34, - 196, - 224, - 731, - 794, - 1120, - 1190, - 1436, - -1 - ], - "answer_end": [ - 177, - 54, - 222, - 264, - 757, - 805, - 1158, - 1212, - 1448, - -1 - ] + "answer_start": [159, 34, 196, 224, 731, 794, 1120, 1190, 1436, -1], + "answer_end": [177, 54, 222, 264, 757, 805, 1158, 1212, 1448, -1] } } }, @@ -2172,30 +1440,8 @@ "Pfc. Andrew Holmes", "Holmes is charged with the premeditated deaths of three civilians" ], - "answer_start": [ - 28, - 28, - 77, - 267, - 363, - 362, - 1229, - 482, - 545, - 662 - ], - "answer_end": [ - 62, - 62, - 181, - 361, - 430, - 481, - 1362, - 541, - 563, - 727 - ] + "answer_start": [28, 28, 77, 267, 363, 362, 1229, 482, 545, 662], + "answer_end": [62, 62, 181, 361, 430, 481, 1362, 541, 563, 727] } } }, @@ -2233,30 +1479,8 @@ "Pfc. Andrew Holmes", "Holmes is charged with the premeditated deaths of three civilians" ], - "answer_start": [ - 28, - 28, - 77, - 267, - 363, - 362, - 1229, - 482, - 545, - 662 - ], - "answer_end": [ - 62, - 62, - 181, - 361, - 430, - 481, - 1362, - 541, - 563, - 727 - ] + "answer_start": [28, 28, 77, 267, 363, 362, 1229, 482, 545, 662], + "answer_end": [62, 62, 181, 361, 430, 481, 1362, 541, 563, 727] } } }, @@ -2294,30 +1518,8 @@ "Pfc. Andrew Holmes", "Holmes is charged with the premeditated deaths of three civilians" ], - "answer_start": [ - 28, - 28, - 77, - 267, - 363, - 362, - 1229, - 482, - 545, - 662 - ], - "answer_end": [ - 62, - 62, - 181, - 361, - 430, - 481, - 1362, - 541, - 563, - 727 - ] + "answer_start": [28, 28, 77, 267, 363, 362, 1229, 482, 545, 662], + "answer_end": [62, 62, 181, 361, 430, 481, 1362, 541, 563, 727] } } }, @@ -2355,30 +1557,8 @@ "38", "just that it has bible references and shows him enjoying gos[e; singing with his wife" ], - "answer_start": [ - 330, - 851, - 851, - 945, - 1175, - 753, - 1472, - 504, - 851, - 6 - ], - "answer_end": [ - 383, - 1085, - 971, - 970, - 1253, - 791, - 1511, - 671, - 884, - 132 - ] + "answer_start": [330, 851, 851, 945, 1175, 753, 1472, 504, 851, 6], + "answer_end": [383, 1085, 971, 970, 1253, 791, 1511, 671, 884, 132] } } }, @@ -2416,30 +1596,8 @@ "38", "just that it has bible references and shows him enjoying gos[e; singing with his wife" ], - "answer_start": [ - 330, - 851, - 851, - 945, - 1175, - 753, - 1472, - 504, - 851, - 6 - ], - "answer_end": [ - 383, - 1085, - 971, - 970, - 1253, - 791, - 1511, - 671, - 884, - 132 - ] + "answer_start": [330, 851, 851, 945, 1175, 753, 1472, 504, 851, 6], + "answer_end": [383, 1085, 971, 970, 1253, 791, 1511, 671, 884, 132] } } }, @@ -2477,30 +1635,8 @@ "38", "just that it has bible references and shows him enjoying gos[e; singing with his wife" ], - "answer_start": [ - 330, - 851, - 851, - 945, - 1175, - 753, - 1472, - 504, - 851, - 6 - ], - "answer_end": [ - 383, - 1085, - 971, - 970, - 1253, - 791, - 1511, - 671, - 884, - 132 - ] + "answer_start": [330, 851, 851, 945, 1175, 753, 1472, 504, 851, 6], + "answer_end": [383, 1085, 971, 970, 1253, 791, 1511, 671, 884, 132] } } }, @@ -2553,42 +1689,12 @@ "no" ], "answer_start": [ - 32, - 37, - 571, - 806, - 278, - 841, - 949, - 971, - 995, - 372, - 806, - 841, - 1618, - 1636, - 1579, - 1654, - 1671 + 32, 37, 571, 806, 278, 841, 949, 971, 995, 372, 806, 841, 1618, 1636, + 1579, 1654, 1671 ], "answer_end": [ - 36, - 67, - 575, - 819, - 282, - 862, - 969, - 1020, - 1020, - 380, - 820, - 863, - 1632, - 1651, - 1597, - 1670, - 1746 + 36, 67, 575, 819, 282, 862, 969, 1020, 1020, 380, 820, 863, 1632, + 1651, 1597, 1670, 1746 ] } } @@ -2642,42 +1748,12 @@ "no" ], "answer_start": [ - 32, - 37, - 571, - 806, - 278, - 841, - 949, - 971, - 995, - 372, - 806, - 841, - 1618, - 1636, - 1579, - 1654, - 1671 + 32, 37, 571, 806, 278, 841, 949, 971, 995, 372, 806, 841, 1618, 1636, + 1579, 1654, 1671 ], "answer_end": [ - 36, - 67, - 575, - 819, - 282, - 862, - 969, - 1020, - 1020, - 380, - 820, - 863, - 1632, - 1651, - 1597, - 1670, - 1746 + 36, 67, 575, 819, 282, 862, 969, 1020, 1020, 380, 820, 863, 1632, + 1651, 1597, 1670, 1746 ] } } @@ -2731,42 +1807,12 @@ "no" ], "answer_start": [ - 32, - 37, - 571, - 806, - 278, - 841, - 949, - 971, - 995, - 372, - 806, - 841, - 1618, - 1636, - 1579, - 1654, - 1671 + 32, 37, 571, 806, 278, 841, 949, 971, 995, 372, 806, 841, 1618, 1636, + 1579, 1654, 1671 ], "answer_end": [ - 36, - 67, - 575, - 819, - 282, - 862, - 969, - 1020, - 1020, - 380, - 820, - 863, - 1632, - 1651, - 1597, - 1670, - 1746 + 36, 67, 575, 819, 282, 862, 969, 1020, 1020, 380, 820, 863, 1632, + 1651, 1597, 1670, 1746 ] } } @@ -2810,32 +1856,10 @@ "at least $500" ], "answer_start": [ - 127, - 759, - 523, - 534, - 534, - 213, - 408, - 456, - 644, - 297, - 48, - 442 + 127, 759, 523, 534, 534, 213, 408, 456, 644, 297, 48, 442 ], "answer_end": [ - 129, - 766, - 526, - 555, - 554, - 220, - 411, - 484, - 649, - 316, - 56, - 455 + 129, 766, 526, 555, 554, 220, 411, 484, 649, 316, 56, 455 ] } } @@ -2879,32 +1903,10 @@ "at least $500" ], "answer_start": [ - 127, - 759, - 523, - 534, - 534, - 213, - 408, - 456, - 644, - 297, - 48, - 442 + 127, 759, 523, 534, 534, 213, 408, 456, 644, 297, 48, 442 ], "answer_end": [ - 129, - 766, - 526, - 555, - 554, - 220, - 411, - 484, - 649, - 316, - 56, - 455 + 129, 766, 526, 555, 554, 220, 411, 484, 649, 316, 56, 455 ] } } @@ -2948,32 +1950,10 @@ "at least $500" ], "answer_start": [ - 127, - 759, - 523, - 534, - 534, - 213, - 408, - 456, - 644, - 297, - 48, - 442 + 127, 759, 523, 534, 534, 213, 408, 456, 644, 297, 48, 442 ], "answer_end": [ - 129, - 766, - 526, - 555, - 554, - 220, - 411, - 484, - 649, - 316, - 56, - 455 + 129, 766, 526, 555, 554, 220, 411, 484, 649, 316, 56, 455 ] } } @@ -3012,30 +1992,8 @@ "threw a ball into the water", "got very wet" ], - "answer_start": [ - 33, - 76, - 381, - 170, - 749, - 552, - 658, - 897, - 1001, - 1189 - ], - "answer_end": [ - 59, - 91, - 400, - 198, - 763, - 561, - 668, - 933, - 1028, - 1201 - ] + "answer_start": [33, 76, 381, 170, 749, 552, 658, 897, 1001, 1189], + "answer_end": [59, 91, 400, 198, 763, 561, 668, 933, 1028, 1201] } } }, @@ -3073,30 +2031,8 @@ "threw a ball into the water", "got very wet" ], - "answer_start": [ - 33, - 76, - 381, - 170, - 749, - 552, - 658, - 897, - 1001, - 1189 - ], - "answer_end": [ - 59, - 91, - 400, - 198, - 763, - 561, - 668, - 933, - 1028, - 1201 - ] + "answer_start": [33, 76, 381, 170, 749, 552, 658, 897, 1001, 1189], + "answer_end": [59, 91, 400, 198, 763, 561, 668, 933, 1028, 1201] } } }, @@ -3134,30 +2070,8 @@ "threw a ball into the water", "got very wet" ], - "answer_start": [ - 33, - 76, - 381, - 170, - 749, - 552, - 658, - 897, - 1001, - 1189 - ], - "answer_end": [ - 59, - 91, - 400, - 198, - 763, - 561, - 668, - 933, - 1028, - 1201 - ] + "answer_start": [33, 76, 381, 170, 749, 552, 658, 897, 1001, 1189], + "answer_end": [59, 91, 400, 198, 763, 561, 668, 933, 1028, 1201] } } }, @@ -3216,48 +2130,12 @@ "No" ], "answer_start": [ - 936, - 906, - 2, - 969, - 1001, - 1017, - 845, - 1129, - 1159, - 1225, - 100, - 181, - 689, - 671, - 2, - 732, - 373, - 215, - 329, - 546 + 936, 906, 2, 969, 1001, 1017, 845, 1129, 1159, 1225, 100, 181, 689, + 671, 2, 732, 373, 215, 329, 546 ], "answer_end": [ - 964, - 933, - 9, - 1016, - 1016, - 1024, - 870, - 1157, - 1222, - 1267, - 112, - 190, - 728, - 683, - 35, - 763, - 474, - 280, - 355, - 634 + 964, 933, 9, 1016, 1016, 1024, 870, 1157, 1222, 1267, 112, 190, 728, + 683, 35, 763, 474, 280, 355, 634 ] } } @@ -3317,48 +2195,12 @@ "No" ], "answer_start": [ - 936, - 906, - 2, - 969, - 1001, - 1017, - 845, - 1129, - 1159, - 1225, - 100, - 181, - 689, - 671, - 2, - 732, - 373, - 215, - 329, - 546 + 936, 906, 2, 969, 1001, 1017, 845, 1129, 1159, 1225, 100, 181, 689, + 671, 2, 732, 373, 215, 329, 546 ], "answer_end": [ - 964, - 933, - 9, - 1016, - 1016, - 1024, - 870, - 1157, - 1222, - 1267, - 112, - 190, - 728, - 683, - 35, - 763, - 474, - 280, - 355, - 634 + 964, 933, 9, 1016, 1016, 1024, 870, 1157, 1222, 1267, 112, 190, 728, + 683, 35, 763, 474, 280, 355, 634 ] } } @@ -3418,48 +2260,12 @@ "No" ], "answer_start": [ - 936, - 906, - 2, - 969, - 1001, - 1017, - 845, - 1129, - 1159, - 1225, - 100, - 181, - 689, - 671, - 2, - 732, - 373, - 215, - 329, - 546 + 936, 906, 2, 969, 1001, 1017, 845, 1129, 1159, 1225, 100, 181, 689, + 671, 2, 732, 373, 215, 329, 546 ], "answer_end": [ - 964, - 933, - 9, - 1016, - 1016, - 1024, - 870, - 1157, - 1222, - 1267, - 112, - 190, - 728, - 683, - 35, - 763, - 474, - 280, - 355, - 634 + 964, 933, 9, 1016, 1016, 1024, 870, 1157, 1222, 1267, 112, 190, 728, + 683, 35, 763, 474, 280, 355, 634 ] } } @@ -3519,48 +2325,12 @@ "unknown" ], "answer_start": [ - 15, - 100, - 186, - 234, - 235, - 451, - 15, - 702, - 992, - 839, - 901, - 711, - 1, - 1039, - 1189, - 1149, - 1039, - 15, - 312, - -1 + 15, 100, 186, 234, 235, 451, 15, 702, 992, 839, 901, 711, 1, 1039, + 1189, 1149, 1039, 15, 312, -1 ], "answer_end": [ - 98, - 184, - 212, - 250, - 295, - 497, - 83, - 882, - 1037, - 900, - 988, - 1074, - 1583, - 1348, - 1197, - 1173, - 1348, - 229, - 450, - -1 + 98, 184, 212, 250, 295, 497, 83, 882, 1037, 900, 988, 1074, 1583, + 1348, 1197, 1173, 1348, 229, 450, -1 ] } } @@ -3620,48 +2390,12 @@ "unknown" ], "answer_start": [ - 15, - 100, - 186, - 234, - 235, - 451, - 15, - 702, - 992, - 839, - 901, - 711, - 1, - 1039, - 1189, - 1149, - 1039, - 15, - 312, - -1 + 15, 100, 186, 234, 235, 451, 15, 702, 992, 839, 901, 711, 1, 1039, + 1189, 1149, 1039, 15, 312, -1 ], "answer_end": [ - 98, - 184, - 212, - 250, - 295, - 497, - 83, - 882, - 1037, - 900, - 988, - 1074, - 1583, - 1348, - 1197, - 1173, - 1348, - 229, - 450, - -1 + 98, 184, 212, 250, 295, 497, 83, 882, 1037, 900, 988, 1074, 1583, + 1348, 1197, 1173, 1348, 229, 450, -1 ] } } @@ -3721,48 +2455,12 @@ "unknown" ], "answer_start": [ - 15, - 100, - 186, - 234, - 235, - 451, - 15, - 702, - 992, - 839, - 901, - 711, - 1, - 1039, - 1189, - 1149, - 1039, - 15, - 312, - -1 + 15, 100, 186, 234, 235, 451, 15, 702, 992, 839, 901, 711, 1, 1039, + 1189, 1149, 1039, 15, 312, -1 ], "answer_end": [ - 98, - 184, - 212, - 250, - 295, - 497, - 83, - 882, - 1037, - 900, - 988, - 1074, - 1583, - 1348, - 1197, - 1173, - 1348, - 229, - 450, - -1 + 98, 184, 212, 250, 295, 497, 83, 882, 1037, 900, 988, 1074, 1583, + 1348, 1197, 1173, 1348, 229, 450, -1 ] } } @@ -3824,50 +2522,12 @@ "\"Follow me! It's urgent!\"" ], "answer_start": [ - 0, - 0, - 74, - 134, - 588, - 883, - 220, - 389, - 678, - 1211, - 1462, - 1548, - 975, - 659, - 1574, - 1636, - 220, - 292, - 293, - 497, - 758 + 0, 0, 74, 134, 588, 883, 220, 389, 678, 1211, 1462, 1548, 975, 659, + 1574, 1636, 220, 292, 293, 497, 758 ], "answer_end": [ - 37, - 75, - 101, - 219, - 609, - 973, - 292, - 495, - 876, - 1346, - 1572, - 1573, - 1045, - 708, - 1636, - 1774, - 292, - 336, - 377, - 575, - 797 + 37, 75, 101, 219, 609, 973, 292, 495, 876, 1346, 1572, 1573, 1045, + 708, 1636, 1774, 292, 336, 377, 575, 797 ] } } @@ -3929,50 +2589,12 @@ "\"Follow me! It's urgent!\"" ], "answer_start": [ - 0, - 0, - 74, - 134, - 588, - 883, - 220, - 389, - 678, - 1211, - 1462, - 1548, - 975, - 659, - 1574, - 1636, - 220, - 292, - 293, - 497, - 758 + 0, 0, 74, 134, 588, 883, 220, 389, 678, 1211, 1462, 1548, 975, 659, + 1574, 1636, 220, 292, 293, 497, 758 ], "answer_end": [ - 37, - 75, - 101, - 219, - 609, - 973, - 292, - 495, - 876, - 1346, - 1572, - 1573, - 1045, - 708, - 1636, - 1774, - 292, - 336, - 377, - 575, - 797 + 37, 75, 101, 219, 609, 973, 292, 495, 876, 1346, 1572, 1573, 1045, + 708, 1636, 1774, 292, 336, 377, 575, 797 ] } } @@ -4034,50 +2656,12 @@ "\"Follow me! It's urgent!\"" ], "answer_start": [ - 0, - 0, - 74, - 134, - 588, - 883, - 220, - 389, - 678, - 1211, - 1462, - 1548, - 975, - 659, - 1574, - 1636, - 220, - 292, - 293, - 497, - 758 + 0, 0, 74, 134, 588, 883, 220, 389, 678, 1211, 1462, 1548, 975, 659, + 1574, 1636, 220, 292, 293, 497, 758 ], "answer_end": [ - 37, - 75, - 101, - 219, - 609, - 973, - 292, - 495, - 876, - 1346, - 1572, - 1573, - 1045, - 708, - 1636, - 1774, - 292, - 336, - 377, - 575, - 797 + 37, 75, 101, 219, 609, 973, 292, 495, 876, 1346, 1572, 1573, 1045, + 708, 1636, 1774, 292, 336, 377, 575, 797 ] } } @@ -4137,48 +2721,12 @@ "no" ], "answer_start": [ - 0, - 47, - 74, - 199, - 284, - 284, - 310, - 402, - 403, - 486, - 362, - 455, - 542, - 577, - 757, - 825, - 164, - 169, - 199, - 262 + 0, 47, 74, 199, 284, 284, 310, 402, 403, 486, 362, 455, 542, 577, 757, + 825, 164, 169, 199, 262 ], "answer_end": [ - 46, - 88, - 104, - 227, - 325, - 325, - 360, - 454, - 454, - 540, - 401, - 481, - 573, - 620, - 796, - 862, - 199, - 199, - 261, - 284 + 46, 88, 104, 227, 325, 325, 360, 454, 454, 540, 401, 481, 573, 620, + 796, 862, 199, 199, 261, 284 ] } } @@ -4238,48 +2786,12 @@ "no" ], "answer_start": [ - 0, - 47, - 74, - 199, - 284, - 284, - 310, - 402, - 403, - 486, - 362, - 455, - 542, - 577, - 757, - 825, - 164, - 169, - 199, - 262 + 0, 47, 74, 199, 284, 284, 310, 402, 403, 486, 362, 455, 542, 577, 757, + 825, 164, 169, 199, 262 ], "answer_end": [ - 46, - 88, - 104, - 227, - 325, - 325, - 360, - 454, - 454, - 540, - 401, - 481, - 573, - 620, - 796, - 862, - 199, - 199, - 261, - 284 + 46, 88, 104, 227, 325, 325, 360, 454, 454, 540, 401, 481, 573, 620, + 796, 862, 199, 199, 261, 284 ] } } @@ -4339,48 +2851,12 @@ "no" ], "answer_start": [ - 0, - 47, - 74, - 199, - 284, - 284, - 310, - 402, - 403, - 486, - 362, - 455, - 542, - 577, - 757, - 825, - 164, - 169, - 199, - 262 + 0, 47, 74, 199, 284, 284, 310, 402, 403, 486, 362, 455, 542, 577, 757, + 825, 164, 169, 199, 262 ], "answer_end": [ - 46, - 88, - 104, - 227, - 325, - 325, - 360, - 454, - 454, - 540, - 401, - 481, - 573, - 620, - 796, - 862, - 199, - 199, - 261, - 284 + 46, 88, 104, 227, 325, 325, 360, 454, 454, 540, 401, 481, 573, 620, + 796, 862, 199, 199, 261, 284 ] } } @@ -4426,34 +2902,10 @@ "the 7-year-old" ], "answer_start": [ - 19, - 103, - 538, - 527, - 58, - 223, - 917, - 990, - 1065, - 1141, - 834, - 813, - 813 + 19, 103, 538, 527, 58, 223, 917, 990, 1065, 1141, 834, 813, 813 ], "answer_end": [ - 59, - 130, - 565, - 565, - 65, - 227, - 946, - 1061, - 1078, - 1183, - 850, - 828, - 829 + 59, 130, 565, 565, 65, 227, 946, 1061, 1078, 1183, 850, 828, 829 ] } } @@ -4499,34 +2951,10 @@ "the 7-year-old" ], "answer_start": [ - 19, - 103, - 538, - 527, - 58, - 223, - 917, - 990, - 1065, - 1141, - 834, - 813, - 813 + 19, 103, 538, 527, 58, 223, 917, 990, 1065, 1141, 834, 813, 813 ], "answer_end": [ - 59, - 130, - 565, - 565, - 65, - 227, - 946, - 1061, - 1078, - 1183, - 850, - 828, - 829 + 59, 130, 565, 565, 65, 227, 946, 1061, 1078, 1183, 850, 828, 829 ] } } @@ -4572,34 +3000,10 @@ "the 7-year-old" ], "answer_start": [ - 19, - 103, - 538, - 527, - 58, - 223, - 917, - 990, - 1065, - 1141, - 834, - 813, - 813 + 19, 103, 538, 527, 58, 223, 917, 990, 1065, 1141, 834, 813, 813 ], "answer_end": [ - 59, - 130, - 565, - 565, - 65, - 227, - 946, - 1061, - 1078, - 1183, - 850, - 828, - 829 + 59, 130, 565, 565, 65, 227, 946, 1061, 1078, 1183, 850, 828, 829 ] } } @@ -4645,34 +3049,10 @@ "no" ], "answer_start": [ - 1032, - 1168, - 1197, - -1, - 991, - 1385, - 1361, - 1405, - 144, - 1511, - 434, - 544, - 714 + 1032, 1168, 1197, -1, 991, 1385, 1361, 1405, 144, 1511, 434, 544, 714 ], "answer_end": [ - 1110, - 1195, - 1221, - -1, - 1044, - 1475, - 1476, - 1437, - 180, - 1590, - 505, - 583, - 753 + 1110, 1195, 1221, -1, 1044, 1475, 1476, 1437, 180, 1590, 505, 583, 753 ] } } @@ -4718,34 +3098,10 @@ "no" ], "answer_start": [ - 1032, - 1168, - 1197, - -1, - 991, - 1385, - 1361, - 1405, - 144, - 1511, - 434, - 544, - 714 + 1032, 1168, 1197, -1, 991, 1385, 1361, 1405, 144, 1511, 434, 544, 714 ], "answer_end": [ - 1110, - 1195, - 1221, - -1, - 1044, - 1475, - 1476, - 1437, - 180, - 1590, - 505, - 583, - 753 + 1110, 1195, 1221, -1, 1044, 1475, 1476, 1437, 180, 1590, 505, 583, 753 ] } } @@ -4791,34 +3147,10 @@ "no" ], "answer_start": [ - 1032, - 1168, - 1197, - -1, - 991, - 1385, - 1361, - 1405, - 144, - 1511, - 434, - 544, - 714 + 1032, 1168, 1197, -1, 991, 1385, 1361, 1405, 144, 1511, 434, 544, 714 ], "answer_end": [ - 1110, - 1195, - 1221, - -1, - 1044, - 1475, - 1476, - 1437, - 180, - 1590, - 505, - 583, - 753 + 1110, 1195, 1221, -1, 1044, 1475, 1476, 1437, 180, 1590, 505, 583, 753 ] } } diff --git a/evals/datasets/coqa-factuality.json b/evals/datasets/coqa-factuality.json index 099334c..521d00b 100644 --- a/evals/datasets/coqa-factuality.json +++ b/evals/datasets/coqa-factuality.json @@ -39,32 +39,10 @@ "no" ], "answer_start": [ - 59, - 18, - 196, - 281, - 428, - 512, - 678, - 647, - 718, - 1035, - 1143, - 965 + 59, 18, 196, 281, 428, 512, 678, 647, 718, 1035, 1143, 965 ], "answer_end": [ - 93, - 80, - 215, - 315, - 490, - 549, - 716, - 676, - 776, - 1097, - 1170, - 1008 + 93, 80, 215, 315, 490, 549, 716, 676, 776, 1097, 1170, 1008 ] } } @@ -109,32 +87,10 @@ "no" ], "answer_start": [ - 59, - 18, - 196, - 281, - 428, - 512, - 678, - 647, - 718, - 1035, - 1143, - 965 + 59, 18, 196, 281, 428, 512, 678, 647, 718, 1035, 1143, 965 ], "answer_end": [ - 93, - 80, - 215, - 315, - 490, - 549, - 716, - 676, - 776, - 1097, - 1170, - 1008 + 93, 80, 215, 315, 490, 549, 716, 676, 776, 1097, 1170, 1008 ] } } @@ -179,32 +135,10 @@ "no" ], "answer_start": [ - 59, - 18, - 196, - 281, - 428, - 512, - 678, - 647, - 718, - 1035, - 1143, - 965 + 59, 18, 196, 281, 428, 512, 678, 647, 718, 1035, 1143, 965 ], "answer_end": [ - 93, - 80, - 215, - 315, - 490, - 549, - 716, - 676, - 776, - 1097, - 1170, - 1008 + 93, 80, 215, 315, 490, 549, 716, 676, 776, 1097, 1170, 1008 ] } } @@ -246,32 +180,8 @@ "yes", "unknown" ], - "answer_start": [ - 37, - 167, - 303, - 281, - 552, - 708, - 964, - 910, - -1, - 1056, - -1 - ], - "answer_end": [ - 43, - 175, - 346, - 302, - 577, - 723, - 995, - 935, - -1, - 1173, - -1 - ] + "answer_start": [37, 167, 303, 281, 552, 708, 964, 910, -1, 1056, -1], + "answer_end": [43, 175, 346, 302, 577, 723, 995, 935, -1, 1173, -1] } } }, @@ -312,32 +222,8 @@ "yes", "unknown" ], - "answer_start": [ - 37, - 167, - 303, - 281, - 552, - 708, - 964, - 910, - -1, - 1056, - -1 - ], - "answer_end": [ - 43, - 175, - 346, - 302, - 577, - 723, - 995, - 935, - -1, - 1173, - -1 - ] + "answer_start": [37, 167, 303, 281, 552, 708, 964, 910, -1, 1056, -1], + "answer_end": [43, 175, 346, 302, 577, 723, 995, 935, -1, 1173, -1] } } }, @@ -378,32 +264,8 @@ "yes", "unknown" ], - "answer_start": [ - 37, - 167, - 303, - 281, - 552, - 708, - 964, - 910, - -1, - 1056, - -1 - ], - "answer_end": [ - 43, - 175, - 346, - 302, - 577, - 723, - 995, - 935, - -1, - 1173, - -1 - ] + "answer_start": [37, 167, 303, 281, 552, 708, 964, 910, -1, 1056, -1], + "answer_end": [43, 175, 346, 302, 577, 723, 995, 935, -1, 1173, -1] } } }, @@ -453,38 +315,12 @@ "\"Thank you\"" ], "answer_start": [ - 19, - 136, - 146, - 169, - 304, - 348, - 214, - 579, - 727, - 423, - 917, - 594, - 1771, - 1667, + 19, 136, 146, 169, 304, 348, 214, 579, 727, 423, 917, 594, 1771, 1667, 1874 ], "answer_end": [ - 115, - 165, - 165, - 185, - 324, - 375, - 302, - 725, - 769, - 544, - 924, - 725, - 1814, - 1769, - 1885 + 115, 165, 165, 185, 324, 375, 302, 725, 769, 544, 924, 725, 1814, + 1769, 1885 ] } } @@ -535,38 +371,12 @@ "\"Thank you\"" ], "answer_start": [ - 19, - 136, - 146, - 169, - 304, - 348, - 214, - 579, - 727, - 423, - 917, - 594, - 1771, - 1667, + 19, 136, 146, 169, 304, 348, 214, 579, 727, 423, 917, 594, 1771, 1667, 1874 ], "answer_end": [ - 115, - 165, - 165, - 185, - 324, - 375, - 302, - 725, - 769, - 544, - 924, - 725, - 1814, - 1769, - 1885 + 115, 165, 165, 185, 324, 375, 302, 725, 769, 544, 924, 725, 1814, + 1769, 1885 ] } } @@ -617,38 +427,12 @@ "\"Thank you\"" ], "answer_start": [ - 19, - 136, - 146, - 169, - 304, - 348, - 214, - 579, - 727, - 423, - 917, - 594, - 1771, - 1667, + 19, 136, 146, 169, 304, 348, 214, 579, 727, 423, 917, 594, 1771, 1667, 1874 ], "answer_end": [ - 115, - 165, - 165, - 185, - 324, - 375, - 302, - 725, - 769, - 544, - 924, - 725, - 1814, - 1769, - 1885 + 115, 165, 165, 185, 324, 375, 302, 725, 769, 544, 924, 725, 1814, + 1769, 1885 ] } } @@ -709,48 +493,12 @@ "A cop" ], "answer_start": [ - 8, - 8, - 8, - 9, - 512, - 856, - 573, - 512, - 512, - 856, - 1041, - 1041, - 1041, - 1217, - 1216, - 1217, - 1217, - 1338, - 8, - 8 + 8, 8, 8, 9, 512, 856, 573, 512, 512, 856, 1041, 1041, 1041, 1217, + 1216, 1217, 1217, 1338, 8, 8 ], "answer_end": [ - 181, - 183, - 66, - 181, - 690, - 1039, - 690, - 662, - 690, - 1039, - 1103, - 1134, - 1165, - 1336, - 1336, - 1336, - 1336, - 1381, - 138, - 65 + 181, 183, 66, 181, 690, 1039, 690, 662, 690, 1039, 1103, 1134, 1165, + 1336, 1336, 1336, 1336, 1381, 138, 65 ] } } @@ -811,48 +559,12 @@ "A cop" ], "answer_start": [ - 8, - 8, - 8, - 9, - 512, - 856, - 573, - 512, - 512, - 856, - 1041, - 1041, - 1041, - 1217, - 1216, - 1217, - 1217, - 1338, - 8, - 8 + 8, 8, 8, 9, 512, 856, 573, 512, 512, 856, 1041, 1041, 1041, 1217, + 1216, 1217, 1217, 1338, 8, 8 ], "answer_end": [ - 181, - 183, - 66, - 181, - 690, - 1039, - 690, - 662, - 690, - 1039, - 1103, - 1134, - 1165, - 1336, - 1336, - 1336, - 1336, - 1381, - 138, - 65 + 181, 183, 66, 181, 690, 1039, 690, 662, 690, 1039, 1103, 1134, 1165, + 1336, 1336, 1336, 1336, 1381, 138, 65 ] } } @@ -913,48 +625,12 @@ "A cop" ], "answer_start": [ - 8, - 8, - 8, - 9, - 512, - 856, - 573, - 512, - 512, - 856, - 1041, - 1041, - 1041, - 1217, - 1216, - 1217, - 1217, - 1338, - 8, - 8 + 8, 8, 8, 9, 512, 856, 573, 512, 512, 856, 1041, 1041, 1041, 1217, + 1216, 1217, 1217, 1338, 8, 8 ], "answer_end": [ - 181, - 183, - 66, - 181, - 690, - 1039, - 690, - 662, - 690, - 1039, - 1103, - 1134, - 1165, - 1336, - 1336, - 1336, - 1336, - 1381, - 138, - 65 + 181, 183, 66, 181, 690, 1039, 690, 662, 690, 1039, 1103, 1134, 1165, + 1336, 1336, 1336, 1336, 1381, 138, 65 ] } } @@ -1011,44 +687,12 @@ "yes" ], "answer_start": [ - 0, - 56, - 190, - 343, - 414, - 565, - 601, - 888, - 906, - 1098, - 1179, - 1265, - 1296, - 1382, - 1436, - 1481, - 1514, - 1546 + 0, 56, 190, 343, 414, 565, 601, 888, 906, 1098, 1179, 1265, 1296, + 1382, 1436, 1481, 1514, 1546 ], "answer_end": [ - 54, - 107, - 341, - 367, - 497, - 600, - 631, - 903, - 965, - 1177, - 1216, - 1291, - 1350, - 1434, - 1454, - 1509, - 1541, - 1576 + 54, 107, 341, 367, 497, 600, 631, 903, 965, 1177, 1216, 1291, 1350, + 1434, 1454, 1509, 1541, 1576 ] } } @@ -1105,44 +749,12 @@ "yes" ], "answer_start": [ - 0, - 56, - 190, - 343, - 414, - 565, - 601, - 888, - 906, - 1098, - 1179, - 1265, - 1296, - 1382, - 1436, - 1481, - 1514, - 1546 + 0, 56, 190, 343, 414, 565, 601, 888, 906, 1098, 1179, 1265, 1296, + 1382, 1436, 1481, 1514, 1546 ], "answer_end": [ - 54, - 107, - 341, - 367, - 497, - 600, - 631, - 903, - 965, - 1177, - 1216, - 1291, - 1350, - 1434, - 1454, - 1509, - 1541, - 1576 + 54, 107, 341, 367, 497, 600, 631, 903, 965, 1177, 1216, 1291, 1350, + 1434, 1454, 1509, 1541, 1576 ] } } @@ -1199,44 +811,12 @@ "yes" ], "answer_start": [ - 0, - 56, - 190, - 343, - 414, - 565, - 601, - 888, - 906, - 1098, - 1179, - 1265, - 1296, - 1382, - 1436, - 1481, - 1514, - 1546 + 0, 56, 190, 343, 414, 565, 601, 888, 906, 1098, 1179, 1265, 1296, + 1382, 1436, 1481, 1514, 1546 ], "answer_end": [ - 54, - 107, - 341, - 367, - 497, - 600, - 631, - 903, - 965, - 1177, - 1216, - 1291, - 1350, - 1434, - 1454, - 1509, - 1541, - 1576 + 54, 107, 341, 367, 497, 600, 631, 903, 965, 1177, 1216, 1291, 1350, + 1434, 1454, 1509, 1541, 1576 ] } } @@ -1283,34 +863,10 @@ "St. George, Tompkinsville, Clifton, and Stapleton" ], "answer_start": [ - 28, - 45, - 80, - 17, - 88, - 322, - 448, - 477, - 600, - 808, - 831, - 894, - 939 + 28, 45, 80, 17, 88, 322, 448, 477, 600, 808, 831, 894, 939 ], "answer_end": [ - 32, - 58, - 88, - 21, - 119, - 356, - 455, - 493, - 620, - 829, - 890, - 906, - 988 + 32, 58, 88, 21, 119, 356, 455, 493, 620, 829, 890, 906, 988 ] } } @@ -1357,34 +913,10 @@ "St. George, Tompkinsville, Clifton, and Stapleton" ], "answer_start": [ - 28, - 45, - 80, - 17, - 88, - 322, - 448, - 477, - 600, - 808, - 831, - 894, - 939 + 28, 45, 80, 17, 88, 322, 448, 477, 600, 808, 831, 894, 939 ], "answer_end": [ - 32, - 58, - 88, - 21, - 119, - 356, - 455, - 493, - 620, - 829, - 890, - 906, - 988 + 32, 58, 88, 21, 119, 356, 455, 493, 620, 829, 890, 906, 988 ] } } @@ -1431,34 +963,10 @@ "St. George, Tompkinsville, Clifton, and Stapleton" ], "answer_start": [ - 28, - 45, - 80, - 17, - 88, - 322, - 448, - 477, - 600, - 808, - 831, - 894, - 939 + 28, 45, 80, 17, 88, 322, 448, 477, 600, 808, 831, 894, 939 ], "answer_end": [ - 32, - 58, - 88, - 21, - 119, - 356, - 455, - 493, - 620, - 829, - 890, - 906, - 988 + 32, 58, 88, 21, 119, 356, 455, 493, 620, 829, 890, 906, 988 ] } } @@ -1503,32 +1011,10 @@ "The flashlight" ], "answer_start": [ - 24, - 71, - 104, - 193, - 396, - 396, - 771, - 864, - 907, - 1641, - 1696, - 1274 + 24, 71, 104, 193, 396, 396, 771, 864, 907, 1641, 1696, 1274 ], "answer_end": [ - 69, - 102, - 130, - 217, - 461, - 461, - 829, - 906, - 1016, - 1677, - 1726, - 1339 + 69, 102, 130, 217, 461, 461, 829, 906, 1016, 1677, 1726, 1339 ] } } @@ -1573,32 +1059,10 @@ "The flashlight" ], "answer_start": [ - 24, - 71, - 104, - 193, - 396, - 396, - 771, - 864, - 907, - 1641, - 1696, - 1274 + 24, 71, 104, 193, 396, 396, 771, 864, 907, 1641, 1696, 1274 ], "answer_end": [ - 69, - 102, - 130, - 217, - 461, - 461, - 829, - 906, - 1016, - 1677, - 1726, - 1339 + 69, 102, 130, 217, 461, 461, 829, 906, 1016, 1677, 1726, 1339 ] } } @@ -1643,32 +1107,10 @@ "The flashlight" ], "answer_start": [ - 24, - 71, - 104, - 193, - 396, - 396, - 771, - 864, - 907, - 1641, - 1696, - 1274 + 24, 71, 104, 193, 396, 396, 771, 864, 907, 1641, 1696, 1274 ], "answer_end": [ - 69, - 102, - 130, - 217, - 461, - 461, - 829, - 906, - 1016, - 1677, - 1726, - 1339 + 69, 102, 130, 217, 461, 461, 829, 906, 1016, 1677, 1726, 1339 ] } } @@ -1729,48 +1171,12 @@ "2, Giordano told authorities that he had been snorkeling with Gardner" ], "answer_start": [ - 211, - 184, - 494, - 48, - 260, - 70, - 345, - 435, - 678, - 772, - 839, - 1021, - 1040, - 1154, - 1111, - 1269, - 1407, - 1506, - 993, - 1111 + 211, 184, 494, 48, 260, 70, 345, 435, 678, 772, 839, 1021, 1040, 1154, + 1111, 1269, 1407, 1506, 993, 1111 ], "answer_end": [ - 224, - 197, - 511, - 57, - 271, - 126, - 348, - 437, - 713, - 778, - 863, - 1033, - 1064, - 1164, - 1119, - 1300, - 1452, - 1508, - 1002, - 1177 + 224, 197, 511, 57, 271, 126, 348, 437, 713, 778, 863, 1033, 1064, + 1164, 1119, 1300, 1452, 1508, 1002, 1177 ] } } @@ -1831,48 +1237,12 @@ "2, Giordano told authorities that he had been snorkeling with Gardner" ], "answer_start": [ - 211, - 184, - 494, - 48, - 260, - 70, - 345, - 435, - 678, - 772, - 839, - 1021, - 1040, - 1154, - 1111, - 1269, - 1407, - 1506, - 993, - 1111 + 211, 184, 494, 48, 260, 70, 345, 435, 678, 772, 839, 1021, 1040, 1154, + 1111, 1269, 1407, 1506, 993, 1111 ], "answer_end": [ - 224, - 197, - 511, - 57, - 271, - 126, - 348, - 437, - 713, - 778, - 863, - 1033, - 1064, - 1164, - 1119, - 1300, - 1452, - 1508, - 1002, - 1177 + 224, 197, 511, 57, 271, 126, 348, 437, 713, 778, 863, 1033, 1064, + 1164, 1119, 1300, 1452, 1508, 1002, 1177 ] } } @@ -1933,48 +1303,12 @@ "2, Giordano told authorities that he had been snorkeling with Gardner" ], "answer_start": [ - 211, - 184, - 494, - 48, - 260, - 70, - 345, - 435, - 678, - 772, - 839, - 1021, - 1040, - 1154, - 1111, - 1269, - 1407, - 1506, - 993, - 1111 + 211, 184, 494, 48, 260, 70, 345, 435, 678, 772, 839, 1021, 1040, 1154, + 1111, 1269, 1407, 1506, 993, 1111 ], "answer_end": [ - 224, - 197, - 511, - 57, - 271, - 126, - 348, - 437, - 713, - 778, - 863, - 1033, - 1064, - 1164, - 1119, - 1300, - 1452, - 1508, - 1002, - 1177 + 224, 197, 511, 57, 271, 126, 348, 437, 713, 778, 863, 1033, 1064, + 1164, 1119, 1300, 1452, 1508, 1002, 1177 ] } } @@ -2014,30 +1348,8 @@ "yes", "unknown" ], - "answer_start": [ - 159, - 34, - 196, - 224, - 731, - 794, - 1120, - 1190, - 1436, - -1 - ], - "answer_end": [ - 177, - 54, - 222, - 264, - 757, - 805, - 1158, - 1212, - 1448, - -1 - ] + "answer_start": [159, 34, 196, 224, 731, 794, 1120, 1190, 1436, -1], + "answer_end": [177, 54, 222, 264, 757, 805, 1158, 1212, 1448, -1] } } }, @@ -2076,30 +1388,8 @@ "yes", "unknown" ], - "answer_start": [ - 159, - 34, - 196, - 224, - 731, - 794, - 1120, - 1190, - 1436, - -1 - ], - "answer_end": [ - 177, - 54, - 222, - 264, - 757, - 805, - 1158, - 1212, - 1448, - -1 - ] + "answer_start": [159, 34, 196, 224, 731, 794, 1120, 1190, 1436, -1], + "answer_end": [177, 54, 222, 264, 757, 805, 1158, 1212, 1448, -1] } } }, @@ -2138,30 +1428,8 @@ "yes", "unknown" ], - "answer_start": [ - 159, - 34, - 196, - 224, - 731, - 794, - 1120, - 1190, - 1436, - -1 - ], - "answer_end": [ - 177, - 54, - 222, - 264, - 757, - 805, - 1158, - 1212, - 1448, - -1 - ] + "answer_start": [159, 34, 196, 224, 731, 794, 1120, 1190, 1436, -1], + "answer_end": [177, 54, 222, 264, 757, 805, 1158, 1212, 1448, -1] } } }, @@ -2200,30 +1468,8 @@ "Pfc. Andrew Holmes", "Holmes is charged with the premeditated deaths of three civilians" ], - "answer_start": [ - 28, - 28, - 77, - 267, - 363, - 362, - 1229, - 482, - 545, - 662 - ], - "answer_end": [ - 62, - 62, - 181, - 361, - 430, - 481, - 1362, - 541, - 563, - 727 - ] + "answer_start": [28, 28, 77, 267, 363, 362, 1229, 482, 545, 662], + "answer_end": [62, 62, 181, 361, 430, 481, 1362, 541, 563, 727] } } }, @@ -2262,30 +1508,8 @@ "Pfc. Andrew Holmes", "Holmes is charged with the premeditated deaths of three civilians" ], - "answer_start": [ - 28, - 28, - 77, - 267, - 363, - 362, - 1229, - 482, - 545, - 662 - ], - "answer_end": [ - 62, - 62, - 181, - 361, - 430, - 481, - 1362, - 541, - 563, - 727 - ] + "answer_start": [28, 28, 77, 267, 363, 362, 1229, 482, 545, 662], + "answer_end": [62, 62, 181, 361, 430, 481, 1362, 541, 563, 727] } } }, @@ -2324,30 +1548,8 @@ "Pfc. Andrew Holmes", "Holmes is charged with the premeditated deaths of three civilians" ], - "answer_start": [ - 28, - 28, - 77, - 267, - 363, - 362, - 1229, - 482, - 545, - 662 - ], - "answer_end": [ - 62, - 62, - 181, - 361, - 430, - 481, - 1362, - 541, - 563, - 727 - ] + "answer_start": [28, 28, 77, 267, 363, 362, 1229, 482, 545, 662], + "answer_end": [62, 62, 181, 361, 430, 481, 1362, 541, 563, 727] } } }, @@ -2386,30 +1588,8 @@ "38", "just that it has bible references and shows him enjoying gos[e; singing with his wife" ], - "answer_start": [ - 330, - 851, - 851, - 945, - 1175, - 753, - 1472, - 504, - 851, - 6 - ], - "answer_end": [ - 383, - 1085, - 971, - 970, - 1253, - 791, - 1511, - 671, - 884, - 132 - ] + "answer_start": [330, 851, 851, 945, 1175, 753, 1472, 504, 851, 6], + "answer_end": [383, 1085, 971, 970, 1253, 791, 1511, 671, 884, 132] } } }, @@ -2448,30 +1628,8 @@ "38", "just that it has bible references and shows him enjoying gos[e; singing with his wife" ], - "answer_start": [ - 330, - 851, - 851, - 945, - 1175, - 753, - 1472, - 504, - 851, - 6 - ], - "answer_end": [ - 383, - 1085, - 971, - 970, - 1253, - 791, - 1511, - 671, - 884, - 132 - ] + "answer_start": [330, 851, 851, 945, 1175, 753, 1472, 504, 851, 6], + "answer_end": [383, 1085, 971, 970, 1253, 791, 1511, 671, 884, 132] } } }, @@ -2510,30 +1668,8 @@ "38", "just that it has bible references and shows him enjoying gos[e; singing with his wife" ], - "answer_start": [ - 330, - 851, - 851, - 945, - 1175, - 753, - 1472, - 504, - 851, - 6 - ], - "answer_end": [ - 383, - 1085, - 971, - 970, - 1253, - 791, - 1511, - 671, - 884, - 132 - ] + "answer_start": [330, 851, 851, 945, 1175, 753, 1472, 504, 851, 6], + "answer_end": [383, 1085, 971, 970, 1253, 791, 1511, 671, 884, 132] } } }, @@ -2587,42 +1723,12 @@ "no" ], "answer_start": [ - 32, - 37, - 571, - 806, - 278, - 841, - 949, - 971, - 995, - 372, - 806, - 841, - 1618, - 1636, - 1579, - 1654, - 1671 + 32, 37, 571, 806, 278, 841, 949, 971, 995, 372, 806, 841, 1618, 1636, + 1579, 1654, 1671 ], "answer_end": [ - 36, - 67, - 575, - 819, - 282, - 862, - 969, - 1020, - 1020, - 380, - 820, - 863, - 1632, - 1651, - 1597, - 1670, - 1746 + 36, 67, 575, 819, 282, 862, 969, 1020, 1020, 380, 820, 863, 1632, + 1651, 1597, 1670, 1746 ] } } @@ -2677,42 +1783,12 @@ "no" ], "answer_start": [ - 32, - 37, - 571, - 806, - 278, - 841, - 949, - 971, - 995, - 372, - 806, - 841, - 1618, - 1636, - 1579, - 1654, - 1671 + 32, 37, 571, 806, 278, 841, 949, 971, 995, 372, 806, 841, 1618, 1636, + 1579, 1654, 1671 ], "answer_end": [ - 36, - 67, - 575, - 819, - 282, - 862, - 969, - 1020, - 1020, - 380, - 820, - 863, - 1632, - 1651, - 1597, - 1670, - 1746 + 36, 67, 575, 819, 282, 862, 969, 1020, 1020, 380, 820, 863, 1632, + 1651, 1597, 1670, 1746 ] } } @@ -2767,42 +1843,12 @@ "no" ], "answer_start": [ - 32, - 37, - 571, - 806, - 278, - 841, - 949, - 971, - 995, - 372, - 806, - 841, - 1618, - 1636, - 1579, - 1654, - 1671 + 32, 37, 571, 806, 278, 841, 949, 971, 995, 372, 806, 841, 1618, 1636, + 1579, 1654, 1671 ], "answer_end": [ - 36, - 67, - 575, - 819, - 282, - 862, - 969, - 1020, - 1020, - 380, - 820, - 863, - 1632, - 1651, - 1597, - 1670, - 1746 + 36, 67, 575, 819, 282, 862, 969, 1020, 1020, 380, 820, 863, 1632, + 1651, 1597, 1670, 1746 ] } } @@ -2847,32 +1893,10 @@ "at least $500" ], "answer_start": [ - 127, - 759, - 523, - 534, - 534, - 213, - 408, - 456, - 644, - 297, - 48, - 442 + 127, 759, 523, 534, 534, 213, 408, 456, 644, 297, 48, 442 ], "answer_end": [ - 129, - 766, - 526, - 555, - 554, - 220, - 411, - 484, - 649, - 316, - 56, - 455 + 129, 766, 526, 555, 554, 220, 411, 484, 649, 316, 56, 455 ] } } @@ -2917,32 +1941,10 @@ "at least $500" ], "answer_start": [ - 127, - 759, - 523, - 534, - 534, - 213, - 408, - 456, - 644, - 297, - 48, - 442 + 127, 759, 523, 534, 534, 213, 408, 456, 644, 297, 48, 442 ], "answer_end": [ - 129, - 766, - 526, - 555, - 554, - 220, - 411, - 484, - 649, - 316, - 56, - 455 + 129, 766, 526, 555, 554, 220, 411, 484, 649, 316, 56, 455 ] } } @@ -2987,32 +1989,10 @@ "at least $500" ], "answer_start": [ - 127, - 759, - 523, - 534, - 534, - 213, - 408, - 456, - 644, - 297, - 48, - 442 + 127, 759, 523, 534, 534, 213, 408, 456, 644, 297, 48, 442 ], "answer_end": [ - 129, - 766, - 526, - 555, - 554, - 220, - 411, - 484, - 649, - 316, - 56, - 455 + 129, 766, 526, 555, 554, 220, 411, 484, 649, 316, 56, 455 ] } } @@ -3052,30 +2032,8 @@ "threw a ball into the water", "got very wet" ], - "answer_start": [ - 33, - 76, - 381, - 170, - 749, - 552, - 658, - 897, - 1001, - 1189 - ], - "answer_end": [ - 59, - 91, - 400, - 198, - 763, - 561, - 668, - 933, - 1028, - 1201 - ] + "answer_start": [33, 76, 381, 170, 749, 552, 658, 897, 1001, 1189], + "answer_end": [59, 91, 400, 198, 763, 561, 668, 933, 1028, 1201] } } }, @@ -3114,30 +2072,8 @@ "threw a ball into the water", "got very wet" ], - "answer_start": [ - 33, - 76, - 381, - 170, - 749, - 552, - 658, - 897, - 1001, - 1189 - ], - "answer_end": [ - 59, - 91, - 400, - 198, - 763, - 561, - 668, - 933, - 1028, - 1201 - ] + "answer_start": [33, 76, 381, 170, 749, 552, 658, 897, 1001, 1189], + "answer_end": [59, 91, 400, 198, 763, 561, 668, 933, 1028, 1201] } } }, @@ -3176,30 +2112,8 @@ "threw a ball into the water", "got very wet" ], - "answer_start": [ - 33, - 76, - 381, - 170, - 749, - 552, - 658, - 897, - 1001, - 1189 - ], - "answer_end": [ - 59, - 91, - 400, - 198, - 763, - 561, - 668, - 933, - 1028, - 1201 - ] + "answer_start": [33, 76, 381, 170, 749, 552, 658, 897, 1001, 1189], + "answer_end": [59, 91, 400, 198, 763, 561, 668, 933, 1028, 1201] } } }, @@ -3259,48 +2173,12 @@ "No" ], "answer_start": [ - 936, - 906, - 2, - 969, - 1001, - 1017, - 845, - 1129, - 1159, - 1225, - 100, - 181, - 689, - 671, - 2, - 732, - 373, - 215, - 329, - 546 + 936, 906, 2, 969, 1001, 1017, 845, 1129, 1159, 1225, 100, 181, 689, + 671, 2, 732, 373, 215, 329, 546 ], "answer_end": [ - 964, - 933, - 9, - 1016, - 1016, - 1024, - 870, - 1157, - 1222, - 1267, - 112, - 190, - 728, - 683, - 35, - 763, - 474, - 280, - 355, - 634 + 964, 933, 9, 1016, 1016, 1024, 870, 1157, 1222, 1267, 112, 190, 728, + 683, 35, 763, 474, 280, 355, 634 ] } } @@ -3361,48 +2239,12 @@ "No" ], "answer_start": [ - 936, - 906, - 2, - 969, - 1001, - 1017, - 845, - 1129, - 1159, - 1225, - 100, - 181, - 689, - 671, - 2, - 732, - 373, - 215, - 329, - 546 + 936, 906, 2, 969, 1001, 1017, 845, 1129, 1159, 1225, 100, 181, 689, + 671, 2, 732, 373, 215, 329, 546 ], "answer_end": [ - 964, - 933, - 9, - 1016, - 1016, - 1024, - 870, - 1157, - 1222, - 1267, - 112, - 190, - 728, - 683, - 35, - 763, - 474, - 280, - 355, - 634 + 964, 933, 9, 1016, 1016, 1024, 870, 1157, 1222, 1267, 112, 190, 728, + 683, 35, 763, 474, 280, 355, 634 ] } } @@ -3463,48 +2305,12 @@ "No" ], "answer_start": [ - 936, - 906, - 2, - 969, - 1001, - 1017, - 845, - 1129, - 1159, - 1225, - 100, - 181, - 689, - 671, - 2, - 732, - 373, - 215, - 329, - 546 + 936, 906, 2, 969, 1001, 1017, 845, 1129, 1159, 1225, 100, 181, 689, + 671, 2, 732, 373, 215, 329, 546 ], "answer_end": [ - 964, - 933, - 9, - 1016, - 1016, - 1024, - 870, - 1157, - 1222, - 1267, - 112, - 190, - 728, - 683, - 35, - 763, - 474, - 280, - 355, - 634 + 964, 933, 9, 1016, 1016, 1024, 870, 1157, 1222, 1267, 112, 190, 728, + 683, 35, 763, 474, 280, 355, 634 ] } } @@ -3565,48 +2371,12 @@ "unknown" ], "answer_start": [ - 15, - 100, - 186, - 234, - 235, - 451, - 15, - 702, - 992, - 839, - 901, - 711, - 1, - 1039, - 1189, - 1149, - 1039, - 15, - 312, - -1 + 15, 100, 186, 234, 235, 451, 15, 702, 992, 839, 901, 711, 1, 1039, + 1189, 1149, 1039, 15, 312, -1 ], "answer_end": [ - 98, - 184, - 212, - 250, - 295, - 497, - 83, - 882, - 1037, - 900, - 988, - 1074, - 1583, - 1348, - 1197, - 1173, - 1348, - 229, - 450, - -1 + 98, 184, 212, 250, 295, 497, 83, 882, 1037, 900, 988, 1074, 1583, + 1348, 1197, 1173, 1348, 229, 450, -1 ] } } @@ -3667,48 +2437,12 @@ "unknown" ], "answer_start": [ - 15, - 100, - 186, - 234, - 235, - 451, - 15, - 702, - 992, - 839, - 901, - 711, - 1, - 1039, - 1189, - 1149, - 1039, - 15, - 312, - -1 + 15, 100, 186, 234, 235, 451, 15, 702, 992, 839, 901, 711, 1, 1039, + 1189, 1149, 1039, 15, 312, -1 ], "answer_end": [ - 98, - 184, - 212, - 250, - 295, - 497, - 83, - 882, - 1037, - 900, - 988, - 1074, - 1583, - 1348, - 1197, - 1173, - 1348, - 229, - 450, - -1 + 98, 184, 212, 250, 295, 497, 83, 882, 1037, 900, 988, 1074, 1583, + 1348, 1197, 1173, 1348, 229, 450, -1 ] } } @@ -3769,48 +2503,12 @@ "unknown" ], "answer_start": [ - 15, - 100, - 186, - 234, - 235, - 451, - 15, - 702, - 992, - 839, - 901, - 711, - 1, - 1039, - 1189, - 1149, - 1039, - 15, - 312, - -1 + 15, 100, 186, 234, 235, 451, 15, 702, 992, 839, 901, 711, 1, 1039, + 1189, 1149, 1039, 15, 312, -1 ], "answer_end": [ - 98, - 184, - 212, - 250, - 295, - 497, - 83, - 882, - 1037, - 900, - 988, - 1074, - 1583, - 1348, - 1197, - 1173, - 1348, - 229, - 450, - -1 + 98, 184, 212, 250, 295, 497, 83, 882, 1037, 900, 988, 1074, 1583, + 1348, 1197, 1173, 1348, 229, 450, -1 ] } } @@ -3873,50 +2571,12 @@ "\"Follow me! It's urgent!\"" ], "answer_start": [ - 0, - 0, - 74, - 134, - 588, - 883, - 220, - 389, - 678, - 1211, - 1462, - 1548, - 975, - 659, - 1574, - 1636, - 220, - 292, - 293, - 497, - 758 + 0, 0, 74, 134, 588, 883, 220, 389, 678, 1211, 1462, 1548, 975, 659, + 1574, 1636, 220, 292, 293, 497, 758 ], "answer_end": [ - 37, - 75, - 101, - 219, - 609, - 973, - 292, - 495, - 876, - 1346, - 1572, - 1573, - 1045, - 708, - 1636, - 1774, - 292, - 336, - 377, - 575, - 797 + 37, 75, 101, 219, 609, 973, 292, 495, 876, 1346, 1572, 1573, 1045, + 708, 1636, 1774, 292, 336, 377, 575, 797 ] } } @@ -3979,50 +2639,12 @@ "\"Follow me! It's urgent!\"" ], "answer_start": [ - 0, - 0, - 74, - 134, - 588, - 883, - 220, - 389, - 678, - 1211, - 1462, - 1548, - 975, - 659, - 1574, - 1636, - 220, - 292, - 293, - 497, - 758 + 0, 0, 74, 134, 588, 883, 220, 389, 678, 1211, 1462, 1548, 975, 659, + 1574, 1636, 220, 292, 293, 497, 758 ], "answer_end": [ - 37, - 75, - 101, - 219, - 609, - 973, - 292, - 495, - 876, - 1346, - 1572, - 1573, - 1045, - 708, - 1636, - 1774, - 292, - 336, - 377, - 575, - 797 + 37, 75, 101, 219, 609, 973, 292, 495, 876, 1346, 1572, 1573, 1045, + 708, 1636, 1774, 292, 336, 377, 575, 797 ] } } @@ -4085,50 +2707,12 @@ "\"Follow me! It's urgent!\"" ], "answer_start": [ - 0, - 0, - 74, - 134, - 588, - 883, - 220, - 389, - 678, - 1211, - 1462, - 1548, - 975, - 659, - 1574, - 1636, - 220, - 292, - 293, - 497, - 758 + 0, 0, 74, 134, 588, 883, 220, 389, 678, 1211, 1462, 1548, 975, 659, + 1574, 1636, 220, 292, 293, 497, 758 ], "answer_end": [ - 37, - 75, - 101, - 219, - 609, - 973, - 292, - 495, - 876, - 1346, - 1572, - 1573, - 1045, - 708, - 1636, - 1774, - 292, - 336, - 377, - 575, - 797 + 37, 75, 101, 219, 609, 973, 292, 495, 876, 1346, 1572, 1573, 1045, + 708, 1636, 1774, 292, 336, 377, 575, 797 ] } } @@ -4189,48 +2773,12 @@ "no" ], "answer_start": [ - 0, - 47, - 74, - 199, - 284, - 284, - 310, - 402, - 403, - 486, - 362, - 455, - 542, - 577, - 757, - 825, - 164, - 169, - 199, - 262 + 0, 47, 74, 199, 284, 284, 310, 402, 403, 486, 362, 455, 542, 577, 757, + 825, 164, 169, 199, 262 ], "answer_end": [ - 46, - 88, - 104, - 227, - 325, - 325, - 360, - 454, - 454, - 540, - 401, - 481, - 573, - 620, - 796, - 862, - 199, - 199, - 261, - 284 + 46, 88, 104, 227, 325, 325, 360, 454, 454, 540, 401, 481, 573, 620, + 796, 862, 199, 199, 261, 284 ] } } @@ -4291,48 +2839,12 @@ "no" ], "answer_start": [ - 0, - 47, - 74, - 199, - 284, - 284, - 310, - 402, - 403, - 486, - 362, - 455, - 542, - 577, - 757, - 825, - 164, - 169, - 199, - 262 + 0, 47, 74, 199, 284, 284, 310, 402, 403, 486, 362, 455, 542, 577, 757, + 825, 164, 169, 199, 262 ], "answer_end": [ - 46, - 88, - 104, - 227, - 325, - 325, - 360, - 454, - 454, - 540, - 401, - 481, - 573, - 620, - 796, - 862, - 199, - 199, - 261, - 284 + 46, 88, 104, 227, 325, 325, 360, 454, 454, 540, 401, 481, 573, 620, + 796, 862, 199, 199, 261, 284 ] } } @@ -4393,48 +2905,12 @@ "no" ], "answer_start": [ - 0, - 47, - 74, - 199, - 284, - 284, - 310, - 402, - 403, - 486, - 362, - 455, - 542, - 577, - 757, - 825, - 164, - 169, - 199, - 262 + 0, 47, 74, 199, 284, 284, 310, 402, 403, 486, 362, 455, 542, 577, 757, + 825, 164, 169, 199, 262 ], "answer_end": [ - 46, - 88, - 104, - 227, - 325, - 325, - 360, - 454, - 454, - 540, - 401, - 481, - 573, - 620, - 796, - 862, - 199, - 199, - 261, - 284 + 46, 88, 104, 227, 325, 325, 360, 454, 454, 540, 401, 481, 573, 620, + 796, 862, 199, 199, 261, 284 ] } } @@ -4481,34 +2957,10 @@ "the 7-year-old" ], "answer_start": [ - 19, - 103, - 538, - 527, - 58, - 223, - 917, - 990, - 1065, - 1141, - 834, - 813, - 813 + 19, 103, 538, 527, 58, 223, 917, 990, 1065, 1141, 834, 813, 813 ], "answer_end": [ - 59, - 130, - 565, - 565, - 65, - 227, - 946, - 1061, - 1078, - 1183, - 850, - 828, - 829 + 59, 130, 565, 565, 65, 227, 946, 1061, 1078, 1183, 850, 828, 829 ] } } @@ -4555,34 +3007,10 @@ "the 7-year-old" ], "answer_start": [ - 19, - 103, - 538, - 527, - 58, - 223, - 917, - 990, - 1065, - 1141, - 834, - 813, - 813 + 19, 103, 538, 527, 58, 223, 917, 990, 1065, 1141, 834, 813, 813 ], "answer_end": [ - 59, - 130, - 565, - 565, - 65, - 227, - 946, - 1061, - 1078, - 1183, - 850, - 828, - 829 + 59, 130, 565, 565, 65, 227, 946, 1061, 1078, 1183, 850, 828, 829 ] } } @@ -4629,34 +3057,10 @@ "the 7-year-old" ], "answer_start": [ - 19, - 103, - 538, - 527, - 58, - 223, - 917, - 990, - 1065, - 1141, - 834, - 813, - 813 + 19, 103, 538, 527, 58, 223, 917, 990, 1065, 1141, 834, 813, 813 ], "answer_end": [ - 59, - 130, - 565, - 565, - 65, - 227, - 946, - 1061, - 1078, - 1183, - 850, - 828, - 829 + 59, 130, 565, 565, 65, 227, 946, 1061, 1078, 1183, 850, 828, 829 ] } } @@ -4703,34 +3107,10 @@ "no" ], "answer_start": [ - 1032, - 1168, - 1197, - -1, - 991, - 1385, - 1361, - 1405, - 144, - 1511, - 434, - 544, - 714 + 1032, 1168, 1197, -1, 991, 1385, 1361, 1405, 144, 1511, 434, 544, 714 ], "answer_end": [ - 1110, - 1195, - 1221, - -1, - 1044, - 1475, - 1476, - 1437, - 180, - 1590, - 505, - 583, - 753 + 1110, 1195, 1221, -1, 1044, 1475, 1476, 1437, 180, 1590, 505, 583, 753 ] } } @@ -4777,34 +3157,10 @@ "no" ], "answer_start": [ - 1032, - 1168, - 1197, - -1, - 991, - 1385, - 1361, - 1405, - 144, - 1511, - 434, - 544, - 714 + 1032, 1168, 1197, -1, 991, 1385, 1361, 1405, 144, 1511, 434, 544, 714 ], "answer_end": [ - 1110, - 1195, - 1221, - -1, - 1044, - 1475, - 1476, - 1437, - 180, - 1590, - 505, - 583, - 753 + 1110, 1195, 1221, -1, 1044, 1475, 1476, 1437, 180, 1590, 505, 583, 753 ] } } @@ -4851,34 +3207,10 @@ "no" ], "answer_start": [ - 1032, - 1168, - 1197, - -1, - 991, - 1385, - 1361, - 1405, - 144, - 1511, - 434, - 544, - 714 + 1032, 1168, 1197, -1, 991, 1385, 1361, 1405, 144, 1511, 434, 544, 714 ], "answer_end": [ - 1110, - 1195, - 1221, - -1, - 1044, - 1475, - 1476, - 1437, - 180, - 1590, - 505, - 583, - 753 + 1110, 1195, 1221, -1, 1044, 1475, 1476, 1437, 180, 1590, 505, 583, 753 ] } } diff --git a/evals/package.json b/evals/package.json index ffcf6a0..a040ecd 100644 --- a/evals/package.json +++ b/evals/package.json @@ -14,7 +14,7 @@ "dependencies": { "autoevals": "workspace:*", "braintrust": "^0.0.140", - "zod": "^3.22.4" + "zod": "^4.2.1" }, "devDependencies": { "@types/node": "^20.10.5", diff --git a/evals/src/autoevals.eval.ts b/evals/src/autoevals.eval.ts index 0d6f55b..879b874 100644 --- a/evals/src/autoevals.eval.ts +++ b/evals/src/autoevals.eval.ts @@ -7,7 +7,7 @@ import { coqaCaseSchema, dataDir, } from "./datasets"; -import { z } from "zod"; +import { z } from "zod/v3"; import { AnswerCorrectness, ClosedQA, diff --git a/evals/src/datasets.ts b/evals/src/datasets.ts index cddf942..23a0ed3 100644 --- a/evals/src/datasets.ts +++ b/evals/src/datasets.ts @@ -1,4 +1,4 @@ -import { z } from "zod"; +import { z } from "zod/v3"; import path from "path"; diff --git a/evals/src/sync_datasets.ts b/evals/src/sync_datasets.ts index 2eaa1b7..0611942 100644 --- a/evals/src/sync_datasets.ts +++ b/evals/src/sync_datasets.ts @@ -1,6 +1,6 @@ import { duckq, getDuckDBConn } from "./duckdb"; -import { z } from "zod"; +import { z } from "zod/v3"; import { coqaSchema, dataDir, diff --git a/js/ragas.ts b/js/ragas.ts index d5a5285..c263648 100644 --- a/js/ragas.ts +++ b/js/ragas.ts @@ -5,10 +5,10 @@ import { Scorer, ScorerArgs } from "./score"; import { DEFAULT_MODEL, LLMArgs } from "./llm"; import { buildOpenAIClient, extractOpenAIArgs } from "./oai"; import OpenAI from "openai"; +import { zodFunction } from "openai/helpers/zod"; import { ListContains } from "./list"; import { EmbeddingSimilarity } from "./string"; -import { z } from "zod"; -import zodToJsonSchema from "zod-to-json-schema"; +import { z } from "zod/v3"; import { makePartial, ScorerWithPartial } from "./partial"; type RagasArgs = { @@ -87,14 +87,11 @@ export const ContextEntityRecall: ScorerWithPartial< }, ], tools: [ - { - type: "function", - function: { - name: "extract_entities", - description: "Extract unique entities from a given text", - parameters: zodToJsonSchema(entitySchema), - }, - }, + zodFunction({ + name: "extract_entities", + description: "Extract unique entities from a given text", + parameters: entitySchema, + }), ], tool_choice: { type: "function", function: { name: "extract_entities" } }, }); @@ -167,14 +164,11 @@ export const ContextRelevancy: ScorerWithPartial = }, ], tools: [ - { - type: "function", - function: { - name: "extract_sentences", - description: "Extract relevant sentences from a given context", - parameters: zodToJsonSchema(relevantSentencesSchema), - }, - }, + zodFunction({ + name: "extract_sentences", + description: "Extract relevant sentences from a given context", + parameters: relevantSentencesSchema, + }), ], tool_choice: { type: "function", @@ -267,13 +261,10 @@ export const ContextRecall: ScorerWithPartial = makePartial( }, ], tools: [ - { - type: "function", - function: { - name: "extract_statements", - parameters: zodToJsonSchema(contextRecallSchema), - }, - }, + zodFunction({ + name: "extract_statements", + parameters: contextRecallSchema, + }), ], tool_choice: { type: "function", @@ -367,15 +358,11 @@ export const ContextPrecision: ScorerWithPartial = }, ], tools: [ - { - type: "function", - function: { - name: "verify", - description: - "Verify if context was useful in arriving at the answer", - parameters: zodToJsonSchema(contextPrecisionSchema), - }, - }, + zodFunction({ + name: "verify", + description: "Verify if context was useful in arriving at the answer", + parameters: contextPrecisionSchema, + }), ], tool_choice: { type: "function", function: { name: "verify" } }, }); @@ -494,14 +481,11 @@ export const Faithfulness: ScorerWithPartial = makePartial( }, ], tools: [ - { - type: "function", - function: { - name: "extract_statements", - description: "Extract statements from an answer given a question", - parameters: zodToJsonSchema(extractedStatementsSchema), - }, - }, + zodFunction({ + name: "extract_statements", + description: "Extract statements from an answer given a question", + parameters: extractedStatementsSchema, + }), ], tool_choice: { type: "function", @@ -525,15 +509,12 @@ export const Faithfulness: ScorerWithPartial = makePartial( }, ], tools: [ - { - type: "function", - function: { - name: "judge_statements", - description: - "Judge whether the statements are faithful to the context", - parameters: zodToJsonSchema(statementFaithfulnessSchema), - }, - }, + zodFunction({ + name: "judge_statements", + description: + "Judge whether the statements are faithful to the context", + parameters: statementFaithfulnessSchema, + }), ], tool_choice: { type: "function", function: { name: "judge_statements" } }, }); @@ -635,15 +616,12 @@ export const AnswerRelevancy: ScorerWithPartial< }, ], tools: [ - { - type: "function", - function: { - name: "generate_question", - description: - "Generate a question for the given answer and identify if the answer is noncommittal", - parameters: zodToJsonSchema(questionGenSchema), - }, - }, + zodFunction({ + name: "generate_question", + description: + "Generate a question for the given answer and identify if the answer is noncommittal", + parameters: questionGenSchema, + }), ], tool_choice: { type: "function", @@ -810,14 +788,11 @@ export const AnswerCorrectness: ScorerWithPartial< }, ], tools: [ - { - type: "function", - function: { - name: "classify_statements", - description: "Classify statements as TP, FP, or FN", - parameters: zodToJsonSchema(answerCorrectnessClassificationSchema), - }, - }, + zodFunction({ + name: "classify_statements", + description: "Classify statements as TP, FP, or FN", + parameters: answerCorrectnessClassificationSchema, + }), ], tool_choice: { type: "function", diff --git a/js/templates.ts b/js/templates.ts index 636f712..cfe8e55 100644 --- a/js/templates.ts +++ b/js/templates.ts @@ -1,4 +1,4 @@ -import { z } from "zod"; +import { z } from "zod/v3"; import * as yaml from "js-yaml"; import battle from "../templates/battle.yaml"; @@ -13,7 +13,7 @@ import translation from "../templates/translation.yaml"; export const modelGradedSpecSchema = z.object({ prompt: z.string(), - choice_scores: z.record(z.number()), + choice_scores: z.record(z.string(), z.number()), model: z.string().optional(), use_cot: z.boolean().optional(), temperature: z.number().optional(), diff --git a/package.json b/package.json index 396d34b..676a5e5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "autoevals", - "version": "0.0.131", + "version": "0.1.0", "description": "Universal library for evaluating AI models", "repository": { "type": "git", @@ -44,7 +44,11 @@ "typedoc": "^0.25.13", "typedoc-plugin-markdown": "^3.17.1", "typescript": "^5.9.2", - "vitest": "^2.1.9" + "vitest": "^2.1.9", + "zod": "3.25.67" + }, + "peerDependencies": { + "zod": "^3.0.0 || ^4.0.0" }, "dependencies": { "ajv": "^8.17.1", @@ -53,9 +57,8 @@ "js-yaml": "^4.1.0", "linear-sum-assignment": "^1.0.7", "mustache": "^4.2.0", - "openai": "^6.3.0", - "zod": "^3.25.76", - "zod-to-json-schema": "^3.24.6" + "openai": "^6.7.0", + "zod-to-json-schema": "3.25.0" }, "packageManager": "pnpm@10.26.2" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 545cc5c..4af6626 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,18 +26,15 @@ importers: specifier: ^4.2.0 version: 4.2.0 openai: - specifier: ^6.3.0 - version: 6.3.0(zod@3.25.76) - zod: - specifier: ^3.25.76 - version: 3.25.76 + specifier: ^6.7.0 + version: 6.15.0(zod@3.25.67) zod-to-json-schema: - specifier: ^3.24.6 - version: 3.24.6(zod@3.25.76) + specifier: 3.25.0 + version: 3.25.0(zod@3.25.67) devDependencies: "@rollup/plugin-yaml": specifier: ^4.1.2 - version: 4.1.2 + version: 4.1.2(rollup@4.46.4) "@types/js-levenshtein": specifier: ^1.1.3 version: 1.1.3 @@ -55,7 +52,7 @@ importers: version: 2.10.5(@types/node@20.19.11)(typescript@5.9.2) tsup: specifier: ^8.5.0 - version: 8.5.0(tsx@3.14.0)(typescript@5.9.2) + version: 8.5.0(postcss@8.5.6)(tsx@3.14.0)(typescript@5.9.2)(yaml@2.7.0) tsx: specifier: ^3.14.0 version: 3.14.0 @@ -64,13 +61,16 @@ importers: version: 0.25.13(typescript@5.9.2) typedoc-plugin-markdown: specifier: ^3.17.1 - version: 3.17.1(typedoc@0.25.13) + version: 3.17.1(typedoc@0.25.13(typescript@5.9.2)) typescript: specifier: ^5.9.2 version: 5.9.2 vitest: specifier: ^2.1.9 - version: 2.1.9(@types/node@20.19.11)(msw@2.10.5) + version: 2.1.9(@types/node@20.19.11)(msw@2.10.5(@types/node@20.19.11)(typescript@5.9.2)) + zod: + specifier: 3.25.67 + version: 3.25.67 evals: dependencies: @@ -81,15 +81,15 @@ importers: specifier: ^0.0.140 version: 0.0.140 zod: - specifier: ^3.22.4 - version: 3.24.2 + specifier: ^4.2.1 + version: 4.2.1 devDependencies: "@types/node": specifier: ^20.10.5 version: 20.17.24 duckdb: specifier: ^1.0.0 - version: 1.2.0 + version: 1.2.0(encoding@0.1.13) tsx: specifier: ^3.14.0 version: 3.14.0 @@ -2418,10 +2418,10 @@ packages: integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==, } - openai@6.3.0: + openai@6.15.0: resolution: { - integrity: sha512-E6vOGtZvdcb4yXQ5jXvDlUG599OhIkb/GjBLZXS+qk0HF+PJReIldEc9hM8Ft81vn+N6dRdFRb7BZNK8bbvXrw==, + integrity: sha512-F1Lvs5BoVvmZtzkUEVyh8mDQPPFolq4F+xdsx/DO8Hee8YF3IGAlZqUIsF+DVGhqf4aU0a3bTghsxB6OIsRy1g==, } hasBin: true peerDependencies: @@ -3349,24 +3349,24 @@ packages: } engines: { node: ">=18" } - zod-to-json-schema@3.24.6: + zod-to-json-schema@3.25.0: resolution: { - integrity: sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==, + integrity: sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==, } peerDependencies: - zod: ^3.24.1 + zod: ^3.25 || ^4 - zod@3.24.2: + zod@3.25.67: resolution: { - integrity: sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==, + integrity: sha512-idA2YXwpCdqUSKRCACDE6ItZD9TZzy3OZMtpfLoh6oPR47lipysRrJfjzMqFxQ3uJuUPyUeWe1r9vLH33xO/Qw==, } - zod@3.25.76: + zod@4.2.1: resolution: { - integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==, + integrity: sha512-0wZ1IRqGGhMP76gLqz8EyfBXKk0J2qo2+H3fi4mcUP/KtTocoX08nmIAHl1Z2kJIZbZee8KOpBCSNPRgauucjw==, } snapshots: @@ -3374,16 +3374,16 @@ snapshots: dependencies: json-schema: 0.4.0 - "@asteasolutions/zod-to-openapi@6.4.0(zod@3.24.2)": + "@asteasolutions/zod-to-openapi@6.4.0(zod@3.25.67)": dependencies: openapi3-ts: 4.4.0 - zod: 3.24.2 + zod: 3.25.67 "@braintrust/core@0.0.44": dependencies: - "@asteasolutions/zod-to-openapi": 6.4.0(zod@3.24.2) + "@asteasolutions/zod-to-openapi": 6.4.0(zod@3.25.67) uuid: 9.0.1 - zod: 3.24.2 + zod: 3.25.67 "@bundled-es-modules/cookie@2.0.1": dependencies: @@ -3617,24 +3617,26 @@ snapshots: dependencies: "@inquirer/core": 10.1.15(@types/node@20.19.11) "@inquirer/type": 3.0.8(@types/node@20.19.11) + optionalDependencies: "@types/node": 20.19.11 "@inquirer/core@10.1.15(@types/node@20.19.11)": dependencies: "@inquirer/figures": 1.0.13 "@inquirer/type": 3.0.8(@types/node@20.19.11) - "@types/node": 20.19.11 ansi-escapes: 4.3.2 cli-width: 4.1.0 mute-stream: 2.0.0 signal-exit: 4.1.0 wrap-ansi: 6.2.0 yoctocolors-cjs: 2.1.2 + optionalDependencies: + "@types/node": 20.19.11 "@inquirer/figures@1.0.13": {} "@inquirer/type@3.0.8(@types/node@20.19.11)": - dependencies: + optionalDependencies: "@types/node": 20.19.11 "@isaacs/cliui@8.0.2": @@ -3672,12 +3674,12 @@ snapshots: "@kwsites/promise-deferred@1.1.1": {} - "@mapbox/node-pre-gyp@2.0.0": + "@mapbox/node-pre-gyp@2.0.0(encoding@0.1.13)": dependencies: consola: 3.4.1 detect-libc: 2.0.3 https-proxy-agent: 7.0.6 - node-fetch: 2.7.0 + node-fetch: 2.7.0(encoding@0.1.13) nopt: 8.1.0 semver: 7.7.1 tar: 7.4.3 @@ -3718,17 +3720,21 @@ snapshots: "@pkgjs/parseargs@0.11.0": optional: true - "@rollup/plugin-yaml@4.1.2": + "@rollup/plugin-yaml@4.1.2(rollup@4.46.4)": dependencies: - "@rollup/pluginutils": 5.2.0 + "@rollup/pluginutils": 5.2.0(rollup@4.46.4) js-yaml: 4.1.0 tosource: 2.0.0-alpha.3 + optionalDependencies: + rollup: 4.46.4 - "@rollup/pluginutils@5.2.0": + "@rollup/pluginutils@5.2.0(rollup@4.46.4)": dependencies: "@types/estree": 1.0.8 estree-walker: 2.0.2 picomatch: 4.0.3 + optionalDependencies: + rollup: 4.46.4 "@rollup/rollup-android-arm-eabi@4.46.4": optional: true @@ -3821,11 +3827,12 @@ snapshots: chai: 5.3.1 tinyrainbow: 1.2.0 - "@vitest/mocker@2.1.9(msw@2.10.5)(vite@5.4.19)": + "@vitest/mocker@2.1.9(msw@2.10.5(@types/node@20.19.11)(typescript@5.9.2))(vite@5.4.19(@types/node@20.19.11))": dependencies: "@vitest/spy": 2.1.9 estree-walker: 3.0.3 magic-string: 0.30.17 + optionalDependencies: msw: 2.10.5(@types/node@20.19.11)(typescript@5.9.2) vite: 5.4.19(@types/node@20.19.11) @@ -3946,7 +3953,7 @@ snapshots: pluralize: 8.0.0 simple-git: 3.27.0 uuid: 9.0.1 - zod: 3.24.2 + zod: 3.25.67 transitivePeerDependencies: - supports-color @@ -4082,9 +4089,9 @@ snapshots: dotenv@16.4.7: {} - duckdb@1.2.0: + duckdb@1.2.0(encoding@0.1.13): dependencies: - "@mapbox/node-pre-gyp": 2.0.0 + "@mapbox/node-pre-gyp": 2.0.0(encoding@0.1.13) node-addon-api: 7.1.1 node-gyp: 9.4.1 transitivePeerDependencies: @@ -4206,7 +4213,7 @@ snapshots: fast-uri@3.0.6: {} fdir@6.5.0(picomatch@4.0.3): - dependencies: + optionalDependencies: picomatch: 4.0.3 fft.js@4.0.4: {} @@ -4547,8 +4554,9 @@ snapshots: picocolors: 1.1.1 strict-event-emitter: 0.5.1 type-fest: 4.41.0 - typescript: 5.9.2 yargs: 17.7.2 + optionalDependencies: + typescript: 5.9.2 transitivePeerDependencies: - "@types/node" @@ -4570,9 +4578,11 @@ snapshots: node-addon-api@7.1.1: {} - node-fetch@2.7.0: + node-fetch@2.7.0(encoding@0.1.13): dependencies: whatwg-url: 5.0.0 + optionalDependencies: + encoding: 0.1.13 node-gyp@9.4.1: dependencies: @@ -4612,9 +4622,9 @@ snapshots: dependencies: wrappy: 1.0.2 - openai@6.3.0(zod@3.25.76): - dependencies: - zod: 3.25.76 + openai@6.15.0(zod@3.25.67): + optionalDependencies: + zod: 3.25.67 openapi3-ts@4.4.0: dependencies: @@ -4659,10 +4669,13 @@ snapshots: pluralize@8.0.0: {} - postcss-load-config@6.0.1(tsx@3.14.0): + postcss-load-config@6.0.1(postcss@8.5.6)(tsx@3.14.0)(yaml@2.7.0): dependencies: lilconfig: 3.1.3 + optionalDependencies: + postcss: 8.5.6 tsx: 3.14.0 + yaml: 2.7.0 postcss@8.5.6: dependencies: @@ -4915,7 +4928,7 @@ snapshots: ts-interface-checker@0.1.13: {} - tsup@8.5.0(tsx@3.14.0)(typescript@5.9.2): + tsup@8.5.0(postcss@8.5.6)(tsx@3.14.0)(typescript@5.9.2)(yaml@2.7.0): dependencies: bundle-require: 5.1.0(esbuild@0.25.9) cac: 6.7.14 @@ -4926,7 +4939,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(tsx@3.14.0) + postcss-load-config: 6.0.1(postcss@8.5.6)(tsx@3.14.0)(yaml@2.7.0) resolve-from: 5.0.0 rollup: 4.46.4 source-map: 0.8.0-beta.0 @@ -4934,6 +4947,8 @@ snapshots: tinyexec: 0.3.2 tinyglobby: 0.2.14 tree-kill: 1.2.2 + optionalDependencies: + postcss: 8.5.6 typescript: 5.9.2 transitivePeerDependencies: - jiti @@ -4953,7 +4968,7 @@ snapshots: type-fest@4.41.0: {} - typedoc-plugin-markdown@3.17.1(typedoc@0.25.13): + typedoc-plugin-markdown@3.17.1(typedoc@0.25.13(typescript@5.9.2)): dependencies: handlebars: 4.7.8 typedoc: 0.25.13(typescript@5.9.2) @@ -5020,18 +5035,17 @@ snapshots: vite@5.4.19(@types/node@20.19.11): dependencies: - "@types/node": 20.19.11 esbuild: 0.21.5 postcss: 8.5.6 rollup: 4.46.4 optionalDependencies: + "@types/node": 20.19.11 fsevents: 2.3.3 - vitest@2.1.9(@types/node@20.19.11)(msw@2.10.5): + vitest@2.1.9(@types/node@20.19.11)(msw@2.10.5(@types/node@20.19.11)(typescript@5.9.2)): dependencies: - "@types/node": 20.19.11 "@vitest/expect": 2.1.9 - "@vitest/mocker": 2.1.9(msw@2.10.5)(vite@5.4.19) + "@vitest/mocker": 2.1.9(msw@2.10.5(@types/node@20.19.11)(typescript@5.9.2))(vite@5.4.19(@types/node@20.19.11)) "@vitest/pretty-format": 2.1.9 "@vitest/runner": 2.1.9 "@vitest/snapshot": 2.1.9 @@ -5050,6 +5064,8 @@ snapshots: vite: 5.4.19(@types/node@20.19.11) vite-node: 2.1.9(@types/node@20.19.11) why-is-node-running: 2.3.0 + optionalDependencies: + "@types/node": 20.19.11 transitivePeerDependencies: - less - lightningcss @@ -5137,10 +5153,10 @@ snapshots: yoctocolors-cjs@2.1.2: {} - zod-to-json-schema@3.24.6(zod@3.25.76): + zod-to-json-schema@3.25.0(zod@3.25.67): dependencies: - zod: 3.25.76 + zod: 3.25.67 - zod@3.24.2: {} + zod@3.25.67: {} - zod@3.25.76: {} + zod@4.2.1: {}