fix: solution key

terryyz · terryyz · commit 229dbc2487ae · 2024-06-18T21:08:32.000+08:00
diff --git a/bigcodebench/evaluate.py b/bigcodebench/evaluate.py
@@ -50,7 +50,7 @@ def get_groundtruth(problems, hashcode, check_gt_only):
     expected_time = {}
     for task_id, problem in tqdm(problems.items()):
         expected_time[task_id] = trusted_exec(
-            problem["complete_prompt"] + "\n" + problem["clean_canonical_solution"],
+            problem["complete_prompt"] + "\n" + problem["canonical_solution"],
             problem["test"],
             problem["task_id"],
         )

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ def get_groundtruth(problems, hashcode, check_gt_only):`
`50`	`50`	`expected_time = {}`
`51`	`51`	`for task_id, problem in tqdm(problems.items()):`
`52`	`52`	`expected_time[task_id] = trusted_exec(`
`53`		`- problem["complete_prompt"] + "\n" + problem["clean_canonical_solution"],`
	`53`	`+ problem["complete_prompt"] + "\n" + problem["canonical_solution"],`
`54`	`54`	`problem["test"],`
`55`	`55`	`problem["task_id"],`
`56`	`56`	`)`