feat: add trust check

Author: terryyz

bigcodebench/evaluate.py

@@ -49,7 +49,7 @@ def get_groundtruth(problems, hashcode, check_gt_only, max_as_limit, max_data_li
     tbegin = time.time()
     expected_time = {}
     for task_id, problem in tqdm(problems.items()):
-        expected_time[task_id] = trusted_exec(
+        expected_time[task_id] = trusted_check(
             problem["complete_prompt"] + "\n" + problem["canonical_solution"],
             problem["test"],
             problem["task_id"],

bigcodebench/gen/util/__init__.py

@@ -2,7 +2,8 @@
 import sys
 import types
 import unittest
-
+import multiprocessing
+from multiprocessing import Array, Value, Manager
 from bigcodebench.eval.utils import (
     create_tempdir,
     reliability_guard,
@@ -12,7 +13,7 @@
 )
 
 
-def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_stack_limit):
+def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_stack_limit, times):
     """Execute trusted code in place."""
 
     with create_tempdir():
@@ -51,15 +52,17 @@ def trusted_exec(code, test_code, task_id, max_as_limit, max_data_limit, max_sta
         start = time.time()
         with safe_environment(), swallow_io(), time_limit(seconds=120):
             suite.run(test_result)
+
+        if len(test_result.failures + test_result.errors) > 0:
+            times.value = -1
+        else:
+            times.value = time.time() - start
 
         # Needed for cleaning up.
         shutil.rmtree = rmtree
         os.rmdir = rmdir
         os.chdir = chdir
 
-        if len(test_result.failures + test_result.errors) > 0:
-            return None
-        return time.time() - start
 
 def trusted_check_exec(code, inputs):
     """Check trusted_exec success."""
@@ -69,3 +72,45 @@ def trusted_check_exec(code, inputs):
     except Exception:
         return False
     return True
+
+
+def trusted_check(
+    code: str,
+    test_code: str,
+    task_id: str,
+    max_as_limit: float,
+    max_data_limit: float,
+    max_stack_limit: float,
+) -> Tuple[str, np.ndarray]:
+    timeout = os.getenv("BIGCODEBENCH_TIMEOUT_PER_TASK", 120) + 1
+    # shared memory objects
+    times = Value("i")
+    manager = Manager()
+
+    p = multiprocessing.Process(
+        target=trusted_exec,
+        args=(
+            code,
+            test_code,
+            task_id,
+            max_as_limit,
+            max_data_limit,
+            max_stack_limit,
+            times,
+        ),
+    )
+    p.start()
+    p.join(timeout=timeout+1)
+    if p.is_alive():
+        p.terminate()
+        time.sleep(0.1)
+    if p.is_alive():
+        p.kill()
+        time.sleep(0.1)
+
+    if times.value == -1:
+        times = -1
+    else:
+        times = times.value
+    
+    return times