add to inspect (#1065)

* adding lcb to inspect-ai

* adding lcb

* adds aime

* add ifeval french

* move prompts to respective eval file

* rename mmlu prompt functoin

* move prompt function to task files

* move prompt function to task files

* prompt function names end with _prompt

* fix missing

* prompt function names end with _prompt

* remove LETTER_INDEX

* fix tests

* revert lcb

* add to inspect

* add mixeval and musr

* Apply suggestion from @NathanHB

* Apply style fixes

* adding tasks to inspect

* adding tasks to inspect

* adds tasks that are defined in inspect

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: NathanHB

src/lighteval/main_inspect.py

@@ -53,6 +53,11 @@ def get_inspect_ai_task(
     name = lighteval_task_config.name
     sample_fields = lighteval_task_config.sample_fields
 
+    if sample_fields is None:
+        raise ValueError(
+            f"Task {name} is not supported by inspect_ai yet. You can either define it or use a different backend, `lighteval --help`"
+        )
+
     dataset_repo = lighteval_task_config.hf_repo
     dataset_subset = lighteval_task_config.hf_subset
     dataset_split = lighteval_task_config.evaluation_splits[0]
@@ -528,12 +533,36 @@ def bundle(log_dir: str, output_dir: str, overwrite: bool = True, repo_id: str |
 
 
 if __name__ == "__main__":
-    task = "lighteval|gsm8k|5,lighteval|gsm8k|1,lighteval|gsm8k|0"
-    task = "lighteval|agieval|0"
-    task = "lighteval|hle|0"
-    task = "lighteval|ifeval|0"
-    task = "lighteval|gpqa|0"
-    task = "lighteval|ifbench_test|0"
-    task = "lighteval|mmlu_pro|0"
+    tasks = [
+        "gsm8k",
+        "agieval",
+        "hle",
+        "ifeval",
+        "gpqa",
+        "ifbench_test",
+        "mmlu_pro",
+        "mixeval",
+        "aimo",
+        "anli",
+        "arc",
+        "arithmetic",
+        "asdiv",
+        "babi_qa",
+        "bbq",
+        "bigbench",
+        "bigbench_hard",
+        "blimp",
+        "bold",
+        "boolq",
+        "civil_comments",
+        "commonsenseqa",
+        "covid_dialog",
+        "dyck_language",
+        "math_500",
+        "musr",
+        "olympiad_bench",
+        "simpleqa",
+        "tiny_benchmarks",
+    ]
     model = "hf-inference-providers/meta-llama/Llama-3.1-8B-Instruct:nebius"
     eval(models=[model], tasks=task)

src/lighteval/tasks/tasks/aimo.py

@@ -17,7 +17,10 @@
 paper:
 """
 
-from lighteval.metrics.metrics import Metrics
+from inspect_ai.dataset import Sample
+from inspect_ai.solver import generate
+
+from lighteval.metrics.metrics import Metrics, math_scorer
 from lighteval.metrics.normalizations import math_normalizer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
@@ -32,9 +35,16 @@ def aimo_prompt(line, task_name: str = None):
     )
 
 
+def record_to_sample(record):
+    return Sample(input=record["problem"], target=str(record["answer"]))
+
+
 task = LightevalTaskConfig(
     name="aimo_progress_prize_1",
     prompt_function=aimo_prompt,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
     hf_subset="",
     hf_repo="lighteval/aimo_progress_prize_1",
     hf_avail_splits=["train"],

src/lighteval/tasks/tasks/anli.py

@@ -22,6 +22,12 @@
 https://arxiv.org/abs/1910.14599
 """
 
+from string import ascii_uppercase
+
+from inspect_ai.dataset import Sample
+from inspect_ai.scorer import choice
+from inspect_ai.solver import multiple_choice
+
 from lighteval.metrics.metrics import Metrics
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
@@ -36,6 +42,12 @@ def anli_prompt(line, task_name: str = None):
     )
 
 
+def record_to_sample(record):
+    choices = ["True", "Neither", "False"]
+    query = f"{record['premise']}\nQuestion: {record['hypothesis']}"
+    return Sample(input=query, target=ascii_uppercase[record["label"]], choices=choices)
+
+
 anli_r1 = LightevalTaskConfig(
     name="anli:r1",
     prompt_function=anli_prompt,
@@ -49,6 +61,9 @@ def anli_prompt(line, task_name: str = None):
     metrics=[Metrics.loglikelihood_acc],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 
@@ -65,6 +80,9 @@ def anli_prompt(line, task_name: str = None):
     metrics=[Metrics.loglikelihood_acc],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 
@@ -81,6 +99,9 @@ def anli_prompt(line, task_name: str = None):
     metrics=[Metrics.loglikelihood_acc],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 TASKS_TABLE = [

src/lighteval/tasks/tasks/arc.py

@@ -22,6 +22,10 @@
 https://arxiv.org/abs/1803.05457
 """
 
+from inspect_ai.dataset import Sample
+from inspect_ai.scorer import choice
+from inspect_ai.solver import multiple_choice
+
 from lighteval.metrics.metrics import Metrics
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
@@ -36,6 +40,14 @@ def arc_prompt(line, task_name: str = None):
     )
 
 
+def record_to_sample(record):
+    query = record["question"].strip()
+    target = record["answerKey"]
+    choices = record["choices"]["text"]
+
+    return Sample(input=query, target=target, choices=choices)
+
+
 arc_challenge = LightevalTaskConfig(
     name="arc:challenge",
     prompt_function=arc_prompt,
@@ -51,6 +63,9 @@ def arc_prompt(line, task_name: str = None):
     ],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 arc_easy = LightevalTaskConfig(
@@ -68,6 +83,9 @@ def arc_prompt(line, task_name: str = None):
     ],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[multiple_choice(cache=True)],
+    scorer=choice(),
 )
 
 TASKS_TABLE = [arc_challenge, arc_easy]

src/lighteval/tasks/tasks/arithmetic.py

@@ -19,15 +19,25 @@
 https://arxiv.org/abs/2005.14165
 """
 
-from lighteval.metrics.metrics import Metrics
+from inspect_ai.dataset import Sample
+from inspect_ai.solver import generate
+
+from lighteval.metrics.metrics import Metrics, math_scorer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
 
+# TODO: convert dataset to parquet
+
+
 def arithmetic_prompt(line, task_name: str = None):
     return Doc(task_name=task_name, query=line["context"], choices=[line["completion"]], gold_index=[0])
 
 
+def record_to_sample(record):
+    return Sample(input=record["context"], target=record["completion"])
+
+
 arithmetic_1dc = LightevalTaskConfig(
     name="arithmetic:1dc",
     prompt_function=arithmetic_prompt,
@@ -41,6 +51,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_2da = LightevalTaskConfig(
@@ -56,6 +69,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_2dm = LightevalTaskConfig(
@@ -71,6 +87,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_2ds = LightevalTaskConfig(
@@ -86,6 +105,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_3da = LightevalTaskConfig(
@@ -101,6 +123,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_3ds = LightevalTaskConfig(
@@ -116,6 +141,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_4da = LightevalTaskConfig(
@@ -131,6 +159,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_4ds = LightevalTaskConfig(
@@ -146,6 +177,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_5da = LightevalTaskConfig(
@@ -161,6 +195,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 arithmetic_5ds = LightevalTaskConfig(
@@ -176,6 +213,9 @@ def arithmetic_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 TASKS_TABLE = [

src/lighteval/tasks/tasks/asdiv.py

@@ -19,7 +19,10 @@
 https://arxiv.org/abs/2410.12853
 """
 
-from lighteval.metrics.metrics import Metrics
+from inspect_ai.dataset import Sample
+from inspect_ai.solver import generate
+
+from lighteval.metrics.metrics import Metrics, math_scorer
 from lighteval.tasks.lighteval_task import LightevalTaskConfig
 from lighteval.tasks.requests import Doc
 
@@ -33,6 +36,12 @@ def asdiv_prompt(line, task_name: str = None):
     )
 
 
+def record_to_sample(record):
+    query = f"{record['body']}\n{record['question']}"
+    target = record["answer"].split(" (")[0]
+    return Sample(input=query, target=target)
+
+
 asdiv = LightevalTaskConfig(
     name="asdiv",
     prompt_function=asdiv_prompt,
@@ -46,6 +55,9 @@ def asdiv_prompt(line, task_name: str = None):
     metrics=[Metrics.exact_match],
     stop_sequence=["\n"],
     version=0,
+    sample_fields=record_to_sample,
+    solver=[generate(cache=True)],
+    scorer=math_scorer(),
 )
 
 TASKS_TABLE = [asdiv]

src/lighteval/tasks/tasks/babi_qa.py

@@ -26,6 +26,9 @@
 from lighteval.tasks.requests import Doc
 
 
+# TODO: clean dataset and convert to inspect-ai
+
+
 def babi_qa_prompt(line, task_name: str = None):
     def process_path(path: str) -> str:
         steps = path.split(",")