diff --git a/benchmarks/arteval_bench/data/benchmark/arteval_tasks.jsonl b/benchmarks/arteval_bench/data/benchmark/arteval_tasks.jsonl index 4d80fe7..d3ad158 100644 --- a/benchmarks/arteval_bench/data/benchmark/arteval_tasks.jsonl +++ b/benchmarks/arteval_bench/data/benchmark/arteval_tasks.jsonl @@ -1,3 +1,4 @@ {"artifact_id": "sosp24_wasabi", "artifact_dir": "sosp24_wasabi", "artifact_readme": "sosp24_wasabi/wasabi/README.md", "artifact_url": "https://github.com/bastoica/wasabi/tree/sosp24-ae", "evaluator": "sosp24_wasabi/wasabi/_agent_eval/main.py", "expected_score": 4, "docer_env": "bastoica/ae-agent-ubuntu24.04:latest"} -{"artifact_id": "osdi24_anvil", "artifact_dir": "osdi24_anvil", "artifact_readme": "sosp23_acto/acto/README.md", "artifact_url": "https://github.com/anvil-verifier/anvil", "evaluator": "osdi24_anvil/anvil/_agent_eval/main.py", "expected_score": 4, "docer_env": "bastoica/ae-agent-ubuntu24.04:latest"} -{"artifact_id": "sosp23_acto", "artifact_dir": "sosp23_acto", "artifact_readme": "sosp23_acto/acto/README.md", "artifact_url": "https://github.com/xlab-uiuc/acto", "evaluator": "sosp23_acto/acto/_agent_eval/main.py", "expected_score": 4, "docer_env": "bastoica/ae-agent-ubuntu24.04:latest"} \ No newline at end of file +{"artifact_id": "osdi24_anvil", "artifact_dir": "osdi24_anvil", "artifact_readme": "osdi24_anvil/anvil/README.md", "artifact_url": "https://github.com/anvil-verifier/anvil", "evaluator": "osdi24_anvil/_agent_eval/main.py", "expected_score": 4, "docer_env": "bastoica/ae-agent-ubuntu24.04:latest"} +{"artifact_id": "sosp23_acto", "artifact_dir": "sosp23_acto", "artifact_readme": "sosp23_acto/acto/README.md", "artifact_url": "https://github.com/xlab-uiuc/acto", "evaluator": "sosp23_acto/_agent_eval/main.py", "expected_score": 4, "docer_env": "bastoica/ae-agent-ubuntu24.04:latest"} +{"artifact_id": "eurosys25_egwalker", "artifact_dir": "eurosys25_egwalker", "artifact_readme": "eurosys25_egwalker/egwalker/README.md", "artifact_url": "https://github.com/josephg/egwalker-paper", "evaluator": "eurosys25_egwalker/_agent_eval/main.py", "expected_score": 4, "docer_env": "bastoica/ae-agent-ubuntu24.04:latest"} \ No newline at end of file diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/main.py b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/main.py new file mode 100644 index 0000000..b67e39a --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/main.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import sys +from typing import Dict + +# from oracle_artifact_build import OracleArtifactBuild +from oracle_env_setup import OracleEnvSetup +# from oracle_benchmark_prep import OracleBenchmarkPrep +# from oracle_experiment_runs import OracleExperimentRuns + +from utils import logger + +def main(): + results: Dict[str, int] = {} + + score = 0 + for cls in (OracleEnvSetup, OracleArtifactBuild, OracleBenchmarkPrep, OracleExperimentRuns): + checker = cls() + ok = checker.run() + name = cls.__name__ + logger.info(f"{name}: {'PASS' if ok else 'FAIL'}") + if ok: + results[name] = 1 + score += 1 + else: + results[name] = 0 + + logger.info(f"Agent scores: {results}") + return score + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_artifact_build.py b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_artifact_build.py new file mode 100644 index 0000000..462a2d9 --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_artifact_build.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +import os +import subprocess +from dataclasses import dataclass +from typing import Iterable, List, Optional, Tuple +from pathlib import Path + +from utils import REPO_DIR +from utils import logger + + +@dataclass(frozen=True) +class BuildTarget: + name: str + repo_key: str + cmd: List[str] + + +BUILD_TARGETS: List[BuildTarget] = [ + BuildTarget( + name="artifact-core", + repo_key="artifact-core", + cmd=[ + "make", + "-j8", + "tools/diamond-types/target/release/dt", + "tools/crdt-converter/target/release/crdt-converter", + "tools/diamond-types/target/release/paper-stats", + "tools/paper-benchmarks/target/memusage/paper-benchmarks", + "tools/paper-benchmarks/target/release/paper-benchmarks", + "tools/ot-bench/target/memusage/ot-bench", + "tools/ot-bench/target/release/ot-bench" + ], + ), +] + + +class OracleArtifactBuild: + + def __init__(self) -> None: + self.repo_dir = REPO_DIR + + def run_shell_command( + self, + cmd: Iterable[str], + cwd: Optional[Path] = None, + ) -> Tuple[int, str, str]: + """ + Run a command and return (rc, stdout, stderr) tuple. + """ + try: + cp = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=str(cwd) if cwd is not None else None, + ) + return cp.returncode, cp.stdout or "", cp.stderr or "" + except FileNotFoundError: + return 127, "", "" + + def build_target(self, target: BuildTarget) -> Optional[str]: + """ + Build a single target using its configured repository and command. + """ + repo_path = Path(os.path.expanduser(self.repo_dir)) + if not repo_path.exists(): + return f"{target.name} repo directory missing" + + rc, out, err = self.run_shell_command(target.cmd, cwd=repo_path) + if rc != 0: + return f"{target.name} build failed (error code: {rc}; error message: {err})" + + return None + + def build_check(self): + """ + Run builds for all configured targets and collect failures. + """ + problems: List[str] = [] + for target in BUILD_TARGETS: + msg = self.build_target(target) + if msg: + problems.append(msg) + if problems: + return False, "; ".join(problems) + return True, "" + + def run(self): + ok, why = self.build_check() + logger.info(f"Build: {'PASS' if ok else 'FAIL' + (' - ' + why if why else '')}") + return ok \ No newline at end of file diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_benchmark_prep.py b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_benchmark_prep.py new file mode 100644 index 0000000..310f7ab --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_benchmark_prep.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, List, Optional, Tuple + +from utils import HOME +from utils import REPO_DIR +from utils import REFERENCE_BENCHMARK_FILE +from utils import logger + + +@dataclass(frozen=True) +class DatasetRef: + filepath: str + sizeinbytes: int + + +class OracleBenchmarkPrep: + + def __init__(self) -> None: + self.home = Path(os.path.expanduser(str(HOME))) + self.repo_path = Path(os.path.expanduser(str(REPO_DIR))) + self.ref_json = Path(os.path.expanduser(str(REFERENCE_BENCHMARK_FILE))) + + def load_json(self, path: Path) -> Tuple[Optional[Any], str]: + """ + Load JSON from disk and return (obj, err). + """ + if not path.exists(): + return None, f"ref json missing: {path}" + try: + with path.open("r", encoding="utf-8") as f: + return json.load(f), "" + except Exception as e: + return None, f"ref json unreadable: {e}" + + def iter_ref_entries(self, obj: Any) -> List[dict]: + """ + Extract benchmark entries from a reference JSON. + """ + if isinstance(obj, list): + return [x for x in obj if isinstance(x, dict)] + if isinstance(obj, dict): + for v in obj.values(): + if isinstance(v, list) and v and all(isinstance(x, dict) for x in v): + return v + return [] + + def parse_entry(self, d: dict) -> Tuple[Optional[DatasetRef], str]: + """ + Parse a single JSON entry into DatasetRef. + """ + if "filepath" not in d: + return None, "missing filepath" + if "sizeinbytes" not in d: + return None, "missing sizeinbytes" + + fp = d.get("filepath", "") + sz = d.get("sizeinbytes", None) + + if not isinstance(fp, str) or not fp: + return None, "invalid filepath" + if not isinstance(sz, int) or sz < 0: + return None, "invalid sizeinbytes" + + return DatasetRef(filepath=fp, sizeinbytes=sz), "" + + def check_entry(self, ref: DatasetRef) -> Optional[str]: + """ + Validate that dataset files exist and matche the expected sizes (in bytes). + """ + rel = Path(ref.filepath) + + if rel.is_absolute(): + return f"{ref.filepath}: absolute paths not allowed" + + p = self.repo_path / rel + if not p.exists(): + return f"{ref.filepath}: missing" + if not p.is_file(): + return f"{ref.filepath}: not a file" + + try: + actual = p.stat().st_size + except OSError as e: + return f"{ref.filepath}: stat failed ({e})" + + if actual != ref.sizeinbytes: + return f"{ref.filepath}: size mismatch (expected {ref.sizeinbytes}, got {actual})" + + return None + + def datasets_check(self) -> Tuple[bool, str]: + """ + Check all referenced dataset files are present and match expected sizes. + """ + obj, err = self.load_json(self.ref_json) + if err: + return False, err + + entries = self.iter_ref_entries(obj) + if not entries: + return False, "no entries found in ref json" + + problems: List[str] = [] + for d in entries: + ref, perr = self.parse_entry(d) + if perr or ref is None: + problems.append(perr or "invalid entry") + continue + + msg = self.check_entry(ref) + if msg: + problems.append(msg) + + if problems: + return False, "; ".join(problems) + return True, "" + + def run(self) -> bool: + ok, why = self.datasets_check() + logger.info(f"Datasets: {'PASS' if ok else 'FAIL' + (' - ' + why if why else '')}") + return ok \ No newline at end of file diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_env_setup.py b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_env_setup.py new file mode 100644 index 0000000..028d5f2 --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_env_setup.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +import subprocess +import re +from dataclasses import dataclass +from typing import Iterable, Optional, Tuple +from pathlib import Path + +from utils import logger + + +Version = Tuple[int, int, int] + + +@dataclass(frozen=True) +class ToolRequirement: + name: str + cmd: list[str] + min_version: Optional[Version] = None + optional: bool = False + + +MIN_RUST_VERSION: Version = (1, 78, 0) + + +TOOL_REQUIREMENTS: list[ToolRequirement] = [ + ToolRequirement( + name="rustc", + cmd=["rustc", "--version"], + min_version=MIN_RUST_VERSION, + ), + ToolRequirement( + name="cargo", + cmd=["cargo", "--version"], + ), + ToolRequirement( + name="node", + cmd=["node", "--version"], + ), + ToolRequirement( + name="make", + cmd=["make", "--version"], + optional=True, + ), +] + + +class OracleEnvSetup: + + def run_shell_command( + self, + cmd: Iterable[str], + cwd: Optional[Path] = None, + ) -> Tuple[int, str, str]: + """ + Run a command and return (rc, stdout, stderr) tuple. + """ + try: + cp = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=str(cwd) if cwd is not None else None, + ) + return cp.returncode, cp.stdout or "", cp.stderr or "" + except FileNotFoundError: + return 127, "", "" + + def parse_version(self, s: str) -> Optional[Version]: + """ + Extract a version number from a string. + """ + m = re.search(r"(?:^|\s)v?(\d+)\.(\d+)(?:\.(\d+))?", s) + if not m: + return None + major = int(m.group(1)) + minor = int(m.group(2)) + patch = int(m.group(3)) if m.group(3) is not None else 0 + return (major, minor, patch) + + def version_lt(self, a: Version, b: Version) -> bool: + return a < b + + def check_tool(self, req: ToolRequirement) -> Tuple[Optional[str], Optional[str]]: + """ + Check a single dependency requirement, including version. + """ + rc, out, err = self.run_shell_command(req.cmd) + combined = (out + "\n" + err).strip() + + if rc == 127: + if req.optional: + return None, f"{req.name} missing (optional)" + return f"{req.name} not found", None + + if rc != 0: + if req.optional: + return None, f"{req.name} check failed (rc={rc}) (optional)" + return f"{req.name} check failed (rc={rc})", None + + if req.min_version is not None: + v = self.parse_version(combined) + if v is None: + return f"{req.name} version parse failed", None + if self.version_lt(v, req.min_version): + return f"{req.name} too old (need >= {req.min_version[0]}.{req.min_version[1]}.{req.min_version[2]})", None + + return None, None + + def build_check(self): + """ + Validate required dependnecies and environment setup. + """ + problems: list[str] = [] + warnings: list[str] = [] + + for req in TOOL_REQUIREMENTS: + problem, warning = self.check_tool(req) + if problem: + problems.append(problem) + if warning: + warnings.append(warning) + + if problems: + return False, "; ".join(problems) + + if warnings: + return True, "WARN: " + "; ".join(warnings) + + return True, "" + + def run(self): + ok, why = self.build_check() + label = "Environment" + if ok and why: + logger.info(f"{label}: PASS - {why}") + return ok + logger.info(f"{label}: {'PASS' if ok else 'FAIL' + (' - ' + why if why else '')}") + return ok \ No newline at end of file diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_experiment_runs.py b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_experiment_runs.py new file mode 100644 index 0000000..b4e2d70 --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/oracle_experiment_runs.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +import json +import os +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from utils import REPO_DIR +from utils import REFERENCE_RESULTS_FILE +from utils import SIMILARITY_RATIO +from utils import logger + + +class OracleExperimentRuns: + def __init__(self) -> None: + self.repo_dir = Path(os.path.expanduser(str(REPO_DIR))) + self.timings_file = self.repo_dir / "results" / "timings.json" + self.reference_file = Path(os.path.expanduser(str(REFERENCE_RESULTS_FILE))) + self.max_mismatches_to_report = (1 - SIMILARITY_RATIO) + + def load_json(self, path: Path) -> Tuple[Optional[Any], str]: + """ + Load JSON from disk and return (obj, err). + """ + if not path.exists(): + return None, f"missing json: {path}" + try: + with path.open("r", encoding="utf-8") as f: + return json.load(f), "" + except Exception as e: + return None, f"unreadable json: {path} ({e})" + + def as_float(self, v: Any) -> Optional[float]: + if isinstance(v, (int, float)): + return float(v) + return None + + def ratios_within_tolerance(self, actual: float, ref: float) -> Tuple[bool, float]: + """ + Check whether two measurements are within tolerance. + """ + if abs(ref) < 1e-12: + if abs(actual) < 1e-12: + return True, 0.0 + return False, float("inf") + + rel_diff = abs(actual - ref) / abs(ref) + return rel_diff <= (1.0 - float(SIMILARITY_RATIO)), rel_diff + + def compare_timings( + self, + actual: Dict[str, Any], + reference: Dict[str, Any], + ) -> Tuple[bool, str]: + """ + Compare current timings with the original, reference timings. + """ + if not isinstance(actual, dict) or not isinstance(reference, dict): + return False, "timings json invalid format (expected object at top-level)" + + missing: List[str] = [] + mismatches: List[str] = [] + total = 0 + ok_count = 0 + + for metric_name, ref_metric in reference.items(): + if not isinstance(ref_metric, dict): + missing.append(f"{metric_name}: invalid reference section (expected object)") + continue + + act_metric = actual.get(metric_name) + if not isinstance(act_metric, dict): + missing.append(f"{metric_name}: missing metric") + continue + + for tag, ref_stats in ref_metric.items(): + if not isinstance(ref_stats, dict): + missing.append(f"{metric_name}.{tag}: invalid reference tag (expected object)") + continue + + act_stats = act_metric.get(tag) + if not isinstance(act_stats, dict): + missing.append(f"{metric_name}.{tag}: missing tag") + continue + + for field, ref_val_raw in ref_stats.items(): + total += 1 + + if field not in act_stats: + missing.append(f"{metric_name}.{tag}.{field}: missing field") + continue + + ref_val = self.as_float(ref_val_raw) + act_val = self.as_float(act_stats.get(field)) + + if ref_val is None: + missing.append(f"{metric_name}.{tag}.{field}: non-numeric reference value") + continue + if act_val is None: + missing.append(f"{metric_name}.{tag}.{field}: non-numeric actual value") + continue + + ok, sim = self.ratios_within_tolerance(act_val, ref_val) + if ok: + ok_count += 1 + else: + mismatches.append( + f"{metric_name}.{tag}.{field}: {act_val} vs {ref_val} (similarity {sim:.3f} < {SIMILARITY_RATIO})" + ) + + if missing or mismatches: + parts: List[str] = [] + summary = f"{ok_count}/{total} fields meet similarity ratio" if total else "0 fields compared" + if missing: + parts.append("missing/invalid: " + "; ".join(missing)) + if mismatches: + parts.append("measurement difference: " + "; ".join(mismatches)) + return False, summary + " - " + " | ".join(parts) + + summary = f"{ok_count}/{total} fields meet similarity ratio" if total else "no reference fields to compare" + return True, summary + + def run(self) -> bool: + actual_obj, err = self.load_json(self.timings_file) + if err: + logger.info(f"Timings: FAIL - {err}") + return False + + ref_obj, err = self.load_json(self.reference_file) + if err: + logger.info(f"Timings: FAIL - {err}") + return False + + ok, why = self.compare_timings(actual_obj, ref_obj) + logger.info(f"Timings: {'PASS' if ok else 'FAIL' + (' - ' + why if why else '')}") + return ok diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/refs/datasets.ref.json b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/refs/datasets.ref.json new file mode 100644 index 0000000..dad4419 --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/refs/datasets.ref.json @@ -0,0 +1,142 @@ +[ + { + "filepath": "datasets/A1-uncompressed.am", + "sizeinbytes": 1445627 + }, + { + "filepath": "datasets/A1.am", + "sizeinbytes": 772717 + }, + { + "filepath": "datasets/A1.dt", + "sizeinbytes": 334914 + }, + { + "filepath": "datasets/A1.json", + "sizeinbytes": 1353802 + }, + { + "filepath": "datasets/A1.yjs", + "sizeinbytes": 308418 + }, + { + "filepath": "datasets/A2-uncompressed.am", + "sizeinbytes": 1113502 + }, + { + "filepath": "datasets/A2.am", + "sizeinbytes": 572699 + }, + { + "filepath": "datasets/A2.dt", + "sizeinbytes": 318046 + }, + { + "filepath": "datasets/A2.json", + "sizeinbytes": 1513005 + }, + { + "filepath": "datasets/A2.yjs", + "sizeinbytes": 506207 + }, + { + "filepath": "datasets/C1-uncompressed.am", + "sizeinbytes": 1572344 + }, + { + "filepath": "datasets/C1.am", + "sizeinbytes": 136380 + }, + { + "filepath": "datasets/C1.dt", + "sizeinbytes": 466960 + }, + { + "filepath": "datasets/C1.json", + "sizeinbytes": 13811442 + }, + { + "filepath": "datasets/C1.yjs", + "sizeinbytes": 845328 + }, + { + "filepath": "datasets/C2-uncompressed.am", + "sizeinbytes": 1732126 + }, + { + "filepath": "datasets/C2.am", + "sizeinbytes": 206973 + }, + { + "filepath": "datasets/C2.dt", + "sizeinbytes": 676591 + }, + { + "filepath": "datasets/C2.json", + "sizeinbytes": 18956694 + }, + { + "filepath": "datasets/C2.yjs", + "sizeinbytes": 725734 + }, + { + "filepath": "datasets/S1-uncompressed.am", + "sizeinbytes": 877862 + }, + { + "filepath": "datasets/S1.am", + "sizeinbytes": 341839 + }, + { + "filepath": "datasets/S1.dt", + "sizeinbytes": 316413 + }, + { + "filepath": "datasets/S1.json", + "sizeinbytes": 3011880 + }, + { + "filepath": "datasets/S1.yjs", + "sizeinbytes": 479708 + }, + { + "filepath": "datasets/S2-uncompressed.am", + "sizeinbytes": 1173041 + }, + { + "filepath": "datasets/S2.am", + "sizeinbytes": 601328 + }, + { + "filepath": "datasets/S2.dt", + "sizeinbytes": 471421 + }, + { + "filepath": "datasets/S2.json", + "sizeinbytes": 1685725 + }, + { + "filepath": "datasets/S2.yjs", + "sizeinbytes": 405640 + }, + { + "filepath": "datasets/S3-uncompressed.am", + "sizeinbytes": 1905274 + }, + { + "filepath": "datasets/S3.am", + "sizeinbytes": 831539 + }, + { + "filepath": "datasets/S3.dt", + "sizeinbytes": 729169 + }, + { + "filepath": "datasets/S3.json", + "sizeinbytes": 2478019 + }, + { + "filepath": "datasets/S3.yjs", + "sizeinbytes": 317745 + } +] diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/refs/timings.ref.json b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/refs/timings.ref.json new file mode 100644 index 0000000..4a1e0fb --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/refs/timings.ref.json @@ -0,0 +1,354 @@ +{ + "dt_merge_norm": { + "S1": { + "mean": 1.765569771388292, + "p10": 1.7634046209090908, + "p90": 1.767358451978022, + "stddev": 0.0035584340677394816 + }, + "S2": { + "mean": 2.679764410588234, + "p10": 2.683664764705882, + "p90": 2.6774855411764706, + "stddev": 0.006751116324473548 + }, + "S3": { + "mean": 3.577152279333332, + "p10": 3.5652497399999996, + "p90": 3.573826073333333, + "stddev": 0.012676478619268544 + }, + "C1": { + "mean": 56.14389554000003, + "p10": 56.4988075, + "p90": 56.285378, + "stddev": 0.5087256032162999 + }, + "C2": { + "mean": 82.63611046, + "p10": 82.652672, + "p90": 82.7274149, + "stddev": 0.18079136579640193 + }, + "A1": { + "mean": 8.860465820000002, + "p10": 8.868881516666665, + "p90": 8.89078035, + "stddev": 0.024928422316188883 + }, + "A2": { + "mean": 23.470546336666665, + "p10": 23.452774033333334, + "p90": 23.431936, + "stddev": 0.024358764370572987 + } + }, + "dt_ff_off": { + "S1": { + "mean": 9.80900226, + "p10": 9.779692516666666, + "p90": 9.830549999999999, + "stddev": 0.016300405979697476 + }, + "S2": { + "mean": 17.09569275333333, + "p10": 17.1315493, + "p90": 17.106342666666666, + "stddev": 0.03845435642474562 + }, + "S3": { + "mean": 24.422856873333334, + "p10": 24.363427299999998, + "p90": 24.3898533, + "stddev": 0.04641843294561104 + }, + "C1": { + "mean": 69.78582748999999, + "p10": 69.93251000000001, + "p90": 69.70866, + "stddev": 0.28612502731811074 + }, + "C2": { + "mean": 95.37582798, + "p10": 95.4350589, + "p90": 95.433982, + "stddev": 0.24025054369615817 + }, + "A1": { + "mean": 23.946847216666676, + "p10": 24.046661666666665, + "p90": 23.997562733333332, + "stddev": 0.08946744675419208 + }, + "A2": { + "mean": 23.741934939999997, + "p10": 23.71631406666667, + "p90": 23.804853966666663, + "stddev": 0.07915570682674 + } + }, + "dt_opt_load": { + "S1": { + "mean": 0.0680419148998607, + "p10": 0.068029822, + "p90": 0.06795833973626372, + "stddev": 0.0005774366506787338 + }, + "S2": { + "mean": 0.03711389266785635, + "p10": 0.03699626912457912, + "p90": 0.037023959291819286, + "stddev": 0.00017199145178924718 + }, + "S3": { + "mean": 0.02878299671947925, + "p10": 0.028789752192513368, + "p90": 0.02880063897866839, + "stddev": 0.00002919116188466282 + }, + "C1": { + "mean": 0.12062605658576642, + "p10": 0.12036285525252524, + "p90": 0.1207069468009768, + "stddev": 0.00010982400989893558 + }, + "C2": { + "mean": 0.1120261903435016, + "p10": 0.11197791646464646, + "p90": 0.11204969639804638, + "stddev": 0.0000964347693773219 + }, + "A1": { + "mean": 0.011675564311794702, + "p10": 0.011741236203208556, + "p90": 0.01167711628442146, + "stddev": 0.000035924562781578904 + }, + "A2": { + "mean": 0.05249885432660977, + "p10": 0.052552646650717696, + "p90": 0.05249511281087334, + "stddev": 0.0000672882007538972 + } + }, + "dt-crdt_process_remote_edits": { + "S1": { + "mean": 17.87260023, + "p10": 17.89071933333333, + "p90": 17.832252999999998, + "stddev": 0.043475484546061954 + }, + "S2": { + "mean": 19.08492903000001, + "p10": 19.124460066666664, + "p90": 19.069909966666664, + "stddev": 0.04766111438497832 + }, + "S3": { + "mean": 26.88948144000002, + "p10": 26.9604711, + "p90": 26.951750999999998, + "stddev": 0.05729642375616823 + }, + "C1": { + "mean": 52.45371103999997, + "p10": 52.433468, + "p90": 52.73901219999999, + "stddev": 0.20021030918371385 + }, + "C2": { + "mean": 64.16709595999998, + "p10": 63.99304479999999, + "p90": 64.33029029999999, + "stddev": 0.17750798796414355 + }, + "A1": { + "mean": 42.650900715000006, + "p10": 42.70575745, + "p90": 42.84124599999999, + "stddev": 0.14850565750944086 + }, + "A2": { + "mean": 26.175318695000005, + "p10": 26.22467195, + "p90": 26.1155135, + "stddev": 0.07095799021343169 + } + }, + "automerge_remote": { + "S1": { + "mean": 620.3062693800001, + "p10": 620.1435849, + "p90": 619.6568222999999, + "stddev": 1.6810659181733942 + }, + "S2": { + "mean": 747.2433299099996, + "p10": 753.3979349, + "p90": 748.1610354999999, + "stddev": 3.135502884514345 + }, + "S3": { + "mean": 1444.166267079999, + "p10": 1443.9552891, + "p90": 1439.9426531, + "stddev": 5.608138176449289 + }, + "C1": { + "mean": 11848.025785339996, + "p10": 11775.7724094, + "p90": 11895.4857794, + "stddev": 42.25654285147792 + }, + "C2": { + "mean": 24636.277552770007, + "p10": 24572.3934124, + "p90": 24662.196266899995, + "stddev": 43.23906403665259 + }, + "A1": { + "mean": 484.5537476, + "p10": 479.14352149999996, + "p90": 482.50125410000004, + "stddev": 5.643829248672304 + }, + "A2": { + "mean": 519.8018028399999, + "p10": 519.0745423999999, + "p90": 520.4351099, + "stddev": 2.191345086833545 + } + }, + "yrs_remote": { + "S1": { + "mean": 8.058310392857145, + "p10": 8.044104428571428, + "p90": 8.055851428571428, + "stddev": 0.007071325984663515 + }, + "S2": { + "mean": 11.166586169999999, + "p10": 11.147594600000001, + "p90": 11.1682734, + "stddev": 0.011708391253536999 + }, + "S3": { + "mean": 9.469344053333334, + "p10": 9.459481833333333, + "p90": 9.457590333333332, + "stddev": 0.01725351818858463 + }, + "C1": { + "mean": 11.781803348000004, + "p10": 11.789621199999997, + "p90": 11.78669084, + "stddev": 0.0186948521303512 + }, + "C2": { + "mean": 8.693525539999996, + "p10": 8.518234816666665, + "p90": 8.714304316666666, + "stddev": 0.06465154549440037 + }, + "A1": { + "mean": 11.874572001999997, + "p10": 11.9125626, + "p90": 11.874057700000005, + "stddev": 0.057867157592226576 + }, + "A2": { + "mean": 12.798531739999998, + "p10": 12.692624499999999, + "p90": 12.70298175, + "stddev": 1.2566901261158037 + } + }, + "ot": { + "S1": { + "mean": 2.3530728677272728, + "p10": 2.3523336818181813, + "p90": 2.355160631818182, + "stddev": 0.005622618387914949 + }, + "S2": { + "mean": 2.840479363888889, + "p10": 2.842858444444444, + "p90": 2.841036383333333, + "stddev": 0.00709593979991191 + }, + "S3": { + "mean": 3.788857682857142, + "p10": 3.764989285714286, + "p90": 3.80589115, + "stddev": 0.014547852990969832 + }, + "C1": { + "mean": 365.49562632999994, + "p10": 364.63688809999996, + "p90": 364.9103132, + "stddev": 1.187256479110188 + }, + "C2": { + "mean": 378.4289899600001, + "p10": 378.328244, + "p90": 379.33611699999994, + "stddev": 0.9557334251883629 + }, + "A1": { + "mean": 6266.4576153000025, + "p10": 6268.2776376, + "p90": 6264.2641024, + "stddev": 4.190739209238828 + }, + "A2": { + "mean": 3664266.2923529, + "p10": 3665429.0829243003, + "p90": 3663200.2680154, + "stddev": 1143.9396128307087 + } + }, + "yjs_remote": { + "S1": { + "mean": 57.41284772514621, + "p10": 56.05801199999951, + "p90": 57.55393799999911, + "stddev": 3.4314588704714035 + }, + "S2": { + "mean": 85.18153684615375, + "p10": 84.64801260000021, + "p90": 87.09783660000171, + "stddev": 2.3329512798384475 + }, + "S3": { + "mean": 79.93072461290346, + "p10": 79.80155780000058, + "p90": 81.33051500000074, + "stddev": 3.5812436457468264 + }, + "C1": { + "mean": 84.10543327118617, + "p10": 85.66311869999772, + "p90": 86.53187949999628, + "stddev": 3.1030385448180624 + }, + "C2": { + "mean": 55.23782650555521, + "p10": 59.1130919000003, + "p90": 56.3074750999978, + "stddev": 3.352232916785458 + }, + "A1": { + "mean": 88.38225315929175, + "p10": 86.0255416, + "p90": 86.39550419998997, + "stddev": 3.3294846740889574 + }, + "A2": { + "mean": 74.19105031579002, + "p10": 74.78094500000589, + "p90": 78.25383420000725, + "stddev": 3.5853106831560817 + } + } +} \ No newline at end of file diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/utils.py b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/utils.py new file mode 100644 index 0000000..284ec89 --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/_agent_eval/utils.py @@ -0,0 +1,29 @@ +# --- CONSTANTS --- # +from pathlib import Path + +HOME = Path.home() / "eurosys25_egwalker" +REPO_DIR = f"{HOME}/egwalker" + +REFERENCE_BENCHMARK_FILE = f"{HOME}/_agent_eval/refs/datasets.ref.json" +REFERENCE_RESULTS_FILE = f"{HOME}/_agent_eval/refs/timings.ref.json" +SIMILARITY_RATIO = 0.75 + + +# --- CUSTOM LOGGER --- # +import logging +import os +from datetime import datetime + +os.makedirs('logs', exist_ok=True) + +LOG_FORMAT = '%(asctime)s | %(levelname)s | %(name)s | %(message)s' +DATE_FORMAT = '%Y-%m-%d %H:%M:%S' + +logger = logging.getLogger("OSDI24-ANVIL-AGENT-EVALUATOR") +logger.setLevel(logging.DEBUG) + +console_handler = logging.StreamHandler() +console_handler.setLevel(logging.INFO) +console_handler.setFormatter(logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT)) + +logger.addHandler(console_handler) \ No newline at end of file diff --git a/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/egwalker/ACM-Reference-Format.bst b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/egwalker/ACM-Reference-Format.bst new file mode 100644 index 0000000..c47cb4c --- /dev/null +++ b/benchmarks/arteval_bench/data/benchmark/eurosys25_egwalker/egwalker/ACM-Reference-Format.bst @@ -0,0 +1,3081 @@ +%%% -*-BibTeX-*- +%%% ==================================================================== +%%% @BibTeX-style-file{ +%%% author = "Nelson H. F. Beebe, Boris Veytsman and Gerald Murray", +%%% version = "2.1", +%%% acmart-version = "1.90", +%%% date = "Mar 26 2023", +%%% filename = "ACM-Reference-Format.bst", +%%% email = "borisv@lk.net, boris@varphi.com", +%%% codetable = "ISO/ASCII", +%%% keywords = "ACM Transactions bibliography style; BibTeX", +%%% license = "public domain", +%%% supported = "yes", +%%% abstract = "", +%%% } +%%% ==================================================================== + +%%% Revision history: see source in git + +ENTRY + { address + advisor + archiveprefix + author + booktitle + chapter + city + date + edition + editor + eprint + eprinttype + eprintclass + howpublished + institution + journal + key + location + month + note + number + organization + pages + primaryclass + publisher + school + series + title + type + volume + year + % New keys recognized + issue % UTAH: used in, e.g., ACM SIGSAM Bulletin and ACM Communications in Computer Algebra + articleno + eid + day % UTAH: needed for newspapers, weeklies, bi-weeklies + doi % UTAH + url % UTAH + bookpages % UTAH + numpages + lastaccessed % UTAH: used only for @Misc{...} + coden % UTAH + isbn % UTAH + isbn-13 % UTAH + issn % UTAH + lccn % UTAH + distinctURL % whether to print url if doi is present + } + {} + { label.year extra.label sort.year sort.label basic.label.year} + +INTEGERS { output.state before.all mid.sentence after.sentence after.block } + +INTEGERS { show-isbn-10-and-13 } % initialized below in begin.bib + +INTEGERS { nameptr namesleft numnames } + +INTEGERS { multiresult } + +INTEGERS { len } + +INTEGERS { last.extra.num } + +STRINGS { s t t.org u } + +STRINGS { last.label next.extra } + +STRINGS { p1 p2 p3 page.count } + + +FUNCTION { not } +{ + { #0 } + { #1 } + if$ +} + +FUNCTION { and } +{ + 'skip$ + { pop$ #0 } + if$ +} + +FUNCTION { or } +{ + { pop$ #1 } + 'skip$ + if$ +} + + +FUNCTION { dump.stack.1 } +{ + duplicate$ "STACK[top] = [" swap$ * "]" * warning$ +} + +FUNCTION { dump.stack.2 } +{ + duplicate$ "STACK[top ] = [" swap$ * "]" * warning$ + swap$ + duplicate$ "STACK[top-1] = [" swap$ * "]" * warning$ + swap$ +} + +FUNCTION { empty.or.unknown } +{ + %% Examine the top stack entry, and push 1 if it is empty, or + %% consists only of whitespace, or is a string beginning with two + %% queries (??), and otherwise, push 0. + %% + %% This function provides a replacement for empty$, with the + %% convenient feature that unknown values marked by two leading + %% queries are treated the same as missing values, and thus, do not + %% appear in the output .bbl file, and yet, their presence in .bib + %% file(s) serves to mark values which are temporarily missing, but + %% are expected to be filled in eventually once more data is + %% obtained. The TeX User Group and BibNet bibliography archives + %% make extensive use of this practice. + %% + %% An empty string cannot serve the same purpose, because just as in + %% statistics data processing, an unknown value is not the same as an + %% empty value. + %% + %% At entry: stack = ... top:[string] + %% At exit: stack = ... top:[0 or 1] + + duplicate$ empty$ + { pop$ #1 } + { #1 #2 substring$ "??" = } + if$ +} + +FUNCTION { empty.or.zero } +{ + %% Examine the top entry and push 1 if it is empty, or is zero + duplicate$ empty$ + { pop$ #1 } + { "0" = } + if$ +} + + +FUNCTION { writeln } +{ + %% In BibTeX style files, the sequences + %% + %% ... "one" "two" output + %% ... "one" "two" output.xxx + %% + %% ship "one" to the output file, possibly following by punctuation, + %% leaving the stack with + %% + %% ... "two" + %% + %% There is thus a one-string lag in output processing that must be + %% carefully handled to avoid duplicating a string in the output + %% file. Unless otherwise noted, all output.xxx functions leave + %% just one new string on the stack, and that model should be born + %% in mind when reading or writing function code. + %% + %% BibTeX's asynchronous buffering of output from strings from the + %% stack is confusing because newline$ bypasses the buffer. It + %% would have been so much easier for newline to be a character + %% rather than a state of the output-in-progress. + %% + %% The documentation in btxhak.dvi is WRONG: it says + %% + %% newline$ Writes onto the bbl file what's accumulated in the + %% output buffer. It writes a blank line if and only + %% if the output buffer is empty. Since write$ does + %% reasonable line breaking, you should use this + %% function only when you want a blank line or an + %% explicit line break. + %% + %% write$ Pops the top (string) literal and writes it on the + %% output buffer (which will result in stuff being + %% written onto the bbl file when the buffer fills + %% up). + %% + %% Examination of the BibTeX source code shows that write$ does + %% indeed behave as claimed, but newline$ sends a newline character + %% directly to the output file, leaving the stack unchanged. The + %% first line "Writes onto ... buffer." is therefore wrong. + %% + %% The original BibTeX style files almost always use "write$ newline$" + %% in that order, so it makes sense to hide that pair in a private + %% function like this one, named after a statement in Pascal, + %% the programming language embedded in the BibTeX Web program. + + write$ % output top-of-stack string + newline$ % immediate write of newline (not via stack) +} + +FUNCTION { init.state.consts } +{ + #0 'before.all := + #1 'mid.sentence := + #2 'after.sentence := + #3 'after.block := +} + +FUNCTION { output.nonnull } +{ % Stack in: ... R S T Stack out: ... R T File out: S + 's := + output.state mid.sentence = + { + ", " * write$ + } + { + output.state after.block = + { + add.period$ writeln + "\newblock " write$ + } + { + output.state before.all = + { + write$ + } + { + add.period$ " " * write$ + } + if$ + } + if$ + mid.sentence 'output.state := + } + if$ + s +} + +FUNCTION { output.nonnull.dot.space } +{ % Stack in: ... R S T Stack out: ... R T File out: S + 's := + output.state mid.sentence = % { ". " * write$ } + { + ". " * write$ + } + { + output.state after.block = + { + add.period$ writeln "\newblock " write$ + } + { + output.state before.all = + { + write$ + } + { + add.period$ " " * write$ + } + if$ + } + if$ + mid.sentence 'output.state := + } + if$ + s +} + +FUNCTION { output.nonnull.remove } +{ % Stack in: ... R S T Stack out: ... R T File out: S + 's := + output.state mid.sentence = + { + " " * write$ + } + { + output.state after.block = + { + add.period$ writeln "\newblock " write$ + } + { + output.state before.all = + { + write$ + } + { + add.period$ " " * write$ + } + if$ + } + if$ + mid.sentence 'output.state := + } + if$ + s +} + +FUNCTION { output.nonnull.removenospace } +{ % Stack in: ... R S T Stack out: ... R T File out: S + 's := + output.state mid.sentence = + { + "" * write$ + } + { + output.state after.block = + { + add.period$ writeln "\newblock " write$ + } + { + output.state before.all = + { + write$ + } + { + add.period$ " " * write$ + } + if$ + } + if$ + mid.sentence 'output.state := + } + if$ + s +} + +FUNCTION { output } +{ % discard top token if empty, else like output.nonnull + duplicate$ empty.or.unknown + 'pop$ + 'output.nonnull + if$ +} + +FUNCTION { output.dot.space } +{ % discard top token if empty, else like output.nonnull.dot.space + duplicate$ empty.or.unknown + 'pop$ + 'output.nonnull.dot.space + if$ +} + +FUNCTION { output.removenospace } +{ % discard top token if empty, else like output.nonnull.removenospace + duplicate$ empty.or.unknown + 'pop$ + 'output.nonnull.removenospace + if$ +} + +FUNCTION { output.check } +{ % like output, but warn if key name on top-of-stack is not set + 't := + duplicate$ empty.or.unknown + { pop$ "empty " t * " in " * cite$ * warning$ } + 'output.nonnull + if$ +} + +FUNCTION { bibinfo.output.check } +{ % like output.check, adding bibinfo field + 't := + duplicate$ empty.or.unknown + { pop$ "empty " t * " in " * cite$ * warning$ } + { "\bibinfo{" t "}{" * * swap$ * "}" * + output.nonnull } + if$ +} + +FUNCTION { output.check.dot.space } +{ % like output.dot.space, but warn if key name on top-of-stack is not set + 't := + duplicate$ empty.or.unknown + { pop$ "empty " t * " in " * cite$ * warning$ } + 'output.nonnull.dot.space + if$ +} + +FUNCTION { fin.block } +{ % functionally, but not logically, identical to fin.entry + add.period$ + writeln +} + +FUNCTION { fin.entry } +{ + add.period$ + writeln +} + +FUNCTION { new.sentence } +{ % update sentence state, with neither output nor stack change + output.state after.block = + 'skip$ + { + output.state before.all = + 'skip$ + { after.sentence 'output.state := } + if$ + } + if$ +} + +FUNCTION { fin.sentence } +{ + add.period$ + write$ + new.sentence + "" +} + +FUNCTION { new.block } +{ + output.state before.all = + 'skip$ + { after.block 'output.state := } + if$ +} + +FUNCTION { output.coden } % UTAH +{ % output non-empty CODEN as one-line sentence (stack untouched) + coden empty.or.unknown + { } + { "\showCODEN{" coden * "}" * writeln } + if$ +} + +% +% Sometimes articleno starts with the word 'Article' or 'Paper. +% (this is a bug of acmdl, sigh) +% We strip them. We assume eid or articleno is already on stack +% + +FUNCTION { strip.articleno.or.eid } +{ + 't := + t #1 #7 substring$ "Article" = + {t #8 t text.length$ substring$ 't :=} + { } + if$ + t #1 #7 substring$ "article" = + {t #8 t text.length$ substring$ 't :=} + { } + if$ + t #1 #5 substring$ "Paper" = + {t #6 t text.length$ substring$ 't :=} + { } + if$ + t #1 #5 substring$ "paper" = + {t #6 t text.length$ substring$ 't :=} + { } + if$ + % Strip any left trailing space or ~ + t #1 #1 substring$ " " = + {t #2 t text.length$ substring$ 't :=} + { } + if$ + t #1 #1 substring$ "~" = + {t #2 t text.length$ substring$ 't :=} + { } + if$ + t +} + + +FUNCTION { format.articleno } +{ + articleno empty.or.unknown not eid empty.or.unknown not and + { "Both articleno and eid are defined for " cite$ * warning$ } + 'skip$ + if$ + articleno empty.or.unknown eid empty.or.unknown and + { "" } + { + numpages empty.or.unknown + { "articleno or eid field, but no numpages field, in " + cite$ * warning$ } + { } + if$ + eid empty.or.unknown + { "Article \bibinfo{articleno}{" articleno strip.articleno.or.eid * "}" * } + { "Article \bibinfo{articleno}{" eid strip.articleno.or.eid * "}" * } + if$ + } + if$ +} + +FUNCTION { format.year } +{ % push year string or "[n.\,d.]" onto output stack + %% Because year is a mandatory field, we always force SOMETHING + %% to be output + "\bibinfo{year}{" + year empty.or.unknown + { "[n.\,d.]" } + { year } + if$ + * "}" * +} + +FUNCTION { format.day.month } +{ % push "day month " or "month " or "" onto output stack + day empty.or.unknown + { + month empty.or.unknown + { "" } + { "\bibinfo{date}{" month * "} " *} + if$ + } + { + month empty.or.unknown + { "" } + { "\bibinfo{date}{" day * " " * month * "} " *} + if$ + } + if$ +} + +FUNCTION { format.day.month.year } % UTAH +{ % if month is empty, push "" else push "(MON.)" or "(DD MON.)" + % Needed for frequent periodicals: 2008. ... New York Times C-1, C-2, C-17 (23 Oct.) + % acm-*.bst addition: prefix parenthesized date string with + % ", Article nnn " + articleno empty.or.unknown eid empty.or.unknown and + { "" } + { output.state after.block = + {", " format.articleno * } + { format.articleno } + if$ + } + if$ + " (" * format.day.month * format.year * ")" * +} + +FUNCTION { output.day.month.year } % UTAH +{ % if month is empty value, do nothing; else output stack top and + % leave with new top string "(MON.)" or "(DD MON.)" + % Needed for frequent periodicals: 2008. ... New York Times C-1, C-2, C-17 (23 Oct.) + format.day.month.year + output.nonnull.remove +} + +FUNCTION { strip.doi } % UTAH +{ % Strip any Web address prefix to recover the bare DOI, leaving the + % result on the output stack, as recommended by CrossRef DOI + % documentation. + % For example, reduce "http://doi.acm.org/10.1145/1534530.1534545" to + % "10.1145/1534530.1534545". A suitable URL is later typeset and + % displayed as the LAST item in the reference list entry. Publisher Web + % sites wrap this with a suitable link to a real URL to resolve the DOI, + % and the master https://doi.org/ address is preferred, since publisher- + % specific URLs can disappear in response to economic events. All + % journals are encouraged by the DOI authorities to use that typeset + % format and link procedures for uniformity across all publications that + % include DOIs in reference lists. + % The numeric prefix is guaranteed to start with "10.", so we use + % that as a test. + % 2017-02-04 Added stripping of https:// (Boris) + doi #1 #3 substring$ "10." = + { doi } + { + doi 't := % get modifiable copy of DOI + + % Change https:// to http:// to strip both prefixes (BV) + + t #1 #8 substring$ "https://" = + { "http://" t #9 t text.length$ #8 - substring$ * 't := } + { } + if$ + + t #1 #7 substring$ "http://" = + { + t #8 t text.length$ #7 - substring$ 't := + + "INTERNAL STYLE-FILE ERROR" 's := + + % search for next "/" and assign its suffix to s + + { t text.length$ } + { + t #1 #1 substring$ "/" = + { + % save rest of string as true DOI (should be 10.xxxx/yyyy) + t #2 t text.length$ #1 - substring$ 's := + "" 't := % empty string t terminates the loop + } + { + % discard first character and continue loop: t <= substring(t,2,last) + t #2 t text.length$ #1 - substring$ 't := + } + if$ + } + while$ + + % check for valid DOI (should be 10.xxxx/yyyy) + s #1 #3 substring$ "10." = + { } + { "unrecognized DOI substring " s * " in DOI value [" * doi * "]" * warning$ } + if$ + + s % push the stripped DOI on the output stack + + } + { + "unrecognized DOI value [" doi * "]" * warning$ + doi % push the unrecognized original DOI on the output stack + } + if$ + } + if$ +} + +% +% Change by BV: added standard prefix to URL +% +FUNCTION { output.doi } % UTAH +{ % output non-empty DOI as one-line sentence (stack untouched) + doi empty.or.unknown + { } + { + %% Use \urldef here for the same reason it is used in output.url, + %% see output.url for further discussion. + "\urldef\tempurl%" writeln + "\url{https://doi.org/" strip.doi * "}" * writeln + "\showDOI{\tempurl}" writeln + } + if$ +} + +FUNCTION { output.isbn } % UTAH +{ % output non-empty ISBN-10 and/or ISBN-13 as one-line sentences (stack untouched) + show-isbn-10-and-13 + { + %% show both 10- and 13-digit ISBNs + isbn empty.or.unknown + { } + { + "\showISBNx{" isbn * "}" * writeln + } + if$ + isbn-13 empty.or.unknown + { } + { + "\showISBNxiii{" isbn-13 * "}" * writeln + } + if$ + } + { + %% show 10-digit ISBNs only if 13-digit ISBNs not available + isbn-13 empty.or.unknown + { + isbn empty.or.unknown + { } + { + "\showISBNx{" isbn * "}" * writeln + } + if$ + } + { + "\showISBNxiii{" isbn-13 * "}" * writeln + } + if$ + } + if$ +} + +FUNCTION { output.issn } % UTAH +{ % output non-empty ISSN as one-line sentence (stack untouched) + issn empty.or.unknown + { } + { "\showISSN{" issn * "}" * writeln } + if$ +} + +FUNCTION { output.issue } +{ % output non-empty issue number as a one-line sentence (stack untouched) + issue empty.or.unknown + { } + { "Issue " issue * "." * writeln } + if$ +} + +FUNCTION { output.lccn } % UTAH +{ % return with stack untouched + lccn empty.or.unknown + { } + { "\showLCCN{" lccn * "}" * writeln } + if$ +} + +FUNCTION { output.note } % UTAH +{ % return with stack empty + note empty.or.unknown + { } + { "\shownote{" note * "}" add.period$ * writeln } + if$ +} + +FUNCTION { output.note.check } % UTAH +{ % return with stack empty + note empty.or.unknown + { "empty note in " cite$ * warning$ } + { "\shownote{" note * "}" add.period$ * writeln } + if$ +} + +FUNCTION { output.eprint } % +{ % return with stack empty + eprint empty.or.unknown + { } + { "\showeprint" + archiveprefix empty.or.unknown + { eprinttype empty.or.unknown + { } + { "[" eprinttype "]" * * * } + if$ + } + { "[" archiveprefix "l" change.case$ "]" * * * } + if$ + "{" eprint "}" * * * + primaryclass empty.or.unknown + { eprintclass empty.or.unknown + { } + { "~[" eprintclass "]" * * * } + if$ + } + { "~[" primaryclass "]" * * * } + if$ + writeln + } + if$ +} + + +% +% Changes by BV 2011/04/15. Do not output +% url if doi is defined +% +% +% Changes by BV 2021/11/26. Output url even if doi is defined +% if distinctURL is not zero. +% +FUNCTION { output.url } % UTAH +{ % return with stack untouched + % output URL and associated lastaccessed fields + doi empty.or.unknown distinctURL empty.or.zero not or + { + url empty.or.unknown + { } + { + %% Use \urldef, outside \showURL, so that %nn, #, etc in URLs work + %% correctly. Put the actual URL on its own line to reduce the + %% likelihood of BibTeX's nasty line wrapping after column 79. + %% \url{} can undo this, but if that doesn't work for some reason + %% the .bbl file would have to be repaired manually. + "\urldef\tempurl%" writeln + "\url{" url * "}" * writeln + + "\showURL{%" writeln + lastaccessed empty.or.unknown + { "" } + { "Retrieved " lastaccessed * " from " * } + if$ + "\tempurl}" * writeln + } + if$ + } + { } + if$ +} + +FUNCTION { output.year.check } +{ % warn if year empty, output top string and leave " YEAR