From 880e32832aa49502bfea39ad6b369fed09be0a2c Mon Sep 17 00:00:00 2001 From: Ofir Press Date: Mon, 23 Feb 2026 17:42:36 +0000 Subject: [PATCH] Add eval.yaml (#2) - Add eval.yaml (8db12b107a1abfd10808f023ed233890b4d7cbae) Co-authored-by: Niels Rogge --- eval.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 eval.yaml diff --git a/eval.yaml b/eval.yaml new file mode 100644 index 0000000..d3ae551 --- /dev/null +++ b/eval.yaml @@ -0,0 +1,11 @@ +name: SWE-Bench Verified +description: SWE-bench Verified is a curated, human-annotated subset of 500 issues from the original SWE-bench, designed to accurately evaluate AI models on real-world software engineering tasks. It fixes issues in the original dataset by ensuring all tasks are solvable, well-defined, and properly tested by human developers. +evaluation_framework: swe-bench-verified + +tasks: + - id: swe_bench_%_resolved + config: default + split: test + - id: swe_bench_average_cost + config: default + split: test \ No newline at end of file