Debian Codegraph @

evaluations/suites/default/tasks/003-self-verification-artifact/task.json

raw
1 {
2 "id": "003-self-verification-artifact",
3 "instruction_file": "input/instruction.txt",
4 "scripted_turns_file": "input/model-turns.json",
5 "working_directory": ".",
6 "full_send": true,
7 "step_limit": 10,
8 "graders": {
9 "outcome_diff": {
10 "enabled": true,
11 "required": true
12 },
13 "outcome_command_output": {
14 "enabled": true,
15 "required": true,
16 "command": [
17 "bash",
18 "-lc",
19 "actual=$(cat result.txt); [ \"$actual\" = \"agentic self verification works\" ] && printf '%s\\n' \"$actual\""
20 ],
21 "expected_exit_code": 0,
22 "stdout_contains": [
23 "agentic self verification works"
24 ]
25 },
26 "transcript_command_trace": {
27 "enabled": true,
28 "required": false,
29 "expected_commands": [
30 "printf 'agentic self verification works\\n' > result.txt",
31 "test -f result.txt && cat result.txt"
32 ],
33 "expected_exit_codes": [
34 0,
35 0
36 ],
37 "max_command_count": 5
38 }
39 }
40 }