Debian Codegraph @

evaluations/suites/default/tasks/004-weak-done-rejection/task.json

raw
1 {
2 "id": "004-weak-done-rejection",
3 "instruction_file": "input/instruction.txt",
4 "scripted_turns_file": "input/model-turns.json",
5 "working_directory": ".",
6 "full_send": true,
7 "step_limit": 10,
8 "graders": {
9 "outcome_diff": {
10 "enabled": true,
11 "required": true
12 },
13 "outcome_command_output": {
14 "enabled": true,
15 "required": true,
16 "command": [
17 "bash",
18 "-lc",
19 "python3 - <<'PY'\nimport json\nwith open('result.json') as f:\n data = json.load(f)\nassert data == {'status': 'ok'}\nprint(data['status'])\nPY"
20 ],
21 "expected_exit_code": 0,
22 "stdout_contains": [
23 "ok"
24 ]
25 },
26 "transcript_command_trace": {
27 "enabled": true,
28 "required": false,
29 "expected_commands": [
30 "printf '{\"status\":\"ok\"}\\n' > result.json",
31 "python3 - <<'PY'\nimport json\nwith open('result.json') as f:\n data = json.load(f)\nassert data == {'status': 'ok'}\nprint(data['status'])\nPY"
32 ],
33 "expected_exit_codes": [
34 0,
35 0
36 ],
37 "max_command_count": 5
38 }
39 }
40 }