Debian Codegraph @

evaluations/suites/protocol-loss-live/tasks/001-invalid-turn-live/task.json

raw
1 {
2 "id": "001-invalid-turn-live",
3 "instruction_file": "input/instruction.txt",
4 "working_directory": ".",
5 "full_send": true,
6 "step_limit": 3,
7 "mode": "live-provider",
8 "graders": {
9 "outcome_diff": {
10 "enabled": true,
11 "required": true
12 },
13 "outcome_command_output": {
14 "enabled": true,
15 "required": true,
16 "command": [
17 "bash",
18 "-lc",
19 "actual=$(cat note.txt); [ \"$actual\" = \"hello\" ] && printf '%s\\n' \"$actual\""
20 ],
21 "expected_exit_code": 0,
22 "stdout_contains": [
23 "hello"
24 ]
25 },
26 "transcript_command_trace": {
27 "enabled": true,
28 "required": false,
29 "expected_exit_codes": [
30 0
31 ],
32 "max_command_count": 3
33 }
34 }
35 }