Debian Codegraph @

evaluations/suites/responses-history-live/tasks/001-seeded-assistant-history/task.json

raw
1 {
2 "id": "001-seeded-assistant-history",
3 "instruction_file": "input/instruction.txt",
4 "seed_transcript_file": "input/seed-transcript.json",
5 "working_directory": ".",
6 "full_send": true,
7 "step_limit": 5,
8 "mode": "live-provider",
9 "graders": {
10 "outcome_diff": {
11 "enabled": true,
12 "required": true
13 },
14 "outcome_command_output": {
15 "enabled": true,
16 "required": true,
17 "command": [
18 "bash",
19 "-lc",
20 "actual=$(cat responses-history-note.txt); [ \"$actual\" = \"responses-history-ok\" ] && printf '%s\\n' \"$actual\""
21 ],
22 "expected_exit_code": 0,
23 "stdout_contains": [
24 "responses-history-ok"
25 ]
26 },
27 "transcript_command_trace": {
28 "enabled": true,
29 "required": false,
30 "expected_exit_codes": [
31 0
32 ],
33 "max_command_count": 3
34 }
35 }
36 }