| 1 |
# Test-suite makefile for open-adventure |
| 2 |
|
| 3 |
# SPDX-FileCopyrightText: Copyright Eric S. Raymond <[email protected]> |
| 4 |
# SPDX-License-Identifier: BSD-2-Clause |
| 5 |
|
| 6 |
# Use absolute path so tests that change working directory still use |
| 7 |
# scripts from parent directory. Note that using $PWD seems to fail |
| 8 |
# here under Gitlab's CI environment. |
| 9 |
PARDIR=$(realpath ..) |
| 10 |
PATH := $(PARDIR):$(realpath .):${PATH} |
| 11 |
GCOV?=gcov |
| 12 |
|
| 13 |
# Make this overrideable so it's easier to test old versions |
| 14 |
advent?=advent |
| 15 |
|
| 16 |
# Defeat annoying behavior under Mac OS X - builtin echo doesn't do -n |
| 17 |
ECHO := /bin/echo |
| 18 |
|
| 19 |
# The TAP filter. Only affects presentation of the test suite messages |
| 20 |
TAPCONSUMER=tapview |
| 21 |
|
| 22 |
# Fall back to safety if our declared TAP consumer does not exist. |
| 23 |
# This is helpful in the CI environment, where it would be better for |
| 24 |
# the logfiles to carry the raw TAP messages. |
| 25 |
TAPFILTER=$(shell command -v $(TAPCONSUMER) || echo cat) |
| 26 |
|
| 27 |
# Find all *.log entries to test |
| 28 |
TESTLOADS := $(shell ls -1 *.log | sed '/.log/s///' | sort) |
| 29 |
|
| 30 |
.PHONY: check clean testlist listcheck savegames savecheck coverage |
| 31 |
.PHONY: buildchecks multifile-regress tap count |
| 32 |
|
| 33 |
check: savecheck |
| 34 |
@make tap | tapview |
| 35 |
@-advent -x 2>/dev/null || exit 0 # Get usage message into coverage tests |
| 36 |
|
| 37 |
.SUFFIXES: .chk |
| 38 |
|
| 39 |
clean: |
| 40 |
rm -fr *~ *.adv scratch.tmp *.ochk advent430 adventure.data |
| 41 |
|
| 42 |
# Show summary lines for all tests. |
| 43 |
testlist: |
| 44 |
@grep '^##' *.log |
| 45 |
listcheck: |
| 46 |
@for f in *.log; do \ |
| 47 |
if ( head -3 $$f | grep -q '^ *##' ); then :; else echo "$$f needs a description"; fi; \ |
| 48 |
done |
| 49 |
|
| 50 |
# Generate bogus savegames. |
| 51 |
cheat_numdie.adv: |
| 52 |
@$(PARDIR)/cheat -d -900 -o cheat_numdie.adv > /tmp/cheat_numdie |
| 53 |
cheat_numdie1000.adv: |
| 54 |
@$(PARDIR)/cheat -d -1000 -o cheat_numdie1000.adv > /tmp/cheat_numdie1000 |
| 55 |
cheat_savetamper.adv: |
| 56 |
@$(PARDIR)/cheat -d 2000 -o cheat_savetamper.adv > /tmp/cheat_savetamper |
| 57 |
resume_badversion.adv: |
| 58 |
@$(PARDIR)/cheat -v -1337 -o resume_badversion.adv > /tmp/cheat_badversion |
| 59 |
thousand_saves.adv: |
| 60 |
@$(PARDIR)/cheat -s -1000 -o thousand_saves.adv > /tmp/cheat_1000saves |
| 61 |
thousand_turns.adv: |
| 62 |
@$(PARDIR)/cheat -t -1000 -o thousand_turns.adv > /tmp/cheat_1000turns |
| 63 |
thousand_limit.adv: |
| 64 |
@$(PARDIR)/cheat -l -1000 -o thousand_limit.adv > /tmp/cheat_1000limit |
| 65 |
SGAMES = cheat_numdie.adv cheat_numdie1000.adv cheat_savetamper.adv resume_badversion.adv \ |
| 66 |
thousand_saves.adv thousand_turns.adv thousand_limit.adv |
| 67 |
|
| 68 |
# Force coverage of cheat edgecases |
| 69 |
scheck1: |
| 70 |
@$(PARDIR)/cheat -QqQ 2> /tmp/coverage_cheat_batopt | true |
| 71 |
@./outcheck.sh "cheat: bogus option for save file generation" |
| 72 |
scheck2: |
| 73 |
@$(PARDIR)/cheat 2>/dev/null | true |
| 74 |
@./outcheck.sh "cheat: No save file specified" |
| 75 |
scheck3: |
| 76 |
@$(PARDIR)/cheat -d 1 2> /tmp/coverage_cheat_nooutput | true |
| 77 |
@./outcheck.sh "cheat: doesn't save because we omit -o" |
| 78 |
scheck4: |
| 79 |
@$(PARDIR)/cheat -o / 2> /tmp/coverage_cheat_badoutput | true |
| 80 |
@./outcheck.sh "cheat: doesn't save to invalid path" |
| 81 |
scheck5: |
| 82 |
@$(advent) -r /badfilename < pitfall.log > /tmp/coverage_advent_readfail 2>&1 || exit 1 |
| 83 |
@./outcheck.sh "cheat: doesn't start with invalid file with -r" |
| 84 |
scheck6: |
| 85 |
@$(advent) -l / < pitfall.log > /tmp/coverage_advent_logfail 2>&1 || exit 1 |
| 86 |
@./outcheck.sh "cheat: doesn't start with invalid file passed to -l" |
| 87 |
scheck7: |
| 88 |
@$(advent) -r thousand_saves.adv < pitfall.log > /tmp/coverage_advent_readfail 2>&1 || exit 1 |
| 89 |
@./outcheck.sh "test -r with valid input" |
| 90 |
SCHECKS = scheck1 scheck2 scheck3 scheck4 scheck5 scheck6 scheck7 |
| 91 |
|
| 92 |
# Don't run this from here, you'll get cryptic warnings and no good result |
| 93 |
# if the advent binary wasn't built with coverage flags. Do "make clean coverage" |
| 94 |
# from the top-level directory. |
| 95 |
coverage: check |
| 96 |
lcov -t "advent" -o $(PARDIR)/advent.info -c -d $(PARDIR) --gcov-tool=$(GCOV) |
| 97 |
genhtml -o $(PARDIR)/coverage/ $(PARDIR)/advent.info |
| 98 |
./coverage_dungeon.py |
| 99 |
|
| 100 |
# Rebuild characterizing tests |
| 101 |
buildchecks: savegames |
| 102 |
$(PARDIR)/cheat -s -1000 -o thousand_saves.adv > /tmp/regress1000saves |
| 103 |
@for file in $(TESTLOADS); do \ |
| 104 |
echo "Remaking $${file}.chk"; \ |
| 105 |
OPTS=`sed -n /#options:/s///p <$${file}.log`; \ |
| 106 |
advent $$OPTS <$${file}.log >$${file}.chk 2>&1 || exit 1; \ |
| 107 |
done; \ |
| 108 |
echo "inven" | advent issue36.log /dev/stdin >multifile.chk; \ |
| 109 |
rm -f scratch.tmp |
| 110 |
|
| 111 |
RUN_TARGETS=$(TESTLOADS:%=run-regress-%) |
| 112 |
$(RUN_TARGETS): run-regress-%: %.log |
| 113 |
@(test=$(<:.log=); legend=$$(sed -n '/^## /s///p' <"$<" 2>/dev/null || echo "(no description)"); \ |
| 114 |
OPTS=`sed -n /#options:/s///p $<`; \ |
| 115 |
$(advent) $$OPTS <$< | tapdiffer "$${test}: $${legend}" "$${test}.chk") |
| 116 |
|
| 117 |
multifile-regress: |
| 118 |
@(echo "inven" | advent issue36.log /dev/stdin) | tapdiffer "multifile: multiple-file test" multifile.chk |
| 119 |
|
| 120 |
TEST_TARGETS = $(SCHECKS) $(RUN_TARGETS) multifile-regress |
| 121 |
|
| 122 |
tap: count $(SGAMES) $(TEST_TARGETS) |
| 123 |
@rm -f scratch.tmp /tmp/coverage* /tmp/cheat* |
| 124 |
count: |
| 125 |
@echo 1..$(words $(TEST_TARGETS)) |
| 126 |
|
| 127 |
# The following machinery tests the game against a binary made from |
| 128 |
# the advent430 branch To use it, switch to that branch, build the |
| 129 |
# binary, run it once to generate adventure.data, then switch back to |
| 130 |
# master leaving advent430 and adventure.data in place (make clean |
| 131 |
# does not remove them). |
| 132 |
# |
| 133 |
# make clean # Clean up object files, laving a bare source tree |
| 134 |
# git checkout advent430 # Check out the advent430 branch |
| 135 |
# make # Build the advent430 binary |
| 136 |
# advent430 # Run it. Answer the novice question and quit |
| 137 |
# make clean # Remove .o files |
| 138 |
# git checkout master # Go back to master branch |
| 139 |
# make # Rebuild advent. |
| 140 |
# |
| 141 |
# The diff file produced has corrected spellings in it. That's what oldfilter |
| 142 |
# is for, to massage out the original spellings and avoid noise diffs. |
| 143 |
# Diffs in amount of whitespace and trailing whitespace are ignored |
| 144 |
# |
| 145 |
# A magic comment of NOCOMPARE in a log file excludes it from this comparison. |
| 146 |
# making it a skipped test in the TAP view. First use of this was to avoid a |
| 147 |
# spurious mismatch on the news text. Other uses avoid spurious mismatches due |
| 148 |
# to bug fixes. |
| 149 |
# |
| 150 |
# When adding more tests, bear in mind that any game that continues after a |
| 151 |
# resurrection will need a NOCOMPARE. At some point in the forward port, |
| 152 |
# resurrection was accidentally changed in a way that messed with the LCG chain. |
| 153 |
# |
| 154 |
# The *.chk files need not be up-to-date for this to work. |
| 155 |
# |
| 156 |
TAPFILTER=tapview |
| 157 |
oldcompare: |
| 158 |
@if [ -f ../advent430 ]; then cp ../advent430 ../adventure.data .; else echo "advent430 nonexistent"; exit 1; fi |
| 159 |
@-(for x in *.log; do \ |
| 160 |
stem=$${x%.log}; \ |
| 161 |
legend=$$(sed -n '/^## /s///p' <$$x 2>/dev/null || echo "(no description)"); \ |
| 162 |
if grep NOCOMPARE $$x >/dev/null; \ |
| 163 |
then echo "not ok - $${stem}.ochk: $${legend} # SKIP"; \ |
| 164 |
else \ |
| 165 |
./advent430 <$${stem}.log | oldfilter >$${stem}.ochk; \ |
| 166 |
../advent <$${stem}.log >$${stem}.log-new; \ |
| 167 |
./newfilter <$${stem}.log-new | tapdiffer -b "$${stem}: $${legend}" $${stem}.ochk; \ |
| 168 |
fi; \ |
| 169 |
done; \ |
| 170 |
echo 1..$(words $(shell ls *.log))) | $(TAPFILTER) |
| 171 |
@rm *.ochk *-new advent430 adventure.data |
| 172 |
|
| 173 |
# List all NOCOMPARE tests. |
| 174 |
residuals: |
| 175 |
@grep -n NOCOMPARE *.log |
| 176 |
|
| 177 |
# end |