Debian Codegraph @

t/Makefile

raw
1 #! /usr/bin/make -rRf
2 #?
3 #? NAME
4 #? Makefile - makefile for testing O-Saft
5 #?
6 #? SYNOPSYS
7 #? make [options] [target] [...]
8 #?
9 #? DESCRIPTION
10 #? For more details please see
11 #? ../Makefile Makefile.help Makefile.pod
12 #? make help.test
13 #?
14 # HACKER's INFO
15 # For details please see
16 # ../Makefile Makefile.help Makefile.template
17 #
18 # Naming conventions for t/Makefile*
19 # * variable names have the prefix TEST and HELP .
20 # * target names for public use have the prefix test.
21 # * individual, special targets have the prefix _test (exceptions exist)
22 #
23 # Summary variables extended in each t/Makefile*
24 # ALL.includes - list of Makefiles
25 # ALL.inc.type - list of types (should be suffix of Makefile's name)
26 # ALL.help.tests - list of documentation (help) texts
27 # ALL.tests - list of targets for testing
28 # ALL.tests.log - list of targets for testing writing to logfile
29 #
30 # Variables specific for each t/Makefile*
31 # _SID.myTYPE - version number
32 # _MYSELF.myTYPE - name of current file
33 # HELP.myTYPE - documentation (help) text for the corresponding file
34 # HELP.myTYPE.all - list of all targets for the corresponding file
35 # HELP.myTYPE.internal - some more documentation (help) text
36 # where myTYPE above is the suffix of the Makefile's filename
37 #
38 # Each t/Makefile* may also redefine following variables
39 # HELP_TYPE - suffix (and type) of Makefile*
40 # HELP_HEAD - descriptive header line, set to HELP_NAME, HELP_RULE
41 #
42 # Special targets for documentation (help) in t/Makefile*
43 # help.test.% - print individual documentation using $(HELP.myTYPE)
44 # help.test.%.all - print all targets using $(HELP.myTYPE.all)
45 # help.test.%.internal - print details using $(HELP.myTYPE.internal)
46 # where % is the suffix of the Makefile's filename
47 #
48 # TODO:
49 # * include Makefile.* should be generic
50 # * unify test.warnings.log and test.tests.log (should use same target)
51 # * rename ALL.tests and ALL.tests.log (too similar to ALL.test)
52 #
53 #? VERSION
54 #? @(#) Makefile 1.142 22/11/13 13:26:27
55 #?
56 #? AUTHOR
57 #? 18-apr-18 Achim Hoffmann
58 #?
59 # -----------------------------------------------------------------------------
60
61 HELP-help.test.test = general testing targets
62
63 _SID.test := 1.142
64
65 _MYSELF.test := t/Makefile
66 ALL.includes += $(_MYSELF.test)
67 ALL.inc.type += test
68 ALL.help.tests += help.test
69
70 export OSAFT_MAKE := avoid writing random data (like date and time string)
71 export LANG := C
72 # LANG ensures same messages and sorting everywhere
73
74 first-test-target-is-default: help.test
75
76 ifndef ALL.Makefiles
77 -include t/Makefile.inc
78 # defines variables if called directly (not from ../Makefile)
79 endif
80
81 help.test: HELP_TYPE = test
82 help.test-v: HELP_TYPE = test
83 help.test-vv: HELP_TYPE = test
84 help.test.all: HELP_TYPE = test
85 help.test.test: HELP_TYPE = test
86 help.test.test-v: HELP_TYPE = test
87 help.test.test-vv: HELP_TYPE = test
88 help.test.test.all: HELP_TYPE = test
89 help.test: HELP_HEAD = $(HELP_RULE)
90 help.test.%: HELP_HEAD = $(HELP_RULE)
91
92 #_____________________________________________________________________________
93 #________________________________________________________________ variables __|
94
95 ifndef test-remove-environment-variables
96 # most environment variables should be ignored, to avoid unexpected
97 # behaviour and to keep our list of variables clean;
98 # forst define list of required variables, then get list of existing
99 # environment variable and remove all except the required ones
100 $(eval _keep = OSAFT_MAKE LANG \
101 DISPLAY HOME PATH PWD SHELL TERM USER NLSPATH \
102 DT_RUNPATH DT_RPATH DT_NEEDED \
103 DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH \
104 LD_CONFIG LD_LIBRARY_PATH LD_PRELOAD LD_RUN_PATH LD_DEBUG \
105 LD_TRACE_LOADED_OBJECTS LD_TRACE_LOADED_OBJECTS_ALL LD_TRACE_PRELINKING \
106 LIBPATH RPATH RUNPATH SHLIB_PATH \
107 OPENSSL_CONF OPENSSL_FIPS OPENSSL_ENGINES OPENSSL \
108 OPENSSL_ALLOW_PROXY OPENSSL_ALLOW_PROXY_CERTS \
109 )
110 $(eval _env = $(shell printenv | awk -F= '{print $$1}'))
111 $(foreach _var, $(filter-out $(_keep),$(_env)), $(eval undefine $(_var)))
112 $(eval undefine _keep)
113 $(eval undefine _env)
114 test-remove-environment-variables = done
115 endif
116
117 # store -n (dry-run) option in own variable
118 TEST.make-n := $(if $(findstring n,$(firstword -$(MAKEFLAGS))),-n)
119
120 # internal variables
121 TEST.dir := t
122 TEST.logdir = $(TEST.dir)/log
123 TEST.logtxt := <<random value replaced by make>>
124 TEST.symlinks := ../contrib ../osaft.pm ../.o-saft.pl ../Net ../docs ../OSaft
125
126 # following may be redefined in included t/Makefile*
127 #TEST.init := --header
128 #TEST.args :=
129
130 # variable for list of hostnames needs to be unique in each Makefile*
131 # otherwise some targets in other Makefile* get confused
132 #TEST.host = localhost
133 #TEST.hosts =
134
135 # following defined in ../Makefile
136 # TEST.do =
137 # SRC.test =
138 # ALL.test =
139
140 # defined in other t/Makefile*
141 # ALL.tests =
142
143 # $(SRC.rc) may not be defined here, hence hardcoded
144
145 ifndef SRC.contrib.dir
146 SRC.contrib.dir = contrib
147 endif
148
149 # Tracing just the log-targets is special. Using $(TRACE.target) is not an
150 # option, because it would als change the final target called with $(MAKE) .
151 # Hence an additional variable is used.
152 # TODO: not public, not documented, variable needs to be defined here
153 TRACE.target.log =
154 # example = echo "\# $*\tcalling: $(MAKE) -s testcmd-$*"
155
156 TEST.today := $(shell date +%Y""%m%d)
157 # current date in ISO format
158
159 # need a usable way to change diff program; EXE.diff may be set on command-line
160 EXE.log-diff := diff
161 EXE.log-xxdiff := xxdiff --geometry 1600x1000
162 EXE.log-mgdiff := mgdiff
163 EXE.log-tkdiff := tkdiff
164 EXE.diffs := diff xxdiff mgdiff tkdiff
165 EXE.diff := tkdiff
166 EXE.log-diff = $(EXE.log-tkdiff)
167 # chaneg EXE.log-diff to use another program
168
169 # need a usable way to change generator for man-pages
170 EXE.gendoc.perldoc := perldoc
171 EXE.gendoc.pod2man := pod2man
172 EXE.gendoc := $(EXE.gendoc.perldoc)
173
174 # default filter for testarg-%.log targets
175 EXE.log-filterarg := cat
176 # default filter for testcmd-%.log targets
177 EXE.log-filtercmd := cat
178
179 _EXE.grep-opt := -q
180 # grep (mainly used in message-% target) should not report matching lines
181 # variable can be reset or redifined for more verbose output
182
183 _EXE.sort-opt := -n -f
184 # newer GNU sort (~2018) changed the default behaviour, for example
185 # case-sensitivity, these options do the traditional sort
186 # TODO: need to check first if sort supports these options
187
188 _EXE.perldoc-opt := -n nroff -T
189 # newer perldoc prints with plain text format by default
190 # -n nroff enforced the traditional behaviour with ASCII-escapes
191
192 _EXE.pod2man-opt := --utf8
193 # --date=DATESTRING
194 # will be used for testing to avoid diff in generated files
195
196 # define script or parts of it as variables, for better human readability
197 # SEE Make:target matching
198 # _EXE.macro_by_line.awk - extract all variable names
199 # _EXE.target_by_line.awk - extract all target names
200 # _EXE.target_to_arr.awk - extract target names and store uniquely in arr[]
201 # _EXE.print_arr_END.awk - print collected data from arr[]
202 # _EXE.print_file.awk - either print separator line or the filename
203 TEST.target_prefix := [a-zA-Z][_a-zA-Z.]
204 # matching target names not starting with _
205 _EXE.macro_by_line.awk := /^[a-zA-Z_][a-zA-Z_.]* *=/{sub(/=/,"",$$1);print $$1}
206 _EXE.target_by_line.awk:= /^[a-zA-Z_][a-zA-Z_.]* *:/{sub(/:/,"",$$1);print $$1}
207 _EXE.target_to_arr.awk = /^.*%/{next} /^$(TEST.target_prefix)*/{arr[$$1]=1}
208 _EXE.print_arr_END.awk := END{for(idx in arr){print idx}}
209 _EXE.print_file.awk := (FNR==1){ x="";for(i=1;i<62;i++){x=sprintf("%s%c",x,"-")};print "\n\t\t\#"x}
210
211 # if target was test*-compare-v or test*-move-v TRACE.target is not empty, so
212 # checking this variable if not empty is sufficent to print verbose information
213 # not tested if following defines could be written as macros
214 # SEE Make:defines with commands
215
216 define EXE.log-compare
217 cd $(TEST.logdir) && \
218 echo "# compare "`ls $(TEST.target_prefix)*.log$(TEST.log-suffix) | wc -l`" file ..." && \
219 for f in $(TEST.target_prefix)*.log; do \
220 [ -f $${f}$(TEST.log-suffix) ] || continue ; \
221 [ -n '$(TRACE.target)' ] && echo "# $(TEST.logdir)/$${f} ..." ; \
222 $(EXE.log-diff) $${f} $${f}$(TEST.log-suffix) >/dev/null 2>&1 ; \
223 done
224 endef
225
226 define EXE.log-compare-hint
227 echo ""
228 echo "# for informations and to show differences with $(EXE.diff), use:"
229 echo " $(MAKE_COMMAND) help.test.log-info"
230 echo " $(MAKE_COMMAND) $(MAKECMDGOALS)-compare"
231 echo " $(MAKE_COMMAND) $(MAKECMDGOALS)-compare-v"
232 echo " $(MAKE_COMMAND) $(MAKECMDGOALS)-compare TEST.log-suffix=$(TEST.log-suffix)"
233 endef
234
235 define EXE.log-move
236 cd $(TEST.logdir) && \
237 for f in $(TEST.target_prefix)*.log; do \
238 [ -f $${f}$(TEST.log-suffix) ] || continue ; \
239 [ -n '$(TRACE.target)' ] && echo "# $(TEST.logdir)/$${f} ..." ; \
240 mv $${f}$(TEST.log-suffix) $${f} ; \
241 done
242 endef
243
244 # FIXME: does not set _stub properly if $* is set
245 define EXE.log-move-hint
246 _stub="log"
247 $(eval _stub="log")
248 $(eval $(shell [ -n "$*" ] && _stub="$*.log" ))
249 echo ""
250 echo "# for informations and to move new files to .log , use:"
251 echo " $(MAKE_COMMAND) help.test.log-info"
252 echo " $(MAKE_COMMAND) test.$(_stub)-move"
253 echo " $(MAKE_COMMAND) test.$(_stub)-move TEST.log-suffix=$(TEST.log-suffix)"
254 endef
255
256
257 HELP.test = $(_NL)\
258 \# The term '%' in the targets names described here, may be any of:$(_NL)\
259 \# $(ALL.inc.type)$(_NL)\
260 \#$(_NL)\
261 \# Following variables are intended to be used on command line:$(_NL)\
262 \# EXE.pl - program to perform tests, default: '$(EXE.pl)'$(_NL)\
263 \# TEST.init - arguments and options always passed to EXE.pl; default: '$(TEST.init)'$(_NL)\
264 \# TEST.args - arguments and options to be passed to EXE.pl per target$(_NL)\
265 \# TEST.host - hostname to be used for tests (when only one is expected)$(_NL)\
266 \# TEST.hosts - list of hostnames to perform tests with$(_NL)\
267 \# TEST.dir - directory in which EXE.pl will be executed; default: '$(TEST.dir)/'$(_NL)\
268 \# TEST.logdir - directory where results of EXE.pl will be stored; default: '$(TEST.logdir)/'$(_NL)\
269 \# TEST.rc - content of RC-file to be used by EXE.pl: '$(TEST.rc)'$(_NL)\
270 \# TEST.target_prefix - prefix of log-filename; see help.test.log-info$(_NL)\
271 \# TEST.log-suffix - suffix of log-filename; see help.test.log-info$(_NL)\
272 \#$(_NL)\
273 \# Notes about some spezial targets:$(_NL)\
274 \# help.test - automatically generated target; alias for help.test.test$(_NL)\
275 \# help.test.test.alls -$(_NL)\
276 \# help.test.test.all - be prepared for huge output$(_NL)\
277 \#$(_NL)\
278 \# Keep in mind that following files are located in '$(TEST.dir)/':$(_NL)\
279 \# .o-saft.pl $(TEST.rc) .o-saft.tcl$(_NL)\
280 \# which are used by EXE.pl or other tools used for the tests.
281
282 # following variable contains (huge) list of all available targets
283 HELP.test.test.all = $(_NL)\
284 \# $(words $(ALL.tests)) targets for tests:$(_NL)\
285 $(ALL.tests)$(_NL)\
286 \#$(_NL)\
287 \# $(words $(ALL.tests.log)) targets which write results to logfiles:$(_NL)\
288 $(ALL.tests.log)$(_NL)\
289 \#$(_NL)\
290 \# summary targets for tests:$(_NL)\
291 $(ALL.help.tests:help.%=%)$(_NL)\
292 \#$(_NL)\
293 \# for more target and details, following targets can be used$(_NL)\
294 $(foreach target,$(ALL.inc.type),t-$(target))$(_NL)
295
296 HELP.test.test = $(HELP.test)
297 HELP.test.all = $(HELP.test.test.all)
298 help.test: help.test.test
299 help.test.all: help.test.test.all
300 @$(TRACE.target)
301 # convenience to satisfy help.test.all
302
303 # help.test.test would only print the HELP-* texts from herein.
304 # All other tests from the individual t/Makefile.* files have their own HELP-*
305 # variable their. help.test.test should also print hints how to get information
306 # about these other targets. Therefore the auxiliary target _help_list exists
307 # in Makefile which is used as dependency here.
308 help.test.test: _help_list
309 help.test.test-v: _help_list
310 help.test.test-v-v: _help_list
311
312 #_____________________________________________________________________________
313 #____________________________________________ target for help in t/Makefile*__|
314
315 HELP-help.test.test.all = print available targets for tests from all Makefiles
316 HELP-help.test.test.alls = print available targets for tests one per line
317 #HELP-help.test.test.all-n = show command for all available test targets
318 HELP-help.test.%.all = print available individual targets for tests '%' from Makefile.*
319
320 # experimental: some targets print too many shell commands
321 help.test.test.all-n:
322 @$(TRACE.target)
323 @$(MAKE) -n $(ALL.tests)
324
325 help.test.test.alls:
326 @$(TRACE.target)
327 @echo " # $(words $(ALL.tests)) targets for tests:"
328 @echo $(ALL.tests) | $(EXE.wordperline) | sort $(_EXE.sort-opt)
329 @echo ""
330
331 help.test.%.all:
332 @$(TRACE.target)
333 @echo " # individual targets for testing $* :"
334 @echo $(ALL.test.$*) | $(EXE.wordperline) | sort $(_EXE.sort-opt)
335 @echo ""
336 @echo "$(HELP.test.$*.all)"
337
338 HELP-help.test.%.internal = show settings for tests '%' from Makefile.*
339 help.test.%.internal:
340 @$(TRACE.target)
341 @echo " # TEST.file: $(TEST.file)"
342 @echo " # TEST.init.$*: $(TEST.init.$*)"
343 @echo " # TEST.$*.hosts: $(TEST.$*.hosts)"
344 @echo " # ALL.test$*: $(words $(ALL.test$*)) : $(ALL.test$*)"
345 @echo " # ALL.test.$*: $(words $(ALL.test.$*)) : $(ALL.test.$*)"
346 @echo $(HELP.$*.internal)
347 @echo " # -------------------------------------------------------------"
348
349 ALL.help.tests.all = $(foreach type,$(ALL.inc.type), help.test.$(type).all)
350 ALL.help.test.internal = $(foreach type,$(ALL.inc.type), help.test.$(type).internal)
351 help.test.targets:
352 @$(MAKE) -s $(ALL.help.tests.all $(ALL.help.test.internal)
353
354 #_____________________________________________________________________________
355 #_____________________________________________________ internal test target __|
356
357 HELP-_internal = _____________________________________ internal test target _
358 HELP-testcmd-test.internal = print GNU Make's internals and global project settings
359 # dummy ' to keep some stupid syntax highlighting happy
360
361 # internal information (nothing related to $(Project))
362 # NOTE: $(_SID*) variables indicate if a sub-makefile was included.
363 # NOTE: $$ needs to be used to ensure that the variables are evaluated when
364 # the targets executes and not when the Makefile is read.
365 # NOTE: when target testcmd-test.internal is called with -n ; the command:
366 # @echo '# $$(MAKE) = $(MAKE)'
367 # it will produce output:
368 # # $(MAKE) = make
369 # SEE GNU Make:MAKE vs. MAKE_COMMAND
370 # TODO: list of $(_SID.*) should be generated using $(ALL.inc.type)
371 test.file-1:
372 test,file-2:
373 test_file-3:
374 testcmd-test.internal: test.file-1 test,file-2 test_file-3
375 @$(TRACE.target)
376 @echo 'test.target: test.file-1 test-file-2 test_file-3'
377 @echo '# SIDs of included Makefile* (file not included if SID missing):'
378 @echo '# $$(_SID) = $(_SID)'
379 @echo '# $$(_SID.help) = $(_SID.help)'
380 @echo '# $$(_SID.test) = $(_SID.test)'
381 @echo '# $$(_SID.critic) = $(_SID.critic)'
382 @echo '# $$(_SID.warnings) = $(_SID.warnings)'
383 @echo '# $$(_SID.cipher) = $(_SID.cipher)'
384 @echo '# $$(_SID.make) = $(_SID.make)'
385 @echo '# $$(_SID.exit) = $(_SID.exit)'
386 @echo '# $$(_SID.init) = $(_SID.init)'
387 @echo '# $$(_SID.misc) = $(_SID.misc)'
388 @echo '# $$(_SID.cmd) = $(_SID.cmd)'
389 @echo '# $$(_SID.dev) = $(_SID.dev)'
390 @echo '# $$(_SID.mod) = $(_SID.mod)'
391 @echo '# $$(_SID.etc) = $(_SID.etc)'
392 @echo '# $$(_SID.ext) = $(_SID.ext)'
393 @echo '# $$(_SID.cgi) = $(_SID.cgi)'
394 @echo '# $$(_SID.gen) = $(_SID.gen)'
395 @echo '# $$(_SID.hlp) = $(_SID.hlp)'
396 @echo '# $$(_SID.inc) = $(_SID.inc)'
397 @echo '# $$(_SID.opt) = $(_SID.opt)'
398 @echo '# $$(_SID.pod) = $(_SID.pod)'
399 @echo '# $$(_SID.tcl) = $(_SID.tcl)'
400 @echo '# $$(_SID.docker) = $(_SID.docker)'
401 @echo '# $$(_SID.legacy) = $(_SID.legacy)'
402 @echo '# $$(_SID.template) = $(_SID.template)'
403 @echo '# show some private make variables:'
404 @echo '# $$(PWD) = $(PWD)'
405 @echo '# $$(PWD.dir) = $(notdir $(PWD))'
406 @echo '# $$(ALL.includes) = $(ALL.includes)'
407 @echo '# $$(ALL.inc.type) = $(ALL.inc.type)'
408 @echo '# $$(TEST.dir) = $(TEST.dir)'
409 @echo '# $$(TEST.logdir) = $(TEST.logdir)'
410 @echo '# $$(TEST.host) = $(TEST.host)'
411 @echo '# $$(TEST.hosts) = $(TEST.hosts)'
412 @echo '# $$(ALL.Makefiles) = $(ALL.Makefiles)'
413 @echo '# show some make variables:'
414 @echo '# $$@ = $@ #'
415 @echo '# $$< = $< #'
416 @echo '# $$? = $? #'
417 @echo '# $$^ = $^ #'
418 @echo '# $$+ = $+ #'
419 @echo '# $$| = $| #'
420 @echo '# $$% = $% #'
421 @echo '# $$* = $* #'
422 @echo '# $$> = $> #'
423 @echo '# $$- = $- #'
424 @echo '# $$D = $D #'
425 @echo '# $$F = $F #'
426 @echo '# $$T = $T #'
427 @echo '# not shown: $$(%D) $$(?D) $$(@D) $$(*D) $$(<D) $$(?D) $$(^D) $$(%F) $$(?F) $$(@F) $$(*F) $$(<F) $$(?F) $$(^F)'
428 @echo '# $$(MAKE) = $(MAKE)'
429 @echo '# $$(CURDIR) = $(CURDIR)'
430 @echo '# $$(MAKE_COMMAND) = $(MAKE_COMMAND)'
431 @echo '# $$(MAKE_VERSION) = $(MAKE_VERSION)'
432 @echo '# $$(MAKE_HOST) = $(MAKE_HOST)'
433 @echo '# $$(MAKE_RESTARTS) = $(MAKE_RESTARTS)'
434 @echo '# $$(MFLAGS) = $(MFLAGS)'
435 @echo '# $$(MAKEFLAGS) = $(MAKEFLAGS)'
436 @echo '# $$(MAKELEVEL) = $(MAKELEVEL)'
437 @echo '# $$(MAKEFILE) = $(MAKEFILE)'
438 @echo '# $$(MAKEFILES) = $(MAKEFILES)'
439 @echo '# $$(MAKEFILE_LIST) = $(MAKEFILE_LIST)'
440 @echo '# $$(MAKEFILE_LIST)F= $(firstword $(MAKEFILE_LIST))'
441 @echo '# $$(MAKEFILE_LIST)L= $(lastword $(MAKEFILE_LIST))'
442 @echo '# $$(MAKEOVERRIDES) = $(MAKEOVERRIDES)'
443 @echo '# $$(MAKECMDGOALS) = $(MAKECMDGOALS)'
444 @echo '# $$(GNUMAKEFLAGS) = $(GNUMAKEFLAGS)'
445 @echo '# $$(.DEFAULT_GOAL) = $(.DEFAULT_GOAL)'
446 @echo '# $$(.INCLUDE_DIRS) = $(.INCLUDE_DIRS)'
447 @echo '# $$(.INTERMEDIATE) = $(.INTERMEDIATE)'
448 @echo '# $$(.PRECIOUS) = $(.PRECIOUS)'
449 @echo '# $$(.RECIPEPREFIX) = $(.RECIPEPREFIX)'
450 @echo '# $$(.SECONDARY) = $(.SECONDARY)'
451 @echo '# $$(.SHELLFLAGS) = $(.SHELLFLAGS)'
452 @echo '# $$(.SUFFIXES) = $(.SUFFIXES)'
453 @echo '# $$(.FEATURES) = $(.FEATURES)'
454 @echo '# $$(.VARIABLES) = $(words $(.VARIABLES)) variables'
455 @echo '# huge list .VARIABLES not shown (includes own names),'
456 @echo '# use: make help.test.makevars'
457 # not printed because different for each call: $(MAKE_TERMOUT) $(MAKE_TERMERR)
458 # not used as most likely not a variable: .DEFAULT .PHONY .SECONDEXPANSION
459
460 # TODO (2022) not yet added to proper variables
461 ALL.help.test += testcmd-test.internal
462 ALL.tests += testcmd-test.internal
463
464 help.test.makevars:
465 @echo '# $$(.VARIABLES) = '
466 @echo '$(.VARIABLES)' | $(EXE.wordperline) | sort $(_EXE.sort-opt)
467
468 .PHONY: help.test.makevars
469
470 #_____________________________________________________________________________
471 #______________________________________________________ targets for testing __|
472
473 # FIXME: missing public (not internal) documentation for pattern rules:
474 # message-%:
475 # no.message-%:
476 # testarg-%:
477 # testcmd-%
478 # test.all-%:
479
480 $(TEST.logdir):
481 @mkdir $@
482
483 HELP-test.links = create required symbolic links in '$(TEST.dir)'
484 # creating links is scary, fails if they exist; therefore ln --force is used,
485 # but this may destroy existing links or files silently
486 # because the target itself is never created, it will always be executed
487 # use this target (rule) with care!
488 ../%:
489 @$(TRACE.target)
490 cd $(TEST.dir) && ln --force -s $@
491
492 test.links: $(TEST.symlinks)
493
494 # target to check for empty hostname list, can also be used for more debugging
495 _no-hosts__not-yet-working:
496 @[ "$(eTEST.hosts)" = "" ] && $(eval _ERR := "no TESTS.hosts defined")
497 @[ "$(eTEST.hosts)" = "from.Makefile.FQDN" ] \
498 && $(eval _ERR := "fake TESTS.hosts defined in $(TEST.file)")
499 @[ -n "$(_ERR)" ] && echo $(_ERR) && exit 2 || echo -n ""
500
501 _no-hosts:
502 @-[ "$(eTEST.hosts)" = "" ] \
503 && echo no TESTS.hosts defined \
504 && exit 2 \
505 || echo -n ""
506 @-[ "$(eTEST.hosts)" = "from.Makefile.FQDN" ] \
507 && echo fake TESTS.hosts defined in $(TEST.file) \
508 && exit 2 \
509 || echo -n ""
510
511 .PHONY: _no-hosts
512
513
514 # Testing for messages or other strings (i.e **WARNING) works as follows:
515 # call $(EXE.pl) with command and/or options in question
516 # then search (grep) output for message (string)
517 # For some behaviours of $(EXE.pl) a RC-file is required.
518 # Different target rules can be mapped to the pattern rule message-%. It gets
519 # the message string from the automatic variable $* , and all arguments for
520 # $(EXE.pl) with following Makefile variables:
521 # $(TEST.init) - command, options to be passed to $(EXE.pl)
522 # $(TEST.args) - command, options and hostname to be passed to $(EXE.pl)
523 # $(TEST.rc) - content of RC-file to be used by $(EXE.pl)
524 # These variables can be set conditinally for each target, see example below.
525 # Some tests are not yet implemented, or difficult to implement. In this case
526 # $(TEST.args) contains a string starting with "TODO:". The message-% target
527 # tests the variable for this string and then simply prints it. Otherwise the
528 # check will be performed (see if - else - fi in message-% rule).
529 # The pattern rule succeeds (returns status 0) if the pattern is found. The
530 # rule fails, if the pattern is not found. That's why it is very important to
531 # define TEST.args propperly. Even the sequence of the arguments may count.
532 #
533 # NOTE: the called $(EXE.pl) may return the expected pattern given by the $*
534 # pattern multiple times, even in not intended output. In this case, the rule
535 # succeeds also, as the pattern is found in the output. Example:
536 # make message-unintended TEST.args="host-unintended +cn"
537 #
538 # NOTE: even TEST.tmp.rc is generated for each call, it will only be used
539 # when requested with the --rc=some.rc option.
540 #
541 # NOTE: if TEST.args contains special characters, syntax errors may occour
542 # when used in the shell. Quoting TEST.args is not possible because then it
543 # becomes a single argument instead of discrete arguments. At least following
544 # special characters should not be used: ; & | # < > ` [ ]
545 #
546 # Example of Usage
547 # pattern rule: warning-%
548 # target rule: warning-049
549 # pattern: 049
550 # arguments: TEST.args = +unknown_command +quit
551 # Example for the target rule with above settings will be:
552 # warning-%: EXE.pl = ../$(SRC.pl)
553 # warning-%: TEST.init = --init-option
554 # warning-049: TEST.args = +unknown_command +quit
555 # Map all pattern rules to message-% pattern rule (recipe need command)
556 # warning-%: message-%
557 # @echo -n ""
558 # The recipe in the pattern rule message-% will be:
559 # $(EXE.pl) $(TEST.init) $(TEST.args) 2>&1 | grep 049
560 # which finally evaluates to:
561 # ../o-saft.pl --init-option +unknown_command +quit 2>&1 | grep 049
562 # +quit command or other command and hostname is needed
563 # for testing the warning message
564
565 _FORCE:
566
567 # target succeeds if message is there
568 message-%:
569 @$(TRACE.target)
570 @-if expr "$(TEST.args)" ":" "^TODO" >/dev/null ; then \
571 echo "$@: $(TEST.args)"; \
572 else \
573 echo "$(TEST.rc)" > $(TEST.tmp.rc) ; \
574 echo "cd $(TEST.dir) && $(EXE.pl) $(TEST.init) $(TEST.args) 2>&1 | grep $(_EXE.grep-opt) $* " ; \
575 cd $(TEST.dir) && $(EXE.pl) $(TEST.init) $(TEST.args) 2>&1 | grep $(_EXE.grep-opt) $* ; \
576 _status=$$? ; \
577 rm -f $(TEST.tmp.rc) ; \
578 exit $$_status ; \
579 fi
580 # following removed from above, too noicy:
581 # echo "echo '$(TEST.rc)' > $(TEST.tmp.rc)" ;
582
583 # target succeeds if message is missing
584 # TODO: need more examples beside those in t/Makefile.cgi
585 no.message-%:
586 @$(TRACE.target)
587 @echo "$(TEST.rc)" > $(TEST.tmp.rc)
588 cd $(TEST.dir) && $(EXE.pl) $(TEST.init) $(TEST.args) 2>&1 | awk '/ $*/{exit 1}'
589 @rm -f $(TEST.tmp.rc)
590
591 # Simple target to calL: $(EXE.pl) $(TEST.init) $(TEST.args)
592 testarg-%:
593 @$(TRACE.target)
594 -cd $(TEST.dir) && $(EXE.pl) $(TEST.init) $(TEST.args)
595
596 # SEE Make:--dry-run
597 # following pattern rule should be unified with testcmd-%.log
598 testarg-%.log: $(TEST.logdir) _FORCE
599 @$(TRACE.target)
600 @$(TRACE.target.log)
601 @$(eval _NEW.log := $(TEST.logdir)/$@-$(TEST.today))
602 @expr "$(MAKEFLAGS)" : n >/dev/null \
603 && echo "$(MAKE) $(MFLAGS) -s testarg-$* 2>&1 | $(EXE.log-filterarg) > $@ 2>&1" \
604 || $(MAKE) $(MFLAGS) -s testarg-$* 2>&1 | $(EXE.log-filterarg) > $@ 2>&1
605 @-diff $(TEST.logdir)/$@ $@ 2>/dev/null \
606 && rm $@ \
607 || mv $@ $(_NEW.log)
608 @-test -f $(TEST.logdir)/$@ || mv $(_NEW.log) $(TEST.logdir)/$@
609 @-ls -l $(TEST.logdir)/testarg-$(*)*
610 # TODO: target should fail if there is a diff
611
612
613 # The goal for test targets is to perform (test) all commands with all hosts.
614 # The hostnames are provided in a simple list: $(TEST.hosts) .
615 # The commands could not be provided in a make variable all together, because
616 # each command may consist of space separated words like: "+info --header".
617 # Hence each command is defined in the variable TEST.args, which will be set
618 # for an individual target, for example: testcmd-001 .There is one target for
619 # each specific test case, which actually is a list of commands and options
620 # for the tool $(EXE.pl) . Note that the trailing DDD (001 in example above)
621 # is just a number to make each target unique.
622 # To feed the hostname to that target, the target is defined as pattern rule
623 # testcmd-001_% , which means that the hostname can be passed like:
624 # testcmd-001_host.some.tld
625 # Now we can simply use: $(TESTS.hosts:%=testcmd-001_%) , which generates one
626 # target for each host in the list. But that would require to build a list for
627 # each such target: testcmd-002_% and testcmd-003_% and so on.
628 # As we want to perform all these targets with all hostnames, this would also
629 # require an additional pattern rule for the hostname part. To avoid such an
630 # addditional pattern rule for each testcmd-DDD, the general pattern rule
631 # testcmd-% is used. It handles all the individual targets which contain the
632 # hostname.
633 # Unfortunatelly, this pattern contains DDD_ (for example 001_host.some.tld)
634 # as hostname. This DDD_ prefix must then be removed in the target command,
635 # the $(shell awk ...) does the dirty work. in detail (as defined below):
636 # # pattern rule to handle all targets testcmd-DDD_HOSTNAME
637 # testcmd-%:
638 # #
639 # # remove prefix DDD- from hostname
640 # $(shell awk 'END{h="$*";sub(/^[^_]*-/,"",h);print h}' /dev/null)
641 # # or
642 # $(shell echo "$*" | awk -F_ '{print $$2}') $(TEST.args)
643 # # c00-aa.tld returns: aa.tld
644 #
645 # NOTE: following does not work proper (in GNU Make), hence the solution above:
646 # testcmd-no1: TEST.args = +quit $*
647 # testcmd-no2: TEST.args := +quit $*
648 #
649 # Example of Usage
650 # pattern rule: testcmd-%
651 # target rule: testcmd-c00
652 # pattern: 00c_aa.tld
653 # arguments: TEST.args = +cipher
654 # hosts tested: aa.tld bb.tld
655 # Example for the target rule in t/Makefile.* with above settings will be:
656 # # define list of hostnames
657 # TEST.hosts = aa.tld bb.tld
658 # # define targets
659 # testcmd-c%: EXE.pl = ../$(SRC.pl)
660 # # EXE.pl must be defined wherever testcmd-% is used/referenced
661 # testcmd-c%: TEST.init = --header
662 # #
663 # # define the commands to be used for $(EXE.pl) in the target
664 # testcmd-c00_%: TEST.args += +cipher --enabled
665 # testcmd-c01_%: TEST.args += +info
666 # #
667 # # dynamically generate list of all testcmd-DDD targets
668 # ALL.ext.cmd = $(shell awk -F_ '/^testcmd-c[0-9]/{print $$1}' t/Makefile.ext)
669 # # returns: testcmd-c00 testcmd-c01
670 # #
671 # # dynamically generate list of all testcmd-DDD for all hostnames
672 # ALL.by_host = $(foreach host,$(TEST.hosts),$(ALL.ext.cmd:%=%_$(host)))
673 # # returns: testcmd-c00_aa.tld testcmd-c01_aa.tld testcmd-c00_bb.tld testcmd_c01-bb.tld
674
675 testcmd-%:
676 @$(TRACE.target)
677 -cd $(TEST.dir) && $(EXE.pl) $(shell echo "$*" | awk -F_ '{print $$NF}') $(TEST.init) $(TEST.args)
678 # TODO: need verbose for executed command
679 # TODO: need to add --no-dns if hostname is an IP
680
681 # Target should create a new logfile, then compare it with the current one. If
682 # diff returns nothing, delete newly created logfile, otherwise rename newly
683 # created file to name which contains the current date.
684 # Finally, if current logfile is/was missing, use newly created one:
685 # "test ... || mv ..." . This ensures that the file exists afterwards.
686 # diff's STDERR is discarded (may return: "file does not exist").
687 # The target's command output is piped to $(EXE.log-filtercmd), which is cat
688 # by default. This filter can be redefined as needed. SEE Make:OSAFT_MAKE
689 # NOTE: all target commands are prefixed with - this avoids that make reports
690 # errors if the command fails (as failture is intended, somehow).
691 # NOTE: testcmd-%.log called from within t/ may return: is up to date.
692
693 testcmd-%.log: $(TEST.logdir) _FORCE
694 @$(TRACE.target)
695 @$(TRACE.target.log)
696 @$(eval _NEW.log := $(TEST.logdir)/$@-$(TEST.today))
697 @expr "$(MAKEFLAGS)" : n >/dev/null \
698 && echo "$(MAKE) $(MFLAGS) -s testcmd-$* 2>&1 | $(EXE.log-filtercmd) > $@ 2>&1" \
699 || $(MAKE) $(MFLAGS) -s testcmd-$* 2>&1 | $(EXE.log-filtercmd) > $@ 2>&1
700 @-diff $(TEST.logdir)/$@ $@ >/dev/null 2>&1 || echo "diff $(TEST.logdir)/$@ ..."
701 @-diff $(TEST.logdir)/$@ $@ 2>/dev/null \
702 && rm $@ \
703 || mv $@ $(_NEW.log)
704 @-test -f $(TEST.logdir)/$@ || mv $(_NEW.log) $(TEST.logdir)/$@
705 @-ls -l $(TEST.logdir)/testcmd-$(*)*
706 # TODO: target should fail if there is a diff
707
708
709 # The pattern rule testcmd-% executes an individual target, for example:
710 # testcmd-c001_localhost . The following pattern executes all matching targets
711 # at once. The grouping is done by the rule's pattern. The pattern is searched
712 # for in $(ALL.tests) .
713 # TODO: more documentation ...
714 # TODO: Examples:
715 # test.pattern-info
716 # test.pattern-+info
717 # test.pattern-+check
718 # test.pattern-+cipher
719 # test.pattern-+ciphers
720 # test.pattern--header
721 #
722 dbx-test.pattern-%:
723 @$(TRACE.target)
724 @$(eval _targets = $(shell echo "$(ALL.tests)" | $(EXE.wordperline) | awk '/$*/{print $$0;}'))
725 @echo "# $(_targets) #"
726 test.pattern-%:
727 @$(TRACE.target)
728 @$(eval _targets = $(shell echo "$(ALL.tests)" | $(EXE.wordperline) | awk '/$*/{print $$0;}'))
729 @-test -n "$(_targets)" \
730 && $(MAKE) $(MFLAGS) -s $(_targets) \
731 || echo "# pattern '$*' does not match any target"
732
733 # TODO: logging not yet tested (01/2019)
734 test.pattern-%.log:
735 @$(TRACE.target)
736 @$(TRACE.target.log)
737 @$(eval _targets = $(shell echo "$(ALL.tests)" | $(EXE.wordperline) | awk '/$*/{print $$0".log";}'))
738 @-test -n "$(_targets)" \
739 && $(MAKE) $(MFLAGS) -s $(_targets) \
740 || echo "# pattern '$*' does not match any target"
741
742
743 # some alias targets (humans tend to be lazy:)
744 # FIXME: following buggy; ends with command: rm test.pattern-%
745 test.pat-%: test.pattern-%
746 @echo -n ""
747 test.patt-%: test.pattern-%
748 @echo -n ""
749 #_____________________________________________________________________________
750 #__________________________________________________ include testing targets __|
751
752 # includes are done explicitly instead of:
753 # include t/Makefile.*
754 # to avoid multiple inlcudes of the same file, which would result in make
755 # errors complaining about target redefinitions
756 # for a detailed description see t/Makefile.template
757
758 ifeq (,$(_SID.gen))
759 -include t/Makefile.gen
760 endif
761
762 ifeq (,$(_SID.help))
763 -include t/Makefile.help
764 endif
765
766 ifeq (,$(_SID.warnings))
767 -include t/Makefile.warnings
768 endif
769
770 ifeq (,$(_SID.cipher))
771 -include t/Makefile.cipher
772 endif
773
774 ifeq (,$(_SID.cmd))
775 -include t/Makefile.cmd
776 endif
777
778 ifeq (,$(_SID.exit))
779 -include t/Makefile.exit
780 endif
781
782 ifeq (,$(_SID.opt))
783 -include t/Makefile.opt
784 endif
785
786 ifeq (,$(_SID.ext))
787 -include t/Makefile.ext
788 endif
789
790 ifeq (,$(_SID.hlp))
791 -include t/Makefile.hlp
792 endif
793
794 ifeq (,$(_SID.cgi))
795 -include t/Makefile.cgi
796 endif
797
798 ifeq (,$(_SID.tcl))
799 -include t/Makefile.tcl
800 endif
801
802 ifeq (,$(_SID.etc))
803 -include t/Makefile.etc
804 endif
805
806 ifeq (,$(_SID.dev))
807 -include t/Makefile.dev
808 endif
809
810 ifeq (,$(_SID.mod))
811 -include t/Makefile.mod
812 endif
813
814 ifeq (,$(_SID.init))
815 -include t/Makefile.init
816 endif
817
818 ifeq (,$(_SID.misc))
819 -include t/Makefile.misc
820 endif
821
822 ifeq (,$(_SID.critic))
823 -include t/Makefile.critic
824 endif
825
826 ifeq (,$(_SID.docker))
827 -include t/Makefile.docker
828 endif
829
830 ifeq (,$(_SID.legacy))
831 -include t/Makefile.legacy
832 endif
833
834 ifeq (,$(_SID.make))
835 -include t/Makefile.make
836 endif
837
838 #_____________________________________________________________________________
839 #_____________________________________________________________________ test __|
840
841 HELP-_test = ______________________________________ targets for testing _
842 HELP-tests = make all tests
843 HELP-test = alias for tests
844 HELP-tests.log = same as tests but store results in '$(TEST.logdir)/'
845 HELP-tests.quick = like tests, but less targets (for development)
846 HELP-tests.quick.log = like tests.log, but less targets (for development)
847 HELP-help.test.all = print available individual targets for testing
848 HELP-test.log.ls = list files in '$(TEST.logdir)/' generated by test.log
849 HELP-test.%.log.ls = list files in '$(TEST.logdir)/' generated by test.EXT.log
850 HELP-test.pattern-% = only execute targets matching '%'
851
852 tests: TRACE.target = echo "\\012\#\# $@: $(EXE.pl) $(TEST.args)"
853 tests: $(ALL.tests)
854 @$(TRACE.target)
855 @$(MAKE) $(MFLAGS) -s $(ALL.tests)
856
857 tests.log: $(ALL.tests.log)
858 @$(TRACE.target)
859 @$(MAKE) $(MFLAGS) -s $(ALL.tests.log)
860 @-$(EXE.log-compare-hint)
861 @-$(EXE.log-move-hint)
862
863 test.tests.log-compare: TEST.target_prefix = test
864 test.tests.log-move: TEST.target_prefix = test
865 test.tests.log: TEST.target_prefix = test
866 test.log-compare: TEST.target_prefix = test
867 test.log-move: TEST.target_prefix = test
868
869 # quick tests for development
870 # the target is named tests.quick instead of test.quick to avoid future naming
871 # conflicts (if there is a Makefile.quick);
872 # unfortunately the targets test*s.quick.log* must then be explicitly defined
873 # because the pattern rules test.%.log* do not match
874 tests.quick:
875 @$(MAKE) $(MFLAGS) -s tests ALL.test.ext= ALL.test.misc= ALL.test.critic=
876 #tests.quick.log:
877 # @$(MAKE) $(MFLAGS) -s tests.log ALL.test.ext= ALL.test.misc ALL.test.critic=
878 # FIXME: tests.quick.log not yet working; need to use variable in each Makefile.*
879 tests.quick.log:
880 @$(TRACE.target)
881 @$(TRACE.target.log)
882 $(MAKE) $(MFLAGS) -s tests.log ALL.tests.log="test.warnings.log test.cmd.log test.exit.log test.opt.log test.cgi.log test.tcl.log"
883 @-$(EXE.log-compare-hint)
884
885 tests.quick.log-compare:
886 @$(TRACE.target)
887 @-$(EXE.log-compare)
888 @-$(EXE.log-move-hint)
889 @echo "# 12 logfiles with differences could be expected."
890
891 tests.quick.log-move:
892 @$(TRACE.target)
893 $(EXE.log-move)
894
895 # aliases for convenience
896 test: tests
897 test.log: tests.log
898 @$(TRACE.target)
899 @echo "# to show differences with $(EXE.diff), use:"
900 @echo " $(MAKE_COMMAND) $@-compare"
901
902 test.log.ls:
903 @$(TRACE.target)
904 @-ls $(TEST.logdir)/test.*log
905
906 test.%.log.ls:
907 @$(TRACE.target)
908 @-ls -l $(TEST.logdir)/test*-$(*)*log
909
910
911 # collect targets in this file, should not be added to ALL.tests
912 ALL.test.self += tests tests.quick tests.log tests.quick.log
913
914 .PHONY: test tests test.log tests.log
915
916 #_____________________________________________________________________________
917 #________________________________________ targets for checking test results __|
918
919 HELP-_compare = _________ targets to compare and rename generated logfiles _
920 HELP-test.log-compare = compare logfiles of previous with latest test
921 HELP-test.log-move = rename logfiles of latest test
922 HELP-help.test.log-info = print more details about test.log.* targets
923
924 HELP.test.log-info = $(_NL)\
925 \# $(HELP-_compare)$(_NL)\
926 test.log-compare \# static rule to compare logfiles$(_NL)\
927 test.log-move \# static rule to rename (move) latest logfiles to "standard" logfile$(_NL)\
928 test.%.log-compare \# pattern rule for test.log-compare$(_NL)\
929 test.%.log-move \# pattern rule for test.log-move$(_NL)\
930 $(_NL)\
931 \#$(_NL)\
932 \# all above targets can be controlled with environment variables:$(_NL)\
933 \# TEST.logdir - directory where to find logfiles$(_NL)\
934 \# TEST.target_prefix - prefix of targets to search logfiles for (files to be matched)$(_NL)\
935 \# TEST.log-suffix - suffix of logfiles to be compared to "standard" logfile$(_NL)\
936 \# EXE.log-diff - program used to show difference of latest and "standard" logfile$(_NL)\
937 \# defaults:$(_NL)\
938 \# TEST.logdir = $(TEST.logdir)$(_NL)\
939 \# TEST.target_prefix = $(TEST.target_prefix)$(_NL)\
940 \# TEST.log-suffix = $(TEST.log-suffix)$(_NL)\
941 \# EXE.log-diff = $(EXE.log-diff)$(_NL)\
942 \# test example with adapted environment variables:$(_NL)\
943 \# $(MAKE_COMMAND) $@ TEST.logdir=/tmp TEST.log-suffix=-0815$(_NL)\
944 \#$(_NL)\
945 \# to see what the targets actually do, use$(_NL)\
946 \# (environment variables may be added as needed, see above):$(_NL)\
947 \# $(MAKE_COMMAND) -n test.log-compare$(_NL)\
948 \# $(MAKE_COMMAND) -n test.log-move$(_NL)\
949
950 help.test.log-info:
951 @echo "$(HELP.test.log-info)"
952
953 # summary variables (mainly used for INSTALL.sh)
954 _ALL.devtools.extern += $(EXE.diffs)
955
956 # default prefix, may be redifined per target
957 TEST.target_prefix = testcmd-
958 TEST.target_logfile = $(TEST.logdir)/testcmd-
959
960 # default suffix for logfiles
961 TEST.log-suffix = -$(TEST.today)
962
963 test.log-compare-hint:
964 @-$(EXE.log-compare-hint)
965
966 test.%.log-compare:
967 @$(TRACE.target)
968 @-$(EXE.log-compare)
969 @-$(EXE.log-move-hint)
970
971 test.%.log-move:
972 @$(TRACE.target)
973 $(EXE.log-move)
974
975 test.log-compare:
976 @$(TRACE.target)
977 @-$(EXE.log-compare)
978 @-$(EXE.log-move-hint)
979
980 test.log-move:
981 @$(TRACE.target)
982 $(EXE.log-move)
983
984 #_____________________________________________________________________________
985 #________________________________________________ collected ALL.* variables __|
986
987 ALL.tests :=
988 ALL.tests.log :=
989
990 # add all tests from included files to ALL.tests += ALL.test.$INCLUDE
991 ifndef ALL-macros-generated
992 $(foreach inc, $(ALL.inc.type), $(eval ALL.tests += $(ALL.test.$(inc))) )
993 $(foreach inc, $(ALL.inc.type), $(eval ALL.tests.log += $(ALL.test.$(inc).log)) )
994 endif
995
996 # satisfy generated variables
997 ALL.test.test := $(ALL.tests)
998 ALL.test.test.log := $(ALL.tests.log)