#! /usr/bin/make -rRf #? #? NAME #? Makefile - makefile for testing O-Saft warning messages #? #? DESCRIPTION #? For more details please see #? ../Makefile Makefile Makefile.help Makefile.pod #? make help.test.warnings #? #? Makefile with targets for testing warnings of o-saft.pl . #? All targets succedd, if the warning message is present, they fail, if #? the warning message is not present. #? #? VERSION #? @(#) Makefile.warnigs 1.73 22/10/29 18:58:26 #? #? AUTHOR #? 18-apr-18 Achim Hoffmann #? # ----------------------------------------------------------------------------- HELP-help.test.warnings = targets for testing '$(SRC.pl)' WARNING messages _SID.warnings := 1.73 _MYSELF.warnings := t/Makefile.warnings ALL.includes += $(_MYSELF.warnings) ALL.inc.type += warnings ALL.help.tests += help.test.warnings first-warn-target-is-default: help.test.warnings ifeq (,$(_SID.test)) -include t/Makefile endif ifndef TEST.host TEST.host = localhost endif help.test.warnings: HELP_TYPE = warnings help.test.warnings-v: HELP_TYPE = warnings help.test.warnings-vv: HELP_TYPE = warnings #_____________________________________________________________________________ #________________________________________________________________ variables __| # extract all line looking like: # die $STR{ERROR}, # warn $STR{ERROR}, # warn $STR{WARN}, # warn $STR{WARN} . # warn($STR{ # warn "**WARNING: # _warn(" # _warn($STR{ # Because perl comments with # cannot be used inside Makefile's define or macro # definitions, the following sequence of the perl code is explained first: # skip assignments # remove opening ( if any # if line mtches die or warn or print # replace die|warn by **WARNING: or **ERROR: # replace _warn by **WARNING: # remove postfix condition # remove ") and anything reight to it # remove _warn (if still there) # TODO: warn and die not detected, if they do not appear as first word in line EXE.extract_warn := perl -nle '\ m/^\s*warn\s+=/ && do {next};\ s/[(]/ / if m/^\s*(?:die|_?warn|print)\s*\(\s*$$STR.[EW]/;\ m/^\s*(?:die|_?warn|print $$STR.[EW])/ && do {\ s/^\s*(?:print|die|warn)\s+$$STR.ERROR.,\s"/**ERROR: /;\ s/^\s*(?:print|die|warn)\s+$$STR.WARN.[,.\s]+"/**WARNING: /;\ s/^\s*(?:print|die|warn)[\s(]+"\$$STR.WARN.[,.\s]+/**WARNING: /;\ s/",\s+"/ /;\ s/^\s*_warn(?:_nosni)?\s*\(\s*"/**WARNING: /;\ s/ +if +\(.*//;\ s/"\s*\)?\s*;?\s*$$//;\ s/^\s*_?warn\s*\(?"//;\ print;\ }' EXE.gen_warning-target := perl -nle '\ s/^..(?:ERROR|WARNING): /warning-/;\ s/:/: TEST.args = TODO:/;\ print;' #_____________________________________________________________________________ #_________________________________________________________ testing warnings __| HELP-_warnings = ____________________________________________ test warnings _ HELP-test.warnings = test **WARNING messages of '$(SRC.pl)' HELP-test.warnings.log = test **WARNING messages of '$(SRC.pl)' and compare with previous one HELP-message-STR = test for specific STR in output of '$(SRC.pl)' HELP-warning-NR = test for specific messages number NR in output of '$(SRC.pl)' HELP-warnings-info = get all **WARNING messages from '$(SRC.pl)' HELP-warnings-show.todo = show not implemented tests for warnings HELP-warnings-show.difficult= show tests for warnings which are difficult to implement HELP-warnings.gen.template = generate template Makefile for testing warning messages HELP.warnings = $(_NL)\ \# Hint: message-STR can test for any string in output, example$(_NL)\ \# $(MAKE_COMMAND) message-Certificate TEST.args='+cn localhost'$(_NL)\ \#$(_NL)\ \# Examples:$(_NL)\ \# $(MAKE_COMMAND) warning-042$(_NL)\ \# $(MAKE_COMMAND) message-Certificate TEST.args='+cn localhost'$(_NL)\ \#$(_NL)\ \# Note that using -n reports a bunch of shell commands. HELP.test.warnings.all = \ \# for more details, use:$(_NL)\ \# $(MAKE_COMMAND) t-warn$(_NL)\ # SEE Make:target name # SEE Make:target name prefix # testing warning messages TEST.tmp.rc := /tmp/o-saft.tmprc test.warnings%: EXE.pl = ../$(SRC.pl) warning-%: EXE.pl = ../$(SRC.pl) warning-%: TEST.init = # How target rules in this Makefile (should) work: # The pattern rule message-% from ./Makefile is used to check for a # specific warning message. warning-%: message-% @$(TRACE.target) # The pattern rule message-% searches (grep) the output of o-saft.pl # for a string, which is the pattern of the target name. # Each target rule defines its own arguments to be passed to o-saft.pl # # Mainly the +quit command is sufficient to get the warning. # The first target rule warning-000 is designed to fail. # All targets rules are summarized in the target test.warnings . # # To log the results of these tests, only one file will be created, not # one for each target rule. The target is test.warnings.log . # # All targets define EXE.pl which is the command to be used. # All targets should define TEST.args with the arguments, which force # the warning message DDD given in the target's name warning-DDD . # Some messages can be tested without any target host, then the +quit # command is added to TEST.args . Others need a target host argument # somehow, "any-host" is used for that. Using "any-host" most likely # causes $(EXE.pl) to fail with more WARNNG messages printed, but they # are silently ignored because the message with the specific number DDD # only is extracted. # If the definition of TEST.args starts with the literal string "TODO:" # the target will only print the text (see message-%). # The "TODO:" texts ending with "difficult ..." mark messages, which # cannot be tested easily. They are currently disabled, means that the # "TODO" text is printed by the target. # # Examples # warning-049: TEST.args = +unknown_command +quit # warning-004: TEST.args = TODO: testing perl status, difficult ... # # Limitations # The name of each individual target rule contains the pattern (049 in # above example), which must be searched for in the output of EXE.pl . # As this is just a simple number in all rules here, the search may re- # sult in false positives. Manual checks of the rule commands (recipe) # are recommended from time to time. # # NOTE: in rare cases the target may succedd wrongly, see description # of message-% pattern rule in t/Makefile. # TODO: the message text assigned to TEST.args is mainly a copy of the # the text used in the source code; should find a way to generate # these assignments (done manually here, which is error-prone) warning-000: TEST.args = --invalid_nr_to_show_failed-status warning-002: TEST.args = TODO: testing openssl returning error, difficult ... warning-003: TEST.args = TODO: testing openssl failed with allocate memory, difficult ... warning-004: TEST.args = TODO: testing perl returned status, difficult ... warning-005: TEST.args = TODO: testing missing IO/Socket/SSL.pm warning-006: TEST.args = TODO: testing missing IO/Socket/INET.pm, disabled ... warning-007: TEST.args = TODO: testing missing Net/DNS.pm warning-008: TEST.args = TODO: testing missing Time/Local.pm #warning-009: TEST.args = free warning-010: TEST.args = TODO: testing die, missing Net/SSLhello.pm warning-011: TEST.args = TODO: testing die, missing Net/SSLinfo.pm warning-012: TEST.args = TODO: testing die, missing Net::SSLeay.pm warning-013: TEST.args = TODO: testing die, missing o-saft-man.pm, disabled ... warning-014: TEST.args = TODO: testing die, missing Net::SSLeay.pm warning-015: TEST.args = TODO: testing die, no ciphers found, may happen with openssl pre 1.0.0, difficult ... # 016, 017: programing error warning-016: TEST.args = TODO: testing undefined cipher description for, difficult ... warning-017: TEST.args = TODO: testing partial match for cipher name found, difficult ... warning-020: TEST.args = TODO: testing die, CGI mode requires strict settings, disabled ... warning-021: TEST.args = --set-score=unknown +quit warning-022: TEST.args = --cfgcmd-is=unknown +quit warning-023: TEST.args = --legacy=key +quit #warning-040: TEST.args = free warning-041: TEST.args = s_client +quit warning-042: TEST.args = +cn /unknown-host +quit warning-043: TEST.args = +cn --port=' ' unknown-host +quit warning-044: TEST.args = +zlib +lzo +open_pgp +fallback +quit warning-045: TEST.args = any-host --port 45 +quit warning-046: TEST.args = --rc=$(TEST.tmp.rc) --v +quit warning-046: TEST.rc = --cfg_cmd=new_command=quit warning-047: TEST.args = +info +cn any-host warning-048: TEST.args = +check +cn any-host warning-049: TEST.args = +unknown_command +quit warning-050: TEST.args = +cn --port= +quit warning-051: TEST.args = --label=unknown +quit warning-052: TEST.args = --rc=$(TEST.tmp.rc) warning-052: TEST.rc = "--option=-with_trailing_spaces " warning-053: TEST.args = --capath='/path with spaces' +quit warning-054: TEST.args = --legacy=unknown_legacy +quit warning-055: TEST.args = --format=unknown_format +quit warning-056: TEST.args = --range=unknown_range +quit warning-057: TEST.args = --cipher-mode=unknown +quit warning-058: TEST.args = --ca-path=unknown +quit warning-059: TEST.args = --ca-path=unknown +quit warning-060: TEST.args = --ca-paths=unknown --cfg-init=ca_files=unknown +quit #warning-061: TEST.args = free warning-062: TEST.args = --cipher=unknown +cipher any-host warning-063: TEST.args = --cipher=UNKNOWN +cipher any-host warning-064: TEST.args = +sts --no-http any-host #warning-065: TEST.args = free warning-066: TEST.args = --ignore-out=cn +cn any-host warning-067: TEST.args = --ignore-out=cn +cn --v any-host warning-068: TEST.args = --ignore-out=cn +cn --v any-host warning-069: TEST.args = --sniname=wrong +cn $(TEST.host) warning-070: TEST.args = TODO: testing configuration file cannot be opened #warning-070: TEST.args = --cfg_cmd=$(TEST.tmp.rc) +quit #warning-070: TEST.args = TODO: need special test target which uses unredable --cfg_cmd=$(TEST.tmp.rc) warning-071: TEST.args = --cfg_unknown=dummy=dummy +quit warning-070: TEST.args = TODO: configuration files are not read in CGI mode, difficult ... #warning-072: TEST.args = --cfg_cmd=$(TEST.tmp.rc) --cgi +quit #warning-072: TEST.rc = --cfg_cmd=new_command=not_allowed_with--cgi warning-073: TEST.args = --cfg_cmd=invalid_default_command=default +quit #?# warning-074: TEST.args = --rc=$(TEST.tmp.rc) +quit #?# warning-074: TEST.rc = "--cfg_cmd=dummy=cn unknown_command" warning-074: TEST.args = --cfg_cmd=dummy=unknown_command +quit warning-075: TEST.args = --cfg-init=ca_path=force-message +quit warning-076: TEST.args = --cfg_score=dummy=invalid_value +quit warning-080: TEST.args = TODO: testing Net::SSLeay < 1.49 warning-081: TEST.args = TODO: ancient Net::SSLeay Net::SSLeay::VERSION < 1.49, difficult ... warning-090: TEST.args = TODO: Net::SSLinfo::next_protos no longer supported, please use Net::SSLinfo::protos_alpn instead, difficult ... warning-091: TEST.args = TODO: help file cannot be opened, difficult ... warning-092: TEST.args = TODO: help file unknown syntax:, difficult ... warning-093: TEST.args = TODO: help file cannot be opened, difficult ... warning-190: TEST.args = TODO: open(OSaft/Doc/file.txt): failed, difficult ... warning-191: TEST.args = TODO: no OSaft/Doc/file.txt found, difficult ... warning-111: TEST.args = --mx --nodns +quit warning-111: TEST.args = TODO: testing missing Net/DNS.pm warning-112: TEST.args = +sts_expired +quit warning-112: TEST.args = TODO: testing missing Time/Local.pm need by +sts_expired #warning-113 .. warning-119: TEST.args = free warning-120: TEST.args = TODO: testing ancient perl warning-121: TEST.args = TODO: testing ancient module warning-122: TEST.args = TODO: testing ancient Net::SSLeay warning-123: TEST.args = TODO: testing ancient IO::Socket warning-124: TEST.args = TODO: testing ancient IO::Socket::SSL warning-125: TEST.args = TODO: testing openssl < 1.0.0 warning-126: TEST.args = TODO: testing missing ALPN functionality warning-127: TEST.args = TODO: testing Net::SSLeay < 1.56, ALPN disabled warning-128: TEST.args = TODO: testing openssl < 1.0.2, ALPN disabled warning-129: TEST.args = TODO: testing missing NPN functionality warning-130: TEST.args = TODO: testing Net::SSLeay < 1.46, NPN disabled #warning-131: TEST.args = free warning-132: TEST.args = TODO: testing openssl < 1.0.1, NPN disabled warning-133: TEST.args = TODO: testing Net::SSLeay without OCSP warning-134: TEST.args = TODO: testing Net::SSLeay without EC warning-135: TEST.args = TODO: testing Net::SSLeay < 1.49 #warning-136: TEST.args = free #warning-137: TEST.args = free #warning-138: TEST.args = free #warning-139: TEST.args = free warning-140: TEST.args = +cipherraw --dtlsv1 +quit warning-141: TEST.args = +cipherraw --dtlsv9 +quit warning-141: TEST.args = TODO: testing wrong or unsupported SSL protocol #warning-142: TEST.args = free warning-143: TEST.args = TODO: testing SSL protocol not supported by Net::SSLeay, difficult ... warning-144: TEST.args = TODO: testing missing openssl s_client support for -alpn or -npn, difficult ... warning-145: TEST.args = TODO: testing missing openssl s_client support for -alpn or -npn, difficult ... warning-146: TEST.args = TODO: testing missing openssl -tlsextdebug option warning-147: TEST.args = TODO: testing missing openssl executable in Net::SSLinfo warning-148: TEST.args = TODO: testing missing openssl version -d failed, difficult ... warning-149: TEST.args = --openssl=/does/not/exist +quit warning-201: TEST.args = +cn unknown-host warning-202: TEST.args = +cn --exit=HOST1 www.skype.com # # scary: need a reliable FQDN here -^^^^^^^^^^^^^ warning-203: TEST.args = TODO: testing connection without SNI (errors), difficult ... warning-204: TEST.args = TODO: testing connection without SNI, difficult ... warning-205: TEST.args = TODO: testing connection failed, difficult ... warning-206: TEST.args = TODO: testing connection witout SNI errors from Net::SSLinfo, difficult ... warning-207: TEST.args = TODO: testing connection with openssl failed, difficult ... warning-208: TEST.args = TODO: testing +check without openssl on Windows, difficult ... warning-209: TEST.args = TODO: testing missing SSL version, difficult ... warning-301: TEST.args = TODO: testing continous connection errors, difficult ... warning-302: TEST.args = TODO: testing max connection errors, difficult ... warning-303: TEST.args = TODO: testing unsupported Net::SSLeay::CTX_v2_new, difficult ... warning-304: TEST.args = TODO: testing unsupported Net::SSLeay::CTX_v3_new, difficult ... warning-305: TEST.args = TODO: testing connection _usesocket failed, difficult ... #warning-306 .. warning-310: TEST.args = free warning-311: TEST.args = TODO: testing empty result from openssl, difficult ... warning-312: TEST.args = TODO: testing strange result from openssl, difficult ... warning-312: TEST.args = TODO: testing unknown result from openssl, difficult ... warning-321: TEST.args = TODO: testing _isbleed failed to connect, difficult ... warning-322: TEST.args = TODO: testing _isbleed with openTcpSSLconnection failed, difficult ... warning-323: TEST.args = TODO: testing heartbleed: no reply, difficult ... warning-324: TEST.args = TODO: --sniname=wrong +cn # # scary: need a reliable FQDN here -^^^^^^^^^^^^^ www.skype.com warning-325: TEST.args = TODO: testing connection failed, HTTP disabled, difficult ... warning-331: TEST.args = TODO: testing _isccs: failed to connect, difficult ... warning-332: TEST.args = TODO: testing _isccs: no reply, difficult ... warning-407: TEST.args = --no-rc +cipher -ciphermode=intern --cipher=IS-UNKNOWN $(TEST.host) warning-408: TEST.args = --no-openssl +cipher-dh --exit=HOST2 $(TEST.host) warning-409: TEST.args = --sslv2 --sni +cipher --exit=HOST4 $(TEST.host) warning-410: TEST.args = --sslv2 --sni +cipher --ciphermode=openssl --exit=HOST4 $(TEST.host) warning-411: TEST.args = TODO: testing checked cipher does not match returned cipher, difficult ... warning-412: TEST.args = TODO: testing missing ciphers in sorted list, difficult ... warning-420: TEST.args = TODO: testing empty value [$cipher, $ssl], check ignored; programming error warning-451: TEST.args = TODO: testing function obsolete, please use cipher_openssl, difficult ... warning-510: TEST.args = TODO: testing --help=unknown-table , difficult ... warning-601: TEST.args = TODO: testing connection failed with protocol error, difficult ... warning-602: TEST.args = TODO: testing connection type name mismatch, difficult ... warning-631: TEST.args = TODO: testing SSL protocol mismatch for cipher, difficult ... warning-641: TEST.args = TODO: testing HTTPS request failed, difficult:& needs target with ALPN ... warning-651: TEST.args = TODO: testing Net::SSLeay >= 1.45 required for getting version warning-652: TEST.args = TODO: testing Net::SSLeay >= 1.33 required for getting subjectAltNames warning-653: TEST.args = TODO: testing Net::SSLeay >= 1.30 required for getting commonName warning-654: TEST.args = TODO: testing Net::SSLeay >= 1.45 required for getting fingerprint_md5 warning-655: TEST.args = TODO: testing Net::SSLeay >= 1.46 required for getting some certificate checks warning-801: TEST.args = TODO: testing connection returning unknown label, difficult ... warning-811: TEST.args = TODO: testing ancient openssl version: using '-msg' option to get DH parameters warning-821: TEST.args = TODO: can not print certificate sizes without a certificate, --no-cert warning-831: TEST.args = --testing-+quit__without__--trace=arg +quit warning-841: TEST.args = TODO: used openssl version differs from compiled Net:SSLeay warning-851: TEST.args = TODO: ancient version Net::SSLeay < 1.49 warning-861: TEST.args = TODO: not all ciphers listed # o-saft.pl: programming error (cipher missing in %cipher) warning-862: TEST.args = TODO: unknown cipher key '$key', key ignored, difficult ... warning-863: TEST.args = TODO: unknown cipher key '$key', key ignored, difficult ... # not yet used in o-saft.pl (1/2019) warning-501: TEST.args = TODO: cannot read OSaft/_ciphers_openssl_all.pm warning-501: EXE.pl = ../OSaft/Ciphers.pm warning-502: TEST.args = TODO: cannot read OSaft/_ciphers_iana.pm warning-502: EXE.pl = ../OSaft/Ciphers.pm warning-503: TEST.args = TODO: cannot read OSaft/_ciphers_osaft.pm warning-503: EXE.pl = ../OSaft/Ciphers.pm warning-504: TEST.args = TODO: DATA line%4d: wrong hex key warning-504: EXE.pl = ../OSaft/Ciphers.pm warning-505: TEST.args = TODO: DATA line%4d: wrong number of TAB-separated fields warning-505: EXE.pl = ../OSaft/Ciphers.pm warning-511: TEST.args = getter=0x42 warning-511: EXE.pl = ../OSaft/Ciphers.pm warning-513: TEST.args = TODO: partial match for cipher name found, difficult ... warning-513: EXE.pl = ../OSaft/Ciphers.pm # Ciphers.pm warning-520: TEST.args = testing ciphers unknown format warning-520: EXE.pl = ../OSaft/Ciphers.pm # Ciphers.pm: programming error warning-521: TEST.args = TODO: no key found for '$txt' warning-521: EXE.pl = ../OSaft/Ciphers.pm #warning-522: TEST.args = TODO: testing duplicate openssl key, difficult ... #warning-522: EXE.pl = ../OSaft/Ciphers.pm ALL.test.warnings = $(shell awk -F: '/^warning-...:/{arr[$$1]=1}$(_EXE.print_arr_END.awk)' $(_MYSELF.warnings)) _TEST.template = t/test.warning.Makefile-template $(_TEST.template): $(SRC.pl) $(SRC.pm) @echo "# generated template targets to test **WARNING messages" > $@ @echo "# targets use the message text as TODO (see Makefile)" >> $@ @echo "# Note: texts for TEST.args should not contain ; | ()" >> $@ @echo "# Note: not all texts my be useful, it's a template!" >> $@ @echo "" >> $@ @$(EXE.extract_warn) $^ \ | $(EXE.gen_warning-target) \ | sort >> $@ @-ls -l $@ warnings.gen.template: $(ALL.Makefiles) $(_TEST.template) # TODO: Makefile dependency does not work, probably need to use $(MAKE) # TODO: hence missing from ALL.test.warnings .PHONY: warnings.gen.template warnings-info: $(SRC.pl) $(SRC.pm) @echo "# generated by $(_MYSELF.warnings) $@" @echo "# list of used error and warning numbers and messages" @$(EXE.extract_warn) $^ | sort warnings-show.todo: grep "^warning..*TODO" $(_MYSELF.warnings) ALL.test.warnings += warnings-info warnings-show.difficult warnings-show.todo warnings-show.difficult: grep "^warning..*difficult" $(_MYSELF.warnings) warnings-%.log: $(TEST.logdir) @$(TRACE.target) @$(TRACE.target.log) @$(eval _NEW.log := $(TEST.logdir)/$@-$(TEST.today)) @$(MAKE) $(MFLAGS) -s warnings-$* > $@ 2>&1 @expr "$(MAKEFLAGS)" : n >/dev/null \ && echo "$(MAKE) $(MFLAGS) -s warnings-$* > $@ 2>&1" \ || $(MAKE) $(MFLAGS) -s warnings-$* > $@ 2>&1 @-diff $(TEST.logdir)/$@ $@ 2>/dev/null \ && rm $@ \ || mv $@ $(_NEW.log) @-test -f $(TEST.logdir)/$@ || mv $(_NEW.log) $(TEST.logdir)/$@ @-ls -l $(TEST.logdir)/warnings-$(*)* # TODO: target should be unified with testcmd-%.log # $(_TEST.warning.log) calls "make -s" to avoid printing of executed commands _TEST.warning.log = $(TEST.logdir)/test.warnings.log-$(TEST.today) $(_TEST.warning.log): @echo "# $(_MYSELF.warnings) 1.73: $(MAKE) test.warnings.log" > $@ @$(MAKE) -s $(ALL.test.warnings) >> $@ 2>&1 # Target should create a new logfile, then compare it with the current one. If # diff returns nothing, delete newly created logfile, otherwise rename newly # created file to name which contains the current date. # Finally, If current logfile is/was missing, use newly created one: # "test ... || mv ..." . This ensures that the file exists afterwards. # FIXME: following target should have $(SRC.pl) as dependency, but that fails # if make is called from within ./t/ directory; hence no dependency test.warnings.log: $(_TEST.warning.log) @$(TRACE.target) @$(TRACE.target.log) @diff $(TEST.logdir)/$@ $(_TEST.warning.log) \ && rm $(_TEST.warning.log) \ || mv $(_TEST.warning.log) $(TEST.logdir)/$@ @-test -f $(TEST.logdir)/$@ || mv $(_TEST.warning.log) $(TEST.logdir)/$@ @ls -l $(TEST.logdir)/$@* # TODO: same target as test.cgi.log ALL.test.warnings.log = test.warnings.log test.warnings: $(ALL.test.warnings)