| 1 |
# |
| 2 |
# Makefile to run all tests for Vim |
| 3 |
# |
| 4 |
|
| 5 |
# The old-style tests (*.in) use shared filenames in the working directory |
| 6 |
# (test.ok, test.out, X*, viminfo), so running them in parallel causes races |
| 7 |
# and spurious failures. |
| 8 |
.NOTPARALLEL: |
| 9 |
|
| 10 |
# Use console or GUI. |
| 11 |
VIMPROG = ../vim |
| 12 |
XXDPROG = ../xxd/xxd |
| 13 |
# VIMPROG = ../gvim |
| 14 |
|
| 15 |
SCRIPTSOURCE = ../../runtime |
| 16 |
|
| 17 |
# Comment out this line to see the verbose output of tests. |
| 18 |
# |
| 19 |
# Catches SwapExists to avoid hanging at the ATTENTION prompt. |
| 20 |
REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' | LC_ALL=C awk '/Executing Test_/{match($$0, "([0-9][0-9]:[0-9][0-9] *)?Executing Test_[^\\)]*\\)"); print substr($$0, RSTART, RLENGTH) "\r"; fflush()}' |
| 21 |
|
| 22 |
# Uncomment this line to use valgrind for memory leaks and extra warnings. |
| 23 |
# The output goes into a file "valgrind.testN" |
| 24 |
# Vim should be compiled with EXITFREE to avoid false warnings. |
| 25 |
# This will make testing about 10 times as slow. |
| 26 |
# VALGRIND = valgrind --tool=memcheck --leak-check=yes --num-callers=45 --log-file=valgrind.$* |
| 27 |
|
| 28 |
# To execute one test, e.g. in gdb, use something like this: |
| 29 |
# run -f -u util/unix.vim --gui-dialog-file guidialog -U NONE --noplugin --not-a-term -S runtest.vim test_name.vim |
| 30 |
|
| 31 |
default: nongui |
| 32 |
|
| 33 |
# The list of tests is common to all systems. |
| 34 |
# This defines SCRIPTS_TINY_OUT, NEW_TESTS and NEW_TESTS_RES. |
| 35 |
include Make_all.mak |
| 36 |
|
| 37 |
# Explicit dependencies. |
| 38 |
test_options_all.res: opt_test.vim |
| 39 |
|
| 40 |
.SUFFIXES: .in .out .res .vim |
| 41 |
|
| 42 |
nongui: nolog tinytests newtests report |
| 43 |
|
| 44 |
gui: nolog tinytests newtests report |
| 45 |
|
| 46 |
tiny: nolog tinytests report |
| 47 |
|
| 48 |
benchmark: $(SCRIPTS_BENCH) |
| 49 |
|
| 50 |
report: |
| 51 |
@# without the +eval feature test_result.log is a copy of test.log |
| 52 |
@/bin/sh -c "if test -f test.log; \ |
| 53 |
then cp test.log test_result.log; \ |
| 54 |
else echo No failures reported > test_result.log; \ |
| 55 |
fi" |
| 56 |
$(VIMPROG) -u NONE $(NO_INITS) -S util/summarize.vim messages $(REDIR_TEST_TO_NULL) |
| 57 |
@rm -f starttime |
| 58 |
@echo |
| 59 |
@echo 'Test results:' |
| 60 |
@cat test_result.log |
| 61 |
@/bin/sh -c "if test -f test.log; \ |
| 62 |
then echo TEST FAILURE; exit 1; \ |
| 63 |
else echo ALL DONE; \ |
| 64 |
fi" |
| 65 |
|
| 66 |
$(SCRIPTS_TINY_OUT) $(NEW_TESTS_RES): $(VIMPROG) |
| 67 |
|
| 68 |
# For $ASAN_OPTIONS and $UBSAN_OPTIONS append the testname to it. |
| 69 |
# This assumes $ASAN_OPTIONS contains log_path as last part of the environment variable |
| 70 |
# For Github CI, those variables are set in .github/workflows/ci.yml |
| 71 |
|
| 72 |
# Execute an individual new style test, e.g.: |
| 73 |
# make test_largefile |
| 74 |
$(NEW_TESTS): |
| 75 |
rm -f [email protected] $(CLEANUP_FILES) |
| 76 |
@MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile [email protected] VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE) |
| 77 |
@cat messages |
| 78 |
@if test -f test.log; then \ |
| 79 |
exit 1; \ |
| 80 |
fi |
| 81 |
|
| 82 |
codestyle: |
| 83 |
rm -f test_codestyle.res $(CLEANUP_FILES) |
| 84 |
@MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile test_codestyle.res VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE) |
| 85 |
@cat messages |
| 86 |
@if test -f test.log; then \ |
| 87 |
exit 1; \ |
| 88 |
fi |
| 89 |
|
| 90 |
# Run only tests specific for Vim9 script |
| 91 |
test_vim9: |
| 92 |
rm -f test_vim9_*.res $(CLEANUP_FILES) |
| 93 |
@MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile $(TEST_VIM9_RES) VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE) |
| 94 |
@cat messages |
| 95 |
@rm -f starttime |
| 96 |
@MAKEFLAGS=--no-print-directory $(MAKE) -f Makefile report VIMPROG=$(VIMPROG) XXDPROG=$(XXDPROG) SCRIPTSOURCE=$(SCRIPTSOURCE) |
| 97 |
@if test -f test.log; then \ |
| 98 |
exit 1; \ |
| 99 |
fi |
| 100 |
|
| 101 |
RM_ON_RUN = test.out X* viminfo |
| 102 |
RM_ON_START = test.ok benchmark.out |
| 103 |
RUN_VIMPROG = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u util/unix.vim $(NO_INITS) -s dotest.in |
| 104 |
|
| 105 |
# Delete files that may interfere with running tests. This includes some files |
| 106 |
# that may result from working on the tests, not only from running them. |
| 107 |
clean: |
| 108 |
-rm -rf *.out *.failed *.res *.rej *.orig XfakeHOME Xdir1 Xfind failed |
| 109 |
-rm -f opt_test.vim test_result.log $(CLEANUP_FILES) |
| 110 |
-rm -f gen_opt_test.log |
| 111 |
-rm -rf $(RM_ON_RUN) $(RM_ON_START) |
| 112 |
-rm -f valgrind.* |
| 113 |
-rm -f asan.* asan_test_* |
| 114 |
-rm -f guidialog guidialogfile |
| 115 |
|
| 116 |
# Delete the files produced by benchmarking, so they can run again. |
| 117 |
benchmarkclean: |
| 118 |
rm -f $(SCRIPTS_BENCH) |
| 119 |
|
| 120 |
nolog: |
| 121 |
-rm -f test_result.log $(CLEANUP_FILES) |
| 122 |
|
| 123 |
|
| 124 |
# Tiny tests. Works even without the +eval feature. |
| 125 |
tinytests: $(SCRIPTS_TINY_OUT) |
| 126 |
|
| 127 |
.in.out: |
| 128 |
-rm -rf $*.failed test.ok $(RM_ON_RUN) |
| 129 |
cp $*.ok test.ok |
| 130 |
@# Sleep a moment to avoid that the xterm title is messed up. |
| 131 |
@# 200 msec is sufficient, but only modern sleep supports a fraction of |
| 132 |
@# a second, fall back to a second if it fails. |
| 133 |
@-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1" |
| 134 |
if test -n "$${ASAN_OPTIONS}"; then \ |
| 135 |
ASAN_OPTIONS="$${ASAN_OPTIONS}_$*" UBSAN_OPTIONS="$${UBSAN_OPTIONS}_$*" $(RUN_VIMPROG) $*.in $(REDIR_TEST_TO_NULL) ; \ |
| 136 |
else \ |
| 137 |
$(RUN_VIMPROG) $*.in $(REDIR_TEST_TO_NULL) ; \ |
| 138 |
fi |
| 139 |
|
| 140 |
@# Check if the test.out file matches test.ok. |
| 141 |
@/bin/sh -c "if test -f test.out; then \ |
| 142 |
if diff test.out $*.ok; \ |
| 143 |
then mv -f test.out $*.out; \ |
| 144 |
else echo $* FAILED >>test.log; mv -f test.out $*.failed; \ |
| 145 |
fi \ |
| 146 |
else echo $* NO OUTPUT >>test.log; \ |
| 147 |
fi" |
| 148 |
@/bin/sh -c "if test -f valgrind; then\ |
| 149 |
mv -f valgrind valgrind.$*; \ |
| 150 |
fi" |
| 151 |
-rm -rf X* test.ok viminfo |
| 152 |
|
| 153 |
|
| 154 |
# New style of tests uses Vim script with assert calls. These are easier |
| 155 |
# to write and a lot easier to read and debug. |
| 156 |
# Limitation: Only works with the +eval feature. |
| 157 |
# Add --gui-dialog-file to avoid getting stuck in a dialog. |
| 158 |
RUN_VIMTEST = VIMRUNTIME=$(SCRIPTSOURCE) $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u util/unix.vim --gui-dialog-file guidialog |
| 159 |
|
| 160 |
newtests: newtestssilent |
| 161 |
@/bin/sh -c "if test -f messages; then cat messages; fi" |
| 162 |
|
| 163 |
newtestssilent: $(NEW_TESTS_RES) |
| 164 |
|
| 165 |
|
| 166 |
.vim.res: |
| 167 |
@echo "$(VIMPROG)" > vimcmd |
| 168 |
@echo "$(RUN_VIMTEST)" >> vimcmd |
| 169 |
if test -n "$${ASAN_OPTIONS}"; then \ |
| 170 |
ASAN_OPTIONS="$${ASAN_OPTIONS}_$*" UBSAN_OPTIONS="$${UBSAN_OPTIONS}_$*" $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL) ; \ |
| 171 |
else \ |
| 172 |
$(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL) ; \ |
| 173 |
fi |
| 174 |
@rm vimcmd |
| 175 |
|
| 176 |
test_gui.res: test_gui.vim |
| 177 |
@echo "$(VIMPROG)" > vimcmd |
| 178 |
@echo "$(RUN_GVIMTEST)" >> vimcmd |
| 179 |
if test -n "$${ASAN_OPTIONS}"; then \ |
| 180 |
ASAN_OPTIONS="$${ASAN_OPTIONS}_$*" UBSAN_OPTIONS="$${UBSAN_OPTIONS}_$*" $(RUN_VIMTEST) -u NONE $(NO_INITS) -S runtest.vim $< ; \ |
| 181 |
else \ |
| 182 |
$(RUN_VIMTEST) -u NONE $(NO_INITS) -S runtest.vim $< ; \ |
| 183 |
fi |
| 184 |
|
| 185 |
@rm vimcmd |
| 186 |
|
| 187 |
test_gui_init.res: test_gui_init.vim |
| 188 |
@echo "$(VIMPROG)" > vimcmd |
| 189 |
@echo "$(RUN_GVIMTEST_WITH_GVIMRC)" >> vimcmd |
| 190 |
if test -n "$${ASAN_OPTIONS}"; then \ |
| 191 |
ASAN_OPTIONS="$${ASAN_OPTIONS}_$*" UBSAN_OPTIONS="$${UBSAN_OPTIONS}_$*" $(RUN_VIMTEST) -u util/gui_preinit.vim -U util/gui_init.vim $(NO_PLUGINS) -S runtest.vim $< ; \ |
| 192 |
else \ |
| 193 |
$(RUN_VIMTEST) -u util/gui_preinit.vim -U util/gui_init.vim $(NO_PLUGINS) -S runtest.vim $< ; \ |
| 194 |
fi |
| 195 |
@rm vimcmd |
| 196 |
|
| 197 |
GEN_OPT_DEPS = util/gen_opt_test.vim ../optiondefs.h ../../runtime/doc/options.txt |
| 198 |
|
| 199 |
opt_test.vim: $(GEN_OPT_DEPS) |
| 200 |
if test -n "$${ASAN_OPTIONS}"; then \ |
| 201 |
ASAN_OPTIONS="$${ASAN_OPTIONS}_$*" UBSAN_OPTIONS="$${UBSAN_OPTIONS}_$*" $(VIMPROG) -e -s -u NONE $(NO_INITS) --nofork --gui-dialog-file guidialog -S $(GEN_OPT_DEPS) ; \ |
| 202 |
else \ |
| 203 |
$(VIMPROG) -e -s -u NONE $(NO_INITS) --nofork --gui-dialog-file guidialog -S $(GEN_OPT_DEPS) ; \ |
| 204 |
fi |
| 205 |
@if test -f gen_opt_test.log; then \ |
| 206 |
cat gen_opt_test.log; \ |
| 207 |
exit 1; \ |
| 208 |
fi |
| 209 |
|
| 210 |
test_xxd.res: |
| 211 |
if test -n "$${ASAN_OPTIONS}"; then \ |
| 212 |
XXD=$(XXDPROG); export XXD; ASAN_OPTIONS="$${ASAN_OPTIONS}_$*" UBSAN_OPTIONS="$${UBSAN_OPTIONS}_$*" $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim test_xxd.vim ; \ |
| 213 |
else \ |
| 214 |
XXD=$(XXDPROG); export XXD; $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim test_xxd.vim ; \ |
| 215 |
fi |
| 216 |
|
| 217 |
test_bench_regexp.res: test_bench_regexp.vim |
| 218 |
-rm -rf benchmark.out $(RM_ON_RUN) |
| 219 |
@# Sleep a moment to avoid that the xterm title is messed up. |
| 220 |
@# 200 msec is sufficient, but only modern sleep supports a fraction of |
| 221 |
@# a second, fall back to a second if it fails. |
| 222 |
@-/bin/sh -c "sleep .2 > /dev/null 2>&1 || sleep 1" |
| 223 |
if test -n "$${ASAN_OPTIONS}"; then \ |
| 224 |
ASAN_OPTIONS="$${ASAN_OPTIONS}_$*" UBSAN_OPTIONS="$${UBSAN_OPTIONS}_$*" $(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL) ; \ |
| 225 |
else \ |
| 226 |
$(RUN_VIMTEST) $(NO_INITS) -S runtest.vim $*.vim $(REDIR_TEST_TO_NULL) ; \ |
| 227 |
fi |
| 228 |
@/bin/sh -c "if test -f benchmark.out; then cat benchmark.out; fi" |