Debian Codegraph @

test/testrunner.tcl

raw
1 #!/bin/sh
2 # Script to runs tests for SQLite. Run with option "help" for more info. \
3 exec tclsh "$0" "$@"
4
5 set dir [pwd]
6 set testdir [file normalize [file dirname $argv0]]
7 set saved $argv
8 set argv [list]
9 source [file join $testdir testrunner_data.tcl]
10
11 # Estimated amount of work required by displaytype, relative to 'tcl'
12 #
13 set estwork(tcl) 1
14 set estwork(fuzz) 22
15 set estwork(bld) 66
16 set estwork(make) 102
17
18 set estworkfile [file join $testdir testrunner_estwork.tcl]
19 if {[file readable $estworkfile]} {
20 source $estworkfile
21 }
22 source [file join $testdir permutations.test]
23 set argv $saved
24 cd $dir
25
26 # This script requires an interpreter that supports [package require sqlite3]
27 # to run. If this is not such an intepreter, see if there is a [testfixture]
28 # in the current directory. If so, run the command using it. If not,
29 # recommend that the user build one.
30 #
31 proc find_interpreter {} {
32 global dir
33 set interpreter [file tail [info nameofexec]]
34 set rc [catch { package require sqlite3 }]
35 if {$rc} {
36 if {[file readable pkgIndex.tcl] && [catch {source pkgIndex.tcl}]==0} {
37 set rc [catch { package require sqlite3 }]
38 }
39 }
40 if {$rc} {
41 if { [string match -nocase testfixture* $interpreter]==0
42 && [file executable ./testfixture]
43 } {
44 puts "Failed to find tcl package sqlite3. Restarting with ./testfixture.."
45 set status [catch {
46 exec [trd_get_bin_name testfixture] [info script] {*}$::argv >@ stdout
47 } msg]
48 exit $status
49 }
50 }
51 if {$rc} {
52 puts "Cannot find tcl package sqlite3: Trying to build it now..."
53 if {$::tcl_platform(platform) eq "windows"} {
54 set bat [open make-tcl-extension.bat w]
55 puts $bat "nmake /f Makefile.msc tclextension"
56 close $bat
57 catch {exec -ignorestderr -- make-tcl-extension.bat}
58 } else {
59 catch {exec make tclextension}
60 }
61 if {[file readable pkgIndex.tcl] && [catch {source pkgIndex.tcl}]==0} {
62 set rc [catch { package require sqlite3 }]
63 }
64 if {$rc==0} {
65 puts "The SQLite tcl extension was successfully built and loaded."
66 puts "Run \"make tclextension-install\" to avoid having to rebuild\
67 it in the future."
68 } else {
69 puts "Unable to build the SQLite tcl extension"
70 }
71 }
72 if {$rc} {
73 puts stderr "Cannot find a working instance of the SQLite tcl extension."
74 puts stderr "Run \"make tclextension\" or \"make testfixture\" and\
75 try again..."
76 exit 1
77 }
78 }
79 find_interpreter
80
81 # Usually this script is run by [testfixture]. But it can also be run
82 # by a regular [tclsh]. For these cases, emulate the [clock_milliseconds]
83 # command.
84 if {[info commands clock_milliseconds]==""} {
85 proc clock_milliseconds {} {
86 clock milliseconds
87 }
88 }
89
90 #-------------------------------------------------------------------------
91 # Usage:
92 #
93 proc usage {} {
94 set a0 [file tail $::argv0]
95
96 puts [string trim [subst -nocommands {
97 Usage:
98 $a0 ?SWITCHES? ?PERMUTATION? ?PATTERNS?
99 $a0 PERMUTATION FILE
100 $a0 errors ?-v|--verbose? ?-s|--summary? ?PATTERN?
101 $a0 help
102 $a0 joblist ?PATTERN?
103 $a0 njob ?NJOB?
104 $a0 retest
105 $a0 script ?-msvc? CONFIG
106 $a0 status ?-d SECS? ?--cls?
107 $a0 halt
108 $a0 estwork
109
110 where SWITCHES are:
111 --buildonly Build test exes but do not run tests
112 --cases DISPLAYNAME Only run test that match DISPLAYNAME
113 --config CONFIGS Only use configs on comma-separate list CONFIGS
114 --dryrun Write what would have happened to testrunner.log
115 --explain Write summary to stdout
116 --fuzzdb FILENAME Additional external fuzzcheck database
117 --jobs NUM Run tests using NUM separate processes
118 --omit CONFIGS Omit configs on comma-separated list CONFIGS
119 --status Show the full "status" report while running
120 --stop-on-coredump Stop running if any test segfaults
121 --stop-on-error Stop running after any reported error
122 --zipvfs ZIPVFSDIR ZIPVFS source directory
123
124 Special values for PERMUTATION include:
125
126 list - show allowed PERMUTATION arguments.
127 mdevtest - tests recommended prior to normal development check-ins.
128 devtest - alias for "mdevtest"
129 release - full release test with various builds.
130 c - tests in test/c directory only.
131 sdevtest - like mdevtest but using ASAN and UBSAN.
132 all - all tcl test scripts, plus a subset of test scripts rerun
133 with various permutations.
134 full - all tcl test scripts.
135 veryquick - a fast subset of the tcl test scripts. This is the default.
136
137 The interpreter that runs this script can be an ordinary "tclsh" as long
138 as "package require sqlite3" works, or it can be "testfixture".
139
140 If no PATTERN arguments are present, all tests specified by the PERMUTATION
141 are run. Otherwise, each pattern is interpreted as a glob pattern. Only
142 those tcl tests for which the final component of the filename matches at
143 least one specified pattern are run. The glob wildcard '*' is prepended
144 to the pattern if it does not start with '^' and appended to every
145 pattern that does not end with '$'. If PATTERN begins with "~", then it
146 is an anti-pattern that only matches tests that do not match PATTERN.
147 Tests or only run if they match one or more patterns and match no
148 anti-patterns.
149
150 If no PATTERN arguments are present, then various fuzztest, threadtest
151 and other tests are run as part of the "release" permutation. These are
152 omitted if any PATTERN arguments are specified on the command line.
153
154 If a PERMUTATION is specified and is followed by the path to a Tcl script
155 instead of a list of patterns, then that single Tcl test script is run
156 with the specified permutation.
157
158 The "status" and "njob" commands are designed to be run from the same
159 directory as a running testrunner.tcl script that is running tests. The
160 "status" command prints a report describing the current state and progress
161 of the tests. Use the "-d N" option to have the status display clear the
162 screen and repeat every N seconds. The "njob" command may be used to query
163 or modify the number of sub-processes the test script uses to run tests.
164
165 The "script" command outputs the script used to build a configuration.
166 Add the "-msvc" option for a Windows-compatible script. For a list of
167 available configurations enter "$a0 script help".
168
169 The "errors" commands shows the output of tests that failed in the
170 most recent run. Complete output is shown if the -v or --verbose options
171 are used. Otherwise, an attempt is made to minimize the output to show
172 only the parts that contain the error messages. The --summary option just
173 shows the jobs that failed. If PATTERN are provided, the error information
174 is only provided for jobs that match PATTERN.
175
176 The "retest" command reruns tests that failed or were never completed
177 by a prior invocation of testrunner.tcl.
178
179 Full documentation here: https://sqlite.org/src/doc/trunk/doc/testrunner.md
180 }]]
181
182 exit 1
183 }
184 #-------------------------------------------------------------------------
185
186 #-------------------------------------------------------------------------
187 # Try to estimate a the number of processes to use.
188 #
189 # Command [guess_number_of_cores] attempts to glean the number of logical
190 # cores. Command [default_njob] returns the default value for the --jobs
191 # switch.
192 #
193 proc guess_number_of_cores {} {
194 if {[catch {number_of_cores} ret]} {
195 set ret 4
196 if {$::tcl_platform(platform) eq "windows"} {
197 catch { set ret $::env(NUMBER_OF_PROCESSORS) }
198 } else {
199 if {$::tcl_platform(os)=="Darwin"} {
200 set cmd "sysctl -n hw.logicalcpu"
201 } else {
202 set cmd "nproc"
203 }
204 catch {
205 set fd [open "|$cmd" r]
206 set ret [gets $fd]
207 close $fd
208 set ret [expr $ret]
209 }
210 }
211 }
212 return $ret
213 }
214
215 proc default_njob {} {
216 global env
217 if {[info exists env(NJOB)] && $env(NJOB)>=1} {
218 return $env(NJOB)
219 }
220 set nCore [guess_number_of_cores]
221 if {$nCore<=2} {
222 set nHelper 1
223 } else {
224 set nHelper [expr int($nCore*0.8)]
225 if {$nHelper>20} {set nHelper 20}
226 }
227 return $nHelper
228 }
229 #-------------------------------------------------------------------------
230
231 #-------------------------------------------------------------------------
232 # Setup various default values in the global TRG() array.
233 #
234 set TRG(dbname) [file normalize testrunner.db]
235 set TRG(logname) [file normalize testrunner.log]
236 set TRG(build.logname) [file normalize testrunner_build.log]
237 set TRG(info_script) [file normalize [info script]]
238 set TRG(timeout) 10000 ;# Default busy-timeout for testrunner.db
239 set TRG(nJob) [default_njob] ;# Default number of helper processes
240 set TRG(patternlist) [list]
241 set TRG(cmdline) $argv
242 set TRG(reporttime) 2000
243 set TRG(fuzztest) 0 ;# is the fuzztest option present.
244 set TRG(zipvfs) "" ;# -zipvfs option, if any
245 set TRG(buildonly) 0 ;# True if --buildonly option
246 set TRG(config) {} ;# Only build the named configurations
247 set TRG(omitconfig) {} ;# Do not build these configurations
248 set TRG(dryrun) 0 ;# True if --dryrun option
249 set TRG(explain) 0 ;# True for the --explain option
250 set TRG(stopOnError) 0 ;# Stop running at first failure
251 set TRG(stopOnCore) 0 ;# Stop on a core-dump
252 set TRG(fullstatus) 0 ;# Full "status" report while running
253 set TRG(case) {} ;# Only run cases matching this GLOB pattern
254
255 switch -nocase -glob -- $tcl_platform(os) {
256 *darwin* {
257 set TRG(platform) osx
258 set TRG(make) make.sh
259 set TRG(makecmd) "bash make.sh"
260 set TRG(testfixture) testfixture
261 set TRG(shell) sqlite3
262 set TRG(run) run.sh
263 set TRG(runcmd) "bash run.sh"
264 }
265 *linux* - MSYS_NT* - MINGW64_NT* - MINGW32_NT* {
266 set TRG(platform) linux
267 set TRG(make) make.sh
268 set TRG(makecmd) "bash make.sh"
269 set TRG(testfixture) testfixture
270 set TRG(shell) sqlite3
271 set TRG(run) run.sh
272 set TRG(runcmd) "bash run.sh"
273 }
274 *openbsd* {
275 set TRG(platform) linux
276 set TRG(make) make.sh
277 set TRG(makecmd) "sh make.sh"
278 set TRG(testfixture) testfixture
279 set TRG(shell) sqlite3
280 set TRG(run) run.sh
281 set TRG(runcmd) "sh run.sh"
282 }
283 *win* {
284 set TRG(platform) win
285 set TRG(make) make.bat
286 set TRG(makecmd) "call make.bat"
287 set TRG(testfixture) testfixture.exe
288 set TRG(shell) sqlite3.exe
289 set TRG(run) run.bat
290 set TRG(runcmd) "run.bat"
291 if {"unix" eq $tcl_platform(platform)} {
292 # Presumably cygwin. This block gets testrunner.tcl started on
293 # Cygwin but then downstream tests all fail, at least in part
294 # because of the discrepancies in build target names which need
295 # .exe on cygwin but not on other Unix-like platforms.
296 set TRG(platform) cygwin
297 set TRG(make) make.sh
298 set TRG(makecmd) "bash make.sh"
299 set TRG(testfixture) testfixture
300 set TRG(shell) sqlite3
301 set TRG(run) run.sh
302 set TRG(runcmd) "bash run.sh"
303 }
304 }
305 default {
306 puts "tcl_platform(os)=$::tcl_platform(os)"
307 error "cannot determine platform!"
308 }
309 }
310 set TRG(testfixture-fullpath) [file join $dir $TRG(testfixture)]
311 set TRG(interp) [info nameofexec]
312 #-------------------------------------------------------------------------
313
314 #-------------------------------------------------------------------------
315 # The database schema used by the testrunner.db database.
316 #
317 set TRG(schema) {
318 DROP TABLE IF EXISTS jobs;
319 DROP TABLE IF EXISTS config;
320
321 /*
322 ** This table contains one row for each job that testrunner.tcl must run
323 ** before the entire test run is finished.
324 **
325 ** jobid:
326 ** Unique identifier for each job. Must be a +ve non-zero number.
327 **
328 ** displaytype:
329 ** 3 or 4 letter mnemonic for the class of tests this belongs to e.g.
330 ** "fuzz", "tcl", "make" etc.
331 **
332 ** displayname:
333 ** Name/description of job. For display purposes.
334 **
335 ** build:
336 ** If the job requires a make.bat/make.sh make wrapper (i.e. to build
337 ** something), the name of the build configuration it uses. See
338 ** testrunner_data.tcl for a list of build configs. e.g. "Win32-MemDebug".
339 **
340 ** dirname:
341 ** If the job should use a well-known directory name for its
342 ** sub-directory instead of an anonymous "testdir[1234...]" sub-dir
343 ** that is deleted after the job is finished.
344 **
345 ** cmd:
346 ** Bash or batch script to run the job.
347 **
348 ** depid:
349 ** The jobid value of a job that this job depends on. This job may not
350 ** be run before its depid job has finished successfully.
351 **
352 ** priority:
353 ** Higher values run first. Sometimes.
354 */
355 CREATE TABLE jobs(
356 /* Fields populated when db is initialized */
357 jobid INTEGER PRIMARY KEY, -- id to identify job
358 displaytype TEXT NOT NULL, -- Type of test (for one line report)
359 displayname TEXT NOT NULL, -- Human readable job name
360 build TEXT NOT NULL DEFAULT '', -- make.sh/make.bat file request, if any
361 dirname TEXT NOT NULL DEFAULT '', -- directory name, if required
362 cmd TEXT NOT NULL, -- shell command to run
363 depid INTEGER, -- identifier of dependency (or '')
364 priority INTEGER NOT NULL, -- higher priority jobs may run earlier
365
366 /* Fields updated as jobs run */
367 starttime INTEGER, -- Start time (milliseconds since 1970)
368 endtime INTEGER, -- End time
369 span INTEGER, -- Total run-time in milliseconds
370 estwork INTEGER, -- Estimated amount of work
371 estkey TEXT, -- Key used to compute estwork
372 state TEXT CHECK( state IN ('','ready','running','done','failed','omit','halt') ),
373 ntest INT, -- Number of test cases run
374 nerr INT, -- Number of errors reported
375 svers TEXT, -- Reported SQLite version
376 pltfm TEXT, -- Host platform reported
377 output TEXT, -- test output
378 cwd TEXT -- working directory for test
379 );
380
381 CREATE TABLE config(
382 name TEXT COLLATE nocase PRIMARY KEY,
383 value
384 ) WITHOUT ROWID;
385
386 CREATE INDEX i1 ON jobs(state, priority);
387 CREATE INDEX i2 ON jobs(depid);
388 }
389 #-------------------------------------------------------------------------
390
391 #--------------------------------------------------------------------------
392 # Check if this script is being invoked to run a single file. If so,
393 # run it.
394 #
395 if {[llength $argv]==2
396 && ([lindex $argv 0]=="" || [info exists ::testspec([lindex $argv 0])])
397 && [file exists [lindex $argv 1]]
398 } {
399 set permutation [lindex $argv 0]
400 set script [file normalize [lindex $argv 1]]
401 set ::argv [list]
402
403 set testdir [file dirname $argv0]
404 source $::testdir/tester.tcl
405
406 if {$permutation=="full"} {
407
408 unset -nocomplain ::G(isquick)
409 reset_db
410
411 } elseif {$permutation!="default" && $permutation!=""} {
412
413 if {[info exists ::testspec($permutation)]==0} {
414 error "no such permutation: $permutation"
415 }
416
417 array set O $::testspec($permutation)
418 set ::G(perm:name) $permutation
419 set ::G(perm:prefix) $O(-prefix)
420 set ::G(isquick) 1
421 set ::G(perm:dbconfig) $O(-dbconfig)
422 set ::G(perm:presql) $O(-presql)
423
424 rename finish_test helper_finish_test
425 proc finish_test {} "
426 uplevel {
427 $O(-shutdown)
428 }
429 helper_finish_test
430 "
431
432 eval $O(-initialize)
433 }
434
435 reset_db
436 source $script
437 exit
438 }
439 #--------------------------------------------------------------------------
440
441 #--------------------------------------------------------------------------
442 # Check if this is the "njob" command:
443 #
444 if {([llength $argv]==2 || [llength $argv]==1)
445 && [string compare -nocase njob [lindex $argv 0]]==0
446 } {
447 sqlite3 mydb $TRG(dbname)
448 if {[llength $argv]==2} {
449 set param [lindex $argv 1]
450 if {[string is integer $param]==0 || $param<0 || $param>128} {
451 puts stderr "parameter must be an integer between 0 and 128"
452 exit 1
453 }
454
455 mydb eval { REPLACE INTO config VALUES('njob', $param); }
456 }
457 set res [mydb one { SELECT value FROM config WHERE name='njob' }]
458 mydb close
459 puts "$res"
460 exit
461 }
462 #--------------------------------------------------------------------------
463
464 #--------------------------------------------------------------------------
465 # Check if this is the "halt" command:
466 #
467 if {[llength $argv]==1
468 && [string compare -nocase halt [lindex $argv 0]]==0
469 } {
470 sqlite3 mydb $TRG(dbname)
471 mydb eval {UPDATE jobs SET state='halt' WHERE state IN ('ready','')}
472 mydb close
473 exit
474 }
475 #--------------------------------------------------------------------------
476
477 #--------------------------------------------------------------------------
478 # Check if this is the "estwork" command:
479 #
480 # Generate (on standard output) a set of estwork() values based on the lastest
481 # test case, that can be used to replace the test/testrunner_estwork.tcl file.
482 #
483 if {[llength $argv]==1
484 && [string compare -nocase estwork [lindex $argv 0]]==0
485 } {
486 sqlite3 mydb $TRG(dbname)
487 set njob [mydb one {SELECT count(*) FROM jobs WHERE state='done'}]
488 if {$njob<1000} {
489 puts "Too few completed jobs to do a work estimate."
490 puts "Have $njob but not need at least 1000."
491 mydb close
492 exit 1
493 }
494 set badjobs [mydb one {SELECT count(*) FROM jobs WHERE state<>'done'}]
495 if {$badjobs} {
496 puts "Database contains $badjobs incomplete jobs."
497 mydb close
498 exit 1
499 }
500 set half [mydb one {SELECT count(*)/2 FROM jobs WHERE displaytype='tcl'}]
501 set scale [mydb one {SELECT span FROM jobs WHERE displaytype='tcl'
502 ORDER BY span LIMIT 1 OFFSET $half}]
503 mydb eval {
504 SELECT estkey, CAST(avg(span)/$scale AS INT) AS cost
505 FROM jobs
506 GROUP BY estkey
507 HAVING cost>=2
508 } {
509 set estwork($estkey) $cost
510 }
511 set avgtcl [mydb one {SELECT avg(span) FROM jobs WHERE displaytype='tcl'}]
512 set estwork(tcl) 1
513 foreach type {bld fuzz make} {
514 set avg [mydb one {SELECT avg(span) FROM jobs WHERE displaytype=$type}]
515 if {$avg!=""} {
516 set estwork($type) [expr {int($avg/$avgtcl)}]
517 }
518 }
519 mydb close
520 puts "# Estimated relative cost of various jobs, based on the \"estkey\" field."
521 puts "# Computed by the \"test/testrunner.tcl estwork\" command."
522 puts "#"
523 foreach key [lsort [array names estwork]] {
524 puts "set [list estwork($key)] $estwork($key)"
525 }
526 exit
527 }
528 #--------------------------------------------------------------------------
529
530 #--------------------------------------------------------------------------
531 # Check if this is the "help" command:
532 #
533 if {[string compare -nocase help [lindex $argv 0]]==0} {
534 usage
535 }
536 #--------------------------------------------------------------------------
537
538 #--------------------------------------------------------------------------
539 # Check if this is the "script" command:
540 #
541 if {[string compare -nocase script [lindex $argv 0]]==0} {
542 if {[llength $argv]!=2 && !([llength $argv]==3&&[lindex $argv 1]=="-msvc")} {
543 usage
544 }
545
546 set bMsvc [expr ([llength $argv]==3)]
547 set config [lindex $argv [expr [llength $argv]-1]]
548
549 puts [trd_buildscript $config [file dirname $testdir] $bMsvc]
550 exit
551 }
552
553 # Compute an elapse time string MM:SS or HH:MM:SS based on the
554 # number of milliseconds in the argument.
555 #
556 proc elapsetime {ms} {
557 if {$ms==""} {set ms 0}
558 set s [expr {int(($ms+500.0)*0.001)}]
559 set hr [expr {$s/3600}]
560 set mn [expr {($s/60)%60}]
561 set sc [expr {$s%60}]
562 if {$hr>0} {
563 return [format %02d:%02d:%02d $hr $mn $sc]
564 } else {
565 return [format %02d:%02d $mn $sc]
566 }
567 }
568
569 # Helper routine for show_status
570 #
571 proc display_job {jobdict {tm ""}} {
572 array set job $jobdict
573 if {[string length $job(displayname)]>65} {
574 set dfname [format %.65s... $job(displayname)]
575 } else {
576 set dfname [format %-68s $job(displayname)]
577 }
578 set dtm ""
579 if {$tm!=""} {
580 set dtm [expr {$tm-$job(starttime)}]
581 set dtm [format %8s [elapsetime $dtm]]
582 } else {
583 set dtm [format %8s ""]
584 }
585 puts " $dfname $dtm"
586 }
587
588 # This procedure shows the "status" page. It uses the database
589 # connect passed in as the "db" parameter. If the "cls" parameter
590 # is true, then VT100 escape codes are used to format the display.
591 #
592 proc show_status {db cls} {
593 global TRG
594 $db eval BEGIN
595 if {[catch {
596 set cmdline [$db one { SELECT value FROM config WHERE name='cmdline' }]
597 set nJob [$db one { SELECT value FROM config WHERE name='njob' }]
598 } msg]} {
599 if {$cls} {puts "\033\[H\033\[2J"}
600 puts "Cannot read database: $TRG(dbname)"
601 return
602 }
603 set now [clock_milliseconds]
604 set tm [$db one {
605 SELECT
606 COALESCE((SELECT value FROM config WHERE name='end'), $now) -
607 (SELECT value FROM config WHERE name='start')
608 }]
609
610 set totalw 0
611 foreach s {"" ready running done failed omit} { set S($s) 0; set W($s) 0; }
612 set workpending 0
613 $db eval {
614 SELECT state, count(*) AS cnt, sum(estwork) AS ew FROM jobs GROUP BY 1
615 } {
616 incr S($state) $cnt
617 incr W($state) $ew
618 incr totalw $ew
619 }
620 set nt 0
621 set ne 0
622 $db eval {
623 SELECT sum(ntest) AS nt, sum(nerr) AS ne FROM jobs HAVING nt>0
624 } break
625 set fin [expr $W(done)+$W(failed)+$W(omit)]
626 if {$cmdline!=""} {set cmdline " $cmdline"}
627
628 if {$cls} {
629 # Move the cursor to the top-left corner. Each iteration will simply
630 # overwrite.
631 puts -nonewline "\033\[H"
632 flush stdout
633 }
634 puts [format %-79.79s "Command: \[testrunner.tcl$cmdline\]"]
635 puts [format %-79.79s "Summary: [elapsetime $tm], $fin/$totalw jobs,\
636 $ne errors, $nt tests"]
637
638 set srcdir [file dirname [file dirname $TRG(info_script)]]
639 set line "Running: $S(running) (max: $nJob)"
640 if {$S(running)>0 && [set pct [expr {int(($fin*100.0)/$totalw)}]]>=4} {
641 set tmleft [expr {($tm/double($fin))*($totalw-$fin)}]
642 if {$tmleft<0.02*$tm} {
643 set tmleft [expr {$tm*0.02}]
644 }
645 set etc " ETC [elapsetime $tmleft]"
646 if {[string length $line]+[string length $etc]<80} {
647 append line $etc
648 }
649 # append line " $pct%"
650 }
651 puts [format %-79.79s $line]
652 if {$S(running)>0} {
653 $db eval {
654 SELECT * FROM jobs WHERE state='running' ORDER BY starttime
655 } job {
656 display_job [array get job] $now
657 }
658 }
659 if {$S(failed)>0} {
660 # $toshow is the number of failures to report. In $cls mode,
661 # status tries to limit the number of failure reported so that
662 # the status display does not overflow a 24-line terminal. It will
663 # always show at least the most recent 4 failures, even if an overflow
664 # is needed. No limit is imposed for a status within $cls.
665 #
666 if {$cls && $S(failed)>18-$S(running)} {
667 set toshow [expr {18-$S(running)}]
668 if {$toshow<4} {set toshow 4}
669 set shown " (must recent $toshow shown)"
670 } else {
671 set toshow $S(failed)
672 set shown ""
673 }
674 puts [format %-79s "Failed: $S(failed) $shown"]
675 $db eval {
676 SELECT * FROM jobs WHERE state='failed'
677 ORDER BY endtime DESC LIMIT $toshow
678 } job {
679 display_job [array get job]
680 }
681 set nOmit [$db one {SELECT count(*) FROM jobs WHERE state='omit'}]
682 if {$nOmit} {
683 puts [format %-79s " ... $nOmit jobs omitted due to failures"]
684 }
685 }
686 if {$cls} {
687 # Clear everything else to the bottom of the screen
688 puts -nonewline "\033\[0J"
689 flush stdout
690 }
691 $db eval COMMIT
692 }
693
694
695
696 #--------------------------------------------------------------------------
697 # Check if this is the "status" command:
698 #
699 if {[llength $argv]>=1
700 && [string compare -nocase status [lindex $argv 0]]==0
701 } {
702 set delay 0
703 set cls 0
704 for {set ii 1} {$ii<[llength $argv]} {incr ii} {
705 set a0 [lindex $argv $ii]
706 if {$a0=="-d" && $ii+1<[llength $argv]} {
707 incr ii
708 set delay [lindex $argv $ii]
709 if {![string is integer -strict $delay]} {
710 puts "Argument to -d should be an integer"
711 exit 1
712 }
713 } elseif {$a0=="-cls" || $a0=="--cls"} {
714 set cls 1
715 } else {
716 puts "unknown option: \"$a0\""
717 exit 1
718 }
719 }
720
721 set once 1
722 while {![file readable $TRG(dbname)]} {
723 if {$delay==0} {
724 puts "Database missing: $TRG(dbname)"
725 exit
726 }
727 if {$once} {
728 set once 0
729 puts "Waiting for testing to start...."
730 flush stdout
731 }
732 after [expr {$delay*1000}]
733 }
734 sqlite3 mydb $TRG(dbname)
735 mydb timeout 2000
736
737 # Clear the whole screen initially.
738 #
739 if {$delay>0 || $cls} {puts -nonewline "\033\[2J"}
740
741 while {1} {
742 show_status mydb [expr {$delay>0 || $cls}]
743 if {$delay<=0} break
744 after [expr {$delay*1000}]
745 }
746 mydb close
747 exit
748 }
749
750 #--------------------------------------------------------------------------
751 # Check if this is the "joblist" command:
752 #
753 if {[llength $argv]>=1
754 && [string compare -nocase "joblist" [lindex $argv 0]]==0
755 } {
756 set pattern {}
757 for {set ii 1} {$ii<[llength $argv]} {incr ii} {
758 set a0 [lindex $argv $ii]
759 if {$pattern==""} {
760 set pattern [string trim $a0 *]
761 } else {
762 puts "unknown option: \"$a0\""
763 exit 1
764 }
765 }
766 set SQL {SELECT displaytype, displayname, state FROM jobs}
767 if {$pattern!=""} {
768 regsub -all {[^a-zA-Z0-9*.-/]} $pattern ? pattern
769 set pattern [string tolower $pattern]
770 append SQL \
771 " WHERE lower(concat(state,' ',displaytype,' ',displayname)) GLOB '*$pattern*'"
772 }
773 append SQL " ORDER BY starttime"
774
775 if {![file readable $TRG(dbname)]} {
776 puts "Database missing: $TRG(dbname)"
777 exit
778 }
779 sqlite3 mydb $TRG(dbname)
780 mydb timeout 2000
781
782 mydb eval $SQL {
783 set label UNKNOWN
784 switch -- $state {
785 ready {set label READY}
786 done {set label DONE}
787 failed {set label FAILED}
788 omit {set label OMIT}
789 running {set label RUNNING}
790 }
791 puts [format {%-7s %-5s %s} $label $displaytype $displayname]
792 }
793 mydb close
794 exit
795 }
796
797 # Scan the output of all jobs looking for the summary lines that
798 # report the number of test cases and the number of errors.
799 # Aggregate these numbers and return them.
800 #
801 proc aggregate_test_counts {db} {
802 set ne 0
803 set nt 0
804 $db eval {SELECT sum(nerr) AS ne, sum(ntest) as nt FROM jobs} break
805 return [list $ne $nt]
806 }
807
808 #--------------------------------------------------------------------------
809 # Check if this is the "errors" command:
810 #
811 if {[llength $argv]>=1
812 && ([string compare -nocase errors [lindex $argv 0]]==0 ||
813 [string match err* [lindex $argv 0]]==1)
814 } {
815 set verbose 0
816 set pattern {}
817 set summary 0
818 for {set ii 1} {$ii<[llength $argv]} {incr ii} {
819 set a0 [lindex $argv $ii]
820 if {$a0=="-v" || $a0=="--verbose" || $a0=="-verbose"} {
821 set verbose 1
822 } elseif {$a0=="-s" || $a0=="--summary" || $a0=="-summary"} {
823 set summary 1
824 } elseif {$pattern==""} {
825 set pattern *[string trim $a0 *]*
826 } else {
827 puts "unknown option: \"$a0\"". Use --help for more info."
828 exit 1
829 }
830 }
831 set cnt 0
832 sqlite3 mydb $TRG(dbname)
833 mydb timeout 5000
834 if {$summary} {
835 set sql "SELECT displayname FROM jobs WHERE state='failed'"
836 } else {
837 set sql "SELECT displaytype, displayname, output FROM jobs \
838 WHERE state='failed'"
839 }
840 if {$pattern!=""} {
841 regsub -all {[^a-zA-Z0-9*/ ?]} $pattern . pattern
842 append sql " AND displayname GLOB '$pattern'"
843 }
844 mydb eval $sql {
845 if {$summary} {
846 puts "FAILED: $displayname"
847 continue
848 }
849 puts "**** $displayname ****"
850 if {$verbose || $displaytype!="tcl"} {
851 puts $output
852 } else {
853 foreach line [split $output \n] {
854 if {[string match {!*} $line] || [string match *failed* $line]} {
855 puts $line
856 }
857 }
858 }
859 incr cnt
860 }
861 if {$pattern==""} {
862 set summary [aggregate_test_counts mydb]
863 mydb close
864 puts "Total [lindex $summary 0] errors out of [lindex $summary 1] tests"
865 } else {
866 mydb close
867 }
868 exit
869 }
870
871 #-------------------------------------------------------------------------
872 # Parse the command line.
873 #
874 for {set ii 0} {$ii < [llength $argv]} {incr ii} {
875 set isLast [expr $ii==([llength $argv]-1)]
876 set a [lindex $argv $ii]
877 set n [string length $a]
878
879 if {[string range $a 0 0]=="-"} {
880 if {($n>2 && [string match "$a*" --jobs]) || $a=="-j"} {
881 incr ii
882 set TRG(nJob) [lindex $argv $ii]
883 if {$isLast} { usage }
884 } elseif {($n>2 && [string match "$a*" --zipvfs]) || $a=="-z"} {
885 incr ii
886 set TRG(zipvfs) [file normalize [lindex $argv $ii]]
887 if {$isLast} { usage }
888 } elseif {($n>2 && [string match "$a*" --buildonly]) || $a=="-b"} {
889 set TRG(buildonly) 1
890 } elseif {($n>2 && [string match "$a*" --config]) || $a=="-c"} {
891 incr ii
892 set TRG(config) [lindex $argv $ii]
893 } elseif {($n>2 && [string match "$a*" --dryrun]) || $a=="-d"} {
894 set TRG(dryrun) 1
895 } elseif {($n>2 && [string match "$a*" --explain]) || $a=="-e"} {
896 set TRG(explain) 1
897 } elseif {$n>2 && [string match "$a*" --omit]} {
898 incr ii
899 set TRG(omitconfig) [lindex $argv $ii]
900 } elseif {$n>2 && [string match "$a*" --cases]} {
901 incr ii
902 set TRG(case) [lindex $argv $ii]
903 } elseif {$n>2 && [string match "$a*" --fuzzdb]} {
904 incr ii
905 set env(FUZZDB) [lindex $argv $ii]
906 } elseif {[string match "$a*" --stop-on-error]} {
907 set TRG(stopOnError) 1
908 } elseif {[string match "$a*" --stop-on-coredump]} {
909 set TRG(stopOnCore) 1
910 } elseif {[string match "$a*" --status]} {
911 if {$tcl_platform(platform) eq "windows"} {
912 puts stdout \
913 "The --status option is not available on Windows. A suggested work-around"
914 puts stdout \
915 "is to run the following command in a separate window:\n"
916 puts stdout " [info nameofexe] $argv0 status -d 2\n"
917 } else {
918 set TRG(fullstatus) 1
919 }
920 } else {
921 usage
922 }
923 } else {
924 lappend TRG(patternlist) [string map {% *} $a]
925 }
926 }
927 set argv [list]
928
929 # This script runs individual tests - tcl scripts or [make xyz] commands -
930 # in directories named "testdir$N", where $N is an integer. This variable
931 # contains a list of integers indicating the directories in use.
932 #
933 # This variable is accessed only via the following commands:
934 #
935 # dirs_nHelper
936 # Return the number of entries currently in the list.
937 #
938 # dirs_freeDir IDIR
939 # Remove value IDIR from the list. It is an error if it is not present.
940 #
941 # dirs_allocDir
942 # Select a value that is not already in the list. Add it to the list
943 # and return it.
944 #
945 set TRG(dirs_in_use) [list]
946
947 proc dirs_nHelper {} {
948 global TRG
949 llength $TRG(dirs_in_use)
950 }
951 proc dirs_freeDir {iDir} {
952 global TRG
953 set out [list]
954 foreach d $TRG(dirs_in_use) {
955 if {$iDir!=$d} { lappend out $d }
956 }
957 if {[llength $out]!=[llength $TRG(dirs_in_use)]-1} {
958 error "dirs_freeDir could not find $iDir"
959 }
960 set TRG(dirs_in_use) $out
961 }
962 proc dirs_allocDir {} {
963 global TRG
964 array set inuse [list]
965 foreach d $TRG(dirs_in_use) {
966 set inuse($d) 1
967 }
968 for {set iRet 0} {[info exists inuse($iRet)]} {incr iRet} { }
969 lappend TRG(dirs_in_use) $iRet
970 return $iRet
971 }
972
973 # Check that directory $dir exists. If it does not, create it. If
974 # it does, delete its contents.
975 #
976 proc create_or_clear_dir {dir} {
977 set dir [file normalize $dir]
978 catch { file mkdir $dir }
979 foreach f [glob -nocomplain [file join $dir *]] {
980 catch { file delete -force $f }
981 }
982 }
983
984 proc build_to_dirname {bname} {
985 set fold [string tolower [string map {- _} $bname]]
986 return "testrunner_build_$fold"
987 }
988
989 #-------------------------------------------------------------------------
990
991 proc r_write_db {tcl} {
992 trdb eval { BEGIN EXCLUSIVE }
993 uplevel $tcl
994 trdb eval { COMMIT }
995 }
996
997 # Obtain a new job to be run by worker $iJob (an integer). A job is
998 # returned as a three element list:
999 #
1000 # {$build $config $file}
1001 #
1002 proc r_get_next_job {iJob} {
1003 global T
1004
1005 if {($iJob%2)} {
1006 set orderby "ORDER BY priority ASC"
1007 } else {
1008 set orderby "ORDER BY priority DESC"
1009 }
1010
1011 set ret [list]
1012
1013 r_write_db {
1014 set query "
1015 SELECT * FROM jobs AS j WHERE state='ready' $orderby LIMIT 1
1016 "
1017 trdb eval $query job {
1018 set tm [clock_milliseconds]
1019 set T($iJob) $tm
1020 set jobid $job(jobid)
1021
1022 set cwd $job(dirname)
1023 if {$cwd==""} {
1024 set cwd [dirname $iJob]
1025 }
1026
1027 trdb eval {
1028 UPDATE jobs
1029 SET starttime=$tm, state='running', cwd=$cwd
1030 WHERE jobid=$jobid
1031 }
1032
1033 set ret [array get job]
1034 }
1035 }
1036
1037 return $ret
1038 }
1039
1040 # Usage:
1041 #
1042 # add_job OPTION ARG OPTION ARG...
1043 #
1044 # where available OPTIONS are:
1045 #
1046 # -displaytype
1047 # -displayname
1048 # -build
1049 # -dirname
1050 # -cmd
1051 # -depid
1052 # -priority
1053 #
1054 # Returns the jobid value for the new job.
1055 #
1056 proc add_job {args} {
1057 global estwork
1058
1059 set options {
1060 -displaytype -displayname -build -dirname
1061 -cmd -depid -priority
1062 }
1063
1064 # Set default values of options.
1065 set A(-dirname) ""
1066 set A(-depid) ""
1067 set A(-priority) 0
1068 set A(-build) ""
1069
1070 array set A $args
1071
1072 # Check all required options are present. And that no extras are present.
1073 foreach o $options {
1074 if {[info exists A($o)]==0} { error "missing required option $o" }
1075 }
1076 foreach o [array names A] {
1077 if {[lsearch -exact $options $o]<0} { error "unrecognized option: $o" }
1078 }
1079
1080 set state ""
1081 if {$A(-depid)==""} { set state ready }
1082 set type $A(-displaytype)
1083 set displayname $A(-displayname)
1084 switch $type {
1085 tcl {
1086 set ek [file tail [lindex $displayname end]]
1087 }
1088 bld {
1089 set ek [lindex $displayname end]
1090 }
1091 fuzz {
1092 set ek [lrange $displayname 1 2]
1093 }
1094 make {
1095 set ek [lindex $displayname end]
1096 }
1097 }
1098 if {[info exists estwork($ek)]} {
1099 set ew $estwork($ek)
1100 } else {
1101 set ew $estwork($type)
1102 }
1103
1104 trdb eval {
1105 INSERT INTO jobs(
1106 displaytype, displayname, build, dirname, cmd, depid, priority,
1107 estwork, estkey, state
1108 ) VALUES (
1109 $type,
1110 $A(-displayname),
1111 $A(-build),
1112 $A(-dirname),
1113 $A(-cmd),
1114 $A(-depid),
1115 $A(-priority),
1116 $ew,
1117 $ek,
1118 $state
1119 )
1120 }
1121
1122 trdb last_insert_rowid
1123 }
1124
1125 # Look to see if $jobcmd matches any of the glob patterns given in
1126 # $patternlist. Return true if there is a match. Return false
1127 # if no match is seen.
1128 #
1129 # An empty patternlist matches everything
1130 #
1131 # Entries of patternlist that begin with "~" mean "match anything that
1132 # does not match the following pattern". For example, a patternlist
1133 # of {fuzzcheck ~san} will match "fuzzcheck" but not "fuzzcheck-asan".
1134 #
1135 proc job_matches_any_pattern {patternlist jobcmd} {
1136 set bMatch 0
1137 set bMiss 0
1138 if {[llength $patternlist]==0} {return 1}
1139 foreach p $patternlist {
1140 if {[string index $p 0] eq "~"} {
1141 set p [string range $p 1 end]
1142 set not 1
1143 } else {
1144 set not 0
1145 }
1146 set p [string trim $p *]
1147 if {[string index $p 0]=="^"} {
1148 set p [string range $p 1 end]
1149 } else {
1150 set p "*$p"
1151 }
1152 if {[string index $p end]=="\$"} {
1153 set p [string range $p 0 end-1]
1154 } else {
1155 set p "$p*"
1156 }
1157 if {$not} {
1158 if {[string match $p $jobcmd]} {return 0}
1159 } else {
1160 if {[string match $p $jobcmd]} {
1161 set bMatch 1
1162 } else {
1163 set bMiss 1
1164 }
1165 }
1166 }
1167 if {!$bMiss} {
1168 set bMatch 1
1169 }
1170 return $bMatch
1171 }
1172
1173
1174 # Argument $build is either an empty string, or else a list of length 3
1175 # describing the job to build testfixture. In the usual form:
1176 #
1177 # {ID DIRNAME DISPLAYNAME}
1178 #
1179 # e.g
1180 #
1181 # {1 /home/user/sqlite/test/testrunner_bld_xyz All-Debug}
1182 #
1183 proc add_tcl_jobs {build config patternlist {shelldepid ""}} {
1184 global TRG
1185 set ntcljob 0
1186
1187 set topdir [file dirname $::testdir]
1188 set testrunner_tcl [file normalize [info script]]
1189
1190 if {$build==""} {
1191 set testfixture $TRG(interp)
1192 } else {
1193 set testfixture [file join [lindex $build 1] $TRG(testfixture)]
1194 }
1195 if {[lindex $build 2]=="Valgrind"} {
1196 set setvar "export OMIT_MISUSE=1\n"
1197 set testfixture "${setvar}valgrind -v --error-exitcode=1 $testfixture"
1198 }
1199
1200 # The ::testspec array is populated by permutations.test
1201 foreach f [dict get $::testspec($config) -files] {
1202
1203 if {![job_matches_any_pattern $patternlist "$config [file tail $f]"]} {
1204 continue
1205 }
1206
1207 if {[file pathtype $f]!="absolute"} { set f [file join $::testdir $f] }
1208 set f [file normalize $f]
1209
1210 set displayname [string map [list $topdir/ {}] $f]
1211 if {$config=="full" || $config=="veryquick"} {
1212 set cmd "$testfixture $f"
1213 } else {
1214 set cmd "$testfixture $testrunner_tcl $config $f"
1215 set displayname "config=$config $displayname"
1216 }
1217 if {$build!=""} {
1218 set displayname "[lindex $build 2] $displayname"
1219 }
1220
1221 set lProp [trd_test_script_properties $f]
1222 set priority 0
1223 if {[lsearch $lProp slow]>=0} { set priority 2 }
1224 if {[lsearch $lProp superslow]>=0} { set priority 4 }
1225
1226 set depid [lindex $build 0]
1227 if {$shelldepid!="" && [lsearch $lProp shell]>=0} { set depid $shelldepid }
1228
1229 incr ntcljob
1230 add_job \
1231 -displaytype tcl \
1232 -displayname $displayname \
1233 -cmd $cmd \
1234 -depid $depid \
1235 -priority $priority
1236 }
1237 if {$ntcljob==0 && [llength $build]>0} {
1238 set bldid [lindex $build 0]
1239 trdb eval {DELETE FROM jobs WHERE rowid=$bldid}
1240 }
1241 }
1242
1243 proc add_build_job {buildname target {postcmd ""} {depid ""}} {
1244 global TRG
1245
1246 set dirname "[string tolower [string map {- _} $buildname]]_$target"
1247 set dirname "testrunner_bld_$dirname"
1248
1249 set cmd "$TRG(makecmd) $target"
1250 if {$postcmd!=""} {
1251 append cmd "\n"
1252 append cmd $postcmd
1253 }
1254
1255 set id [add_job \
1256 -displaytype bld \
1257 -displayname "Build $buildname ($target)" \
1258 -dirname $dirname \
1259 -build $buildname \
1260 -cmd $cmd \
1261 -depid $depid \
1262 -priority 3
1263 ]
1264
1265 list $id [file normalize $dirname] $buildname
1266 }
1267
1268 proc add_shell_build_job {buildname dirname depid} {
1269 global TRG
1270
1271 if {$TRG(platform)=="win"} {
1272 set path [string map {/ \\} "$dirname/"]
1273 set copycmd "xcopy $TRG(shell) $path"
1274 } else {
1275 set copycmd "cp $TRG(shell) $dirname/"
1276 }
1277
1278 return [
1279 add_build_job $buildname $TRG(shell) $copycmd $depid
1280 ]
1281 }
1282
1283
1284 proc add_make_job {bld target} {
1285 global TRG
1286
1287 if {$TRG(platform)=="win"} {
1288 set path [string map {/ \\} [lindex $bld 1]]
1289 set cmd "xcopy /S $path\\* ."
1290 } else {
1291 set cmd "cp -r [lindex $bld 1]/* ."
1292 }
1293 append cmd "\n$TRG(makecmd) $target"
1294
1295 add_job \
1296 -displaytype make \
1297 -displayname "[lindex $bld 2] make $target" \
1298 -cmd $cmd \
1299 -depid [lindex $bld 0] \
1300 -priority 1
1301 }
1302
1303 proc add_fuzztest_jobs {buildname patternlist} {
1304 global env TRG
1305 # puts buildname=$buildname
1306
1307 foreach {interpreter scripts} [trd_fuzztest_data $buildname] {
1308 set bldDone 0
1309 set subcmd [lrange $interpreter 1 end]
1310 set interpreter [lindex $interpreter 0]
1311
1312 # For fuzzcheck-asan and fuzzcheck-ubsan, break up some
1313 # fuzzdata files into multiple slices, for improved
1314 # concurrency.
1315 #
1316 if {[string match *fuzzcheck-*san $interpreter]} {
1317 set newscripts {}
1318 foreach s $scripts {
1319 if {[string match {*fuzzdata[12].db} $s]
1320 && ![string match slice $s]} {
1321 set N 6
1322 for {set i 0} {$i<$N} {incr i} {
1323 lappend newscripts [list --slice $i $N $s]
1324 }
1325 } else {
1326 lappend newscripts $s
1327 }
1328 }
1329 set scripts $newscripts
1330 }
1331
1332 if {[string match fuzzcheck* $interpreter]
1333 && [info exists env(FUZZDB)]
1334 && [file readable $env(FUZZDB)]
1335 && $buildname ne "Windows-Win32Heap"
1336 && $buildname ne "Windows-Memdebug"
1337 } {
1338 set TRG(FUZZDB) $env(FUZZDB)
1339 set fname [file normalize $env(FUZZDB)]
1340 set N [expr {([file size $fname]+4999999)/5000000}]
1341 for {set i 0} {$i<$N} {incr i} {
1342 lappend scripts [list --slice $i $N $fname]
1343 }
1344 }
1345
1346 foreach s $scripts {
1347
1348 # Fuzz data files fuzzdata1.db and fuzzdata2.db are larger than
1349 # the others. So ensure that these are run as a higher priority.
1350 if {[llength $s]==1} {
1351 set tail [file tail $s]
1352 } else {
1353 set fname [lindex $s end]
1354 set tail [lrange $s 0 end-1]
1355 lappend tail [file tail $fname]
1356 }
1357 if {![job_matches_any_pattern $patternlist "$interpreter $tail"]} {
1358 continue
1359 }
1360 if {!$bldDone} {
1361 set bld [add_build_job $buildname $interpreter]
1362 foreach {depid dirname displayname} $bld {}
1363 set bldDone 1
1364 }
1365 if {[string match ?-slice* $tail]} {
1366 set priority 15
1367 } elseif {$tail=="fuzzdata1.db"
1368 || $tail=="fuzzdata2.db"
1369 || $tail=="fuzzdata8.db"} {
1370 set priority 5
1371 } else {
1372 set priority 1
1373 }
1374 add_job \
1375 -displaytype fuzz \
1376 -displayname "$buildname $interpreter $tail" \
1377 -depid $depid \
1378 -cmd "[file join $dirname $interpreter] $subcmd $s" \
1379 -priority $priority
1380 }
1381 }
1382 }
1383
1384 proc add_zipvfs_jobs {} {
1385 global TRG
1386 source [file join $TRG(zipvfs) test zipvfs_testrunner.tcl]
1387
1388 set bld [add_build_job Zipvfs $TRG(testfixture)]
1389 foreach s [zipvfs_testrunner_files] {
1390 set cmd "[file join [lindex $bld 1] $TRG(testfixture)] $s"
1391 add_job \
1392 -displaytype tcl \
1393 -displayname "Zipvfs [file tail $s]" \
1394 -cmd $cmd \
1395 -depid [lindex $bld 0]
1396 }
1397
1398 set ::env(SQLITE_TEST_DIR) $::testdir
1399 }
1400
1401 # Used to add jobs for "mdevtest" and "sdevtest".
1402 #
1403 proc add_devtest_jobs {lBld patternlist} {
1404 global TRG
1405
1406 foreach b $lBld {
1407 set bld [add_build_job $b $TRG(testfixture)]
1408 add_tcl_jobs $bld veryquick $patternlist SHELL
1409 add_fuzztest_jobs $b $patternlist
1410
1411 if {[trdb one "SELECT EXISTS (SELECT 1 FROM jobs WHERE depid='SHELL')"]} {
1412 set sbld [add_shell_build_job $b [lindex $bld 1] [lindex $bld 0]]
1413 set sbldid [lindex $sbld 0]
1414 trdb eval {
1415 UPDATE jobs SET depid=$sbldid WHERE depid='SHELL'
1416 }
1417 }
1418
1419 }
1420 }
1421
1422 # Check to ensure that TRG(interp) is a full-blown "testfixture" and
1423 # not just a "tclsh".
1424 #
1425 # The value of TRG(interp) defaults to whatever interpreter is running
1426 # this script, which might be either tclsh or testfixture. If tclsh is
1427 # running this script, change $TRG(interp) to be an instance of testfixture.
1428 # If no testfixture exists in the directory from which this script is run,
1429 # attempt to build one.
1430 #
1431 # Do not return unless $TRG(interp) is a valid testfixture. If unable
1432 # to find and/or construct one, abort with an error message.
1433 #
1434 proc must_be_testfixture {} {
1435 global TRG
1436 if {[lsearch [info commands] sqlite3_soft_heap_limit]<0} {
1437 if {![file exec $TRG(testfixture-fullpath)]} {
1438 puts "make testfixture"
1439 catch {exec make testfixture >@stdout 2>@stderr}
1440 }
1441 if {![file exec $TRG(testfixture-fullpath)]} {
1442 puts "Requires testfixture, and I was unable to build it."
1443 exit 1
1444 }
1445 set TRG(interp) $TRG(testfixture-fullpath)
1446 }
1447 }
1448
1449 proc add_jobs_from_cmdline {patternlist} {
1450 global TRG
1451
1452 if {$TRG(zipvfs)!=""} {
1453 add_zipvfs_jobs
1454 if {[llength $patternlist]==0} return
1455 }
1456
1457 if {[llength $patternlist]==0} {
1458 set patternlist [list veryquick]
1459 }
1460
1461 set first [lindex $patternlist 0]
1462 switch -- $first {
1463 all {
1464 must_be_testfixture
1465 set patternlist [lrange $patternlist 1 end]
1466 set clist [trd_all_configs]
1467 foreach c $clist {
1468 add_tcl_jobs "" $c $patternlist
1469 }
1470 }
1471
1472 devtest -
1473 mdevtest {
1474 set config_set {
1475 All-O0
1476 All-Debug
1477 }
1478 add_devtest_jobs $config_set [lrange $patternlist 1 end]
1479 }
1480
1481 sdevtest {
1482 set config_set {
1483 All-Sanitize
1484 All-Debug
1485 }
1486 add_devtest_jobs $config_set [lrange $patternlist 1 end]
1487 }
1488
1489 release {
1490 set patternlist [lrange $patternlist 1 end]
1491 foreach b [trd_builds $TRG(platform)] {
1492 if {$TRG(config)!="" && ![regexp "\\y$b\\y" $TRG(config)]} continue
1493 if {[regexp "\\y$b\\y" $TRG(omitconfig)]} continue
1494 set bld [add_build_job $b $TRG(testfixture)]
1495 foreach c [trd_configs $TRG(platform) $b] {
1496 add_tcl_jobs $bld $c $patternlist SHELL
1497 }
1498
1499 foreach e [trd_extras $TRG(platform) $b] {
1500 if {$e=="fuzztest"} {
1501 add_fuzztest_jobs $b $patternlist
1502 } elseif {[job_matches_any_pattern $patternlist $e]} {
1503 add_make_job $bld $e
1504 }
1505 }
1506
1507 if {[trdb one "SELECT EXISTS(SELECT 1
1508 FROM jobs WHERE depid='SHELL')"]} {
1509 set sbld [add_shell_build_job $b [lindex $bld 1] [lindex $bld 0]]
1510 set sbldid [lindex $sbld 0]
1511 trdb eval {
1512 UPDATE jobs SET depid=$sbldid WHERE depid='SHELL'
1513 }
1514 }
1515 }
1516 }
1517
1518 c {
1519 set patternlist [lrange $patternlist 1 end]
1520 foreach b [trd_builds $TRG(platform)] {
1521 add_c_jobs $b $patternlist
1522 }
1523 }
1524
1525 list {
1526 set allperm [array names ::testspec]
1527 lappend allperm all devtest mdevtest sdevtest release list
1528 puts "Allowed values for the PERMUTATION argument: [lsort $allperm]"
1529 exit 0
1530 }
1531
1532 retest {
1533 # no-op
1534 }
1535
1536 default {
1537 must_be_testfixture
1538 if {[info exists ::testspec($first)]} {
1539 add_tcl_jobs "" $first [lrange $patternlist 1 end]
1540 } else {
1541 add_tcl_jobs "" full $patternlist
1542 }
1543 }
1544 }
1545
1546 # If the "--case DISPLAYNAME" option appears on the command-line, mark
1547 # all tests other than DISPLAYNAME as 'omit'.
1548 #
1549 if {[info exists TRG(case)] && $TRG(case) ne ""} {
1550 set jid [trdb one {
1551 SELECT jobid FROM jobs WHERE displayname GLOB $TRG(case)
1552 }]
1553 if {$jid eq ""} {
1554 puts "ERROR: No jobs match \"$TRG(case)\"."
1555 puts "The argument to --cases must GLOB match the jobs.displayname column"
1556 puts "of the testrunner.db database."
1557 trdb eval {UPDATE jobs SET state='omit'}
1558 } else {
1559 trdb eval {
1560 WITH RECURSIVE keepers(jid,did) AS (
1561 SELECT jobid,depid FROM jobs
1562 WHERE displayname GLOB $TRG(case)
1563 UNION
1564 SELECT jobid,depid FROM jobs, keepers WHERE jobid=did
1565 )
1566 DELETE FROM jobs WHERE jobid NOT IN (SELECT jid FROM keepers);
1567 }
1568 }
1569 }
1570 }
1571
1572 # Initializer, or reinitialize, the testrunner.db database file.
1573 #
1574 proc make_new_testset {} {
1575 global TRG
1576
1577 trdb eval {PRAGMA journal_mode=WAL;}
1578 r_write_db {
1579 trdb eval $TRG(schema)
1580 set nJob $TRG(nJob)
1581 set cmdline $TRG(cmdline)
1582 set tm [clock_milliseconds]
1583 trdb eval { REPLACE INTO config VALUES('njob', $nJob ); }
1584 trdb eval { REPLACE INTO config VALUES('cmdline', $cmdline ); }
1585 trdb eval { REPLACE INTO config VALUES('start', $tm ); }
1586
1587 add_jobs_from_cmdline $TRG(patternlist)
1588
1589 }
1590 }
1591
1592 proc mark_job_as_finished {jobid output state endtm} {
1593 set ntest 1
1594 set nerr 0
1595 if {$endtm>0} {
1596 set re {\y(\d+) errors out of (\d+) tests( on [^\n]+\n)?}
1597 if {[regexp $re $output all a b pltfm]} {
1598 set nerr $a
1599 set ntest $b
1600 }
1601 regexp {\ySQLite \d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d [0-9a-fA-F]+} \
1602 $output svers
1603 }
1604 r_write_db {
1605 if {$state=="failed"} {
1606 set childstate omit
1607 if {$nerr<=0} {set nerr 1}
1608 } else {
1609 set childstate ready
1610 }
1611 if {[info exists pltfm]} {set pltfm [string trim $pltfm]}
1612 trdb eval {
1613 UPDATE jobs
1614 SET output=$output, state=$state, endtime=$endtm, span=$endtm-starttime,
1615 ntest=$ntest, nerr=$nerr, svers=$svers, pltfm=$pltfm
1616 WHERE jobid=$jobid;
1617 UPDATE jobs SET state=$childstate
1618 WHERE depid=$jobid AND state!='halt' AND state!='done';
1619 UPDATE config SET value=value+$nerr WHERE name='nfail';
1620 UPDATE config SET value=value+$ntest WHERE name='ntest';
1621 }
1622 }
1623 }
1624
1625 proc script_input_ready {fd iJob jobid} {
1626 global TRG
1627 global O
1628 global T
1629
1630 if {[eof $fd]} {
1631 trdb eval { SELECT * FROM jobs WHERE jobid=$jobid } job {}
1632
1633 # If this job specified a directory name, then delete the run.sh/run.bat
1634 # file from it before continuing. This is because the contents of this
1635 # directory might be copied by some other job, and we don't want to copy
1636 # the run.sh file in this case.
1637 if {$job(dirname)!=""} {
1638 file delete -force [file join $job(dirname) $TRG(run)]
1639 }
1640
1641 set ::done 1
1642 fconfigure $fd -blocking 1
1643 set state "done"
1644 set rc [catch { close $fd } msg]
1645 if {$rc} {
1646 puts [format %-79.79s "FAILED: $job(displayname) ($iJob)"]
1647 set state "failed"
1648 if {$TRG(stopOnError)} {
1649 puts "OUTPUT: $O($iJob)"
1650 exit 1
1651 }
1652 if {$TRG(stopOnCore) && [string first {core dumped} $O($iJob)]>0} {
1653 puts "OUTPUT: $O($iJob)"
1654 exit 1
1655 }
1656 }
1657
1658 set tm [clock_milliseconds]
1659 set jobtm [expr {$tm - $job(starttime)}]
1660
1661 puts $TRG(log) "### $job(displayname) ${jobtm}ms ($state)"
1662 puts $TRG(log) [string trim $O($iJob)]
1663
1664 mark_job_as_finished $jobid $O($iJob) $state $tm
1665
1666 dirs_freeDir $iJob
1667 launch_some_jobs
1668 incr ::wakeup
1669 } else {
1670 set rc [catch { gets $fd line } res]
1671 if {$rc} {
1672 puts "ERROR $res"
1673 }
1674 if {$res>=0} {
1675 append O($iJob) "$line\n"
1676 }
1677 }
1678
1679 }
1680
1681 proc dirname {ii} {
1682 return "testdir$ii"
1683 }
1684
1685 proc launch_another_job {iJob} {
1686 global TRG
1687 global O
1688 global T
1689
1690 set testfixture $TRG(interp)
1691 set script $TRG(info_script)
1692
1693 set O($iJob) ""
1694
1695 set jobdict [r_get_next_job $iJob]
1696 if {$jobdict==""} { return 0 }
1697 array set job $jobdict
1698
1699 set dir $job(dirname)
1700 if {$dir==""} { set dir [dirname $iJob] }
1701 create_or_clear_dir $dir
1702
1703 if {$job(build)!=""} {
1704 set srcdir [file dirname $::testdir]
1705 if {$job(build)=="Zipvfs"} {
1706 set script [zipvfs_testrunner_script]
1707 } else {
1708 set bWin [expr {$TRG(platform)=="win"}]
1709 set script [trd_buildscript $job(build) $srcdir $bWin]
1710 }
1711 set fd [open [file join $dir $TRG(make)] w]
1712 puts $fd $script
1713 close $fd
1714 }
1715
1716 # Add a batch/shell file command to set the directory used for temp
1717 # files to the test's working directory. Otherwise, tests that use
1718 # large numbers of temp files (e.g. zipvfs), might generate temp
1719 # filename collisions.
1720 if {$TRG(platform)=="win"} {
1721 set set_tmp_dir "SET SQLITE_TMPDIR=[file normalize $dir]"
1722 } else {
1723 set set_tmp_dir "export SQLITE_TMPDIR=\"[file normalize $dir]\""
1724 }
1725
1726 if { $TRG(dryrun) } {
1727
1728 mark_job_as_finished $job(jobid) "" done 0
1729 dirs_freeDir $iJob
1730 if {$job(build)!=""} {
1731 puts $TRG(log) "(cd $dir ; $job(cmd) )"
1732 } else {
1733 puts $TRG(log) "$job(cmd)"
1734 }
1735
1736 } else {
1737 set pwd [pwd]
1738 cd $dir
1739 set fd [open $TRG(run) w]
1740 puts $fd $set_tmp_dir
1741 puts $fd $job(cmd)
1742 close $fd
1743 set fd [open "|$TRG(runcmd) 2>@1" r]
1744 cd $pwd
1745
1746 fconfigure $fd -blocking false -translation binary
1747 fileevent $fd readable [list script_input_ready $fd $iJob $job(jobid)]
1748 }
1749
1750 return 1
1751 }
1752
1753 # Show the testing progress report
1754 #
1755 proc progress_report {} {
1756 global TRG
1757
1758 if {$TRG(fullstatus)} {
1759 if {$::tcl_platform(platform) eq "windows"} {
1760 exec [info nameofexe] $::argv0 status --cls
1761 } else {
1762 show_status trdb 1
1763 }
1764 } else {
1765 set tmms [expr [clock_milliseconds] - $TRG(starttime)]
1766 set tm [format "%d" [expr int($tmms/1000.0 + 0.5)]]
1767
1768 set wtotal 0
1769 set wdone 0
1770 r_write_db {
1771 trdb eval {
1772 SELECT displaytype, state, count(*) AS cnt, sum(estwork) AS ew
1773 FROM jobs
1774 GROUP BY 1, 2
1775 } {
1776 set v($state,$displaytype) $cnt
1777 incr t($displaytype) $cnt
1778 incr wtotal $ew
1779 if {$state=="done" || $state=="failed" || $state=="omit"} {
1780 incr wdone $ew
1781 }
1782 }
1783 }
1784
1785 set text ""
1786 foreach j [lsort [array names t]] {
1787 foreach k {done failed running} { incr v($k,$j) 0 }
1788 set fin [expr $v(done,$j) + $v(failed,$j)]
1789 lappend text "${j}($fin/$t($j))"
1790 if {$v(failed,$j)>0} {
1791 lappend text "f$v(failed,$j)"
1792 }
1793 if {$v(running,$j)>0} {
1794 lappend text "r$v(running,$j)"
1795 }
1796 }
1797 set report "[elapsetime $tmms] [join $text { }]"
1798 if {$wdone>0 && [set pct [expr {int(($wdone*100.0)/$wtotal)}]]>=4} {
1799 set tmleft [expr {($tmms/double($wdone))*($wtotal-$wdone)}]
1800 set etc " ETC [elapsetime $tmleft]"
1801 if {[string length $report]+[string length $etc]<80} {
1802 append report $etc
1803 }
1804 # append report " $pct%"
1805 }
1806 puts -nonewline [format %-79.79s $report]\r
1807 flush stdout
1808 }
1809 after $TRG(reporttime) progress_report
1810 }
1811
1812 proc launch_some_jobs {} {
1813 global TRG
1814 set nJob [trdb one { SELECT value FROM config WHERE name='njob' }]
1815
1816 while {[dirs_nHelper]<$nJob} {
1817 set iDir [dirs_allocDir]
1818 if {0==[launch_another_job $iDir]} {
1819 dirs_freeDir $iDir
1820 break;
1821 }
1822 }
1823 }
1824
1825 proc run_testset {} {
1826 global TRG
1827 set ii 0
1828
1829 set TRG(starttime) [clock_milliseconds]
1830 set TRG(log) [open $TRG(logname) w]
1831
1832 launch_some_jobs
1833
1834 if {$TRG(fullstatus)} {puts "\033\[2J"}
1835 progress_report
1836 while {[dirs_nHelper]>0} {
1837 after 500 {incr ::wakeup}
1838 vwait ::wakeup
1839 }
1840 close $TRG(log)
1841 progress_report
1842 puts ""
1843
1844 r_write_db {
1845 set tm [clock_milliseconds]
1846 trdb eval { REPLACE INTO config VALUES('end', $tm ); }
1847 set nErr [trdb one {SELECT count(*) FROM jobs WHERE state='failed'}]
1848 if {$nErr>0} {
1849 puts "$nErr failures:"
1850 trdb eval {
1851 SELECT displayname FROM jobs WHERE state='failed'
1852 } {
1853 puts "FAILED: $displayname"
1854 }
1855 }
1856 set nOmit [trdb one {SELECT count(*) FROM jobs WHERE state='omit'}]
1857 if {$nOmit>0} {
1858 puts "$nOmit jobs skipped due to prior failures"
1859 }
1860 }
1861
1862 puts "Test database is $TRG(dbname)"
1863 puts "Test log is $TRG(logname)"
1864 if {[info exists TRG(FUZZDB)]} {
1865 puts "Extra fuzzcheck data taken from $TRG(FUZZDB)"
1866 }
1867 trdb eval {
1868 SELECT sum(ntest) AS totaltest,
1869 sum(nerr) AS totalerr
1870 FROM jobs
1871 } break
1872 trdb eval {
1873 SELECT max(endtime)-min(starttime) AS totaltime
1874 FROM jobs WHERE endtime>0
1875 } break;
1876 set et [elapsetime $totaltime]
1877 set pltfm {}
1878 trdb eval {
1879 SELECT pltfm, count(*) FROM jobs WHERE pltfm IS NOT NULL
1880 ORDER BY 2 DESC LIMIT 1
1881 } break
1882 if {$totalerr==""} {set totalerr 0}
1883 if {$totaltest==""} {set totaltest 0}
1884 puts "$totalerr errors out of $totaltest tests in $et $pltfm"
1885 trdb eval {
1886 SELECT DISTINCT substr(svers,1,79) as v1 FROM jobs WHERE svers IS NOT NULL
1887 } {puts $v1}
1888
1889 }
1890
1891 # If the argument is "retest", simply rerun all tests from the previous
1892 # run that are marked as one of "ready", "running", "failed", or "omit"
1893 # plus redo any build of dependencies those tests.
1894 #
1895 proc handle_retest {} {
1896 set cnt 0
1897 if {[catch {trdb exists {SELECT jobid FROM jobs}} cnt] || $cnt==0} {
1898 puts "No test available to rerun"
1899 exit 1
1900 }
1901 trdb eval {UPDATE jobs SET state='ready'
1902 WHERE state IN ('running','failed','omit')}
1903 for {set kk 0} {$kk<2} {incr kk} {
1904 trdb eval {
1905 UPDATE jobs SET state='ready'
1906 WHERE jobid IN (SELECT depid FROM jobs WHERE state='ready');
1907 UPDATE jobs SET state='' WHERE state='ready' AND depid<>'';
1908 }
1909 }
1910 }
1911
1912 # Handle the --buildonly option, if it was specified.
1913 #
1914 proc handle_buildonly {} {
1915 global TRG
1916 if {$TRG(buildonly)} {
1917 r_write_db {
1918 trdb eval { DELETE FROM jobs WHERE displaytype!='bld' }
1919 }
1920 }
1921 }
1922
1923 # Handle the --explain option. Provide a human-readable
1924 # explanation of all the tests that are in the trdb database jobs
1925 # table.
1926 #
1927 proc explain_layer {indent depid} {
1928 global TRG
1929 if {$TRG(buildonly)} {
1930 set showtests 0
1931 } else {
1932 set showtests 1
1933 }
1934 trdb eval {SELECT jobid, displayname, displaytype, dirname
1935 FROM jobs WHERE depid=$depid ORDER BY displayname} {
1936 if {$displaytype=="bld"} {
1937 puts "${indent}$displayname in $dirname"
1938 explain_layer "${indent} " $jobid
1939 } elseif {$showtests} {
1940 puts "${indent}$displayname"
1941 }
1942 }
1943 }
1944 proc explain_tests {} {
1945 explain_layer "" ""
1946 }
1947
1948 sqlite3 trdb $TRG(dbname)
1949 trdb timeout $TRG(timeout)
1950 if {[llength $TRG(patternlist)]==1 && $TRG(patternlist) eq "retest"} {
1951 set tm 0
1952 handle_retest
1953 } else {
1954 set tm [lindex [time { make_new_testset }] 0]
1955 }
1956 if {$TRG(explain)} {
1957 explain_tests
1958 } else {
1959 if {$TRG(nJob)>1} {
1960 puts "splitting work across $TRG(nJob) cores"
1961 }
1962 if {$tm>0} {
1963 puts "built testset in [expr $tm/1000]ms.."
1964 }
1965 handle_buildonly
1966 run_testset
1967 }
1968 trdb close