| 1 |
#!/usr/bin/env Rscript |
| 2 |
|
| 3 |
#' Run a 'testme' Test Script |
| 4 |
#' |
| 5 |
#' R usage: |
| 6 |
#' testme("<name>") |
| 7 |
#' |
| 8 |
#' Command-line usage: |
| 9 |
#' tests/test-<name>.R |
| 10 |
#' |
| 11 |
#' Command-line usage without package re-install: |
| 12 |
#' inst/testme/run.R --name=<test_name> |
| 13 |
#' inst/testme/run.R <test-name.R> |
| 14 |
#' |
| 15 |
#' Options: |
| 16 |
#' --package=<pkg> The name of the package being tested |
| 17 |
#' (Environment variable: `R_TESTME_PACKAGE`) |
| 18 |
#' (Default: The `Package` field of the DESCRIPTION file) |
| 19 |
#' --name=<name> The name of the test to run, used to locate the test |
| 20 |
#' script `test-<name>.R` |
| 21 |
#' (Environment variable: `R_TESTME_NAME`) |
| 22 |
#' --not-cran Set environment variable `NOT_CRAN=true` |
| 23 |
#' --coverage=summary Estimate test code coverage with basic summary |
| 24 |
#' --coverage=tally Estimate test code coverage with full tally summary |
| 25 |
#' --coverage=report Estimate test code coverage with full HTML report |
| 26 |
#' --debug Output debug messages |
| 27 |
#' (Environment variable: `R_TESTME_DEBUG`) |
| 28 |
#' |
| 29 |
#' Examples: |
| 30 |
#' testme/test-abc.R |
| 31 |
#' testme/test-abc.R --not-cran |
| 32 |
#' tests/test-cpuLoad.R --coverage=report |
| 33 |
#' |
| 34 |
#' inst/testme/run.R inst/testme/test-abc.R |
| 35 |
#' inst/testme/run.R inst/testme/test-abc.R --coverage |
| 36 |
#' |
| 37 |
#' Environment variables: |
| 38 |
#' * R_TESTME_PACKAGE |
| 39 |
#' * R_TESTME_NAME |
| 40 |
#' * R_TESTME_PATH |
| 41 |
#' * R_TESTME_FILTER_NAME |
| 42 |
#' * R_TESTME_FILTER_TAGS |
| 43 |
#' * R_TESTME_COVERAGE |
| 44 |
#' * R_TESTME_DEBUG |
| 45 |
main <- function() { |
| 46 |
cmd_args <- commandArgs(trailingOnly = TRUE) |
| 47 |
|
| 48 |
pattern <- "--package=([[:alpha:][:alnum:]]+)" |
| 49 |
idx <- grep(pattern, cmd_args) |
| 50 |
if (length(idx) > 0L) { |
| 51 |
stopifnot(length(idx) == 1L) |
| 52 |
testme_package <- gsub(pattern, "\\1", cmd_args[idx]) |
| 53 |
cmd_args <- cmd_args[-idx] |
| 54 |
} else { |
| 55 |
testme_package <- Sys.getenv("R_TESTME_PACKAGE", NA_character_) |
| 56 |
if (is.na(testme_package)) { |
| 57 |
desc <- read.dcf("DESCRIPTION") |
| 58 |
testme_package <- desc[1, "Package"] |
| 59 |
} |
| 60 |
} |
| 61 |
|
| 62 |
pattern <- "--path=([[:alpha:][:alnum:]]+)" |
| 63 |
idx <- grep(pattern, cmd_args) |
| 64 |
if (length(idx) > 0L) { |
| 65 |
stopifnot(length(idx) == 1L) |
| 66 |
path <- gsub(pattern, "\\1", cmd_args[idx]) |
| 67 |
cmd_args <- cmd_args[-idx] |
| 68 |
} else { |
| 69 |
path <- Sys.getenv("R_TESTME_PATH", NA_character_) |
| 70 |
if (is.na(path)) { |
| 71 |
path <- file.path("inst", "testme") |
| 72 |
} |
| 73 |
if (!utils::file_test("-d", path)) { |
| 74 |
stop("There exist no such 'R_TESTME_PATH' folder: ", sQuote(path)) |
| 75 |
} |
| 76 |
} |
| 77 |
|
| 78 |
pattern <- "--name=([[:alpha:][:alnum:]]+)" |
| 79 |
idx <- grep(pattern, cmd_args) |
| 80 |
if (length(idx) > 0L) { |
| 81 |
stopifnot(length(idx) == 1L) |
| 82 |
testme_name <- gsub(pattern, "\\1", cmd_args[idx]) |
| 83 |
cmd_args <- cmd_args[-idx] |
| 84 |
} else { |
| 85 |
testme_name <- NULL |
| 86 |
} |
| 87 |
|
| 88 |
pattern <- "^--not-cran" |
| 89 |
idx <- grep(pattern, cmd_args) |
| 90 |
if (length(idx) > 0L) { |
| 91 |
cmd_args <- cmd_args[-idx] |
| 92 |
Sys.setenv(NOT_CRAN = "TRUE") |
| 93 |
} |
| 94 |
|
| 95 |
pattern <- "^--debug" |
| 96 |
idx <- grep(pattern, cmd_args) |
| 97 |
if (length(idx) > 0L) { |
| 98 |
cmd_args <- cmd_args[-idx] |
| 99 |
Sys.setenv(R_TESTME_DEBUG = "TRUE") |
| 100 |
} |
| 101 |
|
| 102 |
pattern <- "^--coverage(|=([[:alpha:][:alnum:]]+))$" |
| 103 |
idx <- grep(pattern, cmd_args) |
| 104 |
if (length(idx) > 0L) { |
| 105 |
value <- gsub(pattern, "\\2", cmd_args[idx]) |
| 106 |
if (!nzchar(value)) { |
| 107 |
coverage <- "summary" |
| 108 |
} else { |
| 109 |
coverage <- match.arg(value, choices = c("none", "summary", "tally", "report")) |
| 110 |
} |
| 111 |
cmd_args <- cmd_args[-idx] |
| 112 |
} else { |
| 113 |
value <- Sys.getenv("R_TESTME_COVERAGE", "none") |
| 114 |
coverage <- match.arg(value, choices = c("none", "summary", "tally", "report")) |
| 115 |
} |
| 116 |
if (coverage != "none") { |
| 117 |
if (!utils::file_test("-f", "DESCRIPTION")) { |
| 118 |
stop("Current folder does not look like a package folder") |
| 119 |
} |
| 120 |
} |
| 121 |
|
| 122 |
## Fallback for 'testme_name'? |
| 123 |
if (is.null(testme_name)) { |
| 124 |
if (length(cmd_args) > 0) { |
| 125 |
stopifnot(length(cmd_args) == 1L) |
| 126 |
file <- cmd_args[1] |
| 127 |
if (utils::file_test("-f", file)) { |
| 128 |
testme_name <- gsub("(^test-|[.]R$)", "", basename(file)) |
| 129 |
} else { |
| 130 |
stop("No such file: ", file) |
| 131 |
} |
| 132 |
} else { |
| 133 |
testme_name <- Sys.getenv("R_TESTME_NAME", NA_character_) |
| 134 |
if (is.na(testme_name)) { |
| 135 |
stop("testme: Environment variable 'R_TESTME_NAME' is not set") |
| 136 |
} |
| 137 |
} |
| 138 |
} |
| 139 |
|
| 140 |
testme_file <- file.path(path, sprintf("test-%s.R", testme_name)) |
| 141 |
if (!utils::file_test("-f", testme_file)) { |
| 142 |
stop("There exist no such 'testme' file: ", sQuote(testme_file)) |
| 143 |
} |
| 144 |
|
| 145 |
|
| 146 |
## ----------------------------------------------------------------- |
| 147 |
## testme environment |
| 148 |
## ----------------------------------------------------------------- |
| 149 |
on_cran <- function() { |
| 150 |
not_cran <- Sys.getenv("NOT_CRAN", NA_character_) |
| 151 |
if (is.na(not_cran)) { |
| 152 |
not_cran <- FALSE |
| 153 |
} else { |
| 154 |
not_cran <- isTRUE(as.logical(not_cran)) |
| 155 |
} |
| 156 |
!interactive() && !not_cran |
| 157 |
} ## on_cran() |
| 158 |
|
| 159 |
|
| 160 |
## Get test script tags |
| 161 |
tags <- local({ |
| 162 |
lines <- readLines(testme_file, warn = FALSE) |
| 163 |
pattern <- "^#'[[:blank:]]+@tags[[:blank:]]+" |
| 164 |
lines <- grep(pattern, lines, value = TRUE) |
| 165 |
tags <- sub(pattern, "", lines) |
| 166 |
tags |
| 167 |
}) |
| 168 |
if (length(tags) > 0) { |
| 169 |
tags <- sub("[[:blank:]]*$", "", tags) |
| 170 |
tags <- unlist(strsplit(tags, split = "[[:blank:]]+")) |
| 171 |
tags <- sort(unique(tags)) |
| 172 |
} else { |
| 173 |
tags <- character(0L) |
| 174 |
} |
| 175 |
|
| 176 |
debug <- isTRUE(as.logical(Sys.getenv("R_TESTME_DEBUG"))) |
| 177 |
|
| 178 |
coverage <- match.arg(coverage, choices = c("none", "summary", "tally", "report")) |
| 179 |
|
| 180 |
## Create 'testme' environment on the search() path |
| 181 |
testme_config <- list( |
| 182 |
testme = TRUE, |
| 183 |
package = testme_package, |
| 184 |
name = testme_name, |
| 185 |
tags = tags, |
| 186 |
status = "created", |
| 187 |
start = proc.time(), |
| 188 |
script = testme_file, |
| 189 |
path = path, |
| 190 |
on_cran = on_cran(), |
| 191 |
coverage = coverage, |
| 192 |
debug = debug |
| 193 |
) |
| 194 |
if ("testme" %in% search()) detach(name = "testme") |
| 195 |
testme <- attach(testme_config, name = "testme", warn.conflicts = FALSE) |
| 196 |
rm(list = c("tags", "testme_package", "testme_name", "testme_file")) |
| 197 |
|
| 198 |
|
| 199 |
## ----------------------------------------------------------------- |
| 200 |
## Filters |
| 201 |
## ----------------------------------------------------------------- |
| 202 |
## Skip on CRAN? To run these tests, set env var NOT_CRAN=true |
| 203 |
if ("skip_on_cran" %in% tags && on_cran()) { |
| 204 |
testme[["status"]] <- "skipped" |
| 205 |
} |
| 206 |
|
| 207 |
code <- Sys.getenv("R_TESTME_FILTER_NAME", NA_character_) |
| 208 |
if (!is.na(code)) { |
| 209 |
expr <- tryCatch(parse(text = code), error = identity) |
| 210 |
if (inherits(expr, "error")) { |
| 211 |
stop("Syntax error in R_TESTME_FILTER_NAME: ", sQuote(code)) |
| 212 |
} |
| 213 |
|
| 214 |
keep <- tryCatch(eval(expr, envir = testme), error = identity) |
| 215 |
if (inherits(keep, "error")) { |
| 216 |
stop("Evaluation of R_TESTME_FILTER_NAME=%s produced an error: %s", |
| 217 |
sQuote(code), conditionMessage(keep)) |
| 218 |
} |
| 219 |
if (!isTRUE(keep)) testme[["status"]] <- "skipped" |
| 220 |
} |
| 221 |
|
| 222 |
code <- Sys.getenv("R_TESTME_FILTER_TAGS", NA_character_) |
| 223 |
if (!is.na(code)) { |
| 224 |
expr <- tryCatch(parse(text = code), error = identity) |
| 225 |
if (inherits(expr, "error")) { |
| 226 |
stop("Syntax error in R_TESTME_FILTER_TAGS: ", sQuote(code)) |
| 227 |
} |
| 228 |
keep <- tryCatch(eval(expr, envir = testme), error = identity) |
| 229 |
if (inherits(keep, "error")) { |
| 230 |
stop("Evaluation of R_TESTME_FILTER_TAGS=%s produced an error: %s", |
| 231 |
sQuote(code), conditionMessage(keep)) |
| 232 |
} |
| 233 |
if (!isTRUE(keep)) testme[["status"]] <- "skipped" |
| 234 |
} |
| 235 |
|
| 236 |
testme_run_test(testme) |
| 237 |
} ## main() |
| 238 |
|
| 239 |
|
| 240 |
|
| 241 |
#' @param testme A names list |
| 242 |
#' |
| 243 |
testme_run_test <- function(testme) { |
| 244 |
message(sprintf("Test %s ...", sQuote(testme[["name"]]))) |
| 245 |
if (testme[["debug"]]) { |
| 246 |
message("testme:") |
| 247 |
message(paste(utils::capture.output(utils::str(as.list(testme))), collapse = "\n")) |
| 248 |
} |
| 249 |
|
| 250 |
path <- testme[["path"]] |
| 251 |
|
| 252 |
## Process prologue scripts, if they exist |
| 253 |
if (testme[["status"]] != "skipped" && |
| 254 |
utils::file_test("-d", file.path(path, "_prologue"))) { |
| 255 |
testme[["status"]] <- "prologue" |
| 256 |
local({ |
| 257 |
## Find all prologue scripts |
| 258 |
files <- dir(file.path(path, "_prologue"), pattern = "*[.]R$", full.names = TRUE) |
| 259 |
files <- sort(files) |
| 260 |
testme[["prologue_scripts"]] <- files |
| 261 |
|
| 262 |
## Source all prologue scripts inside the 'testme' environment |
| 263 |
expr <- bquote({ |
| 264 |
files <- prologue_scripts |
| 265 |
if (.(testme[["debug"]])) message(sprintf("Sourcing %d prologue scripts ...", length(files))) |
| 266 |
for (kk in seq_along(files)) { |
| 267 |
file <- files[kk] |
| 268 |
if (.(testme[["debug"]])) message(sprintf("%02d/%02d prologue script %s", kk, length(files), sQuote(file))) |
| 269 |
source(file, local = TRUE) |
| 270 |
} |
| 271 |
if (.(testme[["debug"]])) message(sprintf("Sourcing %d prologue scripts ... done", length(files))) |
| 272 |
rm(list = c("kk", "file", "files")) |
| 273 |
}) |
| 274 |
eval(expr, envir = testme) |
| 275 |
}) |
| 276 |
|
| 277 |
# ## In case prologue scripts overwrote some elements in 'testme' |
| 278 |
# for (name in names(testme_config)) { |
| 279 |
# testme[[name]] <- testme_config[[name]] |
| 280 |
# } |
| 281 |
} |
| 282 |
|
| 283 |
|
| 284 |
## Run test script |
| 285 |
## Note, prologue scripts may trigger test to be skipped |
| 286 |
if (testme[["status"]] != "skipped") { |
| 287 |
if (testme[["debug"]]) message("Running test script: ", sQuote(testme[["script"]])) |
| 288 |
testme[["status"]] <- "failed" |
| 289 |
if (testme[["coverage"]] != "none") { |
| 290 |
pkg_env <- pkgload::load_all() |
| 291 |
cov <- covr::environment_coverage(pkg_env[["env"]], test_files = testme[["script"]]) |
| 292 |
## Keep source files with non-zero coverage |
| 293 |
tally <- covr::tally_coverage(cov) |
| 294 |
tally <- subset(tally, value > 0) |
| 295 |
cov <- cov[covr::display_name(cov) %in% unique(tally$filename)] |
| 296 |
testme[["test_coverage"]] <- cov |
| 297 |
} else { |
| 298 |
testme[["test_coverage"]] <- NULL |
| 299 |
source(testme[["script"]], echo = TRUE) |
| 300 |
} |
| 301 |
testme[["status"]] <- "success" |
| 302 |
} |
| 303 |
|
| 304 |
|
| 305 |
## Process epilogue scripts, if they exist |
| 306 |
## Note, epilogue scripts may change status or produce check errors |
| 307 |
if (testme[["status"]] == "success" && |
| 308 |
utils::file_test("-d", file.path(path, "_epilogue"))) { |
| 309 |
testme[["status"]] <- "epilogue" |
| 310 |
local({ |
| 311 |
## Find all epilogue scripts |
| 312 |
files <- dir(file.path(path, "_epilogue"), pattern = "*[.]R$", full.names = TRUE) |
| 313 |
files <- sort(files) |
| 314 |
testme[["epilogue_scripts"]] <- files |
| 315 |
|
| 316 |
## Source all epilogue scripts inside the 'testme' environment |
| 317 |
expr <- bquote({ |
| 318 |
files <- epilogue_scripts |
| 319 |
if (.(testme[["debug"]])) message(sprintf("Sourcing %d epilogue scripts ...", length(files))) |
| 320 |
for (kk in seq_along(files)) { |
| 321 |
file <- files[kk] |
| 322 |
if (.(testme[["debug"]])) message(sprintf("%02d/%02d epilogue script %s", kk, length(files), sQuote(file))) |
| 323 |
source(file, local = TRUE) |
| 324 |
} |
| 325 |
if (.(testme[["debug"]])) message(sprintf("Sourcing %d epilogue scripts ... done", length(files))) |
| 326 |
rm(list = c("kk", "file", "files")) |
| 327 |
}) |
| 328 |
eval(expr, envir = testme) |
| 329 |
}) |
| 330 |
testme[["status"]] <- "success" |
| 331 |
} |
| 332 |
|
| 333 |
testme[["stop"]] <- proc.time() |
| 334 |
dt <- testme[["stop"]] - testme[["start"]] |
| 335 |
dt_str <- sprintf("%s=%.1gs", names(dt), dt) |
| 336 |
message("Test time: ", paste(dt_str, collapse = ", ")) |
| 337 |
|
| 338 |
if ("testme" %in% search()) detach(name = "testme") |
| 339 |
|
| 340 |
cov <- testme[["test_coverage"]] |
| 341 |
if (!is.null(cov)) { |
| 342 |
message("Source files covered by the test script:") |
| 343 |
if (length(cov) > 0) { |
| 344 |
if ("tally" %in% testme[["coverage"]]) { |
| 345 |
tally <- covr::tally_coverage(cov) |
| 346 |
print(tally) |
| 347 |
} |
| 348 |
if ("report" %in% testme[["coverage"]]) { |
| 349 |
html <- covr::report(cov, browse = FALSE) |
| 350 |
browseURL(html) |
| 351 |
Sys.sleep(5.0) |
| 352 |
} |
| 353 |
} else { |
| 354 |
message("* No source files were covered by this test!") |
| 355 |
} |
| 356 |
} |
| 357 |
|
| 358 |
message(sprintf("Test %s ... %s", sQuote(testme[["name"]]), testme[["status"]])) |
| 359 |
} ## testme_run_test() |
| 360 |
|
| 361 |
|
| 362 |
main() |