| 1 |
<!DOCTYPE html> |
| 2 |
|
| 3 |
<html> |
| 4 |
|
| 5 |
<head> |
| 6 |
|
| 7 |
<meta charset="utf-8" /> |
| 8 |
<meta name="generator" content="pandoc" /> |
| 9 |
<meta http-equiv="X-UA-Compatible" content="IE=EDGE" /> |
| 10 |
|
| 11 |
<meta name="viewport" content="width=device-width, initial-scale=1" /> |
| 12 |
|
| 13 |
<meta name="author" content="James E. Pustejovsky" /> |
| 14 |
|
| 15 |
<meta name="date" content="2026-05-01" /> |
| 16 |
|
| 17 |
<title>Meta-analysis with cluster-robust variance estimation</title> |
| 18 |
|
| 19 |
<script>// Pandoc 2.9 adds attributes on both header and div. We remove the former (to |
| 20 |
// be compatible with the behavior of Pandoc < 2.8). |
| 21 |
document.addEventListener('DOMContentLoaded', function(e) { |
| 22 |
var hs = document.querySelectorAll("div.section[class*='level'] > :first-child"); |
| 23 |
var i, h, a; |
| 24 |
for (i = 0; i < hs.length; i++) { |
| 25 |
h = hs[i]; |
| 26 |
if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6 |
| 27 |
a = h.attributes; |
| 28 |
while (a.length > 0) h.removeAttribute(a[0].name); |
| 29 |
} |
| 30 |
}); |
| 31 |
</script> |
| 32 |
|
| 33 |
<style type="text/css"> |
| 34 |
code{white-space: pre-wrap;} |
| 35 |
span.smallcaps{font-variant: small-caps;} |
| 36 |
span.underline{text-decoration: underline;} |
| 37 |
div.column{display: inline-block; vertical-align: top; width: 50%;} |
| 38 |
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} |
| 39 |
ul.task-list{list-style: none;} |
| 40 |
</style> |
| 41 |
|
| 42 |
|
| 43 |
|
| 44 |
<style type="text/css"> |
| 45 |
code { |
| 46 |
white-space: pre; |
| 47 |
} |
| 48 |
.sourceCode { |
| 49 |
overflow: visible; |
| 50 |
} |
| 51 |
</style> |
| 52 |
<style type="text/css" data-origin="pandoc"> |
| 53 |
html { -webkit-text-size-adjust: 100%; } |
| 54 |
pre > code.sourceCode { white-space: pre; position: relative; } |
| 55 |
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; } |
| 56 |
pre > code.sourceCode > span:empty { height: 1.2em; } |
| 57 |
.sourceCode { overflow: visible; } |
| 58 |
code.sourceCode > span { color: inherit; text-decoration: inherit; } |
| 59 |
div.sourceCode { margin: 1em 0; } |
| 60 |
pre.sourceCode { margin: 0; } |
| 61 |
@media screen { |
| 62 |
div.sourceCode { overflow: auto; } |
| 63 |
} |
| 64 |
@media print { |
| 65 |
pre > code.sourceCode { white-space: pre-wrap; } |
| 66 |
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; } |
| 67 |
} |
| 68 |
pre.numberSource code |
| 69 |
{ counter-reset: source-line 0; } |
| 70 |
pre.numberSource code > span |
| 71 |
{ position: relative; left: -4em; counter-increment: source-line; } |
| 72 |
pre.numberSource code > span > a:first-child::before |
| 73 |
{ content: counter(source-line); |
| 74 |
position: relative; left: -1em; text-align: right; vertical-align: baseline; |
| 75 |
border: none; display: inline-block; |
| 76 |
-webkit-touch-callout: none; -webkit-user-select: none; |
| 77 |
-khtml-user-select: none; -moz-user-select: none; |
| 78 |
-ms-user-select: none; user-select: none; |
| 79 |
padding: 0 4px; width: 4em; |
| 80 |
color: #aaaaaa; |
| 81 |
} |
| 82 |
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; } |
| 83 |
div.sourceCode |
| 84 |
{ } |
| 85 |
@media screen { |
| 86 |
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; } |
| 87 |
} |
| 88 |
code span.al { color: #ff0000; font-weight: bold; } |
| 89 |
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } |
| 90 |
code span.at { color: #7d9029; } |
| 91 |
code span.bn { color: #40a070; } |
| 92 |
code span.bu { color: #008000; } |
| 93 |
code span.cf { color: #007020; font-weight: bold; } |
| 94 |
code span.ch { color: #4070a0; } |
| 95 |
code span.cn { color: #880000; } |
| 96 |
code span.co { color: #60a0b0; font-style: italic; } |
| 97 |
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } |
| 98 |
code span.do { color: #ba2121; font-style: italic; } |
| 99 |
code span.dt { color: #902000; } |
| 100 |
code span.dv { color: #40a070; } |
| 101 |
code span.er { color: #ff0000; font-weight: bold; } |
| 102 |
code span.ex { } |
| 103 |
code span.fl { color: #40a070; } |
| 104 |
code span.fu { color: #06287e; } |
| 105 |
code span.im { color: #008000; font-weight: bold; } |
| 106 |
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } |
| 107 |
code span.kw { color: #007020; font-weight: bold; } |
| 108 |
code span.op { color: #666666; } |
| 109 |
code span.ot { color: #007020; } |
| 110 |
code span.pp { color: #bc7a00; } |
| 111 |
code span.sc { color: #4070a0; } |
| 112 |
code span.ss { color: #bb6688; } |
| 113 |
code span.st { color: #4070a0; } |
| 114 |
code span.va { color: #19177c; } |
| 115 |
code span.vs { color: #4070a0; } |
| 116 |
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } |
| 117 |
</style> |
| 118 |
<script> |
| 119 |
// apply pandoc div.sourceCode style to pre.sourceCode instead |
| 120 |
(function() { |
| 121 |
var sheets = document.styleSheets; |
| 122 |
for (var i = 0; i < sheets.length; i++) { |
| 123 |
if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue; |
| 124 |
try { var rules = sheets[i].cssRules; } catch (e) { continue; } |
| 125 |
var j = 0; |
| 126 |
while (j < rules.length) { |
| 127 |
var rule = rules[j]; |
| 128 |
// check if there is a div.sourceCode rule |
| 129 |
if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") { |
| 130 |
j++; |
| 131 |
continue; |
| 132 |
} |
| 133 |
var style = rule.style.cssText; |
| 134 |
// check if color or background-color is set |
| 135 |
if (rule.style.color === '' && rule.style.backgroundColor === '') { |
| 136 |
j++; |
| 137 |
continue; |
| 138 |
} |
| 139 |
// replace div.sourceCode by a pre.sourceCode rule |
| 140 |
sheets[i].deleteRule(j); |
| 141 |
sheets[i].insertRule('pre.sourceCode{' + style + '}', j); |
| 142 |
} |
| 143 |
} |
| 144 |
})(); |
| 145 |
</script> |
| 146 |
|
| 147 |
|
| 148 |
|
| 149 |
|
| 150 |
<style type="text/css">body { |
| 151 |
background-color: #fff; |
| 152 |
margin: 1em auto; |
| 153 |
max-width: 700px; |
| 154 |
overflow: visible; |
| 155 |
padding-left: 2em; |
| 156 |
padding-right: 2em; |
| 157 |
font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; |
| 158 |
font-size: 14px; |
| 159 |
line-height: 1.35; |
| 160 |
} |
| 161 |
#TOC { |
| 162 |
clear: both; |
| 163 |
margin: 0 0 10px 10px; |
| 164 |
padding: 4px; |
| 165 |
width: 400px; |
| 166 |
border: 1px solid #CCCCCC; |
| 167 |
border-radius: 5px; |
| 168 |
background-color: #f6f6f6; |
| 169 |
font-size: 13px; |
| 170 |
line-height: 1.3; |
| 171 |
} |
| 172 |
#TOC .toctitle { |
| 173 |
font-weight: bold; |
| 174 |
font-size: 15px; |
| 175 |
margin-left: 5px; |
| 176 |
} |
| 177 |
#TOC ul { |
| 178 |
padding-left: 40px; |
| 179 |
margin-left: -1.5em; |
| 180 |
margin-top: 5px; |
| 181 |
margin-bottom: 5px; |
| 182 |
} |
| 183 |
#TOC ul ul { |
| 184 |
margin-left: -2em; |
| 185 |
} |
| 186 |
#TOC li { |
| 187 |
line-height: 16px; |
| 188 |
} |
| 189 |
table { |
| 190 |
margin: 1em auto; |
| 191 |
border-width: 1px; |
| 192 |
border-color: #DDDDDD; |
| 193 |
border-style: outset; |
| 194 |
border-collapse: collapse; |
| 195 |
} |
| 196 |
table th { |
| 197 |
border-width: 2px; |
| 198 |
padding: 5px; |
| 199 |
border-style: inset; |
| 200 |
} |
| 201 |
table td { |
| 202 |
border-width: 1px; |
| 203 |
border-style: inset; |
| 204 |
line-height: 18px; |
| 205 |
padding: 5px 5px; |
| 206 |
} |
| 207 |
table, table th, table td { |
| 208 |
border-left-style: none; |
| 209 |
border-right-style: none; |
| 210 |
} |
| 211 |
table thead, table tr.even { |
| 212 |
background-color: #f7f7f7; |
| 213 |
} |
| 214 |
p { |
| 215 |
margin: 0.5em 0; |
| 216 |
} |
| 217 |
blockquote { |
| 218 |
background-color: #f6f6f6; |
| 219 |
padding: 0.25em 0.75em; |
| 220 |
} |
| 221 |
hr { |
| 222 |
border-style: solid; |
| 223 |
border: none; |
| 224 |
border-top: 1px solid #777; |
| 225 |
margin: 28px 0; |
| 226 |
} |
| 227 |
dl { |
| 228 |
margin-left: 0; |
| 229 |
} |
| 230 |
dl dd { |
| 231 |
margin-bottom: 13px; |
| 232 |
margin-left: 13px; |
| 233 |
} |
| 234 |
dl dt { |
| 235 |
font-weight: bold; |
| 236 |
} |
| 237 |
ul { |
| 238 |
margin-top: 0; |
| 239 |
} |
| 240 |
ul li { |
| 241 |
list-style: circle outside; |
| 242 |
} |
| 243 |
ul ul { |
| 244 |
margin-bottom: 0; |
| 245 |
} |
| 246 |
pre, code { |
| 247 |
background-color: #f7f7f7; |
| 248 |
border-radius: 3px; |
| 249 |
color: #333; |
| 250 |
white-space: pre-wrap; |
| 251 |
} |
| 252 |
pre { |
| 253 |
border-radius: 3px; |
| 254 |
margin: 5px 0px 10px 0px; |
| 255 |
padding: 10px; |
| 256 |
} |
| 257 |
pre:not([class]) { |
| 258 |
background-color: #f7f7f7; |
| 259 |
} |
| 260 |
code { |
| 261 |
font-family: Consolas, Monaco, 'Courier New', monospace; |
| 262 |
font-size: 85%; |
| 263 |
} |
| 264 |
p > code, li > code { |
| 265 |
padding: 2px 0px; |
| 266 |
} |
| 267 |
div.figure { |
| 268 |
text-align: center; |
| 269 |
} |
| 270 |
img { |
| 271 |
background-color: #FFFFFF; |
| 272 |
padding: 2px; |
| 273 |
border: 1px solid #DDDDDD; |
| 274 |
border-radius: 3px; |
| 275 |
border: 1px solid #CCCCCC; |
| 276 |
margin: 0 5px; |
| 277 |
} |
| 278 |
h1 { |
| 279 |
margin-top: 0; |
| 280 |
font-size: 35px; |
| 281 |
line-height: 40px; |
| 282 |
} |
| 283 |
h2 { |
| 284 |
border-bottom: 4px solid #f7f7f7; |
| 285 |
padding-top: 10px; |
| 286 |
padding-bottom: 2px; |
| 287 |
font-size: 145%; |
| 288 |
} |
| 289 |
h3 { |
| 290 |
border-bottom: 2px solid #f7f7f7; |
| 291 |
padding-top: 10px; |
| 292 |
font-size: 120%; |
| 293 |
} |
| 294 |
h4 { |
| 295 |
border-bottom: 1px solid #f7f7f7; |
| 296 |
margin-left: 8px; |
| 297 |
font-size: 105%; |
| 298 |
} |
| 299 |
h5, h6 { |
| 300 |
border-bottom: 1px solid #ccc; |
| 301 |
font-size: 105%; |
| 302 |
} |
| 303 |
a { |
| 304 |
color: #0033dd; |
| 305 |
text-decoration: none; |
| 306 |
} |
| 307 |
a:hover { |
| 308 |
color: #6666ff; } |
| 309 |
a:visited { |
| 310 |
color: #800080; } |
| 311 |
a:visited:hover { |
| 312 |
color: #BB00BB; } |
| 313 |
a[href^="http:"] { |
| 314 |
text-decoration: underline; } |
| 315 |
a[href^="https:"] { |
| 316 |
text-decoration: underline; } |
| 317 |
|
| 318 |
code > span.kw { color: #555; font-weight: bold; } |
| 319 |
code > span.dt { color: #902000; } |
| 320 |
code > span.dv { color: #40a070; } |
| 321 |
code > span.bn { color: #d14; } |
| 322 |
code > span.fl { color: #d14; } |
| 323 |
code > span.ch { color: #d14; } |
| 324 |
code > span.st { color: #d14; } |
| 325 |
code > span.co { color: #888888; font-style: italic; } |
| 326 |
code > span.ot { color: #007020; } |
| 327 |
code > span.al { color: #ff0000; font-weight: bold; } |
| 328 |
code > span.fu { color: #900; font-weight: bold; } |
| 329 |
code > span.er { color: #a61717; background-color: #e3d2d2; } |
| 330 |
</style> |
| 331 |
|
| 332 |
|
| 333 |
|
| 334 |
|
| 335 |
</head> |
| 336 |
|
| 337 |
<body> |
| 338 |
|
| 339 |
|
| 340 |
|
| 341 |
|
| 342 |
<h1 class="title toc-ignore">Meta-analysis with cluster-robust variance |
| 343 |
estimation</h1> |
| 344 |
<h4 class="author">James E. Pustejovsky</h4> |
| 345 |
<h4 class="date">2026-05-01</h4> |
| 346 |
|
| 347 |
|
| 348 |
|
| 349 |
<p>This vignette demonstrates how to use the <code>clubSandwich</code> |
| 350 |
package to conduct a meta-analysis of dependent effect sizes with robust |
| 351 |
variance estimation. Tests of meta-regression coefficients and F-tests |
| 352 |
of multiple-coefficient hypotheses are calculated using small-sample |
| 353 |
corrections proposed by Tipton (2015) and Tipton and Pustejovsky (2015). |
| 354 |
The example uses a dataset of effect sizes from a Campbell Collaboration |
| 355 |
systematic review of dropout prevention programs, conducted by Sandra Jo |
| 356 |
Wilson and colleagues (2011).</p> |
| 357 |
<p>The original analysis included a meta-regression with covariates that |
| 358 |
capture methodological, participant, and program characteristics. The |
| 359 |
regression specification used here is similar to Model III from Wilson |
| 360 |
et al. (2011), but treats the <code>evaluator_independence</code> and |
| 361 |
<code>implementation_quality</code> variables as categorical rather than |
| 362 |
interval-level. Also, the original analysis clustered at the level of |
| 363 |
the sample (some studies reported results from multiple samples), |
| 364 |
whereas here we cluster at the study level. The meta-regression can be |
| 365 |
fit in several different ways. We first demonstrate using the |
| 366 |
<code>robumeta</code> package (Fisher & Tipton, 2015) and then using |
| 367 |
the <code>metafor</code> package (Viechtbauer, 2010).</p> |
| 368 |
<div id="robumeta-model" class="section level2"> |
| 369 |
<h2>robumeta model</h2> |
| 370 |
<div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb1-1"><a href="#cb1-1" tabindex="-1"></a><span class="fu">library</span>(clubSandwich)</span> |
| 371 |
<span id="cb1-2"><a href="#cb1-2" tabindex="-1"></a><span class="fu">library</span>(robumeta)</span> |
| 372 |
<span id="cb1-3"><a href="#cb1-3" tabindex="-1"></a><span class="fu">data</span>(dropoutPrevention)</span> |
| 373 |
<span id="cb1-4"><a href="#cb1-4" tabindex="-1"></a></span> |
| 374 |
<span id="cb1-5"><a href="#cb1-5" tabindex="-1"></a><span class="co"># clean formatting</span></span> |
| 375 |
<span id="cb1-6"><a href="#cb1-6" tabindex="-1"></a><span class="fu">names</span>(dropoutPrevention)[<span class="dv">7</span><span class="sc">:</span><span class="dv">8</span>] <span class="ot"><-</span> <span class="fu">c</span>(<span class="st">"eval"</span>,<span class="st">"implement"</span>)</span> |
| 376 |
<span id="cb1-7"><a href="#cb1-7" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>eval) <span class="ot"><-</span> <span class="fu">c</span>(<span class="st">"independent"</span>,<span class="st">"indirect"</span>,<span class="st">"planning"</span>,<span class="st">"delivery"</span>)</span> |
| 377 |
<span id="cb1-8"><a href="#cb1-8" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>implement) <span class="ot"><-</span> <span class="fu">c</span>(<span class="st">"low"</span>,<span class="st">"medium"</span>,<span class="st">"high"</span>)</span> |
| 378 |
<span id="cb1-9"><a href="#cb1-9" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>program_site) <span class="ot"><-</span> <span class="fu">c</span>(<span class="st">"community"</span>,<span class="st">"mixed"</span>,<span class="st">"classroom"</span>,<span class="st">"school"</span>)</span> |
| 379 |
<span id="cb1-10"><a href="#cb1-10" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>study_design) <span class="ot"><-</span> <span class="fu">c</span>(<span class="st">"matched"</span>,<span class="st">"unmatched"</span>,<span class="st">"RCT"</span>)</span> |
| 380 |
<span id="cb1-11"><a href="#cb1-11" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>adjusted) <span class="ot"><-</span> <span class="fu">c</span>(<span class="st">"no"</span>,<span class="st">"yes"</span>)</span> |
| 381 |
<span id="cb1-12"><a href="#cb1-12" tabindex="-1"></a></span> |
| 382 |
<span id="cb1-13"><a href="#cb1-13" tabindex="-1"></a>m3_robu <span class="ot"><-</span> <span class="fu">robu</span>(LOR1 <span class="sc">~</span> study_design <span class="sc">+</span> attrition <span class="sc">+</span> group_equivalence <span class="sc">+</span> adjusted</span> |
| 383 |
<span id="cb1-14"><a href="#cb1-14" tabindex="-1"></a> <span class="sc">+</span> outcome <span class="sc">+</span> eval <span class="sc">+</span> male_pct <span class="sc">+</span> white_pct <span class="sc">+</span> average_age</span> |
| 384 |
<span id="cb1-15"><a href="#cb1-15" tabindex="-1"></a> <span class="sc">+</span> implement <span class="sc">+</span> program_site <span class="sc">+</span> duration <span class="sc">+</span> service_hrs, </span> |
| 385 |
<span id="cb1-16"><a href="#cb1-16" tabindex="-1"></a> <span class="at">data =</span> dropoutPrevention, <span class="at">studynum =</span> studyID, <span class="at">var.eff.size =</span> varLOR, </span> |
| 386 |
<span id="cb1-17"><a href="#cb1-17" tabindex="-1"></a> <span class="at">modelweights =</span> <span class="st">"HIER"</span>)</span> |
| 387 |
<span id="cb1-18"><a href="#cb1-18" tabindex="-1"></a><span class="fu">print</span>(m3_robu)</span></code></pre></div> |
| 388 |
<pre><code>## RVE: Hierarchical Effects Model with Small-Sample Corrections |
| 389 |
## |
| 390 |
## Model: LOR1 ~ study_design + attrition + group_equivalence + adjusted + outcome + eval + male_pct + white_pct + average_age + implement + program_site + duration + service_hrs |
| 391 |
## |
| 392 |
## Number of clusters = 152 |
| 393 |
## Number of outcomes = 385 (min = 1 , mean = 2.53 , median = 1 , max = 30 ) |
| 394 |
## Omega.sq = 0.24907 |
| 395 |
## Tau.sq = 0.1024663 |
| 396 |
## |
| 397 |
## Estimate StdErr t-value dfs P(|t|>) 95% CI.L 95% CI.U Sig |
| 398 |
## 1 X.Intercept. 0.016899 0.615399 0.0275 16.9 0.97841541 -1.28228 1.31608 |
| 399 |
## 2 study_designunmatched -0.002626 0.185142 -0.0142 40.5 0.98875129 -0.37667 0.37141 |
| 400 |
## 3 study_designRCT -0.086872 0.140044 -0.6203 38.6 0.53869676 -0.37024 0.19650 |
| 401 |
## 4 attrition 0.118889 0.247228 0.4809 15.5 0.63732597 -0.40666 0.64444 |
| 402 |
## 5 group_equivalence 0.502463 0.195838 2.5657 28.7 0.01579282 0.10174 0.90318 ** |
| 403 |
## 6 adjustedyes -0.322480 0.125413 -2.5713 33.8 0.01470796 -0.57741 -0.06755 ** |
| 404 |
## 7 outcomeenrolled 0.097059 0.139842 0.6941 16.5 0.49727848 -0.19862 0.39274 |
| 405 |
## 8 outcomegraduation 0.147643 0.134938 1.0942 30.2 0.28253825 -0.12786 0.42315 |
| 406 |
## 9 outcomegraduation.ged 0.258034 0.169134 1.5256 16.3 0.14632629 -0.10006 0.61613 |
| 407 |
## 10 evalindirect -0.765085 0.399109 -1.9170 6.2 0.10212896 -1.73406 0.20389 |
| 408 |
## 11 evalplanning -0.920874 0.346536 -2.6574 5.6 0.04027061 -1.78381 -0.05794 ** |
| 409 |
## 12 evaldelivery -0.916673 0.304303 -3.0124 4.7 0.03212299 -1.71432 -0.11903 ** |
| 410 |
## 13 male_pct 0.167965 0.181538 0.9252 16.4 0.36824526 -0.21609 0.55202 |
| 411 |
## 14 white_pct 0.022915 0.149394 0.1534 21.8 0.87950385 -0.28704 0.33287 |
| 412 |
## 15 average_age 0.037102 0.027053 1.3715 21.2 0.18458247 -0.01913 0.09333 |
| 413 |
## 16 implementmedium 0.411779 0.128898 3.1946 26.7 0.00358205 0.14714 0.67642 *** |
| 414 |
## 17 implementhigh 0.658570 0.123874 5.3164 34.6 0.00000635 0.40699 0.91015 *** |
| 415 |
## 18 program_sitemixed 0.444384 0.172635 2.5741 28.6 0.01550504 0.09109 0.79768 ** |
| 416 |
## 19 program_siteclassroom 0.426658 0.159773 2.6704 37.4 0.01115192 0.10303 0.75028 ** |
| 417 |
## 20 program_siteschool 0.262517 0.160519 1.6354 30.1 0.11236814 -0.06525 0.59028 |
| 418 |
## 21 duration 0.000427 0.000873 0.4895 36.7 0.62736846 -0.00134 0.00220 |
| 419 |
## 22 service_hrs -0.003434 0.005012 -0.6852 36.7 0.49752503 -0.01359 0.00672 |
| 420 |
## --- |
| 421 |
## Signif. codes: < .01 *** < .05 ** < .10 * |
| 422 |
## --- |
| 423 |
## Note: If df < 4, do not trust the results</code></pre> |
| 424 |
<p>Note that <code>robumeta</code> produces small-sample corrected |
| 425 |
standard errors and t-tests, and so there is no need to repeat those |
| 426 |
calculations with <code>clubSandwich</code>. The <code>eval</code> |
| 427 |
variable has four levels, and it might be of interest to test whether |
| 428 |
the average program effects differ by the degree of evaluator |
| 429 |
independence. The null hypothesis in this case is that the 10th, 11th, |
| 430 |
and 12th regression coefficients are all equal to zero. A small-sample |
| 431 |
adjusted F-test for this hypothesis can be obtained as follows. The |
| 432 |
<code>vcov = "CR2"</code> option means that the standard errors will be |
| 433 |
corrected using the bias-reduced linearization estimator described in |
| 434 |
Tipton and Pustejovsky (2015).</p> |
| 435 |
<div class="sourceCode" id="cb3"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb3-1"><a href="#cb3-1" tabindex="-1"></a><span class="fu">Wald_test</span>(m3_robu, <span class="at">constraints =</span> <span class="fu">constrain_zero</span>(<span class="dv">10</span><span class="sc">:</span><span class="dv">12</span>), <span class="at">vcov =</span> <span class="st">"CR2"</span>)</span></code></pre></div> |
| 436 |
<pre><code>## test Fstat df_num df_denom p_val sig |
| 437 |
## HTZ 2.78 3 16.8 0.0732 .</code></pre> |
| 438 |
<p>By default, the <code>Wald_test</code> function provides an F-type |
| 439 |
test with degrees of freedom estimated using the approximate Hotelling’s |
| 440 |
<span class="math inline">\(T^2_Z\)</span> method. The test has less |
| 441 |
than 17 degrees of freedom, even though there are 152 independent |
| 442 |
studies in the data, and has a p-value that is not quite significant at |
| 443 |
conventional levels. The low degrees of freedom are a consequence of the |
| 444 |
fact that one of the levels of <code>evaluator independence</code> has |
| 445 |
only a few effect sizes in it:</p> |
| 446 |
<div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb5-1"><a href="#cb5-1" tabindex="-1"></a><span class="fu">table</span>(dropoutPrevention<span class="sc">$</span>eval)</span></code></pre></div> |
| 447 |
<pre><code>## |
| 448 |
## independent indirect planning delivery |
| 449 |
## 6 33 43 303</code></pre> |
| 450 |
</div> |
| 451 |
<div id="metafor-model" class="section level2"> |
| 452 |
<h2>metafor model</h2> |
| 453 |
<p><code>clubSandwich</code> also works with models fit using the |
| 454 |
<code>metafor</code> package. Here we re-fit the same regression |
| 455 |
specification, but use REML to estimate the variance components |
| 456 |
(<code>robumeta</code> uses a method-of-moments estimator), as well as a |
| 457 |
somewhat different weighting scheme than that used in |
| 458 |
<code>robumeta</code>.</p> |
| 459 |
<div class="sourceCode" id="cb7"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb7-1"><a href="#cb7-1" tabindex="-1"></a><span class="fu">library</span>(metafor)</span> |
| 460 |
<span id="cb7-2"><a href="#cb7-2" tabindex="-1"></a>m3_metafor <span class="ot"><-</span> <span class="fu">rma.mv</span>(LOR1 <span class="sc">~</span> study_design <span class="sc">+</span> attrition <span class="sc">+</span> group_equivalence <span class="sc">+</span> adjusted</span> |
| 461 |
<span id="cb7-3"><a href="#cb7-3" tabindex="-1"></a> <span class="sc">+</span> outcome <span class="sc">+</span> eval</span> |
| 462 |
<span id="cb7-4"><a href="#cb7-4" tabindex="-1"></a> <span class="sc">+</span> male_pct <span class="sc">+</span> white_pct <span class="sc">+</span> average_age</span> |
| 463 |
<span id="cb7-5"><a href="#cb7-5" tabindex="-1"></a> <span class="sc">+</span> implement <span class="sc">+</span> program_site <span class="sc">+</span> duration <span class="sc">+</span> service_hrs, </span> |
| 464 |
<span id="cb7-6"><a href="#cb7-6" tabindex="-1"></a> <span class="at">V =</span> varLOR, <span class="at">random =</span> <span class="fu">list</span>(<span class="sc">~</span> <span class="dv">1</span> <span class="sc">|</span> studyID, <span class="sc">~</span> <span class="dv">1</span> <span class="sc">|</span> studySample),</span> |
| 465 |
<span id="cb7-7"><a href="#cb7-7" tabindex="-1"></a> <span class="at">data =</span> dropoutPrevention)</span> |
| 466 |
<span id="cb7-8"><a href="#cb7-8" tabindex="-1"></a><span class="fu">summary</span>(m3_metafor)</span></code></pre></div> |
| 467 |
<pre><code>## |
| 468 |
## Multivariate Meta-Analysis Model (k = 385; method: REML) |
| 469 |
## |
| 470 |
## logLik Deviance AIC BIC AICc |
| 471 |
## -489.0357 978.0714 1026.0714 1119.5371 1029.6217 |
| 472 |
## |
| 473 |
## Variance Components: |
| 474 |
## |
| 475 |
## estim sqrt nlvls fixed factor |
| 476 |
## sigma^2.1 0.2274 0.4769 152 no studyID |
| 477 |
## sigma^2.2 0.1145 0.3384 317 no studySample |
| 478 |
## |
| 479 |
## Test for Residual Heterogeneity: |
| 480 |
## QE(df = 363) = 1588.4397, p-val < .0001 |
| 481 |
## |
| 482 |
## Test of Moderators (coefficients 2:22): |
| 483 |
## QM(df = 21) = 293.8694, p-val < .0001 |
| 484 |
## |
| 485 |
## Model Results: |
| 486 |
## |
| 487 |
## estimate se zval pval ci.lb ci.ub |
| 488 |
## intrcpt 0.5296 0.7250 0.7304 0.4651 -0.8915 1.9506 |
| 489 |
## study_designunmatched -0.0494 0.1722 -0.2871 0.7741 -0.3870 0.2881 |
| 490 |
## study_designRCT 0.0653 0.1628 0.4010 0.6884 -0.2538 0.3843 |
| 491 |
## attrition -0.1366 0.2429 -0.5623 0.5739 -0.6126 0.3395 |
| 492 |
## group_equivalence 0.4071 0.1573 2.5877 0.0097 0.0988 0.7155 ** |
| 493 |
## adjustedyes -0.3581 0.1532 -2.3371 0.0194 -0.6585 -0.0578 * |
| 494 |
## outcomeenrolled -0.2831 0.0771 -3.6709 0.0002 -0.4343 -0.1320 *** |
| 495 |
## outcomegraduation -0.0913 0.0657 -1.3896 0.1646 -0.2201 0.0375 |
| 496 |
## outcomegraduation/ged 0.6983 0.0805 8.6750 <.0001 0.5406 0.8561 *** |
| 497 |
## evalindirect -0.7530 0.4949 -1.5214 0.1282 -1.7230 0.2171 |
| 498 |
## evalplanning -0.7700 0.4869 -1.5814 0.1138 -1.7242 0.1843 |
| 499 |
## evaldelivery -1.0016 0.4600 -2.1774 0.0294 -1.9033 -0.1000 * |
| 500 |
## male_pct 0.1021 0.1715 0.5951 0.5518 -0.2341 0.4382 |
| 501 |
## white_pct 0.1223 0.1804 0.6777 0.4979 -0.2313 0.4758 |
| 502 |
## average_age 0.0061 0.0291 0.2091 0.8344 -0.0509 0.0631 |
| 503 |
## implementmedium 0.4738 0.1609 2.9445 0.0032 0.1584 0.7892 ** |
| 504 |
## implementhigh 0.6318 0.1471 4.2965 <.0001 0.3436 0.9201 *** |
| 505 |
## program_sitemixed 0.3289 0.2413 1.3631 0.1729 -0.1440 0.8019 |
| 506 |
## program_siteclassroom 0.2920 0.1736 1.6821 0.0926 -0.0482 0.6321 . |
| 507 |
## program_siteschool 0.1616 0.1898 0.8515 0.3945 -0.2104 0.5337 |
| 508 |
## duration 0.0013 0.0009 1.3423 0.1795 -0.0006 0.0031 |
| 509 |
## service_hrs -0.0003 0.0047 -0.0654 0.9478 -0.0096 0.0090 |
| 510 |
## |
| 511 |
## --- |
| 512 |
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1</code></pre> |
| 513 |
<p><code>metafor</code> produces model-based standard errors, t-tests, |
| 514 |
and confidence intervals. The <code>coef_test</code> function from |
| 515 |
<code>clubSandwich</code> will calculate robust standard errors and |
| 516 |
robust t-tests for each of the coefficients:</p> |
| 517 |
<div class="sourceCode" id="cb9"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb9-1"><a href="#cb9-1" tabindex="-1"></a><span class="fu">coef_test</span>(m3_metafor, <span class="at">vcov =</span> <span class="st">"CR2"</span>)</span></code></pre></div> |
| 518 |
<pre><code>## Alternative hypothesis: two-sided |
| 519 |
## Coef. Estimate SE Null value t-stat d.f. (Satt) p-val (Satt) Sig. |
| 520 |
## intrcpt 0.529569 0.724851 0 0.7306 20.08 0.47347 |
| 521 |
## study_designunmatched -0.049434 0.204152 0 -0.2421 58.42 0.80952 |
| 522 |
## study_designRCT 0.065272 0.149146 0 0.4376 53.17 0.66342 |
| 523 |
## attrition -0.136575 0.306429 0 -0.4457 10.52 0.66485 |
| 524 |
## group_equivalence 0.407108 0.210917 0 1.9302 23.10 0.06595 . |
| 525 |
## adjustedyes -0.358124 0.136132 0 -2.6307 43.20 0.01176 * |
| 526 |
## outcomeenrolled -0.283124 0.237199 0 -1.1936 7.08 0.27108 |
| 527 |
## outcomegraduation -0.091295 0.091465 0 -0.9981 9.95 0.34188 |
| 528 |
## outcomegraduation/ged 0.698328 0.364882 0 1.9138 8.02 0.09188 . |
| 529 |
## evalindirect -0.752994 0.447670 0 -1.6820 6.56 0.13929 |
| 530 |
## evalplanning -0.769968 0.403898 0 -1.9063 6.10 0.10446 |
| 531 |
## evaldelivery -1.001648 0.355989 0 -2.8137 4.89 0.03834 * |
| 532 |
## male_pct 0.102055 0.148410 0 0.6877 9.68 0.50782 |
| 533 |
## white_pct 0.122255 0.141470 0 0.8642 16.88 0.39961 |
| 534 |
## average_age 0.006084 0.033387 0 0.1822 15.79 0.85772 |
| 535 |
## implementmedium 0.473789 0.148660 0 3.1871 22.44 0.00419 ** |
| 536 |
## implementhigh 0.631842 0.138073 0 4.5761 28.68 < 0.001 *** |
| 537 |
## program_sitemixed 0.328941 0.196848 0 1.6710 27.47 0.10607 |
| 538 |
## program_siteclassroom 0.291952 0.146014 0 1.9995 42.70 0.05195 . |
| 539 |
## program_siteschool 0.161640 0.171700 0 0.9414 29.27 0.35420 |
| 540 |
## duration 0.001270 0.000978 0 1.2988 31.96 0.20332 |
| 541 |
## service_hrs -0.000309 0.004828 0 -0.0641 49.63 0.94915</code></pre> |
| 542 |
<p>Note that <code>coef_test</code> assumed that it should cluster based |
| 543 |
on <code>studyID</code>, which is the outer-most random effect in the |
| 544 |
metafor model. This can be specified explicitly by including the option |
| 545 |
<code>cluster = dropoutPrevention$studyID</code> in the call.</p> |
| 546 |
<p>The F-test for degree of evaluator independence uses the same syntax |
| 547 |
as before:</p> |
| 548 |
<div class="sourceCode" id="cb11"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb11-1"><a href="#cb11-1" tabindex="-1"></a><span class="fu">Wald_test</span>(m3_metafor, <span class="at">constraints =</span> <span class="fu">constrain_zero</span>(<span class="dv">10</span><span class="sc">:</span><span class="dv">12</span>), <span class="at">vcov =</span> <span class="st">"CR2"</span>)</span></code></pre></div> |
| 549 |
<pre><code>## test Fstat df_num df_denom p_val sig |
| 550 |
## HTZ 2.71 3 18.3 0.0753 .</code></pre> |
| 551 |
<p>Despite some differences in weighting schemes, the p-value is very |
| 552 |
close to the result obtained using <code>robumeta</code>.</p> |
| 553 |
</div> |
| 554 |
<div id="references" class="section level2"> |
| 555 |
<h2>References</h2> |
| 556 |
<p>Fisher, Z., & Tipton, E. (2015). robumeta: An R-package for |
| 557 |
robust variance estimation in meta-analysis. <a href="https://arxiv.org/abs/1503.02220">arXiv:1503.02220</a></p> |
| 558 |
<p>Tipton, E. (2015). Small sample adjustments for robust variance |
| 559 |
estimation with meta-regression. <em>Psychological Methods, 20</em>(3), |
| 560 |
375-393. <a href="https://doi.org/10.1037/met0000011" class="uri">https://doi.org/10.1037/met0000011</a></p> |
| 561 |
<p>Tipton, E., & Pustejovsky, J. E. (2015). Small-sample adjustments |
| 562 |
for tests of moderators and model fit using robust variance estimation |
| 563 |
in meta-regression. <em>Journal of Educational and Behavioral |
| 564 |
Statistics, 40</em>(6), 604-634. <a href="https://doi.org/10.3102/1076998615606099" class="uri">https://doi.org/10.3102/1076998615606099</a></p> |
| 565 |
<p>Viechtbauer, W. (2010). Conducting meta-analyses in R with the |
| 566 |
metafor package. <em>Journal of Statistical Software, 36</em>(3), 1-48. |
| 567 |
URL: <a href="https://doi.org/10.18637/jss.v036.i03" class="uri">https://doi.org/10.18637/jss.v036.i03</a></p> |
| 568 |
<p>Wilson, S. J., Lipsey, M. W., Tanner-Smith, E., Huang, C. H., & |
| 569 |
Steinka-Fry, K. T. (2011). Dropout prevention and intervention programs: |
| 570 |
Effects on school completion and dropout Among school-aged children and |
| 571 |
youth: A systematic review. <em>Campbell Systematic Reviews, 7</em>(1), |
| 572 |
1-61. <a href="https://doi.org/10.4073/csr.2011.8" class="uri">https://doi.org/10.4073/csr.2011.8</a></p> |
| 573 |
</div> |
| 574 |
|
| 575 |
|
| 576 |
|
| 577 |
<!-- code folding --> |
| 578 |
|
| 579 |
|
| 580 |
<!-- dynamically load mathjax for compatibility with self-contained --> |
| 581 |
<script> |
| 582 |
(function () { |
| 583 |
var script = document.createElement("script"); |
| 584 |
script.type = "text/javascript"; |
| 585 |
script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"; |
| 586 |
document.getElementsByTagName("head")[0].appendChild(script); |
| 587 |
})(); |
| 588 |
</script> |
| 589 |
|
| 590 |
</body> |
| 591 |
</html> |