Debian Codegraph @

inst/doc/meta-analysis-with-CRVE.html

raw
1 <!DOCTYPE html>
2
3 <html>
4
5 <head>
6
7 <meta charset="utf-8" />
8 <meta name="generator" content="pandoc" />
9 <meta http-equiv="X-UA-Compatible" content="IE=EDGE" />
10
11 <meta name="viewport" content="width=device-width, initial-scale=1" />
12
13 <meta name="author" content="James E. Pustejovsky" />
14
15 <meta name="date" content="2026-05-01" />
16
17 <title>Meta-analysis with cluster-robust variance estimation</title>
18
19 <script>// Pandoc 2.9 adds attributes on both header and div. We remove the former (to
20 // be compatible with the behavior of Pandoc < 2.8).
21 document.addEventListener('DOMContentLoaded', function(e) {
22 var hs = document.querySelectorAll("div.section[class*='level'] > :first-child");
23 var i, h, a;
24 for (i = 0; i < hs.length; i++) {
25 h = hs[i];
26 if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6
27 a = h.attributes;
28 while (a.length > 0) h.removeAttribute(a[0].name);
29 }
30 });
31 </script>
32
33 <style type="text/css">
34 code{white-space: pre-wrap;}
35 span.smallcaps{font-variant: small-caps;}
36 span.underline{text-decoration: underline;}
37 div.column{display: inline-block; vertical-align: top; width: 50%;}
38 div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
39 ul.task-list{list-style: none;}
40 </style>
41
42
43
44 <style type="text/css">
45 code {
46 white-space: pre;
47 }
48 .sourceCode {
49 overflow: visible;
50 }
51 </style>
52 <style type="text/css" data-origin="pandoc">
53 html { -webkit-text-size-adjust: 100%; }
54 pre > code.sourceCode { white-space: pre; position: relative; }
55 pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
56 pre > code.sourceCode > span:empty { height: 1.2em; }
57 .sourceCode { overflow: visible; }
58 code.sourceCode > span { color: inherit; text-decoration: inherit; }
59 div.sourceCode { margin: 1em 0; }
60 pre.sourceCode { margin: 0; }
61 @media screen {
62 div.sourceCode { overflow: auto; }
63 }
64 @media print {
65 pre > code.sourceCode { white-space: pre-wrap; }
66 pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
67 }
68 pre.numberSource code
69 { counter-reset: source-line 0; }
70 pre.numberSource code > span
71 { position: relative; left: -4em; counter-increment: source-line; }
72 pre.numberSource code > span > a:first-child::before
73 { content: counter(source-line);
74 position: relative; left: -1em; text-align: right; vertical-align: baseline;
75 border: none; display: inline-block;
76 -webkit-touch-callout: none; -webkit-user-select: none;
77 -khtml-user-select: none; -moz-user-select: none;
78 -ms-user-select: none; user-select: none;
79 padding: 0 4px; width: 4em;
80 color: #aaaaaa;
81 }
82 pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
83 div.sourceCode
84 { }
85 @media screen {
86 pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
87 }
88 code span.al { color: #ff0000; font-weight: bold; }
89 code span.an { color: #60a0b0; font-weight: bold; font-style: italic; }
90 code span.at { color: #7d9029; }
91 code span.bn { color: #40a070; }
92 code span.bu { color: #008000; }
93 code span.cf { color: #007020; font-weight: bold; }
94 code span.ch { color: #4070a0; }
95 code span.cn { color: #880000; }
96 code span.co { color: #60a0b0; font-style: italic; }
97 code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; }
98 code span.do { color: #ba2121; font-style: italic; }
99 code span.dt { color: #902000; }
100 code span.dv { color: #40a070; }
101 code span.er { color: #ff0000; font-weight: bold; }
102 code span.ex { }
103 code span.fl { color: #40a070; }
104 code span.fu { color: #06287e; }
105 code span.im { color: #008000; font-weight: bold; }
106 code span.in { color: #60a0b0; font-weight: bold; font-style: italic; }
107 code span.kw { color: #007020; font-weight: bold; }
108 code span.op { color: #666666; }
109 code span.ot { color: #007020; }
110 code span.pp { color: #bc7a00; }
111 code span.sc { color: #4070a0; }
112 code span.ss { color: #bb6688; }
113 code span.st { color: #4070a0; }
114 code span.va { color: #19177c; }
115 code span.vs { color: #4070a0; }
116 code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; }
117 </style>
118 <script>
119 // apply pandoc div.sourceCode style to pre.sourceCode instead
120 (function() {
121 var sheets = document.styleSheets;
122 for (var i = 0; i < sheets.length; i++) {
123 if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue;
124 try { var rules = sheets[i].cssRules; } catch (e) { continue; }
125 var j = 0;
126 while (j < rules.length) {
127 var rule = rules[j];
128 // check if there is a div.sourceCode rule
129 if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") {
130 j++;
131 continue;
132 }
133 var style = rule.style.cssText;
134 // check if color or background-color is set
135 if (rule.style.color === '' && rule.style.backgroundColor === '') {
136 j++;
137 continue;
138 }
139 // replace div.sourceCode by a pre.sourceCode rule
140 sheets[i].deleteRule(j);
141 sheets[i].insertRule('pre.sourceCode{' + style + '}', j);
142 }
143 }
144 })();
145 </script>
146
147
148
149
150 <style type="text/css">body {
151 background-color: #fff;
152 margin: 1em auto;
153 max-width: 700px;
154 overflow: visible;
155 padding-left: 2em;
156 padding-right: 2em;
157 font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
158 font-size: 14px;
159 line-height: 1.35;
160 }
161 #TOC {
162 clear: both;
163 margin: 0 0 10px 10px;
164 padding: 4px;
165 width: 400px;
166 border: 1px solid #CCCCCC;
167 border-radius: 5px;
168 background-color: #f6f6f6;
169 font-size: 13px;
170 line-height: 1.3;
171 }
172 #TOC .toctitle {
173 font-weight: bold;
174 font-size: 15px;
175 margin-left: 5px;
176 }
177 #TOC ul {
178 padding-left: 40px;
179 margin-left: -1.5em;
180 margin-top: 5px;
181 margin-bottom: 5px;
182 }
183 #TOC ul ul {
184 margin-left: -2em;
185 }
186 #TOC li {
187 line-height: 16px;
188 }
189 table {
190 margin: 1em auto;
191 border-width: 1px;
192 border-color: #DDDDDD;
193 border-style: outset;
194 border-collapse: collapse;
195 }
196 table th {
197 border-width: 2px;
198 padding: 5px;
199 border-style: inset;
200 }
201 table td {
202 border-width: 1px;
203 border-style: inset;
204 line-height: 18px;
205 padding: 5px 5px;
206 }
207 table, table th, table td {
208 border-left-style: none;
209 border-right-style: none;
210 }
211 table thead, table tr.even {
212 background-color: #f7f7f7;
213 }
214 p {
215 margin: 0.5em 0;
216 }
217 blockquote {
218 background-color: #f6f6f6;
219 padding: 0.25em 0.75em;
220 }
221 hr {
222 border-style: solid;
223 border: none;
224 border-top: 1px solid #777;
225 margin: 28px 0;
226 }
227 dl {
228 margin-left: 0;
229 }
230 dl dd {
231 margin-bottom: 13px;
232 margin-left: 13px;
233 }
234 dl dt {
235 font-weight: bold;
236 }
237 ul {
238 margin-top: 0;
239 }
240 ul li {
241 list-style: circle outside;
242 }
243 ul ul {
244 margin-bottom: 0;
245 }
246 pre, code {
247 background-color: #f7f7f7;
248 border-radius: 3px;
249 color: #333;
250 white-space: pre-wrap;
251 }
252 pre {
253 border-radius: 3px;
254 margin: 5px 0px 10px 0px;
255 padding: 10px;
256 }
257 pre:not([class]) {
258 background-color: #f7f7f7;
259 }
260 code {
261 font-family: Consolas, Monaco, 'Courier New', monospace;
262 font-size: 85%;
263 }
264 p > code, li > code {
265 padding: 2px 0px;
266 }
267 div.figure {
268 text-align: center;
269 }
270 img {
271 background-color: #FFFFFF;
272 padding: 2px;
273 border: 1px solid #DDDDDD;
274 border-radius: 3px;
275 border: 1px solid #CCCCCC;
276 margin: 0 5px;
277 }
278 h1 {
279 margin-top: 0;
280 font-size: 35px;
281 line-height: 40px;
282 }
283 h2 {
284 border-bottom: 4px solid #f7f7f7;
285 padding-top: 10px;
286 padding-bottom: 2px;
287 font-size: 145%;
288 }
289 h3 {
290 border-bottom: 2px solid #f7f7f7;
291 padding-top: 10px;
292 font-size: 120%;
293 }
294 h4 {
295 border-bottom: 1px solid #f7f7f7;
296 margin-left: 8px;
297 font-size: 105%;
298 }
299 h5, h6 {
300 border-bottom: 1px solid #ccc;
301 font-size: 105%;
302 }
303 a {
304 color: #0033dd;
305 text-decoration: none;
306 }
307 a:hover {
308 color: #6666ff; }
309 a:visited {
310 color: #800080; }
311 a:visited:hover {
312 color: #BB00BB; }
313 a[href^="http:"] {
314 text-decoration: underline; }
315 a[href^="https:"] {
316 text-decoration: underline; }
317
318 code > span.kw { color: #555; font-weight: bold; }
319 code > span.dt { color: #902000; }
320 code > span.dv { color: #40a070; }
321 code > span.bn { color: #d14; }
322 code > span.fl { color: #d14; }
323 code > span.ch { color: #d14; }
324 code > span.st { color: #d14; }
325 code > span.co { color: #888888; font-style: italic; }
326 code > span.ot { color: #007020; }
327 code > span.al { color: #ff0000; font-weight: bold; }
328 code > span.fu { color: #900; font-weight: bold; }
329 code > span.er { color: #a61717; background-color: #e3d2d2; }
330 </style>
331
332
333
334
335 </head>
336
337 <body>
338
339
340
341
342 <h1 class="title toc-ignore">Meta-analysis with cluster-robust variance
343 estimation</h1>
344 <h4 class="author">James E. Pustejovsky</h4>
345 <h4 class="date">2026-05-01</h4>
346
347
348
349 <p>This vignette demonstrates how to use the <code>clubSandwich</code>
350 package to conduct a meta-analysis of dependent effect sizes with robust
351 variance estimation. Tests of meta-regression coefficients and F-tests
352 of multiple-coefficient hypotheses are calculated using small-sample
353 corrections proposed by Tipton (2015) and Tipton and Pustejovsky (2015).
354 The example uses a dataset of effect sizes from a Campbell Collaboration
355 systematic review of dropout prevention programs, conducted by Sandra Jo
356 Wilson and colleagues (2011).</p>
357 <p>The original analysis included a meta-regression with covariates that
358 capture methodological, participant, and program characteristics. The
359 regression specification used here is similar to Model III from Wilson
360 et al. (2011), but treats the <code>evaluator_independence</code> and
361 <code>implementation_quality</code> variables as categorical rather than
362 interval-level. Also, the original analysis clustered at the level of
363 the sample (some studies reported results from multiple samples),
364 whereas here we cluster at the study level. The meta-regression can be
365 fit in several different ways. We first demonstrate using the
366 <code>robumeta</code> package (Fisher &amp; Tipton, 2015) and then using
367 the <code>metafor</code> package (Viechtbauer, 2010).</p>
368 <div id="robumeta-model" class="section level2">
369 <h2>robumeta model</h2>
370 <div class="sourceCode" id="cb1"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb1-1"><a href="#cb1-1" tabindex="-1"></a><span class="fu">library</span>(clubSandwich)</span>
371 <span id="cb1-2"><a href="#cb1-2" tabindex="-1"></a><span class="fu">library</span>(robumeta)</span>
372 <span id="cb1-3"><a href="#cb1-3" tabindex="-1"></a><span class="fu">data</span>(dropoutPrevention)</span>
373 <span id="cb1-4"><a href="#cb1-4" tabindex="-1"></a></span>
374 <span id="cb1-5"><a href="#cb1-5" tabindex="-1"></a><span class="co"># clean formatting</span></span>
375 <span id="cb1-6"><a href="#cb1-6" tabindex="-1"></a><span class="fu">names</span>(dropoutPrevention)[<span class="dv">7</span><span class="sc">:</span><span class="dv">8</span>] <span class="ot">&lt;-</span> <span class="fu">c</span>(<span class="st">&quot;eval&quot;</span>,<span class="st">&quot;implement&quot;</span>)</span>
376 <span id="cb1-7"><a href="#cb1-7" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>eval) <span class="ot">&lt;-</span> <span class="fu">c</span>(<span class="st">&quot;independent&quot;</span>,<span class="st">&quot;indirect&quot;</span>,<span class="st">&quot;planning&quot;</span>,<span class="st">&quot;delivery&quot;</span>)</span>
377 <span id="cb1-8"><a href="#cb1-8" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>implement) <span class="ot">&lt;-</span> <span class="fu">c</span>(<span class="st">&quot;low&quot;</span>,<span class="st">&quot;medium&quot;</span>,<span class="st">&quot;high&quot;</span>)</span>
378 <span id="cb1-9"><a href="#cb1-9" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>program_site) <span class="ot">&lt;-</span> <span class="fu">c</span>(<span class="st">&quot;community&quot;</span>,<span class="st">&quot;mixed&quot;</span>,<span class="st">&quot;classroom&quot;</span>,<span class="st">&quot;school&quot;</span>)</span>
379 <span id="cb1-10"><a href="#cb1-10" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>study_design) <span class="ot">&lt;-</span> <span class="fu">c</span>(<span class="st">&quot;matched&quot;</span>,<span class="st">&quot;unmatched&quot;</span>,<span class="st">&quot;RCT&quot;</span>)</span>
380 <span id="cb1-11"><a href="#cb1-11" tabindex="-1"></a><span class="fu">levels</span>(dropoutPrevention<span class="sc">$</span>adjusted) <span class="ot">&lt;-</span> <span class="fu">c</span>(<span class="st">&quot;no&quot;</span>,<span class="st">&quot;yes&quot;</span>)</span>
381 <span id="cb1-12"><a href="#cb1-12" tabindex="-1"></a></span>
382 <span id="cb1-13"><a href="#cb1-13" tabindex="-1"></a>m3_robu <span class="ot">&lt;-</span> <span class="fu">robu</span>(LOR1 <span class="sc">~</span> study_design <span class="sc">+</span> attrition <span class="sc">+</span> group_equivalence <span class="sc">+</span> adjusted</span>
383 <span id="cb1-14"><a href="#cb1-14" tabindex="-1"></a> <span class="sc">+</span> outcome <span class="sc">+</span> eval <span class="sc">+</span> male_pct <span class="sc">+</span> white_pct <span class="sc">+</span> average_age</span>
384 <span id="cb1-15"><a href="#cb1-15" tabindex="-1"></a> <span class="sc">+</span> implement <span class="sc">+</span> program_site <span class="sc">+</span> duration <span class="sc">+</span> service_hrs, </span>
385 <span id="cb1-16"><a href="#cb1-16" tabindex="-1"></a> <span class="at">data =</span> dropoutPrevention, <span class="at">studynum =</span> studyID, <span class="at">var.eff.size =</span> varLOR, </span>
386 <span id="cb1-17"><a href="#cb1-17" tabindex="-1"></a> <span class="at">modelweights =</span> <span class="st">&quot;HIER&quot;</span>)</span>
387 <span id="cb1-18"><a href="#cb1-18" tabindex="-1"></a><span class="fu">print</span>(m3_robu)</span></code></pre></div>
388 <pre><code>## RVE: Hierarchical Effects Model with Small-Sample Corrections
389 ##
390 ## Model: LOR1 ~ study_design + attrition + group_equivalence + adjusted + outcome + eval + male_pct + white_pct + average_age + implement + program_site + duration + service_hrs
391 ##
392 ## Number of clusters = 152
393 ## Number of outcomes = 385 (min = 1 , mean = 2.53 , median = 1 , max = 30 )
394 ## Omega.sq = 0.24907
395 ## Tau.sq = 0.1024663
396 ##
397 ## Estimate StdErr t-value dfs P(|t|&gt;) 95% CI.L 95% CI.U Sig
398 ## 1 X.Intercept. 0.016899 0.615399 0.0275 16.9 0.97841541 -1.28228 1.31608
399 ## 2 study_designunmatched -0.002626 0.185142 -0.0142 40.5 0.98875129 -0.37667 0.37141
400 ## 3 study_designRCT -0.086872 0.140044 -0.6203 38.6 0.53869676 -0.37024 0.19650
401 ## 4 attrition 0.118889 0.247228 0.4809 15.5 0.63732597 -0.40666 0.64444
402 ## 5 group_equivalence 0.502463 0.195838 2.5657 28.7 0.01579282 0.10174 0.90318 **
403 ## 6 adjustedyes -0.322480 0.125413 -2.5713 33.8 0.01470796 -0.57741 -0.06755 **
404 ## 7 outcomeenrolled 0.097059 0.139842 0.6941 16.5 0.49727848 -0.19862 0.39274
405 ## 8 outcomegraduation 0.147643 0.134938 1.0942 30.2 0.28253825 -0.12786 0.42315
406 ## 9 outcomegraduation.ged 0.258034 0.169134 1.5256 16.3 0.14632629 -0.10006 0.61613
407 ## 10 evalindirect -0.765085 0.399109 -1.9170 6.2 0.10212896 -1.73406 0.20389
408 ## 11 evalplanning -0.920874 0.346536 -2.6574 5.6 0.04027061 -1.78381 -0.05794 **
409 ## 12 evaldelivery -0.916673 0.304303 -3.0124 4.7 0.03212299 -1.71432 -0.11903 **
410 ## 13 male_pct 0.167965 0.181538 0.9252 16.4 0.36824526 -0.21609 0.55202
411 ## 14 white_pct 0.022915 0.149394 0.1534 21.8 0.87950385 -0.28704 0.33287
412 ## 15 average_age 0.037102 0.027053 1.3715 21.2 0.18458247 -0.01913 0.09333
413 ## 16 implementmedium 0.411779 0.128898 3.1946 26.7 0.00358205 0.14714 0.67642 ***
414 ## 17 implementhigh 0.658570 0.123874 5.3164 34.6 0.00000635 0.40699 0.91015 ***
415 ## 18 program_sitemixed 0.444384 0.172635 2.5741 28.6 0.01550504 0.09109 0.79768 **
416 ## 19 program_siteclassroom 0.426658 0.159773 2.6704 37.4 0.01115192 0.10303 0.75028 **
417 ## 20 program_siteschool 0.262517 0.160519 1.6354 30.1 0.11236814 -0.06525 0.59028
418 ## 21 duration 0.000427 0.000873 0.4895 36.7 0.62736846 -0.00134 0.00220
419 ## 22 service_hrs -0.003434 0.005012 -0.6852 36.7 0.49752503 -0.01359 0.00672
420 ## ---
421 ## Signif. codes: &lt; .01 *** &lt; .05 ** &lt; .10 *
422 ## ---
423 ## Note: If df &lt; 4, do not trust the results</code></pre>
424 <p>Note that <code>robumeta</code> produces small-sample corrected
425 standard errors and t-tests, and so there is no need to repeat those
426 calculations with <code>clubSandwich</code>. The <code>eval</code>
427 variable has four levels, and it might be of interest to test whether
428 the average program effects differ by the degree of evaluator
429 independence. The null hypothesis in this case is that the 10th, 11th,
430 and 12th regression coefficients are all equal to zero. A small-sample
431 adjusted F-test for this hypothesis can be obtained as follows. The
432 <code>vcov = &quot;CR2&quot;</code> option means that the standard errors will be
433 corrected using the bias-reduced linearization estimator described in
434 Tipton and Pustejovsky (2015).</p>
435 <div class="sourceCode" id="cb3"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb3-1"><a href="#cb3-1" tabindex="-1"></a><span class="fu">Wald_test</span>(m3_robu, <span class="at">constraints =</span> <span class="fu">constrain_zero</span>(<span class="dv">10</span><span class="sc">:</span><span class="dv">12</span>), <span class="at">vcov =</span> <span class="st">&quot;CR2&quot;</span>)</span></code></pre></div>
436 <pre><code>## test Fstat df_num df_denom p_val sig
437 ## HTZ 2.78 3 16.8 0.0732 .</code></pre>
438 <p>By default, the <code>Wald_test</code> function provides an F-type
439 test with degrees of freedom estimated using the approximate Hotelling’s
440 <span class="math inline">\(T^2_Z\)</span> method. The test has less
441 than 17 degrees of freedom, even though there are 152 independent
442 studies in the data, and has a p-value that is not quite significant at
443 conventional levels. The low degrees of freedom are a consequence of the
444 fact that one of the levels of <code>evaluator independence</code> has
445 only a few effect sizes in it:</p>
446 <div class="sourceCode" id="cb5"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb5-1"><a href="#cb5-1" tabindex="-1"></a><span class="fu">table</span>(dropoutPrevention<span class="sc">$</span>eval)</span></code></pre></div>
447 <pre><code>##
448 ## independent indirect planning delivery
449 ## 6 33 43 303</code></pre>
450 </div>
451 <div id="metafor-model" class="section level2">
452 <h2>metafor model</h2>
453 <p><code>clubSandwich</code> also works with models fit using the
454 <code>metafor</code> package. Here we re-fit the same regression
455 specification, but use REML to estimate the variance components
456 (<code>robumeta</code> uses a method-of-moments estimator), as well as a
457 somewhat different weighting scheme than that used in
458 <code>robumeta</code>.</p>
459 <div class="sourceCode" id="cb7"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb7-1"><a href="#cb7-1" tabindex="-1"></a><span class="fu">library</span>(metafor)</span>
460 <span id="cb7-2"><a href="#cb7-2" tabindex="-1"></a>m3_metafor <span class="ot">&lt;-</span> <span class="fu">rma.mv</span>(LOR1 <span class="sc">~</span> study_design <span class="sc">+</span> attrition <span class="sc">+</span> group_equivalence <span class="sc">+</span> adjusted</span>
461 <span id="cb7-3"><a href="#cb7-3" tabindex="-1"></a> <span class="sc">+</span> outcome <span class="sc">+</span> eval</span>
462 <span id="cb7-4"><a href="#cb7-4" tabindex="-1"></a> <span class="sc">+</span> male_pct <span class="sc">+</span> white_pct <span class="sc">+</span> average_age</span>
463 <span id="cb7-5"><a href="#cb7-5" tabindex="-1"></a> <span class="sc">+</span> implement <span class="sc">+</span> program_site <span class="sc">+</span> duration <span class="sc">+</span> service_hrs, </span>
464 <span id="cb7-6"><a href="#cb7-6" tabindex="-1"></a> <span class="at">V =</span> varLOR, <span class="at">random =</span> <span class="fu">list</span>(<span class="sc">~</span> <span class="dv">1</span> <span class="sc">|</span> studyID, <span class="sc">~</span> <span class="dv">1</span> <span class="sc">|</span> studySample),</span>
465 <span id="cb7-7"><a href="#cb7-7" tabindex="-1"></a> <span class="at">data =</span> dropoutPrevention)</span>
466 <span id="cb7-8"><a href="#cb7-8" tabindex="-1"></a><span class="fu">summary</span>(m3_metafor)</span></code></pre></div>
467 <pre><code>##
468 ## Multivariate Meta-Analysis Model (k = 385; method: REML)
469 ##
470 ## logLik Deviance AIC BIC AICc
471 ## -489.0357 978.0714 1026.0714 1119.5371 1029.6217
472 ##
473 ## Variance Components:
474 ##
475 ## estim sqrt nlvls fixed factor
476 ## sigma^2.1 0.2274 0.4769 152 no studyID
477 ## sigma^2.2 0.1145 0.3384 317 no studySample
478 ##
479 ## Test for Residual Heterogeneity:
480 ## QE(df = 363) = 1588.4397, p-val &lt; .0001
481 ##
482 ## Test of Moderators (coefficients 2:22):
483 ## QM(df = 21) = 293.8694, p-val &lt; .0001
484 ##
485 ## Model Results:
486 ##
487 ## estimate se zval pval ci.lb ci.ub
488 ## intrcpt 0.5296 0.7250 0.7304 0.4651 -0.8915 1.9506
489 ## study_designunmatched -0.0494 0.1722 -0.2871 0.7741 -0.3870 0.2881
490 ## study_designRCT 0.0653 0.1628 0.4010 0.6884 -0.2538 0.3843
491 ## attrition -0.1366 0.2429 -0.5623 0.5739 -0.6126 0.3395
492 ## group_equivalence 0.4071 0.1573 2.5877 0.0097 0.0988 0.7155 **
493 ## adjustedyes -0.3581 0.1532 -2.3371 0.0194 -0.6585 -0.0578 *
494 ## outcomeenrolled -0.2831 0.0771 -3.6709 0.0002 -0.4343 -0.1320 ***
495 ## outcomegraduation -0.0913 0.0657 -1.3896 0.1646 -0.2201 0.0375
496 ## outcomegraduation/ged 0.6983 0.0805 8.6750 &lt;.0001 0.5406 0.8561 ***
497 ## evalindirect -0.7530 0.4949 -1.5214 0.1282 -1.7230 0.2171
498 ## evalplanning -0.7700 0.4869 -1.5814 0.1138 -1.7242 0.1843
499 ## evaldelivery -1.0016 0.4600 -2.1774 0.0294 -1.9033 -0.1000 *
500 ## male_pct 0.1021 0.1715 0.5951 0.5518 -0.2341 0.4382
501 ## white_pct 0.1223 0.1804 0.6777 0.4979 -0.2313 0.4758
502 ## average_age 0.0061 0.0291 0.2091 0.8344 -0.0509 0.0631
503 ## implementmedium 0.4738 0.1609 2.9445 0.0032 0.1584 0.7892 **
504 ## implementhigh 0.6318 0.1471 4.2965 &lt;.0001 0.3436 0.9201 ***
505 ## program_sitemixed 0.3289 0.2413 1.3631 0.1729 -0.1440 0.8019
506 ## program_siteclassroom 0.2920 0.1736 1.6821 0.0926 -0.0482 0.6321 .
507 ## program_siteschool 0.1616 0.1898 0.8515 0.3945 -0.2104 0.5337
508 ## duration 0.0013 0.0009 1.3423 0.1795 -0.0006 0.0031
509 ## service_hrs -0.0003 0.0047 -0.0654 0.9478 -0.0096 0.0090
510 ##
511 ## ---
512 ## Signif. codes: 0 &#39;***&#39; 0.001 &#39;**&#39; 0.01 &#39;*&#39; 0.05 &#39;.&#39; 0.1 &#39; &#39; 1</code></pre>
513 <p><code>metafor</code> produces model-based standard errors, t-tests,
514 and confidence intervals. The <code>coef_test</code> function from
515 <code>clubSandwich</code> will calculate robust standard errors and
516 robust t-tests for each of the coefficients:</p>
517 <div class="sourceCode" id="cb9"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb9-1"><a href="#cb9-1" tabindex="-1"></a><span class="fu">coef_test</span>(m3_metafor, <span class="at">vcov =</span> <span class="st">&quot;CR2&quot;</span>)</span></code></pre></div>
518 <pre><code>## Alternative hypothesis: two-sided
519 ## Coef. Estimate SE Null value t-stat d.f. (Satt) p-val (Satt) Sig.
520 ## intrcpt 0.529569 0.724851 0 0.7306 20.08 0.47347
521 ## study_designunmatched -0.049434 0.204152 0 -0.2421 58.42 0.80952
522 ## study_designRCT 0.065272 0.149146 0 0.4376 53.17 0.66342
523 ## attrition -0.136575 0.306429 0 -0.4457 10.52 0.66485
524 ## group_equivalence 0.407108 0.210917 0 1.9302 23.10 0.06595 .
525 ## adjustedyes -0.358124 0.136132 0 -2.6307 43.20 0.01176 *
526 ## outcomeenrolled -0.283124 0.237199 0 -1.1936 7.08 0.27108
527 ## outcomegraduation -0.091295 0.091465 0 -0.9981 9.95 0.34188
528 ## outcomegraduation/ged 0.698328 0.364882 0 1.9138 8.02 0.09188 .
529 ## evalindirect -0.752994 0.447670 0 -1.6820 6.56 0.13929
530 ## evalplanning -0.769968 0.403898 0 -1.9063 6.10 0.10446
531 ## evaldelivery -1.001648 0.355989 0 -2.8137 4.89 0.03834 *
532 ## male_pct 0.102055 0.148410 0 0.6877 9.68 0.50782
533 ## white_pct 0.122255 0.141470 0 0.8642 16.88 0.39961
534 ## average_age 0.006084 0.033387 0 0.1822 15.79 0.85772
535 ## implementmedium 0.473789 0.148660 0 3.1871 22.44 0.00419 **
536 ## implementhigh 0.631842 0.138073 0 4.5761 28.68 &lt; 0.001 ***
537 ## program_sitemixed 0.328941 0.196848 0 1.6710 27.47 0.10607
538 ## program_siteclassroom 0.291952 0.146014 0 1.9995 42.70 0.05195 .
539 ## program_siteschool 0.161640 0.171700 0 0.9414 29.27 0.35420
540 ## duration 0.001270 0.000978 0 1.2988 31.96 0.20332
541 ## service_hrs -0.000309 0.004828 0 -0.0641 49.63 0.94915</code></pre>
542 <p>Note that <code>coef_test</code> assumed that it should cluster based
543 on <code>studyID</code>, which is the outer-most random effect in the
544 metafor model. This can be specified explicitly by including the option
545 <code>cluster = dropoutPrevention$studyID</code> in the call.</p>
546 <p>The F-test for degree of evaluator independence uses the same syntax
547 as before:</p>
548 <div class="sourceCode" id="cb11"><pre class="sourceCode r"><code class="sourceCode r"><span id="cb11-1"><a href="#cb11-1" tabindex="-1"></a><span class="fu">Wald_test</span>(m3_metafor, <span class="at">constraints =</span> <span class="fu">constrain_zero</span>(<span class="dv">10</span><span class="sc">:</span><span class="dv">12</span>), <span class="at">vcov =</span> <span class="st">&quot;CR2&quot;</span>)</span></code></pre></div>
549 <pre><code>## test Fstat df_num df_denom p_val sig
550 ## HTZ 2.71 3 18.3 0.0753 .</code></pre>
551 <p>Despite some differences in weighting schemes, the p-value is very
552 close to the result obtained using <code>robumeta</code>.</p>
553 </div>
554 <div id="references" class="section level2">
555 <h2>References</h2>
556 <p>Fisher, Z., &amp; Tipton, E. (2015). robumeta: An R-package for
557 robust variance estimation in meta-analysis. <a href="https://arxiv.org/abs/1503.02220">arXiv:1503.02220</a></p>
558 <p>Tipton, E. (2015). Small sample adjustments for robust variance
559 estimation with meta-regression. <em>Psychological Methods, 20</em>(3),
560 375-393. <a href="https://doi.org/10.1037/met0000011" class="uri">https://doi.org/10.1037/met0000011</a></p>
561 <p>Tipton, E., &amp; Pustejovsky, J. E. (2015). Small-sample adjustments
562 for tests of moderators and model fit using robust variance estimation
563 in meta-regression. <em>Journal of Educational and Behavioral
564 Statistics, 40</em>(6), 604-634. <a href="https://doi.org/10.3102/1076998615606099" class="uri">https://doi.org/10.3102/1076998615606099</a></p>
565 <p>Viechtbauer, W. (2010). Conducting meta-analyses in R with the
566 metafor package. <em>Journal of Statistical Software, 36</em>(3), 1-48.
567 URL: <a href="https://doi.org/10.18637/jss.v036.i03" class="uri">https://doi.org/10.18637/jss.v036.i03</a></p>
568 <p>Wilson, S. J., Lipsey, M. W., Tanner-Smith, E., Huang, C. H., &amp;
569 Steinka-Fry, K. T. (2011). Dropout prevention and intervention programs:
570 Effects on school completion and dropout Among school-aged children and
571 youth: A systematic review. <em>Campbell Systematic Reviews, 7</em>(1),
572 1-61. <a href="https://doi.org/10.4073/csr.2011.8" class="uri">https://doi.org/10.4073/csr.2011.8</a></p>
573 </div>
574
575
576
577 <!-- code folding -->
578
579
580 <!-- dynamically load mathjax for compatibility with self-contained -->
581 <script>
582 (function () {
583 var script = document.createElement("script");
584 script.type = "text/javascript";
585 script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
586 document.getElementsByTagName("head")[0].appendChild(script);
587 })();
588 </script>
589
590 </body>
591 </html>