Debian Codegraph @

.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py

raw
1 # Copyright 2009-present MongoDB, Inc.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14
15 from collections import OrderedDict as OD
16 from itertools import chain
17 from typing import ClassVar, Iterable, Literal, Mapping, MutableSequence, Optional, Sequence
18
19 from evergreen_config_generator import Value
20 from evergreen_config_generator.functions import func, s3_put
21 from evergreen_config_generator.tasks import (
22 DependencySpec,
23 MatrixTask,
24 NamedTask,
25 both_or_neither,
26 prohibit,
27 require,
28 )
29 from packaging.version import Version
30
31 from evergreen_config_lib import shell_mongoc
32
33 ToggleStr = Literal['OFF', 'ON']
34 OptToggleStr = Optional[ToggleStr]
35 TopologyStr = Literal['server']
36
37
38 class CompileTask(NamedTask):
39 cls_compile_sh_env: ClassVar[Mapping[str, str]] = {}
40 cls_tags: ClassVar[Sequence[str]] = ()
41 cls_sanitize: ClassVar[Sequence[str]] = ()
42
43 def __init__(
44 self,
45 task_name: str,
46 tags: Iterable[str] = (),
47 config: str = 'debug',
48 compression: str | None = 'default',
49 suffix_commands: Iterable[Value] = (),
50 depends_on: Iterable[DependencySpec] = (),
51 prefix_commands: Iterable[Value] = (),
52 sanitize: Iterable[Literal['undefined', 'address', 'thread']] = (),
53 *,
54 CFLAGS: str | None = None,
55 LDFLAGS: str | None = None,
56 EXTRA_CONFIGURE_FLAGS: str | None = None,
57 SSL: Literal['WINDOWS', 'DARWIN', 'OPENSSL', 'OPENSSL_STATIC', 'OFF', None] = None,
58 ENABLE_SHM_COUNTERS: OptToggleStr = None,
59 CHECK_LOG: OptToggleStr = None,
60 TRACING: OptToggleStr = None,
61 SASL: Literal[None, 'OFF', 'AUTO', 'CYRUS', 'SSPI'] = None,
62 ENABLE_RDTSCP: OptToggleStr = None,
63 SRV: OptToggleStr = None,
64 ):
65 super(CompileTask, self).__init__(task_name=task_name, depends_on=depends_on, tags=tags)
66
67 self.suffix_commands = list(suffix_commands)
68 self.prefix_commands = list(prefix_commands)
69
70 # Environment variables for .evergreen/scripts/compile.sh.
71 self.compile_sh_opt: dict[str, str] = {}
72
73 if config != 'debug':
74 assert config == 'release'
75 self.compile_sh_opt['RELEASE'] = 'ON'
76
77 if CFLAGS:
78 self.compile_sh_opt['CFLAGS'] = CFLAGS
79 if LDFLAGS:
80 self.compile_sh_opt['LDFLAGS'] = LDFLAGS
81 if EXTRA_CONFIGURE_FLAGS:
82 self.compile_sh_opt['EXTRA_CONFIGURE_FLAGS'] = EXTRA_CONFIGURE_FLAGS
83 if SSL:
84 self.compile_sh_opt['SSL'] = SSL
85 if ENABLE_SHM_COUNTERS:
86 self.compile_sh_opt['ENABLE_SHM_COUNTERS'] = ENABLE_SHM_COUNTERS
87 if CHECK_LOG:
88 self.compile_sh_opt['CHECK_LOG'] = CHECK_LOG
89 if TRACING:
90 self.compile_sh_opt['TRACING'] = TRACING
91 if SASL:
92 self.compile_sh_opt['SASL'] = SASL
93 if ENABLE_RDTSCP:
94 self.compile_sh_opt['ENABLE_RDTSCP'] = ENABLE_RDTSCP
95 if SRV:
96 self.compile_sh_opt['SRV'] = SRV
97
98 if compression != 'default':
99 self.compile_sh_opt['SNAPPY'] = 'ON' if compression in ('all', 'snappy') else 'OFF'
100 self.compile_sh_opt['ZLIB'] = 'BUNDLED' if compression in ('all', 'zlib') else 'OFF'
101 self.compile_sh_opt['ZSTD'] = 'ON' if compression in ('all', 'zstd') else 'OFF'
102
103 if sanitize:
104 self.compile_sh_opt['SANITIZE'] = ','.join(sanitize)
105
106 self.compile_sh_opt.update(type(self).cls_compile_sh_env)
107
108 def additional_script_env(self) -> Mapping[str, str]:
109 return {}
110
111 def to_dict(self):
112 task = super(CompileTask, self).to_dict()
113 commands = task['commands']
114 assert isinstance(commands, MutableSequence), task
115
116 commands.extend(self.prefix_commands)
117
118 script = 'env'
119 for opt, value in sorted(self.compile_sh_opt.items()):
120 script += ' %s="%s"' % (opt, value)
121
122 script += ' .evergreen/scripts/compile.sh'
123
124 commands.append(shell_mongoc(script, add_expansions_to_env=True))
125 commands.append(func('upload-build'))
126 commands.extend(self.suffix_commands)
127 return task
128
129 def additional_tags(self) -> Iterable[str]:
130 yield from super().additional_tags()
131 yield from self.cls_tags
132
133
134 class SpecialTask(CompileTask):
135 cls_tags: ClassVar[Sequence[str]] = ['special']
136
137
138 class CompileWithClientSideEncryption(CompileTask):
139 cls_compile_sh_env: ClassVar[Mapping[str, str]] = dict(
140 # Compiling with ClientSideEncryption support requires linking against the library libmongocrypt.
141 COMPILE_LIBMONGOCRYPT='ON',
142 EXTRA_CONFIGURE_FLAGS='-DENABLE_PIC=ON',
143 )
144 cls_tags: ClassVar[Sequence[str]] = 'client-side-encryption', 'special'
145
146
147 class CompileWithClientSideEncryptionAsan(CompileTask):
148 cls_compile_sh_env: ClassVar[Mapping[str, str]] = dict(
149 CFLAGS='-fno-omit-frame-pointer',
150 COMPILE_LIBMONGOCRYPT='ON',
151 CHECK_LOG='ON',
152 PATH='/usr/lib/llvm-3.8/bin:$PATH',
153 )
154 cls_tags: ClassVar[Sequence[str]] = ['client-side-encryption']
155 cls_sanitize: ClassVar[Sequence[str]] = ['address']
156
157
158 class LinkTask(NamedTask):
159 def __init__(
160 self, task_name: str, suffix_commands: Iterable[Value], orchestration: Literal[True, False, 'ssl'] = True
161 ):
162 if orchestration == 'ssl':
163 # Actual value of SSL does not matter here so long as it is not 'nossl'.
164 bootstrap_commands = [func('fetch-det'), func('bootstrap-mongo-orchestration', SSL='openssl')]
165 elif orchestration:
166 bootstrap_commands = [func('fetch-det'), func('bootstrap-mongo-orchestration')]
167 else:
168 bootstrap_commands = []
169
170 super().__init__(
171 task_name=task_name,
172 commands=bootstrap_commands + list(suffix_commands),
173 )
174
175
176 all_tasks = [
177 CompileTask(
178 'hardened-compile',
179 tags=['hardened'],
180 compression=None,
181 CFLAGS='-fno-strict-overflow -D_FORTIFY_SOURCE=2 -fstack-protector-all -O',
182 LDFLAGS='-Wl,-z,relro -Wl,-z,now',
183 EXTRA_CONFIGURE_FLAGS='CMAKE_POSITION_INDEPENDENT_CODE=ON',
184 ),
185 CompileTask('debug-compile-compression-zlib', tags=['zlib', 'compression'], compression='zlib'),
186 CompileTask('debug-compile-compression-snappy', tags=['snappy', 'compression'], compression='snappy'),
187 CompileTask('debug-compile-compression-zstd', tags=['zstd', 'compression'], compression='zstd'),
188 CompileTask('debug-compile-nosasl-nossl', tags=['debug-compile', 'nosasl', 'nossl'], SSL='OFF'),
189 CompileTask('debug-compile-lto', CFLAGS='-flto'),
190 CompileTask('debug-compile-lto-thin', CFLAGS='-flto=thin'),
191 CompileTask('debug-compile-no-counters', tags=['debug-compile', 'no-counters'], ENABLE_SHM_COUNTERS='OFF'),
192 CompileTask('compile-tracing', TRACING='ON', CFLAGS='-Werror -Wno-cast-align'),
193 CompileTask('release-compile', config='release'),
194 CompileTask('debug-compile-nosasl-openssl', tags=['debug-compile', 'nosasl', 'openssl'], SSL='OPENSSL'),
195 CompileTask('debug-compile-nosasl-darwinssl', tags=['debug-compile', 'nosasl', 'darwinssl'], SSL='DARWIN'),
196 CompileTask('debug-compile-nosasl-winssl', tags=['debug-compile', 'nosasl', 'winssl'], SSL='WINDOWS'),
197 CompileTask('debug-compile-sasl-openssl', tags=['debug-compile', 'sasl', 'openssl'], SASL='AUTO', SSL='OPENSSL'),
198 CompileTask('debug-compile-sasl-darwinssl', tags=['debug-compile', 'sasl', 'darwinssl'], SASL='AUTO', SSL='DARWIN'),
199 CompileTask('debug-compile-rdtscp', ENABLE_RDTSCP='ON'),
200 CompileTask('debug-compile-sspi-winssl', tags=['debug-compile', 'sspi', 'winssl'], SASL='SSPI', SSL='WINDOWS'),
201 CompileTask('debug-compile-nosrv', tags=['debug-compile'], SRV='OFF'),
202 LinkTask('link-with-cmake', suffix_commands=[func('link sample program', BUILD_SAMPLE_WITH_CMAKE=1)]),
203 LinkTask(
204 'link-with-cmake-ssl',
205 suffix_commands=[func('link sample program', BUILD_SAMPLE_WITH_CMAKE=1, ENABLE_SSL=1)],
206 ),
207 LinkTask(
208 'link-with-cmake-snappy',
209 suffix_commands=[func('link sample program', BUILD_SAMPLE_WITH_CMAKE=1, ENABLE_SNAPPY='ON')],
210 ),
211 LinkTask('link-with-cmake-mac', suffix_commands=[func('link sample program', BUILD_SAMPLE_WITH_CMAKE=1)]),
212 LinkTask('link-with-cmake-windows', suffix_commands=[func('link sample program MSVC')]),
213 LinkTask(
214 'link-with-cmake-windows-ssl',
215 suffix_commands=[func('link sample program MSVC', ENABLE_SSL=1)],
216 orchestration='ssl',
217 ),
218 LinkTask('link-with-cmake-windows-snappy', suffix_commands=[func('link sample program MSVC', ENABLE_SNAPPY='ON')]),
219 LinkTask('link-with-cmake-mingw', suffix_commands=[func('link sample program mingw')]),
220 LinkTask('link-with-pkg-config', suffix_commands=[func('link sample program')]),
221 LinkTask('link-with-pkg-config-mac', suffix_commands=[func('link sample program')]),
222 LinkTask('link-with-pkg-config-ssl', suffix_commands=[func('link sample program', ENABLE_SSL=1)]),
223 LinkTask('link-with-bson', suffix_commands=[func('link sample program bson')], orchestration=False),
224 LinkTask('link-with-bson-mac', suffix_commands=[func('link sample program bson')], orchestration=False),
225 LinkTask('link-with-bson-windows', suffix_commands=[func('link sample program MSVC bson')], orchestration=False),
226 LinkTask('link-with-bson-mingw', suffix_commands=[func('link sample program mingw bson')], orchestration=False),
227 NamedTask(
228 'rpm-package-build',
229 commands=[
230 shell_mongoc('export IS_PATCH="${is_patch}"\n.evergreen/scripts/check_rpm_spec.sh'),
231 shell_mongoc('.evergreen/scripts/build_snapshot_rpm.sh'),
232 s3_put(
233 local_file='rpm.tar.gz',
234 remote_file='${branch_name}/mongo-c-driver-rpm-packages-${CURRENT_VERSION}.tar.gz',
235 content_type='${content_type|application/x-gzip}',
236 ),
237 s3_put(
238 local_file='rpm.tar.gz',
239 remote_file='${branch_name}/${revision}/${version_id}/${build_id}/${execution}/mongo-c-driver-rpm-packages.tar.gz',
240 content_type='${content_type|application/x-gzip}',
241 ),
242 shell_mongoc(
243 'sudo rm -rf ../build ../mock-result ../rpm.tar.gz\n'
244 'export MOCK_TARGET_CONFIG=centos-stream+epel-10-aarch64\n'
245 '.evergreen/scripts/build_snapshot_rpm.sh'
246 ),
247 ],
248 allowed_requesters=[
249 'ad_hoc',
250 'commit',
251 # 'github_merge_queue'
252 # 'github_pr',
253 # 'github_tag',
254 'patch',
255 'trigger',
256 ],
257 ),
258 CompileTask('debug-compile-with-warnings', CFLAGS='-Werror -Wno-cast-align'),
259 NamedTask(
260 'install-libmongoc-after-libbson',
261 commands=[
262 shell_mongoc(
263 '.evergreen/scripts/install-libmongoc-after-libbson.sh',
264 include_expansions_in_env=['distro_id'],
265 ),
266 ],
267 ),
268 ]
269
270
271 class CoverageTask(MatrixTask):
272 axes = OD(
273 [
274 ('version', ['latest']),
275 ('topology', ['replica_set']),
276 ('auth', [True]),
277 ('sasl', ['sasl']),
278 ('ssl', ['openssl']),
279 ('cse', [False, True]),
280 ]
281 )
282
283 def additional_tags(self) -> Iterable[str]:
284 yield from super().additional_tags()
285 yield 'test-coverage'
286 yield str(self.settings.version)
287 if self.cse:
288 yield 'client-side-encryption'
289
290 def name_parts(self) -> Iterable[str]:
291 yield 'test-coverage'
292 yield self.display('version')
293 yield self.display('topology').replace('_', '-')
294 yield from map(self.display, ('auth', 'sasl', 'ssl'))
295 if self.settings.cse:
296 yield 'cse'
297
298 @property
299 def cse(self) -> bool:
300 return bool(self.settings.cse)
301
302 def post_commands(self) -> Iterable[Value]:
303 if self.cse:
304 yield func(
305 'compile coverage',
306 SASL='AUTO',
307 SSL='OPENSSL',
308 COMPILE_LIBMONGOCRYPT='ON',
309 EXTRA_CONFIGURE_FLAGS='EXTRA_CONFIGURE_FLAGS="-DENABLE_PIC=ON"',
310 )
311 else:
312 yield func('compile coverage', SASL='AUTO', SSL='OPENSSL')
313
314 yield func('fetch-det')
315 yield func(
316 'bootstrap-mongo-orchestration',
317 MONGODB_VERSION=self.settings.version,
318 TOPOLOGY=self.settings.topology,
319 AUTH=self.display('auth'),
320 SSL=self.display('ssl'),
321 )
322 yield func('run-simple-http-server')
323 extra = {'COVERAGE': 'ON'}
324 if self.cse:
325 extra['CLIENT_SIDE_ENCRYPTION'] = 'ON'
326 yield func('csfle-setup')
327 yield func('run-tests', AUTH=self.display('auth'), SSL=self.display('ssl'), **extra)
328 if self.cse:
329 yield func('csfle-teardown')
330 yield func('upload coverage')
331 yield func('update codecov.io')
332
333 def do_is_valid_combination(self) -> bool:
334 # Limit coverage tests to test-coverage-latest-replica-set-auth-sasl-openssl (+ cse).
335 require(self.setting_eq('topology', 'replica_set'))
336 require(self.setting_eq('sasl', 'sasl'))
337 require(self.setting_eq('ssl', 'openssl'))
338 require(self.setting_eq('version', 'latest'))
339 require(self.settings.auth is True)
340 return True
341
342
343 all_tasks = chain(all_tasks, CoverageTask.matrix())
344
345
346 class DNSTask(MatrixTask):
347 axes = OD(
348 [
349 ('auth', [False, True]),
350 ('loadbalanced', [False, True]),
351 ('ssl', ['openssl', 'winssl', 'darwinssl']),
352 ]
353 )
354
355 name_prefix = 'test-dns'
356
357 def additional_dependencies(self) -> Iterable[DependencySpec]:
358 yield self.build_task_name
359
360 @property
361 def build_task_name(self) -> str:
362 sasl = 'sspi' if self.settings.ssl == 'winssl' else 'sasl'
363 return f'debug-compile-{sasl}-{self.display("ssl")}'
364
365 def name_parts(self) -> Iterable[str]:
366 yield 'test-dns'
367 if self.settings.auth:
368 yield 'auth'
369 if self.settings.loadbalanced:
370 yield 'loadbalanced'
371 yield self.display('ssl')
372
373 def post_commands(self) -> Iterable[Value]:
374 yield func('fetch-build', BUILD_NAME=self.build_task_name)
375 yield func('fetch-det')
376 if self.settings.loadbalanced:
377 orchestration = func(
378 'bootstrap-mongo-orchestration',
379 TOPOLOGY='sharded_cluster',
380 AUTH='auth' if self.settings.auth else 'noauth',
381 SSL='ssl',
382 LOAD_BALANCER='on',
383 )
384 else:
385 orchestration = func(
386 'bootstrap-mongo-orchestration',
387 TOPOLOGY='replica_set',
388 AUTH='auth' if self.settings.auth else 'noauth',
389 SSL='ssl',
390 )
391
392 yield orchestration
393
394 dns = 'on'
395 if self.settings.loadbalanced:
396 dns = 'loadbalanced'
397 yield func('fetch-det')
398 yield func('start-load-balancer', MONGODB_URI='mongodb://localhost:27017,localhost:27018')
399 elif self.settings.auth:
400 dns = 'dns-auth'
401 yield func('run-tests', SSL='ssl', AUTH=self.display('auth'), DNS=dns)
402
403 def do_is_valid_combination(self) -> bool:
404 prohibit(bool(self.settings.loadbalanced) and bool(self.settings.auth))
405 # Load balancer tests only run on some Linux hosts in Evergreen until CDRIVER-4041 is resolved.
406 prohibit(bool(self.settings.loadbalanced) and self.settings.ssl in ['darwinssl', 'winssl'])
407 return True
408
409
410 all_tasks = chain(all_tasks, DNSTask.matrix())
411
412
413 class CompressionTask(MatrixTask):
414 axes = OD([('compression', ['zlib', 'snappy', 'zstd'])])
415 name_prefix = 'test-latest-server'
416
417 def additional_dependencies(self) -> Iterable[DependencySpec]:
418 yield self.build_task_name
419
420 @property
421 def build_task_name(self) -> str:
422 return f'debug-compile-{self._compressor_suffix()}'
423
424 def additional_tags(self) -> Iterable[str]:
425 yield from super().additional_tags()
426 yield 'compression'
427 yield 'latest'
428 yield from self._compressor_list()
429
430 def name_parts(self) -> Iterable[str]:
431 return [self.name_prefix, self._compressor_suffix()]
432
433 def post_commands(self) -> Iterable[Value]:
434 yield func('fetch-build', BUILD_NAME=self.build_task_name)
435 yield func('fetch-det')
436 yield func('bootstrap-mongo-orchestration', AUTH='noauth', SSL='nossl')
437 yield func('run-simple-http-server')
438 yield func('run-tests', AUTH='noauth', SSL='nossl', COMPRESSORS=','.join(self._compressor_list()))
439
440 def _compressor_suffix(self):
441 if self.settings.compression == 'zlib':
442 return 'compression-zlib'
443 elif self.settings.compression == 'snappy':
444 return 'compression-snappy'
445 elif self.settings.compression == 'zstd':
446 return 'compression-zstd'
447 else:
448 return 'compression'
449
450 def _compressor_list(self):
451 if self.settings.compression == 'zlib':
452 return ['zlib']
453 elif self.settings.compression == 'snappy':
454 return ['snappy']
455 elif self.settings.compression == 'zstd':
456 return ['zstd']
457 else:
458 return ['snappy', 'zlib', 'zstd']
459
460
461 all_tasks = chain(all_tasks, CompressionTask.matrix())
462
463
464 class SpecialIntegrationTask(NamedTask):
465 def __init__(
466 self,
467 task_name: str,
468 main_dep: str = 'debug-compile-sasl-openssl',
469 uri: str | None = None,
470 tags: Iterable[str] = (),
471 version: str = 'latest',
472 topology: str = 'server',
473 ):
474 self._main_dep = main_dep
475 super().__init__(task_name, depends_on=[self._main_dep], tags=tags)
476 self._uri = uri
477 self._version = version
478 self._topo = topology
479
480 def pre_commands(self) -> Iterable[Value]:
481 yield func('fetch-build', BUILD_NAME=self._main_dep)
482 yield func('fetch-det')
483 yield func('bootstrap-mongo-orchestration', MONGODB_VERSION=self._version, TOPOLOGY=self._topo)
484 yield func('run-simple-http-server')
485 yield func('run-tests', URI=self._uri)
486
487
488 all_tasks = chain(
489 all_tasks,
490 [
491 # Verify that retryWrites=true is ignored with standalone.
492 SpecialIntegrationTask('retry-true-latest-server', uri='mongodb://localhost/?retryWrites=true'),
493 SpecialIntegrationTask('test-latest-server-hardened', 'hardened-compile', tags=['hardened', 'latest']),
494 ],
495 )
496
497
498 class AuthTask(MatrixTask):
499 axes = OD([('sasl', ['sasl', 'sspi', False]), ('ssl', ['openssl', 'darwinssl', 'winssl'])])
500
501 name_prefix = 'authentication-tests'
502
503 def additional_tags(self) -> Iterable[str]:
504 yield from super().additional_tags()
505 yield 'authentication-tests'
506 yield self.display('ssl')
507 yield self.display('sasl')
508
509 def additional_dependencies(self) -> Iterable[DependencySpec]:
510 yield self.build_task_name
511
512 def post_commands(self) -> Iterable[Value]:
513 yield func('fetch-build', BUILD_NAME=self.build_task_name)
514 yield func('run auth tests')
515
516 @property
517 def build_task_name(self) -> str:
518 return f'debug-compile-{self.display("sasl")}-{self.display("ssl")}'
519
520 def name_parts(self) -> Iterable[str]:
521 yield self.name_prefix
522 yield self.display('ssl')
523 if not self.settings.sasl:
524 yield 'nosasl'
525
526 def do_is_valid_combination(self) -> bool:
527 both_or_neither(self.settings.ssl == 'winssl', self.settings.sasl == 'sspi')
528 if not self.settings.sasl:
529 require(self.settings.ssl == 'openssl')
530 return True
531
532
533 all_tasks = chain(all_tasks, AuthTask.matrix())
534
535
536 class PostCompileTask(NamedTask):
537 def __init__(self, name: str, tags: Iterable[str], get_build: str, commands: Iterable[Value]):
538 super().__init__(name, commands=commands, tags=tags, depends_on=[get_build])
539 self._dep = get_build
540
541 def pre_commands(self) -> Iterable[Value]:
542 yield func('fetch-build', BUILD_NAME=self._dep)
543
544
545 all_tasks = chain(
546 all_tasks,
547 [
548 NamedTask(
549 'authentication-tests-asan-memcheck',
550 tags=['authentication-tests', 'asan'],
551 commands=[
552 shell_mongoc(
553 'env SANITIZE=address SASL=AUTO SSL=OPENSSL .evergreen/scripts/compile.sh',
554 add_expansions_to_env=True,
555 ),
556 func('run auth tests', ASAN='on'),
557 ],
558 ),
559 ],
560 )
561
562 # Add API version tasks.
563 for server_version in ['8.0', '7.0', '6.0', '5.0']:
564 all_tasks = chain(
565 all_tasks,
566 [
567 PostCompileTask(
568 'test-versioned-api-' + server_version,
569 tags=['versioned-api', f'{server_version}'],
570 get_build='debug-compile-nosasl-openssl',
571 commands=[
572 func('fetch-det'),
573 func(
574 'bootstrap-mongo-orchestration',
575 TOPOLOGY='server',
576 AUTH='auth',
577 SSL='ssl',
578 MONGODB_VERSION=server_version,
579 REQUIRE_API_VERSION='true',
580 ),
581 func('run-simple-http-server'),
582 func('run-tests', MONGODB_API_VERSION=1, AUTH='auth', SSL='ssl'),
583 ],
584 ),
585 PostCompileTask(
586 'test-versioned-api-accept-version-two-' + server_version,
587 tags=['versioned-api', f'{server_version}'],
588 get_build='debug-compile-nosasl-nossl',
589 commands=[
590 func('fetch-det'),
591 func(
592 'bootstrap-mongo-orchestration',
593 TOPOLOGY='server',
594 AUTH='noauth',
595 SSL='nossl',
596 MONGODB_VERSION=server_version,
597 ORCHESTRATION_FILE='versioned-api-testing.json',
598 ),
599 func('run-simple-http-server'),
600 func('run-tests', MONGODB_API_VERSION=1, AUTH='noauth', SSL='nossl'),
601 ],
602 ),
603 ],
604 )
605
606
607 class IPTask(MatrixTask):
608 axes = OD(
609 [
610 ('version', ['7.0', 'latest']),
611 ('client', ['ipv6', 'ipv4', 'localhost']),
612 ('server', ['ipv6', 'ipv4']),
613 ]
614 )
615
616 name_prefix = 'test'
617
618 def additional_dependencies(self) -> Iterable[DependencySpec]:
619 yield 'debug-compile-nosasl-nossl'
620
621 def additional_tags(self) -> Iterable[str]:
622 yield from super().additional_tags()
623 yield from ('nossl', 'nosasl', 'server', 'ipv4-ipv6', self.settings.version)
624
625 def post_commands(self) -> Iterable[Value]:
626 return [
627 func('fetch-build', BUILD_NAME='debug-compile-nosasl-nossl'),
628 func('fetch-det'),
629 func('bootstrap-mongo-orchestration', MONGODB_VERSION=self.settings.version),
630 func('run-simple-http-server'),
631 func(
632 'run-tests',
633 URI={
634 'ipv6': 'mongodb://[::1]/',
635 'ipv4': 'mongodb://127.0.0.1/',
636 'localhost': 'mongodb://localhost/',
637 }[str(self.settings.client)],
638 ),
639 ]
640
641 def name_parts(self) -> Iterable[str]:
642 return (
643 self.name_prefix,
644 self.display('version'),
645 f'server-{self.display("server")}',
646 f'client-{self.display("client")}',
647 'noauth',
648 'nosasl',
649 'nossl',
650 )
651
652 def do_is_valid_combination(self) -> bool:
653 # This would fail by design.
654 if self.settings.server == 'ipv4':
655 prohibit(self.settings.client == 'ipv6')
656
657 # Default configuration is tested in other variants.
658 if self.settings.server == 'ipv6':
659 prohibit(self.settings.client == 'localhost')
660 return True
661
662
663 all_tasks = chain(all_tasks, IPTask.matrix())
664
665 aws_compile_task = NamedTask(
666 'debug-compile-aws',
667 commands=[
668 shell_mongoc(
669 """
670 set -o errexit
671 set -o pipefail
672
673 # Use ccache if able.
674 . .evergreen/scripts/find-ccache.sh
675 find_ccache_and_export_vars "$(pwd)" || true
676
677 # Ubuntu 20.04 does not have uv.
678 if ! command -v uv >/dev/null; then
679 prefix="$(mktemp -d)"
680 python3 -m pip install --prefix "$prefix" uv
681 PATH="$prefix/bin:$PATH"
682 fi
683
684 # Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 20.04 ECS cluster for testing, which may not have all dependent libraries.
685 uvx cmake -DENABLE_TRACING=ON -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF -S . -B cmake-build
686 uvx cmake --build cmake-build --target test-awsauth
687 """,
688 include_expansions_in_env=['CC'],
689 redirect_standard_error_to_output=True,
690 ),
691 func('upload-build'),
692 ],
693 )
694
695 all_tasks = chain(all_tasks, [aws_compile_task])
696
697
698 class AWSTestTask(MatrixTask):
699 axes = OD(
700 [
701 ('testcase', ['regular', 'ec2', 'ecs', 'lambda', 'assume_role', 'assume_role_with_web_identity']),
702 ('version', ['latest', 'rapid', '8.0', '7.0', '6.0', '5.0', '4.4']),
703 ]
704 )
705
706 name_prefix = 'test-aws-openssl'
707
708 def additional_dependencies(self) -> Iterable[DependencySpec]:
709 yield 'debug-compile-aws'
710
711 def additional_tags(self) -> Iterable[str]:
712 yield from super().additional_tags()
713 yield f'{self.settings.version}'
714 yield 'test-aws'
715
716 def post_commands(self) -> Iterable[Value]:
717 return [
718 func('fetch-build', BUILD_NAME='debug-compile-aws'),
719 func('fetch-det'),
720 func(
721 'bootstrap-mongo-orchestration',
722 AUTH='auth',
723 ORCHESTRATION_FILE='auth-aws.json',
724 MONGODB_VERSION=self.settings.version,
725 TOPOLOGY='server',
726 ),
727 func('run aws tests', TESTCASE=str(self.settings.testcase).upper()),
728 ]
729
730 @property
731 def name(self):
732 return f'{self.name_prefix}-{self.settings.testcase}-{self.settings.version}'
733
734
735 all_tasks = chain(all_tasks, AWSTestTask.matrix())
736
737
738 class OCSPTask(MatrixTask):
739 axes = OD(
740 [
741 (
742 'test',
743 [
744 'test_1',
745 'test_2',
746 'test_3',
747 'test_4',
748 'soft_fail_test',
749 'malicious_server_test_1',
750 'malicious_server_test_2',
751 'cache',
752 ],
753 ),
754 ('delegate', ['delegate', 'nodelegate']),
755 ('cert', ['rsa', 'ecdsa']),
756 ('ssl', ['openssl', 'darwinssl', 'winssl']),
757 ('version', ['latest', 'rapid', '8.0', '7.0', '6.0', '5.0', '4.4']),
758 ]
759 )
760
761 name_prefix = 'test-ocsp'
762
763 @property
764 def build_task_name(self) -> str:
765 return f'debug-compile-nosasl-{self.display("ssl")}'
766
767 def additional_tags(self) -> Iterable[str]:
768 yield from super().additional_tags()
769 yield f'ocsp-{self.display("ssl")}'
770
771 def additional_dependencies(self) -> Iterable[DependencySpec]:
772 yield self.build_task_name
773
774 @property
775 def name(self):
776 return f'ocsp-{self.settings.ssl}-{self.test}-{self.settings.cert}-{self.settings.delegate}-{self.settings.version}'
777
778 @property
779 def test(self) -> str:
780 return str(self.settings.test)
781
782 def post_commands(self) -> Iterable[Value]:
783 yield func('fetch-build', BUILD_NAME=self.build_task_name)
784 yield func('fetch-det')
785
786 stapling = 'mustStaple'
787 if self.test in ['test_3', 'test_4', 'soft_fail_test', 'cache']:
788 stapling = 'disableStapling'
789 if self.test in ['malicious_server_test_1', 'malicious_server_test_2']:
790 stapling = 'mustStaple-disableStapling'
791
792 orchestration = func(
793 'bootstrap-mongo-orchestration',
794 MONGODB_VERSION=self.settings.version,
795 TOPOLOGY='server',
796 SSL='ssl',
797 OCSP='on',
798 ORCHESTRATION_FILE=f'{self.settings.cert}-basic-tls-ocsp-{stapling}.json',
799 )
800
801 # The cache test expects a revoked response from an OCSP responder, exactly like TEST_4.
802 test_column = 'TEST_4' if self.test == 'cache' else str(self.test).upper()
803 use_delegate = 'ON' if self.settings.delegate == 'delegate' else 'OFF'
804
805 yield (
806 shell_mongoc(
807 f"""
808 TEST_COLUMN={test_column} CERT_TYPE={self.settings.cert} USE_DELEGATE={use_delegate} .evergreen/scripts/run-ocsp-responder.sh
809 """
810 )
811 )
812
813 yield (orchestration)
814
815 if self.test == 'cache':
816 yield (
817 shell_mongoc(
818 f"""
819 CERT_TYPE={self.settings.cert} .evergreen/scripts/run-ocsp-cache-test.sh
820 """,
821 redirect_standard_error_to_output=True,
822 )
823 )
824 else:
825 yield (
826 shell_mongoc(
827 f"""
828 TEST_COLUMN={self.test.upper()} CERT_TYPE={self.settings.cert} .evergreen/scripts/run-ocsp-test.sh
829 """,
830 redirect_standard_error_to_output=True,
831 )
832 )
833
834 def to_dict(self):
835 task = super(MatrixTask, self).to_dict()
836
837 # OCSP tests should run with a batchtime of 14 days. Avoid running OCSP
838 # tests in patch builds by default (only in commit builds).
839 task['patchable'] = False
840
841 return task
842
843 # Testing in OCSP has a lot of exceptions.
844 def do_is_valid_combination(self) -> bool:
845 if self.settings.ssl == 'darwinssl':
846 # Secure Transport quietly ignores a must-staple certificate with no stapled response.
847 prohibit(self.test == 'malicious_server_test_2')
848
849 # Why does this fail with Secure Transport (CSSMERR_TP_CERT_SUSPENDED)...?
850 prohibit(self.test == 'test_3')
851
852 # CDRIVER-3759: Secure Transport does not implement soft failure?
853 prohibit(self.test == 'soft_fail_test')
854
855 # Only Server 6.0+ are available on MacOS ARM64.
856 if self.settings.version not in ['rapid', 'latest']:
857 prohibit(Version(self.settings.version) < Version('6.0'))
858
859 if self.settings.ssl == 'darwinssl' or self.settings.ssl == 'winssl':
860 # ECDSA certs can't be loaded (in the PEM format they're stored) on Windows/macOS. Skip them.
861 prohibit(self.settings.cert == 'ecdsa')
862
863 # OCSP stapling is not supported on macOS or Windows.
864 prohibit(self.test in ['test_1', 'test_2', 'cache'])
865
866 if self.test == 'soft_fail_test' or self.test == 'malicious_server_test_2' or self.test == 'cache':
867 prohibit(self.settings.delegate == 'delegate')
868
869 return True
870
871
872 all_tasks = chain(all_tasks, OCSPTask.matrix())
873
874 all_tasks = list(all_tasks)