Debian Codegraph @

dirsrvtests/tests/stress/replication/mmr_01_4m-2h-4c_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2022 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import os
10 import sys
11 import time
12 import datetime
13 import ldap
14 import logging
15 import pytest
16 import threading
17 from lib389._constants import *
18 from lib389.properties import *
19 from lib389.tasks import *
20 from lib389.utils import *
21 from lib389.repltools import ReplTools
22
23 pytestmark = pytest.mark.tier3
24
25 logging.getLogger(__name__).setLevel(logging.INFO)
26 log = logging.getLogger(__name__)
27
28 DEBUGGING = False
29 ADD_DEL_COUNT = 5000
30 MAX_LOOPS = 5
31 TEST_CONVERGE_LATENCY = True
32 CONVERGENCE_TIMEOUT = '60'
33 supplier_list = []
34 hub_list = []
35 con_list = []
36 TEST_START = time.time()
37
38 LAST_DN_IDX = ADD_DEL_COUNT - 1
39 LAST_DN_M1 = 'DEL dn="uid=supplier_1-%d,%s' % (LAST_DN_IDX, DEFAULT_SUFFIX)
40 LAST_DN_M2 = 'DEL dn="uid=supplier_2-%d,%s' % (LAST_DN_IDX, DEFAULT_SUFFIX)
41 LAST_DN_M3 = 'DEL dn="uid=supplier_3-%d,%s' % (LAST_DN_IDX, DEFAULT_SUFFIX)
42 LAST_DN_M4 = 'DEL dn="uid=supplier_4-%d,%s' % (LAST_DN_IDX, DEFAULT_SUFFIX)
43
44
45 class TopologyReplication(object):
46 """The Replication Topology Class"""
47 def __init__(self, supplier1, supplier2, supplier3, supplier4, hub1, hub2,
48 consumer1, consumer2, consumer3, consumer4):
49 """Init"""
50 supplier1.open()
51 self.supplier1 = supplier1
52 supplier2.open()
53 self.supplier2 = supplier2
54 supplier3.open()
55 self.supplier3 = supplier3
56 supplier4.open()
57 self.supplier4 = supplier4
58 hub1.open()
59 self.hub1 = hub1
60 hub2.open()
61 self.hub2 = hub2
62 consumer1.open()
63 self.consumer1 = consumer1
64 consumer2.open()
65 self.consumer2 = consumer2
66 consumer3.open()
67 self.consumer3 = consumer3
68 consumer4.open()
69 self.consumer4 = consumer4
70 supplier_list.append(supplier1.serverid)
71 supplier_list.append(supplier2.serverid)
72 supplier_list.append(supplier3.serverid)
73 supplier_list.append(supplier4.serverid)
74 hub_list.append(hub1.serverid)
75 hub_list.append(hub2.serverid)
76 con_list.append(consumer1.serverid)
77 con_list.append(consumer2.serverid)
78 con_list.append(consumer3.serverid)
79 con_list.append(consumer4.serverid)
80
81
82 @pytest.fixture(scope="module")
83 def topology(request):
84 """Create Replication Deployment"""
85
86 # Creating supplier 1...
87 if DEBUGGING:
88 supplier1 = DirSrv(verbose=True)
89 else:
90 supplier1 = DirSrv(verbose=False)
91 args_instance[SER_HOST] = HOST_SUPPLIER_1
92 args_instance[SER_PORT] = PORT_SUPPLIER_1
93 args_instance[SER_SERVERID_PROP] = SERVERID_SUPPLIER_1
94 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
95 args_supplier = args_instance.copy()
96 supplier1.allocate(args_supplier)
97 instance_supplier1 = supplier1.exists()
98 if instance_supplier1:
99 supplier1.delete()
100 supplier1.create()
101 supplier1.open()
102 supplier1.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.SUPPLIER,
103 replicaId=REPLICAID_SUPPLIER_1)
104
105 # Creating supplier 2...
106 if DEBUGGING:
107 supplier2 = DirSrv(verbose=True)
108 else:
109 supplier2 = DirSrv(verbose=False)
110 args_instance[SER_HOST] = HOST_SUPPLIER_2
111 args_instance[SER_PORT] = PORT_SUPPLIER_2
112 args_instance[SER_SERVERID_PROP] = SERVERID_SUPPLIER_2
113 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
114 args_supplier = args_instance.copy()
115 supplier2.allocate(args_supplier)
116 instance_supplier2 = supplier2.exists()
117 if instance_supplier2:
118 supplier2.delete()
119 supplier2.create()
120 supplier2.open()
121 supplier2.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.SUPPLIER,
122 replicaId=REPLICAID_SUPPLIER_2)
123
124 # Creating supplier 3...
125 if DEBUGGING:
126 supplier3 = DirSrv(verbose=True)
127 else:
128 supplier3 = DirSrv(verbose=False)
129 args_instance[SER_HOST] = HOST_SUPPLIER_3
130 args_instance[SER_PORT] = PORT_SUPPLIER_3
131 args_instance[SER_SERVERID_PROP] = SERVERID_SUPPLIER_3
132 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
133 args_supplier = args_instance.copy()
134 supplier3.allocate(args_supplier)
135 instance_supplier3 = supplier3.exists()
136 if instance_supplier3:
137 supplier3.delete()
138 supplier3.create()
139 supplier3.open()
140 supplier3.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.SUPPLIER,
141 replicaId=REPLICAID_SUPPLIER_3)
142
143 # Creating supplier 4...
144 if DEBUGGING:
145 supplier4 = DirSrv(verbose=True)
146 else:
147 supplier4 = DirSrv(verbose=False)
148 args_instance[SER_HOST] = HOST_SUPPLIER_4
149 args_instance[SER_PORT] = PORT_SUPPLIER_4
150 args_instance[SER_SERVERID_PROP] = SERVERID_SUPPLIER_4
151 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
152 args_supplier = args_instance.copy()
153 supplier4.allocate(args_supplier)
154 instance_supplier4 = supplier4.exists()
155 if instance_supplier4:
156 supplier4.delete()
157 supplier4.create()
158 supplier4.open()
159 supplier4.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.SUPPLIER,
160 replicaId=REPLICAID_SUPPLIER_4)
161
162 # Creating hub 1...
163 if DEBUGGING:
164 hub1 = DirSrv(verbose=True)
165 else:
166 hub1 = DirSrv(verbose=False)
167 args_instance[SER_HOST] = HOST_HUB_1
168 args_instance[SER_PORT] = PORT_HUB_1
169 args_instance[SER_SERVERID_PROP] = SERVERID_HUB_1
170 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
171 args_hub = args_instance.copy()
172 hub1.allocate(args_hub)
173 instance_hub1 = hub1.exists()
174 if instance_hub1:
175 hub1.delete()
176 hub1.create()
177 hub1.open()
178 hub1.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.HUB,
179 replicaId=REPLICAID_HUB_1)
180
181 # Creating hub 2...
182 if DEBUGGING:
183 hub2 = DirSrv(verbose=True)
184 else:
185 hub2 = DirSrv(verbose=False)
186 args_instance[SER_HOST] = HOST_HUB_2
187 args_instance[SER_PORT] = PORT_HUB_2
188 args_instance[SER_SERVERID_PROP] = SERVERID_HUB_2
189 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
190 args_hub = args_instance.copy()
191 hub2.allocate(args_hub)
192 instance_hub2 = hub2.exists()
193 if instance_hub2:
194 hub2.delete()
195 hub2.create()
196 hub2.open()
197 hub2.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.HUB,
198 replicaId=REPLICAID_HUB_2)
199
200 # Creating consumer 1...
201 if DEBUGGING:
202 consumer1 = DirSrv(verbose=True)
203 else:
204 consumer1 = DirSrv(verbose=False)
205 args_instance[SER_HOST] = HOST_CONSUMER_1
206 args_instance[SER_PORT] = PORT_CONSUMER_1
207 args_instance[SER_SERVERID_PROP] = SERVERID_CONSUMER_1
208 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
209 args_consumer = args_instance.copy()
210 consumer1.allocate(args_consumer)
211 instance_consumer1 = consumer1.exists()
212 if instance_consumer1:
213 consumer1.delete()
214 consumer1.create()
215 consumer1.open()
216 consumer1.replica.enableReplication(suffix=SUFFIX,
217 role=ReplicaRole.CONSUMER,
218 replicaId=CONSUMER_REPLICAID)
219
220 # Creating consumer 2...
221 if DEBUGGING:
222 consumer2 = DirSrv(verbose=True)
223 else:
224 consumer2 = DirSrv(verbose=False)
225 args_instance[SER_HOST] = HOST_CONSUMER_2
226 args_instance[SER_PORT] = PORT_CONSUMER_2
227 args_instance[SER_SERVERID_PROP] = SERVERID_CONSUMER_2
228 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
229 args_consumer = args_instance.copy()
230 consumer2.allocate(args_consumer)
231 instance_consumer2 = consumer2.exists()
232 if instance_consumer2:
233 consumer2.delete()
234 consumer2.create()
235 consumer2.open()
236 consumer2.replica.enableReplication(suffix=SUFFIX,
237 role=ReplicaRole.CONSUMER,
238 replicaId=CONSUMER_REPLICAID)
239
240 # Creating consumer 3...
241 if DEBUGGING:
242 consumer3 = DirSrv(verbose=True)
243 else:
244 consumer3 = DirSrv(verbose=False)
245 args_instance[SER_HOST] = HOST_CONSUMER_3
246 args_instance[SER_PORT] = PORT_CONSUMER_3
247 args_instance[SER_SERVERID_PROP] = SERVERID_CONSUMER_3
248 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
249 args_consumer = args_instance.copy()
250 consumer3.allocate(args_consumer)
251 instance_consumer3 = consumer3.exists()
252 if instance_consumer3:
253 consumer3.delete()
254 consumer3.create()
255 consumer3.open()
256 consumer3.replica.enableReplication(suffix=SUFFIX,
257 role=ReplicaRole.CONSUMER,
258 replicaId=CONSUMER_REPLICAID)
259
260 # Creating consumer 4...
261 if DEBUGGING:
262 consumer4 = DirSrv(verbose=True)
263 else:
264 consumer4 = DirSrv(verbose=False)
265 args_instance[SER_HOST] = HOST_CONSUMER_4
266 args_instance[SER_PORT] = PORT_CONSUMER_4
267 args_instance[SER_SERVERID_PROP] = SERVERID_CONSUMER_4
268 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
269 args_consumer = args_instance.copy()
270 consumer4.allocate(args_consumer)
271 instance_consumer4 = consumer4.exists()
272 if instance_consumer4:
273 consumer4.delete()
274 consumer4.create()
275 consumer4.open()
276 consumer4.replica.enableReplication(suffix=SUFFIX,
277 role=ReplicaRole.CONSUMER,
278 replicaId=CONSUMER_REPLICAID)
279
280 #
281 # Create all the agreements
282 #
283
284 # Creating agreement from supplier 1 to supplier 2
285 properties = {RA_NAME: 'meTo_' + supplier2.host + ':' + str(supplier2.port),
286 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
287 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
288 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
289 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
290 m1_m2_agmt = supplier1.agreement.create(suffix=SUFFIX, host=supplier2.host,
291 port=supplier2.port,
292 properties=properties)
293 if not m1_m2_agmt:
294 log.fatal("Fail to create a supplier -> supplier replica agreement")
295 sys.exit(1)
296 log.debug("%s created" % m1_m2_agmt)
297
298 # Creating agreement from supplier 1 to supplier 3
299 properties = {RA_NAME: 'meTo_' + supplier3.host + ':' + str(supplier3.port),
300 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
301 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
302 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
303 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
304 m1_m3_agmt = supplier1.agreement.create(suffix=SUFFIX, host=supplier3.host,
305 port=supplier3.port,
306 properties=properties)
307 if not m1_m3_agmt:
308 log.fatal("Fail to create a supplier -> supplier replica agreement")
309 sys.exit(1)
310 log.debug("%s created" % m1_m3_agmt)
311
312 # Creating agreement from supplier 1 to supplier 4
313 properties = {RA_NAME: 'meTo_' + supplier4.host + ':' + str(supplier4.port),
314 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
315 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
316 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
317 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
318 m1_m4_agmt = supplier1.agreement.create(suffix=SUFFIX, host=supplier4.host,
319 port=supplier4.port,
320 properties=properties)
321 if not m1_m4_agmt:
322 log.fatal("Fail to create a supplier -> supplier replica agreement")
323 sys.exit(1)
324 log.debug("%s created" % m1_m4_agmt)
325
326 # Creating agreement from supplier 1 to hub 1
327 properties = {RA_NAME: 'meTo_' + hub1.host + ':' + str(hub1.port),
328 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
329 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
330 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
331 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
332 m1_h1_agmt = supplier1.agreement.create(suffix=SUFFIX, host=hub1.host,
333 port=hub1.port,
334 properties=properties)
335 if not m1_h1_agmt:
336 log.fatal("Fail to create a supplier -> hub replica agreement")
337 sys.exit(1)
338 log.debug("%s created" % m1_h1_agmt)
339
340 # Creating agreement from supplier 1 to hub 2
341 properties = {RA_NAME: 'meTo_' + hub2.host + ':' + str(hub2.port),
342 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
343 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
344 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
345 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
346 m1_h2_agmt = supplier1.agreement.create(suffix=SUFFIX, host=hub2.host,
347 port=hub2.port,
348 properties=properties)
349 if not m1_h2_agmt:
350 log.fatal("Fail to create a supplier -> hub replica agreement")
351 sys.exit(1)
352 log.debug("%s created" % m1_h2_agmt)
353
354 # Creating agreement from supplier 2 to supplier 1
355 properties = {RA_NAME: 'meTo_' + supplier1.host + ':' + str(supplier1.port),
356 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
357 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
358 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
359 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
360 m2_m1_agmt = supplier2.agreement.create(suffix=SUFFIX, host=supplier1.host,
361 port=supplier1.port,
362 properties=properties)
363 if not m2_m1_agmt:
364 log.fatal("Fail to create a supplier -> supplier replica agreement")
365 sys.exit(1)
366 log.debug("%s created" % m2_m1_agmt)
367
368 # Creating agreement from supplier 2 to supplier 3
369 properties = {RA_NAME: 'meTo_' + supplier3.host + ':' + str(supplier3.port),
370 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
371 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
372 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
373 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
374 m2_m3_agmt = supplier2.agreement.create(suffix=SUFFIX, host=supplier3.host,
375 port=supplier3.port,
376 properties=properties)
377 if not m2_m3_agmt:
378 log.fatal("Fail to create a supplier -> supplier replica agreement")
379 sys.exit(1)
380 log.debug("%s created" % m2_m3_agmt)
381
382 # Creating agreement from supplier 2 to supplier 4
383 properties = {RA_NAME: 'meTo_' + supplier4.host + ':' + str(supplier4.port),
384 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
385 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
386 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
387 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
388 m2_m4_agmt = supplier2.agreement.create(suffix=SUFFIX, host=supplier4.host,
389 port=supplier4.port,
390 properties=properties)
391 if not m2_m4_agmt:
392 log.fatal("Fail to create a supplier -> supplier replica agreement")
393 sys.exit(1)
394 log.debug("%s created" % m2_m4_agmt)
395
396 # Creating agreement from supplier 2 to hub 1
397 properties = {RA_NAME: 'meTo_' + hub1.host + ':' + str(hub1.port),
398 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
399 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
400 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
401 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
402 m2_h1_agmt = supplier2.agreement.create(suffix=SUFFIX, host=hub1.host,
403 port=hub1.port,
404 properties=properties)
405 if not m2_h1_agmt:
406 log.fatal("Fail to create a supplier -> hub replica agreement")
407 sys.exit(1)
408 log.debug("%s created" % m2_h1_agmt)
409
410 # Creating agreement from supplier 2 to hub 2
411 properties = {RA_NAME: 'meTo_' + hub2.host + ':' + str(hub2.port),
412 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
413 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
414 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
415 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
416 m2_h2_agmt = supplier2.agreement.create(suffix=SUFFIX, host=hub2.host,
417 port=hub2.port,
418 properties=properties)
419 if not m2_h2_agmt:
420 log.fatal("Fail to create a supplier -> hub replica agreement")
421 sys.exit(1)
422 log.debug("%s created" % m2_h2_agmt)
423
424 # Creating agreement from supplier 3 to supplier 1
425 properties = {RA_NAME: 'meTo_' + supplier1.host + ':' + str(supplier1.port),
426 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
427 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
428 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
429 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
430 m3_m1_agmt = supplier3.agreement.create(suffix=SUFFIX, host=supplier1.host,
431 port=supplier1.port,
432 properties=properties)
433 if not m3_m1_agmt:
434 log.fatal("Fail to create a supplier -> supplier replica agreement")
435 sys.exit(1)
436 log.debug("%s created" % m3_m1_agmt)
437
438 # Creating agreement from supplier 3 to supplier 2
439 properties = {RA_NAME: 'meTo_' + supplier2.host + ':' + str(supplier2.port),
440 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
441 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
442 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
443 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
444 m3_m2_agmt = supplier3.agreement.create(suffix=SUFFIX, host=supplier2.host,
445 port=supplier2.port,
446 properties=properties)
447 if not m3_m2_agmt:
448 log.fatal("Fail to create a supplier -> supplier replica agreement")
449 sys.exit(1)
450 log.debug("%s created" % m3_m2_agmt)
451
452 # Creating agreement from supplier 3 to supplier 4
453 properties = {RA_NAME: 'meTo_' + supplier4.host + ':' + str(supplier4.port),
454 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
455 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
456 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
457 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
458 m3_m4_agmt = supplier3.agreement.create(suffix=SUFFIX, host=supplier4.host,
459 port=supplier4.port,
460 properties=properties)
461 if not m3_m4_agmt:
462 log.fatal("Fail to create a supplier -> supplier replica agreement")
463 sys.exit(1)
464 log.debug("%s created" % m3_m4_agmt)
465
466 # Creating agreement from supplier 3 to hub 1
467 properties = {RA_NAME: 'meTo_' + hub1.host + ':' + str(hub1.port),
468 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
469 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
470 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
471 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
472 m3_h1_agmt = supplier3.agreement.create(suffix=SUFFIX, host=hub1.host,
473 port=hub1.port,
474 properties=properties)
475 if not m3_h1_agmt:
476 log.fatal("Fail to create a supplier -> hub replica agreement")
477 sys.exit(1)
478 log.debug("%s created" % m3_h1_agmt)
479
480 # Creating agreement from supplier 3 to hub 2
481 properties = {RA_NAME: 'meTo_' + hub2.host + ':' + str(hub2.port),
482 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
483 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
484 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
485 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
486 m3_h2_agmt = supplier3.agreement.create(suffix=SUFFIX, host=hub2.host,
487 port=hub2.port,
488 properties=properties)
489 if not m3_h2_agmt:
490 log.fatal("Fail to create a supplier -> hub replica agreement")
491 sys.exit(1)
492 log.debug("%s created" % m3_h2_agmt)
493
494 # Creating agreement from supplier 4 to supplier 1
495 properties = {RA_NAME: 'meTo_' + supplier1.host + ':' + str(supplier1.port),
496 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
497 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
498 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
499 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
500 m4_m1_agmt = supplier4.agreement.create(suffix=SUFFIX, host=supplier1.host,
501 port=supplier1.port,
502 properties=properties)
503 if not m4_m1_agmt:
504 log.fatal("Fail to create a supplier -> supplier replica agreement")
505 sys.exit(1)
506 log.debug("%s created" % m4_m1_agmt)
507
508 # Creating agreement from supplier 4 to supplier 2
509 properties = {RA_NAME: 'meTo_' + supplier2.host + ':' + str(supplier2.port),
510 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
511 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
512 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
513 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
514 m4_m2_agmt = supplier4.agreement.create(suffix=SUFFIX, host=supplier2.host,
515 port=supplier2.port,
516 properties=properties)
517 if not m4_m2_agmt:
518 log.fatal("Fail to create a supplier -> supplier replica agreement")
519 sys.exit(1)
520 log.debug("%s created" % m4_m2_agmt)
521
522 # Creating agreement from supplier 4 to supplier 3
523 properties = {RA_NAME: 'meTo_' + supplier3.host + ':' + str(supplier3.port),
524 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
525 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
526 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
527 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
528 m4_m3_agmt = supplier4.agreement.create(suffix=SUFFIX, host=supplier3.host,
529 port=supplier3.port,
530 properties=properties)
531 if not m4_m3_agmt:
532 log.fatal("Fail to create a supplier -> supplier replica agreement")
533 sys.exit(1)
534 log.debug("%s created" % m4_m3_agmt)
535
536 # Creating agreement from supplier 4 to hub 1
537 properties = {RA_NAME: 'meTo_' + hub1.host + ':' + str(hub1.port),
538 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
539 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
540 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
541 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
542 m4_h1_agmt = supplier4.agreement.create(suffix=SUFFIX, host=hub1.host,
543 port=hub1.port,
544 properties=properties)
545 if not m4_h1_agmt:
546 log.fatal("Fail to create a supplier -> hub replica agreement")
547 sys.exit(1)
548 log.debug("%s created" % m4_h1_agmt)
549
550 # Creating agreement from supplier 4 to hub 2
551 properties = {RA_NAME: 'meTo_' + hub2.host + ':' + str(hub2.port),
552 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
553 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
554 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
555 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
556 m4_h2_agmt = supplier4.agreement.create(suffix=SUFFIX, host=hub2.host,
557 port=hub2.port,
558 properties=properties)
559 if not m4_h2_agmt:
560 log.fatal("Fail to create a supplier -> hub replica agreement")
561 sys.exit(1)
562 log.debug("%s created" % m4_h2_agmt)
563
564 # Creating agreement from hub 1 to consumer 1
565 properties = {RA_NAME: 'me2_' + consumer1.host + ':' + str(consumer1.port),
566 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
567 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
568 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
569 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
570 h1_c1_agmt = hub1.agreement.create(suffix=SUFFIX, host=consumer1.host,
571 port=consumer1.port,
572 properties=properties)
573 if not h1_c1_agmt:
574 log.fatal("Fail to create a hub -> consumer replica agreement")
575 sys.exit(1)
576 log.debug("%s created" % h1_c1_agmt)
577
578 # Creating agreement from hub 1 to consumer 2
579 properties = {RA_NAME: 'me2_' + consumer2.host + ':' + str(consumer2.port),
580 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
581 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
582 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
583 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
584 h1_c2_agmt = hub1.agreement.create(suffix=SUFFIX, host=consumer2.host,
585 port=consumer2.port,
586 properties=properties)
587 if not h1_c2_agmt:
588 log.fatal("Fail to create a hub -> consumer replica agreement")
589 sys.exit(1)
590 log.debug("%s created" % h1_c2_agmt)
591
592 # Creating agreement from hub 1 to consumer 3
593 properties = {RA_NAME: 'me2_' + consumer3.host + ':' + str(consumer3.port),
594 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
595 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
596 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
597 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
598 h1_c3_agmt = hub1.agreement.create(suffix=SUFFIX, host=consumer3.host,
599 port=consumer3.port,
600 properties=properties)
601 if not h1_c3_agmt:
602 log.fatal("Fail to create a hub -> consumer replica agreement")
603 sys.exit(1)
604 log.debug("%s created" % h1_c3_agmt)
605
606 # Creating agreement from hub 1 to consumer 4
607 properties = {RA_NAME: 'me2_' + consumer4.host + ':' + str(consumer4.port),
608 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
609 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
610 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
611 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
612 h1_c4_agmt = hub1.agreement.create(suffix=SUFFIX, host=consumer4.host,
613 port=consumer4.port,
614 properties=properties)
615 if not h1_c4_agmt:
616 log.fatal("Fail to create a hub -> consumer replica agreement")
617 sys.exit(1)
618 log.debug("%s created" % h1_c4_agmt)
619
620 # Creating agreement from hub 2 to consumer 1
621 properties = {RA_NAME: 'me2_' + consumer1.host + ':' + str(consumer1.port),
622 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
623 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
624 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
625 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
626 h2_c1_agmt = hub2.agreement.create(suffix=SUFFIX, host=consumer1.host,
627 port=consumer1.port,
628 properties=properties)
629 if not h2_c1_agmt:
630 log.fatal("Fail to create a hub -> consumer replica agreement")
631 sys.exit(1)
632 log.debug("%s created" % h2_c1_agmt)
633
634 # Creating agreement from hub 2 to consumer 2
635 properties = {RA_NAME: 'me2_' + consumer2.host + ':' + str(consumer2.port),
636 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
637 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
638 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
639 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
640 h2_c2_agmt = hub2.agreement.create(suffix=SUFFIX, host=consumer2.host,
641 port=consumer2.port,
642 properties=properties)
643 if not h2_c2_agmt:
644 log.fatal("Fail to create a hub -> consumer replica agreement")
645 sys.exit(1)
646 log.debug("%s created" % h2_c2_agmt)
647
648 # Creating agreement from hub 2 to consumer 3
649 properties = {RA_NAME: 'me2_' + consumer3.host + ':' + str(consumer3.port),
650 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
651 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
652 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
653 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
654 h2_c3_agmt = hub2.agreement.create(suffix=SUFFIX, host=consumer3.host,
655 port=consumer3.port,
656 properties=properties)
657 if not h2_c3_agmt:
658 log.fatal("Fail to create a hub -> consumer replica agreement")
659 sys.exit(1)
660 log.debug("%s created" % h2_c3_agmt)
661
662 # Creating agreement from hub 2 to consumer 4
663 properties = {RA_NAME: 'me2_' + consumer4.host + ':' + str(consumer4.port),
664 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
665 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
666 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
667 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
668 h2_c4_agmt = hub2.agreement.create(suffix=SUFFIX, host=consumer4.host,
669 port=consumer4.port,
670 properties=properties)
671 if not h2_c4_agmt:
672 log.fatal("Fail to create a hub -> consumer replica agreement")
673 sys.exit(1)
674 log.debug("%s created" % h2_c4_agmt)
675
676 # Allow the replicas to get situated with the new agreements...
677 time.sleep(5)
678
679 #
680 # Initialize all the agreements
681 #
682 supplier1.agreement.init(SUFFIX, HOST_SUPPLIER_2, PORT_SUPPLIER_2)
683 supplier1.waitForReplInit(m1_m2_agmt)
684 supplier1.agreement.init(SUFFIX, HOST_SUPPLIER_3, PORT_SUPPLIER_3)
685 supplier1.waitForReplInit(m1_m3_agmt)
686 supplier1.agreement.init(SUFFIX, HOST_SUPPLIER_4, PORT_SUPPLIER_4)
687 supplier1.waitForReplInit(m1_m4_agmt)
688 supplier1.agreement.init(SUFFIX, HOST_HUB_1, PORT_HUB_1)
689 supplier1.waitForReplInit(m1_h1_agmt)
690 hub1.agreement.init(SUFFIX, HOST_CONSUMER_1, PORT_CONSUMER_1)
691 hub1.waitForReplInit(h1_c1_agmt)
692 hub1.agreement.init(SUFFIX, HOST_CONSUMER_2, PORT_CONSUMER_2)
693 hub1.waitForReplInit(h1_c2_agmt)
694 hub1.agreement.init(SUFFIX, HOST_CONSUMER_3, PORT_CONSUMER_3)
695 hub1.waitForReplInit(h1_c3_agmt)
696 hub1.agreement.init(SUFFIX, HOST_CONSUMER_4, PORT_CONSUMER_4)
697 hub1.waitForReplInit(h1_c4_agmt)
698 supplier1.agreement.init(SUFFIX, HOST_HUB_2, PORT_HUB_2)
699 supplier1.waitForReplInit(m1_h2_agmt)
700
701 # Check replication is working...
702 if supplier1.testReplication(DEFAULT_SUFFIX, consumer1):
703 log.info('Replication is working.')
704 else:
705 log.fatal('Replication is not working.')
706 assert False
707
708 def fin():
709 """If we are debugging just stop the instances, otherwise remove
710 them
711 """
712 if DEBUGGING:
713 supplier1.stop()
714 supplier2.stop()
715 supplier3.stop()
716 supplier4.stop()
717 hub1.stop()
718 hub2.stop()
719 consumer1.stop()
720 consumer2.stop()
721 consumer3.stop()
722 consumer4.stop()
723 else:
724 supplier1.delete()
725 supplier2.delete()
726 supplier3.delete()
727 supplier4.delete()
728 hub1.delete()
729 hub2.delete()
730 consumer1.delete()
731 consumer2.delete()
732 consumer3.delete()
733 consumer4.delete()
734 request.addfinalizer(fin)
735
736 return TopologyReplication(supplier1, supplier2, supplier3, supplier4, hub1, hub2,
737 consumer1, consumer2, consumer3, consumer4)
738
739
740 class AddDelUsers(threading.Thread):
741 """Add's and delets 50000 entries"""
742 def __init__(self, inst):
743 """
744 Initialize the thread
745 """
746 threading.Thread.__init__(self)
747 self.daemon = True
748 self.inst = inst
749 self.name = inst.serverid
750
751 def run(self):
752 """
753 Start adding users
754 """
755 idx = 0
756
757 log.info('AddDelUsers (%s) Adding and deleting %d entries...' %
758 (self.name, ADD_DEL_COUNT))
759
760 while idx < ADD_DEL_COUNT:
761 RDN_VAL = ('uid=%s-%d' % (self.name, idx))
762 USER_DN = ('%s,%s' % (RDN_VAL, DEFAULT_SUFFIX))
763
764 try:
765 self.inst.add_s(Entry((USER_DN, {'objectclass':
766 'top extensibleObject'.split(),
767 'uid': RDN_VAL})))
768 except ldap.LDAPError as e:
769 log.fatal('AddDelUsers (%s): failed to add (%s) error: %s' %
770 (self.name, USER_DN, str(e)))
771 assert False
772
773 try:
774 self.inst.delete_s(USER_DN)
775 except ldap.LDAPError as e:
776 log.fatal('AddDelUsers (%s): failed to delete (%s) error: %s' %
777 (self.name, USER_DN, str(e)))
778 assert False
779
780 idx += 1
781
782 log.info('AddDelUsers (%s) - Finished at: %s' %
783 (self.name, getDateTime()))
784
785
786 def measureConvergence(topology):
787 """Find and measure the convergence of entries from each supplier
788 """
789
790 replicas = [topology.supplier1, topology.supplier2, topology.supplier3,
791 topology.supplier4, topology.hub1, topology.hub2,
792 topology.consumer1, topology.consumer2, topology.consumer3,
793 topology.consumer4]
794
795 if ADD_DEL_COUNT > 10:
796 interval = int(ADD_DEL_COUNT / 10)
797 else:
798 interval = 1
799
800 for supplier in [('1', topology.supplier1),
801 ('2', topology.supplier2),
802 ('3', topology.supplier3),
803 ('4', topology.supplier4)]:
804 # Start with the first entry
805 entries = ['ADD dn="uid=supplier_%s-0,%s' %
806 (supplier[0], DEFAULT_SUFFIX)]
807
808 # Add incremental entries to the list
809 idx = interval
810 while idx < ADD_DEL_COUNT:
811 entries.append('ADD dn="uid=supplier_%s-%d,%s' %
812 (supplier[0], idx, DEFAULT_SUFFIX))
813 idx += interval
814
815 # Add the last entry to the list (if it was not already added)
816 if idx != (ADD_DEL_COUNT - 1):
817 entries.append('ADD dn="uid=supplier_%s-%d,%s' %
818 (supplier[0], (ADD_DEL_COUNT - 1),
819 DEFAULT_SUFFIX))
820
821 ReplTools.replConvReport(DEFAULT_SUFFIX, entries, supplier[1], replicas)
822
823
824 def test_MMR_Integrity(topology):
825 """Apply load to 4 suppliers at the same time. Perform adds and deletes.
826 If any updates are missed we will see an error 32 in the access logs or
827 we will have entries left over once the test completes.
828 """
829 loop = 0
830
831 ALL_REPLICAS = [topology.supplier1, topology.supplier2, topology.supplier3,
832 topology.supplier4,
833 topology.hub1, topology.hub2,
834 topology.consumer1, topology.consumer2,
835 topology.consumer3, topology.consumer4]
836
837 if TEST_CONVERGE_LATENCY:
838 try:
839 for inst in ALL_REPLICAS:
840 replica = inst.replicas.get(DEFAULT_SUFFIX)
841 replica.set('nsds5ReplicaReleaseTimeout', CONVERGENCE_TIMEOUT)
842 except ldap.LDAPError as e:
843 log.fatal('Failed to set replicas release timeout - error: %s' %
844 (str(e)))
845 assert False
846
847 if DEBUGGING:
848 # Enable Repl logging, and increase the max logs
849 try:
850 for inst in ALL_REPLICAS:
851 inst.enableReplLogging()
852 inst.modify_s("cn=config", [(ldap.MOD_REPLACE,
853 'nsslapd-errorlog-maxlogsperdir',
854 '5')])
855 except ldap.LDAPError as e:
856 log.fatal('Failed to set max logs - error: %s' % (str(e)))
857 assert False
858
859 while loop < MAX_LOOPS:
860 # Remove the current logs so we have a clean set of logs to check.
861 log.info('Pass %d...' % (loop + 1))
862 log.info("Removing logs...")
863 for inst in ALL_REPLICAS:
864 inst.deleteAllLogs()
865
866 # Fire off 4 threads to apply the load
867 log.info("Start adding/deleting: " + getDateTime())
868 startTime = time.time()
869 add_del_m1 = AddDelUsers(topology.supplier1)
870 add_del_m1.start()
871 add_del_m2 = AddDelUsers(topology.supplier2)
872 add_del_m2.start()
873 add_del_m3 = AddDelUsers(topology.supplier3)
874 add_del_m3.start()
875 add_del_m4 = AddDelUsers(topology.supplier4)
876 add_del_m4.start()
877
878 # Wait for threads to finish sending their updates
879 add_del_m1.join()
880 add_del_m2.join()
881 add_del_m3.join()
882 add_del_m4.join()
883 log.info("Finished adding/deleting entries: " + getDateTime())
884
885 #
886 # Loop checking for error 32's, and for convergence to complete
887 #
888 log.info("Waiting for replication to converge...")
889 while True:
890 # First check for error 32's
891 for inst in ALL_REPLICAS:
892 if inst.searchAccessLog(" err=32 "):
893 log.fatal('An add was missed on: ' + inst.serverid)
894 assert False
895
896 # Next check to see if the last update is in the access log
897 converged = True
898 for inst in ALL_REPLICAS:
899 if not inst.searchAccessLog(LAST_DN_M1) or \
900 not inst.searchAccessLog(LAST_DN_M2) or \
901 not inst.searchAccessLog(LAST_DN_M3) or \
902 not inst.searchAccessLog(LAST_DN_M4):
903 converged = False
904 break
905
906 if converged:
907 elapsed_tm = int(time.time() - startTime)
908 convtime = str(datetime.timedelta(seconds=elapsed_tm))
909 log.info('Replication converged at: ' + getDateTime() +
910 ' - Elapsed Time: ' + convtime)
911 break
912 else:
913 # Check if replication is idle
914 replicas = [topology.supplier1, topology.supplier2,
915 topology.supplier3, topology.supplier4,
916 topology.hub1, topology.hub2]
917 if ReplTools.replIdle(replicas, DEFAULT_SUFFIX):
918 # Replication is idle - wait 30 secs for access log buffer
919 time.sleep(30)
920
921 # Now check the access log again...
922 converged = True
923 for inst in ALL_REPLICAS:
924 if not inst.searchAccessLog(LAST_DN_M1) or \
925 not inst.searchAccessLog(LAST_DN_M2) or \
926 not inst.searchAccessLog(LAST_DN_M3) or \
927 not inst.searchAccessLog(LAST_DN_M4):
928 converged = False
929 break
930
931 if converged:
932 elapsed_tm = int(time.time() - startTime)
933 convtime = str(datetime.timedelta(seconds=elapsed_tm))
934 log.info('Replication converged at: ' + getDateTime() +
935 ' - Elapsed Time: ' + convtime)
936 break
937 else:
938 log.fatal('Stopping replication check: ' +
939 getDateTime())
940 log.fatal('Failure: Replication is complete, but we ' +
941 'never converged.')
942 assert False
943
944 # Sleep a bit before the next pass
945 time.sleep(3)
946
947 #
948 # Finally check the CSN's
949 #
950 log.info("Check the CSN's...")
951 if not ReplTools.checkCSNs(ALL_REPLICAS):
952 assert False
953 log.info("All CSN's present and accounted for.")
954
955 #
956 # Print the convergence report
957 #
958 log.info('Measuring convergence...')
959 measureConvergence(topology)
960
961 #
962 # Test complete
963 #
964 log.info('No lingering entries.')
965 log.info('Pass %d complete.' % (loop + 1))
966 elapsed_tm = int(time.time() - TEST_START)
967 convtime = str(datetime.timedelta(seconds=elapsed_tm))
968 log.info('Entire test ran for: ' + convtime)
969
970 loop += 1
971
972 log.info('Test PASSED')
973
974
975 if __name__ == '__main__':
976 # Run isolated
977 # -s for DEBUG mode
978 CURRENT_FILE = os.path.realpath(__file__)
979 pytest.main("-s %s" % CURRENT_FILE)