Debian Codegraph @

dirsrvtests/tests/stress/replication/mmr_01_4m_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2022 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import os
10 import sys
11 import time
12 import datetime
13 import ldap
14 import logging
15 import pytest
16 import threading
17 from lib389._constants import *
18 from lib389.properties import *
19 from lib389.tasks import *
20 from lib389.utils import *
21 from lib389.repltools import ReplTools
22
23 pytestmark = pytest.mark.tier3
24
25 logging.getLogger(__name__).setLevel(logging.DEBUG)
26 log = logging.getLogger(__name__)
27
28 DEBUGGING = False
29 ADD_DEL_COUNT = 50000
30 MAX_LOOPS = 2
31 TEST_CONVERGE_LATENCY = True
32 CONVERGENCE_TIMEOUT = '60'
33 supplier_list = []
34 hub_list = []
35 con_list = []
36 TEST_START = time.time()
37
38 LAST_DN_IDX = ADD_DEL_COUNT - 1
39 LAST_DN_M1 = 'DEL dn="uid=supplier_1-%d,%s' % (LAST_DN_IDX, DEFAULT_SUFFIX)
40 LAST_DN_M2 = 'DEL dn="uid=supplier_2-%d,%s' % (LAST_DN_IDX, DEFAULT_SUFFIX)
41 LAST_DN_M3 = 'DEL dn="uid=supplier_3-%d,%s' % (LAST_DN_IDX, DEFAULT_SUFFIX)
42 LAST_DN_M4 = 'DEL dn="uid=supplier_4-%d,%s' % (LAST_DN_IDX, DEFAULT_SUFFIX)
43
44
45 class TopologyReplication(object):
46 """The Replication Topology Class"""
47 def __init__(self, supplier1, supplier2, supplier3, supplier4):
48 """Init"""
49 supplier1.open()
50 self.supplier1 = supplier1
51 supplier2.open()
52 self.supplier2 = supplier2
53 supplier3.open()
54 self.supplier3 = supplier3
55 supplier4.open()
56 self.supplier4 = supplier4
57
58
59 @pytest.fixture(scope="module")
60 def topology(request):
61 """Create Replication Deployment"""
62
63 # Creating supplier 1...
64 if DEBUGGING:
65 supplier1 = DirSrv(verbose=True)
66 else:
67 supplier1 = DirSrv(verbose=False)
68 args_instance[SER_HOST] = HOST_SUPPLIER_1
69 args_instance[SER_PORT] = PORT_SUPPLIER_1
70 args_instance[SER_SERVERID_PROP] = SERVERID_SUPPLIER_1
71 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
72 args_supplier = args_instance.copy()
73 supplier1.allocate(args_supplier)
74 instance_supplier1 = supplier1.exists()
75 if instance_supplier1:
76 supplier1.delete()
77 supplier1.create()
78 supplier1.open()
79 supplier1.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.SUPPLIER,
80 replicaId=REPLICAID_SUPPLIER_1)
81
82 # Creating supplier 2...
83 if DEBUGGING:
84 supplier2 = DirSrv(verbose=True)
85 else:
86 supplier2 = DirSrv(verbose=False)
87 args_instance[SER_HOST] = HOST_SUPPLIER_2
88 args_instance[SER_PORT] = PORT_SUPPLIER_2
89 args_instance[SER_SERVERID_PROP] = SERVERID_SUPPLIER_2
90 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
91 args_supplier = args_instance.copy()
92 supplier2.allocate(args_supplier)
93 instance_supplier2 = supplier2.exists()
94 if instance_supplier2:
95 supplier2.delete()
96 supplier2.create()
97 supplier2.open()
98 supplier2.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.SUPPLIER,
99 replicaId=REPLICAID_SUPPLIER_2)
100
101 # Creating supplier 3...
102 if DEBUGGING:
103 supplier3 = DirSrv(verbose=True)
104 else:
105 supplier3 = DirSrv(verbose=False)
106 args_instance[SER_HOST] = HOST_SUPPLIER_3
107 args_instance[SER_PORT] = PORT_SUPPLIER_3
108 args_instance[SER_SERVERID_PROP] = SERVERID_SUPPLIER_3
109 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
110 args_supplier = args_instance.copy()
111 supplier3.allocate(args_supplier)
112 instance_supplier3 = supplier3.exists()
113 if instance_supplier3:
114 supplier3.delete()
115 supplier3.create()
116 supplier3.open()
117 supplier3.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.SUPPLIER,
118 replicaId=REPLICAID_SUPPLIER_3)
119
120 # Creating supplier 4...
121 if DEBUGGING:
122 supplier4 = DirSrv(verbose=True)
123 else:
124 supplier4 = DirSrv(verbose=False)
125 args_instance[SER_HOST] = HOST_SUPPLIER_4
126 args_instance[SER_PORT] = PORT_SUPPLIER_4
127 args_instance[SER_SERVERID_PROP] = SERVERID_SUPPLIER_4
128 args_instance[SER_CREATION_SUFFIX] = DEFAULT_SUFFIX
129 args_supplier = args_instance.copy()
130 supplier4.allocate(args_supplier)
131 instance_supplier4 = supplier4.exists()
132 if instance_supplier4:
133 supplier4.delete()
134 supplier4.create()
135 supplier4.open()
136 supplier4.replica.enableReplication(suffix=SUFFIX, role=ReplicaRole.SUPPLIER,
137 replicaId=REPLICAID_SUPPLIER_4)
138
139 #
140 # Create all the agreements
141 #
142 # Creating agreement from supplier 1 to supplier 2
143 properties = {RA_NAME: 'meTo_' + supplier2.host + ':' + str(supplier2.port),
144 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
145 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
146 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
147 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
148 m1_m2_agmt = supplier1.agreement.create(suffix=SUFFIX, host=supplier2.host,
149 port=supplier2.port,
150 properties=properties)
151 if not m1_m2_agmt:
152 log.fatal("Fail to create a supplier -> supplier replica agreement")
153 sys.exit(1)
154 log.debug("%s created" % m1_m2_agmt)
155
156 # Creating agreement from supplier 1 to supplier 3
157 properties = {RA_NAME: 'meTo_' + supplier3.host + ':' + str(supplier3.port),
158 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
159 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
160 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
161 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
162 m1_m3_agmt = supplier1.agreement.create(suffix=SUFFIX, host=supplier3.host,
163 port=supplier3.port,
164 properties=properties)
165 if not m1_m3_agmt:
166 log.fatal("Fail to create a supplier -> supplier replica agreement")
167 sys.exit(1)
168 log.debug("%s created" % m1_m3_agmt)
169
170 # Creating agreement from supplier 1 to supplier 4
171 properties = {RA_NAME: 'meTo_' + supplier4.host + ':' + str(supplier4.port),
172 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
173 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
174 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
175 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
176 m1_m4_agmt = supplier1.agreement.create(suffix=SUFFIX, host=supplier4.host,
177 port=supplier4.port,
178 properties=properties)
179 if not m1_m4_agmt:
180 log.fatal("Fail to create a supplier -> supplier replica agreement")
181 sys.exit(1)
182 log.debug("%s created" % m1_m4_agmt)
183
184 # Creating agreement from supplier 2 to supplier 1
185 properties = {RA_NAME: 'meTo_' + supplier1.host + ':' + str(supplier1.port),
186 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
187 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
188 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
189 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
190 m2_m1_agmt = supplier2.agreement.create(suffix=SUFFIX, host=supplier1.host,
191 port=supplier1.port,
192 properties=properties)
193 if not m2_m1_agmt:
194 log.fatal("Fail to create a supplier -> supplier replica agreement")
195 sys.exit(1)
196 log.debug("%s created" % m2_m1_agmt)
197
198 # Creating agreement from supplier 2 to supplier 3
199 properties = {RA_NAME: 'meTo_' + supplier3.host + ':' + str(supplier3.port),
200 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
201 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
202 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
203 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
204 m2_m3_agmt = supplier2.agreement.create(suffix=SUFFIX, host=supplier3.host,
205 port=supplier3.port,
206 properties=properties)
207 if not m2_m3_agmt:
208 log.fatal("Fail to create a supplier -> supplier replica agreement")
209 sys.exit(1)
210 log.debug("%s created" % m2_m3_agmt)
211
212 # Creating agreement from supplier 2 to supplier 4
213 properties = {RA_NAME: 'meTo_' + supplier4.host + ':' + str(supplier4.port),
214 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
215 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
216 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
217 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
218 m2_m4_agmt = supplier2.agreement.create(suffix=SUFFIX, host=supplier4.host,
219 port=supplier4.port,
220 properties=properties)
221 if not m2_m4_agmt:
222 log.fatal("Fail to create a supplier -> supplier replica agreement")
223 sys.exit(1)
224 log.debug("%s created" % m2_m4_agmt)
225
226 # Creating agreement from supplier 3 to supplier 1
227 properties = {RA_NAME: 'meTo_' + supplier1.host + ':' + str(supplier1.port),
228 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
229 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
230 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
231 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
232 m3_m1_agmt = supplier3.agreement.create(suffix=SUFFIX, host=supplier1.host,
233 port=supplier1.port,
234 properties=properties)
235 if not m3_m1_agmt:
236 log.fatal("Fail to create a supplier -> supplier replica agreement")
237 sys.exit(1)
238 log.debug("%s created" % m3_m1_agmt)
239
240 # Creating agreement from supplier 3 to supplier 2
241 properties = {RA_NAME: 'meTo_' + supplier2.host + ':' + str(supplier2.port),
242 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
243 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
244 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
245 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
246 m3_m2_agmt = supplier3.agreement.create(suffix=SUFFIX, host=supplier2.host,
247 port=supplier2.port,
248 properties=properties)
249 if not m3_m2_agmt:
250 log.fatal("Fail to create a supplier -> supplier replica agreement")
251 sys.exit(1)
252 log.debug("%s created" % m3_m2_agmt)
253
254 # Creating agreement from supplier 3 to supplier 4
255 properties = {RA_NAME: 'meTo_' + supplier4.host + ':' + str(supplier4.port),
256 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
257 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
258 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
259 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
260 m3_m4_agmt = supplier3.agreement.create(suffix=SUFFIX, host=supplier4.host,
261 port=supplier4.port,
262 properties=properties)
263 if not m3_m4_agmt:
264 log.fatal("Fail to create a supplier -> supplier replica agreement")
265 sys.exit(1)
266 log.debug("%s created" % m3_m4_agmt)
267
268 # Creating agreement from supplier 4 to supplier 1
269 properties = {RA_NAME: 'meTo_' + supplier1.host + ':' + str(supplier1.port),
270 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
271 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
272 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
273 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
274 m4_m1_agmt = supplier4.agreement.create(suffix=SUFFIX, host=supplier1.host,
275 port=supplier1.port,
276 properties=properties)
277 if not m4_m1_agmt:
278 log.fatal("Fail to create a supplier -> supplier replica agreement")
279 sys.exit(1)
280 log.debug("%s created" % m4_m1_agmt)
281
282 # Creating agreement from supplier 4 to supplier 2
283 properties = {RA_NAME: 'meTo_' + supplier2.host + ':' + str(supplier2.port),
284 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
285 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
286 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
287 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
288 m4_m2_agmt = supplier4.agreement.create(suffix=SUFFIX, host=supplier2.host,
289 port=supplier2.port,
290 properties=properties)
291 if not m4_m2_agmt:
292 log.fatal("Fail to create a supplier -> supplier replica agreement")
293 sys.exit(1)
294 log.debug("%s created" % m4_m2_agmt)
295
296 # Creating agreement from supplier 4 to supplier 3
297 properties = {RA_NAME: 'meTo_' + supplier3.host + ':' + str(supplier3.port),
298 RA_BINDDN: defaultProperties[REPLICATION_BIND_DN],
299 RA_BINDPW: defaultProperties[REPLICATION_BIND_PW],
300 RA_METHOD: defaultProperties[REPLICATION_BIND_METHOD],
301 RA_TRANSPORT_PROT: defaultProperties[REPLICATION_TRANSPORT]}
302 m4_m3_agmt = supplier4.agreement.create(suffix=SUFFIX, host=supplier3.host,
303 port=supplier3.port,
304 properties=properties)
305 if not m4_m3_agmt:
306 log.fatal("Fail to create a supplier -> supplier replica agreement")
307 sys.exit(1)
308 log.debug("%s created" % m4_m3_agmt)
309
310 # Allow the replicas to get situated with the new agreements...
311 time.sleep(5)
312
313 #
314 # Initialize all the agreements
315 #
316 supplier1.agreement.init(SUFFIX, HOST_SUPPLIER_2, PORT_SUPPLIER_2)
317 supplier1.waitForReplInit(m1_m2_agmt)
318 supplier1.agreement.init(SUFFIX, HOST_SUPPLIER_3, PORT_SUPPLIER_3)
319 supplier1.waitForReplInit(m1_m3_agmt)
320 supplier1.agreement.init(SUFFIX, HOST_SUPPLIER_4, PORT_SUPPLIER_4)
321 supplier1.waitForReplInit(m1_m4_agmt)
322
323 # Check replication is working...
324 if supplier1.testReplication(DEFAULT_SUFFIX, supplier4):
325 log.info('Replication is working.')
326 else:
327 log.fatal('Replication is not working.')
328 assert False
329
330 def fin():
331 """If we are debugging just stop the instances, otherwise remove
332 them
333 """
334 if 1 or DEBUGGING:
335 supplier1.stop()
336 supplier2.stop()
337 supplier3.stop()
338 supplier4.stop()
339 else:
340 supplier1.delete()
341 supplier2.delete()
342 supplier3.delete()
343 supplier4.delete()
344 request.addfinalizer(fin)
345
346 return TopologyReplication(supplier1, supplier2, supplier3, supplier4)
347
348
349 class AddDelUsers(threading.Thread):
350 """Add's and delets 50000 entries"""
351 def __init__(self, inst):
352 """
353 Initialize the thread
354 """
355 threading.Thread.__init__(self)
356 self.daemon = True
357 self.inst = inst
358 self.name = inst.serverid
359
360 def run(self):
361 """
362 Start adding users
363 """
364 idx = 0
365
366 log.info('AddDelUsers (%s) Adding and deleting %d entries...' %
367 (self.name, ADD_DEL_COUNT))
368
369 while idx < ADD_DEL_COUNT:
370 RDN_VAL = ('uid=%s-%d' % (self.name, idx))
371 USER_DN = ('%s,%s' % (RDN_VAL, DEFAULT_SUFFIX))
372
373 try:
374 self.inst.add_s(Entry((USER_DN, {'objectclass':
375 'top extensibleObject'.split(),
376 'uid': RDN_VAL})))
377 except ldap.LDAPError as e:
378 log.fatal('AddDelUsers (%s): failed to add (%s) error: %s' %
379 (self.name, USER_DN, str(e)))
380 assert False
381
382 try:
383 self.inst.delete_s(USER_DN)
384 except ldap.LDAPError as e:
385 log.fatal('AddDelUsers (%s): failed to delete (%s) error: %s' %
386 (self.name, USER_DN, str(e)))
387 assert False
388
389 idx += 1
390
391 log.info('AddDelUsers (%s) - Finished at: %s' %
392 (self.name, getDateTime()))
393
394
395 def measureConvergence(topology):
396 """Find and measure the convergence of entries from each supplier
397 """
398
399 replicas = [topology.supplier1, topology.supplier2, topology.supplier3,
400 topology.supplier4]
401
402 if ADD_DEL_COUNT > 10:
403 interval = int(ADD_DEL_COUNT / 10)
404 else:
405 interval = 1
406
407 for supplier in [('1', topology.supplier1),
408 ('2', topology.supplier2),
409 ('3', topology.supplier3),
410 ('4', topology.supplier4)]:
411 # Start with the first entry
412 entries = ['ADD dn="uid=supplier_%s-0,%s' %
413 (supplier[0], DEFAULT_SUFFIX)]
414
415 # Add incremental entries to the list
416 idx = interval
417 while idx < ADD_DEL_COUNT:
418 entries.append('ADD dn="uid=supplier_%s-%d,%s' %
419 (supplier[0], idx, DEFAULT_SUFFIX))
420 idx += interval
421
422 # Add the last entry to the list (if it was not already added)
423 if idx != (ADD_DEL_COUNT - 1):
424 entries.append('ADD dn="uid=supplier_%s-%d,%s' %
425 (supplier[0], (ADD_DEL_COUNT - 1),
426 DEFAULT_SUFFIX))
427
428 ReplTools.replConvReport(DEFAULT_SUFFIX, entries, supplier[1], replicas)
429
430
431 def test_MMR_Integrity(topology):
432 """Apply load to 4 suppliers at the same time. Perform adds and deletes.
433 If any updates are missed we will see an error 32 in the access logs or
434 we will have entries left over once the test completes.
435 """
436 loop = 0
437
438 ALL_REPLICAS = [topology.supplier1, topology.supplier2, topology.supplier3,
439 topology.supplier4]
440
441 if TEST_CONVERGE_LATENCY:
442 try:
443 for inst in ALL_REPLICAS:
444 replica = inst.replicas.get(DEFAULT_SUFFIX)
445 replica.set('nsds5ReplicaReleaseTimeout', CONVERGENCE_TIMEOUT)
446 except ldap.LDAPError as e:
447 log.fatal('Failed to set replicas release timeout - error: %s' %
448 (str(e)))
449 assert False
450
451 if DEBUGGING:
452 # Enable Repl logging, and increase the max logs
453 try:
454 for inst in ALL_REPLICAS:
455 inst.enableReplLogging()
456 inst.modify_s("cn=config", [(ldap.MOD_REPLACE,
457 'nsslapd-errorlog-maxlogsperdir',
458 '5')])
459 except ldap.LDAPError as e:
460 log.fatal('Failed to set max logs - error: %s' % (str(e)))
461 assert False
462
463 while loop < MAX_LOOPS:
464 # Remove the current logs so we have a clean set of logs to check.
465 log.info('Pass %d...' % (loop + 1))
466 log.info("Removing logs...")
467 for inst in ALL_REPLICAS:
468 inst.deleteAllLogs()
469
470 # Fire off 4 threads to apply the load
471 log.info("Start adding/deleting: " + getDateTime())
472 startTime = time.time()
473 add_del_m1 = AddDelUsers(topology.supplier1)
474 add_del_m1.start()
475 add_del_m2 = AddDelUsers(topology.supplier2)
476 add_del_m2.start()
477 add_del_m3 = AddDelUsers(topology.supplier3)
478 add_del_m3.start()
479 add_del_m4 = AddDelUsers(topology.supplier4)
480 add_del_m4.start()
481
482 # Wait for threads to finish sending their updates
483 add_del_m1.join()
484 add_del_m2.join()
485 add_del_m3.join()
486 add_del_m4.join()
487 log.info("Finished adding/deleting entries: " + getDateTime())
488
489 #
490 # Loop checking for error 32's, and for convergence to complete
491 #
492 log.info("Waiting for replication to converge...")
493 while True:
494 # First check for error 32's
495 for inst in ALL_REPLICAS:
496 if inst.searchAccessLog(" err=32 "):
497 log.fatal('An add was missed on: ' + inst.serverid)
498 assert False
499
500 # Next check to see if the last update is in the access log
501 converged = True
502 for inst in ALL_REPLICAS:
503 if not inst.searchAccessLog(LAST_DN_M1) or \
504 not inst.searchAccessLog(LAST_DN_M2) or \
505 not inst.searchAccessLog(LAST_DN_M3) or \
506 not inst.searchAccessLog(LAST_DN_M4):
507 converged = False
508 break
509
510 if converged:
511 elapsed_tm = int(time.time() - startTime)
512 convtime = str(datetime.timedelta(seconds=elapsed_tm))
513 log.info('Replication converged at: ' + getDateTime() +
514 ' - Elapsed Time: ' + convtime)
515 break
516 else:
517 # Check if replication is idle
518 replicas = [topology.supplier1, topology.supplier2,
519 topology.supplier3, topology.supplier4]
520 if ReplTools.replIdle(replicas, DEFAULT_SUFFIX):
521 # Replication is idle - wait 30 secs for access log buffer
522 time.sleep(30)
523
524 # Now check the access log again...
525 converged = True
526 for inst in ALL_REPLICAS:
527 if not inst.searchAccessLog(LAST_DN_M1) or \
528 not inst.searchAccessLog(LAST_DN_M2) or \
529 not inst.searchAccessLog(LAST_DN_M3) or \
530 not inst.searchAccessLog(LAST_DN_M4):
531 converged = False
532 break
533
534 if converged:
535 elapsed_tm = int(time.time() - startTime)
536 convtime = str(datetime.timedelta(seconds=elapsed_tm))
537 log.info('Replication converged at: ' + getDateTime() +
538 ' - Elapsed Time: ' + convtime)
539 break
540 else:
541 log.fatal('Stopping replication check: ' +
542 getDateTime())
543 log.fatal('Failure: Replication is complete, but we ' +
544 'never converged.')
545 assert False
546
547 # Sleep a bit before the next pass
548 time.sleep(3)
549
550 #
551 # Finally check the CSN's
552 #
553 log.info("Check the CSN's...")
554 if not ReplTools.checkCSNs(ALL_REPLICAS):
555 assert False
556 log.info("All CSN's present and accounted for.")
557
558 #
559 # Print the convergence report
560 #
561 log.info('Measuring convergence...')
562 measureConvergence(topology)
563
564 #
565 # Test complete
566 #
567 log.info('No lingering entries.')
568 log.info('Pass %d complete.' % (loop + 1))
569 elapsed_tm = int(time.time() - TEST_START)
570 convtime = str(datetime.timedelta(seconds=elapsed_tm))
571 log.info('Entire test ran for: ' + convtime)
572
573 loop += 1
574
575 log.info('Test PASSED')
576
577
578 if __name__ == '__main__':
579 # Run isolated
580 # -s for DEBUG mode
581 CURRENT_FILE = os.path.realpath(__file__)
582 pytest.main("-s %s" % CURRENT_FILE)