Debian Codegraph @

dirsrvtests/tests/suites/replication/cleanallruv_stress_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2022 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import ldap
10 import logging
11 import pytest
12 import os
13 import random
14 import time
15 import threading
16 from lib389._constants import DEFAULT_SUFFIX
17 from test389.topologies import topology_m4
18 from lib389.tasks import CleanAllRUVTask
19 from lib389.idm.directorymanager import DirectoryManager
20 from lib389.idm.user import UserAccounts
21 from lib389.replica import ReplicationManager, Replicas
22 from lib389.config import LDBMConfig
23
24 log = logging.getLogger(__name__)
25
26
27 class AddUsers(threading.Thread):
28 def __init__(self, inst, num_users):
29 threading.Thread.__init__(self)
30 self.daemon = True
31 self.inst = inst
32 self.num_users = num_users
33
34 def run(self):
35 """Start adding users"""
36
37 dm = DirectoryManager(self.inst)
38 conn = dm.bind()
39
40 users = UserAccounts(conn, DEFAULT_SUFFIX)
41
42 u_range = list(range(self.num_users))
43 random.shuffle(u_range)
44
45 for idx in u_range:
46 try:
47 users.create(properties={
48 'uid': 'testuser%s' % idx,
49 'cn' : 'testuser%s' % idx,
50 'sn' : 'user%s' % idx,
51 'uidNumber' : '%s' % (1000 + idx),
52 'gidNumber' : '%s' % (1000 + idx),
53 'homeDirectory' : '/home/testuser%s' % idx
54 })
55 # One of the suppliers was probably put into read only mode - just break out
56 except ldap.UNWILLING_TO_PERFORM:
57 break
58 except ldap.ALREADY_EXISTS:
59 pass
60 conn.close()
61
62 def remove_supplier4_agmts(msg, topology_m4):
63 """Remove all the repl agmts to supplier4. """
64
65 log.info('%s: remove all the agreements to supplier 4...' % msg)
66 repl = ReplicationManager(DEFAULT_SUFFIX)
67 # This will delete m4 from the topo *and* remove all incoming agreements
68 # to m4.
69 repl.remove_supplier(topology_m4.ms["supplier4"],
70 [topology_m4.ms["supplier1"], topology_m4.ms["supplier2"], topology_m4.ms["supplier3"]])
71
72 def task_done(topology_m4, task_dn, timeout=60):
73 """Check if the task is complete"""
74
75 attrlist = ['nsTaskLog', 'nsTaskStatus', 'nsTaskExitCode',
76 'nsTaskCurrentItem', 'nsTaskTotalItems']
77 done = False
78 count = 0
79
80 while not done and count < timeout:
81 try:
82 entry = topology_m4.ms["supplier1"].getEntry(task_dn, attrlist=attrlist)
83 if entry is not None:
84 if entry.hasAttr('nsTaskExitCode'):
85 done = True
86 break
87 else:
88 done = True
89 break
90 except ldap.NO_SUCH_OBJECT:
91 done = True
92 break
93 except ldap.LDAPError:
94 break
95 time.sleep(1)
96 count += 1
97
98 return done
99
100 def check_ruvs(msg, topology_m4, m4rid):
101 """Check suppliers 1-3 for supplier 4's rid."""
102 for inst in (topology_m4.ms["supplier1"], topology_m4.ms["supplier2"], topology_m4.ms["supplier3"]):
103 clean = False
104 replicas = Replicas(inst)
105 replica = replicas.get(DEFAULT_SUFFIX)
106 log.info('check_ruvs for replica %s:%s (suffix:rid)' % (replica.get_suffix(), replica.get_rid()))
107
108 count = 0
109 while not clean and count < 20:
110 ruv = replica.get_ruv()
111 if m4rid in ruv._rids:
112 time.sleep(5)
113 count = count + 1
114 else:
115 clean = True
116 if not clean:
117 raise Exception("Supplier %s was not cleaned in time." % inst.serverid)
118 return True
119
120
121 @pytest.mark.flaky(max_runs=2, min_passes=1)
122 def test_stress_clean(topology_m4):
123 """Put each server(m1 - m4) under a stress, and perform the entire clean process
124
125 :id: a8263cd6-f068-4357-86e0-e7c34504c8c5
126 :setup: Replication setup with four suppliers
127 :steps:
128 1. Add a bunch of updates to all suppliers
129 2. Put supplier 4 to read-only mode
130 3. Disable replication on supplier 4
131 4. Remove agreements to supplier 4 from other suppliers
132 5. Run a cleanallruv task on supplier 1
133 6. Check that everything was cleaned
134 :expectedresults:
135 1. Operation should be successful
136 2. Supplier 4 should be put to read-only mode
137 3. Replication on supplier 4 should be disabled
138 4. Agreements to supplier 4 should be removed
139 5. Operation should be successful
140 6. Everything should be cleaned
141 """
142
143 log.info('Running test_stress_clean...')
144 log.info('test_stress_clean: put all the suppliers under load...')
145
146 ldbm_config = LDBMConfig(topology_m4.ms["supplier4"])
147
148 # Put all the suppliers under load
149 # not too high load else it takes a long time to converge and
150 # the test result becomes instable
151 m1_add_users = AddUsers(topology_m4.ms["supplier1"], 200)
152 m1_add_users.start()
153 m2_add_users = AddUsers(topology_m4.ms["supplier2"], 200)
154 m2_add_users.start()
155 m3_add_users = AddUsers(topology_m4.ms["supplier3"], 200)
156 m3_add_users.start()
157 m4_add_users = AddUsers(topology_m4.ms["supplier4"], 200)
158 m4_add_users.start()
159
160 # Allow sometime to get replication flowing in all directions
161 log.info('test_stress_clean: allow some time for replication to get flowing...')
162 time.sleep(5)
163
164 # Put supplier 4 into read only mode
165 ldbm_config.set('nsslapd-readonly', 'on')
166 # We need to wait for supplier 4 to push its changes out
167 log.info('test_stress_clean: allow some time for supplier 4 to push changes out (60 seconds)...')
168 time.sleep(60)
169
170 # Remove the agreements from the other suppliers that point to supplier 4
171 repl = ReplicationManager(DEFAULT_SUFFIX)
172 m4rid = repl.get_rid(topology_m4.ms["supplier4"])
173 remove_supplier4_agmts("test_stress_clean", topology_m4)
174
175 # Run the task
176 cruv_task = CleanAllRUVTask(topology_m4.ms["supplier1"])
177 cruv_task.create(properties={
178 'replica-id': m4rid,
179 'replica-base-dn': DEFAULT_SUFFIX,
180 'replica-force-cleaning': 'no'
181 })
182 cruv_task.wait()
183
184 # Wait for the update to finish
185 log.info('test_stress_clean: wait for all the updates to finish...')
186 m1_add_users.join()
187 m2_add_users.join()
188 m3_add_users.join()
189 m4_add_users.join()
190
191 # Check the other supplier's RUV for 'replica 4'
192 log.info('test_stress_clean: check if all the replicas have been cleaned...')
193 clean = check_ruvs("test_stress_clean", topology_m4, m4rid)
194 assert clean
195
196 log.info('test_stress_clean: PASSED, restoring supplier 4...')
197
198 # Sleep for a bit to replication complete
199 log.info("Sleep for 120 seconds to allow replication to complete...")
200 repl = ReplicationManager(DEFAULT_SUFFIX)
201 repl.test_replication_topology([
202 topology_m4.ms["supplier1"],
203 topology_m4.ms["supplier2"],
204 topology_m4.ms["supplier3"],
205 ], timeout=120)
206
207 # Turn off readonly mode
208 ldbm_config.set('nsslapd-readonly', 'off')
209
210
211 if __name__ == '__main__':
212 # Run isolated
213 # -s for DEBUG mode
214 CURRENT_FILE = os.path.realpath(__file__)
215 pytest.main(["-s", CURRENT_FILE])
216