| 1 |
# --- BEGIN COPYRIGHT BLOCK --- |
| 2 |
# Copyright (C) 2022 Red Hat, Inc. |
| 3 |
# All rights reserved. |
| 4 |
# |
| 5 |
# License: GPL (version 3 or any later version). |
| 6 |
# See LICENSE for details. |
| 7 |
# --- END COPYRIGHT BLOCK --- |
| 8 |
# |
| 9 |
import ldap |
| 10 |
import logging |
| 11 |
import pytest |
| 12 |
import os |
| 13 |
import random |
| 14 |
import time |
| 15 |
import threading |
| 16 |
from lib389._constants import DEFAULT_SUFFIX |
| 17 |
from test389.topologies import topology_m4 |
| 18 |
from lib389.tasks import CleanAllRUVTask |
| 19 |
from lib389.idm.directorymanager import DirectoryManager |
| 20 |
from lib389.idm.user import UserAccounts |
| 21 |
from lib389.replica import ReplicationManager, Replicas |
| 22 |
from lib389.config import LDBMConfig |
| 23 |
|
| 24 |
log = logging.getLogger(__name__) |
| 25 |
|
| 26 |
|
| 27 |
class AddUsers(threading.Thread): |
| 28 |
def __init__(self, inst, num_users): |
| 29 |
threading.Thread.__init__(self) |
| 30 |
self.daemon = True |
| 31 |
self.inst = inst |
| 32 |
self.num_users = num_users |
| 33 |
|
| 34 |
def run(self): |
| 35 |
"""Start adding users""" |
| 36 |
|
| 37 |
dm = DirectoryManager(self.inst) |
| 38 |
conn = dm.bind() |
| 39 |
|
| 40 |
users = UserAccounts(conn, DEFAULT_SUFFIX) |
| 41 |
|
| 42 |
u_range = list(range(self.num_users)) |
| 43 |
random.shuffle(u_range) |
| 44 |
|
| 45 |
for idx in u_range: |
| 46 |
try: |
| 47 |
users.create(properties={ |
| 48 |
'uid': 'testuser%s' % idx, |
| 49 |
'cn' : 'testuser%s' % idx, |
| 50 |
'sn' : 'user%s' % idx, |
| 51 |
'uidNumber' : '%s' % (1000 + idx), |
| 52 |
'gidNumber' : '%s' % (1000 + idx), |
| 53 |
'homeDirectory' : '/home/testuser%s' % idx |
| 54 |
}) |
| 55 |
# One of the suppliers was probably put into read only mode - just break out |
| 56 |
except ldap.UNWILLING_TO_PERFORM: |
| 57 |
break |
| 58 |
except ldap.ALREADY_EXISTS: |
| 59 |
pass |
| 60 |
conn.close() |
| 61 |
|
| 62 |
def remove_supplier4_agmts(msg, topology_m4): |
| 63 |
"""Remove all the repl agmts to supplier4. """ |
| 64 |
|
| 65 |
log.info('%s: remove all the agreements to supplier 4...' % msg) |
| 66 |
repl = ReplicationManager(DEFAULT_SUFFIX) |
| 67 |
# This will delete m4 from the topo *and* remove all incoming agreements |
| 68 |
# to m4. |
| 69 |
repl.remove_supplier(topology_m4.ms["supplier4"], |
| 70 |
[topology_m4.ms["supplier1"], topology_m4.ms["supplier2"], topology_m4.ms["supplier3"]]) |
| 71 |
|
| 72 |
def task_done(topology_m4, task_dn, timeout=60): |
| 73 |
"""Check if the task is complete""" |
| 74 |
|
| 75 |
attrlist = ['nsTaskLog', 'nsTaskStatus', 'nsTaskExitCode', |
| 76 |
'nsTaskCurrentItem', 'nsTaskTotalItems'] |
| 77 |
done = False |
| 78 |
count = 0 |
| 79 |
|
| 80 |
while not done and count < timeout: |
| 81 |
try: |
| 82 |
entry = topology_m4.ms["supplier1"].getEntry(task_dn, attrlist=attrlist) |
| 83 |
if entry is not None: |
| 84 |
if entry.hasAttr('nsTaskExitCode'): |
| 85 |
done = True |
| 86 |
break |
| 87 |
else: |
| 88 |
done = True |
| 89 |
break |
| 90 |
except ldap.NO_SUCH_OBJECT: |
| 91 |
done = True |
| 92 |
break |
| 93 |
except ldap.LDAPError: |
| 94 |
break |
| 95 |
time.sleep(1) |
| 96 |
count += 1 |
| 97 |
|
| 98 |
return done |
| 99 |
|
| 100 |
def check_ruvs(msg, topology_m4, m4rid): |
| 101 |
"""Check suppliers 1-3 for supplier 4's rid.""" |
| 102 |
for inst in (topology_m4.ms["supplier1"], topology_m4.ms["supplier2"], topology_m4.ms["supplier3"]): |
| 103 |
clean = False |
| 104 |
replicas = Replicas(inst) |
| 105 |
replica = replicas.get(DEFAULT_SUFFIX) |
| 106 |
log.info('check_ruvs for replica %s:%s (suffix:rid)' % (replica.get_suffix(), replica.get_rid())) |
| 107 |
|
| 108 |
count = 0 |
| 109 |
while not clean and count < 20: |
| 110 |
ruv = replica.get_ruv() |
| 111 |
if m4rid in ruv._rids: |
| 112 |
time.sleep(5) |
| 113 |
count = count + 1 |
| 114 |
else: |
| 115 |
clean = True |
| 116 |
if not clean: |
| 117 |
raise Exception("Supplier %s was not cleaned in time." % inst.serverid) |
| 118 |
return True |
| 119 |
|
| 120 |
|
| 121 |
@pytest.mark.flaky(max_runs=2, min_passes=1) |
| 122 |
def test_stress_clean(topology_m4): |
| 123 |
"""Put each server(m1 - m4) under a stress, and perform the entire clean process |
| 124 |
|
| 125 |
:id: a8263cd6-f068-4357-86e0-e7c34504c8c5 |
| 126 |
:setup: Replication setup with four suppliers |
| 127 |
:steps: |
| 128 |
1. Add a bunch of updates to all suppliers |
| 129 |
2. Put supplier 4 to read-only mode |
| 130 |
3. Disable replication on supplier 4 |
| 131 |
4. Remove agreements to supplier 4 from other suppliers |
| 132 |
5. Run a cleanallruv task on supplier 1 |
| 133 |
6. Check that everything was cleaned |
| 134 |
:expectedresults: |
| 135 |
1. Operation should be successful |
| 136 |
2. Supplier 4 should be put to read-only mode |
| 137 |
3. Replication on supplier 4 should be disabled |
| 138 |
4. Agreements to supplier 4 should be removed |
| 139 |
5. Operation should be successful |
| 140 |
6. Everything should be cleaned |
| 141 |
""" |
| 142 |
|
| 143 |
log.info('Running test_stress_clean...') |
| 144 |
log.info('test_stress_clean: put all the suppliers under load...') |
| 145 |
|
| 146 |
ldbm_config = LDBMConfig(topology_m4.ms["supplier4"]) |
| 147 |
|
| 148 |
# Put all the suppliers under load |
| 149 |
# not too high load else it takes a long time to converge and |
| 150 |
# the test result becomes instable |
| 151 |
m1_add_users = AddUsers(topology_m4.ms["supplier1"], 200) |
| 152 |
m1_add_users.start() |
| 153 |
m2_add_users = AddUsers(topology_m4.ms["supplier2"], 200) |
| 154 |
m2_add_users.start() |
| 155 |
m3_add_users = AddUsers(topology_m4.ms["supplier3"], 200) |
| 156 |
m3_add_users.start() |
| 157 |
m4_add_users = AddUsers(topology_m4.ms["supplier4"], 200) |
| 158 |
m4_add_users.start() |
| 159 |
|
| 160 |
# Allow sometime to get replication flowing in all directions |
| 161 |
log.info('test_stress_clean: allow some time for replication to get flowing...') |
| 162 |
time.sleep(5) |
| 163 |
|
| 164 |
# Put supplier 4 into read only mode |
| 165 |
ldbm_config.set('nsslapd-readonly', 'on') |
| 166 |
# We need to wait for supplier 4 to push its changes out |
| 167 |
log.info('test_stress_clean: allow some time for supplier 4 to push changes out (60 seconds)...') |
| 168 |
time.sleep(60) |
| 169 |
|
| 170 |
# Remove the agreements from the other suppliers that point to supplier 4 |
| 171 |
repl = ReplicationManager(DEFAULT_SUFFIX) |
| 172 |
m4rid = repl.get_rid(topology_m4.ms["supplier4"]) |
| 173 |
remove_supplier4_agmts("test_stress_clean", topology_m4) |
| 174 |
|
| 175 |
# Run the task |
| 176 |
cruv_task = CleanAllRUVTask(topology_m4.ms["supplier1"]) |
| 177 |
cruv_task.create(properties={ |
| 178 |
'replica-id': m4rid, |
| 179 |
'replica-base-dn': DEFAULT_SUFFIX, |
| 180 |
'replica-force-cleaning': 'no' |
| 181 |
}) |
| 182 |
cruv_task.wait() |
| 183 |
|
| 184 |
# Wait for the update to finish |
| 185 |
log.info('test_stress_clean: wait for all the updates to finish...') |
| 186 |
m1_add_users.join() |
| 187 |
m2_add_users.join() |
| 188 |
m3_add_users.join() |
| 189 |
m4_add_users.join() |
| 190 |
|
| 191 |
# Check the other supplier's RUV for 'replica 4' |
| 192 |
log.info('test_stress_clean: check if all the replicas have been cleaned...') |
| 193 |
clean = check_ruvs("test_stress_clean", topology_m4, m4rid) |
| 194 |
assert clean |
| 195 |
|
| 196 |
log.info('test_stress_clean: PASSED, restoring supplier 4...') |
| 197 |
|
| 198 |
# Sleep for a bit to replication complete |
| 199 |
log.info("Sleep for 120 seconds to allow replication to complete...") |
| 200 |
repl = ReplicationManager(DEFAULT_SUFFIX) |
| 201 |
repl.test_replication_topology([ |
| 202 |
topology_m4.ms["supplier1"], |
| 203 |
topology_m4.ms["supplier2"], |
| 204 |
topology_m4.ms["supplier3"], |
| 205 |
], timeout=120) |
| 206 |
|
| 207 |
# Turn off readonly mode |
| 208 |
ldbm_config.set('nsslapd-readonly', 'off') |
| 209 |
|
| 210 |
|
| 211 |
if __name__ == '__main__': |
| 212 |
# Run isolated |
| 213 |
# -s for DEBUG mode |
| 214 |
CURRENT_FILE = os.path.realpath(__file__) |
| 215 |
pytest.main(["-s", CURRENT_FILE]) |
| 216 |
|