| 1 |
# --- BEGIN COPYRIGHT BLOCK --- |
| 2 |
# Copyright (C) 2022 Red Hat, Inc. |
| 3 |
# All rights reserved. |
| 4 |
# |
| 5 |
# License: GPL (version 3 or any later version). |
| 6 |
# See LICENSE for details. |
| 7 |
# --- END COPYRIGHT BLOCK --- |
| 8 |
# |
| 9 |
import time |
| 10 |
import os |
| 11 |
import pytest |
| 12 |
import ldap |
| 13 |
import logging |
| 14 |
from lib389 import Entry |
| 15 |
from lib389._constants import DEFAULT_BENAME, DEFAULT_SUFFIX |
| 16 |
from lib389.backend import Backends, Backend |
| 17 |
from lib389.mappingTree import MappingTrees |
| 18 |
from lib389.configurations.sample import create_base_domain |
| 19 |
from lib389.idm.domain import Domain |
| 20 |
from lib389.idm.user import UserAccounts, UserAccount |
| 21 |
from lib389.idm.organizationalunit import OrganizationalUnits |
| 22 |
from test389.topologies import topology_m2 as topo_m2 |
| 23 |
from test389.topologies import topology_st |
| 24 |
from lib389.agreement import Agreements |
| 25 |
from lib389.utils import ds_is_older, ensure_bytes |
| 26 |
from lib389.tasks import Tasks,ExportTask, ImportTask |
| 27 |
from test389.topologies import topology_st as topo |
| 28 |
|
| 29 |
pytestmark = pytest.mark.tier1 |
| 30 |
|
| 31 |
|
| 32 |
OUNAME = 'NewOU' |
| 33 |
OUDN = f'ou={OUNAME},ou=people,{DEFAULT_SUFFIX}' |
| 34 |
OUPROPERTIES = { 'ou' : OUNAME } |
| 35 |
USERNAME = 'NewUser' |
| 36 |
USERID = '100' |
| 37 |
USERSDN = f'uid={USERNAME},{OUDN}' |
| 38 |
USERPROPERTIES = { |
| 39 |
'uid': USERNAME, |
| 40 |
'sn': USERNAME, |
| 41 |
'cn': USERNAME, |
| 42 |
'uidNumber': USERID, |
| 43 |
'gidNumber': USERID, |
| 44 |
'homeDirectory': f'/home/{USERNAME}' |
| 45 |
} |
| 46 |
|
| 47 |
# 2 tombstone entry + 1 RUV have 2 records in entryrdn index |
| 48 |
# Each record have 1 rdn + 1 normalized rdn both containing nsuniqueid |
| 49 |
# So 3 * 2 * 2 nsuniqueid substrings are expected |
| 50 |
EXPECTED_NB_NSNIQUEID = 12 |
| 51 |
|
| 52 |
logging.getLogger(__name__).setLevel(logging.INFO) |
| 53 |
log = logging.getLogger(__name__) |
| 54 |
|
| 55 |
def checkdbscancount(inst, pattern, expected_count): |
| 56 |
inst.restart() |
| 57 |
dbscanOut = inst.dbscan(args=['-f', f'{inst.dbdir}/{DEFAULT_BENAME}/entryrdn.db', '-A'], stopping=False) |
| 58 |
count = dbscanOut.count(ensure_bytes(pattern)) |
| 59 |
if count != expected_count: |
| 60 |
log.info(f"dbscan output is: {dbscanOut}") |
| 61 |
assert count == expected_count |
| 62 |
|
| 63 |
|
| 64 |
def test_tombstone(topo_m2): |
| 65 |
""" |
| 66 |
An internal unindexed search was able to crash the server due to missing logging function. |
| 67 |
|
| 68 |
:id: a12eacac-4e35-11ed-8625-482ae39447e5 |
| 69 |
:setup: 2 Suplier instances |
| 70 |
:steps: |
| 71 |
1. Add an OrganizationalUnit |
| 72 |
2. Add an User as child of the new OrganizationalUnit |
| 73 |
3. Modify User's description |
| 74 |
4. Delele User |
| 75 |
5. Delete OrganizationalUnit |
| 76 |
6. Dump supplier1 entryrdn index |
| 77 |
7. Check that nsuniqueid appears three times in the dump result |
| 78 |
8. Export supplier1 with replication data |
| 79 |
9. Import supplier2 with previously exported ldif file |
| 80 |
10. Dump entryrdn index |
| 81 |
11. Check that nsuniqueid appears three times in the dump result |
| 82 |
12. Reindex entryrdn on supplier1 |
| 83 |
13. Dump entryrdn index on supplier |
| 84 |
14. Check that nsuniqueid appears three times in the dump result |
| 85 |
15. Perform bulk import from supplier1 to supplier2 |
| 86 |
16. Wait until bulk import is completed |
| 87 |
17. Dump entryrdn index on supplier |
| 88 |
18. Check that nsuniqueid appears three times in the dump result |
| 89 |
:expectedresults: |
| 90 |
1. Should succeed |
| 91 |
2. Should succeed |
| 92 |
3. Should succeed |
| 93 |
4. Should succeed |
| 94 |
5. Should succeed |
| 95 |
6. Should succeed |
| 96 |
7. Should succeed |
| 97 |
8. Should succeed |
| 98 |
9. Should succeed |
| 99 |
10. Should succeed |
| 100 |
11. Should succeed |
| 101 |
12. Should succeed |
| 102 |
13. Should succeed |
| 103 |
14. Should succeed |
| 104 |
15. Should succeed |
| 105 |
16. Should succeed |
| 106 |
17. Should succeed |
| 107 |
18. Should succeed |
| 108 |
""" |
| 109 |
s1 = topo_m2.ms["supplier1"] |
| 110 |
s2 = topo_m2.ms["supplier2"] |
| 111 |
ldif_dir = s1.get_ldif_dir() |
| 112 |
|
| 113 |
log.info("Create tombstones...") |
| 114 |
ous = OrganizationalUnits(s1, DEFAULT_SUFFIX) |
| 115 |
ou = ous.create(properties=OUPROPERTIES) |
| 116 |
users = UserAccounts(s1, DEFAULT_SUFFIX, rdn=None) |
| 117 |
user = users.create(properties=USERPROPERTIES) |
| 118 |
user.replace('description', 'New Description') |
| 119 |
user.delete() |
| 120 |
ou.delete() |
| 121 |
# Need to restart the server otherwise bdb changes may not be up to date. |
| 122 |
checkdbscancount(s1, 'nsuniqueid', EXPECTED_NB_NSNIQUEID) |
| 123 |
|
| 124 |
log.info("Exporting LDIF online...") |
| 125 |
export_ldif = ldif_dir + '/export.ldif' |
| 126 |
export_task = Backends(s1).export_ldif(be_names=DEFAULT_BENAME, ldif=export_ldif, replication=True) |
| 127 |
export_task.wait() |
| 128 |
|
| 129 |
log.info("Importing LDIF online...") |
| 130 |
import_task = ImportTask(s2) |
| 131 |
import_task.import_suffix_from_ldif(ldiffile=export_ldif, suffix=DEFAULT_SUFFIX) |
| 132 |
import_task.wait() |
| 133 |
checkdbscancount(s1, 'nsuniqueid', EXPECTED_NB_NSNIQUEID) |
| 134 |
|
| 135 |
log.info("Reindex online...") |
| 136 |
task = Tasks(s2) |
| 137 |
task.reindex(suffix=DEFAULT_SUFFIX, args={'wait': True}) |
| 138 |
checkdbscancount(s1, 'nsuniqueid', EXPECTED_NB_NSNIQUEID) |
| 139 |
|
| 140 |
log.info("Bulk import...") |
| 141 |
agmt = Agreements(s1).list()[0] |
| 142 |
agmt.begin_reinit() |
| 143 |
(done, error) = agmt.wait_reinit() |
| 144 |
assert done is True |
| 145 |
assert error is False |
| 146 |
checkdbscancount(s1, 'nsuniqueid', EXPECTED_NB_NSNIQUEID) |
| 147 |
|
| 148 |
|
| 149 |
def create_long_rdn_users(users, users_dict): |
| 150 |
for key in range(0x41, 0x5B): |
| 151 |
# generate a name longer that lmdb key size limit (511) |
| 152 |
longname = chr(key) * 700 |
| 153 |
userproperties = { |
| 154 |
'uid': longname, |
| 155 |
'sn': longname, |
| 156 |
'cn': longname, |
| 157 |
'uidNumber': str(1000+key), |
| 158 |
'gidNumber': str(1000+key), |
| 159 |
'homeDirectory': f'/home/{longname}' |
| 160 |
} |
| 161 |
user = users.create(properties=userproperties) |
| 162 |
users_dict[key] = user |
| 163 |
for user in users_dict.values(): |
| 164 |
assert user.exists() |
| 165 |
|
| 166 |
|
| 167 |
def test_long_rdn(topo_m2): |
| 168 |
""" |
| 169 |
Test operation on entries with rdn longer than lmdb maximum key size (511 bytes). |
| 170 |
|
| 171 |
:id: 47a06e92-e14f-11ee-b492-482ae39447e5 |
| 172 |
:setup: 2 Suplier instances |
| 173 |
:steps: |
| 174 |
1. Add Users with a long name |
| 175 |
2. Check Users exists |
| 176 |
3. Export supplier1 with replication data |
| 177 |
4. Check Users exists |
| 178 |
5. Import supplier2 with previously exported ldif file |
| 179 |
6. Check Users exists |
| 180 |
7. Reindex entryrdn on supplier1 |
| 181 |
8. Check Users exists |
| 182 |
9. Perform bulk import from supplier1 to supplier2 |
| 183 |
10. Wait until bulk import is completed |
| 184 |
11. Check Users exists |
| 185 |
12. Delete Users exists |
| 186 |
13. Create an organizational unit with long ou |
| 187 |
14. Add children with long rdn |
| 188 |
15. Rename the organizational unit |
| 189 |
16. Delete the users with a long rdn |
| 190 |
17. Delete the ou with a long rdn |
| 191 |
:expectedresults: |
| 192 |
1. Should succeed |
| 193 |
2. Should succeed |
| 194 |
3. Should succeed |
| 195 |
4. Should succeed |
| 196 |
5. Should succeed |
| 197 |
6. Should succeed |
| 198 |
7. Should succeed |
| 199 |
8. Should succeed |
| 200 |
9. Should succeed |
| 201 |
10. Should succeed |
| 202 |
11. Should succeed |
| 203 |
12. Should succeed |
| 204 |
13. Should succeed |
| 205 |
14. Should succeed |
| 206 |
15. Should succeed |
| 207 |
16. Should succeed |
| 208 |
17. Should succeed |
| 209 |
""" |
| 210 |
s1 = topo_m2.ms["supplier1"] |
| 211 |
s2 = topo_m2.ms["supplier2"] |
| 212 |
ldif_dir = s1.get_ldif_dir() |
| 213 |
log.info("Create users with a long rdn...") |
| 214 |
users_dict_s1 = {} |
| 215 |
users_dict_s2 = {} |
| 216 |
users = UserAccounts(s1, DEFAULT_SUFFIX, rdn=None) |
| 217 |
create_long_rdn_users(users, users_dict_s1) |
| 218 |
for (key, user) in users_dict_s1.items(): |
| 219 |
users_dict_s2[key] = UserAccount(s2, user.dn) |
| 220 |
|
| 221 |
log.info("Exporting LDIF online...") |
| 222 |
export_ldif = ldif_dir + '/export.ldif' |
| 223 |
export_task = Backends(s1).export_ldif(be_names=DEFAULT_BENAME, ldif=export_ldif, replication=True) |
| 224 |
export_task.wait() |
| 225 |
assert export_task.get_exit_code() == 0 |
| 226 |
for user in users_dict_s1.values(): |
| 227 |
assert user.exists() |
| 228 |
|
| 229 |
log.info("Importing LDIF online...") |
| 230 |
import_task = ImportTask(s2) |
| 231 |
import_task.import_suffix_from_ldif(ldiffile=export_ldif, suffix=DEFAULT_SUFFIX) |
| 232 |
import_task.wait() |
| 233 |
assert import_task.get_exit_code() == 0 |
| 234 |
for user in users_dict_s2.values(): |
| 235 |
assert user.exists() |
| 236 |
|
| 237 |
log.info("Reindex online...") |
| 238 |
task = Tasks(s2) |
| 239 |
task.reindex(suffix=DEFAULT_SUFFIX, args={'wait': True}) |
| 240 |
for user in users_dict_s2.values(): |
| 241 |
assert user.exists() |
| 242 |
|
| 243 |
log.info("Bulk import...") |
| 244 |
agmt = Agreements(s1).list()[0] |
| 245 |
agmt.begin_reinit() |
| 246 |
(done, error) = agmt.wait_reinit() |
| 247 |
assert done is True |
| 248 |
assert error is False |
| 249 |
for user in users_dict_s2.values(): |
| 250 |
assert user.exists() |
| 251 |
|
| 252 |
log.info("Delete the users with a long rdn...") |
| 253 |
for user in users_dict_s1.values(): |
| 254 |
user.delete() |
| 255 |
for user in users_dict_s1.values(): |
| 256 |
assert not user.exists() |
| 257 |
|
| 258 |
log.info("Create an organizational unit with long ou") |
| 259 |
longname = '+' * 700 |
| 260 |
ous = OrganizationalUnits(s1, DEFAULT_SUFFIX) |
| 261 |
ou = ous.create(properties={ 'ou': longname }) |
| 262 |
|
| 263 |
log.info("Add children with long rdn") |
| 264 |
users_dict_s1 = {} |
| 265 |
users = UserAccounts(s1, ou.dn, rdn=None) |
| 266 |
create_long_rdn_users(users, users_dict_s1) |
| 267 |
|
| 268 |
log.info("Rename the organizational unit") |
| 269 |
longname = '-' * 700 |
| 270 |
ou.rename(f'ou={longname}') |
| 271 |
|
| 272 |
# Check that user exists |
| 273 |
users_list = [] |
| 274 |
for user in users_dict_s1.values(): |
| 275 |
renamed_dn = user.dn.replace('+', '-') |
| 276 |
renamed_user = UserAccount(s1, renamed_dn) |
| 277 |
users_list.append(renamed_user) |
| 278 |
for user in users_list: |
| 279 |
assert user.exists() |
| 280 |
|
| 281 |
log.info("Delete the users with a long rdn") |
| 282 |
for user in users_list: |
| 283 |
user.delete() |
| 284 |
for user in users_list: |
| 285 |
assert not user.exists() |
| 286 |
|
| 287 |
log.info("Delete the ou with a long rdn") |
| 288 |
ou.delete() |
| 289 |
assert not ou.exists() |
| 290 |
|
| 291 |
def test_entry_rdn_same_as_suffix(topology_st, request): |
| 292 |
""" |
| 293 |
Test that a reindex is successful even if an entry |
| 294 |
has a RDN that is identical to the suffix |
| 295 |
|
| 296 |
:id: 7f5a38e9-b979-4664-b132-81df0e60f38a |
| 297 |
:setup: standalone |
| 298 |
:steps: |
| 299 |
1. Create a new backend with suffix 'dc=dup_rdn' (ID 1) |
| 300 |
2. Create a dummy entry 'ou=my_org,dc=dup_rdn' (ID 2) |
| 301 |
3. Create the problematic entry 'dc=dup_rdn,dc=dup_rdn' (ID 3) |
| 302 |
4. Create a dummy entry 'ou=my_org,dc=dup_rdn,dc=dup_rdn' (ID 4) |
| 303 |
5. Do an offline reindex |
| 304 |
6. Check that entryrdn contains the key P3 (parent of ID 3) |
| 305 |
7. Check that error log does not contain 'entryrdn_insert_key - Same DN' |
| 306 |
:expectedresults: |
| 307 |
1. Should succeed |
| 308 |
2. Should succeed |
| 309 |
3. Should succeed |
| 310 |
4. Should succeed |
| 311 |
5. Should succeed |
| 312 |
6. Should succeed |
| 313 |
7. Should succeed |
| 314 |
""" |
| 315 |
inst = topology_st.standalone |
| 316 |
|
| 317 |
# Create a suffix 'dc=dup_rdn' |
| 318 |
be_name = 'domain' |
| 319 |
dc_value = 'dup_rdn' |
| 320 |
suffix = 'dc=' + dc_value |
| 321 |
rdn = 'my_org' |
| 322 |
be1 = Backend(inst) |
| 323 |
be1.create(properties={ |
| 324 |
'cn': be_name, |
| 325 |
'nsslapd-suffix': suffix, |
| 326 |
}, |
| 327 |
create_mapping_tree=False |
| 328 |
) |
| 329 |
|
| 330 |
mts = MappingTrees(inst) |
| 331 |
mt = mts.create(properties={ |
| 332 |
'cn': suffix, |
| 333 |
'nsslapd-state': 'backend', |
| 334 |
'nsslapd-backend': be_name, |
| 335 |
}) |
| 336 |
|
| 337 |
# Create the domain entry 'dc=dup_rdn' |
| 338 |
create_base_domain(inst, suffix) |
| 339 |
|
| 340 |
# Create the org ou=my_org,dc=dup_rdn |
| 341 |
ous = OrganizationalUnits(inst, suffix) |
| 342 |
ou = ous.create(properties={ 'ou': rdn }) |
| 343 |
|
| 344 |
# when reindexing entryrdn the following entry |
| 345 |
# (dc=dup_rdn,dc=dup_rdn) Triggers |
| 346 |
# this message. |
| 347 |
# This is because its RDN (dc=dup_rdn) is also |
| 348 |
# the suffix DN |
| 349 |
info_message = 'entryrdn_insert_key - Same DN (dn: %s) is already in the entryrdn file with different' % (ou.dn) |
| 350 |
log.info("In case if the bug still exist this line should be in the error log") |
| 351 |
log.info(" --> " + info_message) |
| 352 |
|
| 353 |
# Create the domain entry 'dc=dup_rdn,dc=dup_rdn' |
| 354 |
trigger_entry = suffix + "," + suffix |
| 355 |
domain = Domain(inst, dn=trigger_entry) |
| 356 |
domain.create(properties={ |
| 357 |
'dc': dc_value, |
| 358 |
'description': 'Entry with RDN identical to suffix' |
| 359 |
}) |
| 360 |
|
| 361 |
# Create the org ou=my_org,dc=dup_rdn,dc=dup_rdn |
| 362 |
ous = OrganizationalUnits(inst, trigger_entry) |
| 363 |
ou = ous.create(properties={ 'ou': rdn }) |
| 364 |
|
| 365 |
|
| 366 |
# Trigger an offline reindex |
| 367 |
log.info('Offline reindex, stopping the server') |
| 368 |
topology_st.standalone.stop() |
| 369 |
|
| 370 |
log.info('Reindex all the suffix') |
| 371 |
topology_st.standalone.db2index(bename=be_name) |
| 372 |
|
| 373 |
# make sure the key 'P3' (parent of 'dc=dup_rdn,dc=dup_rdn') exists |
| 374 |
dbscanout = topology_st.standalone.dbscan(bename=be_name, index='entryrdn') |
| 375 |
log.info(dbscanout) |
| 376 |
assert(ensure_bytes('P3') in ensure_bytes(dbscanout)) |
| 377 |
|
| 378 |
# make sure there is no failure detected/logged in error logs |
| 379 |
if topology_st.standalone.get_db_lib() == "mdb": |
| 380 |
pattern_str = ".*Inconsistent id2entry database.*" |
| 381 |
else: |
| 382 |
pattern_str = ".*entryrdn_insert_key - Same DN.*is already in the entryrdn file with different.*$" |
| 383 |
assert not topology_st.standalone.ds_error_log.match(pattern_str) |
| 384 |
|
| 385 |
|
| 386 |
def fin(): |
| 387 |
topology_st.standalone.restart() |
| 388 |
mt.delete() |
| 389 |
be1.delete() |
| 390 |
|
| 391 |
request.addfinalizer(fin) |
| 392 |
|
| 393 |
if __name__ == "__main__": |
| 394 |
# Run isolated |
| 395 |
# -s for DEBUG mode |
| 396 |
CURRENT_FILE = os.path.realpath(__file__) |
| 397 |
pytest.main("-s %s" % CURRENT_FILE) |