# --- BEGIN COPYRIGHT BLOCK --- # Copyright (C) 2026 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). # See LICENSE for details. # --- END COPYRIGHT BLOCK --- # import pytest, time import os import glob import ldap import pwd from shutil import copyfile, rmtree from datetime import datetime from contextlib import contextmanager, suppress from lib389.tasks import * from lib389.utils import * from lib389.dseutils import get_ldapurl_from_serverid from test389.topologies import topology_m2, topology_st from lib389.replica import * from lib389._constants import * from lib389.properties import TASK_WAIT from lib389.index import * from lib389.mappingTree import * from lib389.backend import * from lib389.dirsrv_log import DirsrvAccessLog from lib389.idm.user import UserAccounts, UserAccount from lib389.idm.organization import Organization from ldap.controls.vlv import VLVRequestControl from ldap.controls.sss import SSSRequestControl from lib389.replica import BootstrapReplicationManager, Replicas, ReplicationManager from lib389.agreement import Agreements from lib389.cli_ctl.dblib import run_dbscan from lib389.idm.organizationalunit import OrganizationalUnits pytestmark = pytest.mark.tier1 logging.getLogger(__name__).setLevel(logging.DEBUG) log = logging.getLogger(__name__) DEBUGGING = os.getenv("DEBUGGING", default=False) STARTING_UID_INDEX = 1000 # VLV search always returns first the entry before the target, # and the entry uid=demo_user,ou=people,dc=example,dc=com # is before the uid=testuserNNNN,dc=example,dc=com entries # so VLV_SEARCH_OFFSET (The difference between the vlv index # and the NNNN value) is: VLV_SEARCH_OFFSET = STARTING_UID_INDEX - 2 DEMO_PW = 'secret12' # A VLV Index with invalid vlvSrch: # ( Using objectClass: extensibleobject instead of objectClass: vlvSrch ) BAD_DSE_DATA = """dn: cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config objectClass: top objectClass: extensibleobject cn: vlvSrch vlvBase: dc=example,dc=com vlvFilter: (uid=*) vlvScope: 2 modifiersName: cn=directory manager createTimestamp: 20240110175704Z modifyTimestamp: 20240110175704Z numSubordinates: 1 dn: cn=vlvIdx,cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config objectClass: top objectClass: vlvIndex cn: vlvIdx vlvSort: cn creatorsName: cn=directory manager modifiersName: cn=directory manager createTimestamp: 20240110175704Z modifyTimestamp: 20240110175704Z """ def open_new_ldapi_conn(dsinstance): ldapurl, certdir = get_ldapurl_from_serverid(dsinstance) assert 'ldapi://' in ldapurl conn = ldap.initialize(ldapurl) conn.sasl_interactive_bind_s("", ldap.sasl.external()) return conn def check_vlv_search(conn, **kwargs): before_count = kwargs.get('before_count', 1) after_count = kwargs.get('after_count', 3) offset = kwargs.get('offset', 3501) basedn = kwargs.get('basedn', DEFAULT_SUFFIX) log.info('Testing that VLV search is returning the expected entries: ' f'basedn={basedn} ' f'before_count={before_count} ' f'after_count={after_count} ' f'offset={offset}') vlv_control = VLVRequestControl(criticality=True, before_count=before_count, after_count=after_count, offset=offset, content_count=0, greater_than_or_equal=None, context_id=None) sss_control = SSSRequestControl(criticality=True, ordering_rules=['cn']) result = conn.search_ext_s( base=basedn, scope=ldap.SCOPE_SUBTREE, filterstr='(uid=*)', serverctrls=[vlv_control, sss_control] ) imin = offset + VLV_SEARCH_OFFSET - before_count imax = offset + VLV_SEARCH_OFFSET + after_count for i, (dn, entry) in enumerate(result, start=imin): assert i <= imax expected_rdn = f'uid=testuser{i},' log.debug(f'found {dn} should starts with {expected_rdn}') assert dn.lower().startswith(expected_rdn) def add_user(inst, users, uid): user_properties = { 'uid': f'testuser{uid}', 'cn': f'testuser{uid}', 'sn': 'user', 'uidNumber': str(uid), 'gidNumber': str(uid), 'homeDirectory': f'/home/testuser{uid}' } users.create(properties=user_properties) def add_users(inst, users_num, suffix=DEFAULT_SUFFIX): users = UserAccounts(inst, suffix) log.info(f'Adding {users_num} users') for i in range(0, users_num): uid = STARTING_UID_INDEX + i add_user(inst, users, uid) def create_vlv_search_and_index(inst, basedn=DEFAULT_SUFFIX, bename='userRoot', scope=ldap.SCOPE_SUBTREE, prefix="vlv", vlvsort="cn"): vlv_searches = VLVSearch(inst) vlv_search_properties = { "objectclass": ["top", "vlvSearch"], "cn": f"{prefix}Srch", "vlvbase": basedn, "vlvfilter": "(uid=*)", "vlvscope": str(scope), } vlv_searches.create( basedn=f"cn={bename},cn=ldbm database,cn=plugins,cn=config", properties=vlv_search_properties ) vlv_index = VLVIndex(inst) vlv_index_properties = { "objectclass": ["top", "vlvIndex"], "cn": f"{prefix}Idx", "vlvsort": vlvsort, } vlv_index.create( basedn=f"cn={prefix}Srch,cn={bename},cn=ldbm database,cn=plugins,cn=config", properties=vlv_index_properties ) return vlv_searches, vlv_index def remove_entries(inst, basedn, filter): # Remove all entries matching the criteriae. conn = open_new_ldapi_conn(inst.serverid) entries = conn.search_s(basedn, ldap.SCOPE_SUBTREE, filter) for entry in entries: conn.delete_s(entry[0]) def cleanup(inst): # Remove the left over from previous tests. remove_entries(inst, "cn=config", "(objectclass=vlvIndex)") remove_entries(inst, "cn=config", "(objectclass=vlvSearch)") remove_entries(inst, DEFAULT_SUFFIX, "(cn=testuser*)") class BackendHandler: def __init__(self, inst, bedict, scope=ldap.SCOPE_ONELEVEL): self.inst = inst self.bedict = bedict self.bes = Backends(inst) self.scope = scope self.data = {} def find_backend(self, bename): for be in self.bes.list(): if be.get_attr_val_utf8_l('cn') == bename: return be return None def cleanup(self): benames = list(self.bedict.keys()) benames.reverse() for bename in benames: be = self.find_backend(bename) if be: be.delete() def setup(self): # Create backends, add vlv index and populate the backends. for bename,suffix in self.bedict.items(): be = self.bes.create(properties={ 'cn': bename, 'nsslapd-suffix': suffix, }) # Add suffix entry Organization(self.inst, dn=suffix).create(properties={ 'o': bename, }) # Configure vlv vlv_search, vlv_index = create_vlv_search_and_index( self.inst, basedn=suffix, bename=bename, scope=self.scope, prefix=f'vlv_1lvl_{bename}') # Reindex reindex_task = Tasks(self.inst) assert reindex_task.reindex( suffix=suffix, attrname=vlv_index.rdn, args={TASK_WAIT: True}, vlv=True ) == 0 # Add ou=People entry OrganizationalUnits(self.inst, suffix).create(properties={'ou': 'People'}) # Add another ou that will be deleted before the export # so that import will change the vlv search basedn entryid ou2 = OrganizationalUnits(self.inst, suffix).create(properties={'ou': 'dummy ou'}) # Add a demo user so that vlv_check is happy dn = f'uid=demo_user,ou=people,{suffix}' UserAccount(self.inst, dn=dn).create( properties= { 'uid': 'demo_user', 'cn': 'Demo user', 'sn': 'Demo user', 'uidNumber': '99998', 'gidNumber': '99998', 'homeDirectory': '/var/empty', 'loginShell': '/bin/false', 'userpassword': DEMO_PW }) # Add regular user add_users(self.inst, 10, suffix=suffix) # Removing ou2 ou2.delete() # And export tasks = Tasks(self.inst) ldif = f'{self.inst.get_ldif_dir()}/db-{bename}.ldif' assert tasks.exportLDIF(suffix=suffix, output_file=ldif, args={TASK_WAIT: True}) == 0 # Add the various parameters in topology_st.belist self.data[bename] = { 'be': be, 'suffix': suffix, 'ldif': ldif, 'vlv_search' : vlv_search, 'vlv_index' : vlv_index, 'dn' : dn} @pytest.fixture def vlv_setup_with_two_backend(topology_st, request): inst = topology_st.standalone beh = BackendHandler(inst, { 'be1': 'o=be1', 'be2': 'o=be2' }) def fin(): # Cleanup function if not DEBUGGING and inst.exists() and inst.status(): beh.cleanup() request.addfinalizer(fin) # Make sure that our backend are not already present. beh.cleanup() # Add userRoot backend to the backend list be0 = beh.find_backend('userroot') be0ldif = f'{inst.get_ldif_dir()}/db-userroot.ldif' assert Tasks(inst).exportLDIF(suffix=DEFAULT_SUFFIX, output_file=be0ldif, args={TASK_WAIT: True}) == 0 beh.data['userroot'] = { 'be': be0, 'suffix': DEFAULT_SUFFIX, 'ldif': be0ldif, } # Then add two new backends beh.setup() topology_st.beh = beh topology_st.belist = list(beh.data.values()) return topology_st @pytest.fixture def vlv_setup_nested_backends(topology_st, request): inst = topology_st.standalone beh = BackendHandler(inst, { 'be1': 'o=be1', 'be2': 'o=be2,o=be1' }, scope=ldap.SCOPE_SUBTREE) def fin(): # Cleanup function if not DEBUGGING and inst.exists() and inst.status(): beh.cleanup() request.addfinalizer(fin) # Make sure that our backend are not already present. beh.cleanup() beh.setup() topology_st.beh = beh return topology_st @pytest.fixture def vlv_setup_with_uid_mr(topology_st, request): inst = topology_st.standalone bename = 'be1' besuffix = f'o={bename}' beh = BackendHandler(inst, { bename: besuffix }) def fin(): # Cleanup function if not DEBUGGING and inst.exists() and inst.status(): beh.cleanup() request.addfinalizer(fin) # Make sure that our backend are not already present. beh.cleanup() # Then add the new backend beh.setup() index = Index(inst, f'cn=uid,cn=index,cn={bename},cn=ldbm database,cn=plugins,cn=config') index.add('nsMatchingRule', '2.5.13.2') reindex_task = Tasks(inst) assert reindex_task.reindex( suffix=besuffix, attrname='uid', args={TASK_WAIT: True} ) == 0 topology_st.beh = beh return topology_st @pytest.fixture def freeipa(topology_st): # generate a standalone instance with same vlv config than freeipa inst = topology_st.standalone datadir = os.path.join(os.path.dirname(__file__), '../../data/freeipa/issue6136') # Stop instance inst.stop() # Copy schema for file in glob.glob(f'{datadir}/schema/*.ldif'): target = f'{inst.schemadir}/{os.path.basename(file)}' log.info(f'Copying Schema {file} to {target}') copyfile(file, target) # append dse file = f'{datadir}/dse.ldif' target = f'{inst.confdir}/{os.path.basename(file)}' log.info(f'Appending ldif {file} to dse {target}') with open(file, 'r') as fin: with open(target, 'a') as fout: fout.write(fin.read()) # import ipaca # Make the ldif filke readable by ns-slapd process file_ref = f'{datadir}/ipaca.ldif' file = f'{inst.ds_paths.ldif_dir}/ipaca.ldif' log.info(f'Copying ldif {file_ref} to {file}') copyfile(file_ref, file) dirsrv = pwd.getpwnam('dirsrv') os.chown(file, dirsrv.pw_uid, dirsrv.pw_gid) # Then import it log.info(f'Importing ldif {file} to ipaca') assert inst.ldif2db('ipaca', None, None, None, file) # restart instance inst.restart() return topology_st def test_bulk_import_when_the_backend_with_vlv_was_recreated(topology_m2): """ Testing bulk import when the backend with VLV was recreated. If the test passes without the server crash, 47966 is verified. :id: 37d42d12-2544-49a0-81ef-7dbfb69edc0f :setup: Replication with two suppliers. :steps: 1. Generate vlvSearch entry 2. Generate vlvIndex entry 3. Delete the backend instance on Supplier 2 4. Delete the agreement, replica, and mapping tree, too. 5. Recreate the backend and the VLV index on Supplier 2. 6. Recreating vlvSrchDn and vlvIndexDn on Supplier 2. :expectedresults: 1. Should Success. 2. Should Success. 3. Should Success. 4. Should Success. 5. Should Success. 6. Should Success. """ M1 = topology_m2.ms["supplier1"] M2 = topology_m2.ms["supplier2"] # generate vlvSearch entry properties_for_search = { "objectclass": ["top", "vlvSearch"], "cn": "vlvSrch", "vlvbase": DEFAULT_SUFFIX, "vlvfilter": "(|(objectclass=*)(objectclass=ldapsubentry))", "vlvscope": "2", } vlv_searches = VLVSearch(M2) userroot_vlvsearch = vlv_searches.create( basedn="cn=userRoot,cn=ldbm database,cn=plugins,cn=config", properties=properties_for_search, ) assert "cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" in M2.getEntry( "cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config").dn # generate vlvIndex entry properties_for_index = { "objectclass": ["top", "vlvIndex"], "cn": "vlvIdx", "vlvsort": "cn ou sn", } vlv_index = VLVIndex(M2) userroot_index = vlv_index.create( basedn="cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config", properties=properties_for_index, ) assert "cn=vlvIdx,cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" in M2.getEntry( "cn=vlvIdx,cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config").dn # Delete the backend instance on Supplier 2." userroot_index.delete() userroot_vlvsearch.delete_all() # delete the agreement, replica, and mapping tree, too. repl = ReplicationManager(DEFAULT_SUFFIX) repl.remove_supplier(M2) MappingTrees(M2).list()[0].delete() Backends(M2).list()[0].delete() # Recreate the backend and the VLV index on Supplier 2. M2.backend.create(DEFAULT_SUFFIX, {BACKEND_NAME: "userRoot"}) M2.mappingtree.create(DEFAULT_SUFFIX, "userRoot") # Recreating vlvSrchDn and vlvIndexDn on Supplier 2. vlv_searches.create( basedn="cn=userRoot,cn=ldbm database,cn=plugins,cn=config", properties=properties_for_search, ) vlv_index.create( basedn="cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config", properties=properties_for_index, ) M2.restart() repl.join_supplier(M1, M2) repl.test_replication(M1, M2, 30) repl.test_replication(M2, M1, 30) entries = M2.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(uid=*)") assert len(entries) > 0 entries = M2.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(objectclass=*)") entries = M2.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(objectclass=*)") def test_vlv_recreation_reindex(topology_st): """Test VLV recreation and reindexing. :id: 29f4567f-4ac0-410f-bc99-a32e217a939f :setup: Standalone instance. :steps: 1. Create new VLVs and do the reindex. 2. Test the new VLVs. 3. Remove the existing VLVs. 4. Create new VLVs (with the same name). 5. Perform online re-indexing of the new VLVs. 6. Test the new VLVs. :expectedresults: 1. Should Success. 2. Should Success. 3. Should Success. 4. Should Success. 5. Should Success. 6. Should Success. """ inst = topology_st.standalone reindex_task = Tasks(inst) # Create and test VLVs vlv_search, vlv_index = create_vlv_search_and_index(inst) assert reindex_task.reindex( suffix=DEFAULT_SUFFIX, attrname=vlv_index.rdn, args={TASK_WAIT: True}, vlv=True ) == 0 add_users(inst, 5000) conn = open_new_ldapi_conn(inst.serverid) assert len(conn.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(uid=*)")) > 0 check_vlv_search(conn) # Remove and recreate VLVs vlv_index.delete() vlv_search.delete() vlv_search, vlv_index = create_vlv_search_and_index(inst) assert reindex_task.reindex( suffix=DEFAULT_SUFFIX, attrname=vlv_index.rdn, args={TASK_WAIT: True}, vlv=True ) == 0 conn = open_new_ldapi_conn(inst.serverid) assert len(conn.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(uid=*)")) > 0 check_vlv_search(conn) def get_db_filename(vlvname, prefix='', iscache=False): # Normalize the vlv db name chars = set('abcdefghijklmnopqrstuvwxyz0123456789') vlvname = ''.join(c for c in vlvname.lower() if c in chars) bename = 'userRoot' cache = '' if iscache: cache = '~recno-cache/' if get_default_db_lib() == "mdb": bename = bename.lower() return f'{prefix}{bename}/{cache}vlv#{vlvname}.db' def verify_keys_in_subdb(vlvname, inst, count): """ Verify that vlv index as the expected number of keys. """ dblib = get_default_db_lib() dbfile = get_db_filename(vlvname, prefix=f'{inst.dbdir}/') output = run_dbscan(['-D', dblib, '-f', dbfile]) # Count all lines except the MDB environment ones. found = output.count('\n') - output.count('MDB environment') if found != count: log.info(f'Running: dbscan -D {dblib} -f "{dbfile}"') log.info(f'dbscan output for vlv {vlvname} is: {output}') assert found == count def verify_vlv_subdb_names(vlvname, inst, count=None): """ Verify that vlv index and cache sub database names are conistent on lmdb. """ dblib = get_default_db_lib() if dblib == "bdb": # Let the time to sync bdb txn log to the db prefix = '' time.sleep(60) else: prefix = f'{inst.dbdir}/' output = run_dbscan(['-D', dblib, '-L', inst.dbdir]) vlvsubdbs = [] nbvlvindex = 0 for line in output.split("\n"): if not line: continue fname = line.split()[0] if 'vlv#' in fname: vlvsubdbs.append(fname) if '/~recno-cache/' not in fname: nbvlvindex += 1 log.info(f'vlv sub databases are: {vlvsubdbs}') assert get_db_filename(vlvname, prefix=prefix) in vlvsubdbs if dblib == "mdb": assert get_db_filename(vlvname, prefix=prefix, iscache=True) in vlvsubdbs assert len(vlvsubdbs) == 2 * nbvlvindex else: assert len(vlvsubdbs) == nbvlvindex if count: verify_keys_in_subdb(vlvname, inst, count) def bootstrap_replication(M1, M2): """ Setup a replication manager entry on M2 and update the M1 -> M2 replication agreement to use it. """ # Create a repl manager entry on M2 repl_manager_password = "Secret12" brm = BootstrapReplicationManager(M2) brm.create(properties={ 'cn': brm.common_name, 'userPassword': repl_manager_password }) # Let M1 -> M2 agreement use that entry replica1 = Replicas(M1).get(DEFAULT_SUFFIX) replica2 = Replicas(M2).get(DEFAULT_SUFFIX) agmt = replica1.get_agreements().list()[0] agmt.replace_many(("nsds5ReplicaBindDN", brm.dn), ("nsds5ReplicaCredentials", repl_manager_password)) replica2.remove_all('nsDS5ReplicaBindDNGroup') # Assign the entry to the replica replica2.replace('nsDS5ReplicaBindDN', brm.dn) def remove_database(inst): # remove the database inst.stop() for file in glob.glob(f'{inst.dbdir}/*'): if os.path.islink(file): continue if os.path.isfile(file): os.remove(file) if os.path.isdir(file): rmtree(file) inst.start() @pytest.mark.skipif(get_default_db_lib() == "bdb", reason="Not supported over bdb") def test_vlv_cache_subdb_names(topology_m2): """ Testing that vlv index cache sub database name is consistent. :id: 7a138006-aa24-11ee-9d5f-482ae39447e5 :setup: Replication with two suppliers. :steps: 1. Generate vlvSearch and vlvIndex entries 2. Add users and wait that they get replicated. 3. Check subdb names and perform vlv searches 4. Destroy supplierr2 database and restart 5. Recreate M2 tree with a new user 6. Reinit M2 from M1 7. Check subdb names and perform vlv searches :expectedresults: 1. Should Success. 2. Should Success. 3. Should Success. 4. Should Success. 5. Should Success. 6. Should Success. 7. Should Success. """ NUM_USERS = 50 M1 = topology_m2.ms["supplier1"] M2 = topology_m2.ms["supplier2"] repl = ReplicationManager(DEFAULT_SUFFIX) # Clean test_bulk_import_when_the_backend_with_vlv_was_recreated leftover that cause trouble entries = [ UserAccount(M1, dn=f'uid=testuser{uid},DEFAULT_SUFFIX') for uid in range(STARTING_UID_INDEX, STARTING_UID_INDEX+NUM_USERS) ] entries.append(VLVIndex(M2, dn='cn=vlvIdx,cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config')) entries.append(VLVSearch(M2, dn='cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config')) for entry in entries: if entry.exists(): entry.delete() repl.wait_for_replication(M1, M2) # Restart the instance to workaround https://github.com/389ds/389-ds-base/issues/6029 if get_default_db_lib() == "bdb": M2.restart() # generate vlvSearch and vlvIndex entries vlv_search, vlv_index = create_vlv_search_and_index(M2) vlvname = vlv_index.get_attr_val_utf8_l('cn') assert "cn=vlvIdx,cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config" in M2.getEntry( "cn=vlvIdx,cn=vlvSrch,cn=userRoot,cn=ldbm database,cn=plugins,cn=config").dn reindex_task = Tasks(M2) assert reindex_task.reindex( suffix=DEFAULT_SUFFIX, attrname=vlv_index.rdn, args={TASK_WAIT: True}, vlv=True ) == 0 # Add users and wait that they get replicated. add_users(M1, NUM_USERS) repl.wait_for_replication(M1, M2) # Check subdb names and perform vlv searches conn = open_new_ldapi_conn(M2.serverid) count = len(conn.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(uid=*)")) assert count > 0 check_vlv_search(conn, offset=23) verify_vlv_subdb_names(vlvname, M2, count=count) # Destroy the database and restart # (Ensure that the db is really destroyed. # Avoid getting tricked if database is not properly cleared by the reimport) # by a double bug (index keys not properly reset) M2.stop() remove_database(M2) M2.start() # Recreate tree with a new user on M2 (To check that its key get removed) # This user is not replicated towards M1 because the db has been # recreated with a different generation number Domain(M2, DEFAULT_SUFFIX).create(properties={'dc': 'example'}) OrganizationalUnits(M2, DEFAULT_SUFFIX).create(properties={'ou': 'People'}) users = UserAccounts(M2, DEFAULT_SUFFIX) add_user(M2, users, 2*STARTING_UID_INDEX) bootstrap_replication(M1, M2) # Reinit M2 from M1 agmt = Agreements(M1).list()[0] agmt.begin_reinit() (done, error) = agmt.wait_reinit() assert done is True assert error is False # Check subdb names and perform vlv searches conn = open_new_ldapi_conn(M2.serverid) count = len(conn.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(uid=*)")) assert count > 0 check_vlv_search(conn, offset=23) verify_vlv_subdb_names(vlvname, M2, count=count) @contextmanager def custom_dse(*args, **kwds): """ A Constext manager to handle a special configuration. That restores the original config when exiting out of the context. Usage: with custom_dse(inst) as newdsename, olddsename: update(newdsename) inst.start() action ... """ inst = args[0] olddsename = f'{inst.ds_paths.config_dir}/dse.ldif.vlvtc1' newdsename = f'{inst.ds_paths.config_dir}/dse.ldif' inst.stop() assert os.path.isfile(newdsename) os.replace(newdsename, olddsename) copyfile(olddsename, newdsename) try: yield (newdsename, olddsename) finally: inst.stop() if os.path.isfile(olddsename): os.remove(newdsename) os.rename(olddsename, newdsename) inst.start() def test_vlv_start_with_bad_dse(topology_st): """ Testing that server does not crash if dse.ldif is in disorder. :id: ceb3e9e0-b607-11ee-845a-482ae39447e5 :setup: Standalone :steps: 1. Append vlvIndex entries then vlvSearch entry in the dse.ldif 2. Restart the server :expectedresults: 1. Should Success. 2. Should Success. """ inst = topology_st.standalone # Use a context to ensure that the instance config is restored and instance # restarted at the end of the test. with custom_dse(inst) as (newdsename, olddsename): # Step 1: Append vlvIndex entries then vlvSearch entry in the dse.ldif with open(newdsename, 'at') as dse: dse.write(BAD_DSE_DATA) # Step 2: Restart the server inst.start() @pytest.mark.skipif(get_default_db_lib() == "bdb", reason="bdb may hang because of github #6029") @pytest.mark.parametrize("prefix", ( 'vlv', 'vl-v' )) @pytest.mark.parametrize("basedn", ( DEFAULT_SUFFIX, f'ou=People,{DEFAULT_SUFFIX}' )) def test_vlv_reindex(topology_st, prefix, basedn): """Test VLV reindexing. :id: d5dc0d8e-cbe6-11ee-95b1-482ae39447e5 :parametrized: yes :setup: Standalone instance. :steps: 1. Cleanup leftover from previous tests 2. Add users 3. Create new VLVs and do the reindex. 4. Test the new VLVs. 5. Restart the server and test again the VLVs :expectedresults: 1. Should Success. 2. Should Success. 3. Should Success. 4. Should Success. 5. Should Success. """ NUM_USERS = 50 inst = topology_st.standalone reindex_task = Tasks(inst) # Clean previous tests leftover that cause trouble cleanup(inst) add_users(inst, NUM_USERS) # Create and reindex VLVs vlv_search, vlv_index = create_vlv_search_and_index(inst, prefix=prefix, basedn=basedn) assert reindex_task.reindex( suffix=DEFAULT_SUFFIX, attrname=vlv_index.rdn, args={TASK_WAIT: True}, vlv=True ) == 0 conn = open_new_ldapi_conn(inst.serverid) count = len(conn.search_s(basedn, ldap.SCOPE_SUBTREE, "(uid=*)")) assert count > 0 verify_vlv_subdb_names(vlv_index.rdn, inst, count=count) check_vlv_search(conn, offset=23, basedn=basedn) # Restart and check again inst.restart() conn = open_new_ldapi_conn(inst.serverid) count = len(conn.search_s(basedn, ldap.SCOPE_SUBTREE, "(uid=*)")) assert count > 0 verify_vlv_subdb_names(vlv_index.rdn, inst, count=count) check_vlv_search(conn, offset=23, basedn=basedn) @pytest.mark.skipif(get_default_db_lib() == "bdb", reason="bdb may hang because of github #6029") @pytest.mark.parametrize("prefix", ( 'vlv', 'vl-v' )) @pytest.mark.parametrize("basedn", ( DEFAULT_SUFFIX, f'ou=People,{DEFAULT_SUFFIX}' )) def test_vlv_offline_import(topology_st, prefix, basedn): """Test VLV after off line import. :id: 8732d7a8-e851-11ee-9d63-482ae39447e5 :parametrized: yes :setup: Standalone instance. :steps: 1. Cleanup leftover from previous tests 2. Add users 3. Create new VLVs and do the reindex. 4. Test the new VLVs. 5. Export userRoot backend 6. Stop the instance 7. Perform off-line import 8. Restart instance and check the vlvs :expectedresults: 1. Should Success. 2. Should Success. 3. Should Success. 4. Should Success. 5. Should Success. 6. Should Success. 7. Should Success. 8. Should Success. """ NUM_USERS = 50 inst = topology_st.standalone tasks = Tasks(inst) # Clean previous tests leftover that cause trouble cleanup(inst) add_users(inst, NUM_USERS) # Create and reindex VLVs vlv_search, vlv_index = create_vlv_search_and_index(inst, prefix=prefix, basedn=basedn) assert tasks.reindex( suffix=DEFAULT_SUFFIX, attrname=vlv_index.rdn, args={TASK_WAIT: True}, vlv=True ) == 0 conn = open_new_ldapi_conn(inst.serverid) count = len(conn.search_s(basedn, ldap.SCOPE_SUBTREE, "(uid=*)")) assert count > 0 verify_vlv_subdb_names(vlv_index.rdn, inst, count=count) check_vlv_search(conn, offset=23, basedn=basedn) # Export to ldif ldif = f'{inst.get_ldif_dir()}/db.ldif' assert tasks.exportLDIF( suffix=DEFAULT_SUFFIX, output_file=ldif, args={TASK_WAIT: True} ) == 0 # Perform off-line import inst.stop() assert inst.ldif2db('userRoot', None, None, None, ldif) # Restart and check again the vlv inst.start() conn = open_new_ldapi_conn(inst.serverid) count = len(conn.search_s(basedn, ldap.SCOPE_SUBTREE, "(uid=*)")) assert count > 0 verify_vlv_subdb_names(vlv_index.rdn, inst, count=count) check_vlv_search(conn, offset=23, basedn=basedn) def test_vlv_as_freeipa_backup_restore(freeipa): """Test vlv as if freeipa restore was performed. :id: fb0621da-e879-11ee-900e-482ae39447e5 :setup: Standalone instance with freeipa test config. :steps: 1. Check that standard search returns 16 valid certificate 2. Check that vlv search returns 16 valid certificate :expectedresults: 1. Should Success. 2. Should Success. """ DATE_LIMIT = '20260323135138Z' EXPECTED_COUNT = 12 inst = freeipa.standalone inst.restart() conn = open_new_ldapi_conn(inst.serverid) basedn = 'ou=certificateRepository,ou=ca,o=ipaca' count = len(conn.search_s(basedn, ldap.SCOPE_SUBTREE, f"(&(certStatus=VALID)(notAfter<={DATE_LIMIT}))")) assert count == EXPECTED_COUNT vlv_control = VLVRequestControl(criticality=True, before_count=200, after_count=0, offset=None, content_count=None, greater_than_or_equal='20260323135138Z', context_id=None) sss_control = SSSRequestControl(criticality=True, ordering_rules=['notAfter']) result = conn.search_ext_s( base=basedn, scope=ldap.SCOPE_SUBTREE, filterstr='(certStatus=VALID)', serverctrls=[vlv_control, sss_control] ) assert len(result) == EXPECTED_COUNT def test_vlv_scope_one_on_two_backends(vlv_setup_with_two_backend): """Test one level scoped vlv on two backends. :id: 7652a2ea-eab0-11ee-891f-482ae39447e5 :setup: Standalone instance with two backens and one level scoped vlv :steps: 1. Check that standard search returns 16 valid certificate 2. Check that vlv search returns 16 valid certificate :expectedresults: 1. Should Success. 2. Should Success. """ inst = vlv_setup_with_two_backend.standalone tasks = Tasks(inst) conn = open_new_ldapi_conn(inst.serverid) assert len(conn.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(uid=*)")) > 0 # Check that vlv are working as excpected for tmpd in vlv_setup_with_two_backend.belist: if 'vlv_index' in tmpd: check_vlv_search(conn, offset=6, basedn=tmpd['suffix']) # Export userRoot backend ldif = f'{inst.get_ldif_dir()}/db.ldif' assert tasks.exportLDIF( suffix=DEFAULT_SUFFIX, output_file=ldif, args={TASK_WAIT: True} ) == 0 # Remove the db remove_database(inst) conn = open_new_ldapi_conn(inst.serverid) # Check that vlv are broken for tmpd in vlv_setup_with_two_backend.belist: if 'vlv_index' in tmpd: with pytest.raises(ldap.NO_SUCH_OBJECT): check_vlv_search(conn, offset=6, basedn=tmpd['suffix']) # Reimport the backends for tmpd in vlv_setup_with_two_backend.belist: assert tasks.importLDIF( suffix=tmpd['suffix'], input_file=tmpd['ldif'], args={TASK_WAIT: True}, ) == 0 # Check that vlv are working as excpected for tmpd in vlv_setup_with_two_backend.belist: if 'vlv_index' in tmpd: check_vlv_search(conn, offset=6, basedn=tmpd['suffix']) def test_vlv_by_keyword(freeipa): """Test vlv search by keyword instead of by range :id: 0e74bbf6-f5c6-11ee-b086-482ae39447e5 :setup: Standalone instance with freeipa test config. :steps: 1. Perform a vlv search returning the six entries with highest serialno that is lesser or equal than "0210" 2. Check that the 6 expected entries are returned. :expectedresults: 1. Should Success. 2. Should Success. """ inst = freeipa.standalone conn = open_new_ldapi_conn(inst.serverid) basedn = 'ou=certificateRepository,ou=ca,o=ipaca' vlv_control = VLVRequestControl(criticality=True, before_count=5, after_count=0, offset=None, content_count=None, greater_than_or_equal='0211', context_id=None) sss_control = SSSRequestControl(criticality=True, ordering_rules=['serialno']) result = conn.search_ext_s( base=basedn, scope=ldap.SCOPE_ONELEVEL, filterstr='(certStatus=*)', serverctrls=[vlv_control, sss_control] ) assert len(result) == 6 dns = [ dn for dn,entry in result ] for idx in range(6,12): assert f'cn={idx},ou=certificateRepository,ou=ca,o=ipaca' in dns def get_timestamp_attr(): current_datetime = datetime.now() return f'tsattr-{current_datetime.timestamp()}'.replace(".", "-") def perform_vlv_search(conn, alog, basedn, vlvcrit=True, ssscrit=True, scope=ldap.SCOPE_SUBTREE, sss='cn', filter='(uid=*)'): timestamp = get_timestamp_attr() vlv_control = VLVRequestControl(criticality=vlvcrit, before_count=1, after_count=1, offset=4, content_count=0, greater_than_or_equal=None, context_id=None) sss_control = SSSRequestControl(criticality=ssscrit, ordering_rules=[sss]) log.info(f'perform_vlv_search: basedn={basedn} ={vlvcrit} ssscrit={ssscrit} sss={sss} filter={filter} timestamp={timestamp}') with suppress(ldap.LDAPError): result = conn.search_ext_s( base=basedn, scope=scope, filterstr=filter, attrlist=[timestamp], serverctrls=[vlv_control, sss_control] ) line = alog.match(f'.*SRCH.*{timestamp}.*') log.info(f'perform_vlv_search: line={line}') match = re.match(r'.*conn=(\d+) op=(\d+).*', line[0]) conn,op = match.group(1,2) log.info(f'perform_vlv_search: conn={conn} op={op} ') lines = ''.join(alog.match(f'.*conn={conn} op={op} .*', after_pattern=f'.*{timestamp}.*')) log.info(f'perform_vlv_search: lines={lines}') return lines def test_vlv_logs(vlv_setup_nested_backends): """Test than VLV abd SORT lines are properply written :id: a1d9ad9e-7cbb-11ef-82e3-083a88554478 :setup: Standalone instance with two backens and one level scoped vlv :steps: 1. Check that standard search returns 16 valid certificate 2. Check that vlv search returns 16 valid certificate :expectedresults: 1. Should Success. 2. Should Success. """ inst = vlv_setup_nested_backends.standalone beh = vlv_setup_nested_backends.beh tasks = Tasks(inst) conn = open_new_ldapi_conn(inst.serverid) conn_demo = open_new_ldapi_conn(inst.serverid) alog = DirsrvAccessLog(inst) dn1 = beh.data['be1']['dn'] dn2 = beh.data['be2']['dn'] suffix1 = beh.data['be1']['suffix'] suffix2 = beh.data['be2']['suffix'] conn_demo.bind_s(dn2, DEMO_PW) VLV_FEATURE_DN = 'oid=2.16.840.1.113730.3.4.9,cn=features,cn=config' VLV_DEFAULT_ACI = b'(targetattr != "aci")(version 3.0; acl "VLV Request Control"; ' + \ b'allow( read , search, compare, proxy ) userdn = "ldap:///all";)' VLV_DENY_ACI = b'(targetattr != "aci")(version 3.0; acl "VLV Request Control"; ' + \ f'deny( read , search, compare, proxy ) userdn = "ldap:///{dn2}";)'.encode('utf8') # Set VLV feature ACI mod = (ldap.MOD_REPLACE, 'aci', [ VLV_DEFAULT_ACI, VLV_DENY_ACI ]) conn.modify_s(VLV_FEATURE_DN, [mod,]) # Invalid ACL res = perform_vlv_search(conn_demo, alog, suffix2) assert 'SORT' in res assert 'VLV' in res assert 'err=50 ' in res # Sucessful VLV res = perform_vlv_search(conn, alog, suffix2) assert 'SORT' in res assert 'VLV' in res assert 'err=0 ' in res # Multiple backends SSS and VLV are critical res = perform_vlv_search(conn, alog, suffix1) assert 'SORT' in res assert 'VLV' in res assert 'err=76 ' in res # Multiple backends SSS is critical VLV is not critical res = perform_vlv_search(conn, alog, suffix1, vlvcrit=False) assert 'SORT' in res assert 'VLV' in res assert 'err=12 ' in res # Multiple backends SSS and VLV are not critical res = perform_vlv_search(conn, alog, suffix1, vlvcrit=False, ssscrit=False) assert 'SORT' in res assert 'VLV' in res assert 'err=0 ' in res def test_vlv_with_mr(vlv_setup_with_uid_mr): """ Testing vlv having specific matching rule :id: 5e04afe2-beec-11ef-aa84-482ae39447e5 :setup: Standalone with uid have a matching rule index :steps: 1. Append vlvIndex entries then vlvSearch entry in the dse.ldif 2. Restart the server :expectedresults: 1. Should Success. 2. Should Success. """ inst = vlv_setup_with_uid_mr.standalone beh = vlv_setup_with_uid_mr.beh bename, besuffix = next(iter(beh.bedict.items())) vlv_searches, vlv_index = create_vlv_search_and_index( inst, basedn=besuffix, bename=bename, vlvsort="uid:2.5.13.2") # Reindex the vlv reindex_task = Tasks(inst) assert reindex_task.reindex( suffix=besuffix, attrname=vlv_index.rdn, args={TASK_WAIT: True}, vlv=True ) == 0 inst.restart() users = UserAccounts(inst, besuffix) user_properties = { 'uid': f'a new testuser', 'cn': f'a new testuser', 'sn': 'user', 'uidNumber': '0', 'gidNumber': '0', 'homeDirectory': 'foo' } user = users.create(properties=user_properties) user.delete() assert inst.status() @pytest.mark.skipif(get_default_db_lib() == "bdb", reason="Hangs on BDB") def test_vlv_long_attribute_value(topology_st, request): """ Test VLV with an entry containing a very long attribute value (2K). :id: 99126fa4-003e-11f1-b7d6-c85309d5c3e3 :setup: Standalone instance. :steps: 1. Cleanup leftover from previous tests 2. Create VLV search and index on cn attribute 3. Reindex VLV 4. Add an entry with a cn attribute having 2K character value 5. Verify the entry was added successfully 6. Perform a VLV search to ensure it still works 7. Add another entry with a cn attribute having 2K character value 8. Verify the entry was added successfully 9. Perform a VLV search to ensure it still works :expectedresults: 1. Should Success. 2. Should Success. 3. Should Success. 4. Should Success. 5. Should Success. 6. Should Success. 7. Should Success. 8. Should Success. 9. Should Success. """ inst = topology_st.standalone reindex_task = Tasks(inst) users_to_delete = [] def fin(): cleanup(inst) # Clean the added users for user in users_to_delete: user.delete() if not DEBUGGING: request.addfinalizer(fin) # Clean previous tests leftover fin() # Create VLV search and index vlv_search, vlv_index = create_vlv_search_and_index(inst) assert reindex_task.reindex( suffix=DEFAULT_SUFFIX, attrname=vlv_index.rdn, args={TASK_WAIT: True}, vlv=True ) == 0 # Add a few regular users first add_users(inst, 10) # Create a very long cn value (2K characters) long_cn_value = 'a' * 2048 + '1' # Add an entry with the long cn attribute users = UserAccounts(inst, DEFAULT_SUFFIX) user_properties = { 'uid': 'longcnuser1', 'cn': long_cn_value, 'sn': 'user1', 'uidNumber': '99999', 'gidNumber': '99999', 'homeDirectory': '/home/longcnuser1' } user = users.create(properties=user_properties) users_to_delete.append(user); # Verify the entry was created and has the long cn value entry = user.get_attr_vals_utf8('cn') assert entry[0] == long_cn_value log.info(f'Successfully created user with cn length: {len(entry[0])}') # Perform VLV search to ensure VLV still works with long attribute values conn = open_new_ldapi_conn(inst.serverid) count = len(conn.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(uid=*)")) assert count > 0 log.info(f'VLV search successful with {count} entries including entry with 2K cn value') # Add another entry with the long cn attribute long_cn_value = 'a' * 2048 + '2' user_properties = { 'uid': 'longcnuser2', 'cn': long_cn_value, 'sn': 'user2', 'uidNumber': '99998', 'gidNumber': '99998', 'homeDirectory': '/home/longcnuser2' } user = users.create(properties=user_properties) users_to_delete.append(user); # Verify the entry was created and has the long cn value entry = user.get_attr_vals_utf8('cn') assert entry[0] == long_cn_value log.info(f'Successfully created user with cn length: {len(entry[0])}') # Perform VLV search to ensure VLV still works with long attribute values conn = open_new_ldapi_conn(inst.serverid) count = len(conn.search_s(DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, "(uid=*)")) assert count > 1 log.info(f'VLV search successful with {count} entries including entry with 2K cn value') if __name__ == "__main__": # Run isolated # -s for DEBUG mode CURRENT_FILE = os.path.realpath(__file__) pytest.main("-s %s" % CURRENT_FILE)