| 1 |
# --- BEGIN COPYRIGHT BLOCK --- |
| 2 |
# Copyright (C) 2025 Red Hat, Inc. |
| 3 |
# All rights reserved. |
| 4 |
# |
| 5 |
# License: GPL (version 3 or any later version). |
| 6 |
# See LICENSE for details. |
| 7 |
# --- END COPYRIGHT BLOCK --- |
| 8 |
# |
| 9 |
import os |
| 10 |
import logging |
| 11 |
import pytest |
| 12 |
import ldap |
| 13 |
import ldif |
| 14 |
import random |
| 15 |
from contextlib import suppress |
| 16 |
from lib389.backend import Backends |
| 17 |
from lib389.cli_base import FakeArgs |
| 18 |
from lib389.cli_ctl.dbgen import dbgen_create_groups |
| 19 |
from lib389.config import BDB_LDBMConfig, LMDB_LDBMConfig |
| 20 |
from lib389._constants import DEFAULT_SUFFIX, PLUGIN_AUTOMEMBER |
| 21 |
from lib389.dirsrv_log import DirsrvErrorLog |
| 22 |
from lib389.tasks import ImportTask |
| 23 |
from test389.topologies import topology_st |
| 24 |
from lib389.utils import get_default_db_lib |
| 25 |
|
| 26 |
|
| 27 |
pytestmark = pytest.mark.tier1 |
| 28 |
|
| 29 |
DEBUGGING = os.getenv("DEBUGGING", default=False) |
| 30 |
logging.getLogger(__name__).setLevel(logging.DEBUG) |
| 31 |
log = logging.getLogger(__name__) |
| 32 |
|
| 33 |
os.environ['DS389_MDB_MAX_SIZE'] = str(200 * 1024 * 1024) |
| 34 |
|
| 35 |
SEED = 195707 |
| 36 |
|
| 37 |
CONFIG_ATTR_PINNED_ENTRIES = 'nsslapd-cache-pinned-entries' |
| 38 |
|
| 39 |
# Ensure that the dn, attributes and values are not sorted |
| 40 |
class LdifScrambler(ldif.LDIFParser): |
| 41 |
|
| 42 |
def __init__(self, ldifin, ldifout): |
| 43 |
self.fdin = open(ldifin, 'rb') |
| 44 |
self.fdout = open(ldifout, 'wt') |
| 45 |
super().__init__(self.fdin) |
| 46 |
self.writer = ldif.LDIFWriter(self.fdout) |
| 47 |
self.others = [] |
| 48 |
self.users = [] |
| 49 |
self.user5 = [] |
| 50 |
self.groups = [] |
| 51 |
self.group5 = [] |
| 52 |
|
| 53 |
def close_fds(self): |
| 54 |
self.fdin.close() |
| 55 |
self.fdout.close() |
| 56 |
|
| 57 |
def handle(self,dn,entry): |
| 58 |
if 'cn=myGroup-5,ou=groups,dc=entrycache_test,dc=example,dc=com' in dn: |
| 59 |
self.group5.append((dn, entry)) |
| 60 |
elif ',ou=groups,dc=entrycache_test,dc=example,dc=com' in dn: |
| 61 |
self.groups.append((dn, entry)) |
| 62 |
elif 'uid=group_entry5-' in dn: |
| 63 |
self.user5.append((dn, entry)) |
| 64 |
elif 'uid=group_entry' in dn: |
| 65 |
self.users.append((dn, entry)) |
| 66 |
else: |
| 67 |
self.others.append((dn, entry)) |
| 68 |
|
| 69 |
def scramble(self): |
| 70 |
self.parse() |
| 71 |
log.debug(f'{len(self.others)} entries in self.others') |
| 72 |
log.debug(f'{len(self.users)} entries in self.users') |
| 73 |
log.debug(f'{len(self.user5)} entries in self.user5') |
| 74 |
log.debug(f'{len(self.groups)} entries in self.groups') |
| 75 |
log.debug(f'{len(self.group5)} entries in self.group5') |
| 76 |
for dn, entry in self.others: |
| 77 |
self.writer.unparse(dn, entry) |
| 78 |
self.scramble_entry(self.users) |
| 79 |
self.scramble_entry(self.groups) |
| 80 |
self.scramble_entry(self.user5) |
| 81 |
self.scramble_entry(self.group5) |
| 82 |
|
| 83 |
def scramble_entry(self, group): |
| 84 |
random.shuffle(group) |
| 85 |
for dn, entry in group: |
| 86 |
for vals in entry.values(): |
| 87 |
random.shuffle(vals) |
| 88 |
self.writer.unparse(dn, entry) |
| 89 |
|
| 90 |
|
| 91 |
@pytest.fixture(scope="function") |
| 92 |
def prepare_be(topology_st, request): |
| 93 |
|
| 94 |
inst = topology_st.standalone |
| 95 |
ldif_file = inst.get_ldif_dir() + '/30ku.ldif' |
| 96 |
ldif_file_scrambled = inst.get_ldif_dir() + '/30ku-scrambled.ldif' |
| 97 |
bename = 'entrycache_test' |
| 98 |
suffix = 'dc=entrycache_test,dc=example,dc=com' |
| 99 |
people_base = f'ou=people,{suffix}' |
| 100 |
groups_base = f'ou=groups,{suffix}' |
| 101 |
|
| 102 |
# Remove the backend if it exists |
| 103 |
bes = Backends(inst) |
| 104 |
with suppress(ldap.NO_SUCH_OBJECT): |
| 105 |
be1 = bes.get(bename) |
| 106 |
be1.delete() |
| 107 |
|
| 108 |
# Creates the backend. |
| 109 |
be1 = bes.create(properties={ 'cn': bename, 'nsslapd-suffix': suffix, }) |
| 110 |
|
| 111 |
# Prepare finalizer |
| 112 |
def fin(): |
| 113 |
be1.delete() |
| 114 |
if os.path.exists(ldif_file): |
| 115 |
os.remove(ldif_file) |
| 116 |
|
| 117 |
if not DEBUGGING: |
| 118 |
request.addfinalizer(fin) |
| 119 |
|
| 120 |
# Generates ldif file with a few clarge groups |
| 121 |
# And preset random seed to have deterministic test |
| 122 |
random.seed(SEED) |
| 123 |
args = FakeArgs() |
| 124 |
args.NAME = 'myGroup' |
| 125 |
args.parent = groups_base |
| 126 |
args.suffix = suffix |
| 127 |
args.number = 5 |
| 128 |
args.num_members = 6000 |
| 129 |
args.create_members = True |
| 130 |
args.member_attr = 'uniquemember' |
| 131 |
args.member_parent = people_base |
| 132 |
args.ldif_file = ldif_file |
| 133 |
dbgen_create_groups(inst, log, args) |
| 134 |
assert os.path.exists(ldif_file) |
| 135 |
|
| 136 |
# Turn off bdb cache autosizing |
| 137 |
if get_default_db_lib() == 'bdb': |
| 138 |
config_ldbm = BDB_LDBMConfig(inst) |
| 139 |
config_ldbm.set('nsslapd-cache-autosize', '0') |
| 140 |
else: |
| 141 |
config_ldbm = LMDB_LDBMConfig(inst) |
| 142 |
config_ldbm.set('nsslapd-cache-autosize', '0') |
| 143 |
|
| 144 |
# Set entry cache large enough to hold all the large groups |
| 145 |
# and with a limited number of entries to trigger eviction |
| 146 |
be1.replace('nsslapd-cachememsize', '8000000' ) |
| 147 |
be1.replace('nsslapd-cachesize', '100' ) |
| 148 |
# Set debugging trace specific for this test |
| 149 |
be1.replace('nsslapd-cache-debug-pattern', f'cn=.*,ou=groups,{suffix}' ) |
| 150 |
inst.config.set('nsslapd-errorlog-level', '266354688') |
| 151 |
inst.config.set('nsslapd-errorlog-logbuffering', 'off') |
| 152 |
inst.plugins.disable(name=PLUGIN_AUTOMEMBER) |
| 153 |
|
| 154 |
|
| 155 |
# import the ldif |
| 156 |
inst.stop() |
| 157 |
LdifScrambler(ldif_file, ldif_file_scrambled).scramble() |
| 158 |
if not inst.ldif2db(bename, None, None, None, ldif_file): |
| 159 |
log.fatal('Failed to import {ldif_file}') |
| 160 |
assert False |
| 161 |
with open(inst.ds_paths.error_log, 'r') as fin: |
| 162 |
lines = fin.readlines() |
| 163 |
with open(inst.ds_paths.error_log, 'w') as fout: |
| 164 |
for line in lines: |
| 165 |
line = line.replace('entrycache_add_int', 'Xntrycache_add_int') |
| 166 |
line = line.replace('entrycache_remove_int', 'Xntrycache_remove_int') |
| 167 |
fout.write(line) |
| 168 |
|
| 169 |
inst.start() |
| 170 |
|
| 171 |
return (bename, suffix, be1, people_base, groups_base) |
| 172 |
|
| 173 |
|
| 174 |
def grab_debug_logs(inst, last_count=0): |
| 175 |
errlog = DirsrvErrorLog(inst) |
| 176 |
msgs = errlog.match('.*entrycache_.*_int') |
| 177 |
new_msgs = msgs[last_count:] |
| 178 |
adds = [ line for line in new_msgs if 'entrycache_add_int' in line ] |
| 179 |
dels= [ line for line in new_msgs if 'entrycache_remove_int' in line ] |
| 180 |
return (new_msgs, adds, dels, len(msgs)) |
| 181 |
|
| 182 |
|
| 183 |
def test_entry_cache_eviction(topology_st, prepare_be): |
| 184 |
"""Test that large groups are not evicted |
| 185 |
|
| 186 |
:id: 550b995e-1c76-11f0-93ed-482ae39447e5 |
| 187 |
:setup: Standalone instance |
| 188 |
:steps: |
| 189 |
1. Create DS instance and prepare a test backend |
| 190 |
2. Search all entries |
| 191 |
3. Check error log that the group are added in cache but not removed |
| 192 |
4. Change the number of entries protected from eviction threshold |
| 193 |
so that no group is preserved. |
| 194 |
5. Search all entries |
| 195 |
6. Check error log that the group are removed except the last one |
| 196 |
7. Change the number of entries protected from eviction threshold |
| 197 |
so that one group is not preserved. |
| 198 |
8. Search all entries |
| 199 |
9. Check error log that the group are preserved except one |
| 200 |
:expectedresults: |
| 201 |
1. Success |
| 202 |
2. Success |
| 203 |
3. Success |
| 204 |
4. Success |
| 205 |
5. Success |
| 206 |
6. Success |
| 207 |
7. Success |
| 208 |
8. Success |
| 209 |
9. Success |
| 210 |
""" |
| 211 |
|
| 212 |
inst = topology_st.standalone |
| 213 |
bename, suffix, be1, people_base, groups_base = prepare_be |
| 214 |
|
| 215 |
log_count = 0 |
| 216 |
|
| 217 |
be1.replace(CONFIG_ATTR_PINNED_ENTRIES, '10') |
| 218 |
# Search all groups then all people to try to evict the groups from entrycache |
| 219 |
inst.search_s(groups_base, ldap.SCOPE_SUBTREE, '(objectclass=groupOfNames)', ['dn'], escapehatch='i am sure') |
| 220 |
inst.search_s(people_base, ldap.SCOPE_SUBTREE, '(objectclass=inetOrgPerson)', ['dn'], escapehatch='i am sure') |
| 221 |
|
| 222 |
# Check error logs |
| 223 |
msgs, adds, dels, log_count = grab_debug_logs(inst, log_count) |
| 224 |
# Should have entrycache_add_int for each group and no entrycache_delete_int |
| 225 |
assert len(adds) == 5 |
| 226 |
assert len(dels) == 0 |
| 227 |
|
| 228 |
be1.replace(CONFIG_ATTR_PINNED_ENTRIES, '0') |
| 229 |
# Search all people to evict groups from entrycache |
| 230 |
inst.search_s(people_base, ldap.SCOPE_SUBTREE, '(objectclass=inetOrgPerson)', ['dn'], escapehatch='i am sure') |
| 231 |
# Check error logs |
| 232 |
msgs, adds, dels, log_count = grab_debug_logs(inst, log_count) |
| 233 |
assert len(dels) == 5 |
| 234 |
|
| 235 |
# Search all groups then all people to try to evict the groups from entrycache |
| 236 |
inst.search_s(groups_base, ldap.SCOPE_SUBTREE, '(objectclass=groupOfNames)', ['dn'], escapehatch='i am sure') |
| 237 |
inst.search_s(people_base, ldap.SCOPE_SUBTREE, '(objectclass=inetOrgPerson)', ['dn'], escapehatch='i am sure') |
| 238 |
# Check error logs |
| 239 |
msgs, adds, dels, log_count = grab_debug_logs(inst, log_count) |
| 240 |
assert len(adds) == 5 |
| 241 |
assert len(dels) == 5 |
| 242 |
|
| 243 |
be1.replace(CONFIG_ATTR_PINNED_ENTRIES, '4') |
| 244 |
# Search all groups then all people to try to evict the groups from entrycache |
| 245 |
inst.search_s(groups_base, ldap.SCOPE_SUBTREE, '(objectclass=groupOfNames)', ['dn'], escapehatch='i am sure') |
| 246 |
inst.search_s(people_base, ldap.SCOPE_SUBTREE, '(objectclass=inetOrgPerson)', ['dn'], escapehatch='i am sure') |
| 247 |
# Check error logs |
| 248 |
# Should have entrycache_add_int for each group and one group removed (only 4 preserved) |
| 249 |
msgs, adds, dels, log_count = grab_debug_logs(inst, log_count) |
| 250 |
assert len(adds) == 5 |
| 251 |
assert len(dels) == 1 |
| 252 |
|
| 253 |
if __name__ == "__main__": |
| 254 |
CURRENT_FILE = os.path.realpath(__file__) |
| 255 |
pytest.main("-s -v %s" % CURRENT_FILE) |