Debian Codegraph @

dirsrvtests/tests/suites/features/entrycache_eviction_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2025 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import os
10 import logging
11 import pytest
12 import ldap
13 import ldif
14 import random
15 from contextlib import suppress
16 from lib389.backend import Backends
17 from lib389.cli_base import FakeArgs
18 from lib389.cli_ctl.dbgen import dbgen_create_groups
19 from lib389.config import BDB_LDBMConfig, LMDB_LDBMConfig
20 from lib389._constants import DEFAULT_SUFFIX, PLUGIN_AUTOMEMBER
21 from lib389.dirsrv_log import DirsrvErrorLog
22 from lib389.tasks import ImportTask
23 from test389.topologies import topology_st
24 from lib389.utils import get_default_db_lib
25
26
27 pytestmark = pytest.mark.tier1
28
29 DEBUGGING = os.getenv("DEBUGGING", default=False)
30 logging.getLogger(__name__).setLevel(logging.DEBUG)
31 log = logging.getLogger(__name__)
32
33 os.environ['DS389_MDB_MAX_SIZE'] = str(200 * 1024 * 1024)
34
35 SEED = 195707
36
37 CONFIG_ATTR_PINNED_ENTRIES = 'nsslapd-cache-pinned-entries'
38
39 # Ensure that the dn, attributes and values are not sorted
40 class LdifScrambler(ldif.LDIFParser):
41
42 def __init__(self, ldifin, ldifout):
43 self.fdin = open(ldifin, 'rb')
44 self.fdout = open(ldifout, 'wt')
45 super().__init__(self.fdin)
46 self.writer = ldif.LDIFWriter(self.fdout)
47 self.others = []
48 self.users = []
49 self.user5 = []
50 self.groups = []
51 self.group5 = []
52
53 def close_fds(self):
54 self.fdin.close()
55 self.fdout.close()
56
57 def handle(self,dn,entry):
58 if 'cn=myGroup-5,ou=groups,dc=entrycache_test,dc=example,dc=com' in dn:
59 self.group5.append((dn, entry))
60 elif ',ou=groups,dc=entrycache_test,dc=example,dc=com' in dn:
61 self.groups.append((dn, entry))
62 elif 'uid=group_entry5-' in dn:
63 self.user5.append((dn, entry))
64 elif 'uid=group_entry' in dn:
65 self.users.append((dn, entry))
66 else:
67 self.others.append((dn, entry))
68
69 def scramble(self):
70 self.parse()
71 log.debug(f'{len(self.others)} entries in self.others')
72 log.debug(f'{len(self.users)} entries in self.users')
73 log.debug(f'{len(self.user5)} entries in self.user5')
74 log.debug(f'{len(self.groups)} entries in self.groups')
75 log.debug(f'{len(self.group5)} entries in self.group5')
76 for dn, entry in self.others:
77 self.writer.unparse(dn, entry)
78 self.scramble_entry(self.users)
79 self.scramble_entry(self.groups)
80 self.scramble_entry(self.user5)
81 self.scramble_entry(self.group5)
82
83 def scramble_entry(self, group):
84 random.shuffle(group)
85 for dn, entry in group:
86 for vals in entry.values():
87 random.shuffle(vals)
88 self.writer.unparse(dn, entry)
89
90
91 @pytest.fixture(scope="function")
92 def prepare_be(topology_st, request):
93
94 inst = topology_st.standalone
95 ldif_file = inst.get_ldif_dir() + '/30ku.ldif'
96 ldif_file_scrambled = inst.get_ldif_dir() + '/30ku-scrambled.ldif'
97 bename = 'entrycache_test'
98 suffix = 'dc=entrycache_test,dc=example,dc=com'
99 people_base = f'ou=people,{suffix}'
100 groups_base = f'ou=groups,{suffix}'
101
102 # Remove the backend if it exists
103 bes = Backends(inst)
104 with suppress(ldap.NO_SUCH_OBJECT):
105 be1 = bes.get(bename)
106 be1.delete()
107
108 # Creates the backend.
109 be1 = bes.create(properties={ 'cn': bename, 'nsslapd-suffix': suffix, })
110
111 # Prepare finalizer
112 def fin():
113 be1.delete()
114 if os.path.exists(ldif_file):
115 os.remove(ldif_file)
116
117 if not DEBUGGING:
118 request.addfinalizer(fin)
119
120 # Generates ldif file with a few clarge groups
121 # And preset random seed to have deterministic test
122 random.seed(SEED)
123 args = FakeArgs()
124 args.NAME = 'myGroup'
125 args.parent = groups_base
126 args.suffix = suffix
127 args.number = 5
128 args.num_members = 6000
129 args.create_members = True
130 args.member_attr = 'uniquemember'
131 args.member_parent = people_base
132 args.ldif_file = ldif_file
133 dbgen_create_groups(inst, log, args)
134 assert os.path.exists(ldif_file)
135
136 # Turn off bdb cache autosizing
137 if get_default_db_lib() == 'bdb':
138 config_ldbm = BDB_LDBMConfig(inst)
139 config_ldbm.set('nsslapd-cache-autosize', '0')
140 else:
141 config_ldbm = LMDB_LDBMConfig(inst)
142 config_ldbm.set('nsslapd-cache-autosize', '0')
143
144 # Set entry cache large enough to hold all the large groups
145 # and with a limited number of entries to trigger eviction
146 be1.replace('nsslapd-cachememsize', '8000000' )
147 be1.replace('nsslapd-cachesize', '100' )
148 # Set debugging trace specific for this test
149 be1.replace('nsslapd-cache-debug-pattern', f'cn=.*,ou=groups,{suffix}' )
150 inst.config.set('nsslapd-errorlog-level', '266354688')
151 inst.config.set('nsslapd-errorlog-logbuffering', 'off')
152 inst.plugins.disable(name=PLUGIN_AUTOMEMBER)
153
154
155 # import the ldif
156 inst.stop()
157 LdifScrambler(ldif_file, ldif_file_scrambled).scramble()
158 if not inst.ldif2db(bename, None, None, None, ldif_file):
159 log.fatal('Failed to import {ldif_file}')
160 assert False
161 with open(inst.ds_paths.error_log, 'r') as fin:
162 lines = fin.readlines()
163 with open(inst.ds_paths.error_log, 'w') as fout:
164 for line in lines:
165 line = line.replace('entrycache_add_int', 'Xntrycache_add_int')
166 line = line.replace('entrycache_remove_int', 'Xntrycache_remove_int')
167 fout.write(line)
168
169 inst.start()
170
171 return (bename, suffix, be1, people_base, groups_base)
172
173
174 def grab_debug_logs(inst, last_count=0):
175 errlog = DirsrvErrorLog(inst)
176 msgs = errlog.match('.*entrycache_.*_int')
177 new_msgs = msgs[last_count:]
178 adds = [ line for line in new_msgs if 'entrycache_add_int' in line ]
179 dels= [ line for line in new_msgs if 'entrycache_remove_int' in line ]
180 return (new_msgs, adds, dels, len(msgs))
181
182
183 def test_entry_cache_eviction(topology_st, prepare_be):
184 """Test that large groups are not evicted
185
186 :id: 550b995e-1c76-11f0-93ed-482ae39447e5
187 :setup: Standalone instance
188 :steps:
189 1. Create DS instance and prepare a test backend
190 2. Search all entries
191 3. Check error log that the group are added in cache but not removed
192 4. Change the number of entries protected from eviction threshold
193 so that no group is preserved.
194 5. Search all entries
195 6. Check error log that the group are removed except the last one
196 7. Change the number of entries protected from eviction threshold
197 so that one group is not preserved.
198 8. Search all entries
199 9. Check error log that the group are preserved except one
200 :expectedresults:
201 1. Success
202 2. Success
203 3. Success
204 4. Success
205 5. Success
206 6. Success
207 7. Success
208 8. Success
209 9. Success
210 """
211
212 inst = topology_st.standalone
213 bename, suffix, be1, people_base, groups_base = prepare_be
214
215 log_count = 0
216
217 be1.replace(CONFIG_ATTR_PINNED_ENTRIES, '10')
218 # Search all groups then all people to try to evict the groups from entrycache
219 inst.search_s(groups_base, ldap.SCOPE_SUBTREE, '(objectclass=groupOfNames)', ['dn'], escapehatch='i am sure')
220 inst.search_s(people_base, ldap.SCOPE_SUBTREE, '(objectclass=inetOrgPerson)', ['dn'], escapehatch='i am sure')
221
222 # Check error logs
223 msgs, adds, dels, log_count = grab_debug_logs(inst, log_count)
224 # Should have entrycache_add_int for each group and no entrycache_delete_int
225 assert len(adds) == 5
226 assert len(dels) == 0
227
228 be1.replace(CONFIG_ATTR_PINNED_ENTRIES, '0')
229 # Search all people to evict groups from entrycache
230 inst.search_s(people_base, ldap.SCOPE_SUBTREE, '(objectclass=inetOrgPerson)', ['dn'], escapehatch='i am sure')
231 # Check error logs
232 msgs, adds, dels, log_count = grab_debug_logs(inst, log_count)
233 assert len(dels) == 5
234
235 # Search all groups then all people to try to evict the groups from entrycache
236 inst.search_s(groups_base, ldap.SCOPE_SUBTREE, '(objectclass=groupOfNames)', ['dn'], escapehatch='i am sure')
237 inst.search_s(people_base, ldap.SCOPE_SUBTREE, '(objectclass=inetOrgPerson)', ['dn'], escapehatch='i am sure')
238 # Check error logs
239 msgs, adds, dels, log_count = grab_debug_logs(inst, log_count)
240 assert len(adds) == 5
241 assert len(dels) == 5
242
243 be1.replace(CONFIG_ATTR_PINNED_ENTRIES, '4')
244 # Search all groups then all people to try to evict the groups from entrycache
245 inst.search_s(groups_base, ldap.SCOPE_SUBTREE, '(objectclass=groupOfNames)', ['dn'], escapehatch='i am sure')
246 inst.search_s(people_base, ldap.SCOPE_SUBTREE, '(objectclass=inetOrgPerson)', ['dn'], escapehatch='i am sure')
247 # Check error logs
248 # Should have entrycache_add_int for each group and one group removed (only 4 preserved)
249 msgs, adds, dels, log_count = grab_debug_logs(inst, log_count)
250 assert len(adds) == 5
251 assert len(dels) == 1
252
253 if __name__ == "__main__":
254 CURRENT_FILE = os.path.realpath(__file__)
255 pytest.main("-s -v %s" % CURRENT_FILE)