Debian Codegraph @

dirsrvtests/tests/suites/clu/dbmon_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2020 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8 #
9 import time
10 import subprocess
11 import pytest
12 import json
13 import glob
14 import re
15
16 from lib389.tasks import *
17 from lib389.utils import *
18 from test389.topologies import topology_st, topology_m2
19 from lib389.cli_conf.monitor import db_monitor
20 from lib389.monitor import MonitorLDBM
21 from lib389.cli_base import FakeArgs, LogCapture
22 from lib389.backend import Backends
23
24 pytestmark = pytest.mark.tier1
25
26 logging.getLogger(__name__).setLevel(logging.DEBUG)
27 log = logging.getLogger(__name__)
28
29 OUTPUT_NO_INDEXES = [
30 'DB Monitor Report',
31 'Database Cache:',
32 'Cache Hit Ratio:',
33 'Free Space:',
34 'Free Percentage:',
35 'RO Page Drops:',
36 'Pages In:',
37 'Pages Out:',
38 'Normalized DN Cache:',
39 'Cache Hit Ratio:',
40 'Free Space:',
41 'Free Percentage:',
42 'DN Count:',
43 'Evictions:',
44 'Backends:',
45 'dc=example,dc=com (userRoot):',
46 'Entry Cache Hit Ratio:',
47 'Entry Cache Count:',
48 'Entry Cache Free Space:',
49 'Entry Cache Free Percentage:',
50 'Entry Cache Average Size:',
51 'DN Cache Hit Ratio:',
52 'DN Cache Count:',
53 'DN Cache Free Space:',
54 'DN Cache Free Percentage:',
55 'DN Cache Average Size:'
56 ]
57
58 OUTPUT_INDEXES = [
59 'DB Monitor Report',
60 'Database Cache:',
61 'Cache Hit Ratio:',
62 'Free Space:',
63 'Free Percentage:',
64 'RO Page Drops:',
65 'Pages In:',
66 'Pages Out:',
67 'Normalized DN Cache:',
68 'Cache Hit Ratio:',
69 'Free Space:',
70 'Free Percentage:',
71 'DN Count:',
72 'Evictions:',
73 'Backends:',
74 'dc=example,dc=com (userRoot):',
75 'Entry Cache Hit Ratio:',
76 'Entry Cache Count:',
77 'Entry Cache Free Space:',
78 'Entry Cache Free Percentage:',
79 'Entry Cache Average Size:',
80 'DN Cache Hit Ratio:',
81 'DN Cache Count:',
82 'DN Cache Free Space:',
83 'DN Cache Free Percentage:',
84 'DN Cache Average Size:',
85 'Indexes:',
86 'Index: aci.db',
87 'Cache Hit:',
88 'Cache Miss:',
89 'Page In:',
90 'Page Out:',
91 'Index: id2entry.db',
92 'Index: objectclass.db',
93 'Index: entryrdn.db'
94 ]
95
96 JSON_OUTPUT = [
97 'date',
98 'dbcache',
99 'hit_ratio',
100 'free',
101 'free_percentage',
102 'roevicts',
103 'pagein',
104 'pageout',
105 'ndncache',
106 'hit_ratio',
107 'free',
108 'free_percentage',
109 'count',
110 'evictions',
111 'backends',
112 'userRoot',
113 '"suffix": "dc=example,dc=com"',
114 'entry_cache_count',
115 'entry_cache_free',
116 'entry_cache_free_percentage',
117 'entry_cache_size',
118 'entry_cache_hit_ratio',
119 'dn_cache_count',
120 'dn_cache_free',
121 'dn_cache_free_percentage',
122 'dn_cache_size',
123 'dn_cache_hit_ratio',
124 'indexes',
125 'name',
126 'objectclass.db',
127 'cachehit',
128 'cachemiss',
129 'pagein',
130 'pageout',
131 'entryrdn.db',
132 'aci.db',
133 'id2entry.db'
134 ]
135
136
137 def clear_log(inst):
138 log.info('Clear the log')
139 inst.logcap.flush()
140
141
142 def _set_dbsizes(inst, dbpagesize, dbcachesize):
143 backends = Backends(inst)
144 backend = backends.get(DEFAULT_BENAME)
145 dir = backend.get_attr_val_utf8('nsslapd-directory')
146 inst.stop()
147 # Export the db to ldif
148 ldif_file = f'{inst.get_ldif_dir()}/db.ldif'
149 inst.db2ldif(bename=DEFAULT_BENAME, suffixes=[DEFAULT_SUFFIX],
150 excludeSuffixes=None, repl_data=False,
151 outputfile=ldif_file, encrypt=False)
152 # modify dse.ldif
153 dse_ldif = DSEldif(inst, serverid=inst.serverid)
154 bdb = 'cn=bdb,cn=config,cn=ldbm database,cn=plugins,cn=config'
155 dse_ldif.replace(bdb, 'nsslapd-db-page-size', str(dbpagesize))
156 dse_ldif.replace(bdb, 'nsslapd-cache-autosize', '0')
157 dse_ldif.replace(bdb, 'nsslapd-cache-autosize-split', '0')
158 dse_ldif.replace(bdb, 'nsslapd-dbcachesize', str(dbcachesize))
159 # remove the database files and the database environment files
160 for d in (dir, inst.ds_paths.db_home_dir):
161 for f in glob.glob(f'{d}/*'):
162 if os.path.isfile(f):
163 os.remove(f)
164 # Reimport the db
165 inst.ldif2db(DEFAULT_BENAME, None, None, None, ldif_file)
166 inst.start()
167
168
169 @pytest.mark.skipif(ds_is_older("1.4.2"), reason="Not implemented")
170 @pytest.mark.skipif(get_default_db_lib() == "mdb", reason="Not supported over mdb")
171 def test_dsconf_dbmon(topology_st):
172 """Test dbmon tool, that was ported from legacy tools to dsconf
173
174 :id: 4d584ba9-12a9-4e90-ba9a-7e103affdac5
175 :setup: Standalone instance
176 :steps:
177 1. Create DS instance
178 2. Run dbmon without --indexes
179 3. Run dbmon with --indexes
180 4. Run dbmon with --json
181 :expectedresults:
182 1. Success
183 2. Success
184 3. Success
185 4. Success
186 """
187
188 standalone = topology_st.standalone
189
190 args = FakeArgs()
191 args.backends = DEFAULT_BENAME
192 args.indexes = False
193 args.json = False
194
195 log.info('Sanity check for syntax')
196 db_monitor(standalone, DEFAULT_SUFFIX, topology_st.logcap.log, args)
197 for item in OUTPUT_NO_INDEXES:
198 assert topology_st.logcap.contains(item)
199
200 clear_log(topology_st)
201
202 log.info('Sanity check for --indexes output')
203 args.indexes = True
204 db_monitor(standalone, DEFAULT_SUFFIX, topology_st.logcap.log, args)
205 for index_item in OUTPUT_INDEXES:
206 assert topology_st.logcap.contains(index_item)
207
208 clear_log(topology_st)
209
210 log.info('Sanity check for --json output')
211 args.json = True
212 db_monitor(standalone, DEFAULT_SUFFIX, topology_st.logcap.log, args)
213 for json_item in JSON_OUTPUT:
214 assert topology_st.logcap.contains(json_item)
215
216 clear_log(topology_st)
217
218
219
220 @pytest.mark.skipif(get_default_db_lib() == "mdb", reason="Not supported over mdb")
221 def test_dbmon_mp_pagesize(topology_st):
222 """Test dbmon tool, that was ported from legacy tools to dsconf
223
224 :id: 20c1e5b0-75a0-11ed-91de-482ae39447e5
225 :setup: Standalone instance
226 :steps:
227 1. Set bdb parameters (pagesize and cachesize)
228 2. Query ldbm database statistics and extract dbpages and dbcachesize values
229 3. Capture dsconf monitor dbmon output and extract dbcache free_percentage value
230 4. Check that free_percentage is computed rightly
231 :expectedresults:
232 1. Success
233 2. Success
234 3. Success
235 4. free_percentage computation should be based on file system prefered block size
236 rather than on db page size.
237
238 """
239
240 inst = topology_st.standalone
241 fspath = inst.ds_paths.db_home_dir
242 os.makedirs(fspath, mode=0o750, exist_ok=True)
243 fs_pagesize = os.statvfs(fspath).f_bsize
244 db_pagesize = 1024*64 # Maximum value supported by bdb
245 if fs_pagesize == db_pagesize:
246 fs_pagesize = db_pagesize / 2;
247 _set_dbsizes(inst, db_pagesize, 80960)
248
249 # Now lets check that we are really in the condition
250 # needed to reproduce RHBZ 2034407
251 ldbm_mon = MonitorLDBM(inst).get_status()
252 dbcachesize = int(ldbm_mon['nsslapd-db-cache-size-bytes'][0])
253 dbpages = int(ldbm_mon['nsslapd-db-pages-in-use'][0])
254
255 args = FakeArgs()
256 args.backends = DEFAULT_BENAME
257 args.indexes = False
258 args.json = True
259 lc = LogCapture()
260 db_monitor(inst, DEFAULT_SUFFIX, lc.log, args)
261 db_mon_as_str = "".join( ( str(rec) for rec in lc.outputs ) )
262 db_mon_as_str = re.sub("^[^{]*{", "{", db_mon_as_str)[:-2]
263 db_mon = json.loads(db_mon_as_str);
264
265 dbmon_free_percentage = int(10 * float(db_mon['dbcache']['free_percentage']))
266 real_free_percentage = int(1000 * ( dbcachesize - dbpages * fs_pagesize ) / dbcachesize)
267 log.info(f'dbcachesize: {dbcachesize}')
268 log.info(f'dbpages: {dbpages}')
269 log.info(f'db_pagesize: {db_pagesize}')
270 log.info(f'fs_pagesize: {fs_pagesize}')
271 log.info(f'dbmon_free_percentage: {dbmon_free_percentage}')
272 log.info(f'real_free_percentage: {real_free_percentage}')
273 assert real_free_percentage == dbmon_free_percentage
274
275
276 def test_ndn_cache_disabled(topology_st):
277 """Test dbmon output when ndn-cache-enabled is turned off
278
279 :id: 760e217c-70e8-4767-b504-dda7ba2e1f64
280 :setup: Standalone instance
281 :steps:
282 1. Run dbmon with nsslapd-ndn-cache-enabled=on (default)
283 2. Verify NDN cache stats are present in the output
284 3. Set nsslapd-ndn-cache-enabled=off and restart
285 4. Run dbmon again and verify NDN cache stats are not present
286 5. Set nsslapd-ndn-cache-enabled=on and restart
287 6. Run dbmon again and verify NDN cache stats are back
288 :expectedresults:
289 1. Success
290 2. Should display NDN cache data
291 3. Success
292 4. Should not display NDN cache data
293 5. Success
294 6. Should display NDN cache data
295 """
296 inst = topology_st.standalone
297 args = FakeArgs()
298 args.backends = None
299 args.indexes = False
300 args.json = True
301 lc = LogCapture()
302
303 log.info("Testing with NDN cache enabled (default)")
304 db_monitor(inst, DEFAULT_SUFFIX, lc.log, args)
305 db_mon_as_str = "".join((str(rec) for rec in lc.outputs))
306 db_mon_as_str = re.sub("^[^{]*{", "{", db_mon_as_str)[:-2]
307 db_mon = json.loads(db_mon_as_str)
308
309 assert 'ndncache' in db_mon
310 assert 'hit_ratio' in db_mon['ndncache']
311 lc.flush()
312
313 log.info("Setting nsslapd-ndn-cache-enabled to OFF")
314 inst.config.set('nsslapd-ndn-cache-enabled', 'off')
315 inst.restart()
316
317 log.info("Testing with NDN cache disabled")
318 db_monitor(inst, DEFAULT_SUFFIX, lc.log, args)
319 db_mon_as_str = "".join((str(rec) for rec in lc.outputs))
320 db_mon_as_str = re.sub("^[^{]*{", "{", db_mon_as_str)[:-2]
321 db_mon = json.loads(db_mon_as_str)
322
323 assert 'ndncache' not in db_mon
324 lc.flush()
325
326 log.info("Setting nsslapd-ndn-cache-enabled to ON")
327 inst.config.set('nsslapd-ndn-cache-enabled', 'on')
328 inst.restart()
329
330 log.info("Testing with NDN cache re-enabled")
331 db_monitor(inst, DEFAULT_SUFFIX, lc.log, args)
332 db_mon_as_str = "".join((str(rec) for rec in lc.outputs))
333 db_mon_as_str = re.sub("^[^{]*{", "{", db_mon_as_str)[:-2]
334 db_mon = json.loads(db_mon_as_str)
335
336 assert 'ndncache' in db_mon
337 assert 'hit_ratio' in db_mon['ndncache']
338 lc.flush()
339
340 args.json = False
341
342 log.info("Testing with NDN cache enabled - non-JSON output")
343 db_monitor(inst, DEFAULT_SUFFIX, lc.log, args)
344 output = "".join((str(rec) for rec in lc.outputs))
345
346 assert "Normalized DN Cache:" in output
347 assert "Cache Hit Ratio:" in output
348 lc.flush()
349
350 log.info("Setting nsslapd-ndn-cache-enabled to OFF")
351 inst.config.set('nsslapd-ndn-cache-enabled', 'off')
352 inst.restart()
353
354 log.info("Testing with NDN cache disabled - non-JSON output")
355 db_monitor(inst, DEFAULT_SUFFIX, lc.log, args)
356 output = "".join((str(rec) for rec in lc.outputs))
357
358 assert "Normalized DN Cache:" not in output
359 lc.flush()
360
361 inst.config.set('nsslapd-ndn-cache-enabled', 'on')
362 inst.restart()
363
364
365 if __name__ == '__main__':
366 # Run isolated
367 # -s for DEBUG mode
368 CURRENT_FILE = os.path.realpath(__file__)
369 pytest.main("-s %s" % CURRENT_FILE)