Debian Codegraph @

dirsrvtests/tests/suites/disk_monitoring/disk_monitoring_test.py

raw
1 # --- BEGIN COPYRIGHT BLOCK ---
2 # Copyright (C) 2025 Red Hat, Inc.
3 # All rights reserved.
4 #
5 # License: GPL (version 3 or any later version).
6 # See LICENSE for details.
7 # --- END COPYRIGHT BLOCK ---
8
9 import os
10 import subprocess
11 import re
12 import time
13 import ldap
14 import pytest
15 import logging
16 from lib389.tasks import *
17 from lib389._constants import *
18 from lib389.utils import ensure_bytes
19 from lib389.backend import Backends
20 from test389.topologies import topology_st as topo
21 from lib389.paths import *
22 from lib389.idm.user import UserAccounts
23
24 DEBUGGING = os.getenv("DEBUGGING", default=False)
25 if DEBUGGING:
26 logging.getLogger(__name__).setLevel(logging.DEBUG)
27 else:
28 logging.getLogger(__name__).setLevel(logging.INFO)
29 log = logging.getLogger(__name__)
30
31 pytestmark = pytest.mark.tier2
32 disk_monitoring_ack = pytest.mark.skipif(not os.environ.get('DISK_MONITORING_ACK', False), reason="Disk monitoring tests may damage system configuration.")
33
34 THRESHOLD_BYTES = 30000000
35
36
37 def presetup(inst):
38 """Presetup function to mount a tmpfs for log directory to simulate disk space limits."""
39
40 log.info("Setting up tmpfs for disk monitoring tests")
41 inst.stop()
42 log_dir = inst.ds_paths.log_dir
43
44 if os.path.exists(log_dir):
45 log.debug(f"Mounting tmpfs on existing directory: {log_dir}")
46 subprocess.call(['mount', '-t', 'tmpfs', '-o', 'size=35M', 'tmpfs', log_dir])
47 else:
48 log.debug(f"Creating and mounting tmpfs on new directory: {log_dir}")
49 os.mkdir(log_dir)
50 subprocess.call(['mount', '-t', 'tmpfs', '-o', 'size=35M', 'tmpfs', log_dir])
51
52 subprocess.call(f'chown {DEFAULT_USER}: -R {log_dir}', shell=True)
53 subprocess.call(f'chown {DEFAULT_USER}: -R {log_dir}/*', shell=True)
54 subprocess.call(f'restorecon -FvvR {log_dir}', shell=True)
55 inst.start()
56 log.info("tmpfs setup completed")
57
58
59 def setupthesystem(inst):
60 """Setup system configuration for disk monitoring tests."""
61
62 log.info("Configuring system for disk monitoring tests")
63 inst.start()
64 inst.config.set('nsslapd-disk-monitoring-grace-period', '1')
65 inst.config.set('nsslapd-accesslog-logbuffering', 'off')
66 inst.config.set('nsslapd-disk-monitoring-threshold', ensure_bytes(str(THRESHOLD_BYTES)))
67 inst.restart()
68 log.info("System configuration completed")
69
70
71 def capture_config(inst):
72 """Capture current configuration values for later restoration."""
73
74 log.info("Capturing current configuration values")
75
76 config_attrs = [
77 'nsslapd-disk-monitoring',
78 'nsslapd-disk-monitoring-threshold',
79 'nsslapd-disk-monitoring-grace-period',
80 'nsslapd-disk-monitoring-logging-critical',
81 'nsslapd-disk-monitoring-readonly-on-threshold',
82 'nsslapd-accesslog-logbuffering',
83 'nsslapd-errorlog-level',
84 'nsslapd-accesslog-logging-enabled',
85 'nsslapd-accesslog-maxlogsize',
86 'nsslapd-accesslog-logrotationtimeunit',
87 'nsslapd-accesslog-level',
88 'nsslapd-external-libs-debug-enabled',
89 'nsslapd-errorlog-logging-enabled'
90 ]
91
92 captured_config = {}
93 for config_attr in config_attrs:
94 try:
95 current_value = inst.config.get_attr_val_utf8(config_attr)
96 captured_config[config_attr] = current_value
97 log.debug(f"Captured {config_attr}: {current_value}")
98 except Exception as e:
99 log.debug(f"Could not capture {config_attr}: {e}")
100 captured_config[config_attr] = None
101
102 log.info("Configuration capture completed")
103 return captured_config
104
105
106 def restore_config(inst, captured_config):
107 """Restore configuration values to previously captured state."""
108
109 log.info("Restoring configuration to captured values")
110
111 for config_attr, original_value in captured_config.items():
112 if original_value is not None:
113 try:
114 current_value = inst.config.get_attr_val_utf8(config_attr)
115 if current_value != original_value:
116 log.debug(f"Restoring {config_attr} from '{current_value}' to '{original_value}'")
117 inst.config.set(config_attr, ensure_bytes(original_value))
118 except Exception as e:
119 log.debug(f"Could not restore {config_attr}: {e}")
120
121 log.info("Configuration restoration completed")
122
123
124 @pytest.fixture(scope="function")
125 def setup(request, topo):
126 """Module-level fixture to setup the test environment."""
127
128 log.info("Starting module setup for disk monitoring tests")
129 inst = topo.standalone
130
131 # Capture current configuration before making any changes
132 original_config = capture_config(inst)
133
134 presetup(inst)
135 setupthesystem(inst)
136
137 def fin():
138 log.info("Running module cleanup for disk monitoring tests")
139 inst.stop()
140 subprocess.call(['umount', '-fl', inst.ds_paths.log_dir])
141 # Restore configuration to original values
142 inst.start()
143 restore_config(inst, original_config)
144 log.info("Module cleanup completed")
145
146 request.addfinalizer(fin)
147
148
149 def wait_for_condition(inst, condition_str, timeout=30):
150 """Wait until the given condition evaluates to False."""
151
152 log.debug(f"Waiting for condition to be False: {condition_str} (timeout: {timeout}s)")
153 start_time = time.time()
154 while time.time() - start_time < timeout:
155 if not eval(condition_str):
156 log.debug(f"Condition satisfied after {time.time() - start_time:.2f}s")
157 return
158 time.sleep(1)
159 raise AssertionError(f"Condition '{condition_str}' still True after {timeout} seconds")
160
161
162 def wait_for_log_entry(inst, message, timeout=30):
163 """Wait for a specific message to appear in the error log."""
164
165 log.debug(f"Waiting for log entry: '{message}' (timeout: {timeout}s)")
166 start_time = time.time()
167 while time.time() - start_time < timeout:
168 with open(inst.errlog, 'r') as log_file:
169 if message in log_file.read():
170 log.debug(f"Found log entry after {time.time() - start_time:.2f}s")
171 return
172 time.sleep(1)
173 raise AssertionError(f"Message '{message}' not found in error log after {timeout} seconds")
174
175
176 def get_avail_bytes(path):
177 """Get available bytes on the filesystem at the given path."""
178
179 stat = os.statvfs(path)
180 return stat.f_bavail * stat.f_bsize
181
182
183 def fill_to_target_avail(path, target_avail_bytes):
184 """Fill the disk to reach the target available bytes by creating a large file."""
185
186 avail = get_avail_bytes(path)
187 fill_bytes = avail - target_avail_bytes
188 log.debug(f"Current available: {avail}, target: {target_avail_bytes}, will create {fill_bytes} byte file")
189 if fill_bytes <= 0:
190 raise ValueError("Already below target avail")
191
192 fill_file = os.path.join(path, 'fill.dd')
193 bs = 4096
194 count = (fill_bytes + bs - 1) // bs # ceil division to ensure enough
195 log.info(f"Creating fill file {fill_file} with {count} blocks of {bs} bytes")
196 subprocess.check_call(['dd', 'if=/dev/zero', f'of={fill_file}', f'bs={bs}', f'count={count}'])
197 return fill_file
198
199
200 @pytest.fixture(scope="function")
201 def reset_logs(topo):
202 """Function-level fixture to reset the error log before each test."""
203
204 log.debug("Resetting error logs before test")
205 topo.standalone.deleteErrorLogs()
206
207
208 def generate_access_log_activity(inst, num_users=10, num_binds=100):
209 """Generate access log activity by creating users and performing binds."""
210
211 log.info(f"Generating access log activity with {num_users} users and {num_binds} binds each")
212 users = UserAccounts(inst, DEFAULT_SUFFIX)
213
214 # Create test users
215 for i in range(num_users):
216 user_properties = {
217 'uid': f'cn=user{i}',
218 'cn': f'cn=user{i}',
219 'sn': f'cn=user{i}',
220 'userPassword': "Itsme123",
221 'uidNumber': f'1{i}',
222 'gidNumber': f'2{i}',
223 'homeDirectory': f'/home/{i}'
224 }
225 users.create(properties=user_properties)
226
227 # Perform bind operations
228 for j in range(num_binds):
229 for user in users.list():
230 user.bind('Itsme123')
231
232 log.info("Access log activity generation completed")
233 return users
234
235
236 @disk_monitoring_ack
237 def test_verify_operation_when_disk_monitoring_is_off(topo, setup, reset_logs):
238 """Verify operation when Disk monitoring is off.
239
240 :id: 73a97536-fe9e-11e8-ba9f-8c16451d917b
241 :setup: Standalone
242 :steps:
243 1. Turn off disk monitoring
244 2. Go below the threshold
245 3. Check DS is up and not entering shutdown mode
246 :expectedresults:
247 1. Success
248 2. Success
249 3. Success
250 """
251 log.info("Starting test_verify_operation_when_disk_monitoring_is_off")
252 inst = topo.standalone
253 fill_file = None
254
255 try:
256 log.info("Disabling disk monitoring")
257 inst.config.set('nsslapd-disk-monitoring', 'off')
258 inst.restart()
259
260 log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)")
261 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES - 1)
262
263 log.info("Verifying server stays up despite being below threshold")
264 wait_for_condition(inst, 'inst.status() != True', 11)
265
266 # Check DS is up and not entering shutdown mode
267 assert inst.status() == True
268 log.info("Verified: server remains operational when disk monitoring is disabled")
269
270 finally:
271 if fill_file and os.path.exists(fill_file):
272 log.debug(f"Cleaning up fill file: {fill_file}")
273 os.remove(fill_file)
274
275 log.info("Test completed successfully")
276
277
278 @disk_monitoring_ack
279 def test_enable_external_libs_debug_log(topo, setup, reset_logs):
280 """Check that OpenLDAP logs are successfully enabled and disabled when disk threshold is reached.
281
282 :id: 121b2b24-ecba-48e2-9ee2-312d929dc8c6
283 :setup: Standalone instance
284 :steps:
285 1. Set nsslapd-external-libs-debug-enabled to "on"
286 2. Go straight below 1/2 of the threshold
287 3. Verify that the external libs debug setting is disabled
288 4. Go back above 1/2 of the threshold
289 5. Verify that the external libs debug setting is enabled back
290 :expectedresults:
291 1. Success
292 2. Success
293 3. Success
294 4. Success
295 5. Success
296 """
297 log.info("Starting test_enable_external_libs_debug_log")
298 inst = topo.standalone
299 fill_file = None
300
301 try:
302 log.info("Configuring disk monitoring and external libs debug")
303 inst.config.set('nsslapd-disk-monitoring', 'on')
304 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off')
305 inst.config.set('nsslapd-external-libs-debug-enabled', 'on')
306 inst.config.set('nsslapd-errorlog-level', '8')
307 inst.restart()
308
309 log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)")
310 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1)
311
312 log.info("Verifying external libs debug is automatically disabled")
313 wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-external-libs-debug-enabled') != 'off'", 31)
314
315 finally:
316 if fill_file and os.path.exists(fill_file):
317 log.debug(f"Cleaning up fill file: {fill_file}")
318 os.remove(fill_file)
319
320 log.info("Verifying external libs debug is re-enabled after freeing space")
321 wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-external-libs-debug-enabled') != 'on'", 31)
322 inst.config.set('nsslapd-external-libs-debug-enabled', 'off')
323
324 log.info("Test completed successfully")
325
326
327 @disk_monitoring_ack
328 def test_free_up_the_disk_space_and_change_ds_config(topo, setup, reset_logs):
329 """Free up the disk space and change DS config.
330
331 :id: 7be4d560-fe9e-11e8-a307-8c16451d917b
332 :setup: Standalone
333 :steps:
334 1. Enable Disk Monitoring plugin and set disk monitoring logging to critical
335 2. Verify no message about loglevel is present in the error log
336 3. Verify no message about disabling logging is present in the error log
337 4. Verify no message about removing rotated logs is present in the error log
338 :expectedresults:
339 1. Success
340 2. Success
341 3. Success
342 4. Success
343 """
344 log.info("Starting test_free_up_the_disk_space_and_change_ds_config")
345 inst = topo.standalone
346
347 log.info("Enabling disk monitoring with critical logging")
348 inst.config.set('nsslapd-disk-monitoring', 'on')
349 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on')
350 inst.config.set('nsslapd-errorlog-level', '8')
351 inst.restart()
352
353 log.info("Verifying no premature disk monitoring messages in error log")
354 with open(inst.errlog, 'r') as err_log:
355 content = err_log.read()
356
357 assert 'temporarily setting error loglevel to zero' not in content
358 assert 'disabling access and audit logging' not in content
359 assert 'deleting rotated logs' not in content
360
361 log.info("Verified: no unexpected disk monitoring messages found")
362 log.info("Test completed successfully")
363
364
365 @disk_monitoring_ack
366 def test_verify_operation_with_nsslapd_disk_monitoring_logging_critical_off(topo, setup, reset_logs):
367 """Verify operation with "nsslapd-disk-monitoring-logging-critical: off".
368
369 :id: 82363bca-fe9e-11e8-9ae7-8c16451d917b
370 :setup: Standalone
371 :steps:
372 1. Verify that verbose logging was set to default level
373 2. Verify that logging is disabled
374 3. Verify that rotated logs were not removed
375 :expectedresults:
376 1. Success
377 2. Success
378 3. Success
379 """
380 log.info("Starting test_verify_operation_with_nsslapd_disk_monitoring_logging_critical_off")
381 inst = topo.standalone
382 fill_file = None
383
384 try:
385 log.info("Configuring disk monitoring with critical logging disabled")
386 inst.config.set('nsslapd-disk-monitoring', 'on')
387 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off')
388 inst.config.set('nsslapd-errorlog-level', '8')
389 inst.restart()
390
391 log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)")
392 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES - 1)
393
394 log.info("Waiting for loglevel to be set to default")
395 wait_for_log_entry(inst, 'temporarily setting error loglevel to the default level', 11)
396
397 log.info("Verifying error log level was set to default")
398 config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-errorlog-level'])
399 current_level = int(re.findall(r'nsslapd-errorlog-level: \d+', str(config_entry))[0].split(' ')[1])
400 assert LOG_DEFAULT == current_level
401
402 log.info("Verifying access logging is disabled")
403 wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') != 'off'", 11)
404 assert inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') == 'off'
405
406 log.info("Verifying expected disk monitoring messages")
407 with open(inst.errlog, 'r') as err_log:
408 content = err_log.read()
409
410 assert 'disabling access and audit logging' in content
411 wait_for_log_entry(inst, 'deleting rotated logs', 11)
412 assert f"Unable to remove file: {inst.ds_paths.log_dir}" not in content
413 assert 'is too far below the threshold' not in content
414
415 log.info("All verifications passed")
416
417 finally:
418 if fill_file and os.path.exists(fill_file):
419 log.debug(f"Cleaning up fill file: {fill_file}")
420 os.remove(fill_file)
421
422 log.info("Test completed successfully")
423
424
425 @disk_monitoring_ack
426 def test_operation_with_nsslapd_disk_monitoring_logging_critical_on_below_half_of_the_threshold(topo, setup, reset_logs):
427 """Verify operation with "nsslapd-disk-monitoring-logging-critical: on" below 1/2 of the threshold.
428 Verify recovery.
429
430 :id: 8940c502-fe9e-11e8-bcc0-8c16451d917b
431 :setup: Standalone
432 :steps:
433 1. Verify that DS goes into shutdown mode
434 2. Verify that DS exited shutdown mode
435 :expectedresults:
436 1. Success
437 2. Success
438 """
439 log.info("Starting test_operation_with_nsslapd_disk_monitoring_logging_critical_on_below_half_of_the_threshold")
440 inst = topo.standalone
441 fill_file = None
442
443 try:
444 log.info("Configuring disk monitoring with critical logging enabled")
445 inst.config.set('nsslapd-disk-monitoring', 'on')
446 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on')
447 inst.restart()
448
449 log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)")
450 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1)
451
452 log.info("Waiting for shutdown mode message")
453 wait_for_log_entry(inst, 'is too far below the threshold', 100)
454
455 log.info("Freeing up disk space")
456 os.remove(fill_file)
457 fill_file = None
458
459 log.info("Waiting for recovery message")
460 wait_for_log_entry(inst, 'Available disk space is now acceptable', 25)
461
462 log.info("Verified: server entered and exited shutdown mode correctly")
463
464 finally:
465 if fill_file and os.path.exists(fill_file):
466 log.debug(f"Cleaning up fill file: {fill_file}")
467 os.remove(fill_file)
468
469 log.info("Test completed successfully")
470
471
472 @disk_monitoring_ack
473 def test_setting_nsslapd_disk_monitoring_logging_critical_to_off(topo, setup, reset_logs):
474 """Setting nsslapd-disk-monitoring-logging-critical to "off".
475
476 :id: 93265ec4-fe9e-11e8-af93-8c16451d917b
477 :setup: Standalone
478 :steps:
479 1. Set nsslapd-disk-monitoring-logging-critical to "off"
480 :expectedresults:
481 1. Success
482 """
483 log.info("Starting test_setting_nsslapd_disk_monitoring_logging_critical_to_off")
484 inst = topo.standalone
485
486 log.info("Setting disk monitoring configuration")
487 inst.config.set('nsslapd-disk-monitoring', 'on')
488 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off')
489 inst.config.set('nsslapd-errorlog-level', '8')
490 inst.restart()
491
492 log.info("Verifying server is running normally")
493 assert inst.status() == True
494
495 log.info("Test completed successfully")
496
497
498 @disk_monitoring_ack
499 def test_operation_with_nsslapd_disk_monitoring_logging_critical_off(topo, setup, reset_logs):
500 """Verify operation with nsslapd-disk-monitoring-logging-critical: off.
501
502 :id: 97985a52-fe9e-11e8-9914-8c16451d917b
503 :setup: Standalone
504 :steps:
505 1. Generate access log activity to create rotated logs
506 2. Go below threshold to trigger disk monitoring
507 3. Verify that verbose logging was set to default level
508 4. Verify that logging is disabled
509 5. Verify that rotated logs were removed
510 :expectedresults:
511 1. Success
512 2. Success
513 3. Success
514 4. Success
515 5. Success
516 """
517 log.info("Starting test_operation_with_nsslapd_disk_monitoring_logging_critical_off")
518 inst = topo.standalone
519 fill_file = None
520 users = None
521
522 try:
523 log.info("Configuring disk monitoring and access log settings")
524 inst.config.set('nsslapd-disk-monitoring', 'on')
525 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off')
526 inst.config.set('nsslapd-errorlog-level', '8')
527 inst.config.set('nsslapd-accesslog-maxlogsize', '1')
528 inst.config.set('nsslapd-accesslog-logrotationtimeunit', 'minute')
529 inst.config.set('nsslapd-accesslog-level', '772')
530 inst.restart()
531
532 log.info("Generating access log activity to create rotated logs")
533 users = generate_access_log_activity(inst, num_users=10, num_binds=100)
534
535 inst.bind_s(DN_DM, PW_DM)
536
537 log.info("Resetting access log settings")
538 inst.config.set('nsslapd-accesslog-maxlogsize', '100')
539 inst.config.set('nsslapd-accesslog-logrotationtimeunit', 'day')
540 inst.config.set('nsslapd-accesslog-level', '256')
541 inst.restart()
542
543 log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)")
544 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES - 1)
545
546 log.info("Waiting for loglevel to be set to default")
547 wait_for_log_entry(inst, 'temporarily setting error loglevel to the default level', 11)
548
549 log.info("Verifying error log level was set to default")
550 config_level = None
551 for _ in range(10):
552 time.sleep(1)
553 config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-errorlog-level'])
554 config_level = int(re.findall(r'nsslapd-errorlog-level: \d+', str(config_entry))[0].split(' ')[1])
555 if LOG_DEFAULT == config_level:
556 break
557 assert LOG_DEFAULT == config_level
558
559 log.info("Verifying access logging is disabled")
560 wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') == 'off'", 20)
561
562 with open(inst.errlog, 'r') as err_log:
563 content = err_log.read()
564 assert 'disabling access and audit logging' in content
565
566 log.info("Verifying rotated logs are removed")
567 wait_for_log_entry(inst, 'deleting rotated logs', 20)
568
569 rotated_logs = re.findall(r'access.\d+-\d+', str(os.listdir(inst.ds_paths.log_dir)))
570 assert not rotated_logs, f"Found unexpected rotated logs: {rotated_logs}"
571
572 with open(inst.errlog, 'r') as err_log:
573 content = err_log.read()
574 assert 'Unable to remove file:' not in content
575 assert 'is too far below the threshold' not in content
576
577 log.info("All verifications passed")
578
579 finally:
580 # Clean up users
581 if users:
582 log.debug("Cleaning up test users")
583 for user in users.list():
584 try:
585 user.delete()
586 except ldap.ALREADY_EXISTS:
587 pass
588
589 if fill_file and os.path.exists(fill_file):
590 log.debug(f"Cleaning up fill file: {fill_file}")
591 os.remove(fill_file)
592
593 log.info("Test completed successfully")
594
595
596 @disk_monitoring_ack
597 def test_operation_with_nsslapd_disk_monitoring_logging_critical_off_below_half_of_the_threshold(topo, setup, reset_logs):
598 """Verify operation with nsslapd-disk-monitoring-logging-critical: off below 1/2 of the threshold.
599 Verify shutdown and recovery.
600
601 :id: 9d4c7d48-fe9e-11e8-b5d6-8c16451d917b
602 :setup: Standalone
603 :steps:
604 1. Go below half threshold to trigger shutdown
605 2. Verify DS shutdown after grace period
606 3. Free space and restart
607 4. Verify logging is re-enabled
608 5. Create rotated logs and enable verbose logging
609 :expectedresults:
610 1. Success
611 2. Success
612 3. Success
613 4. Success
614 5. Success
615 """
616 log.info("Starting test_operation_with_nsslapd_disk_monitoring_logging_critical_off_below_half_of_the_threshold")
617 inst = topo.standalone
618 fill_file = None
619 users = None
620
621 try:
622 log.info("Configuring disk monitoring with critical logging disabled")
623 inst.config.set('nsslapd-disk-monitoring', 'on')
624 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off')
625 inst.restart()
626
627 log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)")
628 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1)
629
630 log.info("Waiting for shutdown messages")
631 wait_for_log_entry(inst, 'is too far below the threshold', 100)
632 wait_for_log_entry(inst, 'Signaling slapd for shutdown', 90)
633
634 log.info("Verifying server shutdown within grace period")
635 for i in range(60):
636 time.sleep(1)
637 if not inst.status():
638 log.info(f"Server shut down after {i+1} seconds")
639 break
640 assert inst.status() == False
641
642 log.info("Freeing disk space and cleaning logs")
643 os.remove(fill_file)
644 fill_file = None
645 open(f'{inst.ds_paths.log_dir}/errors', 'w').close()
646
647 log.info("Starting server after freeing space")
648 inst.start()
649
650 log.info("Verifying logging is re-enabled")
651 assert inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') == 'on'
652 assert inst.config.get_attr_val_utf8('nsslapd-errorlog-logging-enabled') == 'on'
653
654 with open(inst.errlog, 'r') as err_log:
655 content = err_log.read()
656 assert 'disabling access and audit logging' not in content
657
658 log.info("Setting up access log rotation for testing")
659 inst.config.set('nsslapd-accesslog-maxlogsize', '1')
660 inst.config.set('nsslapd-accesslog-logrotationtimeunit', 'minute')
661 inst.config.set('nsslapd-accesslog-level', '772')
662 inst.restart()
663
664 log.info("Creating rotated log files through user activity")
665 users = generate_access_log_activity(inst, num_users=10, num_binds=100)
666
667 log.info("Waiting for log rotation to occur")
668 for i in range(61):
669 time.sleep(1)
670 rotated_logs = re.findall(r'access.\d+-\d+', str(os.listdir(inst.ds_paths.log_dir)))
671 if rotated_logs:
672 log.info(f"Log rotation detected after {i+1} seconds")
673 break
674 assert rotated_logs, "No rotated logs found after waiting"
675
676 inst.bind_s(DN_DM, PW_DM)
677
678 log.info("Enabling verbose logging")
679 inst.config.set('nsslapd-accesslog-maxlogsize', '100')
680 inst.config.set('nsslapd-accesslog-logrotationtimeunit', 'day')
681 inst.config.set('nsslapd-accesslog-level', '256')
682 inst.config.set('nsslapd-errorlog-level', '8')
683 inst.restart()
684
685 log.info("Recovery and setup verification completed")
686
687 finally:
688 # Clean up users
689 if users:
690 log.debug("Cleaning up test users")
691 for user in users.list():
692 try:
693 user.delete()
694 except ldap.ALREADY_EXISTS:
695 pass
696
697 if fill_file and os.path.exists(fill_file):
698 log.debug(f"Cleaning up fill file: {fill_file}")
699 os.remove(fill_file)
700
701 log.info("Test completed successfully")
702
703
704 @disk_monitoring_ack
705 def test_go_straight_below_half_of_the_threshold(topo, setup, reset_logs):
706 """Go straight below 1/2 of the threshold and verify recovery.
707
708 :id: a2a0664c-fe9e-11e8-b220-8c16451d917b
709 :setup: Standalone
710 :steps:
711 1. Go straight below 1/2 of the threshold
712 2. Verify that verbose logging was set to default level
713 3. Verify that logging is disabled
714 4. Verify DS is in shutdown mode
715 5. Verify DS has recovered from shutdown
716 :expectedresults:
717 1. Success
718 2. Success
719 3. Success
720 4. Success
721 5. Success
722 """
723 log.info("Starting test_go_straight_below_half_of_the_threshold")
724 inst = topo.standalone
725 fill_file = None
726
727 try:
728 log.info("Configuring disk monitoring with critical logging disabled")
729 inst.config.set('nsslapd-disk-monitoring', 'on')
730 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off')
731 inst.config.set('nsslapd-errorlog-level', '8')
732 inst.restart()
733
734 # Go straight below half threshold
735 log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)")
736 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1)
737
738 # Verify that verbose logging was set to default level
739 log.info("Waiting for loglevel to be set to default")
740 wait_for_log_entry(inst, 'temporarily setting error loglevel to the default level', 11)
741
742 log.info("Verifying error log level was set to default")
743 config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-errorlog-level'])
744 current_level = int(re.findall(r'nsslapd-errorlog-level: \d+', str(config_entry))[0].split(' ')[1])
745 assert LOG_DEFAULT == current_level
746
747 log.info("Verifying access logging is disabled")
748 wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') != 'off'", 11)
749
750 log.info("Verifying expected disk monitoring messages")
751 wait_for_log_entry(inst, 'disabling access and audit logging', 2)
752 wait_for_log_entry(inst, 'deleting rotated logs', 11)
753
754 with open(inst.errlog, 'r') as err_log:
755 content = err_log.read()
756 assert 'Unable to remove file:' not in content
757
758 log.info("Verifying server enters shutdown mode")
759 wait_for_condition(inst, 'inst.status() != False', 90)
760 wait_for_log_entry(inst, 'is too far below the threshold', 2)
761
762 log.info("Freeing disk space and restarting server")
763 os.remove(fill_file)
764 fill_file = None
765 open(f'{inst.ds_paths.log_dir}/errors', 'w').close()
766 inst.start()
767
768 log.info("Verifying server recovery")
769 wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') != 'on'", 20)
770
771 with open(inst.errlog, 'r') as err_log:
772 content = err_log.read()
773 assert 'disabling access and audit logging' not in content
774
775 log.info("Recovery verification completed")
776
777 finally:
778 if fill_file and os.path.exists(fill_file):
779 log.debug(f"Cleaning up fill file: {fill_file}")
780 os.remove(fill_file)
781
782 log.info("Test completed successfully")
783
784
785 @disk_monitoring_ack
786 def test_readonly_on_threshold(topo, setup, reset_logs):
787 """Verify that nsslapd-disk-monitoring-readonly-on-threshold switches the server to read-only mode.
788
789 :id: 06814c19-ef3c-4800-93c9-c7c6e76fcbb9
790 :customerscenario: True
791 :setup: Standalone
792 :steps:
793 1. Configure readonly on threshold
794 2. Go below threshold and verify backend is read-only
795 3. Go back above threshold and verify backend is read-write
796 :expectedresults:
797 1. Success
798 2. Success
799 3. Success
800 """
801 log.info("Starting test_readonly_on_threshold")
802 inst = topo.standalone
803 fill_file = None
804 test_user = None
805
806 try:
807 backends = Backends(inst)
808 backend_name = backends.list()[0].rdn
809 log.info(f"Testing with backend: {backend_name}")
810
811 log.info("Configuring disk monitoring with readonly on threshold")
812 inst.deleteErrorLogs()
813 inst.config.set('nsslapd-disk-monitoring', 'on')
814 inst.config.set('nsslapd-disk-monitoring-readonly-on-threshold', 'on')
815 inst.restart()
816
817 log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)")
818 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES - 1)
819
820 log.info("Waiting for backend to enter read-only mode")
821 wait_for_log_entry(inst, f"Putting the backend '{backend_name}' to read-only mode", 11)
822
823 log.info("Verifying backend is in read-only mode")
824 users = UserAccounts(topo.standalone, DEFAULT_SUFFIX)
825 try:
826 test_user = users.create_test_user()
827 test_user.delete()
828 assert False, "Expected UNWILLING_TO_PERFORM error for read-only mode"
829 except ldap.UNWILLING_TO_PERFORM as e:
830 if 'database is read-only' not in str(e):
831 raise
832 log.info("Confirmed: backend correctly rejects writes in read-only mode")
833
834 log.info("Freeing disk space")
835 os.remove(fill_file)
836 fill_file = None
837
838 log.info("Waiting for backend to return to read-write mode")
839 wait_for_log_entry(inst, f"Putting the backend '{backend_name}' back to read-write mode", 11)
840
841 log.info("Verifying backend is in read-write mode")
842 test_user = users.create_test_user()
843 assert test_user.exists()
844 test_user.delete()
845 test_user = None
846
847 log.info("Confirmed: backend correctly accepts writes in read-write mode")
848
849 finally:
850 if test_user:
851 try:
852 test_user.delete()
853 except:
854 pass
855
856 if fill_file and os.path.exists(fill_file):
857 log.debug(f"Cleaning up fill file: {fill_file}")
858 os.remove(fill_file)
859
860 log.info("Test completed successfully")
861
862
863 @disk_monitoring_ack
864 def test_readonly_on_threshold_below_half_of_the_threshold(topo, setup, reset_logs):
865 """Go below 1/2 of the threshold when readonly on threshold is enabled.
866
867 :id: 10262663-b41f-420e-a2d0-9532dd54fa7c
868 :customerscenario: True
869 :setup: Standalone
870 :steps:
871 1. Configure readonly on threshold
872 2. Go below half threshold
873 3. Verify backend is read-only and shutdown messages appear
874 4. Free space and verify backend returns to read-write
875 :expectedresults:
876 1. Success
877 2. Success
878 3. Success
879 4. Success
880 """
881 log.info("Starting test_readonly_on_threshold_below_half_of_the_threshold")
882 inst = topo.standalone
883 fill_file = None
884 test_user = None
885
886 try:
887 backends = Backends(inst)
888 backend_name = backends.list()[0].rdn
889 log.info(f"Testing with backend: {backend_name}")
890
891 log.info("Configuring disk monitoring with readonly on threshold")
892 inst.deleteErrorLogs()
893 inst.config.set('nsslapd-disk-monitoring', 'on')
894 inst.config.set('nsslapd-disk-monitoring-readonly-on-threshold', 'on')
895 inst.restart()
896
897 log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)")
898 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1)
899
900 log.info("Waiting for backend to enter read-only mode")
901 wait_for_log_entry(inst, f"Putting the backend '{backend_name}' to read-only mode", 11)
902
903 log.info("Verifying backend is in read-only mode")
904 users = UserAccounts(inst, DEFAULT_SUFFIX)
905 try:
906 test_user = users.create_test_user()
907 test_user.delete()
908 assert False, "Expected UNWILLING_TO_PERFORM error for read-only mode"
909 except ldap.UNWILLING_TO_PERFORM as e:
910 if 'database is read-only' not in str(e):
911 raise
912 log.info("Confirmed: backend correctly rejects writes in read-only mode")
913
914 log.info("Waiting for shutdown threshold message")
915 wait_for_log_entry(inst, 'is too far below the threshold', 51)
916
917 log.info("Freeing disk space")
918 os.remove(fill_file)
919 fill_file = None
920
921 log.info("Waiting for backend to return to read-write mode")
922 wait_for_log_entry(inst, f"Putting the backend '{backend_name}' back to read-write mode", 51)
923
924 log.info("Verifying backend is in read-write mode")
925 test_user = users.create_test_user()
926 assert test_user.exists()
927 test_user.delete()
928 test_user = None
929
930 log.info("Confirmed: backend correctly accepts writes in read-write mode")
931
932 finally:
933 if test_user:
934 try:
935 test_user.delete()
936 except:
937 pass
938
939 if fill_file and os.path.exists(fill_file):
940 log.debug(f"Cleaning up fill file: {fill_file}")
941 os.remove(fill_file)
942
943 log.info("Test completed successfully")
944
945
946 @disk_monitoring_ack
947 def test_below_half_of_the_threshold_not_starting_after_shutdown(topo, setup, reset_logs):
948 """Test that the instance won't start if we are below 1/2 of the threshold.
949
950 :id: cceeaefd-9fa4-45c5-9ac6-9887a0671ef8
951 :customerscenario: True
952 :setup: Standalone
953 :steps:
954 1. Go below half threshold and wait for shutdown
955 2. Try to start the instance and verify it fails
956 3. Free space and verify instance starts successfully
957 :expectedresults:
958 1. Success
959 2. Startup fails as expected
960 3. Success
961 """
962 log.info("Starting test_below_half_of_the_threshold_not_starting_after_shutdown")
963 inst = topo.standalone
964 fill_file = None
965
966 try:
967 log.info("Configuring disk monitoring")
968 inst.deleteErrorLogs()
969 inst.config.set('nsslapd-disk-monitoring', 'on')
970 inst.restart()
971
972 log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)")
973 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1)
974
975 log.info("Waiting for server to shut down due to disk space")
976 wait_for_condition(inst, 'inst.status() == True', 120)
977
978 log.info("Attempting to start instance (should fail)")
979 try:
980 inst.start()
981 assert False, "Instance startup should have failed due to low disk space"
982 except (ValueError, subprocess.CalledProcessError):
983 log.info("Instance startup failed as expected due to low disk space")
984
985 wait_for_log_entry(inst, f'is too far below the threshold({THRESHOLD_BYTES} bytes). Exiting now', 2)
986
987 log.info("Freeing disk space")
988 os.remove(fill_file)
989 fill_file = None
990
991 log.info("Starting instance after freeing space")
992 inst.start()
993 assert inst.status() == True
994 log.info("Instance started successfully after freeing space")
995
996 finally:
997 if fill_file and os.path.exists(fill_file):
998 log.debug(f"Cleaning up fill file: {fill_file}")
999 os.remove(fill_file)
1000
1001 log.info("Test completed successfully")
1002
1003
1004 @disk_monitoring_ack
1005 def test_go_straight_below_4kb(topo, setup, reset_logs):
1006 """Go straight below 4KB and verify behavior.
1007
1008 :id: a855115a-fe9e-11e8-8e91-8c16451d917b
1009 :setup: Standalone
1010 :steps:
1011 1. Go straight below 4KB
1012 2. Verify server behavior
1013 3. Clean space and restart
1014 :expectedresults:
1015 1. Success
1016 2. Success
1017 3. Success
1018 """
1019 log.info("Starting test_go_straight_below_4kb")
1020 inst = topo.standalone
1021 fill_file = None
1022
1023 try:
1024 log.info("Configuring disk monitoring")
1025 inst.config.set('nsslapd-disk-monitoring', 'on')
1026 inst.restart()
1027
1028 log.info("Filling disk to go below 4KB")
1029 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, 4000)
1030
1031 log.info("Waiting for server shutdown due to extreme low disk space")
1032 wait_for_condition(inst, 'inst.status() != False', 11)
1033
1034 log.info("Freeing disk space and restarting")
1035 os.remove(fill_file)
1036 fill_file = None
1037 inst.start()
1038
1039 assert inst.status() == True
1040 log.info("Server restarted successfully after freeing space")
1041
1042 finally:
1043 if fill_file and os.path.exists(fill_file):
1044 log.debug(f"Cleaning up fill file: {fill_file}")
1045 os.remove(fill_file)
1046
1047 log.info("Test completed successfully")
1048
1049
1050 @disk_monitoring_ack
1051 def test_threshold_to_overflow_value(topo, setup, reset_logs):
1052 """Test overflow in nsslapd-disk-monitoring-threshold.
1053
1054 :id: ad60ab3c-fe9e-11e8-88dc-8c16451d917b
1055 :setup: Standalone
1056 :steps:
1057 1. Set nsslapd-disk-monitoring-threshold to overflow value
1058 2. Verify the value is set correctly
1059 :expectedresults:
1060 1. Success
1061 2. Success
1062 """
1063 log.info("Starting test_threshold_to_overflow_value")
1064 inst = topo.standalone
1065
1066 overflow_value = '3000000000'
1067 log.info(f"Setting threshold to overflow value: {overflow_value}")
1068
1069 inst.config.set('nsslapd-disk-monitoring-threshold', ensure_bytes(overflow_value))
1070
1071 config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-disk-monitoring-threshold'])
1072 current_value = re.findall(r'nsslapd-disk-monitoring-threshold: \d+', str(config_entry))[0].split(' ')[1]
1073 assert overflow_value == current_value
1074
1075 log.info(f"Verified: threshold value set to {current_value}")
1076 log.info("Test completed successfully")
1077
1078
1079 @disk_monitoring_ack
1080 def test_threshold_is_reached_to_half(topo, setup, reset_logs):
1081 """Verify RHDS not shutting down when disk monitoring threshold is reached to half.
1082
1083 :id: b2d3665e-fe9e-11e8-b9c0-8c16451d917b
1084 :setup: Standalone
1085 :steps:
1086 1. Configure disk monitoring with critical logging
1087 2. Go below threshold
1088 3. Verify there is no endless loop of error messages
1089 :expectedresults:
1090 1. Success
1091 2. Success
1092 3. Success
1093 """
1094 log.info("Starting test_threshold_is_reached_to_half")
1095 inst = topo.standalone
1096 fill_file = None
1097
1098 try:
1099 log.info("Configuring disk monitoring with critical logging enabled")
1100 inst.config.set('nsslapd-disk-monitoring', 'on')
1101 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on')
1102 inst.config.set('nsslapd-errorlog-level', '8')
1103 inst.config.set('nsslapd-disk-monitoring-threshold', ensure_bytes(str(THRESHOLD_BYTES)))
1104 inst.restart()
1105
1106 log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)")
1107 fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1)
1108
1109 log.info("Waiting for loglevel message and verifying it's not repeated")
1110 wait_for_log_entry(inst, "temporarily setting error loglevel to the default level", 11)
1111
1112 with open(inst.errlog, 'r') as err_log:
1113 content = err_log.read()
1114
1115 message_count = len(re.findall("temporarily setting error loglevel to the default level", content))
1116 assert message_count == 1, f"Expected 1 occurrence of message, found {message_count}"
1117
1118 log.info("Verified: no endless loop of error messages")
1119
1120 finally:
1121 if fill_file and os.path.exists(fill_file):
1122 log.debug(f"Cleaning up fill file: {fill_file}")
1123 os.remove(fill_file)
1124
1125 log.info("Test completed successfully")
1126
1127
1128 @disk_monitoring_ack
1129 @pytest.mark.parametrize("test_input,expected", [
1130 ("nsslapd-disk-monitoring-threshold", '-2'),
1131 ("nsslapd-disk-monitoring-threshold", '9223372036854775808'),
1132 ("nsslapd-disk-monitoring-threshold", '2047'),
1133 ("nsslapd-disk-monitoring-threshold", '0'),
1134 ("nsslapd-disk-monitoring-threshold", '-1294967296'),
1135 ("nsslapd-disk-monitoring-threshold", 'invalid'),
1136 ("nsslapd-disk-monitoring", 'invalid'),
1137 ("nsslapd-disk-monitoring", '1'),
1138 ("nsslapd-disk-monitoring-grace-period", '0'),
1139 ("nsslapd-disk-monitoring-grace-period", '525 948'),
1140 ("nsslapd-disk-monitoring-grace-period", '-1'),
1141 ("nsslapd-disk-monitoring-logging-critical", 'oninvalid'),
1142 ("nsslapd-disk-monitoring-grace-period", '-1'),
1143 ("nsslapd-disk-monitoring-grace-period", '0'),
1144 ])
1145 def test_negagtive_parameterize(topo, setup, reset_logs, test_input, expected):
1146 """Verify that invalid operations are not permitted.
1147
1148 :id: b88efbf8-fe9e-11e8-8499-8c16451d917b
1149 :parametrized: yes
1150 :setup: Standalone
1151 :steps:
1152 1. Try to set invalid configuration values
1153 :expectedresults:
1154 1. Configuration change should fail
1155 """
1156 log.info(f"Starting test_negagtive_parameterize for {test_input}={expected}")
1157 inst = topo.standalone
1158
1159 log.info(f"Attempting to set invalid value: {test_input}={expected}")
1160 with pytest.raises(Exception):
1161 inst.config.set(test_input, ensure_bytes(expected))
1162
1163 log.info("Verified: invalid configuration value was rejected")
1164 log.info("Test completed successfully")
1165
1166
1167 @disk_monitoring_ack
1168 def test_valid_operations_are_permitted(topo, setup, reset_logs):
1169 """Verify that valid operations are permitted.
1170
1171 :id: bd4f83f6-fe9e-11e8-88f4-8c16451d917b
1172 :setup: Standalone
1173 :steps:
1174 1. Perform various valid configuration operations
1175 :expectedresults:
1176 1. All operations should succeed
1177 """
1178 log.info("Starting test_valid_operations_are_permitted")
1179 inst = topo.standalone
1180
1181 log.info("Setting initial disk monitoring configuration")
1182 inst.config.set('nsslapd-disk-monitoring', 'on')
1183 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on')
1184 inst.config.set('nsslapd-errorlog-level', '8')
1185 inst.restart()
1186
1187 log.info("Testing deletion of nsslapd-disk-monitoring-threshold")
1188 inst.modify_s('cn=config', [(ldap.MOD_DELETE, 'nsslapd-disk-monitoring-threshold', '')])
1189
1190 log.info("Testing addition of nsslapd-disk-monitoring-threshold value")
1191 inst.config.add('nsslapd-disk-monitoring-threshold', '2000001')
1192
1193 log.info("Testing deletion of nsslapd-disk-monitoring")
1194 config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-disk-monitoring'])
1195 current_value = str(config_entry[0]).split(' ')[2].split('\n\n')[0]
1196 inst.modify_s('cn=config', [(ldap.MOD_DELETE, 'nsslapd-disk-monitoring', ensure_bytes(current_value))])
1197
1198 log.info("Testing addition of nsslapd-disk-monitoring value")
1199 inst.config.add('nsslapd-disk-monitoring', 'off')
1200
1201 log.info("Testing deletion of nsslapd-disk-monitoring-grace-period")
1202 inst.modify_s('cn=config', [(ldap.MOD_DELETE, 'nsslapd-disk-monitoring-grace-period', '')])
1203
1204 log.info("Testing addition of nsslapd-disk-monitoring-grace-period value")
1205 inst.config.add('nsslapd-disk-monitoring-grace-period', '61')
1206
1207 log.info("Testing deletion of nsslapd-disk-monitoring-logging-critical")
1208 config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-disk-monitoring-logging-critical'])
1209 current_value = str(config_entry[0]).split(' ')[2].split('\n\n')[0]
1210 inst.modify_s('cn=config', [(ldap.MOD_DELETE, 'nsslapd-disk-monitoring-logging-critical', ensure_bytes(current_value))])
1211
1212 log.info("Testing addition of nsslapd-disk-monitoring-logging-critical value")
1213 inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on')
1214
1215 log.info("All valid operations completed successfully")
1216 log.info("Test completed successfully")
1217
1218
1219 if __name__ == '__main__':
1220 CURRENT_FILE = os.path.realpath(__file__)
1221 pytest.main("-s -v %s" % CURRENT_FILE)
1222