| 1 |
# --- BEGIN COPYRIGHT BLOCK --- |
| 2 |
# Copyright (C) 2025 Red Hat, Inc. |
| 3 |
# All rights reserved. |
| 4 |
# |
| 5 |
# License: GPL (version 3 or any later version). |
| 6 |
# See LICENSE for details. |
| 7 |
# --- END COPYRIGHT BLOCK --- |
| 8 |
|
| 9 |
import os |
| 10 |
import subprocess |
| 11 |
import re |
| 12 |
import time |
| 13 |
import ldap |
| 14 |
import pytest |
| 15 |
import logging |
| 16 |
from lib389.tasks import * |
| 17 |
from lib389._constants import * |
| 18 |
from lib389.utils import ensure_bytes |
| 19 |
from lib389.backend import Backends |
| 20 |
from test389.topologies import topology_st as topo |
| 21 |
from lib389.paths import * |
| 22 |
from lib389.idm.user import UserAccounts |
| 23 |
|
| 24 |
DEBUGGING = os.getenv("DEBUGGING", default=False) |
| 25 |
if DEBUGGING: |
| 26 |
logging.getLogger(__name__).setLevel(logging.DEBUG) |
| 27 |
else: |
| 28 |
logging.getLogger(__name__).setLevel(logging.INFO) |
| 29 |
log = logging.getLogger(__name__) |
| 30 |
|
| 31 |
pytestmark = pytest.mark.tier2 |
| 32 |
disk_monitoring_ack = pytest.mark.skipif(not os.environ.get('DISK_MONITORING_ACK', False), reason="Disk monitoring tests may damage system configuration.") |
| 33 |
|
| 34 |
THRESHOLD_BYTES = 30000000 |
| 35 |
|
| 36 |
|
| 37 |
def presetup(inst): |
| 38 |
"""Presetup function to mount a tmpfs for log directory to simulate disk space limits.""" |
| 39 |
|
| 40 |
log.info("Setting up tmpfs for disk monitoring tests") |
| 41 |
inst.stop() |
| 42 |
log_dir = inst.ds_paths.log_dir |
| 43 |
|
| 44 |
if os.path.exists(log_dir): |
| 45 |
log.debug(f"Mounting tmpfs on existing directory: {log_dir}") |
| 46 |
subprocess.call(['mount', '-t', 'tmpfs', '-o', 'size=35M', 'tmpfs', log_dir]) |
| 47 |
else: |
| 48 |
log.debug(f"Creating and mounting tmpfs on new directory: {log_dir}") |
| 49 |
os.mkdir(log_dir) |
| 50 |
subprocess.call(['mount', '-t', 'tmpfs', '-o', 'size=35M', 'tmpfs', log_dir]) |
| 51 |
|
| 52 |
subprocess.call(f'chown {DEFAULT_USER}: -R {log_dir}', shell=True) |
| 53 |
subprocess.call(f'chown {DEFAULT_USER}: -R {log_dir}/*', shell=True) |
| 54 |
subprocess.call(f'restorecon -FvvR {log_dir}', shell=True) |
| 55 |
inst.start() |
| 56 |
log.info("tmpfs setup completed") |
| 57 |
|
| 58 |
|
| 59 |
def setupthesystem(inst): |
| 60 |
"""Setup system configuration for disk monitoring tests.""" |
| 61 |
|
| 62 |
log.info("Configuring system for disk monitoring tests") |
| 63 |
inst.start() |
| 64 |
inst.config.set('nsslapd-disk-monitoring-grace-period', '1') |
| 65 |
inst.config.set('nsslapd-accesslog-logbuffering', 'off') |
| 66 |
inst.config.set('nsslapd-disk-monitoring-threshold', ensure_bytes(str(THRESHOLD_BYTES))) |
| 67 |
inst.restart() |
| 68 |
log.info("System configuration completed") |
| 69 |
|
| 70 |
|
| 71 |
def capture_config(inst): |
| 72 |
"""Capture current configuration values for later restoration.""" |
| 73 |
|
| 74 |
log.info("Capturing current configuration values") |
| 75 |
|
| 76 |
config_attrs = [ |
| 77 |
'nsslapd-disk-monitoring', |
| 78 |
'nsslapd-disk-monitoring-threshold', |
| 79 |
'nsslapd-disk-monitoring-grace-period', |
| 80 |
'nsslapd-disk-monitoring-logging-critical', |
| 81 |
'nsslapd-disk-monitoring-readonly-on-threshold', |
| 82 |
'nsslapd-accesslog-logbuffering', |
| 83 |
'nsslapd-errorlog-level', |
| 84 |
'nsslapd-accesslog-logging-enabled', |
| 85 |
'nsslapd-accesslog-maxlogsize', |
| 86 |
'nsslapd-accesslog-logrotationtimeunit', |
| 87 |
'nsslapd-accesslog-level', |
| 88 |
'nsslapd-external-libs-debug-enabled', |
| 89 |
'nsslapd-errorlog-logging-enabled' |
| 90 |
] |
| 91 |
|
| 92 |
captured_config = {} |
| 93 |
for config_attr in config_attrs: |
| 94 |
try: |
| 95 |
current_value = inst.config.get_attr_val_utf8(config_attr) |
| 96 |
captured_config[config_attr] = current_value |
| 97 |
log.debug(f"Captured {config_attr}: {current_value}") |
| 98 |
except Exception as e: |
| 99 |
log.debug(f"Could not capture {config_attr}: {e}") |
| 100 |
captured_config[config_attr] = None |
| 101 |
|
| 102 |
log.info("Configuration capture completed") |
| 103 |
return captured_config |
| 104 |
|
| 105 |
|
| 106 |
def restore_config(inst, captured_config): |
| 107 |
"""Restore configuration values to previously captured state.""" |
| 108 |
|
| 109 |
log.info("Restoring configuration to captured values") |
| 110 |
|
| 111 |
for config_attr, original_value in captured_config.items(): |
| 112 |
if original_value is not None: |
| 113 |
try: |
| 114 |
current_value = inst.config.get_attr_val_utf8(config_attr) |
| 115 |
if current_value != original_value: |
| 116 |
log.debug(f"Restoring {config_attr} from '{current_value}' to '{original_value}'") |
| 117 |
inst.config.set(config_attr, ensure_bytes(original_value)) |
| 118 |
except Exception as e: |
| 119 |
log.debug(f"Could not restore {config_attr}: {e}") |
| 120 |
|
| 121 |
log.info("Configuration restoration completed") |
| 122 |
|
| 123 |
|
| 124 |
@pytest.fixture(scope="function") |
| 125 |
def setup(request, topo): |
| 126 |
"""Module-level fixture to setup the test environment.""" |
| 127 |
|
| 128 |
log.info("Starting module setup for disk monitoring tests") |
| 129 |
inst = topo.standalone |
| 130 |
|
| 131 |
# Capture current configuration before making any changes |
| 132 |
original_config = capture_config(inst) |
| 133 |
|
| 134 |
presetup(inst) |
| 135 |
setupthesystem(inst) |
| 136 |
|
| 137 |
def fin(): |
| 138 |
log.info("Running module cleanup for disk monitoring tests") |
| 139 |
inst.stop() |
| 140 |
subprocess.call(['umount', '-fl', inst.ds_paths.log_dir]) |
| 141 |
# Restore configuration to original values |
| 142 |
inst.start() |
| 143 |
restore_config(inst, original_config) |
| 144 |
log.info("Module cleanup completed") |
| 145 |
|
| 146 |
request.addfinalizer(fin) |
| 147 |
|
| 148 |
|
| 149 |
def wait_for_condition(inst, condition_str, timeout=30): |
| 150 |
"""Wait until the given condition evaluates to False.""" |
| 151 |
|
| 152 |
log.debug(f"Waiting for condition to be False: {condition_str} (timeout: {timeout}s)") |
| 153 |
start_time = time.time() |
| 154 |
while time.time() - start_time < timeout: |
| 155 |
if not eval(condition_str): |
| 156 |
log.debug(f"Condition satisfied after {time.time() - start_time:.2f}s") |
| 157 |
return |
| 158 |
time.sleep(1) |
| 159 |
raise AssertionError(f"Condition '{condition_str}' still True after {timeout} seconds") |
| 160 |
|
| 161 |
|
| 162 |
def wait_for_log_entry(inst, message, timeout=30): |
| 163 |
"""Wait for a specific message to appear in the error log.""" |
| 164 |
|
| 165 |
log.debug(f"Waiting for log entry: '{message}' (timeout: {timeout}s)") |
| 166 |
start_time = time.time() |
| 167 |
while time.time() - start_time < timeout: |
| 168 |
with open(inst.errlog, 'r') as log_file: |
| 169 |
if message in log_file.read(): |
| 170 |
log.debug(f"Found log entry after {time.time() - start_time:.2f}s") |
| 171 |
return |
| 172 |
time.sleep(1) |
| 173 |
raise AssertionError(f"Message '{message}' not found in error log after {timeout} seconds") |
| 174 |
|
| 175 |
|
| 176 |
def get_avail_bytes(path): |
| 177 |
"""Get available bytes on the filesystem at the given path.""" |
| 178 |
|
| 179 |
stat = os.statvfs(path) |
| 180 |
return stat.f_bavail * stat.f_bsize |
| 181 |
|
| 182 |
|
| 183 |
def fill_to_target_avail(path, target_avail_bytes): |
| 184 |
"""Fill the disk to reach the target available bytes by creating a large file.""" |
| 185 |
|
| 186 |
avail = get_avail_bytes(path) |
| 187 |
fill_bytes = avail - target_avail_bytes |
| 188 |
log.debug(f"Current available: {avail}, target: {target_avail_bytes}, will create {fill_bytes} byte file") |
| 189 |
if fill_bytes <= 0: |
| 190 |
raise ValueError("Already below target avail") |
| 191 |
|
| 192 |
fill_file = os.path.join(path, 'fill.dd') |
| 193 |
bs = 4096 |
| 194 |
count = (fill_bytes + bs - 1) // bs # ceil division to ensure enough |
| 195 |
log.info(f"Creating fill file {fill_file} with {count} blocks of {bs} bytes") |
| 196 |
subprocess.check_call(['dd', 'if=/dev/zero', f'of={fill_file}', f'bs={bs}', f'count={count}']) |
| 197 |
return fill_file |
| 198 |
|
| 199 |
|
| 200 |
@pytest.fixture(scope="function") |
| 201 |
def reset_logs(topo): |
| 202 |
"""Function-level fixture to reset the error log before each test.""" |
| 203 |
|
| 204 |
log.debug("Resetting error logs before test") |
| 205 |
topo.standalone.deleteErrorLogs() |
| 206 |
|
| 207 |
|
| 208 |
def generate_access_log_activity(inst, num_users=10, num_binds=100): |
| 209 |
"""Generate access log activity by creating users and performing binds.""" |
| 210 |
|
| 211 |
log.info(f"Generating access log activity with {num_users} users and {num_binds} binds each") |
| 212 |
users = UserAccounts(inst, DEFAULT_SUFFIX) |
| 213 |
|
| 214 |
# Create test users |
| 215 |
for i in range(num_users): |
| 216 |
user_properties = { |
| 217 |
'uid': f'cn=user{i}', |
| 218 |
'cn': f'cn=user{i}', |
| 219 |
'sn': f'cn=user{i}', |
| 220 |
'userPassword': "Itsme123", |
| 221 |
'uidNumber': f'1{i}', |
| 222 |
'gidNumber': f'2{i}', |
| 223 |
'homeDirectory': f'/home/{i}' |
| 224 |
} |
| 225 |
users.create(properties=user_properties) |
| 226 |
|
| 227 |
# Perform bind operations |
| 228 |
for j in range(num_binds): |
| 229 |
for user in users.list(): |
| 230 |
user.bind('Itsme123') |
| 231 |
|
| 232 |
log.info("Access log activity generation completed") |
| 233 |
return users |
| 234 |
|
| 235 |
|
| 236 |
@disk_monitoring_ack |
| 237 |
def test_verify_operation_when_disk_monitoring_is_off(topo, setup, reset_logs): |
| 238 |
"""Verify operation when Disk monitoring is off. |
| 239 |
|
| 240 |
:id: 73a97536-fe9e-11e8-ba9f-8c16451d917b |
| 241 |
:setup: Standalone |
| 242 |
:steps: |
| 243 |
1. Turn off disk monitoring |
| 244 |
2. Go below the threshold |
| 245 |
3. Check DS is up and not entering shutdown mode |
| 246 |
:expectedresults: |
| 247 |
1. Success |
| 248 |
2. Success |
| 249 |
3. Success |
| 250 |
""" |
| 251 |
log.info("Starting test_verify_operation_when_disk_monitoring_is_off") |
| 252 |
inst = topo.standalone |
| 253 |
fill_file = None |
| 254 |
|
| 255 |
try: |
| 256 |
log.info("Disabling disk monitoring") |
| 257 |
inst.config.set('nsslapd-disk-monitoring', 'off') |
| 258 |
inst.restart() |
| 259 |
|
| 260 |
log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)") |
| 261 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES - 1) |
| 262 |
|
| 263 |
log.info("Verifying server stays up despite being below threshold") |
| 264 |
wait_for_condition(inst, 'inst.status() != True', 11) |
| 265 |
|
| 266 |
# Check DS is up and not entering shutdown mode |
| 267 |
assert inst.status() == True |
| 268 |
log.info("Verified: server remains operational when disk monitoring is disabled") |
| 269 |
|
| 270 |
finally: |
| 271 |
if fill_file and os.path.exists(fill_file): |
| 272 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 273 |
os.remove(fill_file) |
| 274 |
|
| 275 |
log.info("Test completed successfully") |
| 276 |
|
| 277 |
|
| 278 |
@disk_monitoring_ack |
| 279 |
def test_enable_external_libs_debug_log(topo, setup, reset_logs): |
| 280 |
"""Check that OpenLDAP logs are successfully enabled and disabled when disk threshold is reached. |
| 281 |
|
| 282 |
:id: 121b2b24-ecba-48e2-9ee2-312d929dc8c6 |
| 283 |
:setup: Standalone instance |
| 284 |
:steps: |
| 285 |
1. Set nsslapd-external-libs-debug-enabled to "on" |
| 286 |
2. Go straight below 1/2 of the threshold |
| 287 |
3. Verify that the external libs debug setting is disabled |
| 288 |
4. Go back above 1/2 of the threshold |
| 289 |
5. Verify that the external libs debug setting is enabled back |
| 290 |
:expectedresults: |
| 291 |
1. Success |
| 292 |
2. Success |
| 293 |
3. Success |
| 294 |
4. Success |
| 295 |
5. Success |
| 296 |
""" |
| 297 |
log.info("Starting test_enable_external_libs_debug_log") |
| 298 |
inst = topo.standalone |
| 299 |
fill_file = None |
| 300 |
|
| 301 |
try: |
| 302 |
log.info("Configuring disk monitoring and external libs debug") |
| 303 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 304 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off') |
| 305 |
inst.config.set('nsslapd-external-libs-debug-enabled', 'on') |
| 306 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 307 |
inst.restart() |
| 308 |
|
| 309 |
log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)") |
| 310 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1) |
| 311 |
|
| 312 |
log.info("Verifying external libs debug is automatically disabled") |
| 313 |
wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-external-libs-debug-enabled') != 'off'", 31) |
| 314 |
|
| 315 |
finally: |
| 316 |
if fill_file and os.path.exists(fill_file): |
| 317 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 318 |
os.remove(fill_file) |
| 319 |
|
| 320 |
log.info("Verifying external libs debug is re-enabled after freeing space") |
| 321 |
wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-external-libs-debug-enabled') != 'on'", 31) |
| 322 |
inst.config.set('nsslapd-external-libs-debug-enabled', 'off') |
| 323 |
|
| 324 |
log.info("Test completed successfully") |
| 325 |
|
| 326 |
|
| 327 |
@disk_monitoring_ack |
| 328 |
def test_free_up_the_disk_space_and_change_ds_config(topo, setup, reset_logs): |
| 329 |
"""Free up the disk space and change DS config. |
| 330 |
|
| 331 |
:id: 7be4d560-fe9e-11e8-a307-8c16451d917b |
| 332 |
:setup: Standalone |
| 333 |
:steps: |
| 334 |
1. Enable Disk Monitoring plugin and set disk monitoring logging to critical |
| 335 |
2. Verify no message about loglevel is present in the error log |
| 336 |
3. Verify no message about disabling logging is present in the error log |
| 337 |
4. Verify no message about removing rotated logs is present in the error log |
| 338 |
:expectedresults: |
| 339 |
1. Success |
| 340 |
2. Success |
| 341 |
3. Success |
| 342 |
4. Success |
| 343 |
""" |
| 344 |
log.info("Starting test_free_up_the_disk_space_and_change_ds_config") |
| 345 |
inst = topo.standalone |
| 346 |
|
| 347 |
log.info("Enabling disk monitoring with critical logging") |
| 348 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 349 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on') |
| 350 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 351 |
inst.restart() |
| 352 |
|
| 353 |
log.info("Verifying no premature disk monitoring messages in error log") |
| 354 |
with open(inst.errlog, 'r') as err_log: |
| 355 |
content = err_log.read() |
| 356 |
|
| 357 |
assert 'temporarily setting error loglevel to zero' not in content |
| 358 |
assert 'disabling access and audit logging' not in content |
| 359 |
assert 'deleting rotated logs' not in content |
| 360 |
|
| 361 |
log.info("Verified: no unexpected disk monitoring messages found") |
| 362 |
log.info("Test completed successfully") |
| 363 |
|
| 364 |
|
| 365 |
@disk_monitoring_ack |
| 366 |
def test_verify_operation_with_nsslapd_disk_monitoring_logging_critical_off(topo, setup, reset_logs): |
| 367 |
"""Verify operation with "nsslapd-disk-monitoring-logging-critical: off". |
| 368 |
|
| 369 |
:id: 82363bca-fe9e-11e8-9ae7-8c16451d917b |
| 370 |
:setup: Standalone |
| 371 |
:steps: |
| 372 |
1. Verify that verbose logging was set to default level |
| 373 |
2. Verify that logging is disabled |
| 374 |
3. Verify that rotated logs were not removed |
| 375 |
:expectedresults: |
| 376 |
1. Success |
| 377 |
2. Success |
| 378 |
3. Success |
| 379 |
""" |
| 380 |
log.info("Starting test_verify_operation_with_nsslapd_disk_monitoring_logging_critical_off") |
| 381 |
inst = topo.standalone |
| 382 |
fill_file = None |
| 383 |
|
| 384 |
try: |
| 385 |
log.info("Configuring disk monitoring with critical logging disabled") |
| 386 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 387 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off') |
| 388 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 389 |
inst.restart() |
| 390 |
|
| 391 |
log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)") |
| 392 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES - 1) |
| 393 |
|
| 394 |
log.info("Waiting for loglevel to be set to default") |
| 395 |
wait_for_log_entry(inst, 'temporarily setting error loglevel to the default level', 11) |
| 396 |
|
| 397 |
log.info("Verifying error log level was set to default") |
| 398 |
config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-errorlog-level']) |
| 399 |
current_level = int(re.findall(r'nsslapd-errorlog-level: \d+', str(config_entry))[0].split(' ')[1]) |
| 400 |
assert LOG_DEFAULT == current_level |
| 401 |
|
| 402 |
log.info("Verifying access logging is disabled") |
| 403 |
wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') != 'off'", 11) |
| 404 |
assert inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') == 'off' |
| 405 |
|
| 406 |
log.info("Verifying expected disk monitoring messages") |
| 407 |
with open(inst.errlog, 'r') as err_log: |
| 408 |
content = err_log.read() |
| 409 |
|
| 410 |
assert 'disabling access and audit logging' in content |
| 411 |
wait_for_log_entry(inst, 'deleting rotated logs', 11) |
| 412 |
assert f"Unable to remove file: {inst.ds_paths.log_dir}" not in content |
| 413 |
assert 'is too far below the threshold' not in content |
| 414 |
|
| 415 |
log.info("All verifications passed") |
| 416 |
|
| 417 |
finally: |
| 418 |
if fill_file and os.path.exists(fill_file): |
| 419 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 420 |
os.remove(fill_file) |
| 421 |
|
| 422 |
log.info("Test completed successfully") |
| 423 |
|
| 424 |
|
| 425 |
@disk_monitoring_ack |
| 426 |
def test_operation_with_nsslapd_disk_monitoring_logging_critical_on_below_half_of_the_threshold(topo, setup, reset_logs): |
| 427 |
"""Verify operation with "nsslapd-disk-monitoring-logging-critical: on" below 1/2 of the threshold. |
| 428 |
Verify recovery. |
| 429 |
|
| 430 |
:id: 8940c502-fe9e-11e8-bcc0-8c16451d917b |
| 431 |
:setup: Standalone |
| 432 |
:steps: |
| 433 |
1. Verify that DS goes into shutdown mode |
| 434 |
2. Verify that DS exited shutdown mode |
| 435 |
:expectedresults: |
| 436 |
1. Success |
| 437 |
2. Success |
| 438 |
""" |
| 439 |
log.info("Starting test_operation_with_nsslapd_disk_monitoring_logging_critical_on_below_half_of_the_threshold") |
| 440 |
inst = topo.standalone |
| 441 |
fill_file = None |
| 442 |
|
| 443 |
try: |
| 444 |
log.info("Configuring disk monitoring with critical logging enabled") |
| 445 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 446 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on') |
| 447 |
inst.restart() |
| 448 |
|
| 449 |
log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)") |
| 450 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1) |
| 451 |
|
| 452 |
log.info("Waiting for shutdown mode message") |
| 453 |
wait_for_log_entry(inst, 'is too far below the threshold', 100) |
| 454 |
|
| 455 |
log.info("Freeing up disk space") |
| 456 |
os.remove(fill_file) |
| 457 |
fill_file = None |
| 458 |
|
| 459 |
log.info("Waiting for recovery message") |
| 460 |
wait_for_log_entry(inst, 'Available disk space is now acceptable', 25) |
| 461 |
|
| 462 |
log.info("Verified: server entered and exited shutdown mode correctly") |
| 463 |
|
| 464 |
finally: |
| 465 |
if fill_file and os.path.exists(fill_file): |
| 466 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 467 |
os.remove(fill_file) |
| 468 |
|
| 469 |
log.info("Test completed successfully") |
| 470 |
|
| 471 |
|
| 472 |
@disk_monitoring_ack |
| 473 |
def test_setting_nsslapd_disk_monitoring_logging_critical_to_off(topo, setup, reset_logs): |
| 474 |
"""Setting nsslapd-disk-monitoring-logging-critical to "off". |
| 475 |
|
| 476 |
:id: 93265ec4-fe9e-11e8-af93-8c16451d917b |
| 477 |
:setup: Standalone |
| 478 |
:steps: |
| 479 |
1. Set nsslapd-disk-monitoring-logging-critical to "off" |
| 480 |
:expectedresults: |
| 481 |
1. Success |
| 482 |
""" |
| 483 |
log.info("Starting test_setting_nsslapd_disk_monitoring_logging_critical_to_off") |
| 484 |
inst = topo.standalone |
| 485 |
|
| 486 |
log.info("Setting disk monitoring configuration") |
| 487 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 488 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off') |
| 489 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 490 |
inst.restart() |
| 491 |
|
| 492 |
log.info("Verifying server is running normally") |
| 493 |
assert inst.status() == True |
| 494 |
|
| 495 |
log.info("Test completed successfully") |
| 496 |
|
| 497 |
|
| 498 |
@disk_monitoring_ack |
| 499 |
def test_operation_with_nsslapd_disk_monitoring_logging_critical_off(topo, setup, reset_logs): |
| 500 |
"""Verify operation with nsslapd-disk-monitoring-logging-critical: off. |
| 501 |
|
| 502 |
:id: 97985a52-fe9e-11e8-9914-8c16451d917b |
| 503 |
:setup: Standalone |
| 504 |
:steps: |
| 505 |
1. Generate access log activity to create rotated logs |
| 506 |
2. Go below threshold to trigger disk monitoring |
| 507 |
3. Verify that verbose logging was set to default level |
| 508 |
4. Verify that logging is disabled |
| 509 |
5. Verify that rotated logs were removed |
| 510 |
:expectedresults: |
| 511 |
1. Success |
| 512 |
2. Success |
| 513 |
3. Success |
| 514 |
4. Success |
| 515 |
5. Success |
| 516 |
""" |
| 517 |
log.info("Starting test_operation_with_nsslapd_disk_monitoring_logging_critical_off") |
| 518 |
inst = topo.standalone |
| 519 |
fill_file = None |
| 520 |
users = None |
| 521 |
|
| 522 |
try: |
| 523 |
log.info("Configuring disk monitoring and access log settings") |
| 524 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 525 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off') |
| 526 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 527 |
inst.config.set('nsslapd-accesslog-maxlogsize', '1') |
| 528 |
inst.config.set('nsslapd-accesslog-logrotationtimeunit', 'minute') |
| 529 |
inst.config.set('nsslapd-accesslog-level', '772') |
| 530 |
inst.restart() |
| 531 |
|
| 532 |
log.info("Generating access log activity to create rotated logs") |
| 533 |
users = generate_access_log_activity(inst, num_users=10, num_binds=100) |
| 534 |
|
| 535 |
inst.bind_s(DN_DM, PW_DM) |
| 536 |
|
| 537 |
log.info("Resetting access log settings") |
| 538 |
inst.config.set('nsslapd-accesslog-maxlogsize', '100') |
| 539 |
inst.config.set('nsslapd-accesslog-logrotationtimeunit', 'day') |
| 540 |
inst.config.set('nsslapd-accesslog-level', '256') |
| 541 |
inst.restart() |
| 542 |
|
| 543 |
log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)") |
| 544 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES - 1) |
| 545 |
|
| 546 |
log.info("Waiting for loglevel to be set to default") |
| 547 |
wait_for_log_entry(inst, 'temporarily setting error loglevel to the default level', 11) |
| 548 |
|
| 549 |
log.info("Verifying error log level was set to default") |
| 550 |
config_level = None |
| 551 |
for _ in range(10): |
| 552 |
time.sleep(1) |
| 553 |
config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-errorlog-level']) |
| 554 |
config_level = int(re.findall(r'nsslapd-errorlog-level: \d+', str(config_entry))[0].split(' ')[1]) |
| 555 |
if LOG_DEFAULT == config_level: |
| 556 |
break |
| 557 |
assert LOG_DEFAULT == config_level |
| 558 |
|
| 559 |
log.info("Verifying access logging is disabled") |
| 560 |
wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') == 'off'", 20) |
| 561 |
|
| 562 |
with open(inst.errlog, 'r') as err_log: |
| 563 |
content = err_log.read() |
| 564 |
assert 'disabling access and audit logging' in content |
| 565 |
|
| 566 |
log.info("Verifying rotated logs are removed") |
| 567 |
wait_for_log_entry(inst, 'deleting rotated logs', 20) |
| 568 |
|
| 569 |
rotated_logs = re.findall(r'access.\d+-\d+', str(os.listdir(inst.ds_paths.log_dir))) |
| 570 |
assert not rotated_logs, f"Found unexpected rotated logs: {rotated_logs}" |
| 571 |
|
| 572 |
with open(inst.errlog, 'r') as err_log: |
| 573 |
content = err_log.read() |
| 574 |
assert 'Unable to remove file:' not in content |
| 575 |
assert 'is too far below the threshold' not in content |
| 576 |
|
| 577 |
log.info("All verifications passed") |
| 578 |
|
| 579 |
finally: |
| 580 |
# Clean up users |
| 581 |
if users: |
| 582 |
log.debug("Cleaning up test users") |
| 583 |
for user in users.list(): |
| 584 |
try: |
| 585 |
user.delete() |
| 586 |
except ldap.ALREADY_EXISTS: |
| 587 |
pass |
| 588 |
|
| 589 |
if fill_file and os.path.exists(fill_file): |
| 590 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 591 |
os.remove(fill_file) |
| 592 |
|
| 593 |
log.info("Test completed successfully") |
| 594 |
|
| 595 |
|
| 596 |
@disk_monitoring_ack |
| 597 |
def test_operation_with_nsslapd_disk_monitoring_logging_critical_off_below_half_of_the_threshold(topo, setup, reset_logs): |
| 598 |
"""Verify operation with nsslapd-disk-monitoring-logging-critical: off below 1/2 of the threshold. |
| 599 |
Verify shutdown and recovery. |
| 600 |
|
| 601 |
:id: 9d4c7d48-fe9e-11e8-b5d6-8c16451d917b |
| 602 |
:setup: Standalone |
| 603 |
:steps: |
| 604 |
1. Go below half threshold to trigger shutdown |
| 605 |
2. Verify DS shutdown after grace period |
| 606 |
3. Free space and restart |
| 607 |
4. Verify logging is re-enabled |
| 608 |
5. Create rotated logs and enable verbose logging |
| 609 |
:expectedresults: |
| 610 |
1. Success |
| 611 |
2. Success |
| 612 |
3. Success |
| 613 |
4. Success |
| 614 |
5. Success |
| 615 |
""" |
| 616 |
log.info("Starting test_operation_with_nsslapd_disk_monitoring_logging_critical_off_below_half_of_the_threshold") |
| 617 |
inst = topo.standalone |
| 618 |
fill_file = None |
| 619 |
users = None |
| 620 |
|
| 621 |
try: |
| 622 |
log.info("Configuring disk monitoring with critical logging disabled") |
| 623 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 624 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off') |
| 625 |
inst.restart() |
| 626 |
|
| 627 |
log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)") |
| 628 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1) |
| 629 |
|
| 630 |
log.info("Waiting for shutdown messages") |
| 631 |
wait_for_log_entry(inst, 'is too far below the threshold', 100) |
| 632 |
wait_for_log_entry(inst, 'Signaling slapd for shutdown', 90) |
| 633 |
|
| 634 |
log.info("Verifying server shutdown within grace period") |
| 635 |
for i in range(60): |
| 636 |
time.sleep(1) |
| 637 |
if not inst.status(): |
| 638 |
log.info(f"Server shut down after {i+1} seconds") |
| 639 |
break |
| 640 |
assert inst.status() == False |
| 641 |
|
| 642 |
log.info("Freeing disk space and cleaning logs") |
| 643 |
os.remove(fill_file) |
| 644 |
fill_file = None |
| 645 |
open(f'{inst.ds_paths.log_dir}/errors', 'w').close() |
| 646 |
|
| 647 |
log.info("Starting server after freeing space") |
| 648 |
inst.start() |
| 649 |
|
| 650 |
log.info("Verifying logging is re-enabled") |
| 651 |
assert inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') == 'on' |
| 652 |
assert inst.config.get_attr_val_utf8('nsslapd-errorlog-logging-enabled') == 'on' |
| 653 |
|
| 654 |
with open(inst.errlog, 'r') as err_log: |
| 655 |
content = err_log.read() |
| 656 |
assert 'disabling access and audit logging' not in content |
| 657 |
|
| 658 |
log.info("Setting up access log rotation for testing") |
| 659 |
inst.config.set('nsslapd-accesslog-maxlogsize', '1') |
| 660 |
inst.config.set('nsslapd-accesslog-logrotationtimeunit', 'minute') |
| 661 |
inst.config.set('nsslapd-accesslog-level', '772') |
| 662 |
inst.restart() |
| 663 |
|
| 664 |
log.info("Creating rotated log files through user activity") |
| 665 |
users = generate_access_log_activity(inst, num_users=10, num_binds=100) |
| 666 |
|
| 667 |
log.info("Waiting for log rotation to occur") |
| 668 |
for i in range(61): |
| 669 |
time.sleep(1) |
| 670 |
rotated_logs = re.findall(r'access.\d+-\d+', str(os.listdir(inst.ds_paths.log_dir))) |
| 671 |
if rotated_logs: |
| 672 |
log.info(f"Log rotation detected after {i+1} seconds") |
| 673 |
break |
| 674 |
assert rotated_logs, "No rotated logs found after waiting" |
| 675 |
|
| 676 |
inst.bind_s(DN_DM, PW_DM) |
| 677 |
|
| 678 |
log.info("Enabling verbose logging") |
| 679 |
inst.config.set('nsslapd-accesslog-maxlogsize', '100') |
| 680 |
inst.config.set('nsslapd-accesslog-logrotationtimeunit', 'day') |
| 681 |
inst.config.set('nsslapd-accesslog-level', '256') |
| 682 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 683 |
inst.restart() |
| 684 |
|
| 685 |
log.info("Recovery and setup verification completed") |
| 686 |
|
| 687 |
finally: |
| 688 |
# Clean up users |
| 689 |
if users: |
| 690 |
log.debug("Cleaning up test users") |
| 691 |
for user in users.list(): |
| 692 |
try: |
| 693 |
user.delete() |
| 694 |
except ldap.ALREADY_EXISTS: |
| 695 |
pass |
| 696 |
|
| 697 |
if fill_file and os.path.exists(fill_file): |
| 698 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 699 |
os.remove(fill_file) |
| 700 |
|
| 701 |
log.info("Test completed successfully") |
| 702 |
|
| 703 |
|
| 704 |
@disk_monitoring_ack |
| 705 |
def test_go_straight_below_half_of_the_threshold(topo, setup, reset_logs): |
| 706 |
"""Go straight below 1/2 of the threshold and verify recovery. |
| 707 |
|
| 708 |
:id: a2a0664c-fe9e-11e8-b220-8c16451d917b |
| 709 |
:setup: Standalone |
| 710 |
:steps: |
| 711 |
1. Go straight below 1/2 of the threshold |
| 712 |
2. Verify that verbose logging was set to default level |
| 713 |
3. Verify that logging is disabled |
| 714 |
4. Verify DS is in shutdown mode |
| 715 |
5. Verify DS has recovered from shutdown |
| 716 |
:expectedresults: |
| 717 |
1. Success |
| 718 |
2. Success |
| 719 |
3. Success |
| 720 |
4. Success |
| 721 |
5. Success |
| 722 |
""" |
| 723 |
log.info("Starting test_go_straight_below_half_of_the_threshold") |
| 724 |
inst = topo.standalone |
| 725 |
fill_file = None |
| 726 |
|
| 727 |
try: |
| 728 |
log.info("Configuring disk monitoring with critical logging disabled") |
| 729 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 730 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'off') |
| 731 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 732 |
inst.restart() |
| 733 |
|
| 734 |
# Go straight below half threshold |
| 735 |
log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)") |
| 736 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1) |
| 737 |
|
| 738 |
# Verify that verbose logging was set to default level |
| 739 |
log.info("Waiting for loglevel to be set to default") |
| 740 |
wait_for_log_entry(inst, 'temporarily setting error loglevel to the default level', 11) |
| 741 |
|
| 742 |
log.info("Verifying error log level was set to default") |
| 743 |
config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-errorlog-level']) |
| 744 |
current_level = int(re.findall(r'nsslapd-errorlog-level: \d+', str(config_entry))[0].split(' ')[1]) |
| 745 |
assert LOG_DEFAULT == current_level |
| 746 |
|
| 747 |
log.info("Verifying access logging is disabled") |
| 748 |
wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') != 'off'", 11) |
| 749 |
|
| 750 |
log.info("Verifying expected disk monitoring messages") |
| 751 |
wait_for_log_entry(inst, 'disabling access and audit logging', 2) |
| 752 |
wait_for_log_entry(inst, 'deleting rotated logs', 11) |
| 753 |
|
| 754 |
with open(inst.errlog, 'r') as err_log: |
| 755 |
content = err_log.read() |
| 756 |
assert 'Unable to remove file:' not in content |
| 757 |
|
| 758 |
log.info("Verifying server enters shutdown mode") |
| 759 |
wait_for_condition(inst, 'inst.status() != False', 90) |
| 760 |
wait_for_log_entry(inst, 'is too far below the threshold', 2) |
| 761 |
|
| 762 |
log.info("Freeing disk space and restarting server") |
| 763 |
os.remove(fill_file) |
| 764 |
fill_file = None |
| 765 |
open(f'{inst.ds_paths.log_dir}/errors', 'w').close() |
| 766 |
inst.start() |
| 767 |
|
| 768 |
log.info("Verifying server recovery") |
| 769 |
wait_for_condition(inst, "inst.config.get_attr_val_utf8('nsslapd-accesslog-logging-enabled') != 'on'", 20) |
| 770 |
|
| 771 |
with open(inst.errlog, 'r') as err_log: |
| 772 |
content = err_log.read() |
| 773 |
assert 'disabling access and audit logging' not in content |
| 774 |
|
| 775 |
log.info("Recovery verification completed") |
| 776 |
|
| 777 |
finally: |
| 778 |
if fill_file and os.path.exists(fill_file): |
| 779 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 780 |
os.remove(fill_file) |
| 781 |
|
| 782 |
log.info("Test completed successfully") |
| 783 |
|
| 784 |
|
| 785 |
@disk_monitoring_ack |
| 786 |
def test_readonly_on_threshold(topo, setup, reset_logs): |
| 787 |
"""Verify that nsslapd-disk-monitoring-readonly-on-threshold switches the server to read-only mode. |
| 788 |
|
| 789 |
:id: 06814c19-ef3c-4800-93c9-c7c6e76fcbb9 |
| 790 |
:customerscenario: True |
| 791 |
:setup: Standalone |
| 792 |
:steps: |
| 793 |
1. Configure readonly on threshold |
| 794 |
2. Go below threshold and verify backend is read-only |
| 795 |
3. Go back above threshold and verify backend is read-write |
| 796 |
:expectedresults: |
| 797 |
1. Success |
| 798 |
2. Success |
| 799 |
3. Success |
| 800 |
""" |
| 801 |
log.info("Starting test_readonly_on_threshold") |
| 802 |
inst = topo.standalone |
| 803 |
fill_file = None |
| 804 |
test_user = None |
| 805 |
|
| 806 |
try: |
| 807 |
backends = Backends(inst) |
| 808 |
backend_name = backends.list()[0].rdn |
| 809 |
log.info(f"Testing with backend: {backend_name}") |
| 810 |
|
| 811 |
log.info("Configuring disk monitoring with readonly on threshold") |
| 812 |
inst.deleteErrorLogs() |
| 813 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 814 |
inst.config.set('nsslapd-disk-monitoring-readonly-on-threshold', 'on') |
| 815 |
inst.restart() |
| 816 |
|
| 817 |
log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)") |
| 818 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES - 1) |
| 819 |
|
| 820 |
log.info("Waiting for backend to enter read-only mode") |
| 821 |
wait_for_log_entry(inst, f"Putting the backend '{backend_name}' to read-only mode", 11) |
| 822 |
|
| 823 |
log.info("Verifying backend is in read-only mode") |
| 824 |
users = UserAccounts(topo.standalone, DEFAULT_SUFFIX) |
| 825 |
try: |
| 826 |
test_user = users.create_test_user() |
| 827 |
test_user.delete() |
| 828 |
assert False, "Expected UNWILLING_TO_PERFORM error for read-only mode" |
| 829 |
except ldap.UNWILLING_TO_PERFORM as e: |
| 830 |
if 'database is read-only' not in str(e): |
| 831 |
raise |
| 832 |
log.info("Confirmed: backend correctly rejects writes in read-only mode") |
| 833 |
|
| 834 |
log.info("Freeing disk space") |
| 835 |
os.remove(fill_file) |
| 836 |
fill_file = None |
| 837 |
|
| 838 |
log.info("Waiting for backend to return to read-write mode") |
| 839 |
wait_for_log_entry(inst, f"Putting the backend '{backend_name}' back to read-write mode", 11) |
| 840 |
|
| 841 |
log.info("Verifying backend is in read-write mode") |
| 842 |
test_user = users.create_test_user() |
| 843 |
assert test_user.exists() |
| 844 |
test_user.delete() |
| 845 |
test_user = None |
| 846 |
|
| 847 |
log.info("Confirmed: backend correctly accepts writes in read-write mode") |
| 848 |
|
| 849 |
finally: |
| 850 |
if test_user: |
| 851 |
try: |
| 852 |
test_user.delete() |
| 853 |
except: |
| 854 |
pass |
| 855 |
|
| 856 |
if fill_file and os.path.exists(fill_file): |
| 857 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 858 |
os.remove(fill_file) |
| 859 |
|
| 860 |
log.info("Test completed successfully") |
| 861 |
|
| 862 |
|
| 863 |
@disk_monitoring_ack |
| 864 |
def test_readonly_on_threshold_below_half_of_the_threshold(topo, setup, reset_logs): |
| 865 |
"""Go below 1/2 of the threshold when readonly on threshold is enabled. |
| 866 |
|
| 867 |
:id: 10262663-b41f-420e-a2d0-9532dd54fa7c |
| 868 |
:customerscenario: True |
| 869 |
:setup: Standalone |
| 870 |
:steps: |
| 871 |
1. Configure readonly on threshold |
| 872 |
2. Go below half threshold |
| 873 |
3. Verify backend is read-only and shutdown messages appear |
| 874 |
4. Free space and verify backend returns to read-write |
| 875 |
:expectedresults: |
| 876 |
1. Success |
| 877 |
2. Success |
| 878 |
3. Success |
| 879 |
4. Success |
| 880 |
""" |
| 881 |
log.info("Starting test_readonly_on_threshold_below_half_of_the_threshold") |
| 882 |
inst = topo.standalone |
| 883 |
fill_file = None |
| 884 |
test_user = None |
| 885 |
|
| 886 |
try: |
| 887 |
backends = Backends(inst) |
| 888 |
backend_name = backends.list()[0].rdn |
| 889 |
log.info(f"Testing with backend: {backend_name}") |
| 890 |
|
| 891 |
log.info("Configuring disk monitoring with readonly on threshold") |
| 892 |
inst.deleteErrorLogs() |
| 893 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 894 |
inst.config.set('nsslapd-disk-monitoring-readonly-on-threshold', 'on') |
| 895 |
inst.restart() |
| 896 |
|
| 897 |
log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)") |
| 898 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1) |
| 899 |
|
| 900 |
log.info("Waiting for backend to enter read-only mode") |
| 901 |
wait_for_log_entry(inst, f"Putting the backend '{backend_name}' to read-only mode", 11) |
| 902 |
|
| 903 |
log.info("Verifying backend is in read-only mode") |
| 904 |
users = UserAccounts(inst, DEFAULT_SUFFIX) |
| 905 |
try: |
| 906 |
test_user = users.create_test_user() |
| 907 |
test_user.delete() |
| 908 |
assert False, "Expected UNWILLING_TO_PERFORM error for read-only mode" |
| 909 |
except ldap.UNWILLING_TO_PERFORM as e: |
| 910 |
if 'database is read-only' not in str(e): |
| 911 |
raise |
| 912 |
log.info("Confirmed: backend correctly rejects writes in read-only mode") |
| 913 |
|
| 914 |
log.info("Waiting for shutdown threshold message") |
| 915 |
wait_for_log_entry(inst, 'is too far below the threshold', 51) |
| 916 |
|
| 917 |
log.info("Freeing disk space") |
| 918 |
os.remove(fill_file) |
| 919 |
fill_file = None |
| 920 |
|
| 921 |
log.info("Waiting for backend to return to read-write mode") |
| 922 |
wait_for_log_entry(inst, f"Putting the backend '{backend_name}' back to read-write mode", 51) |
| 923 |
|
| 924 |
log.info("Verifying backend is in read-write mode") |
| 925 |
test_user = users.create_test_user() |
| 926 |
assert test_user.exists() |
| 927 |
test_user.delete() |
| 928 |
test_user = None |
| 929 |
|
| 930 |
log.info("Confirmed: backend correctly accepts writes in read-write mode") |
| 931 |
|
| 932 |
finally: |
| 933 |
if test_user: |
| 934 |
try: |
| 935 |
test_user.delete() |
| 936 |
except: |
| 937 |
pass |
| 938 |
|
| 939 |
if fill_file and os.path.exists(fill_file): |
| 940 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 941 |
os.remove(fill_file) |
| 942 |
|
| 943 |
log.info("Test completed successfully") |
| 944 |
|
| 945 |
|
| 946 |
@disk_monitoring_ack |
| 947 |
def test_below_half_of_the_threshold_not_starting_after_shutdown(topo, setup, reset_logs): |
| 948 |
"""Test that the instance won't start if we are below 1/2 of the threshold. |
| 949 |
|
| 950 |
:id: cceeaefd-9fa4-45c5-9ac6-9887a0671ef8 |
| 951 |
:customerscenario: True |
| 952 |
:setup: Standalone |
| 953 |
:steps: |
| 954 |
1. Go below half threshold and wait for shutdown |
| 955 |
2. Try to start the instance and verify it fails |
| 956 |
3. Free space and verify instance starts successfully |
| 957 |
:expectedresults: |
| 958 |
1. Success |
| 959 |
2. Startup fails as expected |
| 960 |
3. Success |
| 961 |
""" |
| 962 |
log.info("Starting test_below_half_of_the_threshold_not_starting_after_shutdown") |
| 963 |
inst = topo.standalone |
| 964 |
fill_file = None |
| 965 |
|
| 966 |
try: |
| 967 |
log.info("Configuring disk monitoring") |
| 968 |
inst.deleteErrorLogs() |
| 969 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 970 |
inst.restart() |
| 971 |
|
| 972 |
log.info(f"Filling disk to go below half threshold ({THRESHOLD_BYTES // 2} bytes)") |
| 973 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1) |
| 974 |
|
| 975 |
log.info("Waiting for server to shut down due to disk space") |
| 976 |
wait_for_condition(inst, 'inst.status() == True', 120) |
| 977 |
|
| 978 |
log.info("Attempting to start instance (should fail)") |
| 979 |
try: |
| 980 |
inst.start() |
| 981 |
assert False, "Instance startup should have failed due to low disk space" |
| 982 |
except (ValueError, subprocess.CalledProcessError): |
| 983 |
log.info("Instance startup failed as expected due to low disk space") |
| 984 |
|
| 985 |
wait_for_log_entry(inst, f'is too far below the threshold({THRESHOLD_BYTES} bytes). Exiting now', 2) |
| 986 |
|
| 987 |
log.info("Freeing disk space") |
| 988 |
os.remove(fill_file) |
| 989 |
fill_file = None |
| 990 |
|
| 991 |
log.info("Starting instance after freeing space") |
| 992 |
inst.start() |
| 993 |
assert inst.status() == True |
| 994 |
log.info("Instance started successfully after freeing space") |
| 995 |
|
| 996 |
finally: |
| 997 |
if fill_file and os.path.exists(fill_file): |
| 998 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 999 |
os.remove(fill_file) |
| 1000 |
|
| 1001 |
log.info("Test completed successfully") |
| 1002 |
|
| 1003 |
|
| 1004 |
@disk_monitoring_ack |
| 1005 |
def test_go_straight_below_4kb(topo, setup, reset_logs): |
| 1006 |
"""Go straight below 4KB and verify behavior. |
| 1007 |
|
| 1008 |
:id: a855115a-fe9e-11e8-8e91-8c16451d917b |
| 1009 |
:setup: Standalone |
| 1010 |
:steps: |
| 1011 |
1. Go straight below 4KB |
| 1012 |
2. Verify server behavior |
| 1013 |
3. Clean space and restart |
| 1014 |
:expectedresults: |
| 1015 |
1. Success |
| 1016 |
2. Success |
| 1017 |
3. Success |
| 1018 |
""" |
| 1019 |
log.info("Starting test_go_straight_below_4kb") |
| 1020 |
inst = topo.standalone |
| 1021 |
fill_file = None |
| 1022 |
|
| 1023 |
try: |
| 1024 |
log.info("Configuring disk monitoring") |
| 1025 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 1026 |
inst.restart() |
| 1027 |
|
| 1028 |
log.info("Filling disk to go below 4KB") |
| 1029 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, 4000) |
| 1030 |
|
| 1031 |
log.info("Waiting for server shutdown due to extreme low disk space") |
| 1032 |
wait_for_condition(inst, 'inst.status() != False', 11) |
| 1033 |
|
| 1034 |
log.info("Freeing disk space and restarting") |
| 1035 |
os.remove(fill_file) |
| 1036 |
fill_file = None |
| 1037 |
inst.start() |
| 1038 |
|
| 1039 |
assert inst.status() == True |
| 1040 |
log.info("Server restarted successfully after freeing space") |
| 1041 |
|
| 1042 |
finally: |
| 1043 |
if fill_file and os.path.exists(fill_file): |
| 1044 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 1045 |
os.remove(fill_file) |
| 1046 |
|
| 1047 |
log.info("Test completed successfully") |
| 1048 |
|
| 1049 |
|
| 1050 |
@disk_monitoring_ack |
| 1051 |
def test_threshold_to_overflow_value(topo, setup, reset_logs): |
| 1052 |
"""Test overflow in nsslapd-disk-monitoring-threshold. |
| 1053 |
|
| 1054 |
:id: ad60ab3c-fe9e-11e8-88dc-8c16451d917b |
| 1055 |
:setup: Standalone |
| 1056 |
:steps: |
| 1057 |
1. Set nsslapd-disk-monitoring-threshold to overflow value |
| 1058 |
2. Verify the value is set correctly |
| 1059 |
:expectedresults: |
| 1060 |
1. Success |
| 1061 |
2. Success |
| 1062 |
""" |
| 1063 |
log.info("Starting test_threshold_to_overflow_value") |
| 1064 |
inst = topo.standalone |
| 1065 |
|
| 1066 |
overflow_value = '3000000000' |
| 1067 |
log.info(f"Setting threshold to overflow value: {overflow_value}") |
| 1068 |
|
| 1069 |
inst.config.set('nsslapd-disk-monitoring-threshold', ensure_bytes(overflow_value)) |
| 1070 |
|
| 1071 |
config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-disk-monitoring-threshold']) |
| 1072 |
current_value = re.findall(r'nsslapd-disk-monitoring-threshold: \d+', str(config_entry))[0].split(' ')[1] |
| 1073 |
assert overflow_value == current_value |
| 1074 |
|
| 1075 |
log.info(f"Verified: threshold value set to {current_value}") |
| 1076 |
log.info("Test completed successfully") |
| 1077 |
|
| 1078 |
|
| 1079 |
@disk_monitoring_ack |
| 1080 |
def test_threshold_is_reached_to_half(topo, setup, reset_logs): |
| 1081 |
"""Verify RHDS not shutting down when disk monitoring threshold is reached to half. |
| 1082 |
|
| 1083 |
:id: b2d3665e-fe9e-11e8-b9c0-8c16451d917b |
| 1084 |
:setup: Standalone |
| 1085 |
:steps: |
| 1086 |
1. Configure disk monitoring with critical logging |
| 1087 |
2. Go below threshold |
| 1088 |
3. Verify there is no endless loop of error messages |
| 1089 |
:expectedresults: |
| 1090 |
1. Success |
| 1091 |
2. Success |
| 1092 |
3. Success |
| 1093 |
""" |
| 1094 |
log.info("Starting test_threshold_is_reached_to_half") |
| 1095 |
inst = topo.standalone |
| 1096 |
fill_file = None |
| 1097 |
|
| 1098 |
try: |
| 1099 |
log.info("Configuring disk monitoring with critical logging enabled") |
| 1100 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 1101 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on') |
| 1102 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 1103 |
inst.config.set('nsslapd-disk-monitoring-threshold', ensure_bytes(str(THRESHOLD_BYTES))) |
| 1104 |
inst.restart() |
| 1105 |
|
| 1106 |
log.info(f"Filling disk to go below threshold ({THRESHOLD_BYTES} bytes)") |
| 1107 |
fill_file = fill_to_target_avail(inst.ds_paths.log_dir, THRESHOLD_BYTES // 2 - 1) |
| 1108 |
|
| 1109 |
log.info("Waiting for loglevel message and verifying it's not repeated") |
| 1110 |
wait_for_log_entry(inst, "temporarily setting error loglevel to the default level", 11) |
| 1111 |
|
| 1112 |
with open(inst.errlog, 'r') as err_log: |
| 1113 |
content = err_log.read() |
| 1114 |
|
| 1115 |
message_count = len(re.findall("temporarily setting error loglevel to the default level", content)) |
| 1116 |
assert message_count == 1, f"Expected 1 occurrence of message, found {message_count}" |
| 1117 |
|
| 1118 |
log.info("Verified: no endless loop of error messages") |
| 1119 |
|
| 1120 |
finally: |
| 1121 |
if fill_file and os.path.exists(fill_file): |
| 1122 |
log.debug(f"Cleaning up fill file: {fill_file}") |
| 1123 |
os.remove(fill_file) |
| 1124 |
|
| 1125 |
log.info("Test completed successfully") |
| 1126 |
|
| 1127 |
|
| 1128 |
@disk_monitoring_ack |
| 1129 |
@pytest.mark.parametrize("test_input,expected", [ |
| 1130 |
("nsslapd-disk-monitoring-threshold", '-2'), |
| 1131 |
("nsslapd-disk-monitoring-threshold", '9223372036854775808'), |
| 1132 |
("nsslapd-disk-monitoring-threshold", '2047'), |
| 1133 |
("nsslapd-disk-monitoring-threshold", '0'), |
| 1134 |
("nsslapd-disk-monitoring-threshold", '-1294967296'), |
| 1135 |
("nsslapd-disk-monitoring-threshold", 'invalid'), |
| 1136 |
("nsslapd-disk-monitoring", 'invalid'), |
| 1137 |
("nsslapd-disk-monitoring", '1'), |
| 1138 |
("nsslapd-disk-monitoring-grace-period", '0'), |
| 1139 |
("nsslapd-disk-monitoring-grace-period", '525 948'), |
| 1140 |
("nsslapd-disk-monitoring-grace-period", '-1'), |
| 1141 |
("nsslapd-disk-monitoring-logging-critical", 'oninvalid'), |
| 1142 |
("nsslapd-disk-monitoring-grace-period", '-1'), |
| 1143 |
("nsslapd-disk-monitoring-grace-period", '0'), |
| 1144 |
]) |
| 1145 |
def test_negagtive_parameterize(topo, setup, reset_logs, test_input, expected): |
| 1146 |
"""Verify that invalid operations are not permitted. |
| 1147 |
|
| 1148 |
:id: b88efbf8-fe9e-11e8-8499-8c16451d917b |
| 1149 |
:parametrized: yes |
| 1150 |
:setup: Standalone |
| 1151 |
:steps: |
| 1152 |
1. Try to set invalid configuration values |
| 1153 |
:expectedresults: |
| 1154 |
1. Configuration change should fail |
| 1155 |
""" |
| 1156 |
log.info(f"Starting test_negagtive_parameterize for {test_input}={expected}") |
| 1157 |
inst = topo.standalone |
| 1158 |
|
| 1159 |
log.info(f"Attempting to set invalid value: {test_input}={expected}") |
| 1160 |
with pytest.raises(Exception): |
| 1161 |
inst.config.set(test_input, ensure_bytes(expected)) |
| 1162 |
|
| 1163 |
log.info("Verified: invalid configuration value was rejected") |
| 1164 |
log.info("Test completed successfully") |
| 1165 |
|
| 1166 |
|
| 1167 |
@disk_monitoring_ack |
| 1168 |
def test_valid_operations_are_permitted(topo, setup, reset_logs): |
| 1169 |
"""Verify that valid operations are permitted. |
| 1170 |
|
| 1171 |
:id: bd4f83f6-fe9e-11e8-88f4-8c16451d917b |
| 1172 |
:setup: Standalone |
| 1173 |
:steps: |
| 1174 |
1. Perform various valid configuration operations |
| 1175 |
:expectedresults: |
| 1176 |
1. All operations should succeed |
| 1177 |
""" |
| 1178 |
log.info("Starting test_valid_operations_are_permitted") |
| 1179 |
inst = topo.standalone |
| 1180 |
|
| 1181 |
log.info("Setting initial disk monitoring configuration") |
| 1182 |
inst.config.set('nsslapd-disk-monitoring', 'on') |
| 1183 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on') |
| 1184 |
inst.config.set('nsslapd-errorlog-level', '8') |
| 1185 |
inst.restart() |
| 1186 |
|
| 1187 |
log.info("Testing deletion of nsslapd-disk-monitoring-threshold") |
| 1188 |
inst.modify_s('cn=config', [(ldap.MOD_DELETE, 'nsslapd-disk-monitoring-threshold', '')]) |
| 1189 |
|
| 1190 |
log.info("Testing addition of nsslapd-disk-monitoring-threshold value") |
| 1191 |
inst.config.add('nsslapd-disk-monitoring-threshold', '2000001') |
| 1192 |
|
| 1193 |
log.info("Testing deletion of nsslapd-disk-monitoring") |
| 1194 |
config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-disk-monitoring']) |
| 1195 |
current_value = str(config_entry[0]).split(' ')[2].split('\n\n')[0] |
| 1196 |
inst.modify_s('cn=config', [(ldap.MOD_DELETE, 'nsslapd-disk-monitoring', ensure_bytes(current_value))]) |
| 1197 |
|
| 1198 |
log.info("Testing addition of nsslapd-disk-monitoring value") |
| 1199 |
inst.config.add('nsslapd-disk-monitoring', 'off') |
| 1200 |
|
| 1201 |
log.info("Testing deletion of nsslapd-disk-monitoring-grace-period") |
| 1202 |
inst.modify_s('cn=config', [(ldap.MOD_DELETE, 'nsslapd-disk-monitoring-grace-period', '')]) |
| 1203 |
|
| 1204 |
log.info("Testing addition of nsslapd-disk-monitoring-grace-period value") |
| 1205 |
inst.config.add('nsslapd-disk-monitoring-grace-period', '61') |
| 1206 |
|
| 1207 |
log.info("Testing deletion of nsslapd-disk-monitoring-logging-critical") |
| 1208 |
config_entry = inst.search_s('cn=config', ldap.SCOPE_SUBTREE, '(objectclass=*)', ['nsslapd-disk-monitoring-logging-critical']) |
| 1209 |
current_value = str(config_entry[0]).split(' ')[2].split('\n\n')[0] |
| 1210 |
inst.modify_s('cn=config', [(ldap.MOD_DELETE, 'nsslapd-disk-monitoring-logging-critical', ensure_bytes(current_value))]) |
| 1211 |
|
| 1212 |
log.info("Testing addition of nsslapd-disk-monitoring-logging-critical value") |
| 1213 |
inst.config.set('nsslapd-disk-monitoring-logging-critical', 'on') |
| 1214 |
|
| 1215 |
log.info("All valid operations completed successfully") |
| 1216 |
log.info("Test completed successfully") |
| 1217 |
|
| 1218 |
|
| 1219 |
if __name__ == '__main__': |
| 1220 |
CURRENT_FILE = os.path.realpath(__file__) |
| 1221 |
pytest.main("-s -v %s" % CURRENT_FILE) |
| 1222 |
|