#!/usr/bin/env python3 #=============================================================================== # Copyright 2015 NetApp, Inc. All Rights Reserved, # contribution by Jorge Mora # # This program is free software; you can redistribute it and/or modify it under # the terms of the GNU General Public License as published by the Free Software # Foundation; either version 2 of the License, or (at your option) any later # version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. #=============================================================================== import os import errno import struct import traceback import nfstest_config as c from nfstest.utils import * from packet.nfs.nfs4_const import * from nfstest.test_util import TestUtil from fcntl import fcntl,F_RDLCK,F_SETLK # Module constants __author__ = "Jorge Mora (%s)" % c.NFSTEST_AUTHOR_EMAIL __copyright__ = "Copyright (C) 2015 NetApp, Inc." __license__ = "GPL v2" __version__ = "1.0" USAGE = """%prog --server [options] Sparse file tests ================= Verify correct functionality of sparse files. These are files which have unallocated or uninitialized data blocks as holes. The new NFSv4.2 operation SEEK is used to search for the next hole or data segment in a file. Basic tests verify the SEEK operation returns the correct offset of the next hole or data with respect to the starting offset given to the seek system call. Verify the SEEK operation is sent to the server with the correct stateid as a READ call. All files have a virtual hole at the end of the file so when searching for the next hole, even if the file does not have a hole, it returns the size of the file. Some tests include testing at the protocol level by taking a packet trace and inspecting the actual packets sent to the server. Negative tests include trying to SEEK starting from an offset beyond the end of the file. Examples: The only required option is --server $ %prog --server 192.168.0.11 Notes: The user id in the local host must have access to run commands as root using the 'sudo' command without the need for a password. Valid only for NFS version 4.2 and above.""" # Test script ID SCRIPT_ID = "SPARSE" SEEK_TESTS = [ "seek01", "seek02", "seek03", "seek04", ] # Include the test groups in the list of test names # so they are displayed in the help TESTNAMES = ["seek"] + SEEK_TESTS TESTGROUPS = { "seek": { "tests": SEEK_TESTS, "desc": "Run all SEEK tests: ", }, } def getlock(fd, lock_type, offset=0, length=0): """Get byte range lock on file given by file descriptor""" lockdata = struct.pack('hhllhh', lock_type, 0, offset, length, 0, 0) out = fcntl(fd, F_SETLK, lockdata) return struct.unpack('hhllhh', out) class SparseTest(TestUtil): """SparseTest object SparseTest() -> New test object Usage: x = SparseTest(testnames=['seek01', 'seek02', ...]) # Run all the tests x.run_tests() x.exit() """ def __init__(self, **kwargs): """Constructor Initialize object's private data. """ TestUtil.__init__(self, **kwargs) self.opts.version = "%prog " + __version__ # Tests are valid for NFSv4.2 and beyond self.opts.set_defaults(nfsversion=4.2) self.scan_options() # Disable createtraces option self.createtraces = False def setup(self, **kwargs): """Setup test environment""" self.umount() self.trace_start() self.mount() # Get block size for mounted volume self.statvfs = os.statvfs(self.mtdir) super(SparseTest, self).setup(**kwargs) # Sparse file definition self.sparsesize = 5 * self.filesize uargs = { "size" : self.sparsesize, "hole_list" : [self.filesize, 3*self.filesize], "hole_size" : self.filesize, "ftype" : FTYPE_SP_DEALLOC, } # Create sparse file where it starts and ends with data self.create_file(**uargs) # Create sparse file where it starts and ends with a hole uargs["hole_list"] = [0, 2*self.filesize, 4*self.filesize] self.create_file(**uargs) self.umount() self.trace_stop() def test_seek(self, fd, whence, msg=""): """Verify SEEK succeeds searching for the next data or hole fd: File descriptor for opened file whence: Search for data when using SEEK_DATA or a hole using SEEK_HOLE msg: String to identify the specific test running and it is appended to the main assertion message [default: ""] """ file_size = self.sfile.filesize if whence == SEEK_DATA: segstr = "data" what = NFS4_CONTENT_DATA # Append offset for last byte on file to test limit condition offset_list = self.sfile.data_offsets + [file_size-1] else: segstr = "hole" what = NFS4_CONTENT_HOLE offset_list = self.sfile.hole_offsets if self.sfile.endhole: # Append offset so the last byte offset is used offset_list += [file_size-1] else: # Append offset so the last byte offset is used # but expecting the implicit hole at the end of the file offset_list += [file_size] eof = 0 offset = 0 for doffset in offset_list: try: self.test_info("==== %s test %02d %s%s" % (self.testname, self.testidx, SEEKmap[whence], msg)) self.testidx += 1 self.trace_start() fmsg = "" werrno = 0 seek_offset = offset self.dprint('DBG3', "SEEK using %s on file %s starting at offset %d" % (SEEKmap[whence], self.absfile, offset)) offset = os.lseek(fd, offset, whence) self.dprint('INFO', "SEEK returned offset %d" % offset) if offset == file_size: # Hole was not found eof = 1 except OSError as werror: werrno = werror.errno fmsg = ", got error [%s] %s" % (errno.errorcode.get(werrno, werrno), os.strerror(werrno)) finally: self.trace_stop() if whence == SEEK_DATA and self.sfile.endhole and doffset == offset_list[-1]: # Looking for data starting on a hole which is the end of the file fmsg = ", expecting ENXIO but it succeeded" expr = werrno == errno.ENXIO tmsg = "SEEK should fail with ENXIO searching for the next %s when file ends in a hole" % segstr self.test(expr, tmsg+msg, failmsg=fmsg) self.set_nfserr_list(nfs4list=[NFS4ERR_NOENT, NFS4ERR_NXIO]) else: tmsg = "SEEK should succeed searching for the next %s" % segstr self.test(werrno == 0, tmsg+msg, failmsg=fmsg) if werrno == 0: fmsg = ", expecting offset %d but got %d" % (doffset, offset) if whence == SEEK_HOLE and offset == file_size: # Found the implicit hole at the end of the file tmsg = "SEEK should return the size of the file when the next hole is not found" else: tmsg = "SEEK should return correct offset when the next %s is found" % segstr self.test(offset == doffset, tmsg+msg, failmsg=fmsg) offset += self.filesize if offset >= file_size: # Use the offset exactly on the last byte of the file offset = file_size - 1 self.trace_open() (pktcall, pktreply) = self.find_nfs_op(OP_SEEK, status=None, last_call=True) self.dprint('DBG7', str(pktcall)) self.dprint('DBG7', str(pktreply)) self.test(pktcall, "SEEK should be sent to the server") if pktcall is None: return seekobj = pktcall.NFSop fmsg = ", expecting %s but got %s" % (self.stid_str(self.stateid), self.stid_str(seekobj.stateid.other)) self.test(seekobj.stateid == self.stateid, "SEEK should be sent with correct stateid", failmsg=fmsg) fmsg = ", expecting %d but got %d" % (seek_offset, seekobj.offset) self.test(seekobj.offset == seek_offset, "SEEK should be sent with correct offset", failmsg=fmsg) fmsg = ", expecting %s but got %s" % (data_content4.get(what,what), seekobj.offset) self.test(seekobj.what == what, "SEEK should be sent with %s" % seekobj.what, failmsg=fmsg) self.test(pktreply, "SEEK should be sent to the client") if pktreply is None: return if whence == SEEK_DATA and self.sfile.endhole and doffset == offset_list[-1]: fmsg = ", expecting NFS4ERR_NXIO but got %s" % pktreply.nfs.status self.test(pktreply and pktreply.nfs.status == NFS4ERR_NXIO, "SEEK should return NFS4ERR_NXIO", failmsg=fmsg) else: fmsg = ", got %s" % pktreply.nfs.status self.test(pktreply and pktreply.nfs.status == 0, "SEEK should return NFS4_OK", failmsg=fmsg) if pktreply and pktreply.nfs.status == 0: idx = pktcall.NFSidx rseekobj = pktreply.nfs.array[idx] fmsg = ", expecting %d but got %d" % (doffset, rseekobj.offset) self.test(rseekobj.offset == doffset, "SEEK should return the correct offset", failmsg=fmsg) fmsg = ", but got %s" % rseekobj.eof self.test(rseekobj.eof == eof, "SEEK should return eof as %s" % nfs_bool[eof], failmsg=fmsg) def seek01(self, whence, lock=False): """Verify SEEK succeeds searching for the next data or hole whence: Search for data when using SEEK_DATA or a hole using SEEK_HOLE lock: Lock file before seeking for the data or hole [default: False] """ for sparseidx in [0,1]: try: fd = None msg = "" if lock: msg = " (locking file)" if sparseidx == 0: self.test_info("<<<<<<<<<< Using sparse file starting and ending with data%s >>>>>>>>>>" % msg) else: self.test_info("<<<<<<<<<< Using sparse file starting and ending with hole%s >>>>>>>>>>" % msg) self.umount() self.trace_start() self.mount() self.sfile = self.sparse_files[sparseidx] self.absfile = self.sfile.absfile self.filename = self.sfile.filename self.dprint('DBG3', "Open file %s for reading" % self.absfile) fd = os.open(self.absfile, os.O_RDONLY) if lock: self.dprint('DBG3', "Lock file %s" % self.absfile) out = getlock(fd, F_RDLCK) self.trace_stop() self.trace_open() self.get_stateid(self.filename) if self.deleg_stateid is not None: self._deleg_granted = True # Search for the data/hole segments self.test_seek(fd, whence, msg) except Exception: self.test(False, traceback.format_exc()) finally: if fd: os.close(fd) self.umount() def seek01_test(self): """Verify SEEK succeeds searching for the next data""" self.test_group("Verify SEEK succeeds searching for the next data") self.testidx = 1 self._deleg_granted = False self.seek01(SEEK_DATA) if not self._deleg_granted: # Run tests with byte range locking self.seek01(SEEK_DATA, lock=True) def seek02_test(self): """Verify SEEK succeeds searching for the next hole""" self.test_group("Verify SEEK succeeds searching for the next hole") self.testidx = 1 self._deleg_granted = False self.seek01(SEEK_HOLE) if not self._deleg_granted: # Run tests with byte range locking self.seek01(SEEK_HOLE, lock=True) def seek03(self, whence, offset, sparseidx=0, lock=False, msg=""): """Verify SEEK fails with ENXIO when offset is beyond the end of the file whence: Search for data when using SEEK_DATA or a hole using SEEK_HOLE offset: Search for data or hole starting from this offset sparseidx: Index of the sparse file to use for the testing [default: 0] lock: Lock file before seeking for the data or hole [default: False] msg: String to identify the specific test running and it is appended to the main assertion message [default: ""] """ try: fd = None self.test_info("==== %s test %02d %s%s" % (self.testname, self.testidx, SEEKmap[whence], msg)) self.testidx += 1 self.umount() self.trace_start() self.mount() # Use sparse file given by the index sfile = self.sparse_files[sparseidx] absfile = sfile.absfile filesize = sfile.filesize if whence == SEEK_DATA: segstr = "data segment" smsg = "using SEEK_DATA" what = NFS4_CONTENT_DATA else: segstr = "hole" smsg = "using SEEK_HOLE" what = NFS4_CONTENT_HOLE if offset < filesize: smsg += " when offset is in the middle of last hole" elif offset == filesize: smsg += " when offset equals to the file size" else: smsg += " when offset is beyond the end of the file" self.dprint('DBG3', "Open file %s for reading" % absfile) fd = os.open(absfile, os.O_RDONLY) if lock: self.dprint('DBG3', "Lock file %s" % self.absfile) out = getlock(fd, F_RDLCK) self.dprint('DBG3', "Search for the next %s on file %s starting at offset %d" % (segstr, absfile, offset)) try: werrno = 0 fmsg = ", expecting ENXIO but it succeeded" o_offset = os.lseek(fd, offset, whence) self.dprint('DBG5', "SEEK returned offset %d" % o_offset) except OSError as werror: werrno = werror.errno fmsg = ", expecting ENXIO but got %s" % errno.errorcode.get(werrno, werrno) expr = werrno == errno.ENXIO tmsg = "SEEK system call should fail with ENXIO %s" % smsg self.test(expr, tmsg+msg, failmsg=fmsg) except Exception: self.test(False, traceback.format_exc()) finally: if fd: os.close(fd) self.umount() self.trace_stop() try: self.set_nfserr_list(nfs4list=[NFS4ERR_NOENT, NFS4ERR_NXIO]) self.trace_open() self.get_stateid(sfile.filename) (pktcall, pktreply) = self.find_nfs_op(OP_SEEK, status=None) self.dprint('DBG7', str(pktcall)) self.dprint('DBG7', str(pktreply)) if pktcall: seekobj = pktcall.NFSop fmsg = ", expecting %s but got %s" % (self.stid_str(self.stateid), self.stid_str(seekobj.stateid.other)) self.test(seekobj.stateid == self.stateid, "SEEK should be sent with correct stateid", failmsg=fmsg) fmsg = ", expecting %d but got %d" % (offset, seekobj.offset) self.test(seekobj.offset == offset, "SEEK should be sent with correct offset", failmsg=fmsg) fmsg = ", expecting %s but got %s" % (data_content4.get(what,what), seekobj.offset) self.test(seekobj.what == what, "SEEK should be sent with %s" % seekobj.what, failmsg=fmsg) else: self.test(False, "SEEK packet call was not found") if not pktreply: self.test(False, "SEEK packet reply was not found") if pktcall and pktreply: idx = pktcall.NFSidx status = pktreply.nfs.array[idx].status expr = status == NFS4ERR_NXIO fmsg = ", expecting NFS4ERR_NXIO but got %s" % nfsstat4.get(status, status) tmsg = "SEEK should fail with NFS4ERR_NXIO %s" % smsg self.test(expr, tmsg+msg, failmsg=fmsg) except Exception: self.test(False, traceback.format_exc()) def seek03_test(self): """Verify SEEK searching for next data fails with ENXIO when offset is beyond the end of the file""" self.test_group("Verify SEEK searching for next data fails with ENXIO when offset is beyond the end of the file") self.testidx = 1 self.seek03(SEEK_DATA, self.sparsesize-2, 1) self.seek03(SEEK_DATA, self.sparsesize, 1) self.seek03(SEEK_DATA, 2*self.sparsesize, 1) if self.deleg_stateid is None: # Run tests with byte range locking msg = " (locking file)" self.seek03(SEEK_DATA, self.sparsesize-2, 1, lock=True, msg=msg) self.seek03(SEEK_DATA, self.sparsesize, 1, lock=True, msg=msg) self.seek03(SEEK_DATA, 2*self.sparsesize, 1, lock=True, msg=msg) def seek04_test(self): """Verify SEEK searching for next hole fails with ENXIO when offset is beyond the end of the file""" self.test_group("Verify SEEK searching for next hole fails with ENXIO when offset is beyond the end of the file") self.testidx = 1 self.seek03(SEEK_HOLE, self.sparsesize, 1) self.seek03(SEEK_HOLE, 2*self.sparsesize, 1) if self.deleg_stateid is None: # Run tests with byte range locking msg = " (locking file)" self.seek03(SEEK_HOLE, self.sparsesize, 1, lock=True, msg=msg) self.seek03(SEEK_HOLE, 2*self.sparsesize, 1, lock=True, msg=msg) ################################################################################ # Entry point x = SparseTest(usage=USAGE, testnames=TESTNAMES, testgroups=TESTGROUPS, sid=SCRIPT_ID) try: x.setup(nfiles=1) # Run all the tests x.run_tests() except Exception: x.test(False, traceback.format_exc()) finally: x.cleanup() x.exit()