Debian Codegraph @

test/nfstest_dio

raw
1 #!/usr/bin/env python3
2 #===============================================================================
3 # Copyright 2012 NetApp, Inc. All Rights Reserved,
4 # contribution by Jorge Mora <[email protected]>
5 #
6 # This program is free software; you can redistribute it and/or modify it under
7 # the terms of the GNU General Public License as published by the Free Software
8 # Foundation; either version 2 of the License, or (at your option) any later
9 # version.
10 #
11 # This program is distributed in the hope that it will be useful, but WITHOUT
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 #===============================================================================
15 import os
16 import time
17 import posix
18 import ctypes
19 import traceback
20 import nfstest_config as c
21 from formatstr import crc32
22 from packet.nfs.nfs3_const import *
23 from packet.nfs.nfs4_const import *
24 from nfstest.test_util import TestUtil
25
26 # Module constants
27 __author__ = "Jorge Mora (%s)" % c.NFSTEST_AUTHOR_EMAIL
28 __copyright__ = "Copyright (C) 2012 NetApp, Inc."
29 __license__ = "GPL v2"
30 __version__ = "1.5"
31
32 USAGE = """%prog --server <server> [options]
33
34 Direct I/O tests
35 ================
36 Functional direct I/O tests verify that every READ/WRITE is sent to the server
37 instead of the client caching the requests. Client bypasses read ahead by
38 sending the READ with only the requested bytes. Verify the client correctly
39 handles eof marker when reading the whole file. Verify client ignores delegation
40 while writing a file.
41
42 Direct I/O on pNFS tests verify the client sends the READ/WRITE to the correct
43 DS or the MDS if using a PAGESIZE aligned buffer or not, respectively.
44
45 Direct I/O data correctness tests verify that a file written with buffered I/O
46 is read correctly with direct I/O. Verify that a file written with direct I/O
47 is read correctly with buffered I/O.
48
49 Vectored I/O tests verify coalescence of multiple vectors into one READ/WRITE
50 packet when all vectors are PAGESIZE aligned. Vectors with different alignments
51 are sent on separate packets.
52
53 Valid for NFSv4.0 and NFSv4.1 including pNFS.
54
55 Examples:
56 The only required option is --server
57 $ %prog --server 192.168.0.11
58
59 Notes:
60 The user id in the local host must have access to run commands as root
61 using the 'sudo' command without the need for a password."""
62
63 # Test script ID
64 SCRIPT_ID = "DIO"
65
66 TESTNAMES = [
67 'eof',
68 'correctness',
69 'fstat',
70 'read',
71 'read_ahead',
72 'basic',
73 'rsize',
74 'wsize',
75 'aligned',
76 'nonaligned',
77 'diffalign',
78 'stripesize',
79 'vectored_io',
80 ]
81
82 # Constants
83 MDS = 0
84 DS = 1
85 SERVER = 2
86 mds_map = {
87 MDS: 'MDS',
88 DS: 'DS',
89 SERVER: 'server',
90 }
91
92 class iovec(ctypes.Structure):
93 """
94 struct iovec {
95 void *iov_base; /* Starting address */
96 size_t iov_len; /* Number of bytes to transfer */
97 };
98 """
99 _fields_ = [
100 ("iov_base", ctypes.c_void_p),
101 ("iov_len", ctypes.c_ulong),
102 ]
103
104 class DioTest(TestUtil):
105 """DioTest object
106
107 DioTest() -> New test object
108
109 Usage:
110 x = DioTest()
111
112 # Verify the client correctly handles eof marker when reading
113 # the end of the file
114 x.verify_eof()
115
116 # Verify client sends a READ request after writing when the file
117 # is open for both read and write
118 x.verify_read()
119
120 # Verify basic direct I/O functionality
121 x.verify_basic_dio(write=True)
122
123 # Vectored I/O test
124 tinfo = [
125 {'size':4096, 'aligned':True, 'server':MDS},
126 {'size':4096, 'aligned':True},
127 {'size':4096, 'aligned':True},
128 ]
129 x.vectored_io(tinfo)
130
131 x.exit()
132 """
133 def __init__(self, **kwargs):
134 """Constructor
135
136 Initialize object's private data.
137 """
138 TestUtil.__init__(self, **kwargs)
139 self.opts.version = "%prog " + __version__
140
141 # Set default script options
142 self.opts.set_defaults(filesize=262144)
143 self.opts.set_defaults(mtopts="hard,intr")
144
145 # Options specific for this test script
146 hmsg = "List of I/O types to test [default: '%default']"
147 self.test_opgroup.add_option("--iotype", default='read,write', help=hmsg)
148 hmsg = "List of buffered I/O types to test [default: '%default']"
149 self.test_opgroup.add_option("--biotype", default='none,read,write', help=hmsg)
150 hmsg = "Use delegation on tests [default: both without and with delegation]"
151 self.test_opgroup.add_option("--withdeleg", default='false,true', help=hmsg)
152 self.scan_options()
153
154 self.r_bsize = 4*self.rsize
155 self.r_mtbsize = self.rsize
156 self.w_bsize = 4*self.wsize
157 self.w_mtbsize = self.wsize
158
159 # Disable createtraces option
160 self.createtraces = False
161
162 # Flag which defines if client is using new style direct I/O where
163 # the use of non-aligned buffers result in sending the I/O to the
164 # DS instead of the MDS
165 self.newstyle = False
166
167 # Flag is True when pNFS is available
168 self.ispnfs = False
169
170 # Flags are True if delegations are available
171 self.read_deleg = False
172 self.write_deleg = False
173
174 if self.nfs_version < 3:
175 self.config("Option nfsversion must be 3 or above")
176
177 # Process --iotype option
178 self.io_list = self.get_list(self.iotype, {'read':False, 'write':True})
179 if self.io_list is None:
180 self.opts.error("invalid type given in --iotype [%s]" % self.iotype)
181
182 # Process --biotype option
183 self.bio_list = self.get_list(self.biotype, {'none':None, 'read':False, 'write':True})
184 if self.bio_list is None:
185 self.opts.error("invalid type given in --biotype [%s]" % self.biotype)
186
187 # Process --withdeleg option
188 self.deleg_list = self.get_list(self.withdeleg, {'false':False, 'true':True})
189 if self.deleg_list is None:
190 self.opts.error("invalid type given in --withdeleg [%s]" % self.withdeleg)
191
192 self.testidx = 1
193 self.fbuffers = []
194
195 # Prototypes for libc functions
196 self.libc.malloc.argtypes = [ctypes.c_long]
197 self.libc.malloc.restype = ctypes.c_void_p
198 self.libc.posix_memalign.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_long, ctypes.c_long]
199 self.libc.posix_memalign.restype = ctypes.c_int
200 self.libc.read.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_long]
201 self.libc.read.restype = ctypes.c_int
202 self.libc.write.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_long]
203 self.libc.write.restype = ctypes.c_int
204 self.libc.lseek.argtypes = [ctypes.c_int, ctypes.c_long, ctypes.c_int]
205 self.libc.lseek.restype = ctypes.c_long
206 self.libc.memcpy.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_long]
207 self.libc.memcpy.restype = ctypes.c_void_p
208 self.libc.readv.argtypes = [ctypes.c_int, ctypes.POINTER(iovec), ctypes.c_int]
209 self.libc.readv.restype = ctypes.c_ulong
210 self.libc.writev.argtypes = [ctypes.c_int, ctypes.POINTER(iovec), ctypes.c_int]
211 self.libc.writev.restype = ctypes.c_ulong
212
213 self.bsize = self.rsize if self.rsize > self.wsize else self.wsize
214
215 def _check_delegations(self):
216 """Check if delegations are granted"""
217 self.umount()
218 self.trace_start()
219 self.mount()
220 try:
221 fd = None
222 ofd = None
223 oofile = self.abspath(self.files[0])
224 self.dprint('DBG4', "Open file %s so open owner sticks around" % oofile)
225 ofd = open(oofile, 'r')
226 self.create_file()
227 rfile = self.abspath(self.files[1])
228 self.dprint('DBG2', "Open file %s for reading" % rfile)
229 fd = open(rfile, 'r')
230 fd.read(self.rsize)
231 finally:
232 if ofd:
233 ofd.close()
234 if fd:
235 fd.close()
236 self.umount()
237 self.trace_stop()
238
239 self.trace_open()
240 self.set_pktlist()
241 (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=self.files[1])
242 self.read_deleg = False if deleg_stateid is None else True
243 self.pktt.rewind()
244 (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=self.files[2])
245 self.write_deleg = False if deleg_stateid is None else True
246 if not self.read_deleg:
247 self.dprint('INFO', "READ delegations are not available -- skipping tests expecting read delegations")
248 if not self.write_deleg:
249 self.dprint('INFO', "WRITE delegations are not available -- skipping tests expecting write delegations")
250 self.pktt.close()
251
252 def setup(self, **kwargs):
253 """Setup test environment"""
254 if self.rsize != self.wsize:
255 raise Exception("CONFIG error: options rsize and wsize must have the same value")
256 elif self.rsize % self.PAGESIZE > 0:
257 raise Exception("CONFIG error: option rsize must be a multiple of %d (PAGESIZE)" % self.PAGESIZE)
258
259 self.umount()
260 self.trace_start()
261 self.mount()
262 statfs = os.statvfs(self.mtpoint)
263
264 if len(self.basename) == 0 and statfs.f_bsize < 3*self.rsize:
265 raise Exception("CONFIG error: mount options rsize and wsize must be greater than or equal to 3 times the value of option rsize")
266
267 super(DioTest, self).setup(**kwargs)
268
269 self.umount()
270 self.trace_stop()
271
272 if self.nfs_version > 3:
273 self.trace_open()
274 self.set_pktlist()
275 (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=self.files[0])
276 (layoutget, layoutget_res) = self.find_layoutget(filehandle)
277 (pktcall, pktreply, dslist) = self.find_getdeviceinfo()
278 if layoutget and len(self.dslist):
279 self.ispnfs = True
280
281 if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
282 if len(pktreply.NFSop.device_addr.versions) > 1:
283 raise Exception("Support for only one device info in NFS flex files layout type")
284 self.r_mtbsize = pktreply.NFSop.device_addr.versions[0].rsize
285 self.w_mtbsize = pktreply.NFSop.device_addr.versions[0].wsize
286 self.r_bsize = 4*self.r_mtbsize
287 self.w_bsize = 4*self.w_mtbsize
288
289 if self.ispnfs:
290 if self.layout is None:
291 raise Exception("Could not find layout")
292 self.stripe_size = self.layout['stripe_size']
293 if self.stripe_size > 0 and self.stripe_size < 3*self.rsize:
294 raise Exception("CONFIG error: option rsize must be less or equal to %d (stripe size / 3)" % int(self.stripe_size/3))
295 self.pktt.close()
296
297 # Check if delegations are granted
298 self._check_delegations()
299
300 # Non-aligned contiguous vectors on aligned offset
301 self.file_handles = {False:None, True:None}
302 tinfo = [
303 {'size':self.rsize, 'aligned':False, 'server':MDS},
304 {'size':self.rsize, 'aligned':False, 'server':MDS},
305 {'size':self.rsize, 'aligned':False, 'server':MDS},
306 ]
307 self.mount()
308 out = self.vectored_io(tinfo, write=False, contiguous=True, check=True)
309 self.umount()
310 self.newstyle = len(out) == 1 and out[0] == 3*self.rsize
311 if self.newstyle:
312 self.dprint('INFO', "Direct I/O data is sent to the DS regardless of buffer alignment")
313
314 def mem_alloc(self, size, aligned=True, fill=False, offset=0):
315 """Allocate buffer.
316
317 size:
318 Number of bytes to allocate
319 aligned:
320 Aligned buffer on a PAGESIZE boundary if true [default: True]
321 fill:
322 Fill buffer with a predetermined pattern
323 offset:
324 Offset used in creating fill data
325
326 Return allocated buffer.
327
328 See also free_buffers()
329 """
330 align_str = ""
331 buffer = None
332 if aligned:
333 # Allocate aligned buffer
334 buffer = ctypes.c_void_p()
335 self.libc.posix_memalign(ctypes.byref(buffer), self.PAGESIZE, size)
336 else:
337 # Make sure the buffer is not aligned
338 align_str = "non-"
339 buffers = []
340 for i in range(30):
341 buffers.append(ctypes.c_void_p())
342 buffers[-1].value = self.libc.malloc(size)
343 if buffers[-1].value & (self.PAGESIZE-1) != 0:
344 # Found non-aligned buffer
345 buffer = buffers.pop()
346 break
347
348 for buf in buffers:
349 # Free all unused buffers
350 self.libc.free(buf)
351
352 if buffer is None:
353 raise Exception("Could not allocate %saligned buffer" % align_str)
354
355 self.dprint('DBG3', "Allocated %saligned buffer of %d bytes @ 0x%x" % (align_str, size, buffer.value))
356 # Save allocated buffer so it can be freed by free_buffers()
357 self.fbuffers.append(buffer)
358 if fill:
359 # Fill buffer
360 data = self.data_pattern(offset, size)
361 pdata = ctypes.create_string_buffer(data)
362 self.libc.memcpy(buffer, pdata, size);
363 return buffer
364
365 def alloc_buffers(self, **kwargs):
366 """Allocate buffers used by do_read and do_write."""
367 size = kwargs.pop('size', 2*self.bsize)
368 # Make sure the buffer is not aligned
369 self.nonaligned_buffer = self.mem_alloc(size, aligned=False)
370 # Allocate aligned buffer
371 self.aligned_buffer = self.mem_alloc(size, aligned=True)
372
373 def free_buffers(self):
374 """Free all allocated buffers created by mem_alloc()."""
375 try:
376 if len(self.fbuffers):
377 while len(self.fbuffers):
378 if self.fbuffers[0] != None:
379 self.dprint('DBG4', "Freeing allocated buffer 0x%x" % self.fbuffers[0].value)
380 self.libc.free(self.fbuffers[0])
381 # Remove buffer from list so this function can be called
382 # multiple times
383 self.fbuffers.pop(0)
384 except Exception:
385 pass
386
387 def do_read(self, fd, offset, size, aligned=True, delay=None):
388 """Wrapper for system call read().
389
390 fd:
391 File descriptor returned from open() system call
392 offset:
393 Start reading at this file position
394 size:
395 Number of bytes to read
396 aligned:
397 Use aligned buffer if true [default: True]
398 delay:
399 Delay read in seconds [default: --iodelay]
400
401 Return data read.
402 """
403 buffer = self.aligned_buffer if aligned else self.nonaligned_buffer
404 self.dprint('DBG3', "Read file %d@%d" % (size, offset))
405 self.libc.lseek(fd, offset, 0)
406 count = self.libc.read(fd, buffer, size)
407 self.dprint('DBG4', "Read returned %d bytes" % count)
408 data = ctypes.string_at(buffer, count)
409 # Slow down traffic for tcpdump to capture all packets
410 self.delay_io(delay)
411 return data
412
413 def do_write(self, fd, offset, size, aligned=True, delay=None):
414 """Wrapper for system call write().
415
416 fd:
417 File descriptor returned from open() system call
418 offset:
419 Start writing at this file position
420 size:
421 Number of bytes to write
422 aligned:
423 Use aligned buffer if true [default: True]
424 delay:
425 Delay write in seconds [default: --iodelay]
426
427 Return number of bytes written.
428 """
429 buffer = self.aligned_buffer if aligned else self.nonaligned_buffer
430 data = self.data_pattern(offset, size)
431 pdata = ctypes.create_string_buffer(data)
432 self.libc.memcpy(buffer, pdata, size);
433 self.dprint('DBG3', "Write file %d@%d" % (size, offset))
434 self.libc.lseek(fd, offset, 0)
435 count = self.libc.write(fd, buffer, size)
436 self.dprint('DBG4', "Write returned %d bytes" % count)
437 # Slow down traffic for tcpdump to capture all packets
438 self.delay_io(delay)
439 return count
440
441 def _get_info(self, direct=True):
442 """Return tuple (O_DIRECT, ' (O_DIRECT)') if direct option is true,
443 (0, '') otherwise.
444 """
445 if direct:
446 info = " (O_DIRECT)"
447 open_args = posix.O_DIRECT
448 else:
449 info = ""
450 open_args = 0
451 return (open_args, info)
452
453 def read_file(self, absfile, rsize=None, direct=True, aligned=True, delay=None):
454 """Read file and compare read data with known data pattern.
455
456 absfile:
457 File to read
458 rsize:
459 Number of bytes to write per call [default: --rsize]
460 direct:
461 Open file using O_DIRECT if true [default: True]
462 aligned:
463 Use aligned buffer if true [default: True]
464 delay:
465 Delay each read in seconds [default: --iodelay]
466
467 Return total number of bytes read.
468 """
469 (open_args, info) = self._get_info(direct)
470 self.dprint('DBG2', "Open file %s for reading%s" % (absfile, info))
471 fd = posix.open(absfile, posix.O_RDONLY|open_args)
472 try:
473 offset = 0
474 rsize = self.rsize if rsize is None else rsize
475 while True:
476 data = self.do_read(fd, offset, rsize, aligned=aligned, delay=delay)
477 count = len(data)
478 cdata = self.data_pattern(offset, count)
479 if count == 0 or data != cdata:
480 break
481 offset += count
482 finally:
483 posix.close(fd)
484 return offset
485
486 def write_file(self, absfile, wsize=None, direct=True, aligned=True, delay=None):
487 """Write file with known data pattern.
488
489 absfile:
490 File to write
491 wsize:
492 Number of bytes to write per call [default: --wsize]
493 direct:
494 Open file using O_DIRECT if true [default: True]
495 aligned:
496 Use aligned buffer if true [default: True]
497 delay:
498 Delay each write in seconds [default: --iodelay]
499
500 Return total number of bytes written.
501 """
502 (open_args, info) = self._get_info(direct)
503 self.dprint('DBG2', "Open file %s for writing%s" % (absfile, info))
504 fd = posix.open(absfile, posix.O_WRONLY|posix.O_CREAT|open_args, 0o644)
505 offset = 0
506 wsize = self.wsize if wsize is None else wsize
507 while offset < self.filesize:
508 count = self.filesize - offset
509 if count > wsize:
510 count = wsize
511 count = self.do_write(fd, offset, wsize, delay=delay)
512 offset += count
513 posix.close(fd)
514 return offset
515
516 def verify_eof(self, aligned=True):
517 """Verify eof marker is handled correctly when reading the end
518 of the file.
519
520 aligned:
521 Use aligned buffer on read() if true [default: True]
522 """
523 try:
524 fd = None
525 align_str = "" if aligned else "non-"
526 self.test_group("Verify eof marker is handled correctly when reading eof using %saligned buffer" % align_str)
527
528 self.umount()
529 self.mount()
530 buffer = self.mem_alloc(2*self.rsize, aligned=aligned)
531 absfile = self.abspath(self.files[0])
532 self.dprint('DBG2', "Open file %s for reading" % absfile)
533 fd = posix.open(absfile, posix.O_RDONLY|posix.O_DIRECT)
534 offset = self.filesize - self.rsize
535 self.dprint('DBG3', "Read file %d@%d" % (self.rsize, offset))
536 self.libc.lseek(fd, offset, 0)
537 count = self.libc.read(fd, buffer, self.rsize)
538 offset += count
539 self.test(count == self.rsize, "READ right before end of file should return correct read count (%d)" % self.rsize, failmsg=", returned read count = %d" % count)
540 self.dprint('DBG3', "Read file %d@%d" % (self.rsize, offset))
541 count = self.libc.read(fd, buffer, self.rsize)
542 self.test(count == 0, "READ at end of file should return read count = 0", failmsg=", returned read count = %d" % count)
543 except Exception:
544 self.test(False, traceback.format_exc())
545 finally:
546 if fd:
547 posix.close(fd)
548 self.free_buffers()
549 self.umount()
550
551 def eof_test(self):
552 """Verify eof marker is handled correctly when reading the end
553 of the file.
554 """
555 self.verify_eof()
556 #self.verify_eof(aligned=False)
557
558 def verify_read(self, read_ahead=False, aligned=True):
559 """Verify READ is sent with only the requested bytes bypassing
560 read ahead when read_ahead is True.
561 Verify READ is sent after writing when the file is open for
562 both read and write when read_ahead is False.
563 """
564 try:
565 fd = None
566 if read_ahead:
567 self.test_group("Verify READ is sent with only the requested bytes bypassing read ahead")
568 else:
569 self.test_group("Verify READ is sent after writing when the file is open for both read and write")
570
571 self.alloc_buffers()
572 self.umount()
573 self.trace_start()
574 self.mount()
575
576 if read_ahead:
577 filename = self.files[0]
578 absfile = self.abspath(filename)
579 self.dprint('DBG2', "Open file %s for reading" % absfile)
580 fd = posix.open(absfile, posix.O_RDONLY|posix.O_DIRECT)
581 else:
582 self.get_filename()
583 filename = self.filename
584 self.dprint('DBG2', "Open file %s for writing" % self.absfile)
585 fd = posix.open(self.absfile, posix.O_RDWR|posix.O_CREAT|posix.O_DIRECT)
586
587 offset = 0
588 for i in range(3):
589 count = self.do_write(fd, offset, self.wsize, aligned=aligned)
590 offset += count
591
592 data = self.do_read(fd, 0, self.rsize, aligned=aligned)
593 self.test(data == self.data_pattern(0, self.rsize), "READ data should be correct")
594 except Exception:
595 self.test(False, traceback.format_exc())
596 return
597 finally:
598 if fd:
599 posix.close(fd)
600 self.umount()
601 self.free_buffers()
602 self.trace_stop()
603
604 try:
605 self.trace_open()
606 self.set_pktlist()
607 (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=filename)
608 stateid = deleg_stateid if deleg_stateid else open_stateid
609 self.find_layoutget(filehandle)
610 (pktcall, pktreply, dslist) = self.find_getdeviceinfo()
611
612 nfs_version = self.nfs_version
613 if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
614 item = pktreply.NFSop.device_addr.versions[0]
615 nfs_version = float("%d.%d" % (item.version, item.minorversion))
616 filehandle = self.layout['filehandles'][0]
617
618 if nfs_version < 4:
619 match_str = "NFS.argop == %d and NFS.fh == b'%s'" % (NFSPROC3_READ, self.pktt.escape(filehandle))
620 else:
621 match_str = "NFS.argop == %d and NFS.stateid.other == b'%s'" % (OP_READ, self.pktt.escape(stateid))
622
623 pkt = self.pktt.match(match_str)
624 self.test(pkt, "READ should be sent to the server")
625 if pkt:
626 roffset = pkt.NFSop.offset
627 rcount = pkt.NFSop.count
628 self.test(roffset == 0 and rcount == self.rsize, "READ should be sent with correct offset (%d) and count (%d)" % (0, self.rsize))
629 if read_ahead:
630 pkt = self.pktt.match(match_str)
631 self.test(not pkt, "Extra READs should not be sent to the server")
632 except Exception:
633 self.test(False, traceback.format_exc())
634 finally:
635 self.pktt.close()
636
637 def read_test(self):
638 """Verify READ is sent after writing when the file is open for
639 both read and write.
640 """
641 self.verify_read()
642
643 def read_ahead_test(self):
644 """Verify READ is sent with only the requested bytes bypassing
645 read ahead.
646 """
647 self.verify_read(read_ahead=True)
648
649 def correctness_test(self):
650 """Verify data correctness when reading/writing using direct I/O.
651 File created with buffered I/O is read correctly with direct I/O.
652 File created with direct I/O is read correctly with buffered I/O.
653 """
654 try:
655 self.test_group("Verify data correctness when reading/writing using direct I/O")
656 iosize = max(self.rsize, self.wsize, int(self.filesize/8))
657 self.alloc_buffers(size=iosize)
658 self.umount()
659 self.mount()
660
661 # Read file using direct I/O on a file created with buffered I/O
662 # and verify data read with known data on file
663 count = self.read_file(self.abspath(self.files[0]), rsize=iosize, delay=0)
664 self.test(count == self.filesize, "File created with buffered I/O is read correctly with direct I/O")
665
666 # Create file using direct I/O
667 self.get_filename()
668 self.write_file(self.absfile, wsize=iosize, delay=0)
669
670 self.umount()
671 self.mount()
672
673 # Verify written data by reading the file using buffered I/O
674 count = self.read_file(self.absfile, rsize=iosize, direct=False, delay=0)
675 self.test(count == self.filesize, "File created with direct I/O is read correctly with buffered I/O")
676 self.umount()
677 except Exception:
678 self.test(False, traceback.format_exc())
679 self.free_buffers()
680
681 def fstat_test(self):
682 """Verify fstat() gets correct file size after writing."""
683 try:
684 fd = None
685 self.test_group("Verify fstat() gets correct file size after writing")
686 wsize = max(self.wsize, int(self.filesize/8))
687 self.alloc_buffers(size=wsize)
688 self.umount()
689 self.mount()
690
691 self.get_filename()
692 self.dprint('DBG2', "Open file %s for writing" % self.absfile)
693 fd = posix.open(self.absfile, posix.O_WRONLY|posix.O_CREAT|posix.O_DIRECT, 0o644)
694
695 idx = 0
696 ngood = 0
697 offset = 0
698 while offset < self.filesize:
699 count = self.do_write(fd, offset, wsize, delay=0)
700 offset += count
701 idx += 1
702 fs = posix.fstat(fd)
703 if fs.st_size == offset:
704 ngood += 1
705 self.test(ngood == idx, "The fstat() should get correct file size after every write")
706
707 # Write at a large offset of 10G
708 offset = 10 * 1024 * 1024 * 1024
709 count = self.do_write(fd, offset, wsize, delay=0)
710 fs = posix.fstat(fd)
711 size = offset + count
712 self.test(fs.st_size==size, "The fstat() should get correct file size after writing at offset = 10G", failmsg="\nexpecting %d, got %d" % (size, fs.st_size))
713
714 except Exception:
715 self.test(False, traceback.format_exc())
716 finally:
717 if fd:
718 posix.close(fd)
719 self.umount()
720 self.free_buffers()
721
722 def verify_basic_dio(self, write, bsize=None, mtbsize=None, nio=None, align_hash=[], buffered_write=None, deleg=False):
723 """Verify basic direct I/O functionality.
724
725 write:
726 Test writing if true, otherwise test reading
727 bsize:
728 Block size used for I/O [default: --rsize/2]
729 mtbsize:
730 Block size to used on mount [default: not specified on mount]
731 nio:
732 Number of READ/WRITE packets each read()/write() request will
733 generate on the wire [default: calculated using bsize and mtbsize]
734 align_hash:
735 List of expected 'alignments' on pNFS, if item is True the I/O
736 is expected to go to the DS, otherwise to the MDS [default: []]
737 buffered_write:
738 Open another file for buffered I/O. If true use buffered write,
739 if false use buffered read [default: None(no buffered I/O)]
740 deleg:
741 Expect to get a delegation if true [default: False]
742 """
743 try:
744 fd = None
745 bfd = None
746 ofd = None
747 io_str = "WRITE" if write else "READ"
748 io_mode = posix.O_WRONLY|posix.O_CREAT if write else posix.O_RDONLY
749 bio_str = "WRITE" if buffered_write else "READ"
750 bio_mode = os.O_WRONLY|os.O_CREAT if buffered_write else os.O_RDONLY
751
752 if not bsize:
753 bsize = int(self.rsize/2)
754 self.alloc_buffers(size=bsize)
755
756 self.umount()
757 self.trace_start()
758 if mtbsize:
759 self.mount(mtopts="hard,intr,rsize=%d,wsize=%d" % (mtbsize, mtbsize))
760 else:
761 self.mount()
762
763 if nio is None:
764 if mtbsize:
765 nio = int(bsize / mtbsize) + (1 if bsize > mtbsize and bsize % mtbsize else 0)
766 else:
767 nio = 1
768 b_size = mtbsize if nio > 1 else bsize
769 else:
770 b_size = int(bsize / nio)
771
772 if write:
773 while len(self.files) < 4:
774 self.get_filename()
775 filename = self.files[3]
776 else:
777 filename = self.files[0]
778 absfile = self.abspath(filename)
779
780 if deleg:
781 oofile = self.abspath(self.files[2])
782 self.dprint('DBG4', "Open file %s so open owner sticks around" % oofile)
783 ofd = open(oofile, 'r')
784
785 self.dprint('DBG2', "Open file %s for %s" % (absfile, io_str))
786 fd = posix.open(absfile, io_mode|posix.O_DIRECT)
787
788 if buffered_write != None:
789 # Open file for buffered I/O
790 if buffered_write:
791 while len(self.files) < 5:
792 self.get_filename()
793 bfile = self.files[4]
794 else:
795 bfile = self.files[1]
796 babsfile = self.abspath(bfile)
797 self.dprint('DBG2', "Open file %s for %s (buffered)" % (babsfile, bio_str))
798 bfd = os.open(babsfile, bio_mode)
799
800 off = 0
801 boffset = 0
802 test_hash = []
803 N = len(align_hash) if len(align_hash) else 3
804 for i in range(N):
805 b_off = off
806 if len(align_hash):
807 aligned = align_hash[i]
808 else:
809 aligned = True
810 for j in range(nio):
811 test_hash.append({'offset':b_off, 'size':b_size, 'aligned':aligned, 'grpidx':i})
812 b_off += b_size
813 if write:
814 count = self.do_write(fd, off, bsize, aligned=aligned)
815 else:
816 data = self.do_read(fd, off, bsize, aligned=aligned)
817 if buffered_write != None:
818 # Buffered I/O
819 if buffered_write:
820 self.dprint('DBG3', "Write file %d@%d (buffered)" % (bsize, boffset))
821 count = os.write(bfd, self.data_pattern(boffset, bsize))
822 self.dprint('DBG4', "Write returned %d bytes" % count)
823 else:
824 self.dprint('DBG3', "Read file %d@%d (buffered)" % (bsize, boffset))
825 data = os.read(bfd, bsize)
826 self.dprint('DBG4', "Read returned %d bytes" % len(data))
827 boffset += bsize
828 # Slow down traffic for tcpdump to capture all packets
829 self.delay_io()
830 off += 2*bsize
831 except Exception:
832 self.test(False, traceback.format_exc())
833 finally:
834 if fd:
835 posix.close(fd)
836 if bfd:
837 os.close(bfd)
838 if deleg:
839 ofd.close()
840 self.umount()
841 self.free_buffers()
842 self.trace_stop()
843
844 try:
845 self.trace_open()
846 self.set_pktlist()
847 (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=filename)
848 save_index = self.pktt.get_index()
849 if deleg_stateid != None:
850 self.dprint('DBG3', "Delegation is granted")
851 stateid = deleg_stateid if deleg_stateid else open_stateid
852 self.find_layoutget(filehandle)
853 (pktcall, pktreply, dslist) = self.find_getdeviceinfo()
854 self.pktt.rewind(save_index)
855
856 nfs_version = self.nfs_version
857 if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
858 item = pktreply.NFSop.device_addr.versions[0]
859 nfs_version = float("%d.%d" % (item.version, item.minorversion))
860 filehandle = self.layout['filehandles'][0]
861
862 if nfs_version < 4:
863 io_op = NFSPROC3_WRITE if write else NFSPROC3_READ
864 else:
865 io_op = OP_WRITE if write else OP_READ
866
867 match_str = "NFS.argop == %d" % io_op
868 if nfs_version < 4:
869 match_str += " and NFS.fh == b'%s'" % self.pktt.escape(filehandle)
870 else:
871 match_str += " and NFS.stateid.other == b'%s'" % self.pktt.escape(stateid)
872
873 idx = 0
874 xids = {}
875 io_h = {}
876 err_h = {}
877 while self.pktt.match(match_str):
878 save_index = self.pktt.get_index()
879 pkt = self.pktt.pkt
880 roffset = pkt.NFSop.offset
881 ipaddr = pkt.ip.dst
882 pobj = pkt.udp if pkt.tcp is None else pkt.tcp
883 port = pobj.dst_port
884 xid = pkt.rpc.xid
885 pktr = self.pktt.match("RPC.type == 1 and RPC.xid == %d" % xid)
886 self.pktt.rewind(save_index)
887 if xids.get(xid, None) is None:
888 # Save xid to keep track of re-transmitted packets
889 xids[xid] = 1
890 else:
891 # Skip re-transmitted packets
892 continue
893 if pktr == "nfs" and pktr.nfs.status != NFS4_OK:
894 # Server returned error for this I/O operation
895 errstr = nfsstat4.get(pktr.nfs.status)
896 if err_h.get(errstr) is None:
897 err_h[errstr] = 1
898 else:
899 err_h[errstr] += 1
900
901 # Get size of I/O data
902 rsize = pkt.NFSop.count
903
904 if io_h.get(roffset) == rsize:
905 # This (offset, size) has already been processed
906 continue
907 else:
908 io_h[roffset] = rsize
909
910 if len(test_hash) <= idx:
911 # Got unexpected READ/WRITE
912 self.test(0, "%s (%d@%d) should not be sent" % (io_str, rsize, roffset))
913 else:
914 # Check READ/WRITE for expected offset and size
915 toffset = test_hash[idx].get('offset')
916 tsize = test_hash[idx].get('size')
917 taligned = test_hash[idx].get('aligned')
918 tgrpidx = test_hash[idx].get('grpidx')
919 expr = (toffset == roffset and tsize == rsize)
920 if not expr and ((mtbsize and bsize > mtbsize) or nio > 1):
921 # The I/O size is greater than the mount block size
922 # check if this write belongs to the same I/O group
923 for item in test_hash:
924 if item.get('grpidx') == tgrpidx:
925 expr = (item.get('offset') == roffset and item.get('size') == rsize)
926 if expr:
927 # I/O belongs to the expected I/O group
928 toffset = item.get('offset')
929 tsize = item.get('size')
930 taligned = item.get('aligned')
931 self.dprint('DBG4', "%s (%d@%d) is sent out of order but within the same I/O call" % (io_str, tsize, toffset))
932 break
933 if len(align_hash) and (self.newstyle or taligned) and self.ispnfs:
934 dsidx = 0
935 ds_index = None
936 for ds in self.dslist:
937 for item in ds:
938 if ipaddr == item['ipaddr'] and port == item['port']:
939 ds_index = dsidx
940 break
941 dsidx += 1
942 if ds_index is None:
943 # XXX this could be a mirror -- mirrors are not supported yet
944 continue
945
946 msg = "%s (%d@%d) should be sent with correct offset and count" % (io_str, tsize, toffset)
947 fmsg = ", got packet (%d@%d)" %(rsize, roffset)
948 self.test(expr, msg, failmsg=fmsg)
949
950 if len(align_hash):
951 if (self.newstyle or taligned) and self.ispnfs:
952 out = self.verify_stripe(toffset, tsize, ds_index)
953 self.test(out, "%s should be sent to the correct DS%s" % (io_str, "" if ds_index is None else "(%d)"%ds_index))
954 else:
955 rserver = MDS if self.ispnfs else SERVER
956 expr = ipaddr == self.server_ipaddr
957 if self.proto in ("tcp", "udp"):
958 expr = expr and port == self.port
959 self.test(expr, "%s should be sent to the %s" % (io_str, mds_map[rserver]))
960 idx += 1
961 for err in err_h:
962 self.test(False, "%s fails with %s, number of failures found: %d" % (io_str, err, err_h[err]))
963
964 for item in test_hash[idx:]:
965 # Check for expected READ/WRITE packets which were not found
966 toffset = item.get('offset')
967 tsize = item.get('size')
968 self.test(0, "%s (%d@%d) should be sent" % (io_str, tsize, toffset))
969
970 if buffered_write != None:
971 self.pktt.rewind()
972 (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=bfile)
973 stateid = deleg_stateid if deleg_stateid else open_stateid
974 self.find_layoutget(filehandle)
975 (pktcall, pktreply, dslist) = self.find_getdeviceinfo()
976 total_size = 0
977 rmatch = True
978 xids = {}
979
980 nfs_version = self.nfs_version
981 if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
982 item = pktreply.NFSop.device_addr.versions[0]
983 nfs_version = float("%d.%d" % (item.version, item.minorversion))
984 filehandle = self.layout['filehandles'][0]
985
986 if nfs_version < 4:
987 bio_op = NFSPROC3_WRITE if buffered_write else NFSPROC3_READ
988 else:
989 bio_op = OP_WRITE if buffered_write else OP_READ
990
991 if nfs_version < 4:
992 match_str = "NFS.argop == %d and NFS.fh == b'%s'" % (bio_op, self.pktt.escape(filehandle))
993 else:
994 match_str = "NFS.argop == %d and NFS.stateid.other == b'%s'" % (bio_op, self.pktt.escape(stateid))
995
996 while self.pktt.match(match_str):
997 pkt = self.pktt.pkt
998 roffset = pkt.NFSop.offset
999 xid = pkt.rpc.xid
1000 if xids.get(xid, None) is None:
1001 # Save xid to keep track of re-transmitted packets
1002 xids[xid] = 1
1003 else:
1004 # Skip re-transmitted packets
1005 continue
1006 rsize = pkt.NFSop.count
1007 if rsize != bsize:
1008 rmatch = False
1009 total_size += rsize
1010 expr = (buffered_write and total_size == boffset) or (total_size >= boffset)
1011 msg = "%ss should be sent with correct size for buffered I/O" % bio_str
1012 fmsg = "; expecting %d, got %d" % (boffset, total_size)
1013 #XXX
1014 #self.test(expr, msg, failmsg=fmsg)
1015 self.test(not rmatch, "%ss should be cached for buffered I/O" % bio_str)
1016 except Exception:
1017 self.test(False, traceback.format_exc())
1018 finally:
1019 self.pktt.close()
1020
1021 def basic_dio(self, testname):
1022 """Verify basic direct I/O functionality for given testname."""
1023 try:
1024 no_striping = self.layout is not None and self.layout.get('stripe_size') == 0
1025
1026 for write in self.io_list:
1027 for deleg in self.deleg_list:
1028 if deleg and (write and not self.write_deleg or not write and not self.read_deleg):
1029 # Skip delegation testing for I/O since delegation
1030 # was not granted for this particular I/O
1031 continue
1032 for bwrite in self.bio_list:
1033 io_str = "WRITE" if write else "READ"
1034 bio_str = "WRITE" if bwrite else "READ"
1035 size_str = "wsize" if write else "rsize"
1036 wr_str = "writing from" if write else "reading into"
1037 dstr = " with deleg" if deleg else ""
1038 bstr = "" if bwrite is None else " and having %s buffered I/O to another file" % bio_str
1039 if self.ispnfs:
1040 ds_str = "correct %s" % mds_map[DS]
1041 if self.newstyle:
1042 mds_str = ds_str
1043 else:
1044 mds_str = mds_map[MDS]
1045 both_str = "both %s and correct %s" % (mds_map[MDS], mds_map[DS])
1046 else:
1047 ds_str = mds_map[SERVER]
1048 mds_str = mds_map[SERVER]
1049 both_str = mds_map[SERVER]
1050
1051 if testname == "basic":
1052 self.test_group("Verify %s packet is sent for each %s%s%s" % (io_str, io_str.lower(), dstr, bstr))
1053 self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg)
1054 elif testname == "rsize" and not write:
1055 self.test_group("Verify multiple %s packets are sent for each %s having request size > %s%s%s" % (io_str, io_str.lower(), size_str, dstr, bstr))
1056 self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.r_bsize, mtbsize=self.r_mtbsize)
1057 elif testname == "wsize" and write:
1058 self.test_group("Verify multiple %s packets are sent for each %s having request size > %s%s%s" % (io_str, io_str.lower(), size_str, dstr, bstr))
1059 self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.w_bsize, mtbsize=self.w_mtbsize)
1060 elif testname == "aligned":
1061 self.test_group("Verify %s is sent to %s when PAGESIZE aligned%s%s" % (io_str, ds_str, dstr, bstr))
1062 self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.rsize, align_hash=[True, True, True, True])
1063 elif testname == "nonaligned":
1064 self.test_group("Verify %s is sent to %s when not PAGESIZE aligned%s%s" % (io_str, mds_str, dstr, bstr))
1065 ahash = [True, True, True, True] if no_striping else [False, False, False, False]
1066 self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.rsize, align_hash=ahash)
1067 elif testname == "diffalign":
1068 self.test_group("Verify %ss are sent to %s on same open using buffers with different alignments%s%s" % (io_str, both_str, dstr, bstr))
1069 ahash = [True, True, True, True] if no_striping else [True, False, True, False]
1070 self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=self.rsize, align_hash=ahash)
1071 elif testname == "stripesize" and self.ispnfs and not no_striping:
1072 self.test_group("Verify multiple %s packets are sent for each %s having request size > stripe size%s%s" % (io_str, io_str.lower(), dstr, bstr))
1073 self.verify_basic_dio(write=write, buffered_write=bwrite, deleg=deleg, bsize=2*self.stripe_size, mtbsize=4*self.stripe_size, nio=2, align_hash=[True, True])
1074 except Exception:
1075 self.test(False, traceback.format_exc())
1076
1077 def basic_test(self):
1078 """Verify a packet is sent for each I/O request."""
1079 self.basic_dio('basic')
1080
1081 def rsize_test(self):
1082 """Verify multiple READ packets are sent for each read request
1083 having request size > rsize.
1084 """
1085 self.basic_dio('rsize')
1086
1087 def wsize_test(self):
1088 """Verify multiple WRITE packets are sent for each write request
1089 having request size > wsize
1090 """
1091 self.basic_dio('wsize')
1092
1093 def aligned_test(self):
1094 """Verify packet is sent to correct DS server when using a memory
1095 which is PAGESIZE aligned.
1096 """
1097 self.basic_dio('aligned')
1098
1099 def nonaligned_test(self):
1100 """Verify packet is sent to the MDS when using a memory which
1101 is not PAGESIZE aligned.
1102 """
1103 self.basic_dio('nonaligned')
1104
1105 def diffalign_test(self):
1106 """Verify packets are sent to both the MDS and correct DS on same open
1107 using buffers with different alignments.
1108 """
1109 self.basic_dio('diffalign')
1110
1111 def stripesize_test(self):
1112 """Verify multiple packets are sent for each request having the
1113 request size greater than stripe size.
1114 """
1115 self.basic_dio('stripesize')
1116
1117 def vectored_io(self, tinfo, offset=0, contiguous=False, write=False, check=False):
1118 """Verify vectored I/O functionality.
1119
1120 tinfo:
1121 List of vector definitions and expected results. Each vector
1122 definition is a dictionary having the following keys:
1123 size: size of vector
1124 aligned: buffer alignment of vector
1125 server: expected server where the vector data is going to
1126 if not specified, this vector data is part of the
1127 previous packet
1128 offset:
1129 Read/write at given offset [default: 0]
1130 contiguous:
1131 Vectors are contiguous if true [default: False]
1132 write:
1133 Test writing if true, otherwise test reading [default: False]
1134 check:
1135 Run the test but don't PASS/FAIL it, just return a list of
1136 all I/O sizes sent to the server [default: False]
1137 """
1138 try:
1139 # Generate test hash
1140 fd = None
1141 off = offset
1142 head_vec = []
1143 test_hash = []
1144 total_size = 0
1145
1146 if self.newstyle and contiguous:
1147 tlist = [item for item in tinfo if item["aligned"]]
1148 if len(tlist) == 0 or len(tlist) == len(tinfo):
1149 # All vectors have the same alignment
1150 tinfo[0]["server"] = DS
1151 for item in tinfo[1:]:
1152 item.pop("server", None)
1153
1154 for item in tinfo:
1155 server = item.get('server')
1156 if server != None:
1157 if self.newstyle and server == MDS:
1158 server = DS
1159 test_hash.append({'server':server, 'size':item['size'], 'offset':off})
1160 else:
1161 test_hash[-1]['size'] += item['size']
1162 align_str = "" if item['aligned'] else "non-"
1163 if not contiguous or len(head_vec) == 0:
1164 head_vec.append("%saligned(%d)" % (align_str, item['size']))
1165 else:
1166 head_vec.append("(%d)" % item['size'])
1167 total_size += item['size']
1168 off += item['size']
1169 if not self.ispnfs:
1170 for item in test_hash:
1171 item['server'] = 2
1172
1173 con_str = "" if contiguous else "non-"
1174 off_str = "@%d " % offset if offset else ""
1175 io_str = "WRITE" if write else "READ"
1176 io_mode = posix.O_WRONLY|posix.O_CREAT if write else posix.O_RDONLY
1177
1178 if not check:
1179 self.test_group("Verify %s %s%scontiguous vector [%s]" % (io_str, off_str, con_str, ", ".join(head_vec)))
1180 sub_testname = "%s_%03d" % (self.testname, self.testidx)
1181 self.dprint("INFO", "Running %s" % sub_testname)
1182 self.testidx += 1
1183
1184 idx = 0
1185 off = offset
1186 buffers = []
1187 nvecs = len(tinfo)
1188 vec_str = "IOvecs("
1189 for item in tinfo:
1190 if contiguous:
1191 if idx == 0:
1192 buffers.append(self.mem_alloc(total_size, aligned=item['aligned'], fill=write, offset=offset))
1193 else:
1194 buffer = ctypes.c_void_p()
1195 buffer.value = buffers[idx-1].value + tinfo[idx-1]['size']
1196 buffers.append(buffer)
1197 else:
1198 buffers.append(self.mem_alloc(item['size'], aligned=item['aligned'], fill=write, offset=off))
1199 off += item['size']
1200 vec_str += "iovec(buffers[%d], %d)," % (idx, item['size'])
1201 idx += 1
1202 vec_str += ")"
1203
1204 # Create array of iovec structures
1205 IOvecs = iovec * nvecs
1206 vectors = eval(vec_str)
1207
1208 self.trace_start()
1209 if write:
1210 while len(self.files) < 4:
1211 self.get_filename()
1212 filename = self.files[3]
1213 else:
1214 filename = self.files[0]
1215 absfile = self.abspath(filename)
1216 self.dprint('DBG2', "Open file %s for %s" % (absfile, io_str))
1217 fd = posix.open(absfile, io_mode|posix.O_DIRECT)
1218 self.libc.lseek(fd, offset, 0)
1219 wr_str = "Writing" if write else "Reading"
1220 fi_str = "to" if write else "from"
1221 self.dprint('DBG3', "%s %d vectors %s file %s " % (wr_str, nvecs, fi_str, absfile))
1222 if write:
1223 count = self.libc.writev(fd, vectors, nvecs)
1224 else:
1225 count = self.libc.readv(fd, vectors, nvecs)
1226 # Slow down traffic for tcpdump to capture all packets
1227 self.delay_io()
1228 except Exception:
1229 self.test(False, traceback.format_exc())
1230 finally:
1231 if fd:
1232 self.dprint('DBG3', "Close file %s " % absfile)
1233 posix.close(fd)
1234 self.trace_stop()
1235
1236 try:
1237 fd = None
1238 self.trace_open()
1239 self.set_pktlist()
1240 nfs_version = self.nfs_version
1241 (filehandle, open_stateid, deleg_stateid) = self.find_open(filename=filename, claimfh=self.file_handles[write], anyclaim=True)
1242 if filehandle:
1243 self.file_handles[write] = filehandle
1244 save_index = self.pktt.get_index()
1245 self.stateid = deleg_stateid if deleg_stateid else open_stateid
1246 self.find_layoutget(filehandle)
1247 (pktcall, pktreply, dslist) = self.find_getdeviceinfo()
1248
1249 if pktreply and pktreply.NFSop.device_addr.type == LAYOUT4_FLEX_FILES:
1250 item = pktreply.NFSop.device_addr.versions[0]
1251 nfs_version = float("%d.%d" % (item.version, item.minorversion))
1252 filehandle = self.layout['filehandles'][0]
1253
1254 self.pktt.rewind(save_index)
1255 elif not check:
1256 if self.pktcall and not self.pktreply:
1257 copstr = "OPEN"
1258 if nfs_version < 4:
1259 copstr = str(self.pktcall.nfs.op)[9:]
1260 self.warning("Could not find %s reply for file" % copstr)
1261 elif nfs_version >= 4 and not self.stateid and not self.pktcall:
1262 self.warning("Could not find OPEN call for file")
1263
1264 dsismds = False
1265 for ds in self.dslist:
1266 for item in ds:
1267 if self.server_ipaddr == item['ipaddr'] and self.port == item['port']:
1268 dsismds = True
1269 break
1270
1271 if nfs_version < 4:
1272 io_op = NFSPROC3_WRITE if write else NFSPROC3_READ
1273 else:
1274 io_op = OP_WRITE if write else OP_READ
1275
1276 match_str = "NFS.argop == %d" % io_op
1277 if nfs_version < 4 and filehandle:
1278 match_str += " and crc32(NFS.fh) == 0x%08x" % crc32(filehandle)
1279 elif filehandle:
1280 # There is a valid state id
1281 match_str += " and crc32(NFS.stateid.other) == 0x%08x" % crc32(self.stateid)
1282
1283 xids = {}
1284 ret = []
1285 received = []
1286 pkt_hash = {}
1287 while self.pktt.match(match_str):
1288 pkt = self.pktt.pkt
1289 ipaddr = pkt.ip.dst
1290 pobj = pkt.udp if pkt.tcp is None else pkt.tcp
1291 port = pobj.dst_port
1292 xid = pkt.rpc.xid
1293 if xids.get(xid, None) is None:
1294 # Save xid to keep track of re-transmitted packets
1295 xids[xid] = 1
1296 else:
1297 # Skip re-transmitted packets
1298 continue
1299 rsize = pkt.NFSop.count
1300 if self.ispnfs:
1301 rserver = MDS if (ipaddr == self.server_ipaddr and port == self.port) else DS
1302 else:
1303 rserver = SERVER
1304 off = pkt.NFSop.offset
1305 pkt_hash[off] = {'server':rserver, 'size':rsize}
1306 received.append("%s(%d)" % (mds_map[rserver], rsize))
1307 ret.append(rsize)
1308
1309 if check:
1310 return ret
1311
1312 dsmsg = " where DS == MDS" if dsismds else ""
1313 self.dprint('INFO', "Client sent [%s]%s" % (", ".join(received), dsmsg))
1314
1315 for item in test_hash:
1316 server = item.get('server')
1317 size = item.get('size')
1318 pktinfo = pkt_hash.pop(item['offset'], None)
1319 if pktinfo is None:
1320 self.test(False, "%s should be sent to the %s with size %d" % (io_str, mds_map[server], size))
1321 else:
1322 rserver = pktinfo.get('server')
1323 rsize = pktinfo.get('size')
1324 srvtest = dsismds or server == rserver
1325 srvrmsg = " not the %s" % mds_map[rserver] if not srvtest else ""
1326 sizemsg = " not %d" % rsize if size != rsize else ""
1327 self.test(srvtest and size == rsize, "%s should be sent to the %s%s with offset %d and size %d%s" % \
1328 (io_str, mds_map[server], srvrmsg, item['offset'], size, sizemsg))
1329 for off in pkt_hash:
1330 rserver = pkt_hash[off].get('server')
1331 rsize = pkt_hash[off].get('size')
1332 self.test(False, "%s should not be sent to the %s with offset %d and size %d" % \
1333 (io_str, mds_map[rserver], off, rsize))
1334
1335 # Check data on all vectors
1336 idx = 0
1337 for item in tinfo:
1338 size = item['size']
1339 data = ctypes.string_at(buffers[idx], size)
1340 if write:
1341 if fd is None:
1342 fd = open(absfile, 'rb')
1343 fd.seek(offset)
1344 rdata = fd.read(size)
1345 self.test(data == rdata, "WRITE vector(%d) data should be correct" % size)
1346 else:
1347 self.test(data == self.data_pattern(offset, size), "READ vector(%d) data should be correct" % size)
1348 offset += size
1349 idx += 1
1350 except Exception:
1351 self.test(False, traceback.format_exc())
1352 finally:
1353 if fd:
1354 fd.close()
1355 self.free_buffers()
1356 self.pktt.close()
1357
1358 def _vectored_io_test(self):
1359 """Verify vectored I/O functionality."""
1360 self.testidx = 1
1361 hsize = int(self.PAGESIZE/2)
1362 for write in self.io_list:
1363 #=======================================================================
1364 # Non-contiguous vectors on aligned offset
1365 tinfo = [
1366 {'size':self.rsize, 'aligned':False, 'server':MDS},
1367 {'size':self.rsize, 'aligned':False, 'server':MDS},
1368 {'size':self.rsize, 'aligned':False, 'server':MDS},
1369 ]
1370 self.vectored_io(tinfo, write=write)
1371
1372 tinfo = [
1373 {'size':self.rsize, 'aligned':True, 'server':DS},
1374 {'size':self.rsize, 'aligned':False, 'server':MDS},
1375 {'size':self.rsize, 'aligned':False, 'server':MDS},
1376 ]
1377 self.vectored_io(tinfo, write=write)
1378
1379 tinfo = [
1380 {'size':self.rsize, 'aligned':False, 'server':MDS},
1381 {'size':self.rsize, 'aligned':True, 'server':MDS},
1382 {'size':self.rsize, 'aligned':False, 'server':MDS},
1383 ]
1384 self.vectored_io(tinfo, write=write)
1385
1386 tinfo = [
1387 {'size':self.rsize, 'aligned':True, 'server':DS},
1388 {'size':self.rsize, 'aligned':True},
1389 {'size':self.rsize, 'aligned':False, 'server':MDS},
1390 ]
1391 self.vectored_io(tinfo, write=write)
1392
1393 tinfo = [
1394 {'size':self.rsize, 'aligned':False, 'server':MDS},
1395 {'size':self.rsize, 'aligned':False, 'server':MDS},
1396 {'size':self.rsize, 'aligned':True, 'server':MDS},
1397 ]
1398 self.vectored_io(tinfo, write=write)
1399
1400 tinfo = [
1401 {'size':self.rsize, 'aligned':True, 'server':DS},
1402 {'size':self.rsize, 'aligned':False, 'server':MDS},
1403 {'size':self.rsize, 'aligned':True, 'server':MDS},
1404 ]
1405 self.vectored_io(tinfo, write=write)
1406
1407 tinfo = [
1408 {'size':self.rsize, 'aligned':False, 'server':MDS},
1409 {'size':self.rsize, 'aligned':True, 'server':MDS},
1410 {'size':self.rsize, 'aligned':True},
1411 ]
1412 self.vectored_io(tinfo, write=write)
1413
1414 tinfo = [
1415 {'size':self.rsize, 'aligned':True, 'server':DS},
1416 {'size':self.rsize, 'aligned':True},
1417 {'size':self.rsize, 'aligned':True},
1418 ]
1419 self.vectored_io(tinfo, write=write)
1420
1421 #=======================================================================
1422 # Non-aligned contiguous vectors on aligned offset
1423 tinfo = [
1424 {'size':self.rsize, 'aligned':False, 'server':MDS},
1425 {'size':self.rsize, 'aligned':False, 'server':MDS},
1426 {'size':self.rsize, 'aligned':False, 'server':MDS},
1427 ]
1428 self.vectored_io(tinfo, write=write, contiguous=True)
1429
1430 #=======================================================================
1431 # Non-aligned contiguous vectors on aligned offset with various sizes
1432 tinfo = [
1433 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1434 {'size':self.rsize, 'aligned':False, 'server':MDS},
1435 {'size':self.rsize, 'aligned':False, 'server':MDS},
1436 ]
1437 self.vectored_io(tinfo, write=write, contiguous=True)
1438
1439 tinfo = [
1440 {'size':self.rsize, 'aligned':False, 'server':MDS},
1441 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1442 {'size':self.rsize, 'aligned':False, 'server':MDS},
1443 ]
1444 self.vectored_io(tinfo, write=write, contiguous=True)
1445
1446 tinfo = [
1447 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1448 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1449 {'size':self.rsize, 'aligned':False, 'server':MDS},
1450 ]
1451 self.vectored_io(tinfo, write=write, contiguous=True)
1452
1453 tinfo = [
1454 {'size':self.rsize, 'aligned':False, 'server':MDS},
1455 {'size':self.rsize, 'aligned':False, 'server':MDS},
1456 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1457 ]
1458 self.vectored_io(tinfo, write=write, contiguous=True)
1459
1460 tinfo = [
1461 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1462 {'size':self.rsize, 'aligned':False, 'server':MDS},
1463 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1464 ]
1465 self.vectored_io(tinfo, write=write, contiguous=True)
1466
1467 tinfo = [
1468 {'size':self.rsize, 'aligned':False, 'server':MDS},
1469 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1470 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1471 ]
1472 self.vectored_io(tinfo, write=write, contiguous=True)
1473
1474 tinfo = [
1475 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1476 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1477 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1478 ]
1479 self.vectored_io(tinfo, write=write, contiguous=True)
1480
1481 #=======================================================================
1482 # Aligned contiguous vectors on aligned offset
1483 tinfo = [
1484 {'size':self.rsize, 'aligned':True, 'server':DS},
1485 {'size':self.rsize, 'aligned':True},
1486 {'size':self.rsize, 'aligned':True},
1487 ]
1488 self.vectored_io(tinfo, write=write, contiguous=True)
1489
1490 #=======================================================================
1491 # Aligned contiguous vectors on aligned offset with various sizes
1492 tinfo = [
1493 {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
1494 {'size':self.rsize, 'aligned':True, 'server':DS},
1495 {'size':self.rsize, 'aligned':True, 'server':DS},
1496 ]
1497 self.vectored_io(tinfo, write=write, contiguous=True)
1498
1499 tinfo = [
1500 {'size':self.rsize, 'aligned':True, 'server':DS},
1501 {'size':self.rsize-hsize, 'aligned':True},
1502 {'size':self.rsize, 'aligned':True, 'server':DS},
1503 ]
1504 self.vectored_io(tinfo, write=write, contiguous=True)
1505
1506 tinfo = [
1507 {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
1508 {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
1509 {'size':self.rsize, 'aligned':True},
1510 ]
1511 self.vectored_io(tinfo, write=write, contiguous=True)
1512
1513 tinfo = [
1514 {'size':self.rsize, 'aligned':True, 'server':DS},
1515 {'size':self.rsize, 'aligned':True},
1516 {'size':self.rsize-hsize, 'aligned':True},
1517 ]
1518 self.vectored_io(tinfo, write=write, contiguous=True)
1519
1520 tinfo = [
1521 {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
1522 {'size':self.rsize, 'aligned':True, 'server':DS},
1523 {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
1524 ]
1525 self.vectored_io(tinfo, write=write, contiguous=True)
1526
1527 tinfo = [
1528 {'size':self.rsize, 'aligned':True, 'server':DS},
1529 {'size':self.rsize-hsize, 'aligned':True},
1530 {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
1531 ]
1532 self.vectored_io(tinfo, write=write, contiguous=True)
1533
1534 tinfo = [
1535 {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
1536 {'size':self.rsize-hsize, 'aligned':True, 'server':DS},
1537 {'size':self.rsize-hsize, 'aligned':True},
1538 ]
1539 self.vectored_io(tinfo, write=write, contiguous=True)
1540
1541 #=======================================================================
1542 # Non-contiguous vectors on non-aligned offset
1543 tinfo = [
1544 {'size':self.rsize, 'aligned':False, 'server':MDS},
1545 {'size':self.rsize, 'aligned':False, 'server':MDS},
1546 {'size':self.rsize, 'aligned':False, 'server':MDS},
1547 ]
1548 self.vectored_io(tinfo, write=write, offset=hsize)
1549
1550 tinfo = [
1551 {'size':self.rsize, 'aligned':True, 'server':MDS},
1552 {'size':self.rsize, 'aligned':False, 'server':MDS},
1553 {'size':self.rsize, 'aligned':False, 'server':MDS},
1554 ]
1555 self.vectored_io(tinfo, write=write, offset=hsize)
1556
1557 tinfo = [
1558 {'size':self.rsize, 'aligned':False, 'server':MDS},
1559 {'size':self.rsize, 'aligned':True, 'server':MDS},
1560 {'size':self.rsize, 'aligned':False, 'server':MDS},
1561 ]
1562 self.vectored_io(tinfo, write=write, offset=hsize)
1563
1564 tinfo = [
1565 {'size':self.rsize, 'aligned':True, 'server':MDS},
1566 {'size':self.rsize, 'aligned':True},
1567 {'size':self.rsize, 'aligned':False, 'server':MDS},
1568 ]
1569 self.vectored_io(tinfo, write=write, offset=hsize)
1570
1571 tinfo = [
1572 {'size':self.rsize, 'aligned':False, 'server':MDS},
1573 {'size':self.rsize, 'aligned':False, 'server':MDS},
1574 {'size':self.rsize, 'aligned':True, 'server':MDS},
1575 ]
1576 self.vectored_io(tinfo, write=write, offset=hsize)
1577
1578 tinfo = [
1579 {'size':self.rsize, 'aligned':True, 'server':MDS},
1580 {'size':self.rsize, 'aligned':False, 'server':MDS},
1581 {'size':self.rsize, 'aligned':True, 'server':MDS},
1582 ]
1583 self.vectored_io(tinfo, write=write, offset=hsize)
1584
1585 tinfo = [
1586 {'size':self.rsize, 'aligned':False, 'server':MDS},
1587 {'size':self.rsize, 'aligned':True, 'server':MDS},
1588 {'size':self.rsize, 'aligned':True},
1589 ]
1590 self.vectored_io(tinfo, write=write, offset=hsize)
1591
1592 tinfo = [
1593 {'size':self.rsize, 'aligned':True, 'server':MDS},
1594 {'size':self.rsize, 'aligned':True},
1595 {'size':self.rsize, 'aligned':True},
1596 ]
1597 self.vectored_io(tinfo, write=write, offset=hsize)
1598
1599 #=======================================================================
1600 # Non-contiguous vectors on non-aligned offset with various sizes
1601 tinfo = [
1602 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1603 {'size':self.rsize, 'aligned':True, 'server':MDS},
1604 {'size':self.rsize, 'aligned':True},
1605 ]
1606 self.vectored_io(tinfo, write=write, offset=hsize)
1607
1608 tinfo = [
1609 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1610 {'size':self.rsize, 'aligned':True, 'server':MDS},
1611 {'size':self.rsize, 'aligned':True},
1612 ]
1613 self.vectored_io(tinfo, write=write, offset=hsize)
1614
1615 tinfo = [
1616 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1617 {'size':self.rsize, 'aligned':False, 'server':MDS},
1618 {'size':self.rsize, 'aligned':True, 'server':MDS},
1619 ]
1620 self.vectored_io(tinfo, write=write, offset=hsize)
1621
1622 tinfo = [
1623 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1624 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1625 {'size':self.rsize, 'aligned':True, 'server':MDS},
1626 ]
1627 self.vectored_io(tinfo, write=write, offset=hsize)
1628
1629 #=======================================================================
1630 # Non-aligned contiguous vectors on non-aligned offset
1631 tinfo = [
1632 {'size':self.rsize, 'aligned':False, 'server':MDS},
1633 {'size':self.rsize, 'aligned':False, 'server':MDS},
1634 {'size':self.rsize, 'aligned':False, 'server':MDS},
1635 ]
1636 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1637
1638 #=======================================================================
1639 # Non-aligned contiguous vectors on non-aligned offset with various sizes
1640 tinfo = [
1641 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1642 {'size':self.rsize, 'aligned':False, 'server':MDS},
1643 {'size':self.rsize, 'aligned':False, 'server':MDS},
1644 ]
1645 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1646
1647 tinfo = [
1648 {'size':self.rsize, 'aligned':False, 'server':MDS},
1649 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1650 {'size':self.rsize, 'aligned':False, 'server':MDS},
1651 ]
1652 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1653
1654 tinfo = [
1655 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1656 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1657 {'size':self.rsize, 'aligned':False, 'server':MDS},
1658 ]
1659 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1660
1661 tinfo = [
1662 {'size':self.rsize, 'aligned':False, 'server':MDS},
1663 {'size':self.rsize, 'aligned':False, 'server':MDS},
1664 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1665 ]
1666 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1667
1668 tinfo = [
1669 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1670 {'size':self.rsize, 'aligned':False, 'server':MDS},
1671 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1672 ]
1673 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1674
1675 tinfo = [
1676 {'size':self.rsize, 'aligned':False, 'server':MDS},
1677 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1678 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1679 ]
1680 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1681
1682 tinfo = [
1683 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1684 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1685 {'size':self.rsize-hsize, 'aligned':False, 'server':MDS},
1686 ]
1687 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1688
1689 #=======================================================================
1690 # Aligned contiguous vectors on non-aligned offset
1691 tinfo = [
1692 {'size':self.rsize, 'aligned':True, 'server':MDS},
1693 {'size':self.rsize, 'aligned':True},
1694 {'size':self.rsize, 'aligned':True},
1695 ]
1696 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1697
1698 #=======================================================================
1699 # Aligned contiguous vectors on non-aligned offset with various sizes
1700 tinfo = [
1701 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1702 {'size':self.rsize, 'aligned':True, 'server':MDS},
1703 {'size':self.rsize, 'aligned':True, 'server':MDS},
1704 ]
1705 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1706
1707 tinfo = [
1708 {'size':self.rsize, 'aligned':True, 'server':MDS},
1709 {'size':self.rsize-hsize, 'aligned':True},
1710 {'size':self.rsize, 'aligned':True, 'server':MDS},
1711 ]
1712 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1713
1714 tinfo = [
1715 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1716 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1717 {'size':self.rsize, 'aligned':True},
1718 ]
1719 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1720
1721 tinfo = [
1722 {'size':self.rsize, 'aligned':True, 'server':MDS},
1723 {'size':self.rsize, 'aligned':True},
1724 {'size':self.rsize-hsize, 'aligned':True},
1725 ]
1726 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1727
1728 tinfo = [
1729 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1730 {'size':self.rsize, 'aligned':True, 'server':MDS},
1731 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1732 ]
1733 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1734
1735 tinfo = [
1736 {'size':self.rsize, 'aligned':True, 'server':MDS},
1737 {'size':self.rsize-hsize, 'aligned':True},
1738 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1739 ]
1740 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1741
1742 tinfo = [
1743 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1744 {'size':self.rsize-hsize, 'aligned':True, 'server':MDS},
1745 {'size':self.rsize-hsize, 'aligned':True},
1746 ]
1747 self.vectored_io(tinfo, write=write, offset=hsize, contiguous=True)
1748
1749 def vectored_io_test(self):
1750 """Verify vectored I/O functionality."""
1751 self.file_handles = {False:None, True:None}
1752 self.umount()
1753 try:
1754 self.mount()
1755 self._vectored_io_test()
1756 finally:
1757 self.umount()
1758
1759 ################################################################################
1760 # Entry point
1761 x = DioTest(usage=USAGE, testnames=TESTNAMES, sid=SCRIPT_ID)
1762
1763 try:
1764 x.setup(nfiles = 2)
1765
1766 # Run all the tests
1767 x.run_tests()
1768 except Exception:
1769 x.test(False, traceback.format_exc())
1770 finally:
1771 x.cleanup()
1772 x.exit()