Debian Codegraph @

test/nfstest_rdma

raw
1 #!/usr/bin/env python3
2 #===============================================================================
3 # Copyright 2020 NetApp, Inc. All Rights Reserved,
4 # contribution by Jorge Mora <[email protected]>
5 #
6 # This program is free software; you can redistribute it and/or modify it under
7 # the terms of the GNU General Public License as published by the Free Software
8 # Foundation; either version 2 of the License, or (at your option) any later
9 # version.
10 #
11 # This program is distributed in the hope that it will be useful, but WITHOUT
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
14 #===============================================================================
15 import os
16 import errno
17 import traceback
18 from formatstr import *
19 import nfstest_config as c
20 from packet.transport.ib import *
21 from packet.nfs.nfs3_const import *
22 from packet.nfs.nfs4_const import *
23 from packet.transport.rdmap import *
24 from packet.transport.ib import OpCode
25 from nfstest.test_util import TestUtil
26 from packet.application.rpcordma_const import *
27 import packet.application.rpc_const as rpc_const
28
29 # Module constants
30 __author__ = "Jorge Mora (%s)" % c.NFSTEST_AUTHOR_EMAIL
31 __copyright__ = "Copyright (C) 2020 NetApp, Inc."
32 __license__ = "GPL v2"
33 __version__ = "1.1"
34
35 USAGE = """%prog --server <server> [options]
36
37 NFS-over-RDMA functional tests
38 ==============================
39 Verify correct functionality of NFS-over-RDMA
40
41 Remote Direct Memory Access (RDMA) provides fast data transfers between
42 servers and storage. NFS over RDMA is best used when a large amount of data
43 needs to be transferred with higher performance than regular NFS. NFS over
44 RDMA is usually used over InfiniBand which provides higher performance and
45 a lower latency.
46
47 Although NFS over RDMA is mostly used over InfiniBand, Ethernet could be
48 used as the link protocol as well. RDMA over Converged Ethernet (RoCE) which
49 allows RDMA over Ethernet by encapsulating the InfiniBand transport packet
50 over Ethernet. RoCE provides a couple of variants: RoCEv1 and RoCEv2.
51 One is RoCEv1 which is an Ethernet link layer protocol and provides RDMA
52 functionality between two hosts in the same Ethernet broadcast domain.
53 While the second is RoCEv2 or RRoCE (Remote RoCE) which is an internet
54 layer protocol so these packets can be routed. RoCEv2 runs over UDP/IPv4
55 or UDP/IPv6. There is also another variant called iWARP which runs over
56 the TCP protocol. Testing is currently supported for all of these variants
57 except for iWARP.
58
59 NFS over RDMA has a couple of extra layers in the packet: InfiniBand layer
60 and RPC-over-RDMA or RPCoRDMA layer. The InfiniBand layer contains the
61 OpCode which specifies the type of RDMA operation to perform and the PSN
62 which is the packet sequence number. The RPCoRDMA layer contains the XID
63 and the RDMA chunk lists. The RDMA read chunk list is used to transfer
64 DDP (Direct Data Placement) data from the NFS client to the server, e.g.,
65 NFS write call. On the other hand, the RDMA write chunk list is used to
66 transfer DDP data from the NFS server back to the client, e.g., NFS read
67 reply. Only certain NFS operations can be transferred using DDP and only
68 the opaque part of the operation is transferred using either RDMA reads
69 or writes while the rest of the NFS packet is transferred via the receive
70 buffer using the RDMA SEND operation. Finally, the RDMA reply chunk is
71 used to transfer replies having a variable length reply which could be
72 larger than the receive buffer and could not be transferred using the
73 write chunk list because it does not contain a single large opaque item.
74
75 Tests are divided into three groups: basic, read and write. The basic tests
76 deal mostly with verifying NFS packets using the reply chunk and some other
77 basic RDMA functionality. The read tests deal with verifying NFS read which
78 in turn verify the RDMA write functionality. Finally, the write tests deal
79 with verifying NFS write which in turn verify the RDMA read functionality.
80 Also, if the NFS read or write is small enough the client could not use the
81 RDMA write or read functionality, but instead could use the receive buffer
82 and transfer the data using the RDMA SEND operations.
83
84 Tests verify the RPCoRDMA layer is sent when necessary and that the RDMA
85 chunk lists are sent with the correct information which includes the number
86 of chunks, number of segments in each chunk and the correct information for
87 each segment. Tests verify each segment information is correct and their
88 corresponding RDMA read or write information which includes correct handle,
89 virtual offset, DMA length and the XDR position for the case of RDMA reads.
90 In addition, the correct number of RDMA I/O fragments is also verified and
91 their corresponding lengths and packet sequence numbers.
92
93 Examples:
94 The only required option is --server
95 $ %prog --server 192.168.0.11
96
97 Notes:
98 The user id in the local host must have access to run commands as root
99 using the 'sudo' command without the need for a password."""
100
101 # Test script ID
102 SCRIPT_ID = "RDMA"
103
104 TESTNAMES_BASIC = ["basic01", "basic02", "basic03", "basic04", "basic05"]
105 TESTNAMES_READ = ["read01", "read02", "read03", "read04"]
106 TESTNAMES_WRITE = ["write01", "write02", "write03", "write04"]
107 TESTNAMES = ["basic"] + TESTNAMES_BASIC + \
108 ["read"] + TESTNAMES_READ + \
109 ["write"] + TESTNAMES_WRITE
110
111 TESTGROUPS = {
112 "basic": {
113 "tests": TESTNAMES_BASIC,
114 "desc": "Run all NFS-over-RDMA basic functionality tests: ",
115 },
116 "read": {
117 "tests": TESTNAMES_READ,
118 "desc": "Run all NFS-over-RDMA functionality tests where file " +
119 "is opened for reading: ",
120 },
121 "write": {
122 "tests": TESTNAMES_WRITE,
123 "desc": "Run all NFS-over-RDMA functionality tests where file " +
124 "is opened for writing: ",
125 },
126 }
127
128 # Line separator
129 LINE_SEP = "="*80
130
131 RDMA_layers = ("ib", "rdmap")
132
133 # I/O operations for both NFSv3 and NFSv4
134 NFSread = { "nfs4":(OP_READ,), "nfs3":(NFSPROC3_READ,) }
135 NFSwrite = { "nfs4":(OP_WRITE,), "nfs3":(NFSPROC3_WRITE,) }
136 NFSrdwr = { "nfs4":(OP_READ, OP_WRITE), "nfs3":(NFSPROC3_READ, NFSPROC3_WRITE) }
137
138 # NFS test types
139 NFS_BASIC = 0
140 NFS_READ = 1
141 NFS_WRITE = 2
142 NFS_EXCHANGE_ID = 3
143 NFS_READDIR = 4
144 NFS_READLINK = 5
145 NFS_GETACL = 6
146
147 # NFSv3/NFSv4 operations for each NFS test type
148 NFSoperations = {
149 NFS_READ : NFSread,
150 NFS_WRITE : NFSwrite,
151 NFS_EXCHANGE_ID : { "nfs4":(OP_EXCHANGE_ID, OP_SETCLIENTID) },
152 NFS_READDIR : { "nfs4":(OP_READDIR,), "nfs3":(NFSPROC3_READDIR, NFSPROC3_READDIRPLUS) },
153 NFS_READLINK : { "nfs4":(OP_READLINK,), "nfs3":(NFSPROC3_READLINK,) },
154 NFS_GETACL : { "nfs4":(OP_GETATTR,) },
155 }
156
157 # Tests only supported in NFSv4.x
158 NFSv4_Only_List = ("basic02", "basic05")
159
160 # RDMA SEND opcode lists
161 SendOnlyList = (SEND_Only, SEND_Only_Immediate, SEND_Only_Invalidate)
162 SendLastList = (SEND_Last, SEND_Last_Immediate, SEND_Last_Invalidate)
163 SendFMList = (SEND_First, SEND_Middle)
164 SendList = SendFMList + SendLastList
165 iWarpSendList = (Send, Send_Invalidate, Send_SE, Send_SE_Invalidate)
166
167 # RDMA Read Response opcode lists
168 ReadRespFMList = (RDMA_READ_Response_First, RDMA_READ_Response_Middle)
169 ReadRespLastList = (RDMA_READ_Response_Last,)
170 ReadResponseList = ReadRespFMList + ReadRespLastList + (RDMA_READ_Response_Only,)
171
172 # RDMA Write opcode lists
173 WriteOnlyList = (RDMA_WRITE_Only, RDMA_WRITE_Only_Immediate)
174 WriteLastList = (RDMA_WRITE_Last, RDMA_WRITE_Last_Immediate)
175 WriteFMList = (RDMA_WRITE_First, RDMA_WRITE_Middle)
176 WriteFOList = (RDMA_WRITE_First,) + WriteOnlyList
177 WriteList = WriteFMList + WriteLastList
178
179 # RDMA opcode lists
180 FirstMiddleList = SendFMList + ReadRespFMList + WriteFMList
181 OnlyList = SendOnlyList + WriteOnlyList + (RDMA_READ_Response_Only,)
182 RWLastList = WriteLastList + ReadRespLastList
183
184 # RDMA I/O types
185 RDMA_READ = 0
186 RDMA_WRITE = 1
187 RDMA_SEND = 2
188
189 # RDMA opcodes for each I/O type
190 RDMA_IO_MAP = {
191 RDMA_READ: {
192 "only": (RDMA_READ_Response_Only,),
193 "first": (RDMA_READ_Response_First,),
194 "middle": (RDMA_READ_Response_Middle,),
195 "last": (RDMA_READ_Response_Last,),
196 },
197 RDMA_WRITE: {
198 "only": WriteOnlyList,
199 "first": (RDMA_WRITE_First,),
200 "middle": (RDMA_WRITE_Middle,),
201 "last": WriteLastList,
202 },
203 RDMA_SEND: {
204 "only": SendOnlyList,
205 "first": (SEND_First,),
206 "middle": (SEND_Middle,),
207 "last": SendLastList,
208 },
209 }
210
211 # Dictionary to convert a number to its name
212 NumNames = {1:"one", 2:"two", 3:"three"}
213
214 def num_name(num):
215 """Return the name of the given number"""
216 return NumNames.get(num, num)
217
218 def chunk_str(count):
219 """Return string representation of the number of chunks given by count"""
220 if count == 0:
221 return "no RDMA"
222 else:
223 return "%s RDMA %s in the" % (num_name(count), plural("chunk", count))
224
225 def get_padding(size):
226 """Get the number of padding bytes"""
227 ndiff = size % 4
228 if ndiff > 0:
229 return (4 - ndiff)
230 return 0
231
232 def get_psn_match(spsn, epsn):
233 """Get the PSN match string for the given range [spsn, epsn)"""
234 if epsn == 0:
235 # Match PSN range at upper limit: [spsn, MAX_PSN]
236 matchstr = "ib.psn >= %d" % spsn
237 elif spsn == 0:
238 # Match PSN range at lower limit: [0, epsn)
239 matchstr = "ib.psn < %d" % epsn
240 elif epsn < spsn:
241 # PSN wrapped around, match range at upper and lower limits:
242 # [spsn, MAX_PSN] and [0, epsn)
243 matchstr = "(ib.psn >= %d or ib.psn < %d)" % (spsn, epsn)
244 else:
245 # Match PSN range: [spsn, epsn)
246 matchstr = "ib.psn >= %d and ib.psn < %d" % (spsn, epsn)
247 return matchstr
248
249 class RdmaTest(TestUtil):
250 """RdmaTest object
251
252 RdmaTest() -> New test object
253
254 Usage:
255 x = RdmaTest(testnames=['write01', ...])
256
257 # Run all the tests
258 x.run_tests()
259 x.exit()
260 """
261 def __init__(self, **kwargs):
262 """Constructor
263
264 Initialize object's private data.
265 """
266 TestUtil.__init__(self, **kwargs)
267
268 # Set default script options
269 self.opts.version = "%prog " + __version__
270 self.opts.set_defaults(nfiles=0)
271 self.opts.set_defaults(mtopts="hard")
272 self.opts.set_defaults(proto="rdma")
273 self.opts.set_defaults(port=20049)
274
275 # Options specific for this test script
276 hmsg = "File size to use for small files [default: %default]"
277 self.test_opgroup.add_option("--small-filesize", default="4k", help=hmsg)
278 hmsg = "File size to use for large files [default: %default]"
279 self.test_opgroup.add_option("--large-filesize", default="1m", help=hmsg)
280 hmsg = "Mark warnings for missing fragments as failures [default: %default]"
281 self.test_opgroup.add_option("--strict", action="store_true", default=False, help=hmsg)
282
283 self.scan_options()
284
285 # Disable createtraces option
286 self.createtraces = False
287
288 self.iosize = None # RDMA I/O size
289 self.reqsize = None # RDMA request size
290 self.maxresponsesize = None # Session max response size
291 self.nfs_read_count = 0 # Number of NFS reads already processed
292 self.nfs_write_xid = {} # XIDs of NFS writes already processed
293 self.read_file_dict = {} # Map size to file name to use for reading
294 self.symlink_list = [] # List of symbolic link names
295 self.testdir_list = [] # List of directory names
296 self.isiwarp = False
297
298 # Calculate file size to use for tests
299 self.small_filesize = int_units(self.small_filesize)
300 self.large_filesize = int_units(self.large_filesize)
301 deltafsize = int(self.small_filesize / 4)
302 self.file_size_list = [int(deltafsize/2)] + [x*deltafsize for x in range(1,4)]
303 self.basic_size_list = self.file_size_list + [self.small_filesize]
304
305 # Files to be created at setup for each test, use either a size or a
306 # list. If size is 0 then the size will be the value in --filesize
307 self.setup_files = {
308 "basic01": self.basic_size_list,
309 "basic02": self.small_filesize,
310 "basic05": self.small_filesize,
311 "read01": self.file_size_list,
312 "read02": self.small_filesize,
313 "read03": self.filesize,
314 "read04": self.large_filesize,
315 }
316
317 # Directories to be created for each test with the number of files
318 # to create in each directory
319 self.setup_dirs = {"basic01":[0], "basic03":[0,10]}
320
321 # Symbolic links to be created at setup, the source file is given by the
322 # size given
323 self.setup_symlinks = {"basic01":self.filesize, "basic04":self.filesize}
324
325 if self.nfs_version < 4:
326 # Remove all tests not supported for NFS version < 4
327 for tname in NFSv4_Only_List:
328 while tname in self.testlist:
329 self.testlist.remove(tname)
330 # Make sure there is at least one test left to run
331 if len(self.testlist) == 0:
332 self.opts.error("tests given in --runtest are not supported for --nfsversion=%s" % self.nfsversion)
333
334 def get_file_size(self, testname=None):
335 """Get file size list for the current test or for the given test name"""
336 results = []
337 if testname is None:
338 # Test name not given so use the current test
339 testname = self.testname
340 # Get the list of file sizes for given test
341 fsize_list = self.setup_files.get(testname, [])
342 # Convert tuple of a single size to a list
343 if isinstance(fsize_list, tuple):
344 fsize_list = list(fsize_list)
345 elif not isinstance(fsize_list, list):
346 fsize_list = [fsize_list]
347 # Convert file sizes to integers, also if size is 0 then
348 # use the size given in --filesize
349 for fsize in fsize_list:
350 results.append(self.filesize if fsize == 0 else int_units(fsize))
351 return results
352
353 def setup(self, **kwargs):
354 """Setup test environment"""
355 dir_list = []
356 fsize_list = []
357 symlk_list = []
358 # Get list of directories, files and symbolic links to create
359 # according to the tests which will be running
360 for testname in self.testlist:
361 dir_list += self.setup_dirs.get(testname, [])
362 fsize_list += self.get_file_size(testname)
363 ssize = self.setup_symlinks.get(testname)
364 if ssize is not None:
365 # Create source file for symbolic link if necessary
366 ssize = self.filesize if ssize == 0 else int_units(ssize)
367 symlk_list.append(ssize)
368 fsize_list.append(ssize)
369
370 testdir_h = {}
371 symlink_h = {}
372 if len(fsize_list) or len(dir_list):
373 # Create file system objects
374 self.umount()
375 self.mount(proto="tcp", port=2049)
376 for fsize in fsize_list:
377 # Create file using the given size if it does not exist yet
378 if self.read_file_dict.get(fsize) is None:
379 self.create_file(size=fsize)
380 self.read_file_dict[fsize] = self.filename
381 for nfiles in dir_list:
382 # Create directory if it does not exist yet
383 if testdir_h.get(nfiles) is None:
384 dirname = self.create_dir()
385 self.testdir_list.append(dirname)
386 # Create all the files given for this directory
387 for i in range(nfiles):
388 self.create_file(size=int(self.small_filesize/8), dir=dirname)
389 testdir_h[nfiles] = 1
390 for fsize in symlk_list:
391 # Create symbolic link if it does not exist yet
392 if symlink_h.get(fsize) is None:
393 # Get file to use as the source
394 filename = self.read_file_dict.get(fsize)
395 srcpath = self.abspath(filename)
396 self.get_filename()
397 self.dprint('DBG3', "Creating symbolic file %s -> %s" % (self.absfile, srcpath))
398 os.symlink(srcpath, self.absfile)
399 self.symlink_list.append(self.filename)
400 symlink_h[fsize] = 1
401 self.umount()
402
403 def get_nfs_ops(self, nfs_iotype):
404 """Get list of operations for the given NFS test type according
405 to the current NFS version mounted
406 """
407 nargs = NFSoperations.get(nfs_iotype)
408 if nargs is None:
409 return ()
410 elif self.nfs_version < 4:
411 return nargs.get("nfs3", ())
412 else:
413 return nargs.get("nfs4", ())
414
415 def read_file(self, **kwargs):
416 """Read the file given by the size"""
417 fd = None
418 size = kwargs.get("size", self.filesize)
419 # Get file path to read matching the size given
420 self.filename = self.read_file_dict.get(size)
421 self.absfile = self.abspath(self.filename)
422 try:
423 self.dprint('DBG2', "Reading file [%s] %d@0" % (self.absfile, size))
424 fd = os.open(self.absfile, os.O_RDONLY)
425 data = os.read(fd, size)
426 finally:
427 if fd:
428 os.close(fd)
429 return data
430
431 def get_nfragments(self, dmalen, iosize=None):
432 """Get the number of fragments for the given DMA length"""
433 # In iWarp the iosize is calculated for each request
434 # thus the value needs to be passed as an argument
435 if dmalen == 0:
436 return 0
437 if iosize is None:
438 iosize = self.iosize
439 return int(dmalen/iosize) + (1 if dmalen%iosize else 0)
440
441 def add_missing_ib_request(self, reqlist, iotype, psn, handle, offset, length):
442 """Add missing IB request and if necessary split it into multiple
443 requests of at most reqsize bytes each
444
445 reqlist:
446 Request list where missing requests will be appended
447 iotype:
448 I/O type of request: RDMA_READ or RDMA_WRITE
449 psn:
450 PSN number of first missing request
451 handle:
452 RETH remote key of request
453 offset:
454 RETH virtual address of request
455 length:
456 Total DMA length for missing requests
457 """
458 # Split request into multiple requests with respect to reqsize
459 while length > 0:
460 size = min(length, self.reqsize)
461 if iotype == RDMA_READ:
462 opcode = RDMA_READ_Request
463 fsize = 0
464 else:
465 opcode = RDMA_WRITE_First if size > self.iosize else RDMA_WRITE_Only
466 fsize = min(size, self.iosize)
467
468 reqlist.append((OpCode(opcode), psn, fsize, handle, offset, size, 0, 1))
469 psn += self.get_nfragments(size)
470 length -= size
471
472 def add_missing_ib_fragment(self, fraglist, iotype, psn, nextpsn, length, nfrags, count):
473 """Add missing IB fragment and if necessary split it into multiple
474 fragments of at most iosize bytes each
475
476 fraglist:
477 Fragment list where missing fragments will be appended
478 iotype:
479 I/O type of fragment: RDMA_READ, RDMA_WRITE or RDMA_SEND
480 psn:
481 PSN number of first fragment in the request
482 nextpsn:
483 PSN number of first missing fragment
484 length:
485 Number of bytes remaining in the request
486 nfrags:
487 Number of fragments in the request
488 count:
489 Number of missing fragments to append
490 """
491 # Split fragment into multiple fragments with respect to iosize
492 lastpsn = psn + nfrags - 1
493 for i in range(count):
494 if nextpsn == psn:
495 if nfrags == 1:
496 # First and only fragment is missing
497 opcode = RDMA_READ_Response_Only if iotype == RDMA_READ else RDMA_WRITE_Only
498 else:
499 # First fragment is missing
500 opcode = RDMA_READ_Response_First if iotype == RDMA_READ else RDMA_WRITE_First
501 elif nextpsn == lastpsn:
502 # Last fragment is missing
503 opcode = RDMA_READ_Response_Last if iotype == RDMA_READ else RDMA_WRITE_Last
504 else:
505 # Middle fragment is missing
506 opcode = RDMA_READ_Response_Middle if iotype == RDMA_READ else RDMA_WRITE_Middle
507 size = min(length, self.iosize)
508 fraglist.append((OpCode(opcode), nextpsn, size, 1))
509 length -= size
510 nextpsn += 1
511 return length
512
513 def sort_ib_fragments(self, fraglist, iotype=None, offset=None, length=None):
514 """Sort IB fragments by PSN, taking care of PSN wrapping around
515 and adding missing fragments to the list.
516
517 fraglist:
518 Fragment list to be sorted. If iotype, offset and length are
519 given, missing requests are added to the list. If only iotype
520 is given, missing fragments are added to the list.
521 iotype:
522 I/O type of fragments in list: RDMA_READ, RDMA_WRITE or RDMA_SEND
523 offset:
524 RETH virtual address of segment
525 length:
526 Total DMA length for segment
527 """
528 if len(fraglist) == 0:
529 return fraglist
530 # Midpoint of valid PSN numbers
531 mpsn = (IB_PSN_MASK>>1)
532 # Get a list of PSN numbers (index==1)
533 psnlist = [x[1] for x in fraglist]
534 psnmin = min(psnlist)
535 psnmax = max(psnlist)
536 retlist = []
537 sortkey = lambda x: x[1]
538 if psnmax - psnmin > mpsn:
539 # PSN wraps around, sort the upper PSN numbers first
540 retlist = sorted([x for x in fraglist if x[1] > mpsn], key=sortkey)
541 # Then sort the lower PSN numbers
542 retlist += sorted([x for x in fraglist if x[1] < mpsn], key=sortkey)
543 else:
544 # List does not wrap around, just sort the whole list
545 retlist = sorted(fraglist, key=sortkey)
546
547 if None not in (iotype, offset, length):
548 # Add missing requests to the list
549 index = 0
550 reqlist = []
551 nextpsn = retlist[0][1] - self.get_nfragments(retlist[0][4] - offset)
552 while length > 0:
553 if index < len(retlist):
554 opc, dpsn, size, handle, dma_off, dma_len, pindex, ismissing = retlist[index]
555 else:
556 # The last request is missing
557 self.add_missing_ib_request(reqlist, iotype, nextpsn, handle, offset, length)
558 break
559 if dma_off > offset:
560 # Add missing request
561 dsize = dma_off - offset
562 self.add_missing_ib_request(reqlist, iotype, nextpsn, handle, offset, dsize)
563 length -= dsize
564 reqlist.append(retlist[index])
565 nextpsn = dpsn + self.get_nfragments(dma_len)
566 offset = dma_off + dma_len
567 length -= dma_len
568 index += 1
569 return reqlist
570 elif iotype in (RDMA_READ, RDMA_WRITE):
571 # Add missing fragments to the list
572 fiolist = [retlist[0]]
573 psn = retlist[0][1] # PSN number of request
574 dma_len = retlist[0][5] # DMA length of request
575 index = 1 # Skip the request, it has already been added to the list
576 # PSN for first fragment -- for RDMA read, the first read response
577 # has the same PSN as the request. On the other hand for RDMA write,
578 # the first write fragment is a middle/last write.
579 nextpsn = psn + (1 if iotype == RDMA_WRITE else 0)
580 # Number of fragments in request
581 nfrags = self.get_nfragments(dma_len)
582 while dma_len > 0:
583 if index < len(retlist):
584 opc, dpsn, size, ismissing = retlist[index]
585 else:
586 # The last fragment is missing
587 count = nfrags - nextpsn + psn
588 self.add_missing_ib_fragment(fiolist, iotype, psn, nextpsn, dma_len, nfrags, count)
589 break
590 if dpsn > nextpsn:
591 # Fragment missing right before the current fragment
592 count = dpsn - nextpsn
593 dma_len = self.add_missing_ib_fragment(fiolist, iotype, psn, nextpsn, dma_len, nfrags, count)
594 fiolist.append(retlist[index])
595 nextpsn = dpsn + 1
596 dma_len -= size
597 index += 1
598 return fiolist
599 elif iotype == RDMA_SEND:
600 # This is a special case since the first and last SEND cannot
601 # be missing because there would be no reassembly.
602 index = 0
603 sndlist = []
604 nextpsn = retlist[0][1] # PSN number of first SEND fragment
605 lastpsn = retlist[-1][1] # PSN number of last SEND fragment
606 while nextpsn <= lastpsn:
607 opc, dpsn, size, ismissing = retlist[index]
608 if dpsn > nextpsn:
609 # Middle fragment missing right before the current fragment
610 count = dpsn - nextpsn
611 for i in range(count):
612 sndlist.append((OpCode(SEND_Middle), nextpsn, self.iosize, 1))
613 nextpsn += 1
614 sndlist.append(retlist[index])
615 nextpsn += 1
616 index += 1
617 return sndlist
618 return retlist
619
620 def get_iosize(self):
621 """Get the I/O size by inspecting any of the First or Middle fragments"""
622 save_index = self.pktt.get_index()
623 if self.pktt.match("ib.opcode in %s" % (FirstMiddleList,)):
624 self.iosize = self.pktt.pkt.ib.psize
625 else:
626 # No First or Middle fragments so get the maximum payload size
627 # in all the Only messages
628 iosize = 100
629 while self.pktt.match("ib.opcode in %s" % (OnlyList,)):
630 iosize = max(iosize, self.pktt.pkt.ib.psize)
631 self.iosize = iosize
632 self.pktt.rewind(save_index)
633 return self.iosize
634
635 def get_maxresponsesize(self):
636 """Get the session maximum response size"""
637 if self.nfs_version > 4:
638 save_index = self.pktt.get_index()
639 pktcall, pktreply = self.find_nfs_op(OP_CREATE_SESSION)
640 if pktreply:
641 self.maxresponsesize = pktreply.NFSop.fore_chan_attrs.maxresponsesize
642 self.pktt.rewind(save_index)
643 return self.maxresponsesize
644
645 def get_chunk_lists(self, rpcordma, rpctype=rpc_const.CALL, display=True):
646 """Return the chunk lists"""
647 if rpcordma is None:
648 return [], [], []
649 read_h = {} # Group each read chunk using the XDR position
650 cname_list = ("reads", "writes", "reply")
651 chunk_lists = {x:[] for x in cname_list}
652
653 # Get the chunk lists from the RPC-over-RDMA layer
654 for cname in cname_list:
655 clist = chunk_lists.get(cname)
656 chunk_list = getattr(rpcordma, cname, [])
657 if chunk_list is None:
658 continue
659 if cname == "reply":
660 # Convert the reply chunk into a list of lists
661 chunk_list = [chunk_list]
662 for chunkobj in chunk_list:
663 if cname == "reads":
664 # This is the read chunk list
665 xdrpos = chunkobj.position
666 cargs = (xdrpos, chunkobj.handle, chunkobj.offset, chunkobj.length)
667 read_h.setdefault(xdrpos, []).append(cargs)
668 elif cname in ("writes", "reply"):
669 # This is the write/reply chunk list
670 clist.append([])
671 for obj in chunkobj.target:
672 clist[-1].append((obj.handle, obj.offset, obj.length))
673
674 # Convert the read list into a list of lists
675 chunk_lists["reads"] = [read_h[x] for x in sorted(read_h.keys())]
676
677 if display:
678 # Display debug info
679 rpctype_str = rpc_const.msg_type[rpctype].lower()
680 for cname in cname_list:
681 clist = chunk_lists.get(cname)
682 cname = cname[:-1] if cname != "reply" else cname
683 rmsg = "xdrpos=%d, " if cname == "read" else ""
684 dmsg = " RDMA %s chunk segment: " + rmsg + "handle=%s, offset=0x%016x, length=%d"
685 if len(clist) > 0:
686 self.dprint('DBG2', "RDMA %s chunks in %s: %d" % (cname, rpctype_str, len(clist)))
687 for chunkobj in clist:
688 self.dprint('DBG2', "* RDMA %s chunk segments: %d" % (cname, len(chunkobj)))
689 for segment in chunkobj:
690 self.dprint('DBG2', dmsg % ((cname,) + segment))
691 return [chunk_lists[x] for x in cname_list]
692
693 def verify_rw_request(self, request, expected):
694 """Verify Read Request or Write First fragment"""
695 opcode, psn, size, handle, offset, dma_len = request
696 dma_psn, dma_offset, dma_length = expected
697 opstr = ib_op_codes.get(opcode)
698
699 self.dprint('DBG2', "%s: psn=%d, size=%d, handle=%s, offset=0x%016x, length=%d" % request)
700 self.test(True, "%s should be sent to client" % opstr)
701
702 expr = offset == dma_offset
703 fmsg = ": expecting 0x%016x, got 0x%016x" % (dma_offset, offset)
704 self.test(expr, "%s should have correct virtual address" % opstr, failmsg=fmsg)
705
706 expr = dma_len <= dma_length
707 fmsg = ": expecting %d, got %d" % (dma_length, dma_len)
708 self.test(expr, "%s should have correct DMA length" % opstr, failmsg=fmsg)
709
710 expr = psn == dma_psn
711 fmsg = ": expecting %s, got %s" % (dma_psn, psn)
712 self.test(expr, "%s should have correct PSN" % opstr, failmsg=fmsg)
713
714 if opcode == RDMA_READ_Request:
715 expr = size == 0
716 fmsg = ": expecting no payload, got %s bytes" % size
717 self.test(expr, "%s should not have any payload data" % opstr, failmsg=fmsg)
718 else:
719 esize = (self.iosize if opcode == RDMA_WRITE_First else dma_length)
720 esize += get_padding(esize)
721 expr = size == esize
722 fmsg = ": expecting %s, got %s" % (esize, size)
723 self.test(expr, "%s should have correct payload size" % opstr, failmsg=fmsg)
724
725 def verify_fragments(self, iotype, iolist, count, rpctype=None):
726 """Verify RDMA I/O fragments"""
727 results = ()
728 rdma_h = RDMA_IO_MAP.get(iotype)
729 # Direction of RDMA I/O fragments
730 if iotype == RDMA_SEND:
731 # For RDMA_SEND use the rpctype to find out correct direction
732 srv_clnt = "server" if rpctype == rpc_const.CALL else "client"
733 else:
734 srv_clnt = "server" if iotype == RDMA_READ else "client"
735 if count > 2:
736 # Results should have First, Middle and Last fragments
737 results = ((rdma_h["first"], 1), (rdma_h["middle"], count-2), (rdma_h["last"], 1))
738 elif count > 1:
739 # Results should only have First and Last fragments
740 results = ((rdma_h["first"], 1), (rdma_h["last"], 1))
741 elif count == 1:
742 # Results should only have one Only fragment
743 results = ((rdma_h["only"], 1),)
744
745 # Get maximum length of opcode name
746 if len(iolist) == 1 and len(iolist[0]) != 4:
747 maxlen = 0
748 else:
749 maxlen = len(max([str(x[0]) for x in iolist if len(x) == 4], key=len))
750
751 # Display all fragments other than the Read Request and Write First
752 # fragment which were already been displayed in verify_rw_request
753 missing_fragments = []
754 for seg in iolist:
755 if len(seg) == 4:
756 opcode, psn, size, ismissing = seg
757 mfrag = " [missing fragment]" if ismissing else ""
758 args = (maxlen, opcode, psn, size, mfrag)
759 self.dprint('DBG3', "%-*s: psn=%d, size=%d%s" % args)
760 if ismissing:
761 missing_fragments.append((psn, size))
762
763 if len(missing_fragments) > 0:
764 msize = sum([x[1] for x in missing_fragments])
765 self.dprint('DBG2', "Missing %d bytes in fragments:" % msize)
766 for psn, size in missing_fragments:
767 self.dprint('DBG3', " Missing %d bytes at PSN %d" % (size, psn))
768
769 # Verify all the fragments
770 for item in results:
771 oplist, ecount = item
772 # Get the number of fragments actually sent for the given OpCode
773 op_list = [x for x in iolist if x[0] in oplist and x[3] == 0]
774 io_count = len(op_list)
775 if io_count > 0:
776 opcode = op_list[0][0]
777 else:
778 opcode = oplist[0]
779
780 # Verify all fragments other than the Read Request and Write First
781 # fragment which were already been verified in verify_rw_request
782 if opcode not in WriteFOList or len(iolist[0]) != 6:
783 amsg = "%s should be sent to %s" % (ib_op_codes.get(opcode), srv_clnt)
784 fmsg = ": expecting %s %s, got %s" % (ecount, plural("fragment", ecount), io_count)
785 expr = io_count == ecount
786 if self.strict or expr:
787 self.test(expr, amsg, failmsg=fmsg)
788 else:
789 self.warning(amsg + fmsg)
790
791 if io_count > 0:
792 expr = True
793 # Get expected I/O size
794 if opcode in SendOnlyList:
795 # There is only one fragment: should be the payload size
796 iosize = iolist[0][2]
797 elif opcode in SendLastList:
798 # Last SEND fragment: should be the payload size since
799 # there is no way to find out the size of the whole I/O
800 iosize = iolist[-1][2]
801 elif opcode in RWLastList:
802 # Last Read/Write fragment: it should be the remainder
803 # size of the whole I/O
804 nbytes = iolist[0][5] % self.iosize
805 iosize = nbytes if nbytes > 0 else self.iosize
806 elif opcode == RDMA_READ_Response_Only:
807 # There is only one fragment: should be the payload size
808 iosize = iolist[-1][2]
809 else:
810 # Use the RDMA I/O size of the mount
811 iosize = self.iosize
812 iosize += get_padding(iosize) # Add padding bytes if any
813 # Verify size of each fragment for given OpCode is correct
814 for item in op_list:
815 size = item[2]
816 expr = expr and size == iosize
817 if not expr:
818 break
819 amsg = "%s should have correct payload size" % ib_op_codes.get(opcode)
820 fmsg = ": expecting %d, got %d" % (iosize, size)
821 self.test(expr, amsg, failmsg=fmsg)
822
823 def verify_ib_segment(self, iotype, handle, offset, length):
824 """Verify IB segment and all its fragments for the given handle"""
825 # Get correct info according to RDMA I/O type (either READ or WRITE)
826 if iotype == RDMA_READ:
827 opreq = (RDMA_READ_Request,)
828 oplist = ReadResponseList
829 else:
830 opreq = WriteFOList # The Write First fragment has the request info
831 oplist = WriteList
832
833 # Search for all read/write requests in the segment
834 requests = []
835 self.reqsize = self.iosize
836 match_str = "ib.opcode in %s and ib.reth.r_key == %s" % (opreq, handle)
837 while self.pktt.match(match_str):
838 pindex = self.pktt.get_index()
839 ibobj = self.pktt.pkt.ib
840 reth = ibobj.reth
841 self.reqsize = max(self.reqsize, reth.dma_len)
842 # Include the packet index in the request
843 requests.append((ibobj.opcode, ibobj.psn, ibobj.psize, reth.r_key, reth.va, reth.dma_len, pindex, 0))
844
845 iostr = "read requests" if iotype == RDMA_READ else "write firsts"
846 self.dprint('DBG2', "RDMA %s for handle %s: %d" % (iostr, handle, len(requests)))
847
848 if len(requests) == 0 and length > 0:
849 if iotype == RDMA_READ:
850 opstr = ib_op_codes.get(RDMA_READ_Request)
851 else:
852 if self.get_nfragments(length) == 1:
853 opstr = ib_op_codes.get(RDMA_WRITE_Only)
854 else:
855 opstr = ib_op_codes.get(RDMA_WRITE_First)
856 self.test(False, "%s should be sent to client" % opstr)
857 elif length == 0:
858 opstr = "read requests" if iotype == RDMA_READ else "writes"
859 fmsg = ": expecting 0, got %d" % len(requests)
860 amsg = "RDMA %s should not be sent to client for segment with DMA length of zero" % opstr
861 self.test(len(requests) == 0, amsg, failmsg=fmsg)
862
863 # Verify all fragments for each request
864 dma_psn = None
865 dma_length = None
866 for req_info in self.sort_ib_fragments(requests, iotype, offset, length):
867 opcode, psn, size, handle, dma_off, dma_len, pindex, ismissing = req_info
868 reqinfo = req_info[:-2]
869
870 if dma_psn is None:
871 # Use the PSN on the request for the first expected PSN
872 dma_psn = psn
873 if dma_length is None:
874 # Use the DMA length on the request for the first expected length
875 dma_length = dma_len
876
877 # Get the number of expected fragments for this request
878 count = self.get_nfragments(dma_len)
879
880 # Get all fragments starting with the PSN from the request and
881 # up to the last expected PSN
882 fragment_list = [reqinfo] # Include the request in the fragment list
883 self.pktt.rewind(pindex)
884 nextpsn = (psn + count) & IB_PSN_MASK
885 mstr = "ib.opcode in %s and %s" % (oplist, get_psn_match(psn, nextpsn))
886 while self.pktt.match(mstr):
887 ib = self.pktt.pkt.ib
888 fragment_list.append((ib.opcode, ib.psn, ib.psize, 0))
889
890 # Verify Read Request or Write First fragment
891 if ismissing:
892 opstr = ib_op_codes.get(opcode)
893 self.dprint('DBG2', "%s: psn=%d, size=%d, handle=%s, offset=0x%016x, length=%d [missing request]" % reqinfo)
894 if self.strict:
895 self.test(False, "%s should be sent to client" % opstr)
896 else:
897 self.warning("%s should be sent to client" % opstr)
898 else:
899 self.verify_rw_request(reqinfo, (dma_psn, offset, dma_length))
900
901 # Verify RDMA I/O fragments
902 fragment_list = self.sort_ib_fragments(fragment_list, iotype)
903 self.verify_fragments(iotype, fragment_list, count)
904
905 dma_psn = nextpsn # Expected PSN for next request
906 offset += dma_len # Expected offset for next request
907 length -= dma_len # Remaining bytes for whole segment
908 # Expected DMA length of next request, the last request for the
909 # segment could be shorter than dma_length and it should be the
910 # remaining bytes for the whole segment
911 dma_length = length
912
913 def add_missing_fragment(self, request_item, opstr, handle, offset, length, iosize):
914 """Add missing fragment"""
915 last_fl = 0
916 iosize = iosize if iosize > 0 else length
917 # Split fragment into multiple fragments with respect to iosize
918 while length > 0:
919 size = min(length, iosize)
920 # Set last flag if last fragment size is less than iosize
921 last_fl = 1 if size < iosize else 0
922 request_item.append((opstr, handle, LongHex(offset), last_fl, size, 1))
923 offset += size
924 length -= size
925 return last_fl
926
927 def verify_iwarp_segment(self, iotype, handle, offset, length):
928 """Verify iWarp segment and all its fragments for the given handle"""
929 # Get correct info according to RDMA I/O type (either READ or WRITE)
930 requests = []
931 reqlen = 0
932 if iotype == RDMA_READ:
933 opres = RDMA_Read_Response
934 # Search for all read requests in the segment
935 match_str = "rdmap.opcode == %d and rdmap.srcstag == %s and " % (RDMA_Read_Request, handle)
936 match_str += "rdmap.srcsto >= %s and rdmap.srcsto < 0x%016x" % (offset, offset + length)
937 while self.pktt.match(match_str):
938 rdmapobj = self.pktt.pkt.rdmap
939 reqlen += rdmapobj.dma_len
940 # Include the packet index in the request
941 requests.append((rdmapobj.opcode, rdmapobj.srcsto, rdmapobj.psize, rdmapobj.sinkstag,
942 rdmapobj.sinksto, rdmapobj.dma_len, self.pktt.pkt.record.index))
943 self.dprint('DBG2', "RDMA read requests for handle %s: %d" % (handle, len(requests)))
944 else:
945 opres = RDMA_Write
946 requests = [(opres, offset, length, handle, offset, length, self.pktt.get_index())]
947
948 opstr = rdmap_op_codes.get(opres)
949 # Direction of RDMA I/O fragments
950 srv_clnt = "server" if iotype == RDMA_READ else "client"
951
952 for reqinfo in requests:
953 # Verify all fragments
954 opreq, soffset, psize, rhandle, roffset, dma_len, pktindex = reqinfo
955 if iotype == RDMA_READ:
956 args = (opreq, handle, soffset, rhandle, roffset, dma_len)
957 self.dprint('DBG2', "%s: src:(%s, %s), sink:(%s, %s), dma_len: %s" % args)
958 self.test(True, "%s should be sent to client" % opreq)
959 amsg = "%s should have correct " % opreq
960 self.test(True, amsg + "virtual address")
961 self.test(dma_len <= length, amsg + "DMA length")
962 fmsg = ": expecting no payload, got %s bytes" % psize
963 self.test(psize == 0, "%s should not have any payload data" % opreq, failmsg=fmsg)
964
965 # Search for all fragments belonging to this request
966 mstr = "rdmap.opcode == %d and rdmap.stag == %s and " % (opres, rhandle)
967 mstr += "rdmap.offset >= %s and rdmap.offset < 0x%016x" % (roffset, roffset + dma_len)
968 iosize = 0
969 fragment_list = []
970 self.pktt.rewind(pktindex)
971 while self.pktt.match(mstr):
972 rdmapobj = self.pktt.pkt.rdmap
973 fragment_list.append((rdmapobj.offset, rdmapobj.lastfl, rdmapobj.psize))
974 iosize = max(iosize, rdmapobj.psize)
975
976 io_count = len(fragment_list)
977 iostr = "reads" if iotype == RDMA_READ else "writes"
978 stagstr = " (stag: %s)" % rhandle if iotype == RDMA_READ else ""
979 self.dprint('DBG2', "RDMA %s for handle %s%s: %d" % (iostr, handle, stagstr, io_count))
980
981 # Sort fragment list by offset and split them up by lastfl==1
982 nextoff = roffset # Expected offset of next fragment
983 request_list = [[]] # List of sub-requests
984 for off, lastfl, size in sorted(fragment_list, key=lambda x: x[0]):
985 if off != nextoff:
986 # Missing fragment found
987 msize = off - nextoff
988 if self.add_missing_fragment(request_list[-1], opstr, rhandle, nextoff, msize, iosize):
989 # Start a new sub-request
990 request_list.append([])
991 # Append fragment to sub-request
992 request_list[-1].append((opstr, rhandle, off, lastfl, size, 0))
993 if lastfl == 1:
994 # Start a new sub-request
995 request_list.append([])
996 nextoff = off + size
997
998 # Total size for all fragments
999 tbytes = sum([x[4] for y in request_list for x in y])
1000 if tbytes < dma_len:
1001 # Add missing fragments for the last request
1002 msize = dma_len - tbytes
1003 self.add_missing_fragment(request_list[-1], opstr, rhandle, nextoff, msize, iosize)
1004
1005 # Drop empty requests that may have been added
1006 request_list = [x for x in request_list if len(x) > 0]
1007
1008 for reqitem in request_list:
1009 # DMA length of sub-request
1010 dmalen = sum([x[4] for x in reqitem])
1011 # Number of sub-requests found
1012 nreqs = len([1 for x in reqitem if x[5] == 0])
1013 if len(request_list) > 1:
1014 # Display only if there are more than one sub-request
1015 self.dprint('DBG2', "RDMA %s for handle %s%s (request): %d" % (iostr, handle, stagstr, nreqs))
1016 psizes = set() # List of unique payload sizes for sub-request
1017 missing_fragments = [] # List of missing fragments
1018 for item in reqitem:
1019 opstr, rhandle, off, lastfl, size, ismissing = item
1020 mfstr = " [missing fragment]" if ismissing else ""
1021 args = item[:-1] + (mfstr,)
1022 self.dprint('DBG3', "%s: stag=%s, offset=%s, last=%d, size=%d%s" % args)
1023 if ismissing:
1024 missing_fragments.append((off, size))
1025 else:
1026 psizes.add(size)
1027
1028 if len(missing_fragments) > 0:
1029 msize = sum([x[1] for x in missing_fragments])
1030 self.dprint('DBG2', "Missing %d bytes in fragments:" % msize)
1031 for off, size in missing_fragments:
1032 self.dprint('DBG3', " Missing %d bytes at offset 0x%016x" % (size, off))
1033
1034 # Calculate the number of expected fragments per sub-request
1035 iosize = iosize if iosize > 0 else dmalen
1036 ecount = self.get_nfragments(dmalen, iosize)
1037 notstr = "not " if ecount == 0 else ""
1038 bmsg = " for segment with DMA length of zero" if ecount == 0 else ""
1039 amsg = "%s should %sbe sent to %s%s" % (opstr, notstr, srv_clnt, bmsg)
1040 fmsg = ": expecting %s %s, got %s" % (ecount, plural("fragment", ecount), nreqs)
1041 expr = nreqs == ecount or ecount == 0
1042 if self.strict or expr:
1043 self.test(expr, amsg, failmsg=fmsg)
1044 else:
1045 self.warning(amsg + fmsg)
1046
1047 if nreqs > 0:
1048 amsg = "%s should have correct " % opstr
1049 # Fragments were matched using the offset
1050 self.test(True, amsg + "virtual address")
1051 # All fragments should have the same size except for the last
1052 # but not necessarily
1053 self.test(len(psizes) <= 2, amsg + "payload size")
1054
1055 def verify_segment(self, iotype, handle, offset, length):
1056 """Verify segment and all its fragments for the given handle"""
1057 if self.isiwarp:
1058 self.verify_iwarp_segment(iotype, handle, offset, length)
1059 else:
1060 self.verify_ib_segment(iotype, handle, offset, length)
1061
1062 def verify_io_op(self, pkt, optype=None, rpctype=None, isrpcordma=True, rdmaproc=RDMA_MSG):
1063 """Verify I/O call/reply operation"""
1064 if pkt is None:
1065 # Not a valid packet
1066 if optype is not None:
1067 iostr = "%s %s" % (self.nfs_op_name(optype), rpc_const.msg_type[rpctype].lower())
1068 if rpctype is None:
1069 srv_clnt = ""
1070 else:
1071 srv_clnt = " to server" if rpctype == rpc_const.CALL else " to client"
1072 self.test(False, "NFS %s should be sent%s" % (iostr, srv_clnt))
1073 return
1074
1075 if self.nfs_version < 4:
1076 # For NFSv3, the NFS read/write is the NFS object
1077 nfsop = pkt.nfs
1078 elif getattr(pkt, "NFSop", None) is not None:
1079 # For NFSv4, the NFS read/write is the NFSop object
1080 nfsop = pkt.NFSop
1081 else:
1082 # NFSop is None so NFS was not matched directly so look
1083 # for the NFS read/write operation object
1084 nfsop = self.getop(pkt, optype)
1085
1086 optype = nfsop.op if optype is None else optype
1087 rpctype = pkt.rpc.type if rpctype is None else rpctype
1088 iostr = "%s %s" % (self.nfs_op_name(optype), rpc_const.msg_type[rpctype].lower())
1089 srv_clnt = "server" if rpctype == rpc_const.CALL else "client"
1090
1091 if optype in self.nfs_op(**NFSrdwr):
1092 # NFS read or write
1093 offstr = "" if rpctype == rpc_const.REPLY else "offset=%d, " % nfsop.offset
1094 self.dprint('DBG2', "Found NFS %s: %scount=%d" % (iostr, offstr, nfsop.count))
1095 else:
1096 self.dprint('DBG2', "Found NFS %s" % iostr)
1097 self.test(pkt, "NFS %s should be sent to %s" % (iostr, srv_clnt))
1098 self.test(pkt in RDMA_layers, "NFS %s should be sent over RDMA" % iostr)
1099 if pkt in RDMA_layers:
1100 if isrpcordma:
1101 expr = pkt == "rpcordma"
1102 self.test(expr, "NFS %s should be sent with RPCoRDMA layer" % iostr)
1103 if expr:
1104 procstr = rdma_proc.get(rdmaproc)
1105 expr = pkt.rpcordma.proc == rdmaproc
1106 self.test(expr, "NFS %s should be sent with %s proc" % (iostr, procstr))
1107 if pkt.rpcordma.proc == RDMA_MSG:
1108 expr = pkt.rpcordma.psize > 0
1109 self.test(expr, "NFS %s should be sent with payload data for %s" % (iostr, procstr))
1110 else:
1111 expr = pkt.rpcordma.psize == 0
1112 self.test(expr, "NFS %s should be sent with no payload data for %s" % (iostr, procstr))
1113 else:
1114 expr = pkt != "rpcordma"
1115 self.test(expr, "NFS %s should be sent with no RPCoRDMA layer" % iostr)
1116 if optype in self.nfs_op(**NFSwrite) and rpctype == rpc_const.CALL:
1117 if self.isiwarp:
1118 expr = pkt.rdmap.opcode == RDMA_Read_Response and pkt.rdmap.lastfl == 1
1119 else:
1120 expr = pkt.ib.opcode in (RDMA_READ_Response_Last, RDMA_READ_Response_Only)
1121 self.test(expr, "NFS %s should be reassembled in the last read response fragment " % iostr)
1122
1123 def verify_chunk_lists(self, rpcordma, optype, rpctype, nreads=0, nwrites=0, nreply=0):
1124 """Verify the RDMA chunk lists"""
1125 if rpcordma is None:
1126 return
1127 iostr = "%s %s" % (self.nfs_op_name(optype), rpc_const.msg_type[rpctype].lower())
1128
1129 ncount = nreads + nwrites
1130 if optype in self.nfs_op(**NFSread) or rpctype == rpc_const.REPLY or ncount == 0:
1131 tmsg = "NFS %s should be sent with " % iostr
1132 else:
1133 tmsg = "RPCoRDMA (NFS %s) should be sent with " % iostr
1134
1135 # Get the RDMA chunk lists
1136 reads, writes, reply = self.get_chunk_lists(rpcordma, display=False)
1137
1138 # Verify the number of chunks in each chunk list
1139 amsg = tmsg + "%s read chunk list" % chunk_str(nreads)
1140 fmsg = ", there are %d read chunks" % len(reads)
1141 self.test(len(reads) == nreads, amsg, failmsg=fmsg)
1142 amsg = tmsg + "%s write chunk list" % chunk_str(nwrites)
1143 fmsg = ", there are %d write chunks" % len(writes)
1144 self.test(len(writes) == nwrites, amsg, failmsg=fmsg)
1145 amsg = tmsg + "%s reply chunk" % chunk_str(nreply)
1146 fmsg = ", there are %d write chunks" % len(reply)
1147 if not self.strict and nreply == 0 and len(reply) > 0:
1148 # Do not fail test if there is an unexpected reply chunk
1149 amsg = tmsg.replace('should', 'may') + "%s reply chunk" % chunk_str(1)
1150 self.test(len(reply) == 1, amsg, failmsg=fmsg)
1151 elif self.strict and len(reply) != nreply:
1152 # If strict is given, log warning if there is an unexpected reply chunk
1153 self.warning(amsg + fmsg)
1154 else:
1155 self.test(len(reply) == nreply, amsg, failmsg=fmsg)
1156
1157 flat_list = []
1158 for clist in reads+writes+reply:
1159 flat_list += clist
1160
1161 if len(flat_list) > 0 and rpctype == rpc_const.CALL:
1162 handle_list = [x[-3] for x in flat_list]
1163 nhandles = len(handle_list) # Number of handles
1164 uhandles = len(set(handle_list)) # Number of unique handles
1165 amsg = tmsg + "all unique RDMA chunk handles"
1166 fmsg = ": expecting %d unique handles but got %d" % (uhandles, nhandles)
1167 self.test(uhandles == nhandles, amsg, failmsg=fmsg)
1168
1169 offset_list = [x[-2] for x in flat_list]
1170 noffsets = len(offset_list) # Number of offsets
1171 uoffsets = len(set(offset_list)) # Number of unique offsets
1172 amsg = tmsg + "all unique RDMA chunk virtual addresses"
1173 fmsg = ": expecting %d unique virtual addresses but got %d" % (uoffsets, noffsets)
1174 self.test(uoffsets == noffsets, amsg, failmsg=fmsg)
1175
1176 afmt = "%sDDP (using %s opcodes)"
1177 msg_h = {0:("no ", "SEND")}
1178 if rpctype == rpc_const.CALL:
1179 if optype in self.nfs_op(**NFSwrite):
1180 # Verify if using DDP and correct XDR position on NFS WRITE call
1181 if len(reads) and len(reads[0]):
1182 expr = reads[0][0][0] <= rpcordma.psize
1183 fmsg = ": xdrpos(%d) should be less than or equal to RPCoRDMA " \
1184 "payload length(%d)" % (reads[0][0][0], rpcordma.psize)
1185 self.test(expr, tmsg + "correct XDR position", failmsg=fmsg)
1186 if reads[0][0][0] == 0:
1187 expr = rpcordma.proc == RDMA_NOMSG
1188 self.test(expr, tmsg + "RDMA_NOMSG proc for a long request (PZRC)")
1189 else:
1190 expr = rpcordma.proc == RDMA_MSG
1191 self.test(expr, tmsg + "RDMA_MSG proc")
1192 if rpcordma.proc == RDMA_MSG:
1193 expr = rpcordma.psize > 0
1194 self.test(expr, tmsg + "payload data for RDMA_MSG")
1195 else:
1196 expr = rpcordma.psize == 0
1197 self.test(expr, tmsg + "no payload data for RDMA_NOMSG")
1198 amsg = afmt % msg_h.get(nreads, ("", "RDMA_READ"))
1199 self.test(len(reads) == nreads, tmsg + amsg)
1200 elif len(reply):
1201 # Verify correct DMA length
1202 tsize = min([x[2] for x in reply[0]])
1203 expr = tsize > 0
1204 self.test(expr, tmsg + "a non-zero DMA length")
1205 if self.maxresponsesize is not None:
1206 tsize = sum([x[2] for x in reply[0]])
1207 expr = tsize < self.maxresponsesize
1208 fmsg = ": %d is not less than %d" % (tsize, self.maxresponsesize)
1209 amsg = tmsg + "a DMA length less than maxresponsesize"
1210 if self.strict or expr:
1211 self.test(expr, amsg, failmsg=fmsg)
1212 else:
1213 self.warning(amsg + fmsg)
1214 elif optype in self.nfs_op(**NFSread):
1215 # Verify if using DDP on NFS READ reply
1216 amsg = afmt % msg_h.get(nwrites, ("", "RDMA_WRITE"))
1217 self.test(len(writes) == nwrites, tmsg + amsg)
1218
1219 def verify_ib_sends(self, pkt, optype, rpctype, rpcordma=None, chunkverify=True):
1220 """Verify I/O is sent using (IB) RDMA SENDs instead of a chunk list"""
1221 if pkt and pkt in RDMA_layers:
1222 send_list = []
1223 # Index of NFS write call or read reply
1224 io_index = pkt.record.index
1225 opcode = pkt.ib.opcode
1226 if opcode in SendOnlyList:
1227 # Nothing to do here, there is only one packet
1228 # for the whole NFS read/write
1229 count = 1
1230 send_list.append((opcode, pkt.ib.psn, pkt.ib.psize, 0))
1231 elif opcode in SendList:
1232 # Find all SEND First, Middle and Last fragments
1233 match_str = "ib.opcode in %s" % (SendList,)
1234 while self.pktt.match(match_str, maxindex=io_index+1):
1235 ibobj = self.pktt.pkt.ib
1236 ib_psn = ibobj.psn
1237 ib_count = ibobj.psize
1238 ib_opcode = ibobj.opcode
1239 send_list.append((ib_opcode, ib_psn, ib_count, 0))
1240 # Make sure to get the SEND_First closest to SEND_Last
1241 index = len(send_list) - 1
1242 for item in reversed(send_list):
1243 if item[0] == SEND_First:
1244 break
1245 index -= 1
1246 if index > 0:
1247 # Remove beginning of list
1248 send_list = send_list[index:]
1249 count = send_list[-1][1] - send_list[0][1] + 1
1250 else:
1251 return
1252
1253 if len(send_list):
1254 if chunkverify:
1255 # Verify the RDMA chunk lists
1256 if rpcordma is None and pkt == "rpcordma":
1257 rpcordma = pkt.rpcordma
1258 self.verify_chunk_lists(rpcordma, optype, rpctype)
1259 # Verify the list of SEND fragments
1260 send_list = self.sort_ib_fragments(send_list, RDMA_SEND)
1261 self.verify_fragments(RDMA_SEND, send_list, count, rpctype=rpctype)
1262
1263 def verify_iwarp_sends(self, pkt, optype, rpctype, rpcordma=None, chunkverify=True):
1264 """Verify I/O is sent using (iWarp) RDMA SENDs instead of a chunk list"""
1265 if pkt and pkt == "rdmap":
1266 send_list = []
1267 # Index of NFS write call or read reply
1268 io_index = pkt.record.index
1269 opcode = pkt.rdmap.opcode
1270 if pkt.rdmap.offset == 0:
1271 # There is only one SEND for this NFS read/write
1272 rdmap = pkt.rdmap
1273 send_list.append((rdmap.opcode, rdmap.offset, rdmap.psize, rdmap.lastfl))
1274 elif opcode in iWarpSendList:
1275 # Find all Send fragments
1276 msn = pkt.ddp.msn
1277 match_str = "ddp.queue == %d and " % pkt.ddp.queue + \
1278 "ddp.msn == %d and " % pkt.ddp.msn + \
1279 "rdmap.opcode in %s" % (iWarpSendList,)
1280 while self.pktt.match(match_str, maxindex=io_index+1):
1281 rdmap = self.pktt.pkt.rdmap
1282 send_list.append((rdmap.opcode, rdmap.offset, rdmap.psize, rdmap.lastfl))
1283
1284 # Filter the SENDs for the NFS read/write, start from the end
1285 # of the list and find the first SEND packet (lastfl == 0)
1286 count = 0
1287 for op, offset, size, lastfl in reversed(send_list):
1288 if count > 0 and lastfl:
1289 break
1290 count += 1
1291 if count < len(send_list):
1292 send_list = send_list[-count:]
1293
1294 if len(send_list):
1295 if chunkverify:
1296 # Verify the RDMA chunk lists
1297 if rpcordma is None and pkt == "rpcordma":
1298 rpcordma = pkt.rpcordma
1299 self.verify_chunk_lists(rpcordma, optype, rpctype)
1300
1301 # Verify the list of Send fragments
1302 srv_clnt = "server" if rpctype == rpc_const.CALL else "client"
1303 iostr = "%s %s" % (self.nfs_op_name(optype), rpc_const.msg_type[rpctype].lower())
1304 self.dprint('DBG2', "RDMA Sends found for %s (MSN=%d): %d" % (iostr, pkt.ddp.msn, len(send_list)))
1305 maxlen = max([len(str(x[1])) for x in send_list])
1306 countlast = 0 # Number of fragments with the last flag set
1307 missfrags = 0 # Number of missing fragments
1308 nextoff = 0 # Next fragment offset
1309 psizes = set() # List of unique payload sizes
1310 for op, offset, size, lastfl in sorted(send_list, key=lambda x: x[1]):
1311 self.dprint('DBG3', "%s: offset=%*s, last=%d, size=%d" % (op, maxlen, offset, lastfl, size))
1312 if lastfl == 1:
1313 countlast += 1
1314 if offset != nextoff:
1315 missfrags += 1
1316 psizes.add(size)
1317 nextoff = offset + size
1318 self.test(len(send_list) > 0, "Send should be sent to %s" % srv_clnt)
1319 amsg = "Send should have correct "
1320 self.test(missfrags == 0, amsg + "offset")
1321 # All fragments should have the same size except for the last
1322 # but not necessarily
1323 self.test(len(psizes) <= 2, amsg + "payload size")
1324 # Fragments were matched using the MSN
1325 self.test(True, amsg + "MSN")
1326 # Only one fragment should have the last flag set
1327 self.test(countlast == 1, amsg + "last flag")
1328
1329 def verify_sends(self, pkt, optype, rpctype, rpcordma=None, chunkverify=True):
1330 """Verify I/O is sent using RDMA SENDs instead of a chunk list"""
1331 if self.isiwarp:
1332 self.verify_iwarp_sends(pkt, optype, rpctype, rpcordma, chunkverify)
1333 else:
1334 self.verify_ib_sends(pkt, optype, rpctype, rpcordma, chunkverify)
1335
1336 def verify_chunk_in_reply(self, rpcordma_c, pktreply, optype, nreads=0, nwrites=0, nreply=0):
1337 """Verify RDMA write chunk list or RDMA reply chunk in the NFS reply"""
1338 if pktreply is None or pktreply != "rpcordma":
1339 # Not an RPC-over-RDMA packet
1340 return
1341
1342 # Get the RDMA chunk lists for both the NFS call and reply
1343 rpcordma_r = pktreply.rpcordma
1344 creads, cwrites, creply = self.get_chunk_lists(rpcordma_c, rpc_const.CALL, display=False)
1345 rreads, rwrites, rreply = self.get_chunk_lists(rpcordma_r, rpc_const.REPLY)
1346 self.verify_chunk_lists(rpcordma_r, optype, rpc_const.REPLY, nreads, nwrites, nreply)
1347
1348 opstr = "NFS %s" % self.nfs_op_name(optype)
1349 tmsg = "%s reply should have correct RDMA segment " % opstr
1350
1351 # Verify either the write chunk list or the reply chunk
1352 call_chunk_list = cwrites if nwrites > 0 else creply
1353 reply_chunk_list = rwrites if nwrites > 0 else rreply
1354
1355 while len(call_chunk_list) and len(reply_chunk_list):
1356 call_chunk = call_chunk_list.pop(0)
1357 reply_chunk = reply_chunk_list.pop(0)
1358
1359 # Verify the reply has the same number of segments as the call
1360 c_count = len(call_chunk)
1361 r_count = len(reply_chunk)
1362 expr = c_count == r_count
1363 fmsg = ": reply "
1364 amsg = "%s reply should be sent with the same number of RDMA chunk segments as the call" % opstr
1365 if c_count > r_count:
1366 count = c_count - r_count
1367 fmsg += "is missing %s %s" % (num_name(count), plural("segment", count))
1368 elif c_count < r_count:
1369 count = r_count - c_count
1370 fmsg += "has %s extra %s" % (num_name(count), plural("segment", count))
1371 self.test(expr, amsg, failmsg=fmsg)
1372
1373 # Verify all segments in the RMA chunk
1374 while len(call_chunk) and len(reply_chunk):
1375 chandle, coffset, clength = call_chunk.pop(0)
1376 rhandle, roffset, rlength = reply_chunk.pop(0)
1377 rstr = "reply" if nreply > 0 else "write"
1378 dargs = (rstr, rhandle, roffset, rlength)
1379 self.dprint('DBG2', "RDMA %s chunk segment: handle=%s, offset=0x%016x, length=%d" % dargs)
1380 fmsg = ": reply handle does not match the call handle"
1381 self.test(rhandle == chandle, tmsg + "handle", failmsg=fmsg)
1382 fmsg = ": reply RDMA offset (%s) should be equal to call RDMA offset (%s)" % (roffset, coffset)
1383 self.test(roffset == coffset, tmsg + "virtual address", failmsg=fmsg)
1384 if nreply > 0 and rpcordma_r.proc == RDMA_MSG:
1385 emsg = " of zero when proc is RDMA_MSG"
1386 expr = rlength == 0
1387 else:
1388 emsg = ""
1389 expr = rlength <= clength
1390 fmsg = ": reply length (%d) should be <= call length (%d)" % (rlength, clength)
1391 self.test(expr, tmsg + "length%s" % emsg, failmsg=fmsg)
1392 self.verify_segment(RDMA_WRITE, rhandle, roffset, rlength)
1393
1394 def verify_rdma_write(self, pktcall):
1395 """Verify RDMA WRITEs whether for a reply chunk or write chunk list"""
1396 self.test_info(LINE_SEP)
1397 self.verify_io_op(pktcall)
1398 save_index = self.pktt.get_index()
1399 if pktcall and pktcall in RDMA_layers:
1400 # This packet is NFS-over-RDMA
1401 optype = pktcall.NFSop.argop
1402 rpcordma = pktcall.rpcordma
1403
1404 # Display RDMA write chunk segments in the call
1405 self.get_chunk_lists(rpcordma, rpc_const.CALL)
1406
1407 # Find the NFS reply
1408 xid = pktcall.rpc.xid
1409 match_str = "rpc.xid == %s or rpcordma.xid == %s" % (xid, xid)
1410 pktreply = self.pktt.match(match_str)
1411 self.pktt.rewind(save_index)
1412
1413 tsize = 0
1414 ck_args = {}
1415 nio_args = {}
1416 if optype in self.nfs_op(**NFSread) and len(rpcordma.writes):
1417 ck_args = {"nwrites":1}
1418 elif rpcordma.reply is not None:
1419 ck_args = {"nreply":1}
1420 if pktreply is not None and pktreply.rpcordma.reply:
1421 tsize = sum([x.length for x in pktreply.rpcordma.reply.target])
1422 if tsize > 0:
1423 nio_args = {"rdmaproc":RDMA_NOMSG}
1424
1425 # Verify the RDMA chunk lists for the call
1426 self.verify_chunk_lists(rpcordma, optype, rpc_const.CALL, **ck_args)
1427
1428 # Verify NFS operation for the reply
1429 self.verify_io_op(pktreply, optype, rpc_const.REPLY, **nio_args)
1430
1431 if len(ck_args):
1432 # NFS operation using the write chunk list or reply chunk
1433 self.verify_chunk_in_reply(pktcall.rpcordma, pktreply, optype, **ck_args)
1434 if rpcordma.reply is not None and tsize == 0:
1435 # The RDMA segment length is zero, thus there are no
1436 # RDMA WRITEs and the reply is sent over SEND_Only
1437 self.verify_sends(pktreply, optype, rpc_const.REPLY, chunkverify=False)
1438 else:
1439 # NFS call does not have a write chunk list or reply chunk
1440 # so verify the reply is sent using SEND operations
1441 self.verify_sends(pktreply, optype, rpc_const.REPLY, rpcordma)
1442 self.pktt.rewind(save_index)
1443
1444 def verify_rdma_reply_chunk(self, nfs_iotype):
1445 """Verify RDMA reply chunk for the given NFS operation"""
1446 # Get correct list of NFSv4 operations or NFSv3 procedures
1447 op_list = self.get_nfs_ops(nfs_iotype)
1448 save_index = self.pktt.get_index()
1449
1450 match_str = "nfs.argop in %s" % (op_list,)
1451 while self.pktt.match(match_str):
1452 nfsop = self.pktt.pkt.NFSop
1453 if self.nfs_version >= 4 and nfsop.argop == OP_GETATTR:
1454 if FATTR4_ACL not in nfsop.attributes:
1455 # Skip all GETATTR packets with no ACL
1456 continue
1457 self.verify_rdma_write(self.pktt.pkt)
1458
1459 self.pktt.rewind(save_index)
1460
1461 def verify_nfs_read(self):
1462 """Verify NFS read is sent over RDMA"""
1463 self.find_nfs_op(self.nfs_op(OP_READ, NFSPROC3_READ), call_only=True)
1464 if self.pktcall:
1465 # Verify RDMA WRITEs for the write chunk list
1466 self.verify_rdma_write(self.pktcall)
1467 self.nfs_read_count += 1
1468 elif self.nfs_read_count == 0:
1469 # Fail only when there are no NFS reads in the packet trace
1470 self.test(False, "NFS READ call should be sent to server")
1471 return bool(self.pktcall)
1472
1473 def verify_nfs_write(self):
1474 """Verify NFS write is sent over RDMA"""
1475 start_index = self.pktt.get_index()
1476 write_count = len(self.nfs_write_xid)
1477 # Search for the NFS write or an RPCoRDMA layer having a read chunk
1478 # list. If the NFS write is matched is because it is small enough to
1479 # use RDMA SENDs instead since the RPCoRDMA layer having the read
1480 # chunk list must come before the NFS write call has been reassembled
1481 mstr1 = "nfs.argop == %d" % self.nfs_op(OP_WRITE, NFSPROC3_WRITE)
1482 mstr2 = "len(rpcordma.reads) > 0"
1483 match_str = "%s or (%s)" % (mstr1, mstr2)
1484
1485 # Search for the first NFS write or the start of an NFS write which
1486 # has not already been processed
1487 rpcordma = None
1488 pkt_rdma_msg = None
1489 while self.pktt.match(match_str):
1490 pkt = self.pktt.pkt
1491 if pkt == "rpc":
1492 xid = pkt.rpc.xid
1493 else:
1494 xid = pkt.rpcordma.xid
1495 if self.nfs_write_xid.get(xid):
1496 # This NFS write has already been processed
1497 continue
1498 else:
1499 # Found next NFS write to process
1500 pkt_rdma_msg = pkt
1501 self.nfs_write_xid[xid] = 1
1502 break
1503 save_index = self.pktt.get_index()
1504 if pkt_rdma_msg:
1505 self.test_info("="*80)
1506 if pkt_rdma_msg and pkt_rdma_msg.NFSop is None:
1507 # NFSop is None so an RPCoRDMA layer with a read chunk list
1508 # was matched -- verify NFS write is sent using RDMA READs
1509 rpcordma = pkt_rdma_msg.rpcordma
1510 self.dprint('DBG2', "Found RPC-over-RDMA packet with %s and a read chunk list" % rpcordma.proc)
1511 self.get_chunk_lists(rpcordma, rpc_const.CALL)
1512 self.verify_chunk_lists(rpcordma, OP_WRITE, rpc_const.CALL, nreads=1)
1513
1514 # Get and verify all read chunk segments
1515 for readobj in rpcordma.reads:
1516 self.pktt.rewind(save_index)
1517 self.verify_segment(RDMA_READ, readobj.handle, readobj.offset, readobj.length)
1518
1519 # Search for the NFS write
1520 self.pktt.rewind(pkt_rdma_msg.record.index)
1521 # Match the XID as well to make sure the correct NFS WRITE is matched
1522 mxidstr = "rpc.xid == 0x%08x" % rpcordma.xid
1523 self.find_nfs_op(self.nfs_op(OP_WRITE, NFSPROC3_WRITE), match=mxidstr, call_only=True)
1524 writecall = self.pktt.pkt
1525 self.verify_io_op(writecall, OP_WRITE, rpc_const.CALL, isrpcordma=False)
1526 elif pkt_rdma_msg and pkt_rdma_msg.NFSop is not None:
1527 # The NFS write was matched so verify NFS write is sent using
1528 # RDMA SENDs instead
1529 writecall = pkt_rdma_msg
1530 self.verify_io_op(writecall, OP_WRITE, rpc_const.CALL)
1531 # Index of NFS write call
1532 write_index = writecall.record.index
1533 self.pktt.rewind(start_index)
1534 self.verify_sends(writecall, OP_WRITE, rpc_const.CALL)
1535 self.pktt.rewind(write_index+1)
1536 elif write_count == 0:
1537 # No NFS writes has been processed so far thus fail the test
1538 # because there is not a single NFS write or RPCoRDMA with a
1539 # read chunk list in the packet trace
1540 self.test(False, "RPC-over-RDMA packet should be sent with RDMA_MSG and a read chunk list")
1541
1542 # Result is True if an NFS write was processed
1543 result = len(self.nfs_write_xid) > write_count
1544 if result and writecall:
1545 # Find the NFS reply and verify it
1546 xid = writecall.rpc.xid
1547 match_str = "rpc.xid == %s or rpcordma.xid == %s" % (xid, xid)
1548 self.pktt.rewind(writecall.record.index+1)
1549 writereply = self.pktt.match(match_str)
1550
1551 if rpcordma:
1552 # NFS Write using RDMA writes, set expected procedure and nreply
1553 # using the RPC-over-RDMA layer of the call
1554 rproc = RDMA_NOMSG if rpcordma.reply else RDMA_MSG
1555 nreply = 1 if rpcordma.reply else 0
1556 elif writecall and writecall.rpcordma.reply:
1557 # NFS Write using RDMA sends and reply chunk is in call
1558 rproc = RDMA_NOMSG
1559 nreply = 1
1560 else:
1561 # No reply chunk in call
1562 rproc = RDMA_MSG
1563 nreply = 0
1564 # Make sure to use correct proc when the reply chunk is empty
1565 if writereply and writereply.rpcordma and writereply.rpcordma.reply:
1566 if sum([x.length for x in writereply.rpcordma.reply.target]) == 0:
1567 # Reply chunk is empty -- no RDMA transfer
1568 rproc = RDMA_MSG
1569 self.verify_io_op(writereply, OP_WRITE, rpc_const.REPLY, rdmaproc=rproc)
1570 rpcordma = writecall.rpcordma if rpcordma is None else rpcordma
1571 self.pktt.rewind(save_index)
1572 self.verify_chunk_in_reply(rpcordma, writereply, OP_WRITE, nreply=nreply)
1573
1574 self.pktt.rewind(save_index)
1575 return result
1576
1577 def verify_nfs_over_rdma(self):
1578 """Verify all NFS packets are sent over RDMA"""
1579 count = 0 # The number of NFS packets in packet trace
1580 ibcount = 0 # The number of NFS-over-RDMA packets in trace (IB)
1581 iwcount = 0 # The number of NFS-over-RDMA packets in trace (iWarp)
1582 save_index = self.pktt.get_index()
1583 self.pktt.rewind(0)
1584 # Get all NFS packets
1585 while self.pktt.match("nfs.op > 0"):
1586 count += 1
1587 if self.pktt.pkt == "ib":
1588 ibcount += 1
1589 elif self.pktt.pkt == "rdmap":
1590 iwcount += 1
1591 if iwcount > ibcount:
1592 self.isiwarp = True
1593 self.dprint('DBG2', "NFS packets: %d" % count)
1594 if ibcount > 0 or iwcount == 0:
1595 self.dprint('DBG2', "NFS-over-RDMA packets: %4d (IB)" % ibcount)
1596 if iwcount > 0 or ibcount == 0:
1597 self.dprint('DBG2', "NFS-over-RDMA packets: %4d (iWarp)" % iwcount)
1598 rcount = ibcount + iwcount
1599 if count > 0:
1600 # Verify all NFS packets are sent over RDMA
1601 fmsg = ", NFS-over-RDMA packets(%d) != NFS packets(%d)" % (rcount, count)
1602 amsg = "All NFS packets should be sent over RDMA"
1603 self.test(rcount == count, amsg, failmsg=fmsg)
1604 self.pktt.rewind(save_index)
1605
1606 def test_rdma_io(self, nfs_iotype, filesizes=None):
1607 """Verify NFS-over-RDMA functionality"""
1608 try:
1609 self.trace_start()
1610 self.mount()
1611 filelist = []
1612
1613 if filesizes is not None:
1614 # Argument filesizes is only used for testing NFS reads or
1615 # NFS writes -- the argument could be either a list, a tuple
1616 # or an integer (if using a single file)
1617 if isinstance(filesizes, (list, tuple)):
1618 filesize_list = filesizes
1619 else:
1620 # Single file size -- convert it into a list
1621 filesize_list = [filesizes]
1622 for fsize in filesize_list:
1623 # For each file size, convert size into an integer if any
1624 # size is given as a string with units
1625 fsize = int_units(fsize)
1626 try:
1627 fmsg = ""
1628 expr = True
1629 if nfs_iotype == NFS_READ:
1630 data = self.read_file(size=fsize)
1631 elif nfs_iotype == NFS_WRITE:
1632 self.create_file(size=fsize, verbose=1, dlevels=["DBG2"])
1633 filelist.append((self.filename, fsize))
1634 except OSError as error:
1635 expr = False
1636 err = error.errno
1637 fmsg = ", got error [%s] %s" % (errno.errorcode.get(err,err), os.strerror(err))
1638 if nfs_iotype == NFS_READ:
1639 self.test(expr, "File should be read correctly", failmsg=fmsg)
1640 doffset, mdata, edata = self.compare_data(data)
1641 self.test(doffset is None, "Data read from file using RDMA mount is correct")
1642 elif nfs_iotype == NFS_WRITE:
1643 self.test(expr, "File should be created", failmsg=fmsg)
1644 if not expr:
1645 return
1646 elif nfs_iotype == NFS_GETACL:
1647 # Verify RDMA reply chunk for a GETATTR asking for ACLs
1648 self.filename = self.read_file_dict.get(self.get_file_size()[0])
1649 self.absfile = self.abspath(self.filename)
1650 self.run_cmd("nfs4_getfacl " + self.absfile)
1651 elif nfs_iotype == NFS_READDIR:
1652 # Verify RDMA reply chunk while reading the contents of
1653 # the directories specified for this test
1654 for dirname in self.testdir_list:
1655 dirpath = self.abspath(dirname)
1656 self.dprint('DBG2', "Reading directory [%s]" % dirpath)
1657 os.listdir(dirpath)
1658 elif nfs_iotype == NFS_READLINK:
1659 # Verify RDMA reply chunk by reading the contents of a
1660 # symbolic link -- which file it is pointing to
1661 symlink = self.abspath(self.symlink_list[0])
1662 self.dprint('DBG2', "Reading symbolic link [%s]" % symlink)
1663 os.readlink(symlink)
1664 elif nfs_iotype == NFS_EXCHANGE_ID:
1665 # Verify RDMA reply chunk on the EXCHANGE_ID or SETCLIENTID
1666 self.read_file(size=self.get_file_size()[0])
1667 elif nfs_iotype == NFS_BASIC:
1668 # Just generate some traffic for this test
1669 dirpath = self.abspath(self.testdir_list[0])
1670 self.dprint('DBG2', "Reading directory [%s]" % dirpath)
1671 os.listdir(self.abspath(self.testdir_list[0]))
1672
1673 symlink = self.abspath(self.symlink_list[0])
1674 self.dprint('DBG2', "Reading symbolic link [%s]" % symlink)
1675 os.readlink(symlink)
1676
1677 for fsize in self.basic_size_list:
1678 self.read_file(size=fsize)
1679 self.create_file(size=fsize)
1680 except Exception:
1681 self.test(False, traceback.format_exc())
1682 return
1683 finally:
1684 self.umount()
1685 self.trace_stop()
1686
1687 if len(filelist):
1688 try:
1689 self.test_info("Compare file data using non-RDMA mount")
1690 self.mount(proto="tcp", port=2049)
1691 for self.filename, fsize in filelist:
1692 self.absfile = self.abspath(self.filename)
1693 self.verify_file_data("Data read from file using non-RDMA mount is correct", filesize=fsize)
1694 except Exception:
1695 self.test(False, traceback.format_exc())
1696 finally:
1697 self.umount()
1698
1699 try:
1700 self.trace_open()
1701 # Use buffered matching -- get all NFS and RDMA packets
1702 self.set_pktlist(layer="ib,rdmap,nfs")
1703 self.get_iosize() # Get the RDMA I/O size if possible
1704 self.get_maxresponsesize() # Get the session maximum response size
1705
1706 # Verify all NFS packets are sent over RDMA
1707 self.verify_nfs_over_rdma()
1708
1709 if nfs_iotype == NFS_READ:
1710 # Verify NFS read is sent over RDMA using either RDMA WRITEs
1711 # or SENDs
1712 self.nfs_read_count = 0
1713 while self.verify_nfs_read(): pass
1714 elif nfs_iotype == NFS_WRITE:
1715 # Verify NFS write is sent over RDMA using either RDMA READs
1716 # or SENDs
1717 self.nfs_write_xid = {}
1718 while self.verify_nfs_write(): pass
1719 elif nfs_iotype != NFS_BASIC:
1720 # Verify RDMA reply chunk on the NFS operation other
1721 # than NFS read or write
1722 self.verify_rdma_reply_chunk(nfs_iotype)
1723 except Exception:
1724 self.test(False, traceback.format_exc())
1725 finally:
1726 self.pktt.close()
1727
1728 def basic01_test(self):
1729 """Verify basic NFS-over-RDMA functionality"""
1730 self.test_group(self.test_description())
1731 self.test_rdma_io(NFS_BASIC)
1732
1733 def basic02_test(self):
1734 """Verify NFS-over-RDMA reply chunk on EXCHANGE_ID/SETCLIENTID"""
1735 self.test_group(self.test_description())
1736 self.test_rdma_io(NFS_EXCHANGE_ID)
1737
1738 def basic03_test(self):
1739 """Verify NFS-over-RDMA reply chunk on READDIR"""
1740 self.test_group(self.test_description())
1741 self.test_rdma_io(NFS_READDIR)
1742
1743 def basic04_test(self):
1744 """Verify NFS-over-RDMA reply chunk on READLINK"""
1745 self.test_group(self.test_description())
1746 self.test_rdma_io(NFS_READLINK)
1747
1748 def basic05_test(self):
1749 """Verify NFS-over-RDMA reply chunk on GETATTR(FATTR4_ACL)"""
1750 self.test_group(self.test_description())
1751 self.test_rdma_io(NFS_GETACL)
1752
1753 def read01_test(self):
1754 """Verify NFS-over-RDMA functionality on a file opened for reading (very small file)"""
1755 self.test_group(self.test_description())
1756 self.test_rdma_io(NFS_READ, filesizes=self.get_file_size())
1757
1758 def read02_test(self):
1759 """Verify NFS-over-RDMA functionality on a file opened for reading (small file)"""
1760 self.test_group(self.test_description())
1761 self.test_rdma_io(NFS_READ, filesizes=self.get_file_size())
1762
1763 def read03_test(self):
1764 """Verify NFS-over-RDMA functionality on a file opened for reading (medium file)"""
1765 self.test_group(self.test_description())
1766 self.test_rdma_io(NFS_READ, filesizes=self.get_file_size())
1767
1768 def read04_test(self):
1769 """Verify NFS-over-RDMA functionality on a file opened for reading (large file)"""
1770 self.test_group(self.test_description())
1771 self.test_rdma_io(NFS_READ, filesizes=self.get_file_size())
1772
1773 def write01_test(self):
1774 """Verify NFS-over-RDMA functionality on a file opened for writing (very small file)"""
1775 self.test_group(self.test_description())
1776 self.test_rdma_io(NFS_WRITE, filesizes=self.file_size_list)
1777
1778 def write02_test(self):
1779 """Verify NFS-over-RDMA functionality on a file opened for writing (small file)"""
1780 self.test_group(self.test_description())
1781 self.test_rdma_io(NFS_WRITE, filesizes=self.small_filesize)
1782
1783 def write03_test(self):
1784 """Verify NFS-over-RDMA functionality on a file opened for writing (medium file)"""
1785 self.test_group(self.test_description())
1786 self.test_rdma_io(NFS_WRITE, filesizes=self.filesize)
1787
1788 def write04_test(self):
1789 """Verify NFS-over-RDMA functionality on a file opened for writing (large file)"""
1790 self.test_group(self.test_description())
1791 self.test_rdma_io(NFS_WRITE, filesizes=self.large_filesize)
1792
1793 ################################################################################
1794 # Entry point
1795 x = RdmaTest(usage=USAGE, testnames=TESTNAMES, testgroups=TESTGROUPS, sid=SCRIPT_ID)
1796
1797 try:
1798 x.setup()
1799
1800 # Run all the tests
1801 x.run_tests()
1802 except Exception:
1803 x.test(False, traceback.format_exc())
1804 finally:
1805 x.proto = "tcp"
1806 x.port = 2049
1807 x.cleanup()
1808 x.exit()