Debian Codegraph @

doc/blkreplay.lyx

raw
1 #LyX 2.0 created this file. For more info see http://www.lyx.org/
2 \lyxformat 413
3 \begin_document
4 \begin_header
5 \textclass scrreprt
6 \begin_preamble
7 \usepackage[dvipsnames]{xcolor}
8 \usepackage{listings}
9 \end_preamble
10 \options abstracton
11 \use_default_options true
12 \begin_modules
13 customHeadersFooters
14 enumitem
15 fixltx2e
16 \end_modules
17 \maintain_unincluded_children false
18 \language english
19 \language_package default
20 \inputencoding auto
21 \fontencoding global
22 \font_roman default
23 \font_sans default
24 \font_typewriter default
25 \font_default_family rmdefault
26 \use_non_tex_fonts false
27 \font_sc false
28 \font_osf false
29 \font_sf_scale 100
30 \font_tt_scale 100
31
32 \graphics default
33 \default_output_format default
34 \output_sync 0
35 \bibtex_command default
36 \index_command default
37 \paperfontsize 10
38 \spacing single
39 \use_hyperref true
40 \pdf_title "blkreplay"
41 \pdf_author "Thomas Schöbel-Theuer"
42 \pdf_bookmarks true
43 \pdf_bookmarksnumbered false
44 \pdf_bookmarksopen false
45 \pdf_bookmarksopenlevel 1
46 \pdf_breaklinks true
47 \pdf_pdfborder true
48 \pdf_colorlinks true
49 \pdf_backref false
50 \pdf_pdfusetitle true
51 \papersize a4paper
52 \use_geometry true
53 \use_amsmath 1
54 \use_esint 1
55 \use_mhchem 1
56 \use_mathdots 1
57 \cite_engine basic
58 \use_bibtopic false
59 \use_indices false
60 \paperorientation portrait
61 \suppress_date false
62 \use_refstyle 1
63 \index Index
64 \shortcut idx
65 \color #008000
66 \end_index
67 \leftmargin 3.7cm
68 \topmargin 2.7cm
69 \rightmargin 2.8cm
70 \bottommargin 2.3cm
71 \secnumdepth 3
72 \tocdepth 3
73 \paragraph_separation indent
74 \paragraph_indentation default
75 \quotes_language english
76 \papercolumns 1
77 \papersides 2
78 \paperpagestyle headings
79 \tracking_changes false
80 \output_changes false
81 \html_math_output 0
82 \html_css_as_file 0
83 \html_be_strict false
84 \end_header
85
86 \begin_body
87
88 \begin_layout Title
89
90 \family typewriter
91 blkreplay
92 \family default
93 and Sonar Diagrams
94 \end_layout
95
96 \begin_layout Subtitle
97 A manual for system administrators,
98 \begin_inset Newline newline
99 \end_inset
100
101 kernel developers,
102 \begin_inset Newline newline
103 \end_inset
104
105 hardware technicians,
106 \begin_inset Newline newline
107 \end_inset
108
109 and experts in IO systems
110 \end_layout
111
112 \begin_layout Author
113 Thomas Schöbel-Theuer (
114 \family typewriter
115 [email protected]
116 \family default
117 )
118 \end_layout
119
120 \begin_layout Date
121 Version 1.0.0
122 \end_layout
123
124 \begin_layout Standard
125 \noindent
126 Copyright (C) 2012 Thomas Schöbel-Theuer / 1&1 Internet AG (see
127 \begin_inset Flex URL
128 status open
129
130 \begin_layout Plain Layout
131
132 http://www.1und1.de
133 \end_layout
134
135 \end_inset
136
137 shortly called 1&1 in the following).
138 \end_layout
139
140 \begin_layout Standard
141 Permission is granted to copy, distribute and/or modify this document under
142 the terms of the GNU Free Documentation License, Version 1.3 or any later
143 version published by the Free Software Foundation; with no Invariant Sections,
144 no Front-Cover Texts, and no Back-Cover Texts.
145 A copy of the license is included in the section entitled
146 \begin_inset Quotes eld
147 \end_inset
148
149
150 \begin_inset CommandInset ref
151 LatexCommand nameref
152 reference "chap:GNU-FDL"
153
154 \end_inset
155
156
157 \begin_inset Quotes erd
158 \end_inset
159
160 .
161 \end_layout
162
163 \begin_layout Abstract
164
165 \family typewriter
166 \begin_inset ERT
167 status open
168
169 \begin_layout Plain Layout
170
171
172 \backslash
173 sloppy
174 \end_layout
175
176 \end_inset
177
178 blkreplay
179 \family default
180 is a GPL'ed toolkit, driving the block layer of Linux (or other Unix-like
181 OSes) while measuring latency and throughput of IO operations for later
182 visualization (so-called
183 \begin_inset Quotes eld
184 \end_inset
185
186 sonar diagrams
187 \begin_inset Quotes erd
188 \end_inset
189
190 and others).
191 \end_layout
192
193 \begin_layout Abstract
194
195 \family typewriter
196 blkreplay
197 \family default
198 comes with a modular and extensible
199 \series bold
200 test suite
201 \series default
202 , automating large projects for testing and/or benchmarking.
203 \end_layout
204
205 \begin_layout Abstract
206
207 \family typewriter
208 blkreplay
209 \family default
210 can be used to
211 \series bold
212 test physical hardware
213 \series default
214 , e.g.
215 compare different brands of hard disks, or RAID controllers / their settings
216 / RAID rebuild performance degradation, or to evaluate the effect of SSD
217 caching, or to compare different block level transports like iSCSI vs Fibrechan
218 nel (over different kinds of storage networks).
219 \end_layout
220
221 \begin_layout Abstract
222 It can compare
223 \series bold
224 virtual hardware
225 \series default
226 (like
227 \family typewriter
228 vmware
229 \family default
230 or
231 \family typewriter
232 XenServer
233 \family default
234 block devices, or any type of block-level
235 \series bold
236 storage virtualization
237 \series default
238 ) to each other or to physical hardware, provided the test setup is handled
239 very carefully
240 \begin_inset Foot
241 status open
242
243 \begin_layout Plain Layout
244 Otherwise you may get
245 \emph on
246 useless fake results
247 \emph default
248 measuring the
249 \emph on
250 cache
251 \emph default
252 performance or even
253 \emph on
254 sparse accesses to holes
255 \emph default
256 instead of the real hardware performance.
257 Such fake results may differ from real results by
258 \emph on
259 factors
260 \emph default
261 , er even by
262 \emph on
263 orders of magnitude
264 \emph default
265 .
266
267 \family typewriter
268 blkreplay
269 \family default
270 comes with thorough descriptions teaching you how to avoid the most common
271 pitfalls.
272 \end_layout
273
274 \end_inset
275
276 .
277 \end_layout
278
279 \begin_layout Abstract
280
281 \family typewriter
282 blkreplay
283 \family default
284 can compare
285 \series bold
286 commercial storage
287 \series default
288 systems from vendors like EMC, Net\SpecialChar \-
289 App, IBM, Hitachi, etc to each other
290 or to cheap off-the-shelf hardware (in order to determine the price/performance
291 ratio), provided the same care
292 \begin_inset Foot
293 status open
294
295 \begin_layout Plain Layout
296 Notice that most commercial storage systems
297 \series bold
298 \emph on
299 are
300 \series default
301 \emph default
302 in fact nothing but virtualized storage, so the above warnings about possible
303
304 \emph on
305 fake results
306 \emph default
307 apply.
308 \end_layout
309
310 \end_inset
311
312 is taken.
313 \end_layout
314
315 \begin_layout Abstract
316 Furthermore, it can be used for tests of the
317 \series bold
318 Linux kernel
319 \series default
320 , e.g.
321 for testing device drivers, comparing IO schedulers at different load patterns,
322 determining the overhead of Linux
323 \family typewriter
324 dm
325 \family default
326 targets or the impact of network problems to DRBD, and much more.
327 \end_layout
328
329 \begin_layout Abstract
330 In addition to artificial loads like random read-write sweeps and various
331 kinds of
332 \emph on
333 overload tests
334 \emph default
335 , it can also replay
336 \series bold
337 natural loads
338 \series default
339 which have been recorded by
340 \family typewriter
341 blktrace
342 \family default
343 at heavily-loaded production servers at big data centers.
344
345 \family typewriter
346 blkreplay
347 \family default
348 comes with a
349 \series bold
350 large collection of natural loads
351 \series default
352 from a wide spectrum of applications (such as web servers, databases, dedicated
353 servers, etc) which have been released to the public by 1&1 under GPL.
354 Some of these natural loads have recorded the real-life disk access behaviour
355 from servers serving thousands of customers in parallel.
356 Static analysis of the workingset behaviour of such natural loads is implemente
357 d.
358 \end_layout
359
360 \begin_layout Abstract
361 At 1&1,
362 \family typewriter
363 blkreplay
364 \family default
365 has even been used as a tool for root cause analysis of incidents: for
366 example, high load peaks presumably stemming from traffic jam (or other
367 sources of overload) were recorded at production sites in real time by
368
369 \family typewriter
370 blktrace
371 \family default
372 , and later replayed in the laboratory (without causing customer impact)
373 seeking for the cause of trouble, or improving the safety margins by choice
374 of better hardware.
375 \end_layout
376
377 \begin_layout Abstract
378 For experts in IO subsystems, visualization techniques like
379 \begin_inset Quotes eld
380 \end_inset
381
382 sonar diagrams
383 \begin_inset Quotes erd
384 \end_inset
385
386 can reveal (parts of) the internal structure of complex IO systems, such
387 as cache hierarchies or other hierarchical storage systems.
388 \end_layout
389
390 \begin_layout Abstract
391 As a community project under GPL,
392 \family typewriter
393 blkreplay
394 \family default
395 is open to contributions from hardware vendors, other data centers, the
396 kernel hacker community, etc.
397 \end_layout
398
399 \begin_layout Standard
400 \begin_inset CommandInset toc
401 LatexCommand tableofcontents
402
403 \end_inset
404
405
406 \end_layout
407
408 \begin_layout Chapter
409 Why Synthetic Benchmarks suck
410 \end_layout
411
412 \begin_layout Standard
413 There are a lot of benchmark tools around, like
414 \family typewriter
415 iozone
416 \family default
417 ,
418 \family typewriter
419 iometer
420 \family default
421 ,
422 \family typewriter
423 iperf
424 \family default
425 ,
426 \family typewriter
427 bonnie
428 \family default
429 (
430 \family typewriter
431 ++
432 \family default
433 ) and many others.
434 \end_layout
435
436 \begin_layout Standard
437 What do they have in
438 \emph on
439 common
440 \emph default
441
442 \begin_inset Foot
443 status open
444
445 \begin_layout Plain Layout
446 Of course, the mentioned performance measuring tools are targeted at inspection
447 of different components of an OS, such as network, filesystem layer, and
448 block IO layer.
449 Here the question is a about
450 \emph on
451 commonality
452 \emph default
453 , not differences.
454 \end_layout
455
456 \end_inset
457
458 ?
459 \end_layout
460
461 \begin_layout Quotation
462 Simply, most of them generate an
463 \series bold
464 artificial load
465 \series default
466 onto your system.
467 \end_layout
468
469 \begin_layout Standard
470 Artificial loads, as opposed to
471 \series bold
472 natural loads
473 \series default
474 , have a main disadvantage: they cannot answer questions like
475 \begin_inset Quotes eld
476 \end_inset
477
478 will my application Z run on system A reliably?
479 \begin_inset Quotes erd
480 \end_inset
481
482
483 \end_layout
484
485 \begin_layout Standard
486 \noindent
487 \align center
488 \begin_inset Tabular
489 <lyxtabular version="3" rows="3" columns="3">
490 <features tabularvalignment="middle">
491 <column alignment="center" valignment="top" width="0">
492 <column alignment="center" valignment="top" width="0">
493 <column alignment="center" valignment="top" width="0">
494 <row>
495 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
496 \begin_inset Text
497
498 \begin_layout Plain Layout
499 question
500 \end_layout
501
502 \end_inset
503 </cell>
504 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
505 \begin_inset Text
506
507 \begin_layout Plain Layout
508 artificial
509 \end_layout
510
511 \end_inset
512 </cell>
513 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
514 \begin_inset Text
515
516 \begin_layout Plain Layout
517 natural
518 \end_layout
519
520 \end_inset
521 </cell>
522 </row>
523 <row>
524 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
525 \begin_inset Text
526
527 \begin_layout Plain Layout
528 Is system A better than B for application Z?
529 \end_layout
530
531 \end_inset
532 </cell>
533 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
534 \begin_inset Text
535
536 \begin_layout Plain Layout
537 \begin_inset Formula $partly$
538 \end_inset
539
540
541 \end_layout
542
543 \end_inset
544 </cell>
545 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
546 \begin_inset Text
547
548 \begin_layout Plain Layout
549 \begin_inset Formula $\sqrt{}$
550 \end_inset
551
552
553 \end_layout
554
555 \end_inset
556 </cell>
557 </row>
558 <row>
559 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
560 \begin_inset Text
561
562 \begin_layout Plain Layout
563 Does application Z run on system A?
564 \end_layout
565
566 \end_inset
567 </cell>
568 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
569 \begin_inset Text
570
571 \begin_layout Plain Layout
572 -
573 \end_layout
574
575 \end_inset
576 </cell>
577 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
578 \begin_inset Text
579
580 \begin_layout Plain Layout
581 \begin_inset Formula $\sqrt{}$
582 \end_inset
583
584
585 \end_layout
586
587 \end_inset
588 </cell>
589 </row>
590 </lyxtabular>
591
592 \end_inset
593
594
595 \end_layout
596
597 \begin_layout Standard
598 \noindent
599 It seems that some people
600 \emph on
601 believe
602 \emph default
603 that synthetic benchmarks can be used even at the position of the dash
604 in the above table.
605 These people are wrong.
606 \end_layout
607
608 \begin_layout Standard
609 Experiences at big data centers at 1&1 show that sometimes the differences
610 between results from artificial benchmarks and real-world application behaviour
611 are very large.
612 We found cases where artificial benchmarks (adjusting parameters like IOPS)
613 suggested that a particular application
614 \emph on
615 should
616 \emph default
617 run on a particular hardware system, but the real application
618 \emph on
619 didn't
620 \emph default
621 : after deployment, a systematic series of incidents
622 \emph on
623 disproved
624 \emph default
625
626 \begin_inset Foot
627 status open
628
629 \begin_layout Plain Layout
630 Sometimes we got results in the other direction: artificial benchmarks suggested
631 that a particular application would
632 \emph on
633 not
634 \emph default
635 run, but in reality it
636 \emph on
637 did
638 \emph default
639 run.
640 \end_layout
641
642 \end_inset
643
644 the validity of the former benchmark results for the
645 \emph on
646 originally intended statement
647 \emph default
648 .
649 The failed prediction from the artificial benchmarks led to a
650 \series bold
651 failed invest
652 \series default
653 .
654 \end_layout
655
656 \begin_layout Standard
657 What can we do about that?
658 \end_layout
659
660 \begin_layout Standard
661 Obviously, parameters like IOPS (even when enriched with attributes like
662
663 \begin_inset Quotes eld
664 \end_inset
665
666 average IOPS
667 \begin_inset Quotes erd
668 \end_inset
669
670 or
671 \begin_inset Quotes eld
672 \end_inset
673
674 peak IOPS
675 \begin_inset Quotes erd
676 \end_inset
677
678 ) are
679 \emph on
680 not
681 \emph default
682 representative for description of the real-world behaviour of applications.
683 Attempts to describe real-world behaviour in mathematical terms of analytic
684 functions have been already made in the 1970's; they failed.
685 All such models can
686 \emph on
687 try
688 \emph default
689 to describe an
690 \emph on
691 approximation
692 \emph default
693 of real-world behaviour, if enough knowledge about the application
694 \emph on
695 would
696 \emph default
697 be available.
698 \end_layout
699
700 \begin_layout Standard
701 So why trying to deal with tools that never can fully describe real-world
702 application behaviour, when there
703 \emph on
704 exist
705 \emph default
706 tools which definitely
707 \emph on
708 can
709 \emph default
710 do?
711 \end_layout
712
713 \begin_layout Standard
714 One of them is
715 \family typewriter
716 blkreplay
717 \family default
718 .
719 \end_layout
720
721 \begin_layout Chapter
722 How
723 \family typewriter
724 blkreplay
725 \family default
726 works
727 \end_layout
728
729 \begin_layout Section
730 Principle
731 \end_layout
732
733 \begin_layout Standard
734 In some sense,
735 \family typewriter
736 blkreplay
737 \family default
738 is just the opposite of the well-known Linux kernel tool
739 \family typewriter
740 blktrace
741 \family default
742 : recordings made by
743 \family typewriter
744 blktrace
745 \family default
746 are simply replayed on
747 \emph on
748 another
749 \emph default
750 block device.
751 \end_layout
752
753 \begin_layout Standard
754 A
755 \family typewriter
756 blktrace
757 \family default
758 record of an IO request contains the following information:
759 \end_layout
760
761 \begin_layout Enumerate
762 timestamp of the IO request (nanosecond resolution)
763 \end_layout
764
765 \begin_layout Enumerate
766 position of the IO request (sector#)
767 \end_layout
768
769 \begin_layout Enumerate
770 length of the IO request (#sectors)
771 \end_layout
772
773 \begin_layout Enumerate
774 direction: R[ead] or W[write]
775 \end_layout
776
777 \begin_layout Standard
778 Notice that
779 \family typewriter
780 blktrace
781 \family default
782 records do
783 \emph on
784 not
785 \emph default
786 contain any data.
787 Therefore,
788 \family typewriter
789 blkreplay
790 \family default
791 must later generate some
792 \emph on
793 fake data
794 \emph default
795 in order to repeat the timely and positionly behaviour of the original
796 recording.
797 By default, NULL blocks enriched with some internal header information
798 are generated.
799 The internal headers may be used for
800 \emph on
801 verification
802 \emph default
803 of the correctness of IO semantics, either by immediate re-read after each
804 write, or in a separate verify pass after the end of an ordinary
805 \family typewriter
806 blkreplay
807 \family default
808 run.
809 \end_layout
810
811 \begin_layout Standard
812 Notice that these NULL blocks will (together with the internal header informatio
813 n)
814 \series bold
815 destroy
816 \series default
817 any previous content (such as filesystem data) on your block device!
818 \end_layout
819
820 \begin_layout Standard
821 Therefore,
822 \emph on
823 never
824 \emph default
825 use
826 \family typewriter
827 blkreplay
828 \family default
829 on production systems.
830 \end_layout
831
832 \begin_layout Labeling
833 \labelwidthstring 00.00.0000
834 \begin_inset Graphics
835 filename images/MatieresToxiques.png
836 lyxscale 50
837 scale 17
838
839 \end_inset
840
841
842 \emph on
843 Always
844 \emph default
845 use
846 \family typewriter
847 blkreplay
848 \family default
849 in the laboratory, always on devices which don't contain any valuable data!
850 \end_layout
851
852 \begin_layout Labeling
853 \labelwidthstring 00.00.0000
854 \begin_inset Graphics
855 filename images/MatieresToxiques.png
856 lyxscale 50
857 scale 17
858
859 \end_inset
860
861 Running
862 \family typewriter
863 blkreplay
864 \family default
865
866 \emph on
867 in parallel
868 \emph default
869 to mounted filesystems on the same device
870 \begin_inset Foot
871 status open
872
873 \begin_layout Plain Layout
874 Although a single Linux kernel instance tries to probihit such a disaster,
875 there are cases where you can
876 \begin_inset Quotes eld
877 \end_inset
878
879 achieve
880 \begin_inset Quotes erd
881 \end_inset
882
883 that effect.
884 Examples are iSCSI connections to the same iSCSI target in parallel.
885 \end_layout
886
887 \end_inset
888
889 will certainly destroy your data and almost certainly crash your kernel.
890 Always run at most a single
891 \family typewriter
892 blkreplay
893 \family default
894 instance on a single device!
895 \end_layout
896
897 \begin_layout Labeling
898 \labelwidthstring 00.00.0000
899 \begin_inset Graphics
900 filename images/MatieresCorrosives.png
901 lyxscale 50
902 scale 17
903
904 \end_inset
905
906 In general,
907 \family typewriter
908 blkreplay
909 \family default
910 is a tool only for
911 \emph on
912 experienced
913 \emph default
914 technicians who know what they do.
915 They should be at least at a senior level.
916 \end_layout
917
918 \begin_layout Section
919 Architecture of
920 \family typewriter
921 blkreplay
922 \end_layout
923
924 \begin_layout Standard
925 The main challenge for
926 \family typewriter
927 blkreplay
928 \family default
929 is to generate
930 \series bold
931 sufficient IO parallelism
932 \series default
933 .
934 \end_layout
935
936 \begin_layout Standard
937 Ordinary production systems like web servers are serving many thousands
938 of HTTP requests per second, which may lead to an IO parallelism at the
939 block level of several hundred outstanding IO requests at the same time.
940 \end_layout
941
942 \begin_layout Standard
943 In order to simulate such a behaviour in the lab, there are principally
944 two alternatives:
945 \end_layout
946
947 \begin_layout Enumerate
948 use the
949 \family typewriter
950 aio
951 \family default
952 interface of the kernel to fire off a large number of IO requests in parallel.
953 \end_layout
954
955 \begin_layout Enumerate
956 use a sufficiently large number of kernel threads or processes in parallel,
957 where each of them fires up only at most one IO request at the same time
958 (blocking IO).
959 \end_layout
960
961 \begin_layout Standard
962 The current version of
963 \family typewriter
964 blkreplay
965 \family default
966 supports only method 2; method 1 is planned for a future release.
967 \end_layout
968
969 \begin_layout Standard
970 Method 2 is motivated by a typical Apache behaviour: it is almost a
971 \begin_inset Quotes eld
972 \end_inset
973
974 fork bomb
975 \begin_inset Quotes erd
976 \end_inset
977
978 , in particular under high connection rates and slow block devices.
979 A high number of ordinary Unix processes is doing conventional
980 \family typewriter
981 read()
982 \family default
983 /
984 \family typewriter
985 write()
986 \family default
987 operations in parallel.
988 \end_layout
989
990 \begin_layout Standard
991 For easy portability even to historic Unix flavours,
992 \family typewriter
993 blkreplay
994 \family default
995 uses ordinary Unix processes generated by simple
996 \family typewriter
997 fork()
998 \family default
999 s and communicating via anonymous pipes, in favour to a shared-memory
1000 \family typewriter
1001 pthreads
1002 \family default
1003 model.
1004 However, future versions may also support
1005 \family typewriter
1006 pthreads
1007 \family default
1008 .
1009 \end_layout
1010
1011 \begin_layout Standard
1012 \noindent
1013 \align center
1014 \begin_inset Graphics
1015 filename architecture.fig
1016 width 75col%
1017
1018 \end_inset
1019
1020
1021 \end_layout
1022
1023 \begin_layout Section
1024 Mode of Operation
1025 \end_layout
1026
1027 \begin_layout Labeling
1028 \labelwidthstring 00.00.0000
1029 \begin_inset Graphics
1030 filename images/MatieresCorrosives.png
1031 lyxscale 50
1032 scale 17
1033
1034 \end_inset
1035
1036 It is
1037 \series bold
1038 essential
1039 \series default
1040 that you understand the concepts described in this chapter.
1041 Otherwise you may produce
1042 \series bold
1043 useless fake results
1044 \series default
1045 , deviating from valid measurements by
1046 \emph on
1047 factors
1048 \emph default
1049 , or even
1050 \emph on
1051 orders of magnitude
1052 \emph default
1053 .
1054 \end_layout
1055
1056 \begin_layout Standard
1057 It is crucial to understand the
1058 \emph on
1059 operating environment
1060 \emph default
1061 where
1062 \family typewriter
1063 blkreplay
1064 \family default
1065 is running.
1066 Please take a look at the following picture:
1067 \end_layout
1068
1069 \begin_layout Standard
1070 \noindent
1071 \align center
1072 \begin_inset Graphics
1073 filename io-layers.fig
1074 width 50text%
1075
1076 \end_inset
1077
1078
1079 \end_layout
1080
1081 \begin_layout Standard
1082 \noindent
1083 In the live system,
1084 \family typewriter
1085 blktrace
1086 \family default
1087 will
1088 \begin_inset Quotes eld
1089 \end_inset
1090
1091 listen
1092 \begin_inset Quotes erd
1093 \end_inset
1094
1095 to the events occurring at point A, and will record them.
1096 In this example, we have a complex system, running virtual machines inside
1097 a hypervisor.
1098 \end_layout
1099
1100 \begin_layout Standard
1101 Notice that all IO requests of the application will not only go through
1102 the filesystem, but also through the buffer / page cache.
1103 The in-memory cache will usually serve most IO requests from the application,
1104 without causing physical IO at the block layer (so-called
1105 \begin_inset Quotes eld
1106 \end_inset
1107
1108 cache hits
1109 \begin_inset Quotes erd
1110 \end_inset
1111
1112 ).
1113 \end_layout
1114
1115 \begin_layout Standard
1116 On some well-tuned production servers, it is no problem to achieve cache
1117 hit rates of 99% or more, leading to a kind of
1118 \begin_inset Quotes eld
1119 \end_inset
1120
1121 gear ratio
1122 \begin_inset Quotes erd
1123 \end_inset
1124
1125 of 1:100, or even 1:1000 (in long-term runs).
1126 Of course, there also exist heavy workloads running on thin servers, where
1127 sometimes less than 1:10 can be achieved.
1128 Even in that case, there will be
1129 \emph on
1130 always
1131 \emph default
1132 some cache hits, for example caused by metadata requests from the filesystem.
1133 In practice, the cache hit rate will never go down to 0%.
1134 Notice that these inevitable cache effects are
1135 \emph on
1136 already included
1137 \emph default
1138 in any
1139 \family typewriter
1140 blktrace
1141 \family default
1142 !
1143 \end_layout
1144
1145 \begin_layout Standard
1146 Now look at the situation in the laboratory.
1147 The application and the filesystem is no longer present, but its
1148 \emph on
1149 effects
1150 \emph default
1151 are simulated by
1152 \family typewriter
1153 blkreplay
1154 \family default
1155 .
1156 Due to the architecture of the Linux kernel, all IO requests will continue
1157 to run through the buffer cache
1158 \begin_inset Foot
1159 status open
1160
1161 \begin_layout Plain Layout
1162 Several commercial Unices used a concept called
1163 \begin_inset Quotes eld
1164 \end_inset
1165
1166 raw device
1167 \begin_inset Quotes erd
1168 \end_inset
1169
1170 which circumvented their buffer cache.
1171 In contrast, the internal structures of Linux device driver are internally
1172 interwoven with the page cache in a rather sticky way.
1173 Instead of
1174 \begin_inset Quotes eld
1175 \end_inset
1176
1177 raw devices
1178 \begin_inset Quotes erd
1179 \end_inset
1180
1181 , Linux uses the concept of
1182 \begin_inset Quotes eld
1183 \end_inset
1184
1185 direct IO
1186 \begin_inset Quotes erd
1187 \end_inset
1188
1189 , which
1190 \emph on
1191 tries
1192 \emph default
1193 to minimize any caching effects.
1194 \end_layout
1195
1196 \end_inset
1197
1198 .
1199 \end_layout
1200
1201 \begin_layout Standard
1202 It should be clear by the very nature of our experiment, that at measuring
1203 point B
1204 \emph on
1205 exactly
1206 \emph default
1207 the same events
1208 \emph on
1209 should
1210 \emph default
1211 happen as had been formerly observed / recorded at point A.
1212 \end_layout
1213
1214 \begin_layout Standard
1215 Thus, the page / buffer cache of the laboratory system
1216 \emph on
1217 must
1218 \emph default
1219 be switched off.
1220 Otherwise, a
1221 \begin_inset Quotes eld
1222 \end_inset
1223
1224 gear ratio
1225 \begin_inset Quotes erd
1226 \end_inset
1227
1228 of 1:10 (or let it be only 1:1.1) would lead to distortions of the measurement
1229 results.
1230 In order to switch off the page / buffer cache,
1231 \family typewriter
1232 blkreplay
1233 \family default
1234 uses
1235 \family typewriter
1236 O_DIRECT
1237 \family default
1238 mode as offered by the Linux kernel.
1239 \end_layout
1240
1241 \begin_layout Standard
1242 Notice that even by these measures, there may remain some subtle differences
1243 between the operations occurring at point A and point B.
1244 The block layer of the Linux kernel does some optimizations, for example
1245 it tries to
1246 \emph on
1247 coalesce
1248 \emph default
1249 adjacent requests, or to
1250 \emph on
1251 split
1252 \emph default
1253 some requests, depending on the capabilities of the hardware.
1254 Usually, these are minor modifications, occurring at less than 1% of all
1255 requests.
1256 However, keep in mind (and check) these effects.
1257 \end_layout
1258
1259 \begin_layout Labeling
1260 \labelwidthstring 00.00.0000
1261 \begin_inset Graphics
1262 filename images/MatieresCorrosives.png
1263 lyxscale 50
1264 scale 17
1265
1266 \end_inset
1267
1268 The
1269 \series bold
1270 elevator strategy
1271 \series default
1272 (aka IO scheduler) of the Linux block layer is a bigger influence factor.
1273 It can
1274 \emph on
1275 reorder
1276 \emph default
1277 requests, and it can even add
1278 \emph on
1279 artificial latencies
1280 \emph default
1281 .
1282 For example,
1283 \family typewriter
1284 CFQ
1285 \family default
1286 adds some speculative latencies in some cases to increase the chances for
1287 request mergers.
1288 Selecting the
1289 \begin_inset Quotes eld
1290 \end_inset
1291
1292 wrong
1293 \begin_inset Foot
1294 status open
1295
1296 \begin_layout Plain Layout
1297 Many sources from the internet claim that
1298 \family typewriter
1299 CFQ
1300 \family default
1301 is the
1302 \begin_inset Quotes eld
1303 \end_inset
1304
1305 best
1306 \begin_inset Quotes erd
1307 \end_inset
1308
1309 IO scheduler.
1310 While this is often true for typical workstation load pattern (and interactive
1311 user expectations), our experience at 1&1 is different with regards to
1312
1313 \emph on
1314 server
1315 \emph default
1316 loads.
1317 Check yourself! Run some
1318 \family typewriter
1319 blkreplay
1320 \family default
1321 comparisons between different IO schedulers.
1322 Hint: there may be even non-linear dependencies from the lower level, whether
1323 you have some RAID with BBU cache, or not.
1324 \end_layout
1325
1326 \end_inset
1327
1328
1329 \begin_inset Quotes erd
1330 \end_inset
1331
1332 scheduler may lead to larger distortions, even to seemingly
1333 \begin_inset Quotes eld
1334 \end_inset
1335
1336 bad
1337 \begin_inset Quotes erd
1338 \end_inset
1339
1340 behaviour (which is
1341 \emph on
1342 not
1343 \emph default
1344 the fault of bad hardware).
1345 In order to really get (almost) the same behaviour at points A and B, you
1346
1347 \emph on
1348 must
1349 \emph default
1350 select the
1351 \family typewriter
1352 NOOP
1353 \family default
1354 or at least the
1355 \family typewriter
1356 DEADLINE
1357 \family default
1358 scheduler.
1359 See
1360 \family typewriter
1361 /sys/dev/block/*/queue/scheduler
1362 \family default
1363 .
1364 \end_layout
1365
1366 \begin_layout Section
1367 Overlapping of IO Requests
1368 \end_layout
1369
1370 \begin_layout Standard
1371 In general, there are two kinds of overlapping between IO requests in real
1372 production systems (at the time when a
1373 \family typewriter
1374 blktrace
1375 \family default
1376 record is made):
1377 \end_layout
1378
1379 \begin_layout Enumerate
1380 timely
1381 \end_layout
1382
1383 \begin_layout Enumerate
1384 positionly
1385 \end_layout
1386
1387 \begin_layout Standard
1388 Both kinds form a two-dimensional space:
1389 \end_layout
1390
1391 \begin_layout Standard
1392 \align center
1393 \begin_inset Tabular
1394 <lyxtabular version="3" rows="3" columns="3">
1395 <features tabularvalignment="middle">
1396 <column alignment="center" valignment="top" width="0">
1397 <column alignment="center" valignment="top" width="0">
1398 <column alignment="center" valignment="top" width="0">
1399 <row>
1400 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1401 \begin_inset Text
1402
1403 \begin_layout Plain Layout
1404 overlapping
1405 \end_layout
1406
1407 \end_inset
1408 </cell>
1409 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1410 \begin_inset Text
1411
1412 \begin_layout Plain Layout
1413 timely yes
1414 \end_layout
1415
1416 \end_inset
1417 </cell>
1418 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
1419 \begin_inset Text
1420
1421 \begin_layout Plain Layout
1422 timely no
1423 \end_layout
1424
1425 \end_inset
1426 </cell>
1427 </row>
1428 <row>
1429 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
1430 \begin_inset Text
1431
1432 \begin_layout Plain Layout
1433 positionly yes
1434 \end_layout
1435
1436 \end_inset
1437 </cell>
1438 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
1439 \begin_inset Text
1440
1441 \begin_layout Plain Layout
1442 -
1443 \end_layout
1444
1445 \end_inset
1446 </cell>
1447 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
1448 \begin_inset Text
1449
1450 \begin_layout Plain Layout
1451 \begin_inset Formula $\sqrt{}$
1452 \end_inset
1453
1454
1455 \end_layout
1456
1457 \end_inset
1458 </cell>
1459 </row>
1460 <row>
1461 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1462 \begin_inset Text
1463
1464 \begin_layout Plain Layout
1465 positionly no
1466 \end_layout
1467
1468 \end_inset
1469 </cell>
1470 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1471 \begin_inset Text
1472
1473 \begin_layout Plain Layout
1474 \begin_inset Formula $\sqrt{}$
1475 \end_inset
1476
1477
1478 \end_layout
1479
1480 \end_inset
1481 </cell>
1482 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
1483 \begin_inset Text
1484
1485 \begin_layout Plain Layout
1486 \begin_inset Formula $\sqrt{}$
1487 \end_inset
1488
1489
1490 \end_layout
1491
1492 \end_inset
1493 </cell>
1494 </row>
1495 </lyxtabular>
1496
1497 \end_inset
1498
1499
1500 \end_layout
1501
1502 \begin_layout Standard
1503 \noindent
1504 Pure timely overlapping (without positionly overlapping) is a frequent case
1505 in almost any IO system (usually called
1506 \begin_inset Quotes eld
1507 \end_inset
1508
1509 IO parallelism
1510 \begin_inset Quotes erd
1511 \end_inset
1512
1513 in folklore).
1514 In opposite, purely positionly overlapping (without timely overlapping)
1515 is also a frequent case, for example when the same sector is re-read after
1516 a while, or re-written when the contents of a file changes frequently.
1517 Completely unrelated requests (neither timely nor positionly overlapping)
1518 are probably the most frequent case in most practical load scenarios at
1519 production sites.
1520 There is no problem with any of them, indicated by the checkmarks.
1521 \end_layout
1522
1523 \begin_layout Standard
1524 However, what about both kinds of overlapping at the same time?
1525 \end_layout
1526
1527 \begin_layout Standard
1528 The case of
1529 \emph on
1530 both
1531 \emph default
1532 timely and positionly overlapping (simultanously) of IO operations is called
1533
1534 \emph on
1535 damaged IO
1536 \emph default
1537 .
1538 \end_layout
1539
1540 \begin_layout Standard
1541 In ordinary OS kernels, damaged IO usually
1542 \emph on
1543 never
1544 \emph default
1545 occurs.
1546 Here are the reasons:
1547 \end_layout
1548
1549 \begin_layout Standard
1550 IO requests are usually generated by in-kernel memory caches like buffer
1551 caches or page caches.
1552 Even in case of databases using Direct IO when submitting requests to the
1553 block layer, their internal database buffer cache works similarly to in-kernel
1554 caches.
1555 It simply makes
1556 \emph on
1557 no sense
1558 \emph default
1559 to write to the same sector twice at the same time, because the result
1560 will be undefined.
1561 A similar argument holds for reads in parallel to writes to/from the same
1562 sector.
1563 \end_layout
1564
1565 \begin_layout Standard
1566 In theory, concurrent reads from the same sector would be possible without
1567 causing harm to data integrity on the block device.
1568 In operating system caches, this would introduce
1569 \emph on
1570 copies
1571 \emph default
1572 of the same data into the buffer or page cache, violating its internal
1573
1574 \emph on
1575 uniqueness
1576 \emph default
1577 properties stating that any sector is cached at most once.
1578 Consequently, this case also never appears
1579 \emph on
1580 in practice
1581 \emph default
1582 at
1583 \emph on
1584 real-life systems(!)
1585 \emph default
1586 .
1587 \end_layout
1588
1589 \begin_layout Standard
1590 However, in terms of
1591 \series bold
1592 correctness of storage semantics
1593 \series default
1594 the case of damaged IO involving read/read to the same sector is
1595 \emph on
1596 allowed
1597 \emph default
1598 .
1599 Probably you already know the following table from database textbooks and
1600 others:
1601 \end_layout
1602
1603 \begin_layout Standard
1604 \align center
1605 \begin_inset Tabular
1606 <lyxtabular version="3" rows="3" columns="3">
1607 <features tabularvalignment="middle">
1608 <column alignment="center" valignment="top" width="0">
1609 <column alignment="center" valignment="top" width="0">
1610 <column alignment="center" valignment="top" width="0">
1611 <row>
1612 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1613 \begin_inset Text
1614
1615 \begin_layout Plain Layout
1616 conflict?
1617 \end_layout
1618
1619 \end_inset
1620 </cell>
1621 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1622 \begin_inset Text
1623
1624 \begin_layout Plain Layout
1625 read
1626 \end_layout
1627
1628 \end_inset
1629 </cell>
1630 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
1631 \begin_inset Text
1632
1633 \begin_layout Plain Layout
1634 write
1635 \end_layout
1636
1637 \end_inset
1638 </cell>
1639 </row>
1640 <row>
1641 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
1642 \begin_inset Text
1643
1644 \begin_layout Plain Layout
1645 read
1646 \end_layout
1647
1648 \end_inset
1649 </cell>
1650 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
1651 \begin_inset Text
1652
1653 \begin_layout Plain Layout
1654 no
1655 \end_layout
1656
1657 \end_inset
1658 </cell>
1659 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
1660 \begin_inset Text
1661
1662 \begin_layout Plain Layout
1663 yes
1664 \end_layout
1665
1666 \end_inset
1667 </cell>
1668 </row>
1669 <row>
1670 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1671 \begin_inset Text
1672
1673 \begin_layout Plain Layout
1674 write
1675 \end_layout
1676
1677 \end_inset
1678 </cell>
1679 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1680 \begin_inset Text
1681
1682 \begin_layout Plain Layout
1683 yes
1684 \end_layout
1685
1686 \end_inset
1687 </cell>
1688 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
1689 \begin_inset Text
1690
1691 \begin_layout Plain Layout
1692 yes
1693 \end_layout
1694
1695 \end_inset
1696 </cell>
1697 </row>
1698 </lyxtabular>
1699
1700 \end_inset
1701
1702
1703 \end_layout
1704
1705 \begin_layout Standard
1706 \noindent
1707 Some block IO systems like DRBD show some misbehaviour in case of concurrent
1708 writes to the same sector, or in some cases they even fail.
1709 Some DRBD versions
1710 \begin_inset Foot
1711 status open
1712
1713 \begin_layout Plain Layout
1714 At present, this seems to be an undocumented behaviour observed by the author.
1715 Even if DRBD's behaviour may change in the future: damaged IO is a bad
1716 idea by itself.
1717 It would be unfair to blame DRBD for
1718 \begin_inset Quotes eld
1719 \end_inset
1720
1721 psychologically unexpected
1722 \begin_inset Quotes erd
1723 \end_inset
1724
1725 behaviour under illegal load patterns, which should never occur.
1726 In general, making code rubust against damaged IO could decrease performance
1727 during ordinary operation.
1728 Thus damaged IO should be avoided at its
1729 \emph on
1730 source
1731 \emph default
1732 .
1733 \end_layout
1734
1735 \end_inset
1736
1737 will at least delay further IO requests for several milliseconds, lowering
1738 IO bandwidth or even leading to temporary hangs.
1739 \end_layout
1740
1741 \begin_layout Standard
1742 So, damaged IO
1743 \emph on
1744 should
1745 \emph default
1746 be avoided under all circumstances.
1747 Failing to do so may result in a disaster; in general, some IO devices
1748 like elder tape drives may even be corrupted as a whole.
1749 \end_layout
1750
1751 \begin_layout Standard
1752 While avoidance of damaged IO is automatically guaranteed in real production
1753 systems by the buffer / cache page of the Linux kernel (or other components
1754 like database memory buffers), our tool
1755 \family typewriter
1756 blkreplay
1757 \family default
1758 must be designed very carefully not to step into that pitfall.
1759 \end_layout
1760
1761 \begin_layout Standard
1762 Why is there a risk that
1763 \family typewriter
1764 blkreplay
1765 \family default
1766 could (accidentally) start some damaged IO?
1767 \end_layout
1768
1769 \begin_layout Standard
1770 Well, replay of the original timing of requests is not always possible.
1771 Even if
1772 \family typewriter
1773 blkreplay
1774 \family default
1775
1776 \emph on
1777 tries
1778 \emph default
1779 to start IO requests in the same timely pattern as at the original site,
1780 a very slow device (or a heavily overloaded device) may delay an IO operation
1781 for a very long time.
1782 In overload scenarios, or in case of iSCSI network hangs, it is possible
1783 that some IO requests may take 5 minutes to complete (or even more, or
1784 even
1785 \emph on
1786 never
1787 \emph default
1788 complete in case of fatal errors).
1789 In such cases, it is not unlikely that a new write to the same sector is
1790 started before the old one has completed.
1791 \end_layout
1792
1793 \begin_layout Standard
1794 In order to avoid damaged IO,
1795 \family typewriter
1796 blkreplay
1797 \family default
1798 uses some in-memory hash tables to detect any (potential) conflicts between
1799 IO requests.
1800 \end_layout
1801
1802 \begin_layout Standard
1803 In order to deal with (potential) problems caused by damaged IO, we use
1804 the following options in
1805 \family typewriter
1806 blkreplay
1807 \family default
1808 to control its behaviour at
1809 \emph on
1810 replay
1811 \emph default
1812 time:
1813 \end_layout
1814
1815 \begin_layout Labeling
1816 \labelwidthstring 00.00.0000
1817
1818 \family typewriter
1819 --strong=0
1820 \family default
1821 For the sake of conflict detection, only write/write conflicts will count.
1822 This mode ensures that the
1823 \begin_inset Quotes eld
1824 \end_inset
1825
1826 storage semantics
1827 \begin_inset Quotes erd
1828 \end_inset
1829
1830 is obeyed with respect to the
1831 \emph on
1832 written data
1833 \emph default
1834
1835 \begin_inset Foot
1836 status open
1837
1838 \begin_layout Plain Layout
1839 This allows to check the end result via
1840 \family typewriter
1841 verify
1842 \family default
1843 modes for correctness.
1844 \end_layout
1845
1846 \end_inset
1847
1848 , but it allows reads to permute with writes (which is
1849 \begin_inset Quotes eld
1850 \end_inset
1851
1852 wrong
1853 \begin_inset Quotes erd
1854 \end_inset
1855
1856 in
1857 \emph on
1858 strong
1859 \emph default
1860 sense, hence the name).
1861 In essence, the following conflict table is used internally for conflict
1862 detection:
1863 \begin_inset Newline newline
1864 \end_inset
1865
1866
1867 \begin_inset ERT
1868 status open
1869
1870 \begin_layout Plain Layout
1871
1872
1873 \backslash
1874 strut
1875 \end_layout
1876
1877 \end_inset
1878
1879
1880 \begin_inset space \hfill{}
1881 \end_inset
1882
1883
1884 \begin_inset Tabular
1885 <lyxtabular version="3" rows="3" columns="3">
1886 <features tabularvalignment="middle">
1887 <column alignment="center" valignment="top" width="0">
1888 <column alignment="center" valignment="top" width="0">
1889 <column alignment="center" valignment="top" width="0">
1890 <row>
1891 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1892 \begin_inset Text
1893
1894 \begin_layout Plain Layout
1895 conflict?
1896 \end_layout
1897
1898 \end_inset
1899 </cell>
1900 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1901 \begin_inset Text
1902
1903 \begin_layout Plain Layout
1904 read
1905 \end_layout
1906
1907 \end_inset
1908 </cell>
1909 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
1910 \begin_inset Text
1911
1912 \begin_layout Plain Layout
1913 write
1914 \end_layout
1915
1916 \end_inset
1917 </cell>
1918 </row>
1919 <row>
1920 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
1921 \begin_inset Text
1922
1923 \begin_layout Plain Layout
1924 read
1925 \end_layout
1926
1927 \end_inset
1928 </cell>
1929 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
1930 \begin_inset Text
1931
1932 \begin_layout Plain Layout
1933 no
1934 \end_layout
1935
1936 \end_inset
1937 </cell>
1938 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
1939 \begin_inset Text
1940
1941 \begin_layout Plain Layout
1942 no
1943 \end_layout
1944
1945 \end_inset
1946 </cell>
1947 </row>
1948 <row>
1949 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1950 \begin_inset Text
1951
1952 \begin_layout Plain Layout
1953 write
1954 \end_layout
1955
1956 \end_inset
1957 </cell>
1958 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
1959 \begin_inset Text
1960
1961 \begin_layout Plain Layout
1962 no
1963 \end_layout
1964
1965 \end_inset
1966 </cell>
1967 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
1968 \begin_inset Text
1969
1970 \begin_layout Plain Layout
1971 yes
1972 \end_layout
1973
1974 \end_inset
1975 </cell>
1976 </row>
1977 </lyxtabular>
1978
1979 \end_inset
1980
1981
1982 \end_layout
1983
1984 \begin_layout Labeling
1985 \labelwidthstring 00.00.0000
1986
1987 \family typewriter
1988 --strong=1
1989 \family default
1990 (default) Use the standard conflict table as known from the literature.
1991 Only read/read is treated as non-conflicting:
1992 \begin_inset Newline newline
1993 \end_inset
1994
1995
1996 \begin_inset ERT
1997 status open
1998
1999 \begin_layout Plain Layout
2000
2001
2002 \backslash
2003 strut
2004 \end_layout
2005
2006 \end_inset
2007
2008
2009 \begin_inset space \hfill{}
2010 \end_inset
2011
2012
2013 \begin_inset Tabular
2014 <lyxtabular version="3" rows="3" columns="3">
2015 <features tabularvalignment="middle">
2016 <column alignment="center" valignment="top" width="0">
2017 <column alignment="center" valignment="top" width="0">
2018 <column alignment="center" valignment="top" width="0">
2019 <row>
2020 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
2021 \begin_inset Text
2022
2023 \begin_layout Plain Layout
2024 conflict?
2025 \end_layout
2026
2027 \end_inset
2028 </cell>
2029 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
2030 \begin_inset Text
2031
2032 \begin_layout Plain Layout
2033 read
2034 \end_layout
2035
2036 \end_inset
2037 </cell>
2038 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
2039 \begin_inset Text
2040
2041 \begin_layout Plain Layout
2042 write
2043 \end_layout
2044
2045 \end_inset
2046 </cell>
2047 </row>
2048 <row>
2049 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
2050 \begin_inset Text
2051
2052 \begin_layout Plain Layout
2053 read
2054 \end_layout
2055
2056 \end_inset
2057 </cell>
2058 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
2059 \begin_inset Text
2060
2061 \begin_layout Plain Layout
2062 no
2063 \end_layout
2064
2065 \end_inset
2066 </cell>
2067 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
2068 \begin_inset Text
2069
2070 \begin_layout Plain Layout
2071 yes
2072 \end_layout
2073
2074 \end_inset
2075 </cell>
2076 </row>
2077 <row>
2078 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
2079 \begin_inset Text
2080
2081 \begin_layout Plain Layout
2082 write
2083 \end_layout
2084
2085 \end_inset
2086 </cell>
2087 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
2088 \begin_inset Text
2089
2090 \begin_layout Plain Layout
2091 yes
2092 \end_layout
2093
2094 \end_inset
2095 </cell>
2096 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
2097 \begin_inset Text
2098
2099 \begin_layout Plain Layout
2100 yes
2101 \end_layout
2102
2103 \end_inset
2104 </cell>
2105 </row>
2106 </lyxtabular>
2107
2108 \end_inset
2109
2110
2111 \end_layout
2112
2113 \begin_layout Labeling
2114 \labelwidthstring 00.00.0000
2115
2116 \family typewriter
2117 --strong=2
2118 \family default
2119
2120 \emph on
2121 All
2122 \emph default
2123 damaged IO is treated as conflicting, even read/read.
2124 This may be useful for simulating the behaviour of
2125 \emph on
2126 real
2127 \emph default
2128 OS caches.
2129 This is equivalent to the following table:
2130 \begin_inset Newline newline
2131 \end_inset
2132
2133
2134 \begin_inset ERT
2135 status open
2136
2137 \begin_layout Plain Layout
2138
2139
2140 \backslash
2141 strut
2142 \end_layout
2143
2144 \end_inset
2145
2146
2147 \begin_inset space \hfill{}
2148 \end_inset
2149
2150
2151 \begin_inset Tabular
2152 <lyxtabular version="3" rows="3" columns="3">
2153 <features tabularvalignment="middle">
2154 <column alignment="center" valignment="top" width="0">
2155 <column alignment="center" valignment="top" width="0">
2156 <column alignment="center" valignment="top" width="0">
2157 <row>
2158 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
2159 \begin_inset Text
2160
2161 \begin_layout Plain Layout
2162 conflict?
2163 \end_layout
2164
2165 \end_inset
2166 </cell>
2167 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
2168 \begin_inset Text
2169
2170 \begin_layout Plain Layout
2171 read
2172 \end_layout
2173
2174 \end_inset
2175 </cell>
2176 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
2177 \begin_inset Text
2178
2179 \begin_layout Plain Layout
2180 write
2181 \end_layout
2182
2183 \end_inset
2184 </cell>
2185 </row>
2186 <row>
2187 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
2188 \begin_inset Text
2189
2190 \begin_layout Plain Layout
2191 read
2192 \end_layout
2193
2194 \end_inset
2195 </cell>
2196 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
2197 \begin_inset Text
2198
2199 \begin_layout Plain Layout
2200 yes
2201 \end_layout
2202
2203 \end_inset
2204 </cell>
2205 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
2206 \begin_inset Text
2207
2208 \begin_layout Plain Layout
2209 yes
2210 \end_layout
2211
2212 \end_inset
2213 </cell>
2214 </row>
2215 <row>
2216 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
2217 \begin_inset Text
2218
2219 \begin_layout Plain Layout
2220 write
2221 \end_layout
2222
2223 \end_inset
2224 </cell>
2225 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
2226 \begin_inset Text
2227
2228 \begin_layout Plain Layout
2229 yes
2230 \end_layout
2231
2232 \end_inset
2233 </cell>
2234 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
2235 \begin_inset Text
2236
2237 \begin_layout Plain Layout
2238 yes
2239 \end_layout
2240
2241 \end_inset
2242 </cell>
2243 </row>
2244 </lyxtabular>
2245
2246 \end_inset
2247
2248
2249 \end_layout
2250
2251 \begin_layout Standard
2252 Indepently from the correctness criterion, the following operation modes
2253 may be selected.
2254 They determine the kind of reaction in case of detected conflicts:
2255 \end_layout
2256
2257 \begin_layout Labeling
2258 \labelwidthstring 00.00.0000
2259
2260 \family typewriter
2261 --with-conflicts
2262 \family default
2263 No countermeasures against damaged IO are taken.
2264 Consequently, is does not matter which
2265 \family typewriter
2266 --strong=
2267 \family default
2268 mode you have selected before.
2269 \end_layout
2270
2271 \begin_layout Labeling
2272 \labelwidthstring 00.00.0000
2273
2274 \family typewriter
2275 --with-drop
2276 \family default
2277 Whenever a new request is conflicting with an old (already issued) request,
2278 it is simply dropped.
2279 This has no side effects, other than that some reads and/or writes may
2280 be missing (depending on
2281 \family typewriter
2282 --strong=
2283 \family default
2284 mode).
2285 This has the lowest overhead of all conflict-avoiding methods.
2286 Depending on properties of the load, the number of dropped requests may
2287 be rather high.
2288 Please check the tail of the result file.
2289 In the statistics section, you will find the number of dropped requests.
2290 If that number is higher than, say, 5%, you should consider one of the
2291 following options.
2292 \end_layout
2293
2294 \begin_layout Labeling
2295 \labelwidthstring 00.00.0000
2296
2297 \family typewriter
2298 --with-partial
2299 \family default
2300 (default) Any conflicting requests (as determined by
2301 \family typewriter
2302 --strong=
2303 \family default
2304 ) are pushed back to an internal pushback list and kept there until the
2305 conflict is gone.
2306 Pushed back requests are immediately submitted as soon as the conflict
2307 has gone away.
2308 This results in a reordering of the
2309 \emph on
2310 affected
2311 \emph default
2312 requests, while
2313 \emph on
2314 trying
2315 \emph default
2316 to replay unaffected ones at original timestamps.
2317 This leads to a partial ordering of requests, which may be
2318 \emph on
2319 very different
2320 \emph default
2321 from the original ordering, and thus may
2322 \begin_inset Quotes eld
2323 \end_inset
2324
2325 violate
2326 \begin_inset Quotes erd
2327 \end_inset
2328
2329 the
2330 \begin_inset Quotes eld
2331 \end_inset
2332
2333 original storage semantics
2334 \begin_inset Quotes erd
2335 \end_inset
2336
2337 if it would make a difference on the replay system.
2338 \begin_inset Newline newline
2339 \end_inset
2340
2341 This mode leads to a 2-class society, where ordinary requests are processed
2342 faster than conflicting ones.
2343 Further details may be found in section
2344 \begin_inset CommandInset ref
2345 LatexCommand ref
2346 reference "sub:Influence-of-Request"
2347
2348 \end_inset
2349
2350 .
2351 \begin_inset Newline newline
2352 \end_inset
2353
2354 Attention! as a side effect, this mode may
2355 \emph on
2356 increase
2357 \emph default
2358 the actual IO parallelism to a larger number than configured via the
2359 \family typewriter
2360 --threads=
2361 \family default
2362 parameter, because some request slots (and in turn, also some threads)
2363 must
2364 \begin_inset Foot
2365 status open
2366
2367 \begin_layout Plain Layout
2368 Otherwise a reactivated request from the pushback list could have to wait
2369 for an ordinary request slot to become free, which would result in an artificia
2370 l delay.
2371 Experiments have show that such kinds of distortions can be serious.
2372 \end_layout
2373
2374 \end_inset
2375
2376 be reserved
2377 \emph on
2378 in advance
2379 \emph default
2380 for pushed-back requests.
2381 When conflicts are gone and the system tries to
2382 \begin_inset Quotes eld
2383 \end_inset
2384
2385 catch up
2386 \begin_inset Quotes erd
2387 \end_inset
2388
2389 , additional IO requests may be submitted.
2390 \begin_inset Newline newline
2391 \end_inset
2392
2393 In practice, this can happen in particular with some Windows loads, where
2394 a lot of writes seem to be repeated.
2395 Frequently, pushed back requests (colored differently for better distinction
2396 from ordinary requests) are the main contributors to delays.
2397 Check them!
2398 \end_layout
2399
2400 \begin_layout Labeling
2401 \labelwidthstring 00.00.0000
2402
2403 \family typewriter
2404 --with-ordering
2405 \family default
2406 The main process spreads its IO requests to the worker processes in a round-rob
2407 in fashion over their anonymous pipes.
2408 Whenever conflicting requests (as determined by
2409 \family typewriter
2410 --strong=
2411 \family default
2412 ) are detected, this spreading process will stop until the conflict has
2413 gone away.
2414 In consequence, the original ordering of requests will be preserved as
2415 much as possible.
2416 As a side effect,
2417 \emph on
2418 all following
2419 \emph default
2420 IO requests are also delayed, even if they don't conflict with anything
2421 else.
2422 This can lead to
2423 \emph on
2424 artificial delays
2425 \emph default
2426 .
2427 \begin_inset Newline newline
2428 \end_inset
2429
2430 In general, this mode is the
2431 \begin_inset Quotes eld
2432 \end_inset
2433
2434 most robust
2435 \begin_inset Quotes erd
2436 \end_inset
2437
2438 one.
2439 \end_layout
2440
2441 \begin_layout Standard
2442 In many cases, you will prefer
2443 \family typewriter
2444 --with-partial
2445 \family default
2446 , which is the default.
2447 It delivers almost the same throughput as
2448 \family typewriter
2449 --with-conflicts
2450 \family default
2451 without the downsides of
2452 \family typewriter
2453 --with-drop
2454 \family default
2455 , while minimizing artificial delays
2456 \begin_inset Foot
2457 status open
2458
2459 \begin_layout Plain Layout
2460 In extreme cases, pushed-back requests (visualized in stronger colors) can
2461 form some kind of
2462 \begin_inset Quotes eld
2463 \end_inset
2464
2465 transitive queues
2466 \begin_inset Quotes erd
2467 \end_inset
2468
2469 , e.g.
2470 when pushback requests depend on other pushback requests transitively.
2471 In such a case, their delays can sum up over a longer time, independently
2472 from
2473 \begin_inset Quotes eld
2474 \end_inset
2475
2476 normal
2477 \begin_inset Quotes erd
2478 \end_inset
2479
2480 requests (clearly visible in the delay diagram).
2481 Although this is often a property of the load, you can
2482 \emph on
2483 try(!)
2484 \emph default
2485 to minimize such an effect by setting the expert option
2486 \family typewriter
2487 ahead_limit
2488 \family default
2489 to values lower than 1s (however too low values will destroy throughput
2490 at all).
2491 Dangerous!
2492 \end_layout
2493
2494 \end_inset
2495
2496 .
2497 \end_layout
2498
2499 \begin_layout Standard
2500 However,
2501 \family typewriter
2502 --with-ordering
2503 \family default
2504 is also useful in many scenarios.
2505 Practical experience from many hardware tests shows that the artificial
2506 delays caused by
2507 \family typewriter
2508 --with-ordering
2509 \family default
2510 seem to be an
2511 \emph on
2512 advantage
2513 \emph default
2514 .
2515 Whenever such stalls occur more than seldomly, they act as an
2516 \series bold
2517 indicator
2518 \series default
2519 for massive IO problems of the test candidate.
2520
2521 \family typewriter
2522 --with-ordering
2523 \family default
2524 is often a kind of
2525 \begin_inset Quotes eld
2526 \end_inset
2527
2528 detector
2529 \begin_inset Quotes erd
2530 \end_inset
2531
2532 for hardware problems, since it visualizes any problems in an eye-catching
2533 way.
2534 \end_layout
2535
2536 \begin_layout Labeling
2537 \labelwidthstring 00.00.0000
2538 \begin_inset Graphics
2539 filename images/MatieresCorrosives.png
2540 lyxscale 50
2541 scale 17
2542
2543 \end_inset
2544
2545 When you start a lot of threads (typically
2546 \begin_inset Formula $\geq256$
2547 \end_inset
2548
2549 ),
2550 \family typewriter
2551 --with-ordering
2552 \family default
2553 may yield
2554 \emph on
2555 better(!)
2556 \emph default
2557 throughput than
2558 \family typewriter
2559 --with-partial
2560 \family default
2561 .
2562 In such cases, the reason is
2563 \emph on
2564 counter-intuitive
2565 \emph default
2566 , as explained in section
2567 \begin_inset CommandInset ref
2568 LatexCommand ref
2569 reference "sub:Influence-of-Number"
2570
2571 \end_inset
2572
2573 : artificial delays caused by
2574 \family typewriter
2575 --with-ordering
2576 \family default
2577 will
2578 \emph on
2579 decrease
2580 \emph default
2581 the
2582 \emph on
2583 average
2584 \emph default
2585 request queue length actually occuring at runtime, which will in turn
2586 \emph on
2587 increase
2588 \emph default
2589 the average throughput, depending on counter-intuitive properties of your
2590 test candidate.
2591 A way to find out is just to run the same benchmarks with
2592 \family typewriter
2593 --threads=16
2594 \family default
2595 or
2596 \family typewriter
2597 --threads=32
2598 \family default
2599 .
2600 In case
2601 \family typewriter
2602 --with-partial
2603 \family default
2604 is now better than
2605 \family typewriter
2606 --with-ordering
2607 \family default
2608 , you likely detected that problem.
2609 Don't draw wrong conclusions from such counter-intuitive effects!
2610 \end_layout
2611
2612 \begin_layout Section
2613 Verification of Storage Semantics
2614 \end_layout
2615
2616 \begin_layout Standard
2617 In order to allow verification of the sector headers and their timestamps
2618 / version stamps,
2619 \family typewriter
2620 blkreplay
2621 \family default
2622 needs some temporary storage where information can be kept for a longer
2623 time than just during IO.
2624 Two temporary files are used:
2625 \family typewriter
2626
2627 \begin_inset Flex URL
2628 status open
2629
2630 \begin_layout Plain Layout
2631
2632 /tmp/verify_table
2633 \end_layout
2634
2635 \end_inset
2636
2637
2638 \family default
2639 and
2640 \family typewriter
2641
2642 \begin_inset Flex URL
2643 status open
2644
2645 \begin_layout Plain Layout
2646
2647 /tmp/completion_table
2648 \end_layout
2649
2650 \end_inset
2651
2652
2653 \family default
2654 .
2655 Both are sparse files, containing a simple (sparse) array of sequence numbers,
2656 indexed by the position (sector#).
2657 Whenever a write is started, a new sequence number is recorded in
2658 \family typewriter
2659
2660 \begin_inset Flex URL
2661 status open
2662
2663 \begin_layout Plain Layout
2664
2665 /tmp/verify_table
2666 \end_layout
2667
2668 \end_inset
2669
2670
2671 \family default
2672 .
2673 Whenever that write is completed, the same number is recorded in
2674 \family typewriter
2675
2676 \begin_inset Flex URL
2677 status open
2678
2679 \begin_layout Plain Layout
2680
2681 /tmp/completion_table
2682 \end_layout
2683
2684 \end_inset
2685
2686
2687 \family default
2688 .
2689 At any time, both tables keep track of the current progress of write operations.
2690
2691 \end_layout
2692
2693 \begin_layout Standard
2694 Whenever the sequence numbers at the same position (sector#) are different
2695 between both files, we know that a write operation has not yet completed.
2696 If they are the same, we know which sequence number should appear in the
2697 header of the corresponding sector.
2698 \end_layout
2699
2700 \begin_layout Standard
2701 There are following variants of verification modes:
2702 \end_layout
2703
2704 \begin_layout Labeling
2705 \labelwidthstring 00.00.0000
2706
2707 \family typewriter
2708 --with-verify
2709 \family default
2710 Whenever a sector is read (by a regular read request) which has been written
2711 before (by an ordinary write request), the read data is checked against
2712 the sequence numbers from the tables.
2713 Any mismatches are reported by the string
2714 \family typewriter
2715 VERIFY ERROR
2716 \family default
2717 in the result file.
2718 Thus,
2719 \family typewriter
2720 zgrep
2721 \begin_inset Quotes eld
2722 \end_inset
2723
2724 VERIFY ERROR
2725 \begin_inset Quotes erd
2726 \end_inset
2727
2728 *.replay.gz
2729 \family default
2730 will show them to you.
2731 In addition, the statistics section at the end of the output file will
2732 contain some valuable information.
2733 \begin_inset Newline newline
2734 \end_inset
2735
2736 Warning! this mode can only reveal errors in the storage semantics if written
2737 blocks are re-read somewhen.
2738 When sectors are just written, but never read, this mode will not detect
2739 anything.
2740 \end_layout
2741
2742 \begin_layout Labeling
2743 \labelwidthstring 00.00.0000
2744
2745 \family typewriter
2746 --with-final-verify
2747 \family default
2748 In addition to
2749 \family typewriter
2750 --with-verify
2751 \family default
2752 , a separate pass will be started at the end of a
2753 \family typewriter
2754 blkreplay
2755 \family default
2756 run.
2757 All sectors which have been touched before, will be checked.
2758 \end_layout
2759
2760 \begin_layout Labeling
2761 \labelwidthstring 00.00.0000
2762
2763 \family typewriter
2764 --with-paranoia
2765 \family default
2766 In addition, any written sector will be immediately re-read during operation.
2767 This doubles the IO rate and leads to extremely high distortions of measurement
2768 results.
2769 \end_layout
2770
2771 \begin_layout Labeling
2772 \labelwidthstring 00.00.0000
2773 \begin_inset Graphics
2774 filename images/MatieresCorrosives.png
2775 lyxscale 50
2776 scale 17
2777
2778 \end_inset
2779
2780 All verify modes will create temporary tables in
2781 \family typewriter
2782 /tmp/
2783 \family default
2784 .
2785 Although the temporary files are sparse, they can use up a significant
2786 amount of storage (depending on the load).
2787 The additional IOs form a
2788 \emph on
2789 sequential bottleneck
2790 \emph default
2791 , and therefore can slowdown
2792 \family typewriter
2793 blkreplay
2794 \family default
2795 considerably.
2796 Please use the verify modes only for
2797 \emph on
2798 validation
2799 \emph default
2800 , but not for measurements / determining performance!
2801 \end_layout
2802
2803 \begin_layout Chapter
2804 How to use
2805 \family typewriter
2806 blkreplay
2807 \end_layout
2808
2809 \begin_layout Labeling
2810 \labelwidthstring 00.00.0000
2811 \begin_inset Graphics
2812 filename images/MatieresToxiques.png
2813 lyxscale 50
2814 scale 17
2815
2816 \end_inset
2817
2818 Running
2819 \family typewriter
2820 blkreplay
2821 \family default
2822 naïvely without reading this chapter may easily lead to
2823 \series bold
2824 completely worthless fake results
2825 \series default
2826 which would be only useful for production of bullshit!
2827 \end_layout
2828
2829 \begin_layout Standard
2830 In science, it would be unethical to produce such bullshit willingly or
2831 even deliberately.
2832 \end_layout
2833
2834 \begin_layout Standard
2835 In industry, usage of such bullshit (even inadvertendly) may easily lead
2836 to failed invests up to millions of Euros / Dollars (depending on the applicati
2837 on and the size of your datacenter).
2838 \end_layout
2839
2840 \begin_layout Standard
2841 Most of the following advice will also apply to other benchmark tools like
2842
2843 \family typewriter
2844 iometer
2845 \family default
2846 .
2847 \end_layout
2848
2849 \begin_layout Section
2850 How to Avoid Common Pitfalls
2851 \begin_inset CommandInset label
2852 LatexCommand label
2853 name "sec:How-to-Avoid"
2854
2855 \end_inset
2856
2857
2858 \end_layout
2859
2860 \begin_layout Labeling
2861 \labelwidthstring 00.00.0000
2862 \begin_inset Graphics
2863 filename images/MatieresToxiques.png
2864 lyxscale 50
2865 scale 17
2866
2867 \end_inset
2868
2869 Don't skip this section! Read it
2870 \emph on
2871 completely
2872 \emph default
2873 , even if you are impatient or under time pressure!
2874 \end_layout
2875
2876 \begin_layout Standard
2877 Modern IO subsystems often use some kind of
2878 \series bold
2879 storage virtualization
2880 \series default
2881 interally.
2882 More often than you can dream of, concepts from storage virtualization
2883 are used (interally) in places where you don't expect them.
2884 \end_layout
2885
2886 \begin_layout Standard
2887 Example: seemingly
2888 \begin_inset Quotes eld
2889 \end_inset
2890
2891 simple
2892 \begin_inset Quotes erd
2893 \end_inset
2894
2895 SSDs or even some USB sticks(!) show some of the behaviours described next.
2896 \end_layout
2897
2898 \begin_layout Subsection
2899 Pitfalls from Storage Virtualization
2900 \begin_inset CommandInset label
2901 LatexCommand label
2902 name "sub:Pitfalls-from-Storage"
2903
2904 \end_inset
2905
2906
2907 \end_layout
2908
2909 \begin_layout Standard
2910 The basic idea of storage virtualization is some kind of
2911 \begin_inset Quotes eld
2912 \end_inset
2913
2914 translation
2915 \begin_inset Quotes erd
2916 \end_inset
2917
2918 (or
2919 \begin_inset Quotes eld
2920 \end_inset
2921
2922 mapping
2923 \begin_inset Quotes erd
2924 \end_inset
2925
2926 ) between
2927 \emph on
2928 virtual storage addresses
2929 \emph default
2930 and
2931 \emph on
2932
2933 \begin_inset Quotes eld
2934 \end_inset
2935
2936 physical
2937 \begin_inset Quotes erd
2938 \end_inset
2939
2940 storage addresses
2941 \emph default
2942 .
2943 \end_layout
2944
2945 \begin_layout Standard
2946 In many
2947 \begin_inset Foot
2948 status open
2949
2950 \begin_layout Plain Layout
2951 A promient exception is classical LVM as implemented by the Linux kernel:
2952 unless you use LVM snapshots or other advanced features, it uses almost
2953 static mappings, and it carries almost no observable overhead in many practical
2954 scenarios.
2955 \end_layout
2956
2957 \end_inset
2958
2959 cases, the address translation / mapping is created on the fly, dynamically
2960 at runtime.
2961 In the following simplified
2962 \begin_inset Foot
2963 status open
2964
2965 \begin_layout Plain Layout
2966 For simplicity, this example assumes that the address translation uses the
2967 same granularity as the benchmark, e.g.
2968 single sectors.
2969 We also don't discuss the internals of the mapping which can also have
2970 a drastic impact on measurement results.
2971 \end_layout
2972
2973 \end_inset
2974
2975 example, the timely order of accesses is marked by increasing numbers,
2976 while the type of IO request is indicated by colors (red = write, blue
2977 = read):
2978 \end_layout
2979
2980 \begin_layout Standard
2981 \noindent
2982 \align center
2983 \begin_inset Graphics
2984 filename storage-virtualization.fig
2985 lyxscale 70
2986 width 80col%
2987
2988 \end_inset
2989
2990
2991 \end_layout
2992
2993 \begin_layout Standard
2994 \noindent
2995 We start with an empty logical address space (often called
2996 \begin_inset Quotes eld
2997 \end_inset
2998
2999 logical volume
3000 \begin_inset Quotes erd
3001 \end_inset
3002
3003 , shortly LV), having a logical size of several terabytes.
3004 Here,
3005 \begin_inset Quotes eld
3006 \end_inset
3007
3008 empty
3009 \begin_inset Quotes erd
3010 \end_inset
3011
3012 means that initially
3013 \emph on
3014 no
3015 \emph default
3016 assignment between logical and physical sector numbers exists.
3017 Now we start a short benchmark (whether
3018 \family typewriter
3019 blkreplay
3020 \family default
3021 or others like
3022 \family typewriter
3023 iometer
3024 \family default
3025 ).
3026 When request #1 (a write) arrives, no physical location exists yet.
3027 Therefore, a new location must be assigned.
3028 In this example, the locations are always taken from the beginning of the
3029 physical address space.
3030 \end_layout
3031
3032 \begin_layout Standard
3033 Thus, all the physical sectors occurring in this example will be allocated
3034 in a very small and compact area at the start of the physical address space.
3035 Imagine the drawing not being true to scale: imagine a total size of several
3036 terabytes (
3037 \begin_inset Formula $>10^{9}$
3038 \end_inset
3039
3040 sectors), and a
3041 \family typewriter
3042 blkreplay
3043 \family default
3044 benchmark touching only a few thousands of sectors.
3045 What will be the effect?
3046 \end_layout
3047
3048 \begin_layout Enumerate
3049 Only a
3050 \emph on
3051 tiny
3052 \emph default
3053 fraction of the physical space will be actually used, usually less than
3054 one per mille or even less than a millionth.
3055 \begin_inset Newline newline
3056 \end_inset
3057
3058 In contrast, real world applications as well as real customers tend to use
3059 up significant space with real data, usually more than 50% (and up to 100%).
3060 \end_layout
3061
3062 \begin_layout Enumerate
3063 Even if accesses to the LV are (pseudo-)random, accesses to the
3064 \begin_inset Quotes eld
3065 \end_inset
3066
3067 physical hot area
3068 \begin_inset Quotes erd
3069 \end_inset
3070
3071 will
3072 \emph on
3073 not
3074 \emph default
3075 remain random: as you can see, they are translated to (purely / almost)
3076
3077 \emph on
3078 sequential
3079 \emph default
3080 access patterns.
3081 If the physical addresses are residing on a mechanical hard disk, (almost)
3082 no seek operations will occur.
3083 Additionally, the physical operations are in
3084 \emph on
3085 ascending
3086 \emph default
3087 order, which is a classical use case for BBU-cached writes and/or readahead
3088 strategies.
3089 Notice that ascending sequential IO on hard disks is usually faster than
3090 random IO by a factor of 100 or even more (depending on hardware and further
3091 factors like RAID, between one and three orders of magnitude).
3092 \begin_inset Newline newline
3093 \end_inset
3094
3095 In contrast, real-world writes will be spreaded much more randomly over
3096 the physical partition, and there will be a significant amount of
3097 \emph on
3098 in-place updates
3099 \emph default
3100 in many practical use cases.
3101 \end_layout
3102
3103 \begin_layout Enumerate
3104 Even worse, read requests need not be mapped at all (indicated by shading
3105 in the drawing).
3106 When reading from an address where never anything had been written before,
3107 NULL blocks may be returned on-the-fly, without causing any physical access
3108 at all
3109 \begin_inset Foot
3110 status open
3111
3112 \begin_layout Plain Layout
3113 A similar effect is known from holes in traditional Unix
3114 \emph on
3115 sparse files
3116 \emph default
3117 .
3118 \end_layout
3119
3120 \end_inset
3121
3122 .
3123 Notice that such
3124 \begin_inset Quotes eld
3125 \end_inset
3126
3127 fake reads
3128 \begin_inset Quotes erd
3129 \end_inset
3130
3131 can be faster than true read accesses by
3132 \emph on
3133 several
3134 \emph default
3135 orders of magnitude.
3136 \end_layout
3137
3138 \begin_layout Enumerate
3139 Even in case read requests are also mapped upon first reference
3140 \begin_inset Foot
3141 status open
3142
3143 \begin_layout Plain Layout
3144 Several commercial storage boxes are known to do so.
3145 However, notice that this behaviour is not documented, and thus not guaranteed
3146 by the vendors.
3147 They sell you a blackbox.
3148 All you can do is to analyze such behaviour if you are curious about their
3149 internals.
3150 In their next firmware release, the behaviour may be already different
3151 without notice.
3152 \end_layout
3153
3154 \end_inset
3155
3156 , perhaps leading to a physical IO (or perhaps not), the same arguments
3157 as for writes apply.
3158 \end_layout
3159
3160 \begin_layout Enumerate
3161 If you
3162 \emph on
3163 repeat
3164 \emph default
3165 the same
3166 \family typewriter
3167 blkreplay
3168 \family default
3169 benchmark once again, immediately after the first run, you will get another
3170 surprise: this time the mapping between logical and physical addresses
3171 already exists, thus you will likely get different results, usually drastically
3172 better, but seldomly slightly worse, depending on the vendor of the storage
3173 virtualization (and on many other factors such as the size of the logical
3174 volume).
3175 In scientific terminology: your experiment is not truly
3176 \emph on
3177 repeatable
3178 \emph default
3179 .
3180 \end_layout
3181
3182 \begin_layout Standard
3183 What can you do about that?
3184 \end_layout
3185
3186 \begin_layout Standard
3187 There is no general solution for all cases.
3188 It depends on the
3189 \emph on
3190 statement
3191 \emph default
3192 you want to prove or disprove by usage of
3193 \family typewriter
3194 blkreplay
3195 \family default
3196 .
3197 \end_layout
3198
3199 \begin_layout Standard
3200 The following is just an
3201 \emph on
3202 approximation
3203 \emph default
3204 if you want to reveal the real-life
3205 \begin_inset Foot
3206 status open
3207
3208 \begin_layout Plain Layout
3209 In real life, customer data or enterprise data is stored on LVs.
3210 Thus benchmarks of empty LVs are
3211 \emph on
3212 completely wrong
3213 \emph default
3214 if you try to reveal real life behaviour.
3215 \end_layout
3216
3217 \end_inset
3218
3219 behaviour of virtualized storage systems:
3220 \end_layout
3221
3222 \begin_layout Enumerate
3223 Whenever you start a new run of a benchmark, you
3224 \series bold
3225 \emph on
3226 must
3227 \emph default
3228 delete
3229 \series default
3230 your old LVs, and create new ones.
3231 Otherwise, your old run will influence the new one in some way you cannot
3232 predict easily.
3233 Remember that the mapping table in the above example is a kind of
3234 \begin_inset Quotes eld
3235 \end_inset
3236
3237 memory
3238 \begin_inset Quotes erd
3239 \end_inset
3240
3241 which records not only the sector numbers occurring in your benchmark,
3242 but even their timely order.
3243 Make sure that this kind of
3244 \begin_inset Quotes eld
3245 \end_inset
3246
3247 memory
3248 \begin_inset Quotes erd
3249 \end_inset
3250
3251 is erased completely
3252 \begin_inset Foot
3253 status open
3254
3255 \begin_layout Plain Layout
3256 Even delete your LVs if you believe that's unnecessary, because you have
3257 obeyed point 2 and have filled it with data to initialize the mapping:
3258 some storage systems make
3259 \emph on
3260 re-assignments
3261 \emph default
3262 of the mapping during your benchmarks.
3263 Because many commercial storage systems are blackboxes, you cannot immediately
3264 see that.
3265 Always keep in mind that usually ordinary benchmarks will only touch a
3266
3267 \emph on
3268 tiny fraction
3269 \emph default
3270 of all physical sectors, compared to real life!
3271 \end_layout
3272
3273 \end_inset
3274
3275 between any runs!
3276 \end_layout
3277
3278 \begin_layout Enumerate
3279 After each fresh creation of a logical volume,
3280 \series bold
3281 fill it with data
3282 \series default
3283 .
3284 This is the only reliable
3285 \begin_inset Foot
3286 status open
3287
3288 \begin_layout Plain Layout
3289 Some storage vendors have internal functions which
3290 \emph on
3291 preallocate
3292 \emph default
3293 the space for a LV.
3294 Don't use them! Don't trust any claims that this would be equivalent to
3295 filling with random data - we found cases where we could
3296 \emph on
3297 disprove
3298 \emph default
3299 such claims, where results differed even by
3300 \emph on
3301 factors
3302 \emph default
3303 .
3304 Just fill your LVs with random data to be sure, even if this delays your
3305 measurements for some hours or even days.
3306 Keep in mind that later production systems will take weeks or even months
3307 to be filled with data before potential problems could show up, so don't
3308 hesitate to resemble such kind of effort in the laboratory.
3309 \end_layout
3310
3311 \end_inset
3312
3313 way.
3314 Best practice is to use tools like
3315 \family typewriter
3316 wipe
3317 \family default
3318 , filling the whole
3319 \begin_inset Foot
3320 status open
3321
3322 \begin_layout Plain Layout
3323 If you are consciously concerned that filling the
3324 \emph on
3325 whole
3326 \emph default
3327 LV might not catch your usecase where (say) only 50% of logical space is
3328 actually used, you
3329 \emph on
3330 could
3331 \emph default
3332 try to fill only 50% of the LV.
3333 However, be
3334 \emph on
3335 sure
3336 \emph default
3337 to fill any blocks which occur in the benchmarks.
3338 Otherwise, the effects described above will almost certainly lead to a
3339
3340 \emph on
3341 higher
3342 \emph default
3343 distortion of your results than just filling with 100%.
3344 Notice that some of the above effects deal with orders of magnitude, not
3345 just a few percent.
3346 \end_layout
3347
3348 \end_inset
3349
3350 LV with random data.
3351 Filling with NULL blocks is not recommended, because some blackbox storage
3352 systems might detect this easily and circumvent it by not creating a mapping
3353 at all (or even erasing an old mapping similar to
3354 \family typewriter
3355 punch
3356 \family default
3357 operations).
3358 \end_layout
3359
3360 \begin_layout Enumerate
3361
3362 \emph on
3363 Immediately
3364 \emph default
3365 after filling with random data, start your benchmark
3366 \series bold
3367 exactly once
3368 \series default
3369 .
3370
3371 \emph on
3372 Never
3373 \emph default
3374 , really
3375 \emph on
3376 never
3377 \emph default
3378 kill a run of
3379 \family typewriter
3380 blkreplay
3381 \family default
3382 (or any other benchmark tool) and restart it.
3383 In case of any error or misbehaviour, you
3384 \emph on
3385 must
3386 \emph default
3387 start over with step 1!
3388 \end_layout
3389
3390 \begin_layout Enumerate
3391 There is a single exception if you really know what you are doing: immediately
3392 after the first run, you may restart the
3393 \emph on
3394 same
3395 \emph default
3396 benchmark once again, in order to reveal some hidden properties of the
3397 mapping.
3398 You
3399 \emph on
3400 must
3401 \emph default
3402 name your output files differently from the first run, and you
3403 \emph on
3404 must not
3405 \emph default
3406 confuse the meaning of the second run with the meaning of the first run.
3407 \end_layout
3408
3409 \begin_layout Subsection
3410 Pitfalls from Caches
3411 \begin_inset CommandInset label
3412 LatexCommand label
3413 name "sub:Pitfalls-from-Caches"
3414
3415 \end_inset
3416
3417
3418 \end_layout
3419
3420 \begin_layout Standard
3421 It is easy to be caught by these pitfalls (even if you try to avoid them
3422 very hard), since caches occur very frequently in almost any type of storage
3423 system, and even in places where you don't expect them.
3424 In addition, some real-life loads have hidden properties you cannot see
3425 at first glance.
3426 \end_layout
3427
3428 \begin_layout Subsubsection
3429 Pitfalls from Cache Operation States
3430 \end_layout
3431
3432 \begin_layout Standard
3433 Many people believe that the most important case is
3434 \emph on
3435 cold caches
3436 \emph default
3437 versus
3438 \emph on
3439 hot caches
3440 \emph default
3441 .
3442 Although this is not completely wrong, it is not always fully true.
3443 There is another more important property of cache states:
3444 \series bold
3445 steady state
3446 \series default
3447 .
3448 \end_layout
3449
3450 \begin_layout Standard
3451 Steady state is not the same as hot state.
3452 When you start your system freshly, many caches will be of course empty
3453 \begin_inset Foot
3454 status open
3455
3456 \begin_layout Plain Layout
3457 There are some exceptions: SSD caches may start in a hot state (caused by
3458 your previous benchmark run), and BBU caches at RAID controllers may also
3459 survive even power failure.
3460 \end_layout
3461
3462 \end_inset
3463
3464 .
3465 An empty cache is always
3466 \begin_inset Quotes eld
3467 \end_inset
3468
3469 cold
3470 \begin_inset Quotes erd
3471 \end_inset
3472
3473 .
3474 During operation, it will be filled with data.
3475 A cache is called
3476 \begin_inset Quotes eld
3477 \end_inset
3478
3479 hot
3480 \begin_inset Quotes erd
3481 \end_inset
3482
3483 , if the
3484 \series bold
3485 cache hit rate
3486 \series default
3487 is
3488 \begin_inset Quotes eld
3489 \end_inset
3490
3491 significantly high enough
3492 \begin_inset Quotes erd
3493 \end_inset
3494
3495 .
3496 What's that? All of these terms are rather vague and depend on the application.
3497 However, some caches may
3498 \emph on
3499 never
3500 \emph default
3501 reach
3502 \begin_inset Quotes eld
3503 \end_inset
3504
3505 hot
3506 \begin_inset Quotes erd
3507 \end_inset
3508
3509 state, for example when the cache design / architecture is
3510 \begin_inset Quotes eld
3511 \end_inset
3512
3513 inefficient
3514 \begin_inset Quotes erd
3515 \end_inset
3516
3517 (cf section
3518 \begin_inset CommandInset ref
3519 LatexCommand ref
3520 reference "A-cache-is"
3521
3522 \end_inset
3523
3524 ).
3525 What then? The term
3526 \begin_inset Quotes eld
3527 \end_inset
3528
3529 hot
3530 \begin_inset Quotes erd
3531 \end_inset
3532
3533 is not the right one for describing that problem: when your cache never
3534 gets hots, your testing candidate will just fail your performance test,
3535 but your test as such will be valid: the result is just telling you that
3536 the cache is not tuned well enough for your application workload.
3537 \end_layout
3538
3539 \begin_layout Standard
3540 What is
3541 \begin_inset Quotes eld
3542 \end_inset
3543
3544 steady state
3545 \begin_inset Quotes erd
3546 \end_inset
3547
3548 ?
3549 \end_layout
3550
3551 \begin_layout Standard
3552 Intuitively, it just means that
3553 \begin_inset Quotes eld
3554 \end_inset
3555
3556 nothing changes (fundamentally) any more
3557 \begin_inset Quotes erd
3558 \end_inset
3559
3560 .
3561 In practice, there is a simple rule of thumb: your benchmark should just
3562 run
3563 \series bold
3564 long enough
3565 \series default
3566 to get into steady state.
3567 In section
3568 \begin_inset CommandInset ref
3569 LatexCommand ref
3570 reference "A-cache-is"
3571
3572 \end_inset
3573
3574 , some theoretical methods are described how to compute the time
3575 \begin_inset Formula $\delta$
3576 \end_inset
3577
3578 until steady state is reached.
3579 In practice, just make a few experiments in order to determine steady state
3580 intuitively.
3581 Many people will get a good feeling for
3582 \begin_inset Quotes eld
3583 \end_inset
3584
3585 steady state
3586 \begin_inset Quotes erd
3587 \end_inset
3588
3589 after some experience.
3590 \end_layout
3591
3592 \begin_layout Standard
3593 Once you know when you have reached the
3594 \begin_inset Quotes eld
3595 \end_inset
3596
3597 golden
3598 \begin_inset Quotes erd
3599 \end_inset
3600
3601 steady state, you can do one of two things:
3602 \end_layout
3603
3604 \begin_layout Enumerate
3605 ignore all benchmark results from the time span before steady state has
3606 been reached.
3607 \end_layout
3608
3609 \begin_layout Enumerate
3610 let the whole benchmark run
3611 \emph on
3612 at least
3613 \emph default
3614 10x as long as the time to reach steady state.
3615 The longer, the better.
3616 \end_layout
3617
3618 \begin_layout Subsubsection
3619 Pitfalls from Cache Size
3620 \end_layout
3621
3622 \begin_layout Standard
3623 The following illustration explains a general problem of cache hierarchies.
3624 Our example demonstrates an ill-designed behaviour: the lower cache is
3625
3626 \emph on
3627 smaller
3628 \emph default
3629 than the upper one.
3630 \end_layout
3631
3632 \begin_layout Standard
3633 \noindent
3634 \align center
3635 \begin_inset Graphics
3636 filename cache-toosmall.fig
3637 lyxscale 50
3638 width 75col%
3639
3640 \end_inset
3641
3642
3643 \end_layout
3644
3645 \begin_layout Standard
3646 \noindent
3647 Now assume that the LV has a size of several terabytes, and the workload
3648 is operating on a large part of that.
3649 On one hand, this is several orders of magnitude larger than the upper
3650 cache, but on the other hand typical application workloads will not access
3651 all sectors with the same uniform probability.
3652 In general, caches are only useful in two cases (non-exclusively):
3653 \end_layout
3654
3655 \begin_layout Enumerate
3656 the cache is strictly larger than the workload.
3657 \end_layout
3658
3659 \begin_layout Enumerate
3660 the workload contains unevenly distributed access frequencies.
3661 \end_layout
3662
3663 \begin_layout Standard
3664 Case 1 occurs only in special cases, such as in-memory databases (or workstation
3665 loads on large RAM machines).
3666 Case 2 appears more often.
3667 \end_layout
3668
3669 \begin_layout Standard
3670 Back to the above example: assume that only case 2 applies to the upper
3671 cache.
3672 Consequently, case 2 will also apply to the lower cache, because the lower
3673 cache is
3674 \emph on
3675 strictly smaller
3676 \emph default
3677 than the upper cache in this ill-designed example.
3678 Now assume we have an
3679 \emph on
3680 inclusive
3681 \emph default
3682
3683 \begin_inset Foot
3684 status open
3685
3686 \begin_layout Plain Layout
3687 In case of
3688 \emph on
3689 exclusive
3690 \emph default
3691 cache hierarchies, the whole picture can be
3692 \emph on
3693 approximately(!)
3694 \emph default
3695 replaced by a simplified one having only one cache level.
3696 The size of the new simplified cache is just the
3697 \emph on
3698 sum
3699 \emph default
3700 of the sizes of both original caches.
3701 \end_layout
3702
3703 \end_inset
3704
3705 cache hierarchy, and are using a standard cache eviction strategy like
3706 LRU.
3707 As a consequence from LRU (or any other strategy avoiding anomalies
3708 \begin_inset Foot
3709 status open
3710
3711 \begin_layout Plain Layout
3712 Probably the best known anomaly is the famous FIFO anomaly, as explained
3713 in most text books about operating systems.
3714 \end_layout
3715
3716 \end_inset
3717
3718 ), any sector present in the lower cache will be also be present in the
3719 upper cache
3720 \begin_inset Foot
3721 status open
3722
3723 \begin_layout Plain Layout
3724 This is just the
3725 \emph on
3726 definition
3727 \emph default
3728 of non-anomaly behaviour.
3729 \end_layout
3730
3731 \end_inset
3732
3733 .
3734 As a further consequence, we see that the lower cache is superfluous: by
3735 removing it, the system could even become faster
3736 \begin_inset Foot
3737 status open
3738
3739 \begin_layout Plain Layout
3740 Notice that there are exceptions.
3741 For example, internal memory caches present at hard disk drives are way
3742 too small to be able to contribute to classical hierarchical caching, but
3743 they can act as cylinder buffers for
3744 \emph on
3745 local aggregation strategies
3746 \emph default
3747 like readahead.
3748 \end_layout
3749
3750 \end_inset
3751
3752 due to less overhead.
3753 \end_layout
3754
3755 \begin_layout Standard
3756 What can we do about that?
3757 \end_layout
3758
3759 \begin_layout Standard
3760 Simply, just design your system in such a way that lower caches are always
3761 strictly larger than higher ones, by a factor of
3762 \begin_inset Formula $k$
3763 \end_inset
3764
3765 .
3766 In order to be useful,
3767 \begin_inset Formula $k\geq2$
3768 \end_inset
3769
3770 should be used, but for really good performance
3771 \begin_inset Formula $k\geq10$
3772 \end_inset
3773
3774 should be selected.
3775 \end_layout
3776
3777 \begin_layout Standard
3778 Probably you already know this, and you think you don't violate it.
3779 However, it is possible you might violate it unwillingly.
3780 The standard case is a server from a data center, equipped with several
3781 gigabytes of RAM.
3782 Almost all of the main memory can be used by the buffer/page caches of
3783 the Linux kernel.
3784 Therefore, you already may have a rather large upper cache you didn't think
3785 about.
3786 As a consequence, caches at the block storage level (e.g.
3787 SSD caches etc) should be larger by an order of magnitude (in this case
3788
3789 \begin_inset Formula $\sim1$
3790 \end_inset
3791
3792 TB or more).
3793 At the time of writing this paper, some commercial storage systems don't
3794 match this seemingly simple requirement.
3795 \end_layout
3796
3797 \begin_layout Standard
3798 There is another variant of this pitfall: records made by
3799 \family typewriter
3800 blktrace
3801 \family default
3802 are measuring the IO traffic
3803 \emph on
3804 below
3805 \emph default
3806 the buffer/page cache in most cases.
3807 Therefore, most (if not all) natural loads obtained by
3808 \family typewriter
3809 blktrace
3810 \family default
3811 contain effects
3812 \begin_inset Foot
3813 status open
3814
3815 \begin_layout Plain Layout
3816 One of the more well-known effects of caches is called
3817 \emph on
3818 cache inversion
3819 \emph default
3820 .
3821 At the time of writing this paper, Wikipedia didn't carry much about it.
3822 Consult a really good textbook or some research papers to learn more about
3823 it.
3824
3825 \end_layout
3826
3827 \end_inset
3828
3829 of the caches of the original system.
3830 In some cases (e.g.
3831 published loads from the
3832 \family typewriter
3833 blktrace
3834 \family default
3835 project), you don't know the original RAM size.
3836 Even if you know, you often cannot tell how large the page cache
3837 \emph on
3838 really
3839 \emph default
3840 was at the time of recording: how much RAM was spent for other purposes
3841 like processes, how much for other filesystems / partitions?
3842 \end_layout
3843
3844 \begin_layout Standard
3845 Even if you knew all that: do you know the
3846 \emph on
3847 workingset size
3848 \emph default
3849 of your application workload? Read on\SpecialChar \ldots{}
3850
3851 \end_layout
3852
3853 \begin_layout Subsection
3854 Pitfalls from Workingset Sizes
3855 \begin_inset CommandInset label
3856 LatexCommand label
3857 name "sub:Pitfalls-from-Workingset"
3858
3859 \end_inset
3860
3861
3862 \end_layout
3863
3864 \begin_layout Standard
3865 The workingset theory has been developped by Denning in the late 1960s,
3866 and has been originally used for the description of the behaviour of hardware-M
3867 MU-based paging / swapping systems and their strategies like LRU.
3868 It is also useful in other areas, such as block storage.
3869 Here is an adaptation of Denning's theory to our needs:
3870 \end_layout
3871
3872 \begin_layout Standard
3873 \begin_inset Formula
3874 \[
3875 WS(t,\delta)=\big\{\text{{set\,\ of\,\ all\,\ sectors\,\ touched\,\ in\,\ the\,\ time\,\ interval\,[t-\delta,t]}}\big\}
3876 \]
3877
3878 \end_inset
3879
3880
3881 \end_layout
3882
3883 \begin_layout Standard
3884 \noindent
3885 Usually,
3886 \begin_inset Formula $\delta$
3887 \end_inset
3888
3889 is treated as a constant, called
3890 \emph on
3891 window size
3892 \emph default
3893 .
3894 Then the workingset
3895 \begin_inset Formula $WS(t,\delta)$
3896 \end_inset
3897
3898 at some point in time
3899 \begin_inset Formula $t$
3900 \end_inset
3901
3902 is simply the set of sectors occurring in a
3903 \family typewriter
3904 blktrace
3905 \family default
3906 during a time window of
3907 \begin_inset Formula $\delta$
3908 \end_inset
3909
3910 seconds
3911 \emph on
3912 before
3913 \emph default
3914
3915 \begin_inset Foot
3916 status open
3917
3918 \begin_layout Plain Layout
3919 Notice: Dennings original theory used the time interval
3920 \begin_inset Formula $[t,t+\delta].$
3921 \end_inset
3922
3923 We find our definition more handy for practical purposes, because we need
3924 no
3925 \begin_inset Quotes eld
3926 \end_inset
3927
3928 lookahead
3929 \begin_inset Quotes erd
3930 \end_inset
3931
3932 into the
3933 \begin_inset Quotes eld
3934 \end_inset
3935
3936 future
3937 \begin_inset Quotes erd
3938 \end_inset
3939
3940 .
3941 \end_layout
3942
3943 \end_inset
3944
3945 the interesting point in time
3946 \begin_inset Formula $t$
3947 \end_inset
3948
3949 .
3950 Notice that we have a
3951 \emph on
3952 set
3953 \emph default
3954 here: it makes no difference how often a particular sector occurs during
3955 the time window
3956 \begin_inset Formula $\delta,$
3957 \end_inset
3958
3959 it just matters
3960 \emph on
3961 whether
3962 \emph default
3963 that sector appears or not.
3964 Also, it makes no difference whether a particular sector is read or written
3965 (or both).
3966 Thus the workingset model is a
3967 \emph on
3968 reduction
3969 \emph default
3970 of the reality to a handy theoretical model, but a model which is known
3971 to
3972 \emph on
3973 preserve
3974 \emph default
3975 some relevant and very interesting properties of the reality.
3976 \end_layout
3977
3978 \begin_layout Description
3979 Example: for any two window sizes
3980 \begin_inset Formula $\delta_{1}<\delta_{2},$
3981 \end_inset
3982
3983 the condition
3984 \begin_inset Formula $WS(t,\delta_{1})\leq WS(t,\delta_{2})$
3985 \end_inset
3986
3987 holds at any point in time
3988 \begin_inset Formula $t$
3989 \end_inset
3990
3991 .
3992 In other words: increasing the window size
3993 \begin_inset Formula $\delta$
3994 \end_inset
3995
3996 will never make the workingset smaller; the workingset can only
3997 \emph on
3998 grow
3999 \emph default
4000 if the window becomes larger.
4001 \end_layout
4002
4003 \begin_layout Standard
4004 The efficiency of block storage caches can be predicted by the workingset
4005 theory in a rather easy and intuitive way.
4006 We assume that accesses to the cache are
4007 \emph on
4008 much faster
4009 \emph default
4010 than accesses to the background storage, such that we can
4011 \emph on
4012 neglect
4013 \emph default
4014 the access times to the cache when compared to accesses to the background
4015 storage.
4016 Then we can model the following interesting property:
4017 \end_layout
4018
4019 \begin_layout Quote
4020 A cache is called to be designed
4021 \series bold
4022 efficiently
4023 \series default
4024
4025 \begin_inset CommandInset label
4026 LatexCommand label
4027 name "A-cache-is"
4028
4029 \end_inset
4030
4031 , iff at any point in time
4032 \begin_inset Formula $t$
4033 \end_inset
4034
4035 the following condition holds:
4036 \begin_inset Newline newline
4037 \end_inset
4038
4039
4040 \begin_inset Formula $|WS(t,\delta)|\leq|\text{{cachesize}}|$
4041 \end_inset
4042
4043 for some
4044 \begin_inset Formula $\delta\geq(\text{{time\,\ to\,\ fill\,\ the\,\ cache\,\ once}})$
4045 \end_inset
4046
4047 .
4048 \end_layout
4049
4050 \begin_layout Standard
4051 The potential painpoint is easy to see: just take
4052 \begin_inset Formula $\delta$
4053 \end_inset
4054
4055 as the time to fill the cache from the background storage, which is
4056 \begin_inset Formula $(\text{{accesstime\,\ to\,\ storage}})\star(\text{{cachesize}})$
4057 \end_inset
4058
4059 .
4060 If there would exist a point in time
4061 \begin_inset Formula $t$
4062 \end_inset
4063
4064 where the workingset
4065 \begin_inset Formula $WS(t,\delta)$
4066 \end_inset
4067
4068 would be
4069 \emph on
4070 larger
4071 \emph default
4072 than could be transferred to/from the storage during that
4073 \emph on
4074 same
4075 \emph default
4076 time window
4077 \begin_inset Formula $\delta$
4078 \end_inset
4079
4080 , the data transfers could become a
4081 \emph on
4082 bottleneck
4083 \emph default
4084 of the system.
4085 Vice versa: if the workingset during window
4086 \begin_inset Formula $\delta$
4087 \end_inset
4088
4089 would always
4090 \emph on
4091 fit
4092 \emph default
4093 into the cache, the cache will usually
4094 \begin_inset Foot
4095 status open
4096
4097 \begin_layout Plain Layout
4098 Of course, this holds only if the workload contains some
4099 \emph on
4100 repetitions
4101 \emph default
4102 during the window
4103 \begin_inset Formula $\delta$
4104 \end_inset
4105
4106 , and if the cache employs some
4107 \begin_inset Quotes eld
4108 \end_inset
4109
4110 good
4111 \begin_inset Quotes erd
4112 \end_inset
4113
4114 replacement strategy like LRU.
4115 The latter assures that
4116 \begin_inset Quotes eld
4117 \end_inset
4118
4119 hot
4120 \begin_inset Quotes erd
4121 \end_inset
4122
4123 pages from the workingset will be kept in the cache.
4124 In addition, keep in mind that neglecting the access times to the cache
4125 could be an
4126 \emph on
4127 oversimplification
4128 \emph default
4129 in many cases.
4130 \end_layout
4131
4132 \end_inset
4133
4134 speed up things.
4135 \end_layout
4136
4137 \begin_layout Standard
4138 Consequences:
4139 \end_layout
4140
4141 \begin_layout Itemize
4142 The
4143 \series bold
4144 workingset behaviour of the application
4145 \series default
4146 is
4147 \emph on
4148 crucial
4149 \emph default
4150 for any storage system.
4151
4152 \end_layout
4153
4154 \begin_layout Itemize
4155 The above argumentation contains an
4156 \emph on
4157 oversimplification
4158 \emph default
4159 : of course, accesses to the cache are not
4160 \begin_inset Quotes eld
4161 \end_inset
4162
4163 indefinitly fast
4164 \begin_inset Quotes erd
4165 \end_inset
4166
4167 as our above neglect assumed.
4168 Therefore, don't take the above inequalities as verbatim inequalities.
4169 Multiply some
4170 \emph on
4171 factor
4172 \emph default
4173 onto them.
4174 In practice, if you really want blastingly fast caches, make sure your
4175 cache is at least
4176 \series bold
4177 one order of magnitude larger
4178 \series default
4179 than the workingset size of your application.
4180 \end_layout
4181
4182 \begin_layout Standard
4183 In order to do that, you would need the know the workingset size of your
4184
4185 \emph on
4186 application
4187 \emph default
4188 .
4189 Please keep in mind that this is not the same as the workingset size measured
4190 by
4191 \family typewriter
4192 blktrace
4193 \family default
4194 :
4195 \end_layout
4196
4197 \begin_layout Standard
4198 \noindent
4199 \align center
4200 \begin_inset Graphics
4201 filename hierarchy.fig
4202 lyxscale 60
4203 width 30col%
4204
4205 \end_inset
4206
4207
4208 \end_layout
4209
4210 \begin_layout Standard
4211 \noindent
4212 As you can see, the measuring point of
4213 \family typewriter
4214 blktrace
4215 \family default
4216 sits
4217 \emph on
4218 below
4219 \emph default
4220 the buffer / page cache, therefore it does not
4221 \emph on
4222 directly
4223 \emph default
4224 measure the application behaviour (in addition to influences from the filesyste
4225 m, like metadata updates etc).
4226 In practice,
4227 \family typewriter
4228 blktrace
4229 \family default
4230 measurements may differ from the
4231 \begin_inset Quotes eld
4232 \end_inset
4233
4234 real
4235 \begin_inset Quotes erd
4236 \end_inset
4237
4238 application workload of stateless webserver designs by
4239 \emph on
4240 several
4241 \emph default
4242 orders of magnitude.
4243 However,
4244 \emph on
4245 all
4246 \emph default
4247 modern OSes employ caches.
4248 Even if we were able to measure the true application workingset in some
4249 way, it would not be relevant for block storage systems, because it would
4250 be
4251 \emph on
4252 impractical
4253 \emph default
4254 to operate those systems without caches.
4255 We need the above picture for understanding the fundamental properties
4256 of
4257 \family typewriter
4258 blktrace
4259 \family default
4260 measurements, and for determining the window size
4261 \begin_inset Formula $\delta$
4262 \end_inset
4263
4264 .
4265 If that is not possible, try to estimate it.
4266 As a very rough estimation, take
4267 \begin_inset Formula $\delta$
4268 \end_inset
4269
4270 as several minutes.
4271 \end_layout
4272
4273 \begin_layout Standard
4274 The
4275 \family typewriter
4276 blkreplay
4277 \family default
4278 suite comes with a small tool to visualize the workingset behaviour as
4279 measured by
4280 \family typewriter
4281 blktrace
4282 \family default
4283 , which is currently the best approximation of the workingset behaviour
4284 of the application we can easily get access to.
4285 Following is an example, showing different window sizes
4286 \begin_inset Formula $\delta$
4287 \end_inset
4288
4289 in the same picture (where 000 means cumulation to
4290 \begin_inset Formula $\infty$
4291 \end_inset
4292
4293 ):
4294 \end_layout
4295
4296 \begin_layout Standard
4297 \noindent
4298 \align center
4299 \begin_inset Graphics
4300 filename images/example1.g30.ws_log.png
4301 lyxscale 30
4302 width 100col%
4303
4304 \end_inset
4305
4306
4307 \end_layout
4308
4309 \begin_layout Labeling
4310 \labelwidthstring 00.00.0000
4311 \begin_inset Graphics
4312 filename images/MatieresCorrosives.png
4313 lyxscale 50
4314 scale 17
4315
4316 \end_inset
4317
4318 Spend some time on it! Your replay needs to last
4319 \emph on
4320 vastly longer
4321 \emph default
4322 than the
4323 \begin_inset Formula $\delta_{\text{{old}}}$
4324 \end_inset
4325
4326 needed to descibe the steady state of the original buffer/page cache, as
4327 well as the
4328 \begin_inset Formula $\delta_{\text{{new}}}$
4329 \end_inset
4330
4331 to describe the steady state of your replay system.
4332 Otherwise, you can get fake results which differ from real practical performanc
4333 e by factors, or even orders of magnitude.
4334 As a rough rule of thumb, any replay of natural loads should take at least
4335 one hour.
4336 Better, make a few measurements lasting 8 hours, or even 24 hours, and
4337 check whether the results differ more than expected (besides natural variations
4338 ).
4339 \end_layout
4340
4341 \begin_layout Subsection
4342 Pitfalls from Replay Device Sizes and Others
4343 \begin_inset CommandInset label
4344 LatexCommand label
4345 name "sub:Pitfalls-from-Replay"
4346
4347 \end_inset
4348
4349
4350 \end_layout
4351
4352 \begin_layout Standard
4353 There is a simple intuitive rule: your replay device for
4354 \family typewriter
4355 blkreplay
4356 \family default
4357 should have (almost) the
4358 \emph on
4359 same
4360 \emph default
4361 size as the original device where
4362 \family typewriter
4363 blktrace
4364 \family default
4365 has taken its measurement from
4366 \begin_inset Foot
4367 status open
4368
4369 \begin_layout Plain Layout
4370 If you don't know this, just make a test run with
4371 \family typewriter
4372 blkreplay
4373 \family default
4374 and check the output file.
4375 At the end, you will find a human-readable statistics showing the highest
4376 original block number occurring in the replay, as well as some other interestin
4377 g numbers.
4378 \end_layout
4379
4380 \end_inset
4381
4382 .
4383 \end_layout
4384
4385 \begin_layout Standard
4386 For easy checking, our output graphics will display the so-called
4387 \series bold
4388 wraparound_factor
4389 \series default
4390 in the lower right corner: it shows the ratio between the max block# occuring
4391 in the load, and the size of the replay device.
4392 Ideally, it should be near
4393 \begin_inset Formula $1.0$
4394 \end_inset
4395
4396 .
4397 When it deviates from this by more than a factor of two (or less than one
4398 halve), it is printed in purple color to warn you.
4399 \end_layout
4400
4401 \begin_layout Standard
4402 Some people don't take this seriously, and some don't even believe that
4403 this can have a
4404 \emph on
4405 tremendous
4406 \emph default
4407 effect.
4408 \end_layout
4409
4410 \begin_layout Standard
4411 Practical experience at 1&1 tells that the above rule is
4412 \emph on
4413 valid
4414 \emph default
4415 , and that results
4416 \emph on
4417 will
4418 \emph default
4419 almost certainly vary.
4420 The bias can be
4421 \emph on
4422 considerable
4423 \emph default
4424 .
4425 \end_layout
4426
4427 \begin_layout Standard
4428 Example: the original
4429 \family typewriter
4430 blktrace
4431 \family default
4432 recording was taken from a production server equipped with 20 TB RAID.
4433 Since in the lab we had only a smaller system with 4 TB,
4434 \family typewriter
4435 blktrace
4436 \family default
4437 measurements were run despite the smaller size.
4438 Whenever
4439 \family typewriter
4440 blkreplay
4441 \family default
4442 tries to start an IO request outside the LV size, it just
4443 \emph on
4444 remaps
4445 \emph default
4446 the sector number modulo the (new) LV size.
4447 Therefore, results
4448 \emph on
4449 appear
4450 \emph default
4451 to be valid, since you cannot see any
4452 \begin_inset Quotes eld
4453 \end_inset
4454
4455 big
4456 \begin_inset Quotes erd
4457 \end_inset
4458
4459 holes or anomalies.
4460 You will find out the difference only if you compare to the
4461 \emph on
4462 correct
4463 \emph default
4464 setup.
4465 When repeating the
4466 \emph on
4467 same
4468 \emph default
4469 measurements with correct LV sizes of ~20 TB, there is a
4470 \emph on
4471 significant
4472 \emph default
4473 difference.
4474 \end_layout
4475
4476 \begin_layout Standard
4477 Some people think that such differences can be attributed solely to the
4478 natural differences in
4479 \emph on
4480 spindle count
4481 \emph default
4482
4483 \begin_inset Foot
4484 status open
4485
4486 \begin_layout Plain Layout
4487 When using the same hard disk model, a 20 TB RAID must contain 5 times as
4488 much spindles as an equivalent 4 TB system.
4489 It is clear that additional spindles can benefit random IO.
4490 But those effects are
4491 \series bold
4492 non-linear
4493 \series default
4494 !
4495 \end_layout
4496
4497 \end_inset
4498
4499 , and therefore it would not hurt if different models of hard disks were
4500 used.
4501 Although there
4502 \emph on
4503 are
4504 \emph default
4505 some effects by spindle count, that opinion is wrong.
4506 In order to disprove such a
4507 \begin_inset Quotes eld
4508 \end_inset
4509
4510 theory
4511 \begin_inset Quotes erd
4512 \end_inset
4513
4514 , just build two different RAIDs with same spindle count, but fundamentally
4515 \begin_inset Foot
4516 status open
4517
4518 \begin_layout Plain Layout
4519 Of course, there are a lot of disk drives (in the same
4520 \begin_inset Quotes eld
4521 \end_inset
4522
4523 class
4524 \begin_inset Quotes erd
4525 \end_inset
4526
4527 ) showing only minor differences.
4528 A
4529 \begin_inset Quotes eld
4530 \end_inset
4531
4532 fundamental
4533 \begin_inset Quotes erd
4534 \end_inset
4535
4536 difference is for example between a cheap SATA disk versus a smaller but
4537 faster 15k SAS disk.
4538
4539 \end_layout
4540
4541 \end_inset
4542
4543 different disk drive models (resulting in different total capacity), and
4544 compare (under otherwise equal conditions).
4545 \end_layout
4546
4547 \begin_layout Standard
4548 In short, any of the following factors can influence the performance, independen
4549 tly from each other (and in no particular order):
4550 \end_layout
4551
4552 \begin_layout Itemize
4553 Total capacity, just by itself.
4554 If you don't believe it, just create LVs with 1/10 size of the physical
4555 storage and compare (on the
4556 \emph on
4557 same
4558 \emph default
4559 hardware) with results from the
4560 \emph on
4561 full
4562 \emph default
4563 physical storage size.
4564 Of course, the original recording must stem from sufficiently large devices,
4565 otherwise your
4566 \begin_inset Quotes eld
4567 \end_inset
4568
4569 disproof
4570 \begin_inset Quotes erd
4571 \end_inset
4572
4573 will be false.
4574 \end_layout
4575
4576 \begin_layout Itemize
4577 Model/class of disk drives.
4578 \end_layout
4579
4580 \begin_layout Itemize
4581 Number of spindles.
4582 \end_layout
4583
4584 \begin_layout Itemize
4585 RAID level.
4586 \end_layout
4587
4588 \begin_layout Itemize
4589 Vendor / firmware version of the controller.
4590 \end_layout
4591
4592 \begin_layout Itemize
4593 Interconnect technology, such as SATA vs SAS.
4594 \end_layout
4595
4596 \begin_layout Itemize
4597 any caches in the hierarchy, such as different BBU cache sizes or different
4598 parameters.
4599 \end_layout
4600
4601 \begin_layout Standard
4602 ...
4603 and probably many others.
4604 \end_layout
4605
4606 \begin_layout Labeling
4607 \labelwidthstring 00.00.0000
4608 \begin_inset Graphics
4609 filename images/MatieresCorrosives.png
4610 lyxscale 50
4611 scale 17
4612
4613 \end_inset
4614
4615 General rule:
4616 \emph on
4617 never
4618 \emph default
4619 expect any of these effects to be linear.
4620 Almost always, they are
4621 \series bold
4622 non-linear
4623 \series default
4624
4625 \begin_inset Foot
4626 status open
4627
4628 \begin_layout Plain Layout
4629 Non-linear effects cannot be combined with each other in a predictable way,
4630 at least in general.
4631 \end_layout
4632
4633 \end_inset
4634
4635 .
4636 Anyone claiming something else must
4637 \series bold
4638 prove
4639 \series default
4640 it!
4641 \end_layout
4642
4643 \begin_layout Standard
4644 Consequence: anyone violating the above rule produces
4645 \series bold
4646 invalid
4647 \series default
4648 results, unless proven the opposite!
4649 \end_layout
4650
4651 \begin_layout Subsection
4652 Pitfalls from Parallelism in IO Systems
4653 \end_layout
4654
4655 \begin_layout Subsubsection
4656 Pitfalls from
4657 \emph on
4658 missing
4659 \emph default
4660 Parallelism
4661 \end_layout
4662
4663 \begin_layout Standard
4664 This pitfall is usually trapping less people, because some intuitive sense
4665 for the effects of IO parallelism is more widespread.
4666 Even if you are already aware of this pitfall, read on.
4667 There are some subtleties.
4668 \end_layout
4669
4670 \begin_layout Standard
4671 \noindent
4672 \align center
4673 \begin_inset Graphics
4674 filename consolidation.fig
4675 lyxscale 50
4676 width 30text%
4677
4678 \end_inset
4679
4680
4681 \end_layout
4682
4683 \begin_layout Standard
4684 \noindent
4685 A frequent use case is
4686 \emph on
4687 storage consolidation
4688 \emph default
4689 .
4690 Up to
4691 \begin_inset Formula $k$
4692 \end_inset
4693
4694 CPU nodes are connected to some
4695 \begin_inset Quotes eld
4696 \end_inset
4697
4698 central
4699 \begin_inset Quotes erd
4700 \end_inset
4701
4702 storage via some kind of storage network.
4703 \end_layout
4704
4705 \begin_layout Standard
4706 You may want to evaluate such an architecture in advance with help of
4707 \family typewriter
4708 blkreplay
4709 \family default
4710 , in order to avoid failed invests.
4711 The
4712 \family typewriter
4713 blkreplay
4714 \family default
4715 suite will help you, because its supervisor scripts like
4716 \family typewriter
4717 tree-replay.sh
4718 \family default
4719 are ready to run masses of
4720 \family typewriter
4721 blkreplay
4722 \family default
4723 instances in parallel to each other, and on different nodes.
4724 \end_layout
4725
4726 \begin_layout Standard
4727 For obvious reasons, you should determine the optimum consolidation factor
4728
4729 \begin_inset Formula $k$
4730 \end_inset
4731
4732 for each given hardware candidate.
4733 If you vary the factor
4734 \begin_inset Formula $k$
4735 \end_inset
4736
4737 , you will get at least two effects:
4738 \end_layout
4739
4740 \begin_layout Enumerate
4741 With higher
4742 \begin_inset Formula $k$
4743 \end_inset
4744
4745 , you need to build up more CPU nodes and more network connections.
4746 Otherwise you will neglect effects from IO parallelism (which don't scale
4747 linearly
4748 \begin_inset Foot
4749 status open
4750
4751 \begin_layout Plain Layout
4752 There is no general way to predict the behaviour of an unknown system! Many
4753 non-linear effects show some kind of
4754 \begin_inset Quotes eld
4755 \end_inset
4756
4757 binary
4758 \begin_inset Quotes erd
4759 \end_inset
4760
4761 behaviour, suddenly collapsing at some point where you didn't expect it.
4762 \end_layout
4763
4764 \end_inset
4765
4766 ), at multiple places of the picture: at potential bottlenecks at the CPU
4767 nodes, at potential bottlenecks inside your storage network, and at lots
4768 of potential bottlenecks in your central storage system.
4769 \begin_inset Newline newline
4770 \end_inset
4771
4772 For example, it is completely wrong to just double the load on
4773 \begin_inset Formula $k/2$
4774 \end_inset
4775
4776 nodes.
4777 Experiences with some test candidates at 1&1 show that results can differ
4778 enormously from results from single loads, each on
4779 \begin_inset Formula $k$
4780 \end_inset
4781
4782 nodes.
4783 \end_layout
4784
4785 \begin_layout Enumerate
4786 The total capacity of the central storage will also vary with
4787 \begin_inset Formula $k$
4788 \end_inset
4789
4790 .
4791 As known from section
4792 \begin_inset CommandInset ref
4793 LatexCommand ref
4794 reference "sub:Pitfalls-from-Replay"
4795
4796 \end_inset
4797
4798 , this will not only influence results by itself, but in addition by further
4799 effects like spindle count, non-scaling of caches with factor
4800 \begin_inset Formula $k$
4801 \end_inset
4802
4803 , and many others.
4804 \end_layout
4805
4806 \begin_layout Subsubsection
4807 Pitfalls from
4808 \emph on
4809 too high
4810 \emph default
4811 IO Parallelism
4812 \end_layout
4813
4814 \begin_layout Standard
4815 You can tune the number of
4816 \family typewriter
4817 blkreplay
4818 \family default
4819 threads via the parameter
4820 \family typewriter
4821 threads=\SpecialChar \ldots{}
4822
4823 \family default
4824 (see appendix
4825 \begin_inset CommandInset ref
4826 LatexCommand ref
4827 reference "chap:Config-File-Parameters"
4828
4829 \end_inset
4830
4831 ).
4832 As a side effect, this will also influence the maximum number of outstanding
4833 IO requests which can be
4834 \begin_inset Quotes eld
4835 \end_inset
4836
4837 on the fly
4838 \begin_inset Quotes erd
4839 \end_inset
4840
4841 in parallel, together with the replay parallelism (number of
4842 \family typewriter
4843 blkreplay
4844 \family default
4845 instances running in parallel on the same physical system).
4846 \end_layout
4847
4848 \begin_layout Standard
4849 Many people believe that an increasing number of outstanding IO requests
4850 will improve overall throughput.
4851 \end_layout
4852
4853 \begin_layout Standard
4854 However, some devices / drivers / IO schedulers may respond in some
4855 \emph on
4856 counter-productive
4857 \emph default
4858 way when hammered with too many IO requests in parallel.
4859 Sometimes, your throughput can even
4860 \emph on
4861 collapse
4862 \emph default
4863 to less than 1/10 of the maximum.
4864 There are many possible reasons for this unexpected behaviour.
4865 \end_layout
4866
4867 \begin_layout Standard
4868 In most cases, you will notice collapsing effects only during
4869 \series bold
4870 overload situations
4871 \series default
4872 .
4873 During normal operation, you will not notice anything, because the device
4874 can
4875 \emph on
4876 catch up
4877 \emph default
4878 with the IO demands.
4879 There are almost no queues, because the
4880 \emph on
4881 average service time
4882 \emph default
4883 is shorter than the average request rate.
4884 \end_layout
4885
4886 \begin_layout Standard
4887 As soon as the device gets overloaded, the situation will change rapidly.
4888 Masses of
4889 \family typewriter
4890 blkreplay
4891 \family default
4892 threads are trying to fire off their requests in time, starting to overlap
4893 now because the service time increases.
4894 Suddenly, some queuing will take place, somewhere.
4895 In mathematical theory, the queue lengths could even grow indefinitely
4896 whenever the average request rate is higher than the average service time.
4897 In practice, there will be some limits somewhere.
4898 \end_layout
4899
4900 \begin_layout Standard
4901 Imagine you are going shopping in a supermarket.
4902 When the cashier girl cannot catch up with the demand from the customers,
4903 their shopping trolleys will start to form a queue in front of her.
4904 Now assume the following behaviour: the longer the queue, the
4905 \emph on
4906 slower
4907 \emph default
4908 she will work.
4909 Imagine that! After a while, customer satisfaction will go down to zero,
4910 because the queue will get longer and longer.
4911 And the longer, the slower she will work.
4912 And so on.
4913 There is no escape, other than stopping to buy anything from there.
4914 \end_layout
4915
4916 \begin_layout Standard
4917 A similar behaviour can be observed in some IO systems.
4918 Once a queue has formed, there is almost no escape from a behaviour similar
4919 to
4920 \series bold
4921 traffic jam
4922 \series default
4923 .
4924 If you cannot change the system, your only chance is to reduce load, or
4925 even to remove the load at all.
4926 \end_layout
4927
4928 \begin_layout Standard
4929 Sometimes it is even hard to trigger such behaviour.
4930 A systems appears to run smoothly for months, but suddenly it collapses.
4931 \end_layout
4932
4933 \begin_layout Standard
4934 If you have such a system (or cannot be sure to have one
4935 \begin_inset Foot
4936 status open
4937
4938 \begin_layout Plain Layout
4939 To find out, we recommend the artifical loads
4940 \family typewriter
4941 *bursts*.load.gz
4942 \family default
4943 for creating anal kinds of overload.
4944 \end_layout
4945
4946 \end_inset
4947
4948 ) and want to simulate its behaviour in the laboratory, you should be carefully
4949 tuning the
4950 \family typewriter
4951 threads
4952 \family default
4953 parameter.
4954 In reality, the number of outstanding requests is often limited in some
4955 way.
4956 For example, a typical workstation load caused by a single user has often
4957 some intrinsically limited IO parallelism, similar to a limited number
4958 of shopping trolleys in the supermarket.
4959 On the other hand, some server loads (such as those caused by Apache) can
4960
4961 \family typewriter
4962 fork()
4963 \family default
4964 off a high number of threads.
4965
4966 \end_layout
4967
4968 \begin_layout Standard
4969 So it is quite possible to observe some traffic jam behaviour in practice,
4970 depending on your application.
4971 It is quite possible that such behaviour is
4972 \series bold
4973 relevant
4974 \series default
4975 for IT operations, at least for
4976 \emph on
4977 some
4978 \emph default
4979 applications (but not for all).
4980 \end_layout
4981
4982 \begin_layout Labeling
4983 \labelwidthstring 00.00.0000
4984 \noindent
4985 \begin_inset Graphics
4986 filename images/MatieresToxiques.png
4987 lyxscale 50
4988 scale 17
4989
4990 \end_inset
4991
4992 Taking any of these
4993 \series bold
4994 non-linear
4995 \series default
4996 (and sometimes even
4997 \emph on
4998 binary collapsing
4999 \emph default
5000 ) effects not seriously and/or using a wrong number of threads / replay
5001 parallelism can easily lead to
5002 \series bold
5003 invalid results
5004 \series default
5005 , and in turn to failed invests!
5006 \end_layout
5007
5008 \begin_layout Standard
5009 Hint: a frequent case are distributed systems by
5010 \emph on
5011 themselves
5012 \emph default
5013 .
5014 They tend to produce
5015 \series bold
5016 queues
5017 \series default
5018 at times and in places where you don't expect them, and they tend to produce
5019 avalanche-like negative effects (self-amplifying), similar to suddenly
5020 appearing
5021 \series bold
5022 traffic jams
5023 \series default
5024 where you cannot determine one single reason in isolation.
5025 \end_layout
5026
5027 \begin_layout Section
5028 Recommended Setup and Usage
5029 \end_layout
5030
5031 \begin_layout Subsection
5032 Planning Phase
5033 \end_layout
5034
5035 \begin_layout Labeling
5036 \labelwidthstring 00.00.0000
5037 \begin_inset Graphics
5038 filename images/MatieresToxiques.png
5039 lyxscale 50
5040 scale 17
5041
5042 \end_inset
5043
5044 Never try to plan a project without deep knowledge of the pitfalls described
5045 in section
5046 \begin_inset CommandInset ref
5047 LatexCommand ref
5048 reference "sec:How-to-Avoid"
5049
5050 \end_inset
5051
5052 .
5053 In addition, some experience with
5054 \family typewriter
5055 blkreplay
5056 \family default
5057 is helpful.
5058 In order to gain such experience, consider a
5059 \emph on
5060 test project
5061 \emph default
5062 just for playing around, and for getting familiar with the pitfalls.
5063 \end_layout
5064
5065 \begin_layout Subsubsection
5066 Describe the Scope of Project
5067 \end_layout
5068
5069 \begin_layout Standard
5070 Before starting, you should get conscious with yourself.
5071
5072 \emph on
5073 What exactly
5074 \emph default
5075 is the question you want to answer with help of
5076 \family typewriter
5077 blkreplay
5078 \family default
5079 ?
5080 \end_layout
5081
5082 \begin_layout Standard
5083 Write down the question both as shortly, as well as precisely as possible.
5084 Here are some examples:
5085 \end_layout
5086
5087 \begin_layout Itemize
5088 Compare hardware vendor
5089 \family typewriter
5090 A
5091 \family default
5092 with
5093 \family typewriter
5094 B
5095 \family default
5096 and
5097 \family typewriter
5098 C
5099 \family default
5100 for my production workload X.
5101 \end_layout
5102
5103 \begin_layout Itemize
5104 Compare hardware vendor
5105 \family typewriter
5106 A
5107 \family default
5108 with
5109 \family typewriter
5110 B
5111 \family default
5112 and
5113 \family typewriter
5114 C
5115 \family default
5116 in general.
5117 \end_layout
5118
5119 \begin_layout Itemize
5120 Debug kernel module
5121 \family typewriter
5122 xxx
5123 \family default
5124 .
5125 \end_layout
5126
5127 \begin_layout Itemize
5128 Compare iSCSI with Fibrechannel for my production workload X.
5129 \end_layout
5130
5131 \begin_layout Itemize
5132 \SpecialChar \ldots{}
5133
5134 \end_layout
5135
5136 \begin_layout Standard
5137 Next, write down a precise description of your intended
5138 \series bold
5139 test environment
5140 \series default
5141 .
5142 Best practice is to name hardware vendors, models, all components (including
5143 intermediate gear like network switches), and so on.
5144 \end_layout
5145
5146 \begin_layout Standard
5147 Last step: describe all the parameters which know of, which
5148 \emph on
5149 could
5150 \emph default
5151 have an influence onto your test results.
5152 Example:
5153 \end_layout
5154
5155 \begin_layout Standard
5156 \noindent
5157 \align center
5158
5159 \size small
5160 \begin_inset Tabular
5161 <lyxtabular version="3" rows="7" columns="2">
5162 <features tabularvalignment="middle">
5163 <column alignment="left" valignment="top" width="0pt">
5164 <column alignment="center" valignment="top" width="0">
5165 <row>
5166 <cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
5167 \begin_inset Text
5168
5169 \begin_layout Plain Layout
5170
5171 \size small
5172 Parameter
5173 \end_layout
5174
5175 \end_inset
5176 </cell>
5177 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
5178 \begin_inset Text
5179
5180 \begin_layout Plain Layout
5181
5182 \size small
5183 Varying?
5184 \end_layout
5185
5186 \end_inset
5187 </cell>
5188 </row>
5189 <row>
5190 <cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
5191 \begin_inset Text
5192
5193 \begin_layout Plain Layout
5194
5195 \size small
5196 16 GB RAM in storage node
5197 \end_layout
5198
5199 \end_inset
5200 </cell>
5201 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
5202 \begin_inset Text
5203
5204 \begin_layout Plain Layout
5205
5206 \size small
5207 no
5208 \end_layout
5209
5210 \end_inset
5211 </cell>
5212 </row>
5213 <row>
5214 <cell alignment="left" valignment="top" topline="true" leftline="true" usebox="none">
5215 \begin_inset Text
5216
5217 \begin_layout Plain Layout
5218
5219 \size small
5220 10GBit vs 1 GBit network speed
5221 \end_layout
5222
5223 \end_inset
5224 </cell>
5225 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
5226 \begin_inset Text
5227
5228 \begin_layout Plain Layout
5229
5230 \size small
5231 yes
5232 \end_layout
5233
5234 \end_inset
5235 </cell>
5236 </row>
5237 <row>
5238 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
5239 \begin_inset Text
5240
5241 \begin_layout Plain Layout
5242
5243 \size small
5244 Cheap 2TB SATA vs expensive 600GB raptor disk (model
5245 \begin_inset Quotes eld
5246 \end_inset
5247
5248 tyrannosaurus rex
5249 \begin_inset Quotes erd
5250 \end_inset
5251
5252 )
5253 \end_layout
5254
5255 \end_inset
5256 </cell>
5257 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
5258 \begin_inset Text
5259
5260 \begin_layout Plain Layout
5261
5262 \size small
5263 yes
5264 \end_layout
5265
5266 \end_inset
5267 </cell>
5268 </row>
5269 <row>
5270 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
5271 \begin_inset Text
5272
5273 \begin_layout Plain Layout
5274 RAID-Level
5275 \begin_inset Formula $\in\{1,5,6\}$
5276 \end_inset
5277
5278
5279 \end_layout
5280
5281 \end_inset
5282 </cell>
5283 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
5284 \begin_inset Text
5285
5286 \begin_layout Plain Layout
5287 yes
5288 \end_layout
5289
5290 \end_inset
5291 </cell>
5292 </row>
5293 <row>
5294 <cell alignment="center" valignment="top" topline="true" leftline="true" usebox="none">
5295 \begin_inset Text
5296
5297 \begin_layout Plain Layout
5298 RAID Stripesize
5299 \begin_inset Formula $\in\{16,32,64,128\}$
5300 \end_inset
5301
5302 kB
5303 \end_layout
5304
5305 \end_inset
5306 </cell>
5307 <cell alignment="center" valignment="top" topline="true" leftline="true" rightline="true" usebox="none">
5308 \begin_inset Text
5309
5310 \begin_layout Plain Layout
5311 yes
5312 \end_layout
5313
5314 \end_inset
5315 </cell>
5316 </row>
5317 <row>
5318 <cell alignment="left" valignment="top" topline="true" bottomline="true" leftline="true" usebox="none">
5319 \begin_inset Text
5320
5321 \begin_layout Plain Layout
5322
5323 \size small
5324 \SpecialChar \ldots{}
5325
5326 \end_layout
5327
5328 \end_inset
5329 </cell>
5330 <cell alignment="center" valignment="top" topline="true" bottomline="true" leftline="true" rightline="true" usebox="none">
5331 \begin_inset Text
5332
5333 \begin_layout Plain Layout
5334
5335 \size small
5336 \SpecialChar \ldots{}
5337
5338 \end_layout
5339
5340 \end_inset
5341 </cell>
5342 </row>
5343 </lyxtabular>
5344
5345 \end_inset
5346
5347
5348 \end_layout
5349
5350 \begin_layout Subsubsection
5351 Describe the Setup of your Experiment
5352 \end_layout
5353
5354 \begin_layout Standard
5355 In many cases, the parameters described in your table will make up a multi-dimen
5356 sional problem space (cartesian product) which is too large to be explored
5357 exhaustively.
5358 As explained in section
5359 \begin_inset CommandInset ref
5360 LatexCommand ref
5361 reference "sub:Pitfalls-from-Replay"
5362
5363 \end_inset
5364
5365 , many of them will influence your results
5366 \emph on
5367 non-linearly
5368 \emph default
5369 .
5370 \end_layout
5371
5372 \begin_layout Standard
5373 Thus, you will have to consider the following general strategies:
5374 \end_layout
5375
5376 \begin_layout Itemize
5377 You may fix some of the parameters to particular values.
5378 Although this saves time, you may miss an opportunity to find an optimal
5379 solution.
5380 \end_layout
5381
5382 \begin_layout Itemize
5383 You may select some/enough random samples from your multi-dimensional problem
5384 space and try them randomly (Monte Carlo methods).
5385 \end_layout
5386
5387 \begin_layout Itemize
5388 Stepwise refinement: explore the multidimensional space by varying exactly
5389
5390 \emph on
5391 one
5392 \emph default
5393 paramenter at once.
5394 This is slow, but you can be sure of the effects caused by this.
5395 \end_layout
5396
5397 \begin_layout Subsubsection
5398 Select
5399 \family typewriter
5400 blkreplay
5401 \family default
5402 Load
5403 \begin_inset CommandInset label
5404 LatexCommand label
5405 name "sub:Select-blkreplay-Load"
5406
5407 \end_inset
5408
5409
5410 \end_layout
5411
5412 \begin_layout Standard
5413 Depending on your project, you should consider both artificial and natural
5414 loads, but not too many of them.
5415 Usually, more than three loads are impractical for an ambitious project
5416 (unless you want to compare masses of loads on the same reference hardware).
5417 \end_layout
5418
5419 \begin_layout Standard
5420 If your project tries to answer a question for some specific workload X,
5421 you should just record that workload if you can do so (see chapter
5422 \begin_inset CommandInset ref
5423 LatexCommand ref
5424 reference "chap:How-to-use"
5425
5426 \end_inset
5427
5428 ).
5429 \end_layout
5430
5431 \begin_layout Standard
5432 If you cannot obtain your real workload in advance, you have to select one
5433 from the
5434 \family typewriter
5435 blkreplay
5436 \family default
5437 project (or other sources) which comes as close as possible to your (intended)
5438 natural workload.
5439 \end_layout
5440
5441 \begin_layout Standard
5442 For the sake of static comparison of workloads,
5443 \family typewriter
5444 cd
5445 \family default
5446 to a directory containing your
5447 \family typewriter
5448 *.load.gz
5449 \family default
5450 files and issue the following command:
5451 \end_layout
5452
5453 \begin_layout Quotation
5454
5455 \family typewriter
5456 /path/to/graph.sh --static myname.load.gz
5457 \end_layout
5458
5459 \begin_layout Standard
5460 This will produce some
5461 \family typewriter
5462 .png
5463 \family default
5464 graphics, describing the throughput and workingset behaviour of your load
5465 (cf section
5466 \begin_inset CommandInset ref
5467 LatexCommand ref
5468 reference "sub:Pitfalls-from-Workingset"
5469
5470 \end_inset
5471
5472 ).
5473 \end_layout
5474
5475 \begin_layout Labeling
5476 \labelwidthstring 00.00.0000
5477 \begin_inset Graphics
5478 filename images/MatieresCorrosives.png
5479 lyxscale 50
5480 scale 17
5481
5482 \end_inset
5483
5484 You need enough space in
5485 \family typewriter
5486 /tmp/
5487 \family default
5488 (or in another
5489 \family typewriter
5490 $TMPDIR
5491 \family default
5492 ) for temporary intermediate files.
5493 If your
5494 \family typewriter
5495 *.load.gz
5496 \family default
5497 file is very large (several hours or even days), you may need several gigabytes.
5498 Please don't interrupt
5499 \family typewriter
5500 graph.sh
5501 \family default
5502 as it spawns lots of subprocesses and creates lots of temporary files.
5503 Currently, there are no checks for free space in
5504 \family typewriter
5505 /tmp/
5506 \family default
5507 , so running out of space may produce wrong results
5508 \emph on
5509 silently
5510 \emph default
5511 .
5512 As a countermeasure, run
5513 \family typewriter
5514 watch df /tmp/
5515 \family default
5516 in a separate window during your run of
5517 \family typewriter
5518 graph.sh
5519 \family default
5520 .
5521 \end_layout
5522
5523 \begin_layout Standard
5524 Hint: some
5525 \family typewriter
5526 blktrace
5527 \family default
5528 recordings (but not all) contain some timing information about the original
5529 IO latencies as measured at the original site.
5530 Use
5531 \family typewriter
5532 /path/to/graph.sh --dynamic myname.load.gz
5533 \family default
5534 to create some additional graphics about them.
5535 \end_layout
5536
5537 \begin_layout Subsubsection
5538 Selection of Replay Duration
5539 \end_layout
5540
5541 \begin_layout Standard
5542 This is a hairy problem, as already described in section
5543 \begin_inset CommandInset ref
5544 LatexCommand ref
5545 reference "sub:Pitfalls-from-Workingset"
5546
5547 \end_inset
5548
5549 .
5550 Often, you cannot run 24h replays for many hundred times.
5551 \end_layout
5552
5553 \begin_layout Standard
5554 If you want to be sure that a particular load will run even under
5555 \emph on
5556 worst-case
5557 \emph default
5558 conditions, you should definitly select some appropriate time window around
5559
5560 \emph on
5561 load peaks
5562 \emph default
5563 , measured both in throughput as well as in workingset size.
5564 \end_layout
5565
5566 \begin_layout Standard
5567 In addition, you can try the following strategies:
5568 \end_layout
5569
5570 \begin_layout Itemize
5571 For explorative phases in your project, such as determining the optimum
5572 in your parameter space, you can try to minimize running times as much
5573 as possible.
5574 But not too much.
5575 Otherwise you will be caught by the pitfalls described in section
5576 \begin_inset CommandInset ref
5577 LatexCommand ref
5578 reference "sec:How-to-Avoid"
5579
5580 \end_inset
5581
5582 .
5583 \end_layout
5584
5585 \begin_layout Itemize
5586 For final verification of your results, you should repeat benchmarks with
5587 a longer window (at least 8h or 24h).
5588 \end_layout
5589
5590 \begin_layout Subsubsection
5591 Total Project Time
5592 \end_layout
5593
5594 \begin_layout Standard
5595 Working with huge parameter spaces is not all you have to consider.
5596 Setup of different RAID levels, re-initialization after changes of stripe
5597 sizes, filling LVs with random data, etc, may take a very long time, in
5598 addition to the benchmark themselves.
5599 Don't forget that! Your only chance are nights and weekends, if you manage
5600 to run something unattendedly.
5601 But predictions are sometimes wrong.
5602 In addition, something may fail and then needs to be restarted.
5603 Calculate some spare time for that!
5604 \end_layout
5605
5606 \begin_layout Standard
5607 If there is a high time pressure in the project timeline, you probably will
5608 have to
5609 \emph on
5610 rework
5611 \emph default
5612 some parts of your project plan.
5613 \end_layout
5614
5615 \begin_layout Labeling
5616 \labelwidthstring 00.00.0000
5617 \begin_inset Graphics
5618 filename images/MatieresCorrosives.png
5619 lyxscale 50
5620 scale 17
5621
5622 \end_inset
5623
5624 Planning is crucial! When you find any discrepancies, try to re-think your
5625 plan as a whole, not just some parts of it.
5626 \end_layout
5627
5628 \begin_layout Subsection
5629 Setup Phase
5630 \end_layout
5631
5632 \begin_layout Subsubsection
5633 Lab setup
5634 \end_layout
5635
5636 \begin_layout Standard
5637 Ensure that all your hard- and software components are ready in the lab
5638 and operational.
5639 \end_layout
5640
5641 \begin_layout Standard
5642 In addition, you need some workstation (or server) where the
5643 \family typewriter
5644 blkreplay
5645 \family default
5646 suite is checked out.
5647 Do the following steps:
5648 \end_layout
5649
5650 \begin_layout Itemize
5651 Ensure that
5652 \family typewriter
5653 gcc
5654 \family default
5655 ,
5656 \family typewriter
5657 make
5658 \family default
5659 ,
5660 \family typewriter
5661 gnuplot
5662 \family default
5663 , and some standard tools like
5664 \family typewriter
5665 grep
5666 \family default
5667 /
5668 \family typewriter
5669 gawk
5670 \family default
5671 are installed.
5672 If you need multiple machine architectures (such as
5673 \family typewriter
5674 x86_64
5675 \family default
5676 and
5677 \family typewriter
5678 i386
5679 \family default
5680 ) in parallel, ensure that
5681 \family typewriter
5682 gcc
5683 \family default
5684 can cross-compile via flags
5685 \family typewriter
5686 -m32
5687 \family default
5688 /
5689 \family typewriter
5690 -m64
5691 \family default
5692 and that the appropriate libs are installed.
5693 \end_layout
5694
5695 \begin_layout Itemize
5696
5697 \family typewriter
5698 git clone https://github.com/schoebel/blkreplay
5699 \end_layout
5700
5701 \begin_layout Itemize
5702
5703 \family typewriter
5704 cd blkreplay
5705 \end_layout
5706
5707 \begin_layout Itemize
5708
5709 \family typewriter
5710 ./configure
5711 \end_layout
5712
5713 \begin_layout Itemize
5714
5715 \family typewriter
5716 make
5717 \end_layout
5718
5719 \begin_layout Itemize
5720 Attention!
5721 \family typewriter
5722 make install
5723 \family default
5724 is not yet supported.
5725 Just leave everything in place.
5726 You can either put
5727 \family typewriter
5728
5729 \begin_inset Flex URL
5730 status open
5731
5732 \begin_layout Plain Layout
5733
5734 /path/to/blkreplay/scripts/
5735 \end_layout
5736
5737 \end_inset
5738
5739
5740 \family default
5741 into your
5742 \family typewriter
5743 $PATH
5744 \family default
5745 , or call the scripts via hard path.
5746 \end_layout
5747
5748 \begin_layout Standard
5749 Ensure that all your test machines are reachable as
5750 \family typewriter
5751 root
5752 \family default
5753 via
5754 \family typewriter
5755 ssh
5756 \family default
5757 from your central workplace, without need for any password prompt.
5758 In order to achieve that, you should consider
5759 \family typewriter
5760 ssh-agent
5761 \family default
5762 , in addition so some tweaking of
5763 \begin_inset Flex URL
5764 status open
5765
5766 \begin_layout Plain Layout
5767
5768 /etc/ssh/ssh_config
5769 \end_layout
5770
5771 \end_inset
5772
5773 (and probably
5774 \begin_inset Flex URL
5775 status open
5776
5777 \begin_layout Plain Layout
5778
5779 /etc/ssh/sshd_config
5780 \end_layout
5781
5782 \end_inset
5783
5784 on each of the target hosts).
5785 \end_layout
5786
5787 \begin_layout Standard
5788 On your workstation, you should have enough disk space to store your results.
5789 Create a subdirectory there for your project.
5790 Copy
5791 \begin_inset Flex URL
5792 status open
5793
5794 \begin_layout Plain Layout
5795
5796 /path/to/blkreplay/example-run/default-main.conf
5797 \end_layout
5798
5799 \end_inset
5800
5801 (and possibly other
5802 \family typewriter
5803 *.conf
5804 \family default
5805 files) to that new subdirectory, and finally
5806 \family typewriter
5807 cd
5808 \family default
5809 to it.
5810 For the rest of your life, you will be working there ;)
5811 \end_layout
5812
5813 \begin_layout Standard
5814 You can now either call
5815 \family typewriter
5816 /path/to/blkreplay/scripts/
5817 \emph on
5818 something
5819 \emph default
5820 .sh
5821 \family default
5822 as hard paths as indicated in the following examples, or you may put
5823 \family typewriter
5824 /path/to/blkreplay/scripts/
5825 \family default
5826 into your
5827 \family typewriter
5828 $PATH
5829 \family default
5830 .
5831 \end_layout
5832
5833 \begin_layout Standard
5834 Customization of
5835 \family typewriter
5836 default-main.conf
5837 \family default
5838 is described in the following.
5839 \end_layout
5840
5841 \begin_layout Subsubsection
5842 Configuration Files
5843 \end_layout
5844
5845 \begin_layout Standard
5846 You should edit
5847 \family typewriter
5848 default-main.conf
5849 \family default
5850 to reflect the default setup for your project.
5851 If you want to run multiple variants of your default setup, you can do
5852 so by creating additional files like
5853 \family typewriter
5854 \emph on
5855 something
5856 \emph default
5857 .conf
5858 \family default
5859 as well as a subdirectory
5860 \family typewriter
5861 \emph on
5862 something
5863 \emph default
5864 /
5865 \family default
5866 (having the same name without suffix
5867 \family typewriter
5868 .conf
5869 \family default
5870 ).
5871 When you later start your benchmark, the values from
5872 \family typewriter
5873 \emph on
5874 something
5875 \emph default
5876 .conf
5877 \family default
5878 will override those from
5879 \family typewriter
5880 default-main.conf
5881 \family default
5882 .
5883 It is highly recommended to override only
5884 \emph on
5885 one
5886 \emph default
5887 parameter inside
5888 \family typewriter
5889 \emph on
5890 something
5891 \emph default
5892 .conf
5893 \family default
5894 , otherwise it may become difficult to reveal the real impact of changed
5895 parameters onto your test candidate.
5896 \end_layout
5897
5898 \begin_layout Standard
5899 In general, you may override
5900 \emph on
5901 any
5902 \emph default
5903 parameter from
5904 \family typewriter
5905 default-main.conf
5906 \family default
5907 , even hostnames, or input files
5908 \family typewriter
5909 *.load.gz
5910 \family default
5911 , or whatever.
5912 \end_layout
5913
5914 \begin_layout Subsubsection
5915 Meaning of the Config File Parameters
5916 \end_layout
5917
5918 \begin_layout Standard
5919 The meaning of the parameters is documented in the following places:
5920 \end_layout
5921
5922 \begin_layout Enumerate
5923 Comments inside
5924 \family typewriter
5925 default-main.conf
5926 \family default
5927 should provide enough information for experienced administrators, at least
5928 for a quick start, and should guide you through the most basic steps.
5929 \end_layout
5930
5931 \begin_layout Enumerate
5932 The same information is available in appendix
5933 \begin_inset CommandInset ref
5934 LatexCommand ref
5935 reference "chap:Config-File-Parameters"
5936
5937 \end_inset
5938
5939 .
5940 \end_layout
5941
5942 \begin_layout Enumerate
5943 Last but not least: read the sources, if you are in doubt about anything.
5944 \end_layout
5945
5946 \begin_layout Subsection
5947 Benchmark Phase
5948 \end_layout
5949
5950 \begin_layout Standard
5951 The basic idea is simple: after customization of
5952 \family typewriter
5953 default-main.conf
5954 \family default
5955 (and probably other
5956 \family typewriter
5957 default-*.conf
5958 \family default
5959 files when using additional modules), you create a new subdirectory for
5960 each benchmark run.
5961 \end_layout
5962
5963 \begin_layout Standard
5964 Whenever you call
5965 \family typewriter
5966
5967 \begin_inset Flex URL
5968 status open
5969
5970 \begin_layout Plain Layout
5971
5972 /path/to/blkreplay/scripts/tree-replay.sh
5973 \end_layout
5974
5975 \end_inset
5976
5977
5978 \family default
5979 (without parameters), a whole bundle of benchmarks will be started, one
5980 for each
5981 \emph on
5982 leaf
5983 \emph default
5984
5985 \begin_inset Foot
5986 status open
5987
5988 \begin_layout Plain Layout
5989 A leaf has no further subtree inside it.
5990 \end_layout
5991
5992 \end_inset
5993
5994 subdirectory (starting from
5995 \family typewriter
5996 cwd
5997 \family default
5998 ), provided that for each (intermedidate) subdirectory name
5999 \family typewriter
6000 \emph on
6001 xxx
6002 \family default
6003 \emph default
6004 there exists some
6005 \family typewriter
6006 \emph on
6007 xxx
6008 \emph default
6009 .conf
6010 \family default
6011 in the current working directory or in one of its parents.
6012 In the whole subdirectory structure, any directory ending with
6013 \family typewriter
6014 .old
6015 \family default
6016 , or including the substring
6017 \family typewriter
6018 ignore
6019 \family default
6020 , or containg a file
6021 \family typewriter
6022 skip
6023 \family default
6024 will be ignored.
6025 \end_layout
6026
6027 \begin_layout Labeling
6028 \labelwidthstring 00.00.0000
6029 Example: you have created a subdirectory
6030 \family typewriter
6031 ./short/
6032 \family default
6033 as well as two nested subdirectories
6034 \family typewriter
6035 ./short/model1/
6036 \family default
6037 and
6038 \family typewriter
6039 ./short/model2/
6040 \family default
6041 .
6042 You further have prepared the config files
6043 \family typewriter
6044 short.conf
6045 \family default
6046 ,
6047 \family typewriter
6048 model1.conf
6049 \family default
6050 and
6051 \family typewriter
6052 model2.conf
6053 \family default
6054 , existing in the current working directory or in some parent directory
6055 down the
6056 \family typewriter
6057 ../
6058 \family default
6059 chain.
6060 Then exactly
6061 \emph on
6062 two
6063 \emph default
6064 benchmark runs will be started, namely in
6065 \family typewriter
6066 ./short/model1/
6067 \family default
6068 and in
6069 \family typewriter
6070 ./short/model2/
6071 \family default
6072 .
6073 The benchmark running in
6074 \family typewriter
6075 ./short/model1/
6076 \family default
6077 will include the following
6078 \family typewriter
6079 *.conf
6080 \family default
6081 files, in the following order:
6082 \family typewriter
6083 default-main.conf
6084 \family default
6085 ,
6086 \family typewriter
6087 short.conf
6088 \family default
6089 , and finally
6090 \family typewriter
6091 model1.conf
6092 \family default
6093 .
6094 Each of the specialized config files may override any previous setting,
6095 but it is highly recommended to change only one parameter at a time and
6096 to use short but expressive names.
6097 Notice: the intermediate directory
6098 \family typewriter
6099 ./short/
6100 \family default
6101 is no leaf (since it contains some subdirectories), therefore no benchmark
6102 will be started inside it.
6103 Later, you just need to create
6104 \family typewriter
6105 ./long/
6106 \family default
6107 as well as
6108 \family typewriter
6109 ./long/model1/
6110 \family default
6111 and
6112 \family typewriter
6113 ./long/model2/
6114 \family default
6115 and some
6116 \family typewriter
6117 long.conf
6118 \family default
6119 in order to repeat the same benchmarks with a longer
6120 \family typewriter
6121 replay_duration
6122 \family default
6123 setting.
6124 \end_layout
6125
6126 \begin_layout Standard
6127 Hint: using
6128 \begin_inset Quotes eld
6129 \end_inset
6130
6131 intuitive
6132 \begin_inset Quotes erd
6133 \end_inset
6134
6135 names like
6136 \family typewriter
6137 short
6138 \family default
6139 and
6140 \family typewriter
6141 long
6142 \family default
6143 bears some danger.
6144 A few years later, you will not remember what they exactly have meant.
6145 Looking into
6146 \family typewriter
6147 *.conf
6148 \family default
6149 will not help other people, for example if you publish your benchmark results
6150 somewhere.
6151 Therefore it may be wise to use
6152 \begin_inset Quotes eld
6153 \end_inset
6154
6155 speaking
6156 \begin_inset Quotes erd
6157 \end_inset
6158
6159 names like
6160 \family typewriter
6161 duration_600
6162 \family default
6163 , at least if you have more than two variants.
6164 On the other hand,
6165 \begin_inset Quotes eld
6166 \end_inset
6167
6168 intuitive
6169 \begin_inset Quotes erd
6170 \end_inset
6171
6172 names are better for presentation to some less-deeply involved audience.
6173 Take some time for creating well-designed names for
6174 \family typewriter
6175 *.conf
6176 \family default
6177 and your directory hierarchy! Changing that names later is cumbersome.
6178 Better to design your names in advance in a systematic (but simple) way.
6179 \end_layout
6180
6181 \begin_layout Standard
6182 On large investigation projects, deeply nested structures may be necessary,
6183 involving different loads, different hardware, different hardware setup,
6184 etc.
6185 Not all of them are currently automated.
6186 You can use the generic module mechanism to extend the default scripts
6187 with further functionality, to push automation further.
6188 \end_layout
6189
6190 \begin_layout Standard
6191 However, not all setup tasks
6192 \emph on
6193 can
6194 \emph default
6195 be automated at all.
6196 Some of them like forcing physical RAID degradation must be started by
6197 physically removing a disk, which cannot be automated (other than buying
6198 extremely expensive robots).
6199 Therefore, you may include some human-readable dialogs inside your
6200 \family typewriter
6201 *.conf
6202 \family default
6203 files (in shell script syntax), or in some new modules you have written.
6204 In any case, it is advisable to write some script code to
6205 \emph on
6206 check some preconditions
6207 \emph default
6208 (such as RAID status) in order to prevent wrong measurements.
6209 \end_layout
6210
6211 \begin_layout Standard
6212 In general,
6213 \family typewriter
6214 tree-replay.sh
6215 \family default
6216 will never repeat any benchmark which has already completed (i.e.
6217 there exists an output file
6218 \family typewriter
6219 *.replay.gz
6220 \family default
6221 in that leaf directory).
6222 This allows an incremental style of working.
6223 \end_layout
6224
6225 \begin_layout Labeling
6226 \labelwidthstring 00.00.0000
6227 Continued
6228 \begin_inset space ~
6229 \end_inset
6230
6231 example: normally, you can call
6232 \family typewriter
6233 tree-replay.sh
6234 \family default
6235 again, but nothing happens, because all leaf directories already contain
6236 some results.
6237 However, then you find a problem with the results in
6238 \family typewriter
6239 ./short/model1/
6240 \family default
6241 , so you want to repeat that benchmark without deleting your first (questionable
6242 ) results.
6243 Now you create a subdirectory
6244 \family typewriter
6245 ./short/model1/try1.old/
6246 \family default
6247 and move your results there.
6248 As said above, any directory names ending with
6249 \family typewriter
6250 .old
6251 \family default
6252 or containing the substring
6253 \family typewriter
6254 ignore
6255 \family default
6256 are ignored, so the directory
6257 \family typewriter
6258 ./short/model1/
6259 \family default
6260 will continue to count as a leaf (despite its newer subdirectory, which
6261 is just ignored).
6262 Since the
6263 \family typewriter
6264 *.replay.gz
6265 \family default
6266 files are now missing in
6267 \family typewriter
6268 ./short/model1/
6269 \family default
6270 due to the
6271 \family typewriter
6272 mv
6273 \family default
6274 ,
6275 \family typewriter
6276 tree-replay.sh
6277 \family default
6278 will repeat that single benchmark there.
6279 \end_layout
6280
6281 \begin_layout Standard
6282 Here are some useful hints:
6283 \end_layout
6284
6285 \begin_layout Itemize
6286 You may skip any directory by creating a file
6287 \family typewriter
6288 skip
6289 \family default
6290 inside it.
6291
6292 \family typewriter
6293 touch ./short/model1/skip
6294 \family default
6295 will disable that directory.
6296 Later, you can remove that file in order to fire off that benchmark.
6297 \end_layout
6298
6299 \begin_layout Itemize
6300
6301 \family typewriter
6302 skip
6303 \family default
6304 files are also working in intermediate directories like
6305 \family typewriter
6306 ./short/
6307 \family default
6308 , disabling the whole subtree in one step.
6309 \end_layout
6310
6311 \begin_layout Itemize
6312 Design your
6313 \family typewriter
6314 *.conf
6315 \family default
6316 files such that arbitrary combinations are
6317 \emph on
6318 possible
6319 \emph default
6320 (cartesian product).
6321 In contrast, your directory hierarchy need not (and, in many cases,
6322 \emph on
6323 will not
6324 \emph default
6325 ) exploit the
6326 \emph on
6327 full
6328 \emph default
6329 cartesian product.
6330 \end_layout
6331
6332 \begin_layout Itemize
6333 You may create a new leaf directory (somewhere in the subtree) even in parallel
6334 to an already running benchmark.
6335 Whenever the currently running benchmark has completed,
6336 \family typewriter
6337 tree-replay.sh
6338 \family default
6339 will re-scan the subdirectory structure, find any freshly created leaf
6340 directories, and determine which benchmark to start next.
6341 All leaf directory names which have not yet completed are sorted alphabetically
6342 , and the first name according to ASCII sort order is taken first.
6343 \end_layout
6344
6345 \begin_layout Itemize
6346 When 10 or more variants could appear somewhere (even after a while), use
6347 leading zeros in any names like
6348 \family typewriter
6349 v001
6350 \family default
6351 ,
6352 \family typewriter
6353 v002
6354 \family default
6355 etc to ensure that ASCII sort order is the same as numerical order.
6356 \end_layout
6357
6358 \begin_layout Itemize
6359 Sometimes the ASCII sort order of names like
6360 \family typewriter
6361 short
6362 \family default
6363 vs
6364 \family typewriter
6365 long
6366 \family default
6367 is boring, because you want to run the
6368 \family typewriter
6369 short
6370 \family default
6371 benchmarks first.
6372 As a workaround in larger projects, add some numerical prefixes like in
6373
6374 \family typewriter
6375 01_short
6376 \family default
6377 vs
6378 \family typewriter
6379 09_long
6380 \family default
6381 (leaving some numerical space such that you can later add
6382 \family typewriter
6383 05_medium
6384 \family default
6385 ).
6386 As a side effect, this also improves the ASCII sort order of your later
6387
6388 \family typewriter
6389 *.png
6390 \family default
6391 graphics.
6392 \end_layout
6393
6394 \begin_layout Itemize
6395 You can rearrange the order in another way: just create an empty file
6396 \family typewriter
6397 prio-nnn
6398 \family default
6399 in a leaf directory, where
6400 \family typewriter
6401 nnn
6402 \family default
6403 is a number denoting a priority class.
6404 Priority classes are overriding the global ASCII sort order.
6405 Directories having the lowest priority class are run first, while directories
6406 without any class are run last.
6407 Inside of each class, the ASCII sort order is obeyed.
6408 \end_layout
6409
6410 \begin_layout Itemize
6411 When you design your
6412 \family typewriter
6413 *.conf
6414 \family default
6415 files systematically as a cartesian product, in theory it makes no difference
6416 whether you permute some directory components (e.g
6417 \family typewriter
6418 ./model1/short/
6419 \family default
6420 instead of
6421 \family typewriter
6422 ./short/model1/
6423 \family default
6424 ).
6425 However, in practice it influences the ASCII sort order (taking the
6426 \emph on
6427 full
6428 \emph default
6429 path) and therefore the order in which your benchmarks are run.
6430 \end_layout
6431
6432 \begin_layout Itemize
6433 When some benchmark fails, just delete the corresponding output files.
6434 On the next cycle,
6435 \family typewriter
6436 tree-replay.sh
6437 \family default
6438 will detect the missing files and just restart that benchmark (possibly
6439 among others).
6440 \end_layout
6441
6442 \begin_layout Itemize
6443
6444 \family typewriter
6445 \emph on
6446 xxx
6447 \emph default
6448 .conf
6449 \family default
6450 files can even reside in some parent directory of the current working directory.
6451 This way, you need not copy your
6452 \family typewriter
6453 .conf
6454 \family default
6455 files inside a complex directory hierarchy (even spanning multiple projects).
6456 However, only the
6457 \family typewriter
6458 \emph on
6459 xxx
6460 \emph default
6461 .conf
6462 \family default
6463 files corresponding to directories reachable from the current working directory
6464 will be included.
6465 Example: if you go to a leaf of your subtree and start
6466 \family typewriter
6467 tree-replay.sh
6468 \family default
6469 there, no
6470 \family typewriter
6471 *.conf
6472 \family default
6473 file other than
6474 \family typewriter
6475 default-*.conf
6476 \family default
6477 will be included.
6478 This may produce different results than expected.
6479 Make sure you start your replays always in the same base directory!
6480 \end_layout
6481
6482 \begin_layout Labeling
6483 \labelwidthstring 00.00.0000
6484 \noindent
6485 \begin_inset Graphics
6486 filename images/MatieresCorrosives.png
6487 lyxscale 50
6488 scale 17
6489
6490 \end_inset
6491
6492 It is easy to misconfigure almost anything by accident.
6493 Check each step you make.
6494 In particular, run tools like
6495 \family typewriter
6496 \size footnotesize
6497 top
6498 \family default
6499 \size default
6500 ,
6501 \family typewriter
6502 \size footnotesize
6503 xosview
6504 \family default
6505 \size default
6506 ,
6507 \family typewriter
6508 \size footnotesize
6509 iostat
6510 \family default
6511 \size default
6512 ,
6513 \family typewriter
6514 \size footnotesize
6515 watch df /tmp/
6516 \family default
6517 \size default
6518 etc on
6519 \emph on
6520 all(!)
6521 \emph default
6522 your involved machines in order to get a chance for noticing when anything
6523 goes wrong!
6524 \emph on
6525 Never
6526 \emph default
6527 , really
6528 \emph on
6529 never
6530 \emph default
6531 run
6532 \family typewriter
6533 \size footnotesize
6534 tree-replay.sh
6535 \family default
6536 \size default
6537
6538 \series bold
6539 blindly
6540 \series default
6541 !
6542 \end_layout
6543
6544 \begin_layout Subsection
6545 Visualization of Results
6546 \end_layout
6547
6548 \begin_layout Labeling
6549 \labelwidthstring 00.00.0000
6550 \noindent
6551 \begin_inset Graphics
6552 filename images/MatieresCorrosives.png
6553 lyxscale 50
6554 scale 17
6555
6556 \end_inset
6557
6558 You need enough space in
6559 \family typewriter
6560 /tmp/
6561 \family default
6562 (or in another
6563 \family typewriter
6564 $TMPDIR
6565 \family default
6566 ) for temporary intermediate files.
6567 If your
6568 \family typewriter
6569 blkreplay
6570 \family default
6571 run was very long (several hours or even days), or if your replay had a
6572 high degree of parallelism, you may need several gigabytes, in extreme
6573 cases even several hundreds of gigabytes (as well as rather long running
6574 times -- please don't interrupt
6575 \family typewriter
6576 graph.sh
6577 \family default
6578 as it spawns lots of subprocesses and creates lots of temporary files).
6579 Currently, there are no checks for free space in
6580 \family typewriter
6581 /tmp/
6582 \family default
6583 , so running out of space may produce wrong results
6584 \emph on
6585 silently
6586 \emph default
6587 .
6588 As a countermeasure, run
6589 \family typewriter
6590 watch df /tmp/
6591 \family default
6592 in a separate window during your run of
6593 \family typewriter
6594 graph.sh
6595 \family default
6596 .
6597 \end_layout
6598
6599 \begin_layout Standard
6600 If you have enabled the module
6601 \family typewriter
6602 graph
6603 \family default
6604 in the config file
6605 \family typewriter
6606 default-graph.conf
6607 \family default
6608 , the following steps will be carried out automatically for you.
6609 Alternatively or in addition, the script
6610 \family typewriter
6611 tree-graph.sh
6612 \family default
6613 can be used to (re-)create all
6614 \family typewriter
6615 *.png
6616 \family default
6617 graphics in a whole directory hierarchy, analogously to
6618 \family typewriter
6619 tree-replay.sh
6620 \family default
6621 (and even sourcing the same
6622 \family typewriter
6623 *.conf
6624 \family default
6625 files in the same way).
6626 In case you need some individual graphics, read on for details.
6627 \end_layout
6628
6629 \begin_layout Standard
6630 After a run of
6631 \family typewriter
6632 tree-replay.sh
6633 \family default
6634 ,
6635 \family typewriter
6636 cd
6637 \family default
6638 to one of the subdirectories where your result files
6639 \family typewriter
6640 *.replay.gz
6641 \family default
6642 have been produced.
6643 There should be as many
6644 \family typewriter
6645 *.replay.gz
6646 \family default
6647 files as there was replay parallelism (on multiple devices in parallel).
6648 Check that.
6649 In addition, check that no errors are inside them, for example by typing:
6650 \end_layout
6651
6652 \begin_layout Quotation
6653
6654 \family typewriter
6655 zgrep ERROR *.replay.gz
6656 \end_layout
6657
6658 \begin_layout Standard
6659 If all is right, issue the following command:
6660 \end_layout
6661
6662 \begin_layout Quotation
6663
6664 \family typewriter
6665 /path/to/graph.sh *.replay.gz
6666 \end_layout
6667
6668 \begin_layout Standard
6669 This will produce
6670 \family typewriter
6671 *.png
6672 \family default
6673 files, which you can inspect with any graphical viewer like
6674 \family typewriter
6675 eog
6676 \family default
6677 /
6678 \family typewriter
6679 konqueror
6680 \family default
6681 etc, print via
6682 \family typewriter
6683 lpr
6684 \family default
6685 , or even work on with graphical editors like
6686 \family typewriter
6687 gimp
6688 \family default
6689 .
6690 \end_layout
6691
6692 \begin_layout Standard
6693 All files
6694 \family typewriter
6695 *.replay.gz
6696 \family default
6697 will be taken together, to form a single result from contemporary replays
6698 on several devices in parallel.
6699 This means: where possible, results from multiple replay devices will be
6700 merged together into a single graphics.
6701 \end_layout
6702
6703 \begin_layout Description
6704 Hint: If you want to view only a single particular device (or zoom into
6705 it), just call
6706 \family typewriter
6707
6708 \begin_inset Flex URL
6709 status open
6710
6711 \begin_layout Plain Layout
6712
6713 /path/to/graph.sh
6714 \end_layout
6715
6716 \end_inset
6717
6718
6719 \family default
6720 with a single argument.
6721 \end_layout
6722
6723 \begin_layout Standard
6724 As output, multiple kinds of graphics are produced.
6725 Each one starts with the same prefix, but has another suffix.
6726 For example,
6727 \family typewriter
6728 yourname.g01.latency.realtime.png
6729 \family default
6730 is a graphics file showing the measured latencies in realtime.
6731 The numbering part
6732 \family typewriter
6733 .g01.
6734
6735 \family default
6736 etc is for sorting in the shell, such that the
6737 \begin_inset Quotes eld
6738 \end_inset
6739
6740 most interesting
6741 \begin_inset Foot
6742 status open
6743
6744 \begin_layout Plain Layout
6745 For many people; of course, there may be different needs.
6746 Feel free to rename your result files as you like.
6747 \end_layout
6748
6749 \end_inset
6750
6751
6752 \begin_inset Quotes erd
6753 \end_inset
6754
6755 graphics will come first.
6756 \end_layout
6757
6758 \begin_layout Standard
6759 By default, parameters matching
6760 \family typewriter
6761 *.load.gz
6762 \family default
6763 will produce graphics containing only
6764 \emph on
6765 static
6766 \emph default
6767 analyses.
6768 When matching
6769 \family typewriter
6770 *.replay.gz
6771 \family default
6772 , only
6773 \emph on
6774 dynamic
6775 \emph default
6776 ones are produced.
6777 In order additionally switch on any (or both), just add one of the options
6778
6779 \family typewriter
6780 --static
6781 \family default
6782 or
6783 \family typewriter
6784 --dynamic
6785 \family default
6786 .
6787 \end_layout
6788
6789 \begin_layout Standard
6790 There is one exception: the throughput graphics are produced always, independent
6791 ly from
6792 \family typewriter
6793 --static
6794 \family default
6795 or
6796 \family typewriter
6797 --dynamic
6798 \family default
6799 .
6800 In the following, the meaning of different suffixes occurring in the output
6801 filenames is described:
6802 \end_layout
6803
6804 \begin_layout Description
6805
6806 \family typewriter
6807 *.thrp.*.png
6808 \family default
6809
6810 \begin_inset ERT
6811 status open
6812
6813 \begin_layout Plain Layout
6814
6815
6816 \backslash
6817 strut
6818 \end_layout
6819
6820 \end_inset
6821
6822
6823 \begin_inset Newline newline
6824 \end_inset
6825
6826 The throughput is displayed on the y axis.
6827 There is an overview variant, where the overall requested throughput is
6828 depicted green, while the actual throughput (when running in dynamic mode)
6829 is orange, for comparison.
6830 When both are exactly identical (which occurs for example when you have
6831 an extremely fast disk or a very low-demanding load), don't be confused
6832 when the orange line completely covers the green one (a property of
6833 \family typewriter
6834 gnuplot
6835 \family default
6836 ).
6837 There are also more detailed variants where the read vs write throughput
6838 is distinct.
6839 \end_layout
6840
6841 \begin_layout Subsubsection
6842 Static Analysis (
6843 \family typewriter
6844 --static
6845 \family default
6846 )
6847 \end_layout
6848
6849 \begin_layout Description
6850
6851 \family typewriter
6852 *.ws_log.*.png
6853 \family default
6854
6855 \begin_inset ERT
6856 status open
6857
6858 \begin_layout Plain Layout
6859
6860
6861 \backslash
6862 strut
6863 \end_layout
6864
6865 \end_inset
6866
6867
6868 \begin_inset Newline newline
6869 \end_inset
6870
6871 On the y axis, the workingset size (cf section
6872 \begin_inset CommandInset ref
6873 LatexCommand ref
6874 reference "sub:Pitfalls-from-Workingset"
6875
6876 \end_inset
6877
6878 ) is displayed in logarithmic scale.
6879 Multiple window sizes
6880 \begin_inset Formula $\delta$
6881 \end_inset
6882
6883 are displayed together in one graphics:
6884 \family typewriter
6885 001
6886 \family default
6887 means
6888 \begin_inset Formula $\delta=1s$
6889 \end_inset
6890
6891 ,
6892 \family typewriter
6893 006
6894 \family default
6895 means 6 seconds,
6896 \family typewriter
6897 060
6898 \family default
6899 means 60 seconds, and
6900 \family typewriter
6901 000
6902 \family default
6903 means
6904 \begin_inset Formula $\delta=\infty$
6905 \end_inset
6906
6907 .
6908 The latter is nothing but cumulation of
6909 \emph on
6910 all
6911 \emph default
6912 occurring sector numbers into one set
6913 \begin_inset Foot
6914 status open
6915
6916 \begin_layout Plain Layout
6917 The slope of the
6918 \family typewriter
6919 000
6920 \family default
6921 line is an indicator for the
6922 \begin_inset Quotes eld
6923 \end_inset
6924
6925 repetitiveness
6926 \begin_inset Quotes erd
6927 \end_inset
6928
6929 of the workload.
6930 \end_layout
6931
6932 \end_inset
6933
6934 .
6935 \end_layout
6936
6937 \begin_layout Description
6938
6939 \family typewriter
6940 *.ws_lin.*.png
6941 \family default
6942
6943 \begin_inset ERT
6944 status open
6945
6946 \begin_layout Plain Layout
6947
6948
6949 \backslash
6950 strut
6951 \end_layout
6952
6953 \end_inset
6954
6955
6956 \begin_inset Newline newline
6957 \end_inset
6958
6959 Like
6960 \family typewriter
6961 ws_log
6962 \family default
6963 , but the y axis is in linear scale.
6964 May be useful for detecting more fine-grained behaviour in peaks.
6965 \end_layout
6966
6967 \begin_layout Description
6968
6969 \family typewriter
6970 *.sum_dist.*.png
6971 \family default
6972
6973 \begin_inset ERT
6974 status open
6975
6976 \begin_layout Plain Layout
6977
6978
6979 \backslash
6980 strut
6981 \end_layout
6982
6983 \end_inset
6984
6985
6986 \begin_inset Newline newline
6987 \end_inset
6988
6989 On the y axis, the
6990 \emph on
6991 distance
6992 \emph default
6993 between the lowest and the highest sector number occurring in each workingset
6994 window is displayed.
6995 As a result, we can see something like a
6996 \begin_inset Quotes eld
6997 \end_inset
6998
6999 total seek distance
7000 \begin_inset Quotes erd
7001 \end_inset
7002
7003 , as if a disk elevator strategy
7004 \emph on
7005 would
7006 \emph default
7007 sort all requests inside a workingset window according to sector numbers,
7008 in order to optimize throughput in favour of latency.
7009 For experts, this can reveal some interesting internal property of the
7010 workload.
7011 \end_layout
7012
7013 \begin_layout Description
7014
7015 \family typewriter
7016 *.avg_dist.*.png
7017 \family default
7018
7019 \begin_inset ERT
7020 status open
7021
7022 \begin_layout Plain Layout
7023
7024
7025 \backslash
7026 strut
7027 \end_layout
7028
7029 \end_inset
7030
7031
7032 \begin_inset Newline newline
7033 \end_inset
7034
7035 Same as
7036 \family typewriter
7037 sum_dist
7038 \family default
7039 , but the average distance is displayed (normalization against the workingset
7040 size).
7041 This results in something like an idealized
7042 \begin_inset Quotes eld
7043 \end_inset
7044
7045 average seek distance
7046 \begin_inset Quotes erd
7047 \end_inset
7048
7049 during each time window.
7050 \end_layout
7051
7052 \begin_layout Description
7053
7054 \family typewriter
7055 *.rqsize.*.png
7056 \family default
7057
7058 \begin_inset ERT
7059 status open
7060
7061 \begin_layout Plain Layout
7062
7063
7064 \backslash
7065 strut
7066 \end_layout
7067
7068 \end_inset
7069
7070
7071 \begin_inset Newline newline
7072 \end_inset
7073
7074 Displays a histogram of the request sizes (#sectors) occurring in the load,
7075 in units of sectors.
7076 \end_layout
7077
7078 \begin_layout Description
7079
7080 \family typewriter
7081 *.rqpos.*.png
7082 \family default
7083
7084 \begin_inset ERT
7085 status open
7086
7087 \begin_layout Plain Layout
7088
7089
7090 \backslash
7091 strut
7092 \end_layout
7093
7094 \end_inset
7095
7096
7097 \begin_inset Newline newline
7098 \end_inset
7099
7100 Displays a histogram of the request position (sector#) occurring in the
7101 load, in coarse units of whole GiB.
7102 The coarse units are necessary to avoid too much jitter in the y axis.
7103 \end_layout
7104
7105 \begin_layout Description
7106
7107 \family typewriter
7108 *.turns.*.png
7109 \family default
7110 (both static and dynamic variants)
7111 \begin_inset ERT
7112 status open
7113
7114 \begin_layout Plain Layout
7115
7116
7117 \backslash
7118 strut
7119 \end_layout
7120
7121 \end_inset
7122
7123
7124 \begin_inset Newline newline
7125 \end_inset
7126
7127 A turn is a request where the sector number is
7128 \emph on
7129 lower
7130 \emph default
7131 than the sector number of its predecessor.
7132 For a mechanical hard disk, a turn means that it has to
7133 \begin_inset Quotes eld
7134 \end_inset
7135
7136 seek backwards
7137 \begin_inset Quotes erd
7138 \end_inset
7139
7140 , which is often more time-consuming than seeking forward.
7141 Imagine an old-fashioned magnetic tape (e.g.
7142 DAT), then you will get some feeling for the meaning of a turn.
7143 Turns are an interesting measure for the locality behaviour of the load
7144 (or the replay result).
7145 There is no freak that some IO schedulers try to minimize the number of
7146 turns (for example the famous sawtooth strategy).
7147 In this graphics, the y axis shows the
7148 \emph on
7149 relative
7150 \emph default
7151 number of turns occurring during a time window of 1 seconds (in percent,
7152 related to the total number of requests occurring during that time window).
7153
7154 \end_layout
7155
7156 \begin_layout Subsubsection
7157 Dynamic Analysis (
7158 \family typewriter
7159 --dynamic
7160 \family default
7161 )
7162 \end_layout
7163
7164 \begin_layout Description
7165
7166 \family typewriter
7167 *.latency.*.png
7168 \family default
7169
7170 \begin_inset ERT
7171 status open
7172
7173 \begin_layout Plain Layout
7174
7175
7176 \backslash
7177 strut
7178 \end_layout
7179
7180 \end_inset
7181
7182
7183 \begin_inset Newline newline
7184 \end_inset
7185
7186 On the y axis, latencies are displayed.
7187 \end_layout
7188
7189 \begin_layout Description
7190
7191 \family typewriter
7192 *.delay.*.png
7193 \family default
7194
7195 \begin_inset ERT
7196 status open
7197
7198 \begin_layout Plain Layout
7199
7200
7201 \backslash
7202 strut
7203 \end_layout
7204
7205 \end_inset
7206
7207
7208 \begin_inset Newline newline
7209 \end_inset
7210
7211 On the y axis, the delays between the intended starting time and the real
7212 starting time are displayed.
7213 \end_layout
7214
7215 \begin_layout Description
7216
7217 \family typewriter
7218 *.thrp.*.png
7219 \family default
7220
7221 \begin_inset ERT
7222 status open
7223
7224 \begin_layout Plain Layout
7225
7226
7227 \backslash
7228 strut
7229 \end_layout
7230
7231 \end_inset
7232
7233
7234 \begin_inset Newline newline
7235 \end_inset
7236
7237 On the y axis, the troughput (IOPS) is displayed.
7238 \end_layout
7239
7240 \begin_layout Description
7241
7242 \family typewriter
7243 *.flying.png
7244 \family default
7245
7246 \begin_inset ERT
7247 status open
7248
7249 \begin_layout Plain Layout
7250
7251
7252 \backslash
7253 strut
7254 \end_layout
7255
7256 \end_inset
7257
7258
7259 \begin_inset Newline newline
7260 \end_inset
7261
7262 The y axis shows the total number of requests which are currently
7263 \begin_inset Quotes eld
7264 \end_inset
7265
7266 on the fly
7267 \begin_inset Quotes erd
7268 \end_inset
7269
7270 .
7271 \begin_inset Newline newline
7272 \end_inset
7273
7274 Examples:
7275 \family typewriter
7276 *.latency.flying.png
7277 \family default
7278 is derived from the latencies, and thus showing the number of requests
7279 currently submitted to the device, while
7280 \family typewriter
7281 *.delay.flying.png
7282 \family default
7283 is derived from the delay, showing the number of requests currently waiting
7284 in the logical
7285 \begin_inset Quotes eld
7286 \end_inset
7287
7288 delay queue
7289 \begin_inset Quotes erd
7290 \end_inset
7291
7292 .
7293 \end_layout
7294
7295 \begin_layout Description
7296
7297 \family typewriter
7298 *.smooth.*
7299 \family default
7300
7301 \begin_inset ERT
7302 status open
7303
7304 \begin_layout Plain Layout
7305
7306
7307 \backslash
7308 strut
7309 \end_layout
7310
7311 \end_inset
7312
7313
7314 \begin_inset Newline newline
7315 \end_inset
7316
7317 This internal infix indicates that the y axis has been smoothed, in order
7318 to be able to see anything in wildly jiggling data.
7319 Example:
7320 \family typewriter
7321 *.smooth.latency.flying.png
7322 \family default
7323 .
7324 \end_layout
7325
7326 \begin_layout Description
7327
7328 \family typewriter
7329 *.realtime.png
7330 \family default
7331
7332 \begin_inset ERT
7333 status open
7334
7335 \begin_layout Plain Layout
7336
7337
7338 \backslash
7339 strut
7340 \end_layout
7341
7342 \end_inset
7343
7344
7345 \begin_inset Newline newline
7346 \end_inset
7347
7348 The x axis is ordered according to the real starting timestamps which have
7349 actually occurred during replay, possibly containing any delays.
7350 Notice: when multiple concurrent
7351 \family typewriter
7352 *.replay.gz
7353 \family default
7354 have been supplied as an argument, they are
7355 \emph on
7356 merged
7357 \emph default
7358 (since their timestamps are usually from the same range), similar to the
7359 effect of overlay slides.
7360 \end_layout
7361
7362 \begin_layout Description
7363
7364 \family typewriter
7365 *.setpoint.png
7366 \family default
7367
7368 \begin_inset ERT
7369 status open
7370
7371 \begin_layout Plain Layout
7372
7373
7374 \backslash
7375 strut
7376 \end_layout
7377
7378 \end_inset
7379
7380
7381 \begin_inset Newline newline
7382 \end_inset
7383
7384 The x axis is ordered according to the
7385 \emph on
7386 intended
7387 \emph default
7388 request submission time (starting point), i.e.
7389 when the request
7390 \emph on
7391 should have
7392 \emph default
7393 started (ignoring any delays).
7394 As before, multiple
7395 \family typewriter
7396 *.replay.gz
7397 \family default
7398 are merged.
7399 \end_layout
7400
7401 \begin_layout Description
7402
7403 \family typewriter
7404 *.completed.png
7405 \family default
7406
7407 \begin_inset ERT
7408 status open
7409
7410 \begin_layout Plain Layout
7411
7412
7413 \backslash
7414 strut
7415 \end_layout
7416
7417 \end_inset
7418
7419
7420 \begin_inset Newline newline
7421 \end_inset
7422
7423 The x axis is ordered by the completion time of the requests.
7424 Note that this may differ from the submission time.
7425 \end_layout
7426
7427 \begin_layout Description
7428
7429 \family typewriter
7430 *.points.png
7431 \family default
7432
7433 \begin_inset ERT
7434 status open
7435
7436 \begin_layout Plain Layout
7437
7438
7439 \backslash
7440 strut
7441 \end_layout
7442
7443 \end_inset
7444
7445
7446 \begin_inset Newline newline
7447 \end_inset
7448
7449 The x axis is ordered by
7450 \emph on
7451 requests
7452 \emph default
7453 , not timestamps.
7454 As a result, requests on the x axis are
7455 \emph on
7456 equidistant
7457 \emph default
7458 , even in case of heavy throughput differences.
7459 For
7460 \family typewriter
7461 *.replay.gz
7462 \family default
7463 , the order is the
7464 \emph on
7465 completion
7466 \emph default
7467 order.
7468 This graphics is useful as a kind of
7469 \begin_inset Quotes eld
7470 \end_inset
7471
7472 looking glass
7473 \begin_inset Quotes erd
7474 \end_inset
7475
7476 , to reveal more details from performance hot spots.
7477 Notice: multiple concurrent replays are
7478 \emph on
7479 not
7480 \emph default
7481 merged on the x axis (as is the case with
7482 \family typewriter
7483 *.realtime.png
7484 \family default
7485 ,
7486 \family typewriter
7487 *.setpoint.png
7488 \family default
7489 and
7490 \family typewriter
7491 *.completed.png
7492 \family default
7493 ).
7494 Instead, the runs are just concatenated (pasted together sequentially)
7495 in the same order as in the corresponding files.
7496 \end_layout
7497
7498 \begin_layout Description
7499
7500 \family typewriter
7501 *.bins.png
7502 \family default
7503
7504 \begin_inset ERT
7505 status open
7506
7507 \begin_layout Plain Layout
7508
7509
7510 \backslash
7511 strut
7512 \end_layout
7513
7514 \end_inset
7515
7516
7517 \begin_inset Newline newline
7518 \end_inset
7519
7520 The new y axis is now a
7521 \emph on
7522 histogram
7523 \emph default
7524 showing absolute frequency occurring at the former y axis, while the new
7525 x axis now carries the role of the former y axis.
7526 \begin_inset Newline newline
7527 \end_inset
7528
7529 Examples:
7530 \family typewriter
7531 *.latency.bins.png
7532 \family default
7533 shows the latencies on the x axis (while the former
7534 \family typewriter
7535 *.latency.realtime.png
7536 \family default
7537 had it on the y axis), while the new y axis now shows the absolute frequency
7538 of that latency (how often that latency occurs, independently from the
7539 former replay timestamps).
7540 \end_layout
7541
7542 \begin_layout Description
7543
7544 \family typewriter
7545 *.xy.png
7546 \family default
7547
7548 \begin_inset ERT
7549 status open
7550
7551 \begin_layout Plain Layout
7552
7553
7554 \backslash
7555 strut
7556 \end_layout
7557
7558 \end_inset
7559
7560
7561 \begin_inset Newline newline
7562 \end_inset
7563
7564 The x axis as well as the y axis are displaying non-timely data.
7565 This is useful for visualization of
7566 \emph on
7567 corrolations
7568 \emph default
7569 .
7570 The following variants exist:
7571 \end_layout
7572
7573 \begin_deeper
7574 \begin_layout Description
7575
7576 \family typewriter
7577 *.latency.xy.png
7578 \family default
7579
7580 \begin_inset ERT
7581 status open
7582
7583 \begin_layout Plain Layout
7584
7585
7586 \backslash
7587 strut
7588 \end_layout
7589
7590 \end_inset
7591
7592
7593 \begin_inset Newline newline
7594 \end_inset
7595
7596 While latencies are displayed at the y axis, the x axis inidicates the number
7597 of flying requests (request queue length).
7598 Thus you can see the corrolation between request queue length and service
7599 time.
7600 \end_layout
7601
7602 \begin_layout Description
7603
7604 \family typewriter
7605 *.delay.xy.png
7606 \family default
7607
7608 \begin_inset ERT
7609 status open
7610
7611 \begin_layout Plain Layout
7612
7613
7614 \backslash
7615 strut
7616 \end_layout
7617
7618 \end_inset
7619
7620
7621 \begin_inset Newline newline
7622 \end_inset
7623
7624 Similarly, but shows the corrolation between request queue length and delays
7625 (the time needed to
7626 \emph on
7627 enter
7628 \emph default
7629 the request queue at all).
7630 \end_layout
7631
7632 \begin_layout Description
7633
7634 \family typewriter
7635 *.latency.delay.xy.png
7636 \family default
7637
7638 \begin_inset ERT
7639 status open
7640
7641 \begin_layout Plain Layout
7642
7643
7644 \backslash
7645 strut
7646 \end_layout
7647
7648 \end_inset
7649
7650
7651 \begin_inset Newline newline
7652 \end_inset
7653
7654 Immediate corrolation between latency and delay.
7655 \end_layout
7656
7657 \end_deeper
7658 \begin_layout Standard
7659 Hint: the internal data format of
7660 \family typewriter
7661 *.replay.gz
7662 \family default
7663 is the same as
7664 \family typewriter
7665 *.load.gz
7666 \family default
7667 .
7668 Thus you can use
7669 \family typewriter
7670 /path/to/graph.sh --static *.replay.gz
7671 \family default
7672 to additionally create the same static analysis graphics as described in
7673 section
7674 \begin_inset CommandInset ref
7675 LatexCommand ref
7676 reference "sub:Select-blkreplay-Load"
7677
7678 \end_inset
7679
7680 .
7681 In difference to analysis of
7682 \family typewriter
7683 *.load.gz
7684 \family default
7685 , this time you may have selected a different time window, and you may have
7686 merged multiple replays together.
7687 \end_layout
7688
7689 \begin_layout Section
7690 Human Interpretation of Results
7691 \end_layout
7692
7693 \begin_layout Standard
7694 Many existing benchmark tools try to deliver a single number as a result,
7695 such as
7696 \begin_inset Formula $x$
7697 \end_inset
7698
7699
7700 \family typewriter
7701 hyper-ops
7702 \family default
7703 or
7704 \begin_inset Formula $x$
7705 \end_inset
7706
7707
7708 \family typewriter
7709 hell-stones
7710 \family default
7711 .
7712 Such numbers cannot describe the reality, which has multiple dimensions
7713 and is much more complex.
7714 Sometimes, new dimensions are even discovered to have an influence.
7715 \end_layout
7716
7717 \begin_layout Standard
7718 By default,
7719 \family typewriter
7720 blkreplay
7721 \family default
7722 does not produce any (single) number, but graphics, showing the behaviour
7723 over time (or in other dimensions).
7724 Although some of those graphics like
7725 \family typewriter
7726 *thrp*
7727 \family default
7728 could be used for production of numbers, most of them serve as input to
7729 the human brain: the human
7730 \begin_inset Quotes eld
7731 \end_inset
7732
7733 neural network
7734 \begin_inset Quotes erd
7735 \end_inset
7736
7737 can be
7738 \emph on
7739 trained
7740 \emph default
7741 to detect
7742 \series bold
7743 hidden properties
7744 \series default
7745 in your measurement data, which cannot be (easily) detected by current
7746 AI technology.
7747 \end_layout
7748
7749 \begin_layout Subsection
7750 Sonar Diagrams
7751 \end_layout
7752
7753 \begin_layout Standard
7754 The following example shows the replay latencies of a natural load over
7755 real time.
7756 Note that the y axis is logarithmic scale (otherwise you would'nt see too
7757 much in the lower bands):
7758 \end_layout
7759
7760 \begin_layout Standard
7761 \noindent
7762 \align center
7763 \begin_inset Graphics
7764 filename images/example1.g01.latency.realtime.png
7765 lyxscale 20
7766 width 100col%
7767
7768 \end_inset
7769
7770
7771 \end_layout
7772
7773 \begin_layout Standard
7774 \noindent
7775 Reads are depicted as blue dots, while writes are colored red.
7776 In case of
7777 \family typewriter
7778 --with-partial
7779 \family default
7780 , pushed-back requests are displayed additionally
7781 \begin_inset Foot
7782 status open
7783
7784 \begin_layout Plain Layout
7785 Notice that the set of purple
7786 \begin_inset Quotes eld
7787 \end_inset
7788
7789 pushes
7790 \begin_inset Quotes erd
7791 \end_inset
7792
7793 is always a
7794 \emph on
7795 subset
7796 \emph default
7797 of the
7798 \begin_inset Quotes eld
7799 \end_inset
7800
7801 writes
7802 \begin_inset Quotes erd
7803 \end_inset
7804
7805 .
7806 They are only visible because they are printed later (hiding the previous
7807 red dots).
7808 \end_layout
7809
7810 \end_inset
7811
7812 in purple.
7813 Together, they form some kind of
7814 \begin_inset Quotes eld
7815 \end_inset
7816
7817 clouds
7818 \begin_inset Quotes erd
7819 \end_inset
7820
7821 , showing some
7822 \begin_inset Quotes eld
7823 \end_inset
7824
7825 density
7826 \begin_inset Quotes erd
7827 \end_inset
7828
7829 both in time scale, as well as in the
7830 \begin_inset Quotes eld
7831 \end_inset
7832
7833 latency scale
7834 \begin_inset Quotes erd
7835 \end_inset
7836
7837 .
7838 \end_layout
7839
7840 \begin_layout Standard
7841 It requires some experience to interpret sonar diagrams.
7842 \end_layout
7843
7844 \begin_layout Itemize
7845 \noindent
7846 As you can see, there are two main
7847 \series bold
7848
7849 \begin_inset Quotes eld
7850 \end_inset
7851
7852 bands
7853 \begin_inset Quotes erd
7854 \end_inset
7855
7856
7857 \series default
7858 optically visible to the human interpretation.
7859 These look similar to
7860 \begin_inset Quotes eld
7861 \end_inset
7862
7863 fish swarms
7864 \begin_inset Quotes erd
7865 \end_inset
7866
7867 occuring in sonar diagrams; hence the name
7868 \begin_inset Quotes eld
7869 \end_inset
7870
7871 sonar diagram
7872 \begin_inset Quotes erd
7873 \end_inset
7874
7875 .
7876 \end_layout
7877
7878 \begin_layout Itemize
7879 Most obviously,
7880 \series bold
7881 cache hierarchies
7882 \series default
7883 can be identified visually, by looking at the
7884 \begin_inset Quotes eld
7885 \end_inset
7886
7887 bands
7888 \begin_inset Quotes erd
7889 \end_inset
7890
7891 .
7892 In addition, the natural fluctuation of the load can be seen.
7893 Even better, it is quite interesting how the test candidate is responding
7894 to the fluctuation, even over time.
7895 \end_layout
7896
7897 \begin_layout Itemize
7898 Any differences between read and write behaviour will immediately catch
7899 your eye.
7900 \end_layout
7901
7902 \begin_layout Itemize
7903 The red
7904 \begin_inset Quotes eld
7905 \end_inset
7906
7907 needles
7908 \begin_inset Quotes erd
7909 \end_inset
7910
7911 are indications of queueing.
7912 In difference to a single
7913 \begin_inset Quotes eld
7914 \end_inset
7915
7916 hang
7917 \begin_inset Quotes erd
7918 \end_inset
7919
7920 which would just
7921 \begin_inset Quotes eld
7922 \end_inset
7923
7924 fly above
7925 \begin_inset Quotes erd
7926 \end_inset
7927
7928 other requests, intermediate latencies from
7929 \begin_inset Quotes eld
7930 \end_inset
7931
7932 fellow requests
7933 \begin_inset Quotes erd
7934 \end_inset
7935
7936 are also present, spanning almost the full range from
7937 \begin_inset Quotes eld
7938 \end_inset
7939
7940 fast
7941 \begin_inset Quotes erd
7942 \end_inset
7943
7944 requests to the queued ones.
7945 \end_layout
7946
7947 \begin_layout Itemize
7948 In case of the strongest needle, you can even see a
7949 \begin_inset Quotes eld
7950 \end_inset
7951
7952 lash
7953 \begin_inset Quotes erd
7954 \end_inset
7955
7956 behind it.
7957 There are no
7958 \begin_inset Quotes eld
7959 \end_inset
7960
7961 faster
7962 \begin_inset Quotes erd
7963 \end_inset
7964
7965 requests (not even reads) during this lash (i.e.
7966 the space below it appears empty - even as if there were an interruption
7967 in the time axis).
7968 This is an indication of a
7969 \emph on
7970 global
7971 \emph default
7972 queuing strategy.
7973 \end_layout
7974
7975 \begin_layout Standard
7976 The same latencies are now displayed as a histogram.
7977 A histogram is a non-timely survey of the absolute frequencies occurring
7978 in the previous graphics.
7979 \end_layout
7980
7981 \begin_layout Standard
7982 \noindent
7983 \align center
7984 \begin_inset Graphics
7985 filename images/example1.g06.latency.bins.png
7986 lyxscale 20
7987 width 100col%
7988
7989 \end_inset
7990
7991
7992 \end_layout
7993
7994 \begin_layout Standard
7995 \noindent
7996 You can identify the
7997 \begin_inset Quotes eld
7998 \end_inset
7999
8000 bands
8001 \begin_inset Quotes erd
8002 \end_inset
8003
8004 a second time by looking at the camel-like
8005 \begin_inset Quotes eld
8006 \end_inset
8007
8008 humps
8009 \begin_inset Quotes erd
8010 \end_inset
8011
8012 .
8013 In addition to the sonar diagram, you can identify the
8014 \begin_inset Quotes eld
8015 \end_inset
8016
8017 hight
8018 \begin_inset Quotes erd
8019 \end_inset
8020
8021 and
8022 \begin_inset Quotes eld
8023 \end_inset
8024
8025 thickness
8026 \begin_inset Quotes erd
8027 \end_inset
8028
8029 of a hump more easily.
8030 Some very small additional humps can also be found, probably an indicator
8031 for further caches or probably for some scheduler timeouts.
8032 \end_layout
8033
8034 \begin_layout Subsection
8035 Delay Diagrams
8036 \end_layout
8037
8038 \begin_layout Standard
8039 As already explained, delays are the difference between the
8040 \emph on
8041 intended/demanded
8042 \emph default
8043 starting time of an IO requests, and its
8044 \emph on
8045 real
8046 \emph default
8047 starting time.
8048 Huge delays are almost always an indication that the system cannot
8049 \begin_inset Quotes eld
8050 \end_inset
8051
8052 catch up
8053 \begin_inset Quotes erd
8054 \end_inset
8055
8056 with the application demands.
8057 Here is an example:
8058 \end_layout
8059
8060 \begin_layout Standard
8061 \noindent
8062 \align center
8063 \begin_inset Graphics
8064 filename images/example1.g04.delay.setpoint.png
8065 lyxscale 20
8066 width 100col%
8067
8068 \end_inset
8069
8070
8071 \end_layout
8072
8073 \begin_layout Standard
8074 \noindent
8075 Here are some interpretations:
8076 \end_layout
8077
8078 \begin_layout Itemize
8079 You will immediately notice the
8080 \begin_inset Quotes eld
8081 \end_inset
8082
8083 runaway tail
8084 \begin_inset Quotes erd
8085 \end_inset
8086
8087 after the purple indicator line, which is just a strong indication that
8088 overall throughput is too low.
8089 \end_layout
8090
8091 \begin_layout Itemize
8092 Even before, you can see some
8093 \begin_inset Quotes eld
8094 \end_inset
8095
8096 needles
8097 \begin_inset Quotes erd
8098 \end_inset
8099
8100 with a
8101 \begin_inset Quotes eld
8102 \end_inset
8103
8104 lash
8105 \begin_inset Quotes erd
8106 \end_inset
8107
8108 behind.
8109 Another indication, even occurring in a section with rather
8110 \begin_inset Quotes eld
8111 \end_inset
8112
8113 low
8114 \begin_inset Quotes erd
8115 \end_inset
8116
8117 throughput demands.
8118 \end_layout
8119
8120 \begin_layout Standard
8121 As before, there is also a histogram variant of delays:
8122 \end_layout
8123
8124 \begin_layout Standard
8125 \noindent
8126 \align center
8127 \begin_inset Graphics
8128 filename images/example1.g07.delay.bins.png
8129 lyxscale 20
8130 width 100col%
8131
8132 \end_inset
8133
8134
8135 \end_layout
8136
8137 \begin_layout Subsection
8138 Throughput Diagrams
8139 \end_layout
8140
8141 \begin_layout Standard
8142 \noindent
8143 In order to compare this with the actual throughput, look at the following
8144 picture, where demanded throughput is colored green, while actually delivered
8145 throughput is orange:
8146 \end_layout
8147
8148 \begin_layout Standard
8149 \noindent
8150 \align center
8151 \begin_inset Graphics
8152 filename images/example1.g000.overview.thrp.png
8153 lyxscale 20
8154 width 100col%
8155
8156 \end_inset
8157
8158
8159 \family typewriter
8160 *.latency.flying
8161 \end_layout
8162
8163 \begin_layout Standard
8164 \noindent
8165 This picture is a strong indication that single IOPS numbers are not sufficient
8166 for really describing the actual IO throughput behaviour of a system.
8167 Our timely plot can tell you a lot more about the system.
8168 More examples can be found in section
8169 \begin_inset CommandInset ref
8170 LatexCommand ref
8171 reference "sub:Overload-with-(Derived)"
8172
8173 \end_inset
8174
8175 .
8176 \end_layout
8177
8178 \begin_layout Subsection
8179 Flying Diagrams
8180 \end_layout
8181
8182 \begin_layout Standard
8183 These are useful to inspect the actual request queue length over time.
8184 The following graphics exhibits the typical
8185 \begin_inset Quotes eld
8186 \end_inset
8187
8188 jitter
8189 \begin_inset Quotes erd
8190 \end_inset
8191
8192 of the queue length during replay of natural loads having a lot of conflicts,
8193 and using
8194 \family typewriter
8195 --with-ordering
8196 \family default
8197 :
8198 \end_layout
8199
8200 \begin_layout Standard
8201 \noindent
8202 \align center
8203 \begin_inset Graphics
8204 filename images/example1.g08.latency.flying.png
8205 lyxscale 20
8206 width 100col%
8207
8208 \end_inset
8209
8210
8211 \end_layout
8212
8213 \begin_layout Standard
8214 \noindent
8215 Notice: this is only the view of
8216 \family typewriter
8217 blkreplay
8218 \family default
8219 .
8220 The physical device may be actually loaded with
8221 \emph on
8222 less
8223 \emph default
8224 requests in parallel, because intermediate layers (such as drivers, iSCSI,
8225 etc) may
8226 \series bold
8227 limit
8228 \series default
8229 the IO parallelism.
8230 When
8231 \family typewriter
8232 blkreplay
8233 \family default
8234 fires off a request, it need not hit the device immediately.
8235 There are a lot of opportunities to be queued for some time in another
8236 places.
8237 \end_layout
8238
8239 \begin_layout Subsection
8240 Corrolation Diagrams
8241 \end_layout
8242
8243 \begin_layout Standard
8244 Most of them are for advanced users and experts.
8245 They can visualize internal relationships between interesting quantities,
8246 such as observed request queue length versus observed latency:
8247 \end_layout
8248
8249 \begin_layout Standard
8250 \noindent
8251 \align center
8252 \begin_inset Graphics
8253 filename images/example1.g10.latency.xy.png
8254 lyxscale 20
8255 width 100col%
8256
8257 \end_inset
8258
8259
8260 \end_layout
8261
8262 \begin_layout Standard
8263 \noindent
8264 Rough interpretation: under overload, write requests cannot be started in
8265 time, and when they are started, they will take a long time.
8266 Read requests are seemingly behaving better.
8267 \end_layout
8268
8269 \begin_layout Section
8270 Advanced Features
8271 \end_layout
8272
8273 \begin_layout Subsection
8274 Modules
8275 \end_layout
8276
8277 \begin_layout Standard
8278 The script
8279 \family typewriter
8280 tree-replay.sh
8281 \family default
8282 can be extended by plugins, aka modules.
8283 Some modules are delivered with the
8284 \family typewriter
8285 blkreplay
8286 \family default
8287 suite, but you can add your own ones (and you should consider submitting
8288 them to the opensource project
8289 \family typewriter
8290 blkreplay
8291 \family default
8292 at
8293 \family typewriter
8294 github.com
8295 \family default
8296 if they could be useful for a broader audience).
8297 \end_layout
8298
8299 \begin_layout Standard
8300 Here is a list of already existing modules:
8301 \end_layout
8302
8303 \begin_layout Labeling
8304 \labelwidthstring 00.00.0000
8305
8306 \family typewriter
8307 iscsi_target_iet
8308 \family default
8309 When enabled, your
8310 \family typewriter
8311 iet
8312 \family default
8313 iSCSI target will be automatically configured for you, automatically generating
8314 iSCSI
8315 \family typewriter
8316 IQN
8317 \family default
8318 names for you.
8319 Usually this module is used in combination with
8320 \family typewriter
8321 iscsi_initiator
8322 \family default
8323 .
8324 \end_layout
8325
8326 \begin_layout Labeling
8327 \labelwidthstring 00.00.0000
8328
8329 \family typewriter
8330 iscsi_initiator
8331 \family default
8332 When enabled, your
8333 \family typewriter
8334 blkreplay
8335 \family default
8336 runs will automatically act on iSCSI
8337 \family typewriter
8338 IQN
8339 \family default
8340 identifiers instead of on device names, transparently creating iSCSI connection
8341 s for you.
8342 It can be used standalone, for example to connect to commercial storage
8343 boxes.
8344 When combined with module
8345 \family typewriter
8346 iscsi_target_
8347 \family default
8348 *, even the
8349 \family typewriter
8350 IQN
8351 \family default
8352 name generation will be automated transparently (in this case you just
8353 supply the device names present at the
8354 \family typewriter
8355 iscsi_target
8356 \family default
8357 host).
8358 \end_layout
8359
8360 \begin_layout Labeling
8361 \labelwidthstring 00.00.0000
8362
8363 \family typewriter
8364 recreate_lvm
8365 \family default
8366 You just supply some physical devices (PVs) in
8367 \family typewriter
8368 $replay_device_list
8369 \family default
8370 , and this module will transparently (re-)create an LVM volume group (VG)
8371 for you, together with some configurable number of logical volumes (LVs)
8372 inside it, optionally even with LVM striping.
8373 \end_layout
8374
8375 \begin_layout Labeling
8376 \labelwidthstring 00.00.0000
8377
8378 \family typewriter
8379 create_lv
8380 \family default
8381 Here you can put in your code for re-creating logical volumes on an arbitrary
8382 storage manager.
8383 You
8384 \series bold
8385 need
8386 \series default
8387 this
8388 \series bold
8389 absolutely
8390 \series default
8391 whenever you are benchmarking virtualized storage; see section
8392 \begin_inset CommandInset ref
8393 LatexCommand ref
8394 reference "sub:Pitfalls-from-Storage"
8395
8396 \end_inset
8397
8398 .
8399 \end_layout
8400
8401 \begin_layout Labeling
8402 \labelwidthstring 00.00.0000
8403
8404 \family typewriter
8405 scheduler
8406 \family default
8407 By default (when enabled), the IO scheduler will be set to
8408 \family typewriter
8409 noop
8410 \family default
8411 (or another value) on all relevant target machines.
8412 \end_layout
8413
8414 \begin_layout Labeling
8415 \labelwidthstring 00.00.0000
8416
8417 \family typewriter
8418 wipe
8419 \family default
8420 When enabled, all your devices will be filled with random data before the
8421 benchmark starts.
8422
8423 \series bold
8424 Always
8425 \series default
8426 use this when benchmarking virtualized storage; see section
8427 \begin_inset CommandInset ref
8428 LatexCommand ref
8429 reference "sub:Pitfalls-from-Storage"
8430
8431 \end_inset
8432
8433 .
8434 \end_layout
8435
8436 \begin_layout Labeling
8437 \labelwidthstring 00.00.0000
8438
8439 \family typewriter
8440 bbu_megaraid
8441 \family default
8442 When enabled, your LSI megaraid conrollers will be programmed to enable/disable
8443 BBU write caching.
8444 The tool
8445 \family typewriter
8446 MegaCli
8447 \family default
8448 must be installed.
8449 \end_layout
8450
8451 \begin_layout Labeling
8452 \labelwidthstring 00.00.0000
8453
8454 \family typewriter
8455 graph
8456 \family default
8457 By default (when enabled), the script
8458 \family typewriter
8459 graph.sh
8460 \family default
8461 will be called automatically for you at the end of each
8462 \family typewriter
8463 blkreplay
8464 \family default
8465 run.
8466 \end_layout
8467
8468 \begin_layout Labeling
8469 \labelwidthstring 00.00.0000
8470
8471 \family typewriter
8472 pipe_*
8473 \family default
8474 When enabled, these filtering modules modify the
8475 \family typewriter
8476 *.load.gz
8477 \family default
8478 input at runtime before it reaches
8479 \family typewriter
8480 blkreplay
8481 \family default
8482 .
8483 Following pipe filters are available:
8484 \end_layout
8485
8486 \begin_deeper
8487 \begin_layout Labeling
8488 \labelwidthstring 00.00.0000
8489
8490 \family typewriter
8491 pipe_repeat
8492 \family default
8493 Repeat the same input files forever, or until
8494 \family typewriter
8495 $replay_duration
8496 \family default
8497 is reached.
8498 This is useful if your
8499 \family typewriter
8500 *.load.gz
8501 \family default
8502 is too short for a longer replay.
8503 However keep in mind that the
8504 \emph on
8505 workingset
8506 \emph default
8507 will not increase, because exactly the
8508 \emph on
8509 same
8510 \emph default
8511 IO operations will be repeated after a while.
8512 In order to circumvent that, combine this module with
8513 \family typewriter
8514 pipe_slip
8515 \family default
8516 .
8517 \end_layout
8518
8519 \begin_layout Labeling
8520 \labelwidthstring 00.00.0000
8521
8522 \family typewriter
8523 pipe_slip
8524 \family default
8525 Add some offset to the sector# after a bunch of operations.
8526 Use this in combination with
8527 \family typewriter
8528 pipe_repeat
8529 \family default
8530 to produce a
8531 \begin_inset Quotes eld
8532 \end_inset
8533
8534 slowly moving
8535 \begin_inset Quotes erd
8536 \end_inset
8537
8538 behaviour of the sector numbers, such that the next cycle will repeat the
8539 operations at a slightly higher position.
8540 Details see appendix
8541 \begin_inset CommandInset ref
8542 LatexCommand ref
8543 reference "sub:File-default-pipe_slip.conf"
8544
8545 \end_inset
8546
8547 .
8548 \end_layout
8549
8550 \begin_layout Labeling
8551 \labelwidthstring 00.00.0000
8552
8553 \family typewriter
8554 pipe_subst
8555 \family default
8556 Replace
8557 \family typewriter
8558 R
8559 \family default
8560 [reads] with
8561 \family typewriter
8562 W
8563 \family default
8564 [writes] or vice versa.
8565 \end_layout
8566
8567 \begin_layout Labeling
8568 \labelwidthstring 00.00.0000
8569
8570 \family typewriter
8571 pipe_spread
8572 \family default
8573 Increase the region where the sector numbers will run about (corresponding
8574 to the size of a LV) by some factor.
8575 Details see appendix
8576 \begin_inset CommandInset ref
8577 LatexCommand ref
8578 reference "sub:File-default-pipe_spread.conf"
8579
8580 \end_inset
8581
8582 .
8583 \end_layout
8584
8585 \begin_layout Labeling
8586 \labelwidthstring 00.00.0000
8587
8588 \family typewriter
8589 pipe_resize
8590 \family default
8591 Increase the request size (#sectors) by some factor, or clip it to some
8592 bounds.
8593 Details see appendix
8594 \begin_inset CommandInset ref
8595 LatexCommand ref
8596 reference "sub:File-default-pipe_resize.conf"
8597
8598 \end_inset
8599
8600 .
8601 \end_layout
8602
8603 \begin_layout Labeling
8604 \labelwidthstring 00.00.0000
8605
8606 \family typewriter
8607 pipe_cmd
8608 \family default
8609 Insert arbitrary shell commands into the input pipeline.
8610 \end_layout
8611
8612 \end_deeper
8613 \begin_layout Standard
8614 Detailed documentation of the module parameters can be found in
8615 \family typewriter
8616 default-*.conf
8617 \family default
8618 , or in appendix
8619 \begin_inset CommandInset ref
8620 LatexCommand ref
8621 reference "sec:Module-Parameters"
8622
8623 \end_inset
8624
8625 .
8626 \end_layout
8627
8628 \begin_layout Standard
8629 In order to write your own module, just look at the existing ones and take
8630 them as a kind of
8631 \begin_inset Quotes eld
8632 \end_inset
8633
8634 template
8635 \begin_inset Quotes erd
8636 \end_inset
8637
8638 .
8639 Modules are simply
8640 \family typewriter
8641 /bin/bash
8642 \family default
8643 code residing in
8644 \family typewriter
8645 scripts/modules/
8646 \family default
8647 and starting with a number
8648 \begin_inset Foot
8649 status open
8650
8651 \begin_layout Plain Layout
8652 The number is used to determine the ASCII sort order, and thus the order
8653 in which modules are
8654 \family typewriter
8655 source
8656 \family default
8657 d.
8658 \end_layout
8659
8660 \end_inset
8661
8662 , followed by an underscore, and finally
8663 \family typewriter
8664 \emph on
8665 modulename
8666 \emph default
8667 .sh
8668 \family default
8669 .
8670 Any existing module is automatically
8671 \family typewriter
8672 source
8673 \family default
8674 d by
8675 \family typewriter
8676 tree-replay.sh
8677 \family default
8678 , provided that some config file
8679 \family typewriter
8680 default-
8681 \emph on
8682 modulename
8683 \emph default
8684 .conf
8685 \family default
8686 exists in your working directory (or in any parent directory down the
8687 \family typewriter
8688 ../
8689 \family default
8690 path).
8691 \end_layout
8692
8693 \begin_layout Standard
8694 Modules are sourced only once at the start of
8695 \family typewriter
8696 tree-replay.sh
8697 \family default
8698 .
8699 Thus they should only define some shell
8700 \family typewriter
8701 function
8702 \family default
8703 s and change some list variables.
8704 The shell functions will be called later for each leaf directory.
8705 The names of your functions should obey the following conventions:
8706 \end_layout
8707
8708 \begin_layout Labeling
8709 \labelwidthstring 00.00.0000
8710
8711 \family typewriter
8712 \emph on
8713 modulename
8714 \emph default
8715 _prepare()
8716 \family default
8717 is called during the preparation phase.
8718 Typically, you can set some variables here, initialize some controllers,
8719 setup your network, create some iSCSI connections, or the like.
8720 \end_layout
8721
8722 \begin_layout Labeling
8723 \labelwidthstring 00.00.0000
8724
8725 \family typewriter
8726 \emph on
8727 modulename
8728 \emph default
8729 _setup()
8730 \family default
8731 The setup phase starts after
8732 \emph on
8733 all
8734 \emph default
8735 prepare functions of all modules have been executed.
8736 For example, some device setup operations are only possible after some
8737 iSCSI connections have been already established.
8738 Therefore, the plugin methodology offers you a separate phase in order
8739 to guarantee some overall order on operations even from unknown foreign
8740 modules.
8741 \end_layout
8742
8743 \begin_layout Labeling
8744 \labelwidthstring 00.00.0000
8745
8746 \family typewriter
8747 \emph on
8748 modulename
8749 \emph default
8750 _run()
8751 \family default
8752 Typically, you won't do anything here, because the running phase is usually
8753 the task of the
8754 \family typewriter
8755 main
8756 \family default
8757 module.
8758 Only in very special cases, you would need to hook yourself into this phase.
8759 \end_layout
8760
8761 \begin_layout Labeling
8762 \labelwidthstring 00.00.0000
8763
8764 \family typewriter
8765 \emph on
8766 modulename
8767 \emph default
8768 _cleanup()
8769 \family default
8770 Typically, you will de-configure something here, remove some temporary
8771 files, check your results, create some statistics, etc.
8772 \end_layout
8773
8774 \begin_layout Labeling
8775 \labelwidthstring 00.00.0000
8776
8777 \family typewriter
8778 \emph on
8779 modulename
8780 \emph default
8781 _finish()
8782 \family default
8783 Typically, network connections etc will be shutdown only here, because
8784 some operations from the cleanup phase could be relying on them being active
8785 yet.
8786 \end_layout
8787
8788 \begin_layout Standard
8789 For each phase, there exists a variable
8790 \family typewriter
8791 \emph on
8792 phasename
8793 \emph default
8794 _list
8795 \family default
8796 determining the order in which the functions will be called.
8797 In most cases, you will just
8798 \emph on
8799 append
8800 \emph default
8801 your function name to that list.
8802 In some cases, you may want to
8803 \emph on
8804 pre
8805 \emph default
8806 pend instead.
8807 Look into the existing code of small modules like
8808 \family typewriter
8809 graph
8810 \family default
8811 to get an impression, and read the
8812 \family typewriter
8813 main()
8814 \family default
8815 function in the
8816 \family typewriter
8817 main
8818 \family default
8819 module.
8820 \end_layout
8821
8822 \begin_layout Section
8823 Lowlevel Details and Expert Usage
8824 \end_layout
8825
8826 \begin_layout Standard
8827 Ordinary users should skip this section.
8828 \end_layout
8829
8830 \begin_layout Subsection
8831 Internal Overhead
8832 \end_layout
8833
8834 \begin_layout Standard
8835 Some new SSDs are promising IO rates of more than 100.000 IOPS when operated
8836 over extremely fast IO channels.
8837 When you try to achieve such high rates, you may probably stumble over
8838 several difficulties.
8839 \end_layout
8840
8841 \begin_layout Standard
8842 For example, running too much threads / processes on the same file handle
8843 or on the same pipe can lead to serious lock contention in the Linux kernel,
8844 limiting overall throughput.
8845 Although some people might argue that
8846 \family typewriter
8847 blkreplay
8848 \family default
8849 itself could produce an
8850 \begin_inset Quotes eld
8851 \end_inset
8852
8853 artificial
8854 \begin_inset Quotes erd
8855 \end_inset
8856
8857 bottleneck, please keep in mind that you need some
8858 \emph on
8859 real applications
8860 \emph default
8861 which were capable of producing / consuming such extremely high IO traffic.
8862 Real applications usually don't use AIO, where such problems occur less
8863 frequently.
8864 No wonder that benchmark results published on the internet are often based
8865 on AIO, which is far from practice.
8866 \end_layout
8867
8868 \begin_layout Standard
8869 Anyway, here is some advice how to deal with extremely high IO traffic.
8870 For
8871 \begin_inset Quotes eld
8872 \end_inset
8873
8874 normal
8875 \begin_inset Quotes erd
8876 \end_inset
8877
8878 IO rates in the region of a few 10.000, the following is usually not needed.
8879 Otherwise, you should be prepared to look into internals of the kernel,
8880 compile your own high-speed custom kernels with some debugging options
8881 switched off / better tuning of schedulers etc, and possibly use tools
8882 like
8883 \family typewriter
8884 oprofile
8885 \family default
8886 to check what is really going on.
8887 \end_layout
8888
8889 \begin_layout Standard
8890 In order to determine the internal overhead of
8891 \family typewriter
8892 blkreplay
8893 \family default
8894 , play around with the following parameters:
8895 \end_layout
8896
8897 \begin_layout Labeling
8898 \labelwidthstring 00.00.0000
8899
8900 \family typewriter
8901 --threads=\SpecialChar \ldots{}
8902
8903 \family default
8904 (or
8905 \family typewriter
8906 threads=\SpecialChar \ldots{}
8907
8908 \family default
8909 as a shell variable in
8910 \family typewriter
8911 *.conf
8912 \family default
8913 ) The most important setting.
8914 Although counter-intuitive, too high numbers of threads may increase the
8915 overall overhead and thus reduce throughput.
8916 Be prepared to degrade this to rather low numbers in order to really achieve
8917 extremely high IO throughput.
8918 Usually, no more than 32 should be used.
8919 \end_layout
8920
8921 \begin_layout Labeling
8922 \labelwidthstring 00.00.0000
8923
8924 \family typewriter
8925 --dry-run
8926 \family default
8927 (or, as a shell variable,
8928 \family typewriter
8929 dry_run
8930 \family default
8931 with underscore instead of minus) Omit any
8932 \family typewriter
8933 read()
8934 \family default
8935 and
8936 \family typewriter
8937 write()
8938 \family default
8939 operations, but continue to use
8940 \family typewriter
8941 lseek64()
8942 \family default
8943 on the file handles.
8944 This way, you can determine the internal overhead in isolation.
8945 \end_layout
8946
8947 \begin_layout Labeling
8948 \labelwidthstring 00.00.0000
8949
8950 \family typewriter
8951 --fake-io
8952 \family default
8953 Additionally omit even
8954 \family typewriter
8955 lseek64()
8956 \family default
8957 operations, as well as creation of header data.
8958 Compare results with
8959 \family typewriter
8960 --dry-run
8961 \family default
8962 .
8963 \end_layout
8964
8965 \begin_layout Labeling
8966 \labelwidthstring 00.00.0000
8967
8968 \family typewriter
8969 --fan-out=\SpecialChar \ldots{}
8970
8971 \family default
8972 In order to reduce parallelism on
8973 \family typewriter
8974 pipe()
8975 \family default
8976 channels,
8977 \family typewriter
8978 blkreplay
8979 \family default
8980 spawns internal distributor threads, forming a tree with limited fan-out
8981 degree.
8982 Too high numbers will increase pipe overhead, while too low numbers will
8983 increase thread and scheduling overhead.
8984 The optimum may depend on hardware (e.g.
8985 NUMA) and kernel version.
8986 \end_layout
8987
8988 \begin_layout Labeling
8989 \labelwidthstring 00.00.0000
8990
8991 \family typewriter
8992 --no-dispatcher
8993 \family default
8994 Omit distributor threads for the answer channels completely.
8995 When combined with high numbers of threads, this may increase the kernel
8996 overhead to almost unusable regions.
8997 Use this for kernel tuning.
8998 \end_layout
8999
9000 \begin_layout Labeling
9001 \labelwidthstring 00.00.0000
9002
9003 \family typewriter
9004 --bottleneck=\SpecialChar \ldots{}
9005
9006 \family default
9007 Limit the number of flying IO requests.
9008 Has a similar effect than
9009 \family typewriter
9010 --threads
9011 \family default
9012 , but is not the same.
9013 Can be used for determining internal kernel and even hardware overheads
9014 such as TLB thrashing (e.g.
9015 by keeping
9016 \family typewriter
9017 --bottleneck
9018 \family default
9019 constant and increasing
9020 \family typewriter
9021 --threads
9022 \family default
9023 ).
9024 Could be useful for kernel and hardware tuning.
9025 \begin_inset Newline newline
9026 \end_inset
9027
9028
9029 \series bold
9030 Hint:
9031 \series default
9032 you can even set this to much higher numbers than
9033 \family typewriter
9034 --threads
9035 \family default
9036 , in order to pre-fill the communication pipes with some ahead requests
9037 (minimizing the risk of waiting times).
9038 However, when the number is way too high, a
9039 \emph on
9040 coprocess deadlock
9041 \emph default
9042 could occur (see the literature about coprocesses).
9043 \end_layout
9044
9045 \begin_layout Labeling
9046 \labelwidthstring 00.00.0000
9047
9048 \family typewriter
9049 --speedup=\SpecialChar \ldots{}
9050
9051 \family default
9052 A very dangerous option when used with natural loads.
9053 Leads to heavy distortions of the relationship between timely and positionly
9054 behaviour; in essence
9055 \emph on
9056 a natural load will not remain a natural load any more
9057 \emph default
9058 .
9059 For artificial loads, however, such as
9060 \family typewriter
9061 *bursts*
9062 \family default
9063 , it is useful to reach top IOPS regions.
9064 \end_layout
9065
9066 \begin_layout Standard
9067 In any case, extremely high IOPS rates are likely to be reached only by
9068 running multiple
9069 \family typewriter
9070 blkreplay
9071 \family default
9072 instances in parallel.
9073 \end_layout
9074
9075 \begin_layout Chapter
9076 How to use
9077 \family typewriter
9078 blktrace
9079 \family default
9080 for Recording of Natural Loads
9081 \begin_inset CommandInset label
9082 LatexCommand label
9083 name "chap:How-to-use"
9084
9085 \end_inset
9086
9087
9088 \end_layout
9089
9090 \begin_layout Standard
9091 The tool
9092 \family typewriter
9093 blktrace
9094 \family default
9095 , as well as its friends like
9096 \family typewriter
9097 blkparse
9098 \family default
9099 , are described elsewhere.
9100 Here are some additions from a practical viewpoint of IT operations:
9101 \end_layout
9102
9103 \begin_layout Itemize
9104 You will need a kernel with compile-time option
9105 \family typewriter
9106 CONFIG_BLK_DEV_IO_TRACE
9107 \family default
9108 enabled.
9109 Although many newer Linux distros have it already enabled by default, you
9110 may stumble over elder systems / distributions where it was missing (so
9111 you may need to install and/or recompile a different kernel).
9112 \end_layout
9113
9114 \begin_layout Itemize
9115 Please note that some
9116 \family typewriter
9117 man
9118 \family default
9119 pages about
9120 \family typewriter
9121 blktrace
9122 \family default
9123 contain unclear / misleading information about the physical unit of the
9124
9125 \family typewriter
9126 -b
9127 \family default
9128 /
9129 \family typewriter
9130 --buffer-size
9131 \family default
9132 option, which may lead to serious problems (including kernel hangups /
9133 crashes) if you (accidentally) try to allocate more kernel memory than
9134 physically available, and/or your production server is heavily loaded.
9135 Unless you are recording extremely high-performance loads, the default
9136 settings will succeed; so there will be no risk for IT operations.
9137 \end_layout
9138
9139 \begin_layout Itemize
9140 Tracing multiple devices in parallel is possible, but sometimes works differentl
9141 y than (inconsistently) described in the
9142 \family typewriter
9143 man
9144 \family default
9145 , possibly leading to kernel problems (depending on versions etc).
9146 Before recording blocktraces at production sites, we recommend to test
9147 it somewhere in advance.
9148 \end_layout
9149
9150 \begin_layout Standard
9151 Please note that you need enough filesystem space for recording, residing
9152 on a
9153 \emph on
9154 different
9155 \emph default
9156 partition than you want to record the IO traffic! Otherwise, your measurements
9157 might be distorted.
9158 \end_layout
9159
9160 \begin_layout Standard
9161 You should record at least 8 hours of operation; better 24h.
9162 \end_layout
9163
9164 \begin_layout Standard
9165 Once
9166 \family typewriter
9167 blktrace
9168 \family default
9169 has finished, some files
9170 \family typewriter
9171 \emph on
9172 yourname
9173 \emph default
9174 .blktrace.{0..7}
9175 \family default
9176 (or other numbers, corresponding to the number of CPUs) will exist in the
9177 current working directory of your production machine.
9178 Please copy all of them to your workstation before proceeding (never risk
9179 more than necessary on production systems).
9180 \end_layout
9181
9182 \begin_layout Standard
9183 After copying, issue the following command:
9184 \end_layout
9185
9186 \begin_layout Quote
9187
9188 \family typewriter
9189 /path/to/blkreplay/scripts/conv_blktrace_to_load.sh
9190 \emph on
9191 yourname
9192 \end_layout
9193
9194 \begin_layout Standard
9195 This will produce an output file
9196 \family typewriter
9197 \emph on
9198 yourname
9199 \emph default
9200 .load.gz
9201 \family default
9202 in your working directory.
9203 You should seriously consider renaming it to
9204 \family typewriter
9205 \emph on
9206 some_better_name
9207 \emph default
9208 .load.gz
9209 \family default
9210 .
9211 \end_layout
9212
9213 \begin_layout Standard
9214 The result file will contain as much IO requests as the script could find
9215 in the
9216 \family typewriter
9217 blktrace
9218 \family default
9219 recording, but it will not contain any original timings (durations of the
9220 requests, aka latencies).
9221 The latency column is simply set to
9222 \begin_inset Formula $0.0$
9223 \end_inset
9224
9225 everywhere.
9226 \end_layout
9227
9228 \begin_layout Standard
9229 There is a reason for it.
9230 Extracting timing information
9231 \emph on
9232 reliably
9233 \emph default
9234 from a
9235 \family typewriter
9236 blktrace
9237 \family default
9238 recording is difficult.
9239 The problem is that each driver / subsystem in den kernel uses different
9240 trace action characters to indicate the start of an IO request (as well
9241 as for indicating various intermediate stages of request processing).
9242 Request completion is
9243 \emph on
9244 another
9245 \emph default
9246 event denoted by character '
9247 \family typewriter
9248 C
9249 \family default
9250 '.
9251 Depending on kernel version / driver, the completion events may be even
9252 missing at all (some devices on very old kernels, which nevertheless occur
9253 in practice).
9254 More seriously, it is a hard task to determine pairs of events, i.e.
9255 determining which completion event belongs to which submission event.
9256 There is no concept of a
9257 \begin_inset Quotes eld
9258 \end_inset
9259
9260 request ID
9261 \begin_inset Quotes erd
9262 \end_inset
9263
9264 which would allow this easily.
9265 You have to guess matching pairs from their sector numbers / length etc.
9266 Guessing may be wrong due to ambiguities.
9267 In general, a completion could sometimes even belong to multiple submission
9268 events.
9269 In addition, when you start and/or end your
9270 \family typewriter
9271 blktrace
9272 \family default
9273 recording at a heavily loaded
9274 \emph on
9275 production
9276 \emph default
9277 server (which is just the most interesting case), some pairs will very
9278 likely remain incomplete at the start and at the end.
9279 \end_layout
9280
9281 \begin_layout Standard
9282 Even under
9283 \begin_inset Quotes eld
9284 \end_inset
9285
9286 normal
9287 \begin_inset Quotes erd
9288 \end_inset
9289
9290 circumstances, we found many cases where non-matching events led to an
9291 incomplete
9292 \family typewriter
9293 *.load.gz
9294 \family default
9295 result.
9296 Therefore we decided to omit timings from our default conversion script,
9297 at least for now
9298 \begin_inset Foot
9299 status open
9300
9301 \begin_layout Plain Layout
9302 Please submit any improvements to the author as patches / git pull requests.
9303 \end_layout
9304
9305 \end_inset
9306
9307 .
9308 However, there is another script which tries to
9309 \emph on
9310 guess
9311 \emph default
9312 the timings as best as it can:
9313 \end_layout
9314
9315 \begin_layout Quote
9316
9317 \family typewriter
9318 /path/to/blkreplay/scripts/conv_blktrace_to_load_with_guessed_timing.sh
9319 \emph on
9320 yourname
9321 \end_layout
9322
9323 \begin_layout Standard
9324 \noindent
9325 The latter script will name its output
9326 \family typewriter
9327 \emph on
9328 yourname
9329 \emph default
9330 .guessed.load.gz,
9331 \family default
9332 which is different from the former one.
9333 There is a reason for it.
9334 Please don't remove the
9335 \family typewriter
9336 *guessed*
9337 \family default
9338 infix when renaming that result file.
9339 We found cases where the difference was considerable.
9340 \end_layout
9341
9342 \begin_layout Labeling
9343 \labelwidthstring 00.00.0000
9344 \begin_inset Graphics
9345 filename images/MatieresCorrosives.png
9346 lyxscale 50
9347 scale 17
9348
9349 \end_inset
9350
9351 Very often, some single guessed timings are obviously wrong.
9352 You can see it at the sonar diagram.
9353 \end_layout
9354
9355 \begin_layout Standard
9356 Please use the
9357 \family typewriter
9358 *guessed*
9359 \family default
9360 version only for
9361 \series bold
9362 analysis
9363 \series default
9364 , such as producing
9365 \family typewriter
9366 *.latency.flying
9367 \family default
9368 or
9369 \family typewriter
9370 *.smooth.latency.flying
9371 \family default
9372 from
9373 \family typewriter
9374 graph.sh --static --dynamic
9375 \family default
9376 in order to determine the original IO parallelism and other interesting
9377 properties.
9378 \end_layout
9379
9380 \begin_layout Labeling
9381 \labelwidthstring 00.00.0000
9382 \begin_inset Graphics
9383 filename images/MatieresToxiques.png
9384 lyxscale 50
9385 scale 17
9386
9387 \end_inset
9388
9389 Only use the
9390 \family typewriter
9391 *.load.gz
9392 \family default
9393 output stemming from
9394 \family typewriter
9395 conv_blktrace_to_load.sh
9396 \family default
9397 as input for
9398 \family typewriter
9399 blkreplay
9400 \family default
9401 .
9402 Never use
9403 \family typewriter
9404 *.guessed.load.gz
9405 \family default
9406 .
9407 You may be missing some essential behaviour which could be crucial for
9408 detection of incidents!
9409 \end_layout
9410
9411 \begin_layout Standard
9412 When your load contains some interesting effects and could be of broader
9413 interest to a larger community, please contact the author of this paper
9414 by Email in order to publish your load at
9415 \family typewriter
9416 www.blkreplay.org
9417 \family default
9418 .
9419 In particular, unusual and novel application areas, heavy ordinary workloads,
9420 and recordings of incidents, are of a broader interest.
9421 \end_layout
9422
9423 \begin_layout Chapter
9424 Experiences with some Setups and some Loads
9425 \end_layout
9426
9427 \begin_layout Section
9428 Overload Tests
9429 \end_layout
9430
9431 \begin_layout Standard
9432 Overload tests try to reveal the behaviour of a system under exceptionary
9433 conditions, similar to a worst-case scenario.
9434 \end_layout
9435
9436 \begin_layout Standard
9437 Overload tests can (and should) be done with natural loads (such as
9438 \begin_inset Flex URL
9439 status open
9440
9441 \begin_layout Plain Layout
9442
9443 http://www.blkreplay.org/loads/natural/1and1/natural-derived/
9444 \end_layout
9445
9446 \end_inset
9447
9448 ).
9449 In addition, it is one of the rare places where
9450 \emph on
9451 artificial
9452 \emph default
9453 loads can reach
9454 \emph on
9455 some
9456 \emph default
9457 justification.
9458 \end_layout
9459
9460 \begin_layout Subsection
9461 Overload with Artificial Bursts
9462 \end_layout
9463
9464 \begin_layout Standard
9465 The following examples are using the artificial loads from
9466 \begin_inset Flex URL
9467 status open
9468
9469 \begin_layout Plain Layout
9470
9471 http://www.blkreplay.org/loads/artificial/random.bursts/
9472 \end_layout
9473
9474 \end_inset
9475
9476 , which are rather
9477 \begin_inset Quotes eld
9478 \end_inset
9479
9480 anal
9481 \begin_inset Quotes erd
9482 \end_inset
9483
9484 ones (see file
9485 \family typewriter
9486 00README
9487 \family default
9488 there).
9489 The load is linearly increasing over time, until the test candidate will
9490
9491 \begin_inset Quotes eld
9492 \end_inset
9493
9494 collapse
9495 \begin_inset Quotes erd
9496 \end_inset
9497
9498 under the overload.
9499 \end_layout
9500
9501 \begin_layout Standard
9502 We use a single
9503 \family typewriter
9504 blkreplay
9505 \family default
9506 instance, and compare between an old SATA RAID-6 and a newer RAID-6 with
9507 faster SAS disks (15k RPM).
9508 For now, we consistently use
9509 \family typewriter
9510 --threads=128
9511 \family default
9512 .
9513 All results can be found in the directory
9514 \family typewriter
9515 blkreplay.git/example-run/
9516 \family default
9517 and subdirs.
9518 \end_layout
9519
9520 \begin_layout Standard
9521 Here is the sonar diagram for the SATA system.
9522 It
9523 \emph on
9524 should
9525 \emph default
9526 have run 300s, but it actually run much longer in realtime:
9527 \end_layout
9528
9529 \begin_layout Standard
9530 \noindent
9531 \align center
9532 \begin_inset Graphics
9533 filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g01.latency.realtime.png
9534 lyxscale 50
9535 width 100col%
9536
9537 \end_inset
9538
9539
9540 \end_layout
9541
9542 \begin_layout Standard
9543 \noindent
9544 At the beginning, the load is rather low, and the system can catch up.
9545 Soon, it reaches a kind of
9546 \begin_inset Quotes eld
9547 \end_inset
9548
9549 maximum throughput
9550 \begin_inset Quotes erd
9551 \end_inset
9552
9553 where the latencies go up to about 1 second.
9554 You can see on the realtime x axis that the replay is delayed as a whole.
9555 This can be better seen at the delay diagram:
9556 \end_layout
9557
9558 \begin_layout Standard
9559 \noindent
9560 \align center
9561 \begin_inset Graphics
9562 filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g04.delay.setpoint.png
9563 lyxscale 50
9564 width 100col%
9565
9566 \end_inset
9567
9568
9569 \end_layout
9570
9571 \begin_layout Standard
9572 \noindent
9573 Here the x axis shows the
9574 \emph on
9575 intended
9576 \emph default
9577 starting time of each request, which is deliberately concentrated at whole
9578 seconds (with gaps inbetween), a property of of the
9579 \family typewriter
9580 *bursts*
9581 \family default
9582 loads.
9583 The
9584 \begin_inset Quotes eld
9585 \end_inset
9586
9587 collapsing point
9588 \begin_inset Quotes erd
9589 \end_inset
9590
9591 is about 60s, which means that it roughly can deliver 600 random IOPS.
9592 This can be even better seen at the overwiew
9593 \family typewriter
9594 thrp
9595 \family default
9596 comparing the actual with the demanded throughput, as well as the detailed
9597
9598 \family typewriter
9599 thrp
9600 \family default
9601 diagram of the actual throughput ordered by completion time of the requests:
9602 \end_layout
9603
9604 \begin_layout Standard
9605 \noindent
9606 \align center
9607 \begin_inset Graphics
9608 filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g000.overview.thrp.png
9609 lyxscale 50
9610 width 100col%
9611
9612 \end_inset
9613
9614
9615 \end_layout
9616
9617 \begin_layout Standard
9618 \noindent
9619 \align center
9620 \begin_inset Graphics
9621 filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g00.thrp.completed.png
9622 lyxscale 50
9623 width 100col%
9624
9625 \end_inset
9626
9627
9628 \end_layout
9629
9630 \begin_layout Standard
9631 \noindent
9632 Now we turn to better hardware: the SAS system uses smaller disks with 15k
9633 RPM, and thus needs more spindles to reach the same capacity.
9634 If we would run the same test for 300s (as in the SATA case), we would
9635 not notice any collapsing.
9636 Thus we double
9637 \family typewriter
9638 replay_duration
9639 \family default
9640 to 600s in order to be able to see the collapsing point.
9641 Please take this into account when looking at the following graphics:
9642 \end_layout
9643
9644 \begin_layout Standard
9645 \noindent
9646 \align center
9647 \begin_inset Graphics
9648 filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g01.latency.realtime.png
9649 lyxscale 50
9650 width 100col%
9651
9652 \end_inset
9653
9654
9655 \end_layout
9656
9657 \begin_layout Standard
9658 \noindent
9659 Notice that the system is even able to reproduce the bursting behaviour
9660 of the load, much better than the SATA system.
9661 Only at higher request rates, some
9662 \begin_inset Quotes eld
9663 \end_inset
9664
9665 blurring clouds
9666 \begin_inset Quotes erd
9667 \end_inset
9668
9669 appear on the x axis, until it finally collapses under too high load.
9670 It is amazing to see the
9671 \begin_inset Quotes eld
9672 \end_inset
9673
9674 binary
9675 \begin_inset Quotes erd
9676 \end_inset
9677
9678 behaviour at the collapsing point: as soon as the actual throughput rate
9679 cannot catch up with the request rate, the internal queues will suddenly
9680 grow up to some internal maximum, and thus lead to latencies around 0.1
9681 seconds, which is better than SATA by an order of magnitude.
9682 Of course, the same effect can be observed at the delay diagram and the
9683 detailed throughput diagram:
9684 \end_layout
9685
9686 \begin_layout Standard
9687 \noindent
9688 \align center
9689 \begin_inset Graphics
9690 filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g03.latency.points.png
9691 lyxscale 50
9692 width 100col%
9693
9694 \end_inset
9695
9696
9697 \end_layout
9698
9699 \begin_layout Standard
9700 \noindent
9701 \align center
9702 \begin_inset Graphics
9703 filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g00.thrp.completed.png
9704 lyxscale 50
9705 width 100col%
9706
9707 \end_inset
9708
9709
9710 \end_layout
9711
9712 \begin_layout Subsection
9713 Overload with (Derived) Natural Loads
9714 \begin_inset CommandInset label
9715 LatexCommand label
9716 name "sub:Overload-with-(Derived)"
9717
9718 \end_inset
9719
9720
9721 \end_layout
9722
9723 \begin_layout Standard
9724 Overload tests with natural loads are probably the most interesting ones.
9725 They can tell you a lot upon both the test candidate, as well as upon your
9726 application.
9727 \end_layout
9728
9729 \begin_layout Standard
9730 In many cases, natural loads are varying very much over time.
9731 Frequently, the variance in the IOPS rate can span an order of magnitude,
9732 or even more.
9733 Typical reasons are cron jobs or nightly backups on an otherwise (almost)
9734 idle server.
9735 \end_layout
9736
9737 \begin_layout Standard
9738 In order to find the peaks in the load, you have to draw
9739 \family typewriter
9740 blktrace
9741 \family default
9742 s from production servers for at least 24h.
9743 After conversion to
9744 \family typewriter
9745 *.load.gz
9746 \family default
9747 , it is rather cumbersome to find the load peaks by hand in order to get
9748 suitable input data for
9749 \emph on
9750 overload
9751 \emph default
9752 tests.
9753 Thus, we have developed the script
9754 \family typewriter
9755 create_derived_load.sh
9756 \family default
9757 which will do the following for you:
9758 \end_layout
9759
9760 \begin_layout Enumerate
9761 It splits the input file(s) into snippets of equal length in realtime (by
9762 default, 5min length each).
9763 \end_layout
9764
9765 \begin_layout Enumerate
9766 It sorts the snippets according to IOPS in reverse order, i.e.
9767 the
9768 \begin_inset Quotes eld
9769 \end_inset
9770
9771 heaviest
9772 \begin_inset Quotes erd
9773 \end_inset
9774
9775 snippet will come first.
9776 \end_layout
9777
9778 \begin_layout Enumerate
9779 It distributes the snippets to a number of output files in a round-robin
9780 fashion (while adjusting the timestamps to the new order).
9781 \end_layout
9782
9783 \begin_layout Standard
9784 As a result, the output files will start with the heaviest IOPS load first,
9785 and later declining.
9786 This methodology has a number of advantages:
9787 \end_layout
9788
9789 \begin_layout Itemize
9790 The overload part is easily accessible at the beginning of each output file
9791 (interesting part first).
9792 \end_layout
9793
9794 \begin_layout Itemize
9795 Thanks to the distribution to many output files, higher replay parallelism
9796 is possible in an uncorrolated way.
9797 \end_layout
9798
9799 \begin_layout Itemize
9800 In case your test candidate should get overloaded: you can easily see its
9801
9802 \emph on
9803 recovery behaviour
9804 \emph default
9805 from overload, thanks to the declination of the load over time.
9806 Will it recover at all, and how fast?
9807 \end_layout
9808
9809 \begin_layout Standard
9810 The latter is an important property of any enterprise-grade storage system.
9811 \end_layout
9812
9813 \begin_layout Standard
9814 Here are some example comparisons between an old SATA system and a newer
9815 SAS system, both equipped with hardware RAID-6, and loaded with a replay
9816 parallelism of 4.
9817 You can find the complete results with additional graphics as a tarball
9818 at
9819 \begin_inset Flex URL
9820 status open
9821
9822 \begin_layout Plain Layout
9823
9824 http://www.blkreplay.org/examples/
9825 \end_layout
9826
9827 \end_inset
9828
9829 .
9830 \end_layout
9831
9832 \begin_layout Standard
9833 \noindent
9834 \begin_inset Graphics
9835 filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
9836 lyxscale 50
9837 width 100col%
9838
9839 \end_inset
9840
9841
9842 \end_layout
9843
9844 \begin_layout Standard
9845 \noindent
9846 \begin_inset Graphics
9847 filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sas/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sas.vg-test.g000.overview.thrp.png
9848 lyxscale 50
9849 width 100col%
9850
9851 \end_inset
9852
9853
9854 \end_layout
9855
9856 \begin_layout Standard
9857 \noindent
9858 It is interesting to observe the behaviour of the SATA system in contrast
9859 to the high-performance SAS: at first, the real IOPS rate (orange) cannot
9860 catch up with the demanded rate (green).
9861 However, when the demanded rate decreases under the green line after some
9862 time, the orange line becomes higher than the green one until the backlog
9863 has been worked up.
9864 Finally, the system can (almost) run in sync with the lower demands, and
9865 the orange line is hiding the green one.
9866 \end_layout
9867
9868 \begin_layout Standard
9869 The same effects can also be seen in the corresponding delay diagrams, where
9870 you can even notice multiple top lines stemming from overlay of replays
9871 in parallel, and even watch their different fallback points:
9872 \end_layout
9873
9874 \begin_layout Standard
9875 \noindent
9876 \begin_inset Graphics
9877 filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sata.bbu-1.vg-test.g04.delay.setpoint.png
9878 lyxscale 50
9879 width 100col%
9880
9881 \end_inset
9882
9883
9884 \end_layout
9885
9886 \begin_layout Standard
9887 \noindent
9888 \begin_inset Graphics
9889 filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sas/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sas.vg-test.g04.delay.setpoint.png
9890 lyxscale 50
9891 width 100col%
9892
9893 \end_inset
9894
9895
9896 \end_layout
9897
9898 \begin_layout Section
9899 Influence of Replay Parameters
9900 \end_layout
9901
9902 \begin_layout Standard
9903 The following examples are run on the rather slow SATA system.
9904 By default, we use
9905 \family typewriter
9906 threads=128
9907 \family default
9908 and 4
9909 \family typewriter
9910 blkreplay
9911 \family default
9912 instances, operating in parallel on 4 LVM devices created from the same
9913 volume group.
9914 Therefore, the total IO parallelism is
9915 \begin_inset Formula $128*4=512$
9916 \end_inset
9917
9918 .
9919 Exceptions from that are denoted.
9920 \end_layout
9921
9922 \begin_layout Subsection
9923 Influence of Request Ordering
9924 \begin_inset CommandInset label
9925 LatexCommand label
9926 name "sub:Influence-of-Request"
9927
9928 \end_inset
9929
9930
9931 \end_layout
9932
9933 \begin_layout Standard
9934 We start with the simplest case: mode
9935 \family typewriter
9936 --with-conflicts
9937 \family default
9938 fires up all requests without guaranteeing any order (besides that requests
9939 are never started too early), which may lead to damaged IO.
9940 Here is the sonar diagram as well as the throughput diagram:
9941 \end_layout
9942
9943 \begin_layout Standard
9944 \noindent
9945 \begin_inset Graphics
9946 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
9947 lyxscale 50
9948 width 100col%
9949
9950 \end_inset
9951
9952
9953 \end_layout
9954
9955 \begin_layout Standard
9956 \noindent
9957 \begin_inset Graphics
9958 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
9959 lyxscale 50
9960 width 100col%
9961
9962 \end_inset
9963
9964
9965 \end_layout
9966
9967 \begin_layout Standard
9968 \noindent
9969 It is clear that the SATA cannot catch up with the high demands here.
9970 But why? Is it the damaged IO?
9971 \end_layout
9972
9973 \begin_layout Standard
9974 A first hint is the rather
9975 \begin_inset Quotes eld
9976 \end_inset
9977
9978 thick
9979 \begin_inset Quotes erd
9980 \end_inset
9981
9982 red
9983 \begin_inset Quotes eld
9984 \end_inset
9985
9986 cloud carpet
9987 \begin_inset Quotes erd
9988 \end_inset
9989
9990 in the latency diagram near 1s, indicating some queueing behaviour.
9991 Interestingly, this carpet falls a little down at the end, when the replay
9992 parallelism decreases due to different termination of different
9993 \family typewriter
9994 blkreplay
9995 \family default
9996 instances.
9997 \end_layout
9998
9999 \begin_layout Standard
10000 Now we turn to
10001 \family typewriter
10002 --with-drop
10003 \family default
10004 .
10005 Surprisingly, there is only a small difference, although damaged IO is
10006 avoided at the cost of some lost requests:
10007 \end_layout
10008
10009 \begin_layout Standard
10010 \noindent
10011 \begin_inset Graphics
10012 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
10013 lyxscale 50
10014 width 100col%
10015
10016 \end_inset
10017
10018
10019 \end_layout
10020
10021 \begin_layout Standard
10022 \noindent
10023 \begin_inset Graphics
10024 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10025 lyxscale 50
10026 width 100col%
10027
10028 \end_inset
10029
10030
10031 \end_layout
10032
10033 \begin_layout Standard
10034 \noindent
10035 This is a clear indication that damaged IO cannot be the reason for the
10036 delays, since it is avoided by
10037 \family typewriter
10038 --with-drop
10039 \family default
10040 .
10041 \end_layout
10042
10043 \begin_layout Standard
10044 Now we turn to
10045 \family typewriter
10046 --with-partial
10047 \family default
10048 .
10049 Most of the time, there is almost the same behaviour as before.
10050 Only the tail is different, because the actual running times are more diverging
10051 between the four
10052 \family typewriter
10053 blkreplay
10054 \family default
10055 instances started in parallel:
10056 \end_layout
10057
10058 \begin_layout Standard
10059 \noindent
10060 \begin_inset Graphics
10061 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
10062 lyxscale 50
10063 width 100col%
10064
10065 \end_inset
10066
10067
10068 \end_layout
10069
10070 \begin_layout Standard
10071 \noindent
10072 \begin_inset Graphics
10073 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10074 lyxscale 50
10075 width 100col%
10076
10077 \end_inset
10078
10079
10080 \end_layout
10081
10082 \begin_layout Standard
10083 \noindent
10084 The tail may look disturbing to some people.
10085 It has a rather
10086 \emph on
10087 low
10088 \emph default
10089 IOPS rate.
10090 In case of delays,
10091 \family typewriter
10092 blkreplay
10093 \family default
10094 should try to catch up as fast as possible, shouldn't it? What the heck
10095 is going on there? Is there a bug in
10096 \family typewriter
10097 blkreplay
10098 \family default
10099 ?
10100 \end_layout
10101
10102 \begin_layout Standard
10103 Examination of
10104 \family typewriter
10105 *.lv-data3.replay.gz
10106 \family default
10107 from
10108 \begin_inset Flex URL
10109 status open
10110
10111 \begin_layout Plain Layout
10112
10113 http://www.blkreplay.org/examples/
10114 \end_layout
10115
10116 \end_inset
10117
10118 tells us that the real last tail consists almost exclusively of pushed-back
10119 requests.
10120 In detail, there are lots of requests depending on each other
10121 \emph on
10122 transitively
10123 \emph default
10124 , reaching over many generations.
10125 They have accumulated over a long time, because
10126 \family typewriter
10127 --with-partial
10128 \family default
10129 is
10130 \emph on
10131 defined
10132 \emph default
10133 to do so.
10134 The tail is nothing but catch-up of pushed-back requests which could not
10135 be run earlier, in order to obey the storage semantics.
10136 Many of them are mutually hindering each other.
10137 The ordinary non-conflicting requests form a different class than then
10138 conflicting ones, leading to a 2-class society with different chances to
10139 be processed (unfairness).
10140 Thus one of the both classes finishes earlier than the other.
10141 This is a disadvantage of
10142 \family typewriter
10143 --with-partial
10144 \family default
10145 you should know.
10146 \end_layout
10147
10148 \begin_layout Standard
10149 On the other hand, this can be turned into an advantage, because you can
10150 use
10151 \family typewriter
10152 --with-partial
10153 \family default
10154 for detection of such a behaviour.
10155 Think of the race between Achilles and the turtle.
10156 If you find a much slower turtle, your test candidate has probably some
10157 problems with that
10158 \emph on
10159 specific
10160 \emph default
10161 load, but not necessarily with other loads at the same IOPS level.
10162 \end_layout
10163
10164 \begin_layout Standard
10165 Hint: the 2-class society caused by
10166 \family typewriter
10167 --with-partial
10168 \family default
10169 is often visible in the delay diagrams.
10170 Check out yourself!
10171 \end_layout
10172
10173 \begin_layout Standard
10174 Let us return to the comparion of request ordering modes: more prominent
10175 differences can be found when we turn to
10176 \family typewriter
10177 --with-ordering
10178 \family default
10179 .
10180 Now the system can catch up
10181 \emph on
10182 in average
10183 \emph default
10184 (but not in general at load peaks).
10185 The sonar diagram shows that the
10186 \begin_inset Quotes eld
10187 \end_inset
10188
10189 thick clouds
10190 \begin_inset Quotes erd
10191 \end_inset
10192
10193 at about 1s have fallen down and have become thinner in densitiy:
10194 \end_layout
10195
10196 \begin_layout Standard
10197 \noindent
10198 \begin_inset Graphics
10199 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
10200 lyxscale 50
10201 width 100col%
10202
10203 \end_inset
10204
10205
10206 \end_layout
10207
10208 \begin_layout Standard
10209 \noindent
10210 \begin_inset Graphics
10211 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10212 lyxscale 50
10213 width 100col%
10214
10215 \end_inset
10216
10217
10218 \end_layout
10219
10220 \begin_layout Standard
10221 \noindent
10222 This rises the question for reasons.
10223 Why should
10224 \family typewriter
10225 --with-ordering
10226 \family default
10227 be better than
10228 \family typewriter
10229 --with-partial
10230 \family default
10231 or even
10232 \family typewriter
10233 --with-drop
10234 \family default
10235 ? Full ordering will usually
10236 \emph on
10237 reduce
10238 \emph default
10239 the IO parallelism when compared to partial ordering.
10240 The full answer is in sections
10241 \begin_inset CommandInset ref
10242 LatexCommand ref
10243 reference "sub:Influence-of-Number"
10244
10245 \end_inset
10246
10247 and
10248 \begin_inset CommandInset ref
10249 LatexCommand ref
10250 reference "sub:Influence-of-Bottlenecks"
10251
10252 \end_inset
10253
10254 , but for now you can take the small
10255 \begin_inset Quotes eld
10256 \end_inset
10257
10258 gaps
10259 \begin_inset Quotes erd
10260 \end_inset
10261
10262 on the x axis as a hint that there must be something related to small micro-sta
10263 lls, which seem to
10264 \emph on
10265 accelerate
10266 \emph default
10267 overall throughput in some
10268 \series bold
10269 counter-intuitive
10270 \series default
10271 way.
10272 The micro-stalls are easily explainable by the submission delays caused
10273 by
10274 \family typewriter
10275 --with-ordering
10276 \family default
10277 , reducing the IO parallelism in case of conflicts.
10278 But why does that
10279 \emph on
10280 accelerate
10281 \emph default
10282 the overall throughput?
10283 \end_layout
10284
10285 \begin_layout Standard
10286 Before proceeding, we take a closer look at the micro-stall effect.
10287 It is better visible in
10288 \family typewriter
10289 *.latency.flying
10290 \family default
10291 showing the
10292 \emph on
10293 actually achieved IO parallelism
10294 \emph default
10295 during the replay.
10296 Here is the comparison between
10297 \family typewriter
10298 --with-conflicts
10299 \family default
10300 , and
10301 \family typewriter
10302 --with-drop
10303 \family default
10304 , and
10305 \family typewriter
10306 --with-
10307 \family default
10308 partial:
10309 \end_layout
10310
10311 \begin_layout Standard
10312 \noindent
10313 \begin_inset Graphics
10314 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
10315 lyxscale 50
10316 width 100col%
10317
10318 \end_inset
10319
10320
10321 \end_layout
10322
10323 \begin_layout Standard
10324 \noindent
10325 \begin_inset Graphics
10326 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
10327 lyxscale 50
10328 width 100col%
10329
10330 \end_inset
10331
10332
10333 \end_layout
10334
10335 \begin_layout Standard
10336 \noindent
10337 \begin_inset Graphics
10338 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
10339 lyxscale 50
10340 width 100col%
10341
10342 \end_inset
10343
10344
10345 \end_layout
10346
10347 \begin_layout Standard
10348 \noindent
10349 The sum of both reads and writes (olive line) indicates that the RAID controller
10350 is filled with a maximum of
10351 \begin_inset Formula $128*4=512$
10352 \end_inset
10353
10354 requests (caused by
10355 \family typewriter
10356 threads=128
10357 \family default
10358 and a replay parallelism of
10359 \begin_inset Formula $4$
10360 \end_inset
10361
10362 ) most of the time, except at the tail.
10363 \end_layout
10364
10365 \begin_layout Standard
10366 And now, compare these with
10367 \family typewriter
10368 --with-ordering
10369 \family default
10370 :
10371 \end_layout
10372
10373 \begin_layout Standard
10374 \noindent
10375 \begin_inset Graphics
10376 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
10377 lyxscale 50
10378 width 100col%
10379
10380 \end_inset
10381
10382
10383 \end_layout
10384
10385 \begin_layout Standard
10386 \noindent
10387 It is clear that by definition of
10388 \family typewriter
10389 ordering
10390 \family default
10391 , some kind of stalls have to be introduced to guarantee some virtual
10392 \begin_inset Quotes eld
10393 \end_inset
10394
10395 strict storage semantics
10396 \begin_inset Quotes erd
10397 \end_inset
10398
10399 .
10400 It is also clear that these micro-stalls will sometimes reduce the actual
10401 IO parallelism, which is clearly visible in the graphics.
10402 However, can these micro-stalls be made
10403 \emph on
10404 responsible
10405 \emph default
10406 for an
10407 \emph on
10408 increase
10409 \emph default
10410 of overall throughput?
10411 \end_layout
10412
10413 \begin_layout Standard
10414 In order to investigate that, we turn to the influence of the
10415 \family typewriter
10416 threads=
10417 \family default
10418 parameter.
10419 \end_layout
10420
10421 \begin_layout Subsection
10422 Influence of Number of Threads
10423 \begin_inset CommandInset label
10424 LatexCommand label
10425 name "sub:Influence-of-Number"
10426
10427 \end_inset
10428
10429
10430 \end_layout
10431
10432 \begin_layout Standard
10433 The following throughput graphics are a direct comparison between 8, 16,
10434 32, 64, 128, 256 and 512 threads on the SATA system using
10435 \family typewriter
10436 --with-conflicts
10437 \family default
10438 , which guarantees the best possible IO parallelism and avoids any non-linear
10439 influences from any conflict-handling strategies:
10440 \end_layout
10441
10442 \begin_layout Standard
10443 \noindent
10444 \begin_inset Graphics
10445 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-008/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-008.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10446 lyxscale 50
10447 width 100col%
10448
10449 \end_inset
10450
10451
10452 \end_layout
10453
10454 \begin_layout Standard
10455 \noindent
10456 \begin_inset Graphics
10457 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-016/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-016.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10458 lyxscale 50
10459 width 100col%
10460
10461 \end_inset
10462
10463
10464 \end_layout
10465
10466 \begin_layout Standard
10467 \noindent
10468 \begin_inset Graphics
10469 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10470 lyxscale 50
10471 width 100col%
10472
10473 \end_inset
10474
10475
10476 \end_layout
10477
10478 \begin_layout Standard
10479 \noindent
10480 \begin_inset Graphics
10481 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-064/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-064.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10482 lyxscale 50
10483 width 100col%
10484
10485 \end_inset
10486
10487
10488 \end_layout
10489
10490 \begin_layout Standard
10491 \noindent
10492 \begin_inset Graphics
10493 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10494 lyxscale 50
10495 width 100col%
10496
10497 \end_inset
10498
10499
10500 \end_layout
10501
10502 \begin_layout Standard
10503 \noindent
10504 \begin_inset Graphics
10505 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-256/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-256.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10506 lyxscale 50
10507 width 100col%
10508
10509 \end_inset
10510
10511
10512 \end_layout
10513
10514 \begin_layout Standard
10515 \noindent
10516 \begin_inset Graphics
10517 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-512/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-512.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10518 lyxscale 50
10519 width 100col%
10520
10521 \end_inset
10522
10523
10524 \end_layout
10525
10526 \begin_layout Standard
10527 \noindent
10528 It is clear that a too low number of threads, such as 8, can hinder overall
10529 throughput (even when in reality the total IO parallelism is
10530 \begin_inset Formula $8*4=32$
10531 \end_inset
10532
10533 by taking the number of
10534 \family typewriter
10535 blkreplay
10536 \family default
10537 instances into account).
10538 It is therefore no wonder that 16 through 64 threads perform better than
10539 8.
10540 But, why is there a counter-productive break-down when increasing the number
10541 of threads after that? The effect is rather strong: 512 threads will decrease
10542 throughput almost by a factor of three when compared to the
10543 \begin_inset Quotes eld
10544 \end_inset
10545
10546 best
10547 \begin_inset Quotes erd
10548 \end_inset
10549
10550 setting.
10551 This is no peanuts! What the hell is going on here?
10552 \end_layout
10553
10554 \begin_layout Standard
10555 The answer can only be buried in the internals of the hardware and/or its
10556 firmware and/or its driver.
10557 Typically, some systems have problems when filled with too many requests
10558 in parallel.
10559 Here, we see a typical
10560 \series bold
10561 non-linear behaviour
10562 \series default
10563 as explained in section
10564 \begin_inset CommandInset ref
10565 LatexCommand ref
10566 reference "sub:Pitfalls-from-Replay"
10567
10568 \end_inset
10569
10570 .
10571 \end_layout
10572
10573 \begin_layout Standard
10574 Consequences: the
10575 \family typewriter
10576 threads=
10577 \family default
10578 parameter can be very important.
10579 And its
10580 \begin_inset Quotes eld
10581 \end_inset
10582
10583 optimum
10584 \begin_inset Quotes erd
10585 \end_inset
10586
10587 can non-linearly depend from the request ordering parameters.
10588 Here, you see a proof for the warnings posed in section
10589 \begin_inset CommandInset ref
10590 LatexCommand ref
10591 reference "sub:Pitfalls-from-Replay"
10592
10593 \end_inset
10594
10595 .
10596 \end_layout
10597
10598 \begin_layout Labeling
10599 \labelwidthstring 00.00.0000
10600 \begin_inset Graphics
10601 filename images/MatieresToxiques.png
10602 lyxscale 50
10603 scale 17
10604
10605 \end_inset
10606
10607
10608 \emph on
10609 Never
10610 \emph default
10611 adjust the
10612 \family typewriter
10613 threads=
10614 \family default
10615 parameter to the value delivering the
10616 \begin_inset Quotes eld
10617 \end_inset
10618
10619 best
10620 \begin_inset Quotes erd
10621 \end_inset
10622
10623 performance when comparing different systems / vendors.
10624 Doing so will be a
10625 \emph on
10626 lie(!)
10627 \emph default
10628 to your audience! Always use the
10629 \family typewriter
10630 threads=
10631 \family default
10632 setting which corresponds to your
10633 \series bold
10634 real application
10635 \series default
10636 based on
10637 \series bold
10638 knowledge
10639 \series default
10640 (by examining the submission IO parallelism at the original
10641 \family typewriter
10642 blktrace
10643 \family default
10644 recording site, e.g.
10645 by looking at the
10646 \family typewriter
10647 *.flying
10648 \family default
10649 graphics produced from the original
10650 \family typewriter
10651 blktrace
10652 \family default
10653 recording), and
10654 \series bold
10655 not
10656 \series default
10657 based on any assumptions which could be faked in any direction! Be responsible
10658 and tell your audience
10659 \emph on
10660 why
10661 \emph default
10662 you select a specific value, and what would happen if you selected a different
10663 one! Or, provide a series of different
10664 \family typewriter
10665 threads=
10666 \family default
10667 settings telling the
10668 \emph on
10669 whole
10670 \emph default
10671 story!
10672 \end_layout
10673
10674 \begin_layout Subsection
10675 Influence of BBU units at RAID controllers
10676 \end_layout
10677
10678 \begin_layout Standard
10679 The following comparison is an example for the performace degradation resulting
10680 from defective (or missing) BBU units:
10681 \end_layout
10682
10683 \begin_layout Standard
10684 \noindent
10685 \begin_inset Graphics
10686 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10687 lyxscale 50
10688 width 100col%
10689
10690 \end_inset
10691
10692
10693 \end_layout
10694
10695 \begin_layout Standard
10696 \noindent
10697 \begin_inset Graphics
10698 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-0/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-0.vg-test.g000.overview.thrp.png
10699 lyxscale 50
10700 width 100col%
10701
10702 \end_inset
10703
10704
10705 \end_layout
10706
10707 \begin_layout Standard
10708 \noindent
10709 In essence, the missing BBU will slow down write requests.
10710 This can be easily seen when comparing the sonar diagrams (just look at
10711 the colors and where they appear):
10712 \end_layout
10713
10714 \begin_layout Standard
10715 \noindent
10716 \begin_inset Graphics
10717 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
10718 lyxscale 50
10719 width 100col%
10720
10721 \end_inset
10722
10723
10724 \end_layout
10725
10726 \begin_layout Standard
10727 \noindent
10728 \begin_inset Graphics
10729 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-0/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-0.vg-test.g01.latency.realtime.png
10730 lyxscale 50
10731 width 100col%
10732
10733 \end_inset
10734
10735
10736 \end_layout
10737
10738 \begin_layout Standard
10739 \noindent
10740 More interesting differences can be found in the tarball at
10741 \begin_inset Flex URL
10742 status open
10743
10744 \begin_layout Plain Layout
10745
10746 http://www.blkreplay.org/examples/
10747 \end_layout
10748
10749 \end_inset
10750
10751 .
10752 \end_layout
10753
10754 \begin_layout Subsection
10755 Influence of Bottlenecks
10756 \begin_inset CommandInset label
10757 LatexCommand label
10758 name "sub:Influence-of-Bottlenecks"
10759
10760 \end_inset
10761
10762
10763 \end_layout
10764
10765 \begin_layout Standard
10766 The expert option
10767 \family typewriter
10768 --bottleneck=<number>
10769 \family default
10770 can be used to limit the number of requests submitted to the worker threads,
10771 which is usually higher than the number of threads.
10772 In each individual pipeline connecting the main thread to each worker thread,
10773 up to 8 requests may be queued in advance.
10774 \end_layout
10775
10776 \begin_layout Standard
10777 In the following example using
10778 \family typewriter
10779 --with-conflicts
10780 \family default
10781 and
10782 \family typewriter
10783 --threads=128
10784 \family default
10785 , we also limit the number of requests by
10786 \family typewriter
10787 --bottleneck=128
10788 \family default
10789 , which leads to almost the same effect as the above discussion regarding
10790
10791 \family typewriter
10792 --with-ordering
10793 \family default
10794 :
10795 \end_layout
10796
10797 \begin_layout Standard
10798 \noindent
10799 \begin_inset Graphics
10800 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g01.latency.realtime.png
10801 lyxscale 50
10802 width 100col%
10803
10804 \end_inset
10805
10806
10807 \end_layout
10808
10809 \begin_layout Standard
10810 \noindent
10811 \begin_inset Graphics
10812 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10813 lyxscale 50
10814 width 100col%
10815
10816 \end_inset
10817
10818
10819 \end_layout
10820
10821 \begin_layout Standard
10822 \noindent
10823 This result is unexpected.
10824 In theory, it should not make any difference whether there is 1 request
10825 in between the main thread and a worker, or many.
10826 However, in practice it makes a difference: the
10827 \emph on
10828 communication latencies
10829 \emph default
10830 over the pipelines are avoided when the pipeline remains always filled
10831 with some requests in advance.
10832 \end_layout
10833
10834 \begin_layout Standard
10835 When
10836 \family typewriter
10837 --bottleneck
10838 \family default
10839 has exactly the same value as
10840 \family typewriter
10841 --threads
10842 \family default
10843 , a
10844 \begin_inset Quotes eld
10845 \end_inset
10846
10847 logical request
10848 \begin_inset Quotes erd
10849 \end_inset
10850
10851 can be either working in a worker thread, or being processed by the main
10852 thread, or underway over the pipes.
10853 It cannot be present at more than one of these places at the same time.
10854 \end_layout
10855
10856 \begin_layout Standard
10857 An example, just for demonstration of the principle: Assume that the IO
10858 system can catch up (at least at the beginning), and that the IO latencies
10859 are
10860 \emph on
10861 constant
10862 \emph default
10863 .
10864 Assume that the communication latencies are exactly the same constant.
10865 What will be the effect? One half of the requests will always reside in
10866 the IO system, but the other half will reside in the pipe communication
10867 infrastructure, in average.
10868 Did you get the point?
10869 \end_layout
10870
10871 \begin_layout Standard
10872 Now, assume that both the IO latencies as well as the communication latencies
10873 are varying (e.g.
10874 due to kernel scheduling etc).
10875 Then the queue length might be staggering around, as observed in the following
10876 graphics (first is the default bottleneck setting
10877 \family typewriter
10878 --bottleneck=$(( threads * 8 ))
10879 \family default
10880 , second is
10881 \family typewriter
10882 --bottleneck=${threads}
10883 \family default
10884 ):
10885 \end_layout
10886
10887 \begin_layout Standard
10888 \noindent
10889 \begin_inset Graphics
10890 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
10891 lyxscale 50
10892 width 100col%
10893
10894 \end_inset
10895
10896
10897 \end_layout
10898
10899 \begin_layout Standard
10900 \noindent
10901 \begin_inset Graphics
10902 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g08.latency.flying.png
10903 lyxscale 50
10904 width 100col%
10905
10906 \end_inset
10907
10908
10909 \end_layout
10910
10911 \begin_layout Standard
10912 \noindent
10913 As before, the SATA system shows its counter-productive behaviour when too
10914 many IO requests are fired up in parallel.
10915 Once the number of requests has reduced, it works better.
10916 But somewhen it staggers to the other extreme, flipping around.
10917 And to the opposite.
10918 \end_layout
10919
10920 \begin_layout Standard
10921 Interesting side note: similar behaviour can be observed in real-life production
10922 systems, because natural loads (as opposed to standard benchmark tests)
10923 often vary by orders of magnitude.
10924 When spuriously plagued by seemingly
10925 \begin_inset Quotes eld
10926 \end_inset
10927
10928 unmotivated
10929 \begin_inset Quotes erd
10930 \end_inset
10931
10932 incidents, look for places where non-linear and counter-productive behaviour
10933 may cause unexpected (and possibly self-amplifying) effects!
10934 \end_layout
10935
10936 \begin_layout Subsection
10937 Influence of
10938 \family typewriter
10939 strong
10940 \family default
10941 Mode
10942 \begin_inset CommandInset label
10943 LatexCommand label
10944 name "sub:Influence-of-strong"
10945
10946 \end_inset
10947
10948
10949 \end_layout
10950
10951 \begin_layout Standard
10952 Here is a direct comparison between all three
10953 \family typewriter
10954 --strong=
10955 \family default
10956 modes, when combined with
10957 \family typewriter
10958 --with-partial
10959 \family default
10960 .
10961 The effect is highly dependable from the load, and usually rather small
10962 (when compared to the drastic effects demonstrated in some of the previous
10963 sections).
10964 But don't take this as given! Better check for it.
10965 First, we compare at
10966 \family typewriter
10967 --threads=128
10968 \family default
10969 :
10970 \end_layout
10971
10972 \begin_layout Standard
10973 \noindent
10974 \begin_inset Graphics
10975 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10976 lyxscale 50
10977 width 100col%
10978
10979 \end_inset
10980
10981
10982 \end_layout
10983
10984 \begin_layout Standard
10985 \noindent
10986 \begin_inset Graphics
10987 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
10988 lyxscale 50
10989 width 100col%
10990
10991 \end_inset
10992
10993
10994 \end_layout
10995
10996 \begin_layout Standard
10997 \noindent
10998 \begin_inset Graphics
10999 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
11000 lyxscale 50
11001 width 100col%
11002
11003 \end_inset
11004
11005
11006 \end_layout
11007
11008 \begin_layout Standard
11009 \noindent
11010 The
11011 \begin_inset Quotes eld
11012 \end_inset
11013
11014 long tail
11015 \begin_inset Quotes erd
11016 \end_inset
11017
11018 explained in section
11019 \begin_inset CommandInset ref
11020 LatexCommand ref
11021 reference "sub:Influence-of-Request"
11022
11023 \end_inset
11024
11025 is disturbing.
11026 The 2-class treatment inherent to
11027 \family typewriter
11028 --with-partial
11029 \family default
11030 leads to a high number of
11031 \emph on
11032 transitively
11033 \emph default
11034 pushed-back requests, racing against the ordinary requests.
11035 This race is in some sense
11036 \begin_inset Quotes eld
11037 \end_inset
11038
11039 non-deterministic
11040 \begin_inset Quotes erd
11041 \end_inset
11042
11043 and may depend non-linearly from the
11044 \family typewriter
11045 --strong=
11046 \family default
11047 mode.
11048 Therefore, we reduce the 2-class treatment by turning to
11049 \family typewriter
11050 --threads=32
11051 \family default
11052 :
11053 \end_layout
11054
11055 \begin_layout Standard
11056 \noindent
11057 \begin_inset Graphics
11058 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
11059 lyxscale 50
11060 width 100col%
11061
11062 \end_inset
11063
11064
11065 \end_layout
11066
11067 \begin_layout Standard
11068 \noindent
11069 \begin_inset Graphics
11070 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
11071 lyxscale 50
11072 width 100col%
11073
11074 \end_inset
11075
11076
11077 \end_layout
11078
11079 \begin_layout Standard
11080 \noindent
11081 \begin_inset Graphics
11082 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
11083 lyxscale 50
11084 width 100col%
11085
11086 \end_inset
11087
11088
11089 \end_layout
11090
11091 \begin_layout Standard
11092 \noindent
11093 Now the tail effect has almost vanished, because the better throughput leads
11094 to less queueing, which in turn decrease the chances for a conflict to
11095 actually occur during the time window
11096 \family typewriter
11097 --ahead_limit=
11098 \family default
11099 (default
11100 \begin_inset Formula $1$
11101 \end_inset
11102
11103 second).
11104 In consequence, any self-amplifying
11105 \begin_inset Quotes eld
11106 \end_inset
11107
11108 traffic jam effects
11109 \begin_inset Quotes erd
11110 \end_inset
11111
11112 are much lower.
11113 As expected, the actual throughput decreases with
11114 \family typewriter
11115 strong
11116 \family default
11117 er mode, but the effect is really tiny, probably below measurement tolerances.
11118 In order to really see something, we take a closer look at the sonar diagrams:
11119 \end_layout
11120
11121 \begin_layout Standard
11122 \noindent
11123 \begin_inset Graphics
11124 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
11125 lyxscale 50
11126 width 100col%
11127
11128 \end_inset
11129
11130
11131 \end_layout
11132
11133 \begin_layout Standard
11134 \noindent
11135 \begin_inset Graphics
11136 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
11137 lyxscale 50
11138 width 100col%
11139
11140 \end_inset
11141
11142
11143 \end_layout
11144
11145 \begin_layout Standard
11146 \noindent
11147 \begin_inset Graphics
11148 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
11149 lyxscale 50
11150 width 100col%
11151
11152 \end_inset
11153
11154
11155 \end_layout
11156
11157 \begin_layout Standard
11158 \noindent
11159 As expected, the number of
11160 \family typewriter
11161 Write Pushbacks
11162 \family default
11163 is increasing with
11164 \family typewriter
11165 --strong=
11166 \family default
11167 mode, and
11168 \family typewriter
11169 Read Pushbacks
11170 \family default
11171 come also into play.
11172 But the differences are minor.
11173 A litte more impressive are the differences at
11174 \family typewriter
11175 *.latency.flying
11176 \family default
11177 , where the
11178 \begin_inset Quotes eld
11179 \end_inset
11180
11181 staggering effect
11182 \begin_inset Quotes erd
11183 \end_inset
11184
11185 is clearly increasing with growing
11186 \family typewriter
11187 --strong=
11188 \family default
11189 mode:
11190 \end_layout
11191
11192 \begin_layout Standard
11193 \noindent
11194 \begin_inset Graphics
11195 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
11196 lyxscale 50
11197 width 100col%
11198
11199 \end_inset
11200
11201
11202 \end_layout
11203
11204 \begin_layout Standard
11205 \noindent
11206 \begin_inset Graphics
11207 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
11208 lyxscale 50
11209 width 100col%
11210
11211 \end_inset
11212
11213
11214 \end_layout
11215
11216 \begin_layout Standard
11217 \noindent
11218 \begin_inset Graphics
11219 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
11220 lyxscale 50
11221 width 100col%
11222
11223 \end_inset
11224
11225
11226 \end_layout
11227
11228 \begin_layout Standard
11229 \noindent
11230 More details can be explored when looking at the purple
11231 \begin_inset Quotes eld
11232 \end_inset
11233
11234 Write Pushes
11235 \begin_inset Quotes erd
11236 \end_inset
11237
11238 / deep-blue
11239 \begin_inset Quotes eld
11240 \end_inset
11241
11242 Read Pushes
11243 \begin_inset Quotes erd
11244 \end_inset
11245
11246 displayed in further graphics at
11247 \begin_inset Flex URL
11248 status open
11249
11250 \begin_layout Plain Layout
11251
11252 http://www.blkreplay.org/examples/
11253 \end_layout
11254
11255 \end_inset
11256
11257 .
11258 \end_layout
11259
11260 \begin_layout Chapter
11261 \start_of_appendix
11262 Config File Parameters
11263 \begin_inset CommandInset label
11264 LatexCommand label
11265 name "chap:Config-File-Parameters"
11266
11267 \end_inset
11268
11269
11270 \end_layout
11271
11272 \begin_layout Standard
11273 \begin_inset ERT
11274 status open
11275
11276 \begin_layout Plain Layout
11277
11278
11279 \backslash
11280 lstset{ %
11281 \end_layout
11282
11283 \begin_layout Plain Layout
11284
11285 language=bash,
11286 \end_layout
11287
11288 \begin_layout Plain Layout
11289
11290 basicstyle=
11291 \backslash
11292 tt
11293 \backslash
11294 footnotesize,
11295 \end_layout
11296
11297 \begin_layout Plain Layout
11298
11299 % numbers=left,
11300 \end_layout
11301
11302 \begin_layout Plain Layout
11303
11304 % numberstyle=
11305 \backslash
11306 tiny
11307 \backslash
11308 color{gray},
11309 \end_layout
11310
11311 \begin_layout Plain Layout
11312
11313 % stepnumber=2,
11314 \end_layout
11315
11316 \begin_layout Plain Layout
11317
11318 % numbersep=5pt,
11319 \end_layout
11320
11321 \begin_layout Plain Layout
11322
11323 %backgroundcolor=
11324 \backslash
11325 color{white},
11326 \end_layout
11327
11328 \begin_layout Plain Layout
11329
11330 showspaces=false,
11331 \end_layout
11332
11333 \begin_layout Plain Layout
11334
11335 showstringspaces=true,
11336 \end_layout
11337
11338 \begin_layout Plain Layout
11339
11340 showtabs=true,
11341 \end_layout
11342
11343 \begin_layout Plain Layout
11344
11345 %frame=single,
11346 \end_layout
11347
11348 \begin_layout Plain Layout
11349
11350 %rulecolor=
11351 \backslash
11352 color{black},
11353 \end_layout
11354
11355 \begin_layout Plain Layout
11356
11357 tabsize=8,
11358 \end_layout
11359
11360 \begin_layout Plain Layout
11361
11362 captionpos=t,
11363 \end_layout
11364
11365 \begin_layout Plain Layout
11366
11367 breaklines=true,
11368 \end_layout
11369
11370 \begin_layout Plain Layout
11371
11372 breakatwhitespace=false,
11373 \end_layout
11374
11375 \begin_layout Plain Layout
11376
11377 title=
11378 \backslash
11379 lstname,
11380 \end_layout
11381
11382 \begin_layout Plain Layout
11383
11384 keywordstyle=
11385 \backslash
11386 color{brown},
11387 \end_layout
11388
11389 \begin_layout Plain Layout
11390
11391 commentstyle=
11392 \backslash
11393 color{OliveGreen},
11394 \end_layout
11395
11396 \begin_layout Plain Layout
11397
11398 }
11399 \end_layout
11400
11401 \end_inset
11402
11403
11404 \end_layout
11405
11406 \begin_layout Section
11407 Basic Parameters
11408 \end_layout
11409
11410 \begin_layout Subsection
11411 File
11412 \family typewriter
11413 user_modules.conf
11414 \family default
11415 :
11416 \end_layout
11417
11418 \begin_layout Standard
11419 \noindent
11420 \begin_inset ERT
11421 status open
11422
11423 \begin_layout Plain Layout
11424
11425
11426 \backslash
11427 lstinputlisting[language=bash]{example-run/user_modules.conf}
11428 \end_layout
11429
11430 \end_inset
11431
11432
11433 \end_layout
11434
11435 \begin_layout Subsection
11436 File
11437 \family typewriter
11438 default-main.conf
11439 \family default
11440 :
11441 \end_layout
11442
11443 \begin_layout Standard
11444 \noindent
11445 \begin_inset ERT
11446 status open
11447
11448 \begin_layout Plain Layout
11449
11450
11451 \backslash
11452 lstinputlisting[language=bash]{example-run/default-main.conf}
11453 \end_layout
11454
11455 \end_inset
11456
11457
11458 \end_layout
11459
11460 \begin_layout Section
11461 Ordinary Module Parameters
11462 \begin_inset CommandInset label
11463 LatexCommand label
11464 name "sec:Module-Parameters"
11465
11466 \end_inset
11467
11468
11469 \end_layout
11470
11471 \begin_layout Standard
11472 The following list is in the order of activation.
11473 \end_layout
11474
11475 \begin_layout Subsection
11476 File
11477 \family typewriter
11478 default-recreate_lvm.conf
11479 \end_layout
11480
11481 \begin_layout Standard
11482 \noindent
11483 \begin_inset ERT
11484 status open
11485
11486 \begin_layout Plain Layout
11487
11488
11489 \backslash
11490 lstinputlisting[language=bash]{example-run/default-recreate_lvm.conf}
11491 \end_layout
11492
11493 \end_inset
11494
11495
11496 \end_layout
11497
11498 \begin_layout Subsection
11499 File
11500 \family typewriter
11501 default-create_lv.conf
11502 \end_layout
11503
11504 \begin_layout Standard
11505 \noindent
11506 \begin_inset ERT
11507 status open
11508
11509 \begin_layout Plain Layout
11510
11511
11512 \backslash
11513 lstinputlisting[language=bash]{example-run/default-create_lv.conf}
11514 \end_layout
11515
11516 \end_inset
11517
11518
11519 \end_layout
11520
11521 \begin_layout Subsection
11522 File
11523 \family typewriter
11524 default-iscsi_target_iet.conf
11525 \end_layout
11526
11527 \begin_layout Standard
11528 \noindent
11529 \begin_inset ERT
11530 status open
11531
11532 \begin_layout Plain Layout
11533
11534
11535 \backslash
11536 lstinputlisting[language=bash]{example-run/default-iscsi_target_iet.conf}
11537 \end_layout
11538
11539 \end_inset
11540
11541
11542 \end_layout
11543
11544 \begin_layout Subsection
11545 File
11546 \family typewriter
11547 default-iscsi_initiator.conf
11548 \end_layout
11549
11550 \begin_layout Standard
11551 \noindent
11552 \begin_inset ERT
11553 status open
11554
11555 \begin_layout Plain Layout
11556
11557
11558 \backslash
11559 lstinputlisting[language=bash]{example-run/default-iscsi_initiator.conf}
11560 \end_layout
11561
11562 \end_inset
11563
11564
11565 \end_layout
11566
11567 \begin_layout Subsection
11568 File
11569 \family typewriter
11570 default-scheduler.conf
11571 \end_layout
11572
11573 \begin_layout Standard
11574 \noindent
11575 \begin_inset ERT
11576 status open
11577
11578 \begin_layout Plain Layout
11579
11580
11581 \backslash
11582 lstinputlisting[language=bash]{example-run/default-scheduler.conf}
11583 \end_layout
11584
11585 \end_inset
11586
11587
11588 \end_layout
11589
11590 \begin_layout Subsection
11591 File
11592 \family typewriter
11593 default-wipe.conf
11594 \end_layout
11595
11596 \begin_layout Standard
11597 \noindent
11598 \begin_inset ERT
11599 status open
11600
11601 \begin_layout Plain Layout
11602
11603
11604 \backslash
11605 lstinputlisting[language=bash]{example-run/default-wipe.conf}
11606 \end_layout
11607
11608 \end_inset
11609
11610
11611 \end_layout
11612
11613 \begin_layout Subsection
11614 File
11615 \family typewriter
11616 default-bbu_megaraid.conf
11617 \end_layout
11618
11619 \begin_layout Standard
11620 \noindent
11621 \begin_inset ERT
11622 status open
11623
11624 \begin_layout Plain Layout
11625
11626
11627 \backslash
11628 lstinputlisting[language=bash]{example-run/default-wipe.conf}
11629 \end_layout
11630
11631 \end_inset
11632
11633
11634 \end_layout
11635
11636 \begin_layout Subsection
11637 File
11638 \family typewriter
11639 default-graph.conf
11640 \end_layout
11641
11642 \begin_layout Standard
11643 \noindent
11644 \begin_inset ERT
11645 status open
11646
11647 \begin_layout Plain Layout
11648
11649
11650 \backslash
11651 lstinputlisting[language=bash]{example-run/default-graph.conf}
11652 \end_layout
11653
11654 \end_inset
11655
11656
11657 \end_layout
11658
11659 \begin_layout Section
11660 Pipe Module Parameters
11661 \begin_inset CommandInset label
11662 LatexCommand label
11663 name "sec:Pipe-Module-Parameters"
11664
11665 \end_inset
11666
11667
11668 \end_layout
11669
11670 \begin_layout Subsection
11671 File
11672 \family typewriter
11673 default-pipe_repeat.conf
11674 \end_layout
11675
11676 \begin_layout Standard
11677 \noindent
11678 \begin_inset ERT
11679 status open
11680
11681 \begin_layout Plain Layout
11682
11683
11684 \backslash
11685 lstinputlisting[language=bash]{example-run/default-pipe_repeat.conf}
11686 \end_layout
11687
11688 \end_inset
11689
11690
11691 \end_layout
11692
11693 \begin_layout Subsection
11694 File
11695 \family typewriter
11696 default-pipe_slip.conf
11697 \begin_inset CommandInset label
11698 LatexCommand label
11699 name "sub:File-default-pipe_slip.conf"
11700
11701 \end_inset
11702
11703
11704 \end_layout
11705
11706 \begin_layout Standard
11707 \noindent
11708 \begin_inset ERT
11709 status open
11710
11711 \begin_layout Plain Layout
11712
11713
11714 \backslash
11715 lstinputlisting[language=bash]{example-run/default-pipe_slip.conf}
11716 \end_layout
11717
11718 \end_inset
11719
11720
11721 \end_layout
11722
11723 \begin_layout Subsection
11724 File
11725 \family typewriter
11726 default-pipe_subst.conf
11727 \end_layout
11728
11729 \begin_layout Standard
11730 \noindent
11731 \begin_inset ERT
11732 status open
11733
11734 \begin_layout Plain Layout
11735
11736
11737 \backslash
11738 lstinputlisting[language=bash]{example-run/default-pipe_subst.conf}
11739 \end_layout
11740
11741 \end_inset
11742
11743
11744 \end_layout
11745
11746 \begin_layout Subsection
11747 File
11748 \family typewriter
11749 default-pipe_spread.conf
11750 \begin_inset CommandInset label
11751 LatexCommand label
11752 name "sub:File-default-pipe_spread.conf"
11753
11754 \end_inset
11755
11756
11757 \end_layout
11758
11759 \begin_layout Standard
11760 \noindent
11761 \begin_inset ERT
11762 status open
11763
11764 \begin_layout Plain Layout
11765
11766
11767 \backslash
11768 lstinputlisting[language=bash]{example-run/default-pipe_spread.conf}
11769 \end_layout
11770
11771 \end_inset
11772
11773
11774 \end_layout
11775
11776 \begin_layout Subsection
11777 File
11778 \family typewriter
11779 default-pipe_resize.conf
11780 \begin_inset CommandInset label
11781 LatexCommand label
11782 name "sub:File-default-pipe_resize.conf"
11783
11784 \end_inset
11785
11786
11787 \end_layout
11788
11789 \begin_layout Standard
11790 \noindent
11791 \begin_inset ERT
11792 status open
11793
11794 \begin_layout Plain Layout
11795
11796
11797 \backslash
11798 lstinputlisting[language=bash]{example-run/default-pipe_resize.conf}
11799 \end_layout
11800
11801 \end_inset
11802
11803
11804 \end_layout
11805
11806 \begin_layout Subsection
11807 File
11808 \family typewriter
11809 default-pipe_cmd.conf
11810 \end_layout
11811
11812 \begin_layout Standard
11813 \noindent
11814 \begin_inset ERT
11815 status open
11816
11817 \begin_layout Plain Layout
11818
11819
11820 \backslash
11821 lstinputlisting[language=bash]{example-run/default-pipe_cmd.conf}
11822 \end_layout
11823
11824 \end_inset
11825
11826
11827 \end_layout
11828
11829 \begin_layout Chapter
11830 File Format
11831 \end_layout
11832
11833 \begin_layout Standard
11834 The
11835 \family typewriter
11836 *.load.gz
11837 \family default
11838 and
11839 \family typewriter
11840 *.replay.gz
11841 \family default
11842 files are in essence compressed CSV files, using
11843 \family typewriter
11844
11845 \begin_inset Quotes eld
11846 \end_inset
11847
11848 ;
11849 \begin_inset Quotes erd
11850 \end_inset
11851
11852
11853 \family default
11854 as delimiter.
11855 However, errors and comments can also occur in the file, violating the
11856 classical CSV format.
11857 \end_layout
11858
11859 \begin_layout Standard
11860 In order to extract classical CSV, use the following command:
11861 \end_layout
11862
11863 \begin_layout Quotation
11864
11865 \family typewriter
11866 zgrep
11867 \begin_inset Quotes eld
11868 \end_inset
11869
11870 ;
11871 \begin_inset Quotes erd
11872 \end_inset
11873
11874
11875 \emph on
11876 myname
11877 \emph default
11878 .replay.gz | less
11879 \end_layout
11880
11881 \begin_layout Standard
11882 The first line of the output contains the column names (and could be removed
11883 by inserting
11884 \family typewriter
11885 | tail -n +2
11886 \family default
11887 into the pipeline).
11888 In
11889 \emph on
11890 theory
11891 \emph default
11892 , you could even use MS Excel to work on the above output.
11893 However, in practice the file will be so extremely large that interactive
11894 workstation tools will likely start swapping your system to death, or even
11895 bombing.
11896 \end_layout
11897
11898 \begin_layout Standard
11899 So the usual way to operate on the output is a
11900 \begin_inset Quotes eld
11901 \end_inset
11902
11903 pipe and filters style
11904 \begin_inset Quotes erd
11905 \end_inset
11906
11907 , using stream-processing tools like
11908 \family typewriter
11909 sed
11910 \family default
11911 ,
11912 \family typewriter
11913 awk
11914 \family default
11915 ,
11916 \family typewriter
11917 perl
11918 \family default
11919 , and so on.
11920 \end_layout
11921
11922 \begin_layout Standard
11923 Both
11924 \family typewriter
11925 *.load.gz
11926 \family default
11927 and
11928 \family typewriter
11929 *.replay.gz
11930 \family default
11931 use the same format.
11932 However,
11933 \family typewriter
11934 *.load.gz
11935 \family default
11936
11937 \series bold
11938 \emph on
11939 must
11940 \series default
11941 \emph default
11942 be sorted numerically according to the timestamp, while
11943 \family typewriter
11944 *.replay.gz
11945 \family default
11946 is usually unsorted (roughly showing the order of request completion in
11947 most cases, which is different from the submission timestamp ordering).
11948 In order to re-use
11949 \family typewriter
11950 *.replay.gz
11951 \family default
11952 again as input for another run of
11953 \family typewriter
11954 blkreplay
11955 \family default
11956 , you must sort it with
11957 \family typewriter
11958 sort -n
11959 \family default
11960 .
11961 In some cases, public
11962 \family typewriter
11963 blktrace
11964 \family default
11965 recordings at
11966 \family typewriter
11967 www.blktrace.org/loads/
11968 \family default
11969 will contain only constants 0.0 as delays and latencies.
11970 \end_layout
11971
11972 \begin_layout Standard
11973 Feel free to filter your results, or create your own artifical loads.
11974 \end_layout
11975
11976 \begin_layout Standard
11977 \noindent
11978 Enjoy.
11979 \end_layout
11980
11981 \begin_layout Chapter
11982 Validation of the
11983 \family typewriter
11984 blkreplay
11985 \family default
11986 Tool
11987 \end_layout
11988
11989 \begin_layout Standard
11990 Some effort has been spent to check that
11991 \family typewriter
11992 blkreplay
11993 \family default
11994 really produces the intended kind of IO load on the target system, and
11995 that its measurement results are likely to be valid.
11996 However, there is no guarantee against hidden bugs, misconfigurations,
11997 or other types of accidents.
11998 See also the disclaimer of the GPL license.
11999 \end_layout
12000
12001 \begin_layout Standard
12002 The following tools try the
12003 \emph on
12004 help
12005 \emph default
12006 you by making
12007 \family typewriter
12008 blkreplay
12009 \family default
12010 as transparent to the public as possible, but cannot form an absolute guarantee
12011 against all possible sources of errors.
12012 \end_layout
12013
12014 \begin_layout Standard
12015 Please report any bugs to the author, or, even better, send patches or git
12016 pull requests.
12017 \end_layout
12018
12019 \begin_layout Section
12020 Running
12021 \family typewriter
12022 blktrace
12023 \family default
12024 during
12025 \family typewriter
12026 blkreplay
12027 \end_layout
12028
12029 \begin_layout Standard
12030 The timeliness of IO requests is already measured and displayed by the
12031 \family typewriter
12032 *.latency.*
12033 \family default
12034 graphics.
12035 The following steps will thus concentrate on
12036 \emph on
12037 completeness
12038 \emph default
12039 .
12040 \end_layout
12041
12042 \begin_layout Standard
12043 In order to check that all IO requests present in a
12044 \family typewriter
12045 *.load.gz
12046 \family default
12047 file are actually applied to the test candidate, you can (and should) re-record
12048 the IO load during an execution of
12049 \family typewriter
12050 blkreplay
12051 \family default
12052 via
12053 \family typewriter
12054 blktrace
12055 \family default
12056 .
12057 The following steps are to be done manually:
12058 \end_layout
12059
12060 \begin_layout Enumerate
12061 Start
12062 \family typewriter
12063 blktrace
12064 \family default
12065 manually, as described in chapter
12066 \begin_inset CommandInset ref
12067 LatexCommand ref
12068 reference "chap:How-to-use"
12069
12070 \end_inset
12071
12072 ,
12073 \emph on
12074 before
12075 \emph default
12076 you start
12077 \family typewriter
12078 tree-replay.sh
12079 \family default
12080 .
12081 Be sure to catch all involved devices on all involved hosts.
12082 \end_layout
12083
12084 \begin_layout Enumerate
12085 Either kill
12086 \family typewriter
12087 blktrace
12088 \family default
12089 with Ctrl-C
12090 \emph on
12091 after
12092 \emph default
12093 tree-replay.sh is done, or use the
12094 \family typewriter
12095 -w
12096 \family default
12097 option to specify a
12098 \emph on
12099 strictly longer
12100 \emph default
12101 (plus some safety margin) recording time.
12102 \begin_inset Newline newline
12103 \end_inset
12104
12105 Attention! some versions of
12106 \family typewriter
12107 blktrace
12108 \family default
12109 seem to loose some requests in some buffers, at least under some conditions.
12110 Usually the number of lost events is rather low, about 10.
12111 Keep that in mind if you later discover some purpoted discrepancies!
12112 \end_layout
12113
12114 \begin_layout Enumerate
12115 Copy the resulting
12116 \family typewriter
12117 *.blktrace.[0-9]+
12118 \family default
12119 file to your workstation.
12120 \end_layout
12121
12122 \begin_layout Enumerate
12123 Invoke
12124 \family typewriter
12125 /path/to/blkreplay/scripts/check_replay_against_blktrace.sh
12126 \emph on
12127 myresult
12128 \emph default
12129 .replay.gz
12130 \emph on
12131 mytrace-prefix
12132 \family default
12133 \emph default
12134 (exactly two parameters).
12135 \begin_inset Newline newline
12136 \end_inset
12137
12138 Hint: the coincidence between
12139 \family typewriter
12140 \emph on
12141 myresult
12142 \emph default
12143 .replay.gz
12144 \family default
12145 and
12146 \family typewriter
12147 \emph on
12148 myload
12149 \emph default
12150 .load.gz
12151 \family default
12152 must be checked separately, using tools like
12153 \family typewriter
12154 cut
12155 \family default
12156 and
12157 \family typewriter
12158 diff
12159 \family default
12160 .
12161 \end_layout
12162
12163 \begin_layout Enumerate
12164 Enjoy the result in form of a wide side-by-side
12165 \family typewriter
12166 diff
12167 \family default
12168 output.
12169 \end_layout
12170
12171 \begin_layout Standard
12172 Hint: no output means that no differences have been found.
12173 The timeliness as well as the order of requests is
12174 \emph on
12175 not
12176 \emph default
12177 checked.
12178 Any difference means that a request is missing somewhere, or has been altered
12179 in some way.
12180 Expect some minor differences (usully less than 1%, but sometimes more)
12181 from request splitting by some device drivers.
12182 \end_layout
12183
12184 \begin_layout Standard
12185 Keep in mind that sometimes
12186 \family typewriter
12187 blktrace
12188 \family default
12189 looses some events, leading to
12190 \begin_inset Quotes eld
12191 \end_inset
12192
12193 false positives
12194 \begin_inset Quotes erd
12195 \end_inset
12196
12197 .
12198 You should take this as a validation of the validation tool ;)
12199 \end_layout
12200
12201 \begin_layout Section
12202 Verbosity Graphics
12203 \end_layout
12204
12205 \begin_layout Standard
12206 Experts only.
12207 It is easy to draw wrong conclusions from this.
12208 \end_layout
12209
12210 \begin_layout Standard
12211 Enable the following parameters:
12212 \family typewriter
12213 verbose=10;
12214 \family default
12215
12216 \family typewriter
12217 enable_compress_ssh=1; graph_options="--static --verbose"
12218 \family default
12219 in your respective
12220 \family typewriter
12221 *.conf
12222 \family default
12223 files.
12224 \end_layout
12225
12226 \begin_layout Standard
12227 The resulting
12228 \family typewriter
12229 *.replay.gz
12230 \family default
12231 file will now contain
12232 \emph on
12233 huge masses
12234 \emph default
12235 of lines starting with
12236 \family typewriter
12237 INFO:
12238 \family default
12239 and carrying lots of internal variables, similar to debug / tracing output.
12240 \end_layout
12241
12242 \begin_layout Labeling
12243 \labelwidthstring 00.00.0000
12244 \noindent
12245 \begin_inset Graphics
12246 filename images/MatieresCorrosives.png
12247 lyxscale 50
12248 scale 17
12249
12250 \end_inset
12251
12252 The sheer masses of data written to
12253 \family typewriter
12254 stdout
12255 \family default
12256 (as such) my lead to distortions of your measurements, e.g.
12257 artificial delays or even hangs.
12258 Always compare to
12259 \family typewriter
12260 --verbose=0
12261 \family default
12262 under otherwise identical conditions, to be sure! In order to check whether
12263
12264 \family typewriter
12265 stdout
12266 \family default
12267 could have become a bottleneck, please search for the lines with
12268 \family typewriter
12269 flush_total=n.nnnnnnnnn
12270 \family default
12271 (some timing with nanosecond resolution).
12272 In particular, the main thread (additionally indicated by
12273 \family typewriter
12274 role='main'
12275 \family default
12276 ) should have spent much less time in
12277 \family typewriter
12278 fflush(stdout)
12279 \family default
12280 than total running time.
12281 When the network is the bottleneck,
12282 \family typewriter
12283 enable_compress_ssh=1
12284 \family default
12285 may help; otherwise disabling could be better -- please check.
12286 \end_layout
12287
12288 \begin_layout Standard
12289 \noindent
12290 In addition, the following types of internal visualizations will be produced:
12291 \end_layout
12292
12293 \begin_layout Description
12294 \noindent
12295
12296 \family typewriter
12297 *.submit_level.png
12298 \family default
12299
12300 \begin_inset ERT
12301 status open
12302
12303 \begin_layout Plain Layout
12304
12305
12306 \backslash
12307 strut
12308 \end_layout
12309
12310 \end_inset
12311
12312
12313 \begin_inset Newline newline
12314 \end_inset
12315
12316 Shows the actual filling level of the pipelines between the main thread
12317 and the workers.
12318 The x axis is in units of requests, and in
12319 \emph on
12320 completion order
12321 \emph default
12322 (different from the submission order).
12323 \end_layout
12324
12325 \begin_layout Standard
12326 \noindent
12327 \begin_inset Graphics
12328 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g50.submit_level.png
12329 lyxscale 50
12330 width 100col%
12331
12332 \end_inset
12333
12334
12335 \end_layout
12336
12337 \begin_layout Description
12338 \noindent
12339
12340 \family typewriter
12341 *.pushback_level.png
12342 \family default
12343
12344 \begin_inset ERT
12345 status open
12346
12347 \begin_layout Plain Layout
12348
12349
12350 \backslash
12351 strut
12352 \end_layout
12353
12354 \end_inset
12355
12356
12357 \begin_inset Newline newline
12358 \end_inset
12359
12360 Shows the current number of requests on the pushback list.
12361 Only meaningful at
12362 \family typewriter
12363 --with-partial
12364 \family default
12365 .
12366 \end_layout
12367
12368 \begin_layout Description
12369 \noindent
12370
12371 \family typewriter
12372 *.submit_ahead.png
12373 \family default
12374
12375 \begin_inset ERT
12376 status open
12377
12378 \begin_layout Plain Layout
12379
12380
12381 \backslash
12382 strut
12383 \end_layout
12384
12385 \end_inset
12386
12387
12388 \begin_inset Newline newline
12389 \end_inset
12390
12391 Shows the time difference between submission to the pipelines (which is
12392 usually ahead of the execution time, but not always) and the intended execution
12393 timestamp.
12394 \end_layout
12395
12396 \begin_layout Standard
12397 \noindent
12398 \begin_inset Graphics
12399 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g52.submit_ahead.png
12400 lyxscale 50
12401 width 100col%
12402
12403 \end_inset
12404
12405
12406 \end_layout
12407
12408 \begin_layout Labeling
12409 \labelwidthstring 00.00.0000
12410 \noindent
12411 \begin_inset Graphics
12412 filename images/MatieresCorrosives.png
12413 lyxscale 50
12414 scale 17
12415
12416 \end_inset
12417
12418 Don't draw wrong conclusions from negative values here! They
12419 \emph on
12420 can
12421 \emph default
12422 be a sign of
12423 \begin_inset Quotes eld
12424 \end_inset
12425
12426 too late
12427 \begin_inset Quotes erd
12428 \end_inset
12429
12430 , but the
12431 \emph on
12432 reason
12433 \emph default
12434 is almost always in the test candidate, just when ordinary delays are produced.
12435 \end_layout
12436
12437 \begin_layout Description
12438 \noindent
12439
12440 \family typewriter
12441 *.submit_lag.png
12442 \family default
12443
12444 \begin_inset ERT
12445 status open
12446
12447 \begin_layout Plain Layout
12448
12449
12450 \backslash
12451 strut
12452 \end_layout
12453
12454 \end_inset
12455
12456
12457 \begin_inset Newline newline
12458 \end_inset
12459
12460 The time difference between submission to the pipelines, and reception by
12461 the worker thread (aka communication latency).
12462 \end_layout
12463
12464 \begin_layout Standard
12465 \begin_inset Graphics
12466 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g53.submit_lag.png
12467 lyxscale 50
12468 width 100col%
12469
12470 \end_inset
12471
12472
12473 \end_layout
12474
12475 \begin_layout Description
12476 \noindent
12477
12478 \family typewriter
12479 *.submit_lag_cumul.png
12480 \family default
12481
12482 \begin_inset ERT
12483 status open
12484
12485 \begin_layout Plain Layout
12486
12487
12488 \backslash
12489 strut
12490 \end_layout
12491
12492 \end_inset
12493
12494
12495 \begin_inset Newline newline
12496 \end_inset
12497
12498 The same, but cumulized over time.
12499 \end_layout
12500
12501 \begin_layout Standard
12502 \noindent
12503 \begin_inset Graphics
12504 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g53.submit_lag_cumul.png
12505 lyxscale 50
12506 width 100col%
12507
12508 \end_inset
12509
12510
12511 \end_layout
12512
12513 \begin_layout Labeling
12514 \labelwidthstring 00.00.0000
12515 \noindent
12516 \begin_inset Graphics
12517 filename images/MatieresCorrosives.png
12518 lyxscale 50
12519 scale 17
12520
12521 \end_inset
12522
12523 Notice than cumulation of time lags belonging to different threads is
12524 \begin_inset Quotes eld
12525 \end_inset
12526
12527 unfair
12528 \begin_inset Quotes erd
12529 \end_inset
12530
12531 by concept.
12532 Large values can regularly occur because of queueing in the submit pipelines.
12533 Complaints about high values make only sense in case of
12534 \family typewriter
12535 $(( bottleneck <= threads )).
12536 \end_layout
12537
12538 \begin_layout Description
12539 \noindent
12540
12541 \family typewriter
12542 *.answer_lag.png
12543 \family default
12544
12545 \begin_inset ERT
12546 status open
12547
12548 \begin_layout Plain Layout
12549
12550
12551 \backslash
12552 strut
12553 \end_layout
12554
12555 \end_inset
12556
12557
12558 \begin_inset Newline newline
12559 \end_inset
12560
12561 The time difference between submission to the answer pipeline caused by
12562 a worker thread, and reception by the main thread (aka communication latency).
12563 \end_layout
12564
12565 \begin_layout Standard
12566 \begin_inset Graphics
12567 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g54.answer_lag.png
12568 lyxscale 50
12569 width 100col%
12570
12571 \end_inset
12572
12573
12574 \end_layout
12575
12576 \begin_layout Description
12577 \noindent
12578
12579 \family typewriter
12580 *.answer_lag_cumul.png
12581 \family default
12582
12583 \begin_inset ERT
12584 status open
12585
12586 \begin_layout Plain Layout
12587
12588
12589 \backslash
12590 strut
12591 \end_layout
12592
12593 \end_inset
12594
12595
12596 \begin_inset Newline newline
12597 \end_inset
12598
12599 Dito, cumulated over time.
12600 \end_layout
12601
12602 \begin_layout Standard
12603 \begin_inset Graphics
12604 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g54.answer_lag_cumul.png
12605 lyxscale 50
12606 width 100col%
12607
12608 \end_inset
12609
12610
12611 \end_layout
12612
12613 \begin_layout Labeling
12614 \labelwidthstring 00.00.0000
12615 \noindent
12616 \begin_inset Graphics
12617 filename images/MatieresCorrosives.png
12618 lyxscale 50
12619 scale 17
12620
12621 \end_inset
12622
12623 Similar warning as before: high values occur regularly in
12624 \family typewriter
12625 --with-ordering
12626 \family default
12627 mode.
12628 Only be startled in
12629 \emph on
12630 some
12631 \emph default
12632 other cases!
12633 \end_layout
12634
12635 \begin_layout Description
12636 \noindent
12637
12638 \family typewriter
12639 *.input_wait[_cumul].png
12640 \family default
12641
12642 \begin_inset ERT
12643 status open
12644
12645 \begin_layout Plain Layout
12646
12647
12648 \backslash
12649 strut
12650 \end_layout
12651
12652 \end_inset
12653
12654
12655 \begin_inset Newline newline
12656 \end_inset
12657
12658 The time differences before and after
12659 \family typewriter
12660 fgets()
12661 \family default
12662 (aka stalls in
12663 \family typewriter
12664 stdin
12665 \family default
12666 ).
12667 When these numbers get too high, this time you should be alerted quite
12668 rightly.
12669 \end_layout
12670
12671 \begin_layout Standard
12672 \begin_inset Graphics
12673 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g55.input_wait.png
12674 lyxscale 50
12675 width 100col%
12676
12677 \end_inset
12678
12679
12680 \end_layout
12681
12682 \begin_layout Standard
12683 \noindent
12684 \begin_inset Graphics
12685 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g55.input_wait_cumul.png
12686 lyxscale 50
12687 width 100col%
12688
12689 \end_inset
12690
12691
12692 \end_layout
12693
12694 \begin_layout Description
12695 \noindent
12696
12697 \family typewriter
12698 *.answer_wait[_cumul].png
12699 \family default
12700
12701 \begin_inset ERT
12702 status open
12703
12704 \begin_layout Plain Layout
12705
12706
12707 \backslash
12708 strut
12709 \end_layout
12710
12711 \end_inset
12712
12713
12714 \begin_inset Newline newline
12715 \end_inset
12716
12717 The time difference of the main thread it is spending for answer wait.
12718 Not to be confused with communication latencies! High number will occur
12719 regularly.
12720 It is even a sign of good performance when the cumulation almost reaches
12721 the total processing time.
12722 \end_layout
12723
12724 \begin_layout Standard
12725 \begin_inset Graphics
12726 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g56.answer_wait.png
12727 lyxscale 50
12728 width 100col%
12729
12730 \end_inset
12731
12732
12733 \end_layout
12734
12735 \begin_layout Standard
12736 \noindent
12737 \begin_inset Graphics
12738 filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g56.answer_wait_cumul.png
12739 lyxscale 50
12740 width 100col%
12741
12742 \end_inset
12743
12744
12745 \end_layout
12746
12747 \begin_layout Chapter
12748 GNU Free Documentation License
12749 \begin_inset CommandInset label
12750 LatexCommand label
12751 name "chap:GNU-FDL"
12752
12753 \end_inset
12754
12755
12756 \end_layout
12757
12758 \begin_layout Standard
12759 \noindent
12760
12761 \family typewriter
12762 \size footnotesize
12763 \begin_inset ERT
12764 status open
12765
12766 \begin_layout Plain Layout
12767
12768
12769 \backslash
12770 lstinputlisting{fdl.txt}
12771 \end_layout
12772
12773 \end_inset
12774
12775
12776 \end_layout
12777
12778 \end_body
12779 \end_document