#LyX 2.0 created this file. For more info see http://www.lyx.org/
\lyxformat 413
\begin_document
\begin_header
\textclass scrreprt
\begin_preamble
\usepackage[dvipsnames]{xcolor}
\usepackage{listings}
\end_preamble
\options abstracton
\use_default_options true
\begin_modules
customHeadersFooters
enumitem
fixltx2e
\end_modules
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman default
\font_sans default
\font_typewriter default
\font_default_family rmdefault
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100
\font_tt_scale 100
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize 10
\spacing single
\use_hyperref true
\pdf_title "blkreplay"
\pdf_author "Thomas Schöbel-Theuer"
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen false
\pdf_bookmarksopenlevel 1
\pdf_breaklinks true
\pdf_pdfborder true
\pdf_colorlinks true
\pdf_backref false
\pdf_pdfusetitle true
\papersize a4paper
\use_geometry true
\use_amsmath 1
\use_esint 1
\use_mhchem 1
\use_mathdots 1
\cite_engine basic
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\use_refstyle 1
\index Index
\shortcut idx
\color #008000
\end_index
\leftmargin 3.7cm
\topmargin 2.7cm
\rightmargin 2.8cm
\bottommargin 2.3cm
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\quotes_language english
\papercolumns 1
\papersides 2
\paperpagestyle headings
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header
\begin_body
\begin_layout Title
\family typewriter
blkreplay
\family default
and Sonar Diagrams
\end_layout
\begin_layout Subtitle
A manual for system administrators,
\begin_inset Newline newline
\end_inset
kernel developers,
\begin_inset Newline newline
\end_inset
hardware technicians,
\begin_inset Newline newline
\end_inset
and experts in IO systems
\end_layout
\begin_layout Author
Thomas Schöbel-Theuer (
\family typewriter
tst@1und1.de
\family default
)
\end_layout
\begin_layout Date
Version 1.0.0
\end_layout
\begin_layout Standard
\noindent
Copyright (C) 2012 Thomas Schöbel-Theuer / 1&1 Internet AG (see
\begin_inset Flex URL
status open
\begin_layout Plain Layout
http://www.1und1.de
\end_layout
\end_inset
shortly called 1&1 in the following).
\end_layout
\begin_layout Standard
Permission is granted to copy, distribute and/or modify this document under
the terms of the GNU Free Documentation License, Version 1.3 or any later
version published by the Free Software Foundation; with no Invariant Sections,
no Front-Cover Texts, and no Back-Cover Texts.
A copy of the license is included in the section entitled
\begin_inset Quotes eld
\end_inset
\begin_inset CommandInset ref
LatexCommand nameref
reference "chap:GNU-FDL"
\end_inset
\begin_inset Quotes erd
\end_inset
.
\end_layout
\begin_layout Abstract
\family typewriter
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
sloppy
\end_layout
\end_inset
blkreplay
\family default
is a GPL'ed toolkit, driving the block layer of Linux (or other Unix-like
OSes) while measuring latency and throughput of IO operations for later
visualization (so-called
\begin_inset Quotes eld
\end_inset
sonar diagrams
\begin_inset Quotes erd
\end_inset
and others).
\end_layout
\begin_layout Abstract
\family typewriter
blkreplay
\family default
comes with a modular and extensible
\series bold
test suite
\series default
, automating large projects for testing and/or benchmarking.
\end_layout
\begin_layout Abstract
\family typewriter
blkreplay
\family default
can be used to
\series bold
test physical hardware
\series default
, e.g.
compare different brands of hard disks, or RAID controllers / their settings
/ RAID rebuild performance degradation, or to evaluate the effect of SSD
caching, or to compare different block level transports like iSCSI vs Fibrechan
nel (over different kinds of storage networks).
\end_layout
\begin_layout Abstract
It can compare
\series bold
virtual hardware
\series default
(like
\family typewriter
vmware
\family default
or
\family typewriter
XenServer
\family default
block devices, or any type of block-level
\series bold
storage virtualization
\series default
) to each other or to physical hardware, provided the test setup is handled
very carefully
\begin_inset Foot
status open
\begin_layout Plain Layout
Otherwise you may get
\emph on
useless fake results
\emph default
measuring the
\emph on
cache
\emph default
performance or even
\emph on
sparse accesses to holes
\emph default
instead of the real hardware performance.
Such fake results may differ from real results by
\emph on
factors
\emph default
, er even by
\emph on
orders of magnitude
\emph default
.
\family typewriter
blkreplay
\family default
comes with thorough descriptions teaching you how to avoid the most common
pitfalls.
\end_layout
\end_inset
.
\end_layout
\begin_layout Abstract
\family typewriter
blkreplay
\family default
can compare
\series bold
commercial storage
\series default
systems from vendors like EMC, Net\SpecialChar \-
App, IBM, Hitachi, etc to each other
or to cheap off-the-shelf hardware (in order to determine the price/performance
ratio), provided the same care
\begin_inset Foot
status open
\begin_layout Plain Layout
Notice that most commercial storage systems
\series bold
\emph on
are
\series default
\emph default
in fact nothing but virtualized storage, so the above warnings about possible
\emph on
fake results
\emph default
apply.
\end_layout
\end_inset
is taken.
\end_layout
\begin_layout Abstract
Furthermore, it can be used for tests of the
\series bold
Linux kernel
\series default
, e.g.
for testing device drivers, comparing IO schedulers at different load patterns,
determining the overhead of Linux
\family typewriter
dm
\family default
targets or the impact of network problems to DRBD, and much more.
\end_layout
\begin_layout Abstract
In addition to artificial loads like random read-write sweeps and various
kinds of
\emph on
overload tests
\emph default
, it can also replay
\series bold
natural loads
\series default
which have been recorded by
\family typewriter
blktrace
\family default
at heavily-loaded production servers at big data centers.
\family typewriter
blkreplay
\family default
comes with a
\series bold
large collection of natural loads
\series default
from a wide spectrum of applications (such as web servers, databases, dedicated
servers, etc) which have been released to the public by 1&1 under GPL.
Some of these natural loads have recorded the real-life disk access behaviour
from servers serving thousands of customers in parallel.
Static analysis of the workingset behaviour of such natural loads is implemente
d.
\end_layout
\begin_layout Abstract
At 1&1,
\family typewriter
blkreplay
\family default
has even been used as a tool for root cause analysis of incidents: for
example, high load peaks presumably stemming from traffic jam (or other
sources of overload) were recorded at production sites in real time by
\family typewriter
blktrace
\family default
, and later replayed in the laboratory (without causing customer impact)
seeking for the cause of trouble, or improving the safety margins by choice
of better hardware.
\end_layout
\begin_layout Abstract
For experts in IO subsystems, visualization techniques like
\begin_inset Quotes eld
\end_inset
sonar diagrams
\begin_inset Quotes erd
\end_inset
can reveal (parts of) the internal structure of complex IO systems, such
as cache hierarchies or other hierarchical storage systems.
\end_layout
\begin_layout Abstract
As a community project under GPL,
\family typewriter
blkreplay
\family default
is open to contributions from hardware vendors, other data centers, the
kernel hacker community, etc.
\end_layout
\begin_layout Standard
\begin_inset CommandInset toc
LatexCommand tableofcontents
\end_inset
\end_layout
\begin_layout Chapter
Why Synthetic Benchmarks suck
\end_layout
\begin_layout Standard
There are a lot of benchmark tools around, like
\family typewriter
iozone
\family default
,
\family typewriter
iometer
\family default
,
\family typewriter
iperf
\family default
,
\family typewriter
bonnie
\family default
(
\family typewriter
++
\family default
) and many others.
\end_layout
\begin_layout Standard
What do they have in
\emph on
common
\emph default
\begin_inset Foot
status open
\begin_layout Plain Layout
Of course, the mentioned performance measuring tools are targeted at inspection
of different components of an OS, such as network, filesystem layer, and
block IO layer.
Here the question is a about
\emph on
commonality
\emph default
, not differences.
\end_layout
\end_inset
?
\end_layout
\begin_layout Quotation
Simply, most of them generate an
\series bold
artificial load
\series default
onto your system.
\end_layout
\begin_layout Standard
Artificial loads, as opposed to
\series bold
natural loads
\series default
, have a main disadvantage: they cannot answer questions like
\begin_inset Quotes eld
\end_inset
will my application Z run on system A reliably?
\begin_inset Quotes erd
\end_inset
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
question
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
artificial
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
natural
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Is system A better than B for application Z?
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\begin_inset Formula $partly$
\end_inset
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
Does application Z run on system A?
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
-
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Standard
\noindent
It seems that some people
\emph on
believe
\emph default
that synthetic benchmarks can be used even at the position of the dash
in the above table.
These people are wrong.
\end_layout
\begin_layout Standard
Experiences at big data centers at 1&1 show that sometimes the differences
between results from artificial benchmarks and real-world application behaviour
are very large.
We found cases where artificial benchmarks (adjusting parameters like IOPS)
suggested that a particular application
\emph on
should
\emph default
run on a particular hardware system, but the real application
\emph on
didn't
\emph default
: after deployment, a systematic series of incidents
\emph on
disproved
\emph default
\begin_inset Foot
status open
\begin_layout Plain Layout
Sometimes we got results in the other direction: artificial benchmarks suggested
that a particular application would
\emph on
not
\emph default
run, but in reality it
\emph on
did
\emph default
run.
\end_layout
\end_inset
the validity of the former benchmark results for the
\emph on
originally intended statement
\emph default
.
The failed prediction from the artificial benchmarks led to a
\series bold
failed invest
\series default
.
\end_layout
\begin_layout Standard
What can we do about that?
\end_layout
\begin_layout Standard
Obviously, parameters like IOPS (even when enriched with attributes like
\begin_inset Quotes eld
\end_inset
average IOPS
\begin_inset Quotes erd
\end_inset
or
\begin_inset Quotes eld
\end_inset
peak IOPS
\begin_inset Quotes erd
\end_inset
) are
\emph on
not
\emph default
representative for description of the real-world behaviour of applications.
Attempts to describe real-world behaviour in mathematical terms of analytic
functions have been already made in the 1970's; they failed.
All such models can
\emph on
try
\emph default
to describe an
\emph on
approximation
\emph default
of real-world behaviour, if enough knowledge about the application
\emph on
would
\emph default
be available.
\end_layout
\begin_layout Standard
So why trying to deal with tools that never can fully describe real-world
application behaviour, when there
\emph on
exist
\emph default
tools which definitely
\emph on
can
\emph default
do?
\end_layout
\begin_layout Standard
One of them is
\family typewriter
blkreplay
\family default
.
\end_layout
\begin_layout Chapter
How
\family typewriter
blkreplay
\family default
works
\end_layout
\begin_layout Section
Principle
\end_layout
\begin_layout Standard
In some sense,
\family typewriter
blkreplay
\family default
is just the opposite of the well-known Linux kernel tool
\family typewriter
blktrace
\family default
: recordings made by
\family typewriter
blktrace
\family default
are simply replayed on
\emph on
another
\emph default
block device.
\end_layout
\begin_layout Standard
A
\family typewriter
blktrace
\family default
record of an IO request contains the following information:
\end_layout
\begin_layout Enumerate
timestamp of the IO request (nanosecond resolution)
\end_layout
\begin_layout Enumerate
position of the IO request (sector#)
\end_layout
\begin_layout Enumerate
length of the IO request (#sectors)
\end_layout
\begin_layout Enumerate
direction: R[ead] or W[write]
\end_layout
\begin_layout Standard
Notice that
\family typewriter
blktrace
\family default
records do
\emph on
not
\emph default
contain any data.
Therefore,
\family typewriter
blkreplay
\family default
must later generate some
\emph on
fake data
\emph default
in order to repeat the timely and positionly behaviour of the original
recording.
By default, NULL blocks enriched with some internal header information
are generated.
The internal headers may be used for
\emph on
verification
\emph default
of the correctness of IO semantics, either by immediate re-read after each
write, or in a separate verify pass after the end of an ordinary
\family typewriter
blkreplay
\family default
run.
\end_layout
\begin_layout Standard
Notice that these NULL blocks will (together with the internal header informatio
n)
\series bold
destroy
\series default
any previous content (such as filesystem data) on your block device!
\end_layout
\begin_layout Standard
Therefore,
\emph on
never
\emph default
use
\family typewriter
blkreplay
\family default
on production systems.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresToxiques.png
lyxscale 50
scale 17
\end_inset
\emph on
Always
\emph default
use
\family typewriter
blkreplay
\family default
in the laboratory, always on devices which don't contain any valuable data!
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresToxiques.png
lyxscale 50
scale 17
\end_inset
Running
\family typewriter
blkreplay
\family default
\emph on
in parallel
\emph default
to mounted filesystems on the same device
\begin_inset Foot
status open
\begin_layout Plain Layout
Although a single Linux kernel instance tries to probihit such a disaster,
there are cases where you can
\begin_inset Quotes eld
\end_inset
achieve
\begin_inset Quotes erd
\end_inset
that effect.
Examples are iSCSI connections to the same iSCSI target in parallel.
\end_layout
\end_inset
will certainly destroy your data and almost certainly crash your kernel.
Always run at most a single
\family typewriter
blkreplay
\family default
instance on a single device!
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
In general,
\family typewriter
blkreplay
\family default
is a tool only for
\emph on
experienced
\emph default
technicians who know what they do.
They should be at least at a senior level.
\end_layout
\begin_layout Section
Architecture of
\family typewriter
blkreplay
\end_layout
\begin_layout Standard
The main challenge for
\family typewriter
blkreplay
\family default
is to generate
\series bold
sufficient IO parallelism
\series default
.
\end_layout
\begin_layout Standard
Ordinary production systems like web servers are serving many thousands
of HTTP requests per second, which may lead to an IO parallelism at the
block level of several hundred outstanding IO requests at the same time.
\end_layout
\begin_layout Standard
In order to simulate such a behaviour in the lab, there are principally
two alternatives:
\end_layout
\begin_layout Enumerate
use the
\family typewriter
aio
\family default
interface of the kernel to fire off a large number of IO requests in parallel.
\end_layout
\begin_layout Enumerate
use a sufficiently large number of kernel threads or processes in parallel,
where each of them fires up only at most one IO request at the same time
(blocking IO).
\end_layout
\begin_layout Standard
The current version of
\family typewriter
blkreplay
\family default
supports only method 2; method 1 is planned for a future release.
\end_layout
\begin_layout Standard
Method 2 is motivated by a typical Apache behaviour: it is almost a
\begin_inset Quotes eld
\end_inset
fork bomb
\begin_inset Quotes erd
\end_inset
, in particular under high connection rates and slow block devices.
A high number of ordinary Unix processes is doing conventional
\family typewriter
read()
\family default
/
\family typewriter
write()
\family default
operations in parallel.
\end_layout
\begin_layout Standard
For easy portability even to historic Unix flavours,
\family typewriter
blkreplay
\family default
uses ordinary Unix processes generated by simple
\family typewriter
fork()
\family default
s and communicating via anonymous pipes, in favour to a shared-memory
\family typewriter
pthreads
\family default
model.
However, future versions may also support
\family typewriter
pthreads
\family default
.
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename architecture.fig
width 75col%
\end_inset
\end_layout
\begin_layout Section
Mode of Operation
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
It is
\series bold
essential
\series default
that you understand the concepts described in this chapter.
Otherwise you may produce
\series bold
useless fake results
\series default
, deviating from valid measurements by
\emph on
factors
\emph default
, or even
\emph on
orders of magnitude
\emph default
.
\end_layout
\begin_layout Standard
It is crucial to understand the
\emph on
operating environment
\emph default
where
\family typewriter
blkreplay
\family default
is running.
Please take a look at the following picture:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename io-layers.fig
width 50text%
\end_inset
\end_layout
\begin_layout Standard
\noindent
In the live system,
\family typewriter
blktrace
\family default
will
\begin_inset Quotes eld
\end_inset
listen
\begin_inset Quotes erd
\end_inset
to the events occurring at point A, and will record them.
In this example, we have a complex system, running virtual machines inside
a hypervisor.
\end_layout
\begin_layout Standard
Notice that all IO requests of the application will not only go through
the filesystem, but also through the buffer / page cache.
The in-memory cache will usually serve most IO requests from the application,
without causing physical IO at the block layer (so-called
\begin_inset Quotes eld
\end_inset
cache hits
\begin_inset Quotes erd
\end_inset
).
\end_layout
\begin_layout Standard
On some well-tuned production servers, it is no problem to achieve cache
hit rates of 99% or more, leading to a kind of
\begin_inset Quotes eld
\end_inset
gear ratio
\begin_inset Quotes erd
\end_inset
of 1:100, or even 1:1000 (in long-term runs).
Of course, there also exist heavy workloads running on thin servers, where
sometimes less than 1:10 can be achieved.
Even in that case, there will be
\emph on
always
\emph default
some cache hits, for example caused by metadata requests from the filesystem.
In practice, the cache hit rate will never go down to 0%.
Notice that these inevitable cache effects are
\emph on
already included
\emph default
in any
\family typewriter
blktrace
\family default
!
\end_layout
\begin_layout Standard
Now look at the situation in the laboratory.
The application and the filesystem is no longer present, but its
\emph on
effects
\emph default
are simulated by
\family typewriter
blkreplay
\family default
.
Due to the architecture of the Linux kernel, all IO requests will continue
to run through the buffer cache
\begin_inset Foot
status open
\begin_layout Plain Layout
Several commercial Unices used a concept called
\begin_inset Quotes eld
\end_inset
raw device
\begin_inset Quotes erd
\end_inset
which circumvented their buffer cache.
In contrast, the internal structures of Linux device driver are internally
interwoven with the page cache in a rather sticky way.
Instead of
\begin_inset Quotes eld
\end_inset
raw devices
\begin_inset Quotes erd
\end_inset
, Linux uses the concept of
\begin_inset Quotes eld
\end_inset
direct IO
\begin_inset Quotes erd
\end_inset
, which
\emph on
tries
\emph default
to minimize any caching effects.
\end_layout
\end_inset
.
\end_layout
\begin_layout Standard
It should be clear by the very nature of our experiment, that at measuring
point B
\emph on
exactly
\emph default
the same events
\emph on
should
\emph default
happen as had been formerly observed / recorded at point A.
\end_layout
\begin_layout Standard
Thus, the page / buffer cache of the laboratory system
\emph on
must
\emph default
be switched off.
Otherwise, a
\begin_inset Quotes eld
\end_inset
gear ratio
\begin_inset Quotes erd
\end_inset
of 1:10 (or let it be only 1:1.1) would lead to distortions of the measurement
results.
In order to switch off the page / buffer cache,
\family typewriter
blkreplay
\family default
uses
\family typewriter
O_DIRECT
\family default
mode as offered by the Linux kernel.
\end_layout
\begin_layout Standard
Notice that even by these measures, there may remain some subtle differences
between the operations occurring at point A and point B.
The block layer of the Linux kernel does some optimizations, for example
it tries to
\emph on
coalesce
\emph default
adjacent requests, or to
\emph on
split
\emph default
some requests, depending on the capabilities of the hardware.
Usually, these are minor modifications, occurring at less than 1% of all
requests.
However, keep in mind (and check) these effects.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
The
\series bold
elevator strategy
\series default
(aka IO scheduler) of the Linux block layer is a bigger influence factor.
It can
\emph on
reorder
\emph default
requests, and it can even add
\emph on
artificial latencies
\emph default
.
For example,
\family typewriter
CFQ
\family default
adds some speculative latencies in some cases to increase the chances for
request mergers.
Selecting the
\begin_inset Quotes eld
\end_inset
wrong
\begin_inset Foot
status open
\begin_layout Plain Layout
Many sources from the internet claim that
\family typewriter
CFQ
\family default
is the
\begin_inset Quotes eld
\end_inset
best
\begin_inset Quotes erd
\end_inset
IO scheduler.
While this is often true for typical workstation load pattern (and interactive
user expectations), our experience at 1&1 is different with regards to
\emph on
server
\emph default
loads.
Check yourself! Run some
\family typewriter
blkreplay
\family default
comparisons between different IO schedulers.
Hint: there may be even non-linear dependencies from the lower level, whether
you have some RAID with BBU cache, or not.
\end_layout
\end_inset
\begin_inset Quotes erd
\end_inset
scheduler may lead to larger distortions, even to seemingly
\begin_inset Quotes eld
\end_inset
bad
\begin_inset Quotes erd
\end_inset
behaviour (which is
\emph on
not
\emph default
the fault of bad hardware).
In order to really get (almost) the same behaviour at points A and B, you
\emph on
must
\emph default
select the
\family typewriter
NOOP
\family default
or at least the
\family typewriter
DEADLINE
\family default
scheduler.
See
\family typewriter
/sys/dev/block/*/queue/scheduler
\family default
.
\end_layout
\begin_layout Section
Overlapping of IO Requests
\end_layout
\begin_layout Standard
In general, there are two kinds of overlapping between IO requests in real
production systems (at the time when a
\family typewriter
blktrace
\family default
record is made):
\end_layout
\begin_layout Enumerate
timely
\end_layout
\begin_layout Enumerate
positionly
\end_layout
\begin_layout Standard
Both kinds form a two-dimensional space:
\end_layout
\begin_layout Standard
\align center
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
overlapping
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
timely yes
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
timely no
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
positionly yes
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
-
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
positionly no
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\begin_inset Formula $\sqrt{}$
\end_inset
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Standard
\noindent
Pure timely overlapping (without positionly overlapping) is a frequent case
in almost any IO system (usually called
\begin_inset Quotes eld
\end_inset
IO parallelism
\begin_inset Quotes erd
\end_inset
in folklore).
In opposite, purely positionly overlapping (without timely overlapping)
is also a frequent case, for example when the same sector is re-read after
a while, or re-written when the contents of a file changes frequently.
Completely unrelated requests (neither timely nor positionly overlapping)
are probably the most frequent case in most practical load scenarios at
production sites.
There is no problem with any of them, indicated by the checkmarks.
\end_layout
\begin_layout Standard
However, what about both kinds of overlapping at the same time?
\end_layout
\begin_layout Standard
The case of
\emph on
both
\emph default
timely and positionly overlapping (simultanously) of IO operations is called
\emph on
damaged IO
\emph default
.
\end_layout
\begin_layout Standard
In ordinary OS kernels, damaged IO usually
\emph on
never
\emph default
occurs.
Here are the reasons:
\end_layout
\begin_layout Standard
IO requests are usually generated by in-kernel memory caches like buffer
caches or page caches.
Even in case of databases using Direct IO when submitting requests to the
block layer, their internal database buffer cache works similarly to in-kernel
caches.
It simply makes
\emph on
no sense
\emph default
to write to the same sector twice at the same time, because the result
will be undefined.
A similar argument holds for reads in parallel to writes to/from the same
sector.
\end_layout
\begin_layout Standard
In theory, concurrent reads from the same sector would be possible without
causing harm to data integrity on the block device.
In operating system caches, this would introduce
\emph on
copies
\emph default
of the same data into the buffer or page cache, violating its internal
\emph on
uniqueness
\emph default
properties stating that any sector is cached at most once.
Consequently, this case also never appears
\emph on
in practice
\emph default
at
\emph on
real-life systems(!)
\emph default
.
\end_layout
\begin_layout Standard
However, in terms of
\series bold
correctness of storage semantics
\series default
the case of damaged IO involving read/read to the same sector is
\emph on
allowed
\emph default
.
Probably you already know the following table from database textbooks and
others:
\end_layout
\begin_layout Standard
\align center
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
conflict?
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
read
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
write
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
read
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
no
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
write
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Standard
\noindent
Some block IO systems like DRBD show some misbehaviour in case of concurrent
writes to the same sector, or in some cases they even fail.
Some DRBD versions
\begin_inset Foot
status open
\begin_layout Plain Layout
At present, this seems to be an undocumented behaviour observed by the author.
Even if DRBD's behaviour may change in the future: damaged IO is a bad
idea by itself.
It would be unfair to blame DRBD for
\begin_inset Quotes eld
\end_inset
psychologically unexpected
\begin_inset Quotes erd
\end_inset
behaviour under illegal load patterns, which should never occur.
In general, making code rubust against damaged IO could decrease performance
during ordinary operation.
Thus damaged IO should be avoided at its
\emph on
source
\emph default
.
\end_layout
\end_inset
will at least delay further IO requests for several milliseconds, lowering
IO bandwidth or even leading to temporary hangs.
\end_layout
\begin_layout Standard
So, damaged IO
\emph on
should
\emph default
be avoided under all circumstances.
Failing to do so may result in a disaster; in general, some IO devices
like elder tape drives may even be corrupted as a whole.
\end_layout
\begin_layout Standard
While avoidance of damaged IO is automatically guaranteed in real production
systems by the buffer / cache page of the Linux kernel (or other components
like database memory buffers), our tool
\family typewriter
blkreplay
\family default
must be designed very carefully not to step into that pitfall.
\end_layout
\begin_layout Standard
Why is there a risk that
\family typewriter
blkreplay
\family default
could (accidentally) start some damaged IO?
\end_layout
\begin_layout Standard
Well, replay of the original timing of requests is not always possible.
Even if
\family typewriter
blkreplay
\family default
\emph on
tries
\emph default
to start IO requests in the same timely pattern as at the original site,
a very slow device (or a heavily overloaded device) may delay an IO operation
for a very long time.
In overload scenarios, or in case of iSCSI network hangs, it is possible
that some IO requests may take 5 minutes to complete (or even more, or
even
\emph on
never
\emph default
complete in case of fatal errors).
In such cases, it is not unlikely that a new write to the same sector is
started before the old one has completed.
\end_layout
\begin_layout Standard
In order to avoid damaged IO,
\family typewriter
blkreplay
\family default
uses some in-memory hash tables to detect any (potential) conflicts between
IO requests.
\end_layout
\begin_layout Standard
In order to deal with (potential) problems caused by damaged IO, we use
the following options in
\family typewriter
blkreplay
\family default
to control its behaviour at
\emph on
replay
\emph default
time:
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--strong=0
\family default
For the sake of conflict detection, only write/write conflicts will count.
This mode ensures that the
\begin_inset Quotes eld
\end_inset
storage semantics
\begin_inset Quotes erd
\end_inset
is obeyed with respect to the
\emph on
written data
\emph default
\begin_inset Foot
status open
\begin_layout Plain Layout
This allows to check the end result via
\family typewriter
verify
\family default
modes for correctness.
\end_layout
\end_inset
, but it allows reads to permute with writes (which is
\begin_inset Quotes eld
\end_inset
wrong
\begin_inset Quotes erd
\end_inset
in
\emph on
strong
\emph default
sense, hence the name).
In essence, the following conflict table is used internally for conflict
detection:
\begin_inset Newline newline
\end_inset
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset space \hfill{}
\end_inset
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
conflict?
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
read
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
write
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
read
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
no
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
no
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
write
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
no
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--strong=1
\family default
(default) Use the standard conflict table as known from the literature.
Only read/read is treated as non-conflicting:
\begin_inset Newline newline
\end_inset
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset space \hfill{}
\end_inset
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
conflict?
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
read
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
write
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
read
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
no
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
write
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--strong=2
\family default
\emph on
All
\emph default
damaged IO is treated as conflicting, even read/read.
This may be useful for simulating the behaviour of
\emph on
real
\emph default
OS caches.
This is equivalent to the following table:
\begin_inset Newline newline
\end_inset
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset space \hfill{}
\end_inset
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
conflict?
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
read
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
write
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
read
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
write
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Standard
Indepently from the correctness criterion, the following operation modes
may be selected.
They determine the kind of reaction in case of detected conflicts:
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--with-conflicts
\family default
No countermeasures against damaged IO are taken.
Consequently, is does not matter which
\family typewriter
--strong=
\family default
mode you have selected before.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--with-drop
\family default
Whenever a new request is conflicting with an old (already issued) request,
it is simply dropped.
This has no side effects, other than that some reads and/or writes may
be missing (depending on
\family typewriter
--strong=
\family default
mode).
This has the lowest overhead of all conflict-avoiding methods.
Depending on properties of the load, the number of dropped requests may
be rather high.
Please check the tail of the result file.
In the statistics section, you will find the number of dropped requests.
If that number is higher than, say, 5%, you should consider one of the
following options.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--with-partial
\family default
(default) Any conflicting requests (as determined by
\family typewriter
--strong=
\family default
) are pushed back to an internal pushback list and kept there until the
conflict is gone.
Pushed back requests are immediately submitted as soon as the conflict
has gone away.
This results in a reordering of the
\emph on
affected
\emph default
requests, while
\emph on
trying
\emph default
to replay unaffected ones at original timestamps.
This leads to a partial ordering of requests, which may be
\emph on
very different
\emph default
from the original ordering, and thus may
\begin_inset Quotes eld
\end_inset
violate
\begin_inset Quotes erd
\end_inset
the
\begin_inset Quotes eld
\end_inset
original storage semantics
\begin_inset Quotes erd
\end_inset
if it would make a difference on the replay system.
\begin_inset Newline newline
\end_inset
This mode leads to a 2-class society, where ordinary requests are processed
faster than conflicting ones.
Further details may be found in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Request"
\end_inset
.
\begin_inset Newline newline
\end_inset
Attention! as a side effect, this mode may
\emph on
increase
\emph default
the actual IO parallelism to a larger number than configured via the
\family typewriter
--threads=
\family default
parameter, because some request slots (and in turn, also some threads)
must
\begin_inset Foot
status open
\begin_layout Plain Layout
Otherwise a reactivated request from the pushback list could have to wait
for an ordinary request slot to become free, which would result in an artificia
l delay.
Experiments have show that such kinds of distortions can be serious.
\end_layout
\end_inset
be reserved
\emph on
in advance
\emph default
for pushed-back requests.
When conflicts are gone and the system tries to
\begin_inset Quotes eld
\end_inset
catch up
\begin_inset Quotes erd
\end_inset
, additional IO requests may be submitted.
\begin_inset Newline newline
\end_inset
In practice, this can happen in particular with some Windows loads, where
a lot of writes seem to be repeated.
Frequently, pushed back requests (colored differently for better distinction
from ordinary requests) are the main contributors to delays.
Check them!
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--with-ordering
\family default
The main process spreads its IO requests to the worker processes in a round-rob
in fashion over their anonymous pipes.
Whenever conflicting requests (as determined by
\family typewriter
--strong=
\family default
) are detected, this spreading process will stop until the conflict has
gone away.
In consequence, the original ordering of requests will be preserved as
much as possible.
As a side effect,
\emph on
all following
\emph default
IO requests are also delayed, even if they don't conflict with anything
else.
This can lead to
\emph on
artificial delays
\emph default
.
\begin_inset Newline newline
\end_inset
In general, this mode is the
\begin_inset Quotes eld
\end_inset
most robust
\begin_inset Quotes erd
\end_inset
one.
\end_layout
\begin_layout Standard
In many cases, you will prefer
\family typewriter
--with-partial
\family default
, which is the default.
It delivers almost the same throughput as
\family typewriter
--with-conflicts
\family default
without the downsides of
\family typewriter
--with-drop
\family default
, while minimizing artificial delays
\begin_inset Foot
status open
\begin_layout Plain Layout
In extreme cases, pushed-back requests (visualized in stronger colors) can
form some kind of
\begin_inset Quotes eld
\end_inset
transitive queues
\begin_inset Quotes erd
\end_inset
, e.g.
when pushback requests depend on other pushback requests transitively.
In such a case, their delays can sum up over a longer time, independently
from
\begin_inset Quotes eld
\end_inset
normal
\begin_inset Quotes erd
\end_inset
requests (clearly visible in the delay diagram).
Although this is often a property of the load, you can
\emph on
try(!)
\emph default
to minimize such an effect by setting the expert option
\family typewriter
ahead_limit
\family default
to values lower than 1s (however too low values will destroy throughput
at all).
Dangerous!
\end_layout
\end_inset
.
\end_layout
\begin_layout Standard
However,
\family typewriter
--with-ordering
\family default
is also useful in many scenarios.
Practical experience from many hardware tests shows that the artificial
delays caused by
\family typewriter
--with-ordering
\family default
seem to be an
\emph on
advantage
\emph default
.
Whenever such stalls occur more than seldomly, they act as an
\series bold
indicator
\series default
for massive IO problems of the test candidate.
\family typewriter
--with-ordering
\family default
is often a kind of
\begin_inset Quotes eld
\end_inset
detector
\begin_inset Quotes erd
\end_inset
for hardware problems, since it visualizes any problems in an eye-catching
way.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
When you start a lot of threads (typically
\begin_inset Formula $\geq256$
\end_inset
),
\family typewriter
--with-ordering
\family default
may yield
\emph on
better(!)
\emph default
throughput than
\family typewriter
--with-partial
\family default
.
In such cases, the reason is
\emph on
counter-intuitive
\emph default
, as explained in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Number"
\end_inset
: artificial delays caused by
\family typewriter
--with-ordering
\family default
will
\emph on
decrease
\emph default
the
\emph on
average
\emph default
request queue length actually occuring at runtime, which will in turn
\emph on
increase
\emph default
the average throughput, depending on counter-intuitive properties of your
test candidate.
A way to find out is just to run the same benchmarks with
\family typewriter
--threads=16
\family default
or
\family typewriter
--threads=32
\family default
.
In case
\family typewriter
--with-partial
\family default
is now better than
\family typewriter
--with-ordering
\family default
, you likely detected that problem.
Don't draw wrong conclusions from such counter-intuitive effects!
\end_layout
\begin_layout Section
Verification of Storage Semantics
\end_layout
\begin_layout Standard
In order to allow verification of the sector headers and their timestamps
/ version stamps,
\family typewriter
blkreplay
\family default
needs some temporary storage where information can be kept for a longer
time than just during IO.
Two temporary files are used:
\family typewriter
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/tmp/verify_table
\end_layout
\end_inset
\family default
and
\family typewriter
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/tmp/completion_table
\end_layout
\end_inset
\family default
.
Both are sparse files, containing a simple (sparse) array of sequence numbers,
indexed by the position (sector#).
Whenever a write is started, a new sequence number is recorded in
\family typewriter
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/tmp/verify_table
\end_layout
\end_inset
\family default
.
Whenever that write is completed, the same number is recorded in
\family typewriter
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/tmp/completion_table
\end_layout
\end_inset
\family default
.
At any time, both tables keep track of the current progress of write operations.
\end_layout
\begin_layout Standard
Whenever the sequence numbers at the same position (sector#) are different
between both files, we know that a write operation has not yet completed.
If they are the same, we know which sequence number should appear in the
header of the corresponding sector.
\end_layout
\begin_layout Standard
There are following variants of verification modes:
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--with-verify
\family default
Whenever a sector is read (by a regular read request) which has been written
before (by an ordinary write request), the read data is checked against
the sequence numbers from the tables.
Any mismatches are reported by the string
\family typewriter
VERIFY ERROR
\family default
in the result file.
Thus,
\family typewriter
zgrep
\begin_inset Quotes eld
\end_inset
VERIFY ERROR
\begin_inset Quotes erd
\end_inset
*.replay.gz
\family default
will show them to you.
In addition, the statistics section at the end of the output file will
contain some valuable information.
\begin_inset Newline newline
\end_inset
Warning! this mode can only reveal errors in the storage semantics if written
blocks are re-read somewhen.
When sectors are just written, but never read, this mode will not detect
anything.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--with-final-verify
\family default
In addition to
\family typewriter
--with-verify
\family default
, a separate pass will be started at the end of a
\family typewriter
blkreplay
\family default
run.
All sectors which have been touched before, will be checked.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--with-paranoia
\family default
In addition, any written sector will be immediately re-read during operation.
This doubles the IO rate and leads to extremely high distortions of measurement
results.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
All verify modes will create temporary tables in
\family typewriter
/tmp/
\family default
.
Although the temporary files are sparse, they can use up a significant
amount of storage (depending on the load).
The additional IOs form a
\emph on
sequential bottleneck
\emph default
, and therefore can slowdown
\family typewriter
blkreplay
\family default
considerably.
Please use the verify modes only for
\emph on
validation
\emph default
, but not for measurements / determining performance!
\end_layout
\begin_layout Chapter
How to use
\family typewriter
blkreplay
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresToxiques.png
lyxscale 50
scale 17
\end_inset
Running
\family typewriter
blkreplay
\family default
naïvely without reading this chapter may easily lead to
\series bold
completely worthless fake results
\series default
which would be only useful for production of bullshit!
\end_layout
\begin_layout Standard
In science, it would be unethical to produce such bullshit willingly or
even deliberately.
\end_layout
\begin_layout Standard
In industry, usage of such bullshit (even inadvertendly) may easily lead
to failed invests up to millions of Euros / Dollars (depending on the applicati
on and the size of your datacenter).
\end_layout
\begin_layout Standard
Most of the following advice will also apply to other benchmark tools like
\family typewriter
iometer
\family default
.
\end_layout
\begin_layout Section
How to Avoid Common Pitfalls
\begin_inset CommandInset label
LatexCommand label
name "sec:How-to-Avoid"
\end_inset
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresToxiques.png
lyxscale 50
scale 17
\end_inset
Don't skip this section! Read it
\emph on
completely
\emph default
, even if you are impatient or under time pressure!
\end_layout
\begin_layout Standard
Modern IO subsystems often use some kind of
\series bold
storage virtualization
\series default
interally.
More often than you can dream of, concepts from storage virtualization
are used (interally) in places where you don't expect them.
\end_layout
\begin_layout Standard
Example: seemingly
\begin_inset Quotes eld
\end_inset
simple
\begin_inset Quotes erd
\end_inset
SSDs or even some USB sticks(!) show some of the behaviours described next.
\end_layout
\begin_layout Subsection
Pitfalls from Storage Virtualization
\begin_inset CommandInset label
LatexCommand label
name "sub:Pitfalls-from-Storage"
\end_inset
\end_layout
\begin_layout Standard
The basic idea of storage virtualization is some kind of
\begin_inset Quotes eld
\end_inset
translation
\begin_inset Quotes erd
\end_inset
(or
\begin_inset Quotes eld
\end_inset
mapping
\begin_inset Quotes erd
\end_inset
) between
\emph on
virtual storage addresses
\emph default
and
\emph on
\begin_inset Quotes eld
\end_inset
physical
\begin_inset Quotes erd
\end_inset
storage addresses
\emph default
.
\end_layout
\begin_layout Standard
In many
\begin_inset Foot
status open
\begin_layout Plain Layout
A promient exception is classical LVM as implemented by the Linux kernel:
unless you use LVM snapshots or other advanced features, it uses almost
static mappings, and it carries almost no observable overhead in many practical
scenarios.
\end_layout
\end_inset
cases, the address translation / mapping is created on the fly, dynamically
at runtime.
In the following simplified
\begin_inset Foot
status open
\begin_layout Plain Layout
For simplicity, this example assumes that the address translation uses the
same granularity as the benchmark, e.g.
single sectors.
We also don't discuss the internals of the mapping which can also have
a drastic impact on measurement results.
\end_layout
\end_inset
example, the timely order of accesses is marked by increasing numbers,
while the type of IO request is indicated by colors (red = write, blue
= read):
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename storage-virtualization.fig
lyxscale 70
width 80col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
We start with an empty logical address space (often called
\begin_inset Quotes eld
\end_inset
logical volume
\begin_inset Quotes erd
\end_inset
, shortly LV), having a logical size of several terabytes.
Here,
\begin_inset Quotes eld
\end_inset
empty
\begin_inset Quotes erd
\end_inset
means that initially
\emph on
no
\emph default
assignment between logical and physical sector numbers exists.
Now we start a short benchmark (whether
\family typewriter
blkreplay
\family default
or others like
\family typewriter
iometer
\family default
).
When request #1 (a write) arrives, no physical location exists yet.
Therefore, a new location must be assigned.
In this example, the locations are always taken from the beginning of the
physical address space.
\end_layout
\begin_layout Standard
Thus, all the physical sectors occurring in this example will be allocated
in a very small and compact area at the start of the physical address space.
Imagine the drawing not being true to scale: imagine a total size of several
terabytes (
\begin_inset Formula $>10^{9}$
\end_inset
sectors), and a
\family typewriter
blkreplay
\family default
benchmark touching only a few thousands of sectors.
What will be the effect?
\end_layout
\begin_layout Enumerate
Only a
\emph on
tiny
\emph default
fraction of the physical space will be actually used, usually less than
one per mille or even less than a millionth.
\begin_inset Newline newline
\end_inset
In contrast, real world applications as well as real customers tend to use
up significant space with real data, usually more than 50% (and up to 100%).
\end_layout
\begin_layout Enumerate
Even if accesses to the LV are (pseudo-)random, accesses to the
\begin_inset Quotes eld
\end_inset
physical hot area
\begin_inset Quotes erd
\end_inset
will
\emph on
not
\emph default
remain random: as you can see, they are translated to (purely / almost)
\emph on
sequential
\emph default
access patterns.
If the physical addresses are residing on a mechanical hard disk, (almost)
no seek operations will occur.
Additionally, the physical operations are in
\emph on
ascending
\emph default
order, which is a classical use case for BBU-cached writes and/or readahead
strategies.
Notice that ascending sequential IO on hard disks is usually faster than
random IO by a factor of 100 or even more (depending on hardware and further
factors like RAID, between one and three orders of magnitude).
\begin_inset Newline newline
\end_inset
In contrast, real-world writes will be spreaded much more randomly over
the physical partition, and there will be a significant amount of
\emph on
in-place updates
\emph default
in many practical use cases.
\end_layout
\begin_layout Enumerate
Even worse, read requests need not be mapped at all (indicated by shading
in the drawing).
When reading from an address where never anything had been written before,
NULL blocks may be returned on-the-fly, without causing any physical access
at all
\begin_inset Foot
status open
\begin_layout Plain Layout
A similar effect is known from holes in traditional Unix
\emph on
sparse files
\emph default
.
\end_layout
\end_inset
.
Notice that such
\begin_inset Quotes eld
\end_inset
fake reads
\begin_inset Quotes erd
\end_inset
can be faster than true read accesses by
\emph on
several
\emph default
orders of magnitude.
\end_layout
\begin_layout Enumerate
Even in case read requests are also mapped upon first reference
\begin_inset Foot
status open
\begin_layout Plain Layout
Several commercial storage boxes are known to do so.
However, notice that this behaviour is not documented, and thus not guaranteed
by the vendors.
They sell you a blackbox.
All you can do is to analyze such behaviour if you are curious about their
internals.
In their next firmware release, the behaviour may be already different
without notice.
\end_layout
\end_inset
, perhaps leading to a physical IO (or perhaps not), the same arguments
as for writes apply.
\end_layout
\begin_layout Enumerate
If you
\emph on
repeat
\emph default
the same
\family typewriter
blkreplay
\family default
benchmark once again, immediately after the first run, you will get another
surprise: this time the mapping between logical and physical addresses
already exists, thus you will likely get different results, usually drastically
better, but seldomly slightly worse, depending on the vendor of the storage
virtualization (and on many other factors such as the size of the logical
volume).
In scientific terminology: your experiment is not truly
\emph on
repeatable
\emph default
.
\end_layout
\begin_layout Standard
What can you do about that?
\end_layout
\begin_layout Standard
There is no general solution for all cases.
It depends on the
\emph on
statement
\emph default
you want to prove or disprove by usage of
\family typewriter
blkreplay
\family default
.
\end_layout
\begin_layout Standard
The following is just an
\emph on
approximation
\emph default
if you want to reveal the real-life
\begin_inset Foot
status open
\begin_layout Plain Layout
In real life, customer data or enterprise data is stored on LVs.
Thus benchmarks of empty LVs are
\emph on
completely wrong
\emph default
if you try to reveal real life behaviour.
\end_layout
\end_inset
behaviour of virtualized storage systems:
\end_layout
\begin_layout Enumerate
Whenever you start a new run of a benchmark, you
\series bold
\emph on
must
\emph default
delete
\series default
your old LVs, and create new ones.
Otherwise, your old run will influence the new one in some way you cannot
predict easily.
Remember that the mapping table in the above example is a kind of
\begin_inset Quotes eld
\end_inset
memory
\begin_inset Quotes erd
\end_inset
which records not only the sector numbers occurring in your benchmark,
but even their timely order.
Make sure that this kind of
\begin_inset Quotes eld
\end_inset
memory
\begin_inset Quotes erd
\end_inset
is erased completely
\begin_inset Foot
status open
\begin_layout Plain Layout
Even delete your LVs if you believe that's unnecessary, because you have
obeyed point 2 and have filled it with data to initialize the mapping:
some storage systems make
\emph on
re-assignments
\emph default
of the mapping during your benchmarks.
Because many commercial storage systems are blackboxes, you cannot immediately
see that.
Always keep in mind that usually ordinary benchmarks will only touch a
\emph on
tiny fraction
\emph default
of all physical sectors, compared to real life!
\end_layout
\end_inset
between any runs!
\end_layout
\begin_layout Enumerate
After each fresh creation of a logical volume,
\series bold
fill it with data
\series default
.
This is the only reliable
\begin_inset Foot
status open
\begin_layout Plain Layout
Some storage vendors have internal functions which
\emph on
preallocate
\emph default
the space for a LV.
Don't use them! Don't trust any claims that this would be equivalent to
filling with random data - we found cases where we could
\emph on
disprove
\emph default
such claims, where results differed even by
\emph on
factors
\emph default
.
Just fill your LVs with random data to be sure, even if this delays your
measurements for some hours or even days.
Keep in mind that later production systems will take weeks or even months
to be filled with data before potential problems could show up, so don't
hesitate to resemble such kind of effort in the laboratory.
\end_layout
\end_inset
way.
Best practice is to use tools like
\family typewriter
wipe
\family default
, filling the whole
\begin_inset Foot
status open
\begin_layout Plain Layout
If you are consciously concerned that filling the
\emph on
whole
\emph default
LV might not catch your usecase where (say) only 50% of logical space is
actually used, you
\emph on
could
\emph default
try to fill only 50% of the LV.
However, be
\emph on
sure
\emph default
to fill any blocks which occur in the benchmarks.
Otherwise, the effects described above will almost certainly lead to a
\emph on
higher
\emph default
distortion of your results than just filling with 100%.
Notice that some of the above effects deal with orders of magnitude, not
just a few percent.
\end_layout
\end_inset
LV with random data.
Filling with NULL blocks is not recommended, because some blackbox storage
systems might detect this easily and circumvent it by not creating a mapping
at all (or even erasing an old mapping similar to
\family typewriter
punch
\family default
operations).
\end_layout
\begin_layout Enumerate
\emph on
Immediately
\emph default
after filling with random data, start your benchmark
\series bold
exactly once
\series default
.
\emph on
Never
\emph default
, really
\emph on
never
\emph default
kill a run of
\family typewriter
blkreplay
\family default
(or any other benchmark tool) and restart it.
In case of any error or misbehaviour, you
\emph on
must
\emph default
start over with step 1!
\end_layout
\begin_layout Enumerate
There is a single exception if you really know what you are doing: immediately
after the first run, you may restart the
\emph on
same
\emph default
benchmark once again, in order to reveal some hidden properties of the
mapping.
You
\emph on
must
\emph default
name your output files differently from the first run, and you
\emph on
must not
\emph default
confuse the meaning of the second run with the meaning of the first run.
\end_layout
\begin_layout Subsection
Pitfalls from Caches
\begin_inset CommandInset label
LatexCommand label
name "sub:Pitfalls-from-Caches"
\end_inset
\end_layout
\begin_layout Standard
It is easy to be caught by these pitfalls (even if you try to avoid them
very hard), since caches occur very frequently in almost any type of storage
system, and even in places where you don't expect them.
In addition, some real-life loads have hidden properties you cannot see
at first glance.
\end_layout
\begin_layout Subsubsection
Pitfalls from Cache Operation States
\end_layout
\begin_layout Standard
Many people believe that the most important case is
\emph on
cold caches
\emph default
versus
\emph on
hot caches
\emph default
.
Although this is not completely wrong, it is not always fully true.
There is another more important property of cache states:
\series bold
steady state
\series default
.
\end_layout
\begin_layout Standard
Steady state is not the same as hot state.
When you start your system freshly, many caches will be of course empty
\begin_inset Foot
status open
\begin_layout Plain Layout
There are some exceptions: SSD caches may start in a hot state (caused by
your previous benchmark run), and BBU caches at RAID controllers may also
survive even power failure.
\end_layout
\end_inset
.
An empty cache is always
\begin_inset Quotes eld
\end_inset
cold
\begin_inset Quotes erd
\end_inset
.
During operation, it will be filled with data.
A cache is called
\begin_inset Quotes eld
\end_inset
hot
\begin_inset Quotes erd
\end_inset
, if the
\series bold
cache hit rate
\series default
is
\begin_inset Quotes eld
\end_inset
significantly high enough
\begin_inset Quotes erd
\end_inset
.
What's that? All of these terms are rather vague and depend on the application.
However, some caches may
\emph on
never
\emph default
reach
\begin_inset Quotes eld
\end_inset
hot
\begin_inset Quotes erd
\end_inset
state, for example when the cache design / architecture is
\begin_inset Quotes eld
\end_inset
inefficient
\begin_inset Quotes erd
\end_inset
(cf section
\begin_inset CommandInset ref
LatexCommand ref
reference "A-cache-is"
\end_inset
).
What then? The term
\begin_inset Quotes eld
\end_inset
hot
\begin_inset Quotes erd
\end_inset
is not the right one for describing that problem: when your cache never
gets hots, your testing candidate will just fail your performance test,
but your test as such will be valid: the result is just telling you that
the cache is not tuned well enough for your application workload.
\end_layout
\begin_layout Standard
What is
\begin_inset Quotes eld
\end_inset
steady state
\begin_inset Quotes erd
\end_inset
?
\end_layout
\begin_layout Standard
Intuitively, it just means that
\begin_inset Quotes eld
\end_inset
nothing changes (fundamentally) any more
\begin_inset Quotes erd
\end_inset
.
In practice, there is a simple rule of thumb: your benchmark should just
run
\series bold
long enough
\series default
to get into steady state.
In section
\begin_inset CommandInset ref
LatexCommand ref
reference "A-cache-is"
\end_inset
, some theoretical methods are described how to compute the time
\begin_inset Formula $\delta$
\end_inset
until steady state is reached.
In practice, just make a few experiments in order to determine steady state
intuitively.
Many people will get a good feeling for
\begin_inset Quotes eld
\end_inset
steady state
\begin_inset Quotes erd
\end_inset
after some experience.
\end_layout
\begin_layout Standard
Once you know when you have reached the
\begin_inset Quotes eld
\end_inset
golden
\begin_inset Quotes erd
\end_inset
steady state, you can do one of two things:
\end_layout
\begin_layout Enumerate
ignore all benchmark results from the time span before steady state has
been reached.
\end_layout
\begin_layout Enumerate
let the whole benchmark run
\emph on
at least
\emph default
10x as long as the time to reach steady state.
The longer, the better.
\end_layout
\begin_layout Subsubsection
Pitfalls from Cache Size
\end_layout
\begin_layout Standard
The following illustration explains a general problem of cache hierarchies.
Our example demonstrates an ill-designed behaviour: the lower cache is
\emph on
smaller
\emph default
than the upper one.
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename cache-toosmall.fig
lyxscale 50
width 75col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Now assume that the LV has a size of several terabytes, and the workload
is operating on a large part of that.
On one hand, this is several orders of magnitude larger than the upper
cache, but on the other hand typical application workloads will not access
all sectors with the same uniform probability.
In general, caches are only useful in two cases (non-exclusively):
\end_layout
\begin_layout Enumerate
the cache is strictly larger than the workload.
\end_layout
\begin_layout Enumerate
the workload contains unevenly distributed access frequencies.
\end_layout
\begin_layout Standard
Case 1 occurs only in special cases, such as in-memory databases (or workstation
loads on large RAM machines).
Case 2 appears more often.
\end_layout
\begin_layout Standard
Back to the above example: assume that only case 2 applies to the upper
cache.
Consequently, case 2 will also apply to the lower cache, because the lower
cache is
\emph on
strictly smaller
\emph default
than the upper cache in this ill-designed example.
Now assume we have an
\emph on
inclusive
\emph default
\begin_inset Foot
status open
\begin_layout Plain Layout
In case of
\emph on
exclusive
\emph default
cache hierarchies, the whole picture can be
\emph on
approximately(!)
\emph default
replaced by a simplified one having only one cache level.
The size of the new simplified cache is just the
\emph on
sum
\emph default
of the sizes of both original caches.
\end_layout
\end_inset
cache hierarchy, and are using a standard cache eviction strategy like
LRU.
As a consequence from LRU (or any other strategy avoiding anomalies
\begin_inset Foot
status open
\begin_layout Plain Layout
Probably the best known anomaly is the famous FIFO anomaly, as explained
in most text books about operating systems.
\end_layout
\end_inset
), any sector present in the lower cache will be also be present in the
upper cache
\begin_inset Foot
status open
\begin_layout Plain Layout
This is just the
\emph on
definition
\emph default
of non-anomaly behaviour.
\end_layout
\end_inset
.
As a further consequence, we see that the lower cache is superfluous: by
removing it, the system could even become faster
\begin_inset Foot
status open
\begin_layout Plain Layout
Notice that there are exceptions.
For example, internal memory caches present at hard disk drives are way
too small to be able to contribute to classical hierarchical caching, but
they can act as cylinder buffers for
\emph on
local aggregation strategies
\emph default
like readahead.
\end_layout
\end_inset
due to less overhead.
\end_layout
\begin_layout Standard
What can we do about that?
\end_layout
\begin_layout Standard
Simply, just design your system in such a way that lower caches are always
strictly larger than higher ones, by a factor of
\begin_inset Formula $k$
\end_inset
.
In order to be useful,
\begin_inset Formula $k\geq2$
\end_inset
should be used, but for really good performance
\begin_inset Formula $k\geq10$
\end_inset
should be selected.
\end_layout
\begin_layout Standard
Probably you already know this, and you think you don't violate it.
However, it is possible you might violate it unwillingly.
The standard case is a server from a data center, equipped with several
gigabytes of RAM.
Almost all of the main memory can be used by the buffer/page caches of
the Linux kernel.
Therefore, you already may have a rather large upper cache you didn't think
about.
As a consequence, caches at the block storage level (e.g.
SSD caches etc) should be larger by an order of magnitude (in this case
\begin_inset Formula $\sim1$
\end_inset
TB or more).
At the time of writing this paper, some commercial storage systems don't
match this seemingly simple requirement.
\end_layout
\begin_layout Standard
There is another variant of this pitfall: records made by
\family typewriter
blktrace
\family default
are measuring the IO traffic
\emph on
below
\emph default
the buffer/page cache in most cases.
Therefore, most (if not all) natural loads obtained by
\family typewriter
blktrace
\family default
contain effects
\begin_inset Foot
status open
\begin_layout Plain Layout
One of the more well-known effects of caches is called
\emph on
cache inversion
\emph default
.
At the time of writing this paper, Wikipedia didn't carry much about it.
Consult a really good textbook or some research papers to learn more about
it.
\end_layout
\end_inset
of the caches of the original system.
In some cases (e.g.
published loads from the
\family typewriter
blktrace
\family default
project), you don't know the original RAM size.
Even if you know, you often cannot tell how large the page cache
\emph on
really
\emph default
was at the time of recording: how much RAM was spent for other purposes
like processes, how much for other filesystems / partitions?
\end_layout
\begin_layout Standard
Even if you knew all that: do you know the
\emph on
workingset size
\emph default
of your application workload? Read on\SpecialChar \ldots{}
\end_layout
\begin_layout Subsection
Pitfalls from Workingset Sizes
\begin_inset CommandInset label
LatexCommand label
name "sub:Pitfalls-from-Workingset"
\end_inset
\end_layout
\begin_layout Standard
The workingset theory has been developped by Denning in the late 1960s,
and has been originally used for the description of the behaviour of hardware-M
MU-based paging / swapping systems and their strategies like LRU.
It is also useful in other areas, such as block storage.
Here is an adaptation of Denning's theory to our needs:
\end_layout
\begin_layout Standard
\begin_inset Formula
\[
WS(t,\delta)=\big\{\text{{set\,\ of\,\ all\,\ sectors\,\ touched\,\ in\,\ the\,\ time\,\ interval\,[t-\delta,t]}}\big\}
\]
\end_inset
\end_layout
\begin_layout Standard
\noindent
Usually,
\begin_inset Formula $\delta$
\end_inset
is treated as a constant, called
\emph on
window size
\emph default
.
Then the workingset
\begin_inset Formula $WS(t,\delta)$
\end_inset
at some point in time
\begin_inset Formula $t$
\end_inset
is simply the set of sectors occurring in a
\family typewriter
blktrace
\family default
during a time window of
\begin_inset Formula $\delta$
\end_inset
seconds
\emph on
before
\emph default
\begin_inset Foot
status open
\begin_layout Plain Layout
Notice: Dennings original theory used the time interval
\begin_inset Formula $[t,t+\delta].$
\end_inset
We find our definition more handy for practical purposes, because we need
no
\begin_inset Quotes eld
\end_inset
lookahead
\begin_inset Quotes erd
\end_inset
into the
\begin_inset Quotes eld
\end_inset
future
\begin_inset Quotes erd
\end_inset
.
\end_layout
\end_inset
the interesting point in time
\begin_inset Formula $t$
\end_inset
.
Notice that we have a
\emph on
set
\emph default
here: it makes no difference how often a particular sector occurs during
the time window
\begin_inset Formula $\delta,$
\end_inset
it just matters
\emph on
whether
\emph default
that sector appears or not.
Also, it makes no difference whether a particular sector is read or written
(or both).
Thus the workingset model is a
\emph on
reduction
\emph default
of the reality to a handy theoretical model, but a model which is known
to
\emph on
preserve
\emph default
some relevant and very interesting properties of the reality.
\end_layout
\begin_layout Description
Example: for any two window sizes
\begin_inset Formula $\delta_{1}<\delta_{2},$
\end_inset
the condition
\begin_inset Formula $WS(t,\delta_{1})\leq WS(t,\delta_{2})$
\end_inset
holds at any point in time
\begin_inset Formula $t$
\end_inset
.
In other words: increasing the window size
\begin_inset Formula $\delta$
\end_inset
will never make the workingset smaller; the workingset can only
\emph on
grow
\emph default
if the window becomes larger.
\end_layout
\begin_layout Standard
The efficiency of block storage caches can be predicted by the workingset
theory in a rather easy and intuitive way.
We assume that accesses to the cache are
\emph on
much faster
\emph default
than accesses to the background storage, such that we can
\emph on
neglect
\emph default
the access times to the cache when compared to accesses to the background
storage.
Then we can model the following interesting property:
\end_layout
\begin_layout Quote
A cache is called to be designed
\series bold
efficiently
\series default
\begin_inset CommandInset label
LatexCommand label
name "A-cache-is"
\end_inset
, iff at any point in time
\begin_inset Formula $t$
\end_inset
the following condition holds:
\begin_inset Newline newline
\end_inset
\begin_inset Formula $|WS(t,\delta)|\leq|\text{{cachesize}}|$
\end_inset
for some
\begin_inset Formula $\delta\geq(\text{{time\,\ to\,\ fill\,\ the\,\ cache\,\ once}})$
\end_inset
.
\end_layout
\begin_layout Standard
The potential painpoint is easy to see: just take
\begin_inset Formula $\delta$
\end_inset
as the time to fill the cache from the background storage, which is
\begin_inset Formula $(\text{{accesstime\,\ to\,\ storage}})\star(\text{{cachesize}})$
\end_inset
.
If there would exist a point in time
\begin_inset Formula $t$
\end_inset
where the workingset
\begin_inset Formula $WS(t,\delta)$
\end_inset
would be
\emph on
larger
\emph default
than could be transferred to/from the storage during that
\emph on
same
\emph default
time window
\begin_inset Formula $\delta$
\end_inset
, the data transfers could become a
\emph on
bottleneck
\emph default
of the system.
Vice versa: if the workingset during window
\begin_inset Formula $\delta$
\end_inset
would always
\emph on
fit
\emph default
into the cache, the cache will usually
\begin_inset Foot
status open
\begin_layout Plain Layout
Of course, this holds only if the workload contains some
\emph on
repetitions
\emph default
during the window
\begin_inset Formula $\delta$
\end_inset
, and if the cache employs some
\begin_inset Quotes eld
\end_inset
good
\begin_inset Quotes erd
\end_inset
replacement strategy like LRU.
The latter assures that
\begin_inset Quotes eld
\end_inset
hot
\begin_inset Quotes erd
\end_inset
pages from the workingset will be kept in the cache.
In addition, keep in mind that neglecting the access times to the cache
could be an
\emph on
oversimplification
\emph default
in many cases.
\end_layout
\end_inset
speed up things.
\end_layout
\begin_layout Standard
Consequences:
\end_layout
\begin_layout Itemize
The
\series bold
workingset behaviour of the application
\series default
is
\emph on
crucial
\emph default
for any storage system.
\end_layout
\begin_layout Itemize
The above argumentation contains an
\emph on
oversimplification
\emph default
: of course, accesses to the cache are not
\begin_inset Quotes eld
\end_inset
indefinitly fast
\begin_inset Quotes erd
\end_inset
as our above neglect assumed.
Therefore, don't take the above inequalities as verbatim inequalities.
Multiply some
\emph on
factor
\emph default
onto them.
In practice, if you really want blastingly fast caches, make sure your
cache is at least
\series bold
one order of magnitude larger
\series default
than the workingset size of your application.
\end_layout
\begin_layout Standard
In order to do that, you would need the know the workingset size of your
\emph on
application
\emph default
.
Please keep in mind that this is not the same as the workingset size measured
by
\family typewriter
blktrace
\family default
:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename hierarchy.fig
lyxscale 60
width 30col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
As you can see, the measuring point of
\family typewriter
blktrace
\family default
sits
\emph on
below
\emph default
the buffer / page cache, therefore it does not
\emph on
directly
\emph default
measure the application behaviour (in addition to influences from the filesyste
m, like metadata updates etc).
In practice,
\family typewriter
blktrace
\family default
measurements may differ from the
\begin_inset Quotes eld
\end_inset
real
\begin_inset Quotes erd
\end_inset
application workload of stateless webserver designs by
\emph on
several
\emph default
orders of magnitude.
However,
\emph on
all
\emph default
modern OSes employ caches.
Even if we were able to measure the true application workingset in some
way, it would not be relevant for block storage systems, because it would
be
\emph on
impractical
\emph default
to operate those systems without caches.
We need the above picture for understanding the fundamental properties
of
\family typewriter
blktrace
\family default
measurements, and for determining the window size
\begin_inset Formula $\delta$
\end_inset
.
If that is not possible, try to estimate it.
As a very rough estimation, take
\begin_inset Formula $\delta$
\end_inset
as several minutes.
\end_layout
\begin_layout Standard
The
\family typewriter
blkreplay
\family default
suite comes with a small tool to visualize the workingset behaviour as
measured by
\family typewriter
blktrace
\family default
, which is currently the best approximation of the workingset behaviour
of the application we can easily get access to.
Following is an example, showing different window sizes
\begin_inset Formula $\delta$
\end_inset
in the same picture (where 000 means cumulation to
\begin_inset Formula $\infty$
\end_inset
):
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename images/example1.g30.ws_log.png
lyxscale 30
width 100col%
\end_inset
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
Spend some time on it! Your replay needs to last
\emph on
vastly longer
\emph default
than the
\begin_inset Formula $\delta_{\text{{old}}}$
\end_inset
needed to descibe the steady state of the original buffer/page cache, as
well as the
\begin_inset Formula $\delta_{\text{{new}}}$
\end_inset
to describe the steady state of your replay system.
Otherwise, you can get fake results which differ from real practical performanc
e by factors, or even orders of magnitude.
As a rough rule of thumb, any replay of natural loads should take at least
one hour.
Better, make a few measurements lasting 8 hours, or even 24 hours, and
check whether the results differ more than expected (besides natural variations
).
\end_layout
\begin_layout Subsection
Pitfalls from Replay Device Sizes and Others
\begin_inset CommandInset label
LatexCommand label
name "sub:Pitfalls-from-Replay"
\end_inset
\end_layout
\begin_layout Standard
There is a simple intuitive rule: your replay device for
\family typewriter
blkreplay
\family default
should have (almost) the
\emph on
same
\emph default
size as the original device where
\family typewriter
blktrace
\family default
has taken its measurement from
\begin_inset Foot
status open
\begin_layout Plain Layout
If you don't know this, just make a test run with
\family typewriter
blkreplay
\family default
and check the output file.
At the end, you will find a human-readable statistics showing the highest
original block number occurring in the replay, as well as some other interestin
g numbers.
\end_layout
\end_inset
.
\end_layout
\begin_layout Standard
For easy checking, our output graphics will display the so-called
\series bold
wraparound_factor
\series default
in the lower right corner: it shows the ratio between the max block# occuring
in the load, and the size of the replay device.
Ideally, it should be near
\begin_inset Formula $1.0$
\end_inset
.
When it deviates from this by more than a factor of two (or less than one
halve), it is printed in purple color to warn you.
\end_layout
\begin_layout Standard
Some people don't take this seriously, and some don't even believe that
this can have a
\emph on
tremendous
\emph default
effect.
\end_layout
\begin_layout Standard
Practical experience at 1&1 tells that the above rule is
\emph on
valid
\emph default
, and that results
\emph on
will
\emph default
almost certainly vary.
The bias can be
\emph on
considerable
\emph default
.
\end_layout
\begin_layout Standard
Example: the original
\family typewriter
blktrace
\family default
recording was taken from a production server equipped with 20 TB RAID.
Since in the lab we had only a smaller system with 4 TB,
\family typewriter
blktrace
\family default
measurements were run despite the smaller size.
Whenever
\family typewriter
blkreplay
\family default
tries to start an IO request outside the LV size, it just
\emph on
remaps
\emph default
the sector number modulo the (new) LV size.
Therefore, results
\emph on
appear
\emph default
to be valid, since you cannot see any
\begin_inset Quotes eld
\end_inset
big
\begin_inset Quotes erd
\end_inset
holes or anomalies.
You will find out the difference only if you compare to the
\emph on
correct
\emph default
setup.
When repeating the
\emph on
same
\emph default
measurements with correct LV sizes of ~20 TB, there is a
\emph on
significant
\emph default
difference.
\end_layout
\begin_layout Standard
Some people think that such differences can be attributed solely to the
natural differences in
\emph on
spindle count
\emph default
\begin_inset Foot
status open
\begin_layout Plain Layout
When using the same hard disk model, a 20 TB RAID must contain 5 times as
much spindles as an equivalent 4 TB system.
It is clear that additional spindles can benefit random IO.
But those effects are
\series bold
non-linear
\series default
!
\end_layout
\end_inset
, and therefore it would not hurt if different models of hard disks were
used.
Although there
\emph on
are
\emph default
some effects by spindle count, that opinion is wrong.
In order to disprove such a
\begin_inset Quotes eld
\end_inset
theory
\begin_inset Quotes erd
\end_inset
, just build two different RAIDs with same spindle count, but fundamentally
\begin_inset Foot
status open
\begin_layout Plain Layout
Of course, there are a lot of disk drives (in the same
\begin_inset Quotes eld
\end_inset
class
\begin_inset Quotes erd
\end_inset
) showing only minor differences.
A
\begin_inset Quotes eld
\end_inset
fundamental
\begin_inset Quotes erd
\end_inset
difference is for example between a cheap SATA disk versus a smaller but
faster 15k SAS disk.
\end_layout
\end_inset
different disk drive models (resulting in different total capacity), and
compare (under otherwise equal conditions).
\end_layout
\begin_layout Standard
In short, any of the following factors can influence the performance, independen
tly from each other (and in no particular order):
\end_layout
\begin_layout Itemize
Total capacity, just by itself.
If you don't believe it, just create LVs with 1/10 size of the physical
storage and compare (on the
\emph on
same
\emph default
hardware) with results from the
\emph on
full
\emph default
physical storage size.
Of course, the original recording must stem from sufficiently large devices,
otherwise your
\begin_inset Quotes eld
\end_inset
disproof
\begin_inset Quotes erd
\end_inset
will be false.
\end_layout
\begin_layout Itemize
Model/class of disk drives.
\end_layout
\begin_layout Itemize
Number of spindles.
\end_layout
\begin_layout Itemize
RAID level.
\end_layout
\begin_layout Itemize
Vendor / firmware version of the controller.
\end_layout
\begin_layout Itemize
Interconnect technology, such as SATA vs SAS.
\end_layout
\begin_layout Itemize
any caches in the hierarchy, such as different BBU cache sizes or different
parameters.
\end_layout
\begin_layout Standard
...
and probably many others.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
General rule:
\emph on
never
\emph default
expect any of these effects to be linear.
Almost always, they are
\series bold
non-linear
\series default
\begin_inset Foot
status open
\begin_layout Plain Layout
Non-linear effects cannot be combined with each other in a predictable way,
at least in general.
\end_layout
\end_inset
.
Anyone claiming something else must
\series bold
prove
\series default
it!
\end_layout
\begin_layout Standard
Consequence: anyone violating the above rule produces
\series bold
invalid
\series default
results, unless proven the opposite!
\end_layout
\begin_layout Subsection
Pitfalls from Parallelism in IO Systems
\end_layout
\begin_layout Subsubsection
Pitfalls from
\emph on
missing
\emph default
Parallelism
\end_layout
\begin_layout Standard
This pitfall is usually trapping less people, because some intuitive sense
for the effects of IO parallelism is more widespread.
Even if you are already aware of this pitfall, read on.
There are some subtleties.
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename consolidation.fig
lyxscale 50
width 30text%
\end_inset
\end_layout
\begin_layout Standard
\noindent
A frequent use case is
\emph on
storage consolidation
\emph default
.
Up to
\begin_inset Formula $k$
\end_inset
CPU nodes are connected to some
\begin_inset Quotes eld
\end_inset
central
\begin_inset Quotes erd
\end_inset
storage via some kind of storage network.
\end_layout
\begin_layout Standard
You may want to evaluate such an architecture in advance with help of
\family typewriter
blkreplay
\family default
, in order to avoid failed invests.
The
\family typewriter
blkreplay
\family default
suite will help you, because its supervisor scripts like
\family typewriter
tree-replay.sh
\family default
are ready to run masses of
\family typewriter
blkreplay
\family default
instances in parallel to each other, and on different nodes.
\end_layout
\begin_layout Standard
For obvious reasons, you should determine the optimum consolidation factor
\begin_inset Formula $k$
\end_inset
for each given hardware candidate.
If you vary the factor
\begin_inset Formula $k$
\end_inset
, you will get at least two effects:
\end_layout
\begin_layout Enumerate
With higher
\begin_inset Formula $k$
\end_inset
, you need to build up more CPU nodes and more network connections.
Otherwise you will neglect effects from IO parallelism (which don't scale
linearly
\begin_inset Foot
status open
\begin_layout Plain Layout
There is no general way to predict the behaviour of an unknown system! Many
non-linear effects show some kind of
\begin_inset Quotes eld
\end_inset
binary
\begin_inset Quotes erd
\end_inset
behaviour, suddenly collapsing at some point where you didn't expect it.
\end_layout
\end_inset
), at multiple places of the picture: at potential bottlenecks at the CPU
nodes, at potential bottlenecks inside your storage network, and at lots
of potential bottlenecks in your central storage system.
\begin_inset Newline newline
\end_inset
For example, it is completely wrong to just double the load on
\begin_inset Formula $k/2$
\end_inset
nodes.
Experiences with some test candidates at 1&1 show that results can differ
enormously from results from single loads, each on
\begin_inset Formula $k$
\end_inset
nodes.
\end_layout
\begin_layout Enumerate
The total capacity of the central storage will also vary with
\begin_inset Formula $k$
\end_inset
.
As known from section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Replay"
\end_inset
, this will not only influence results by itself, but in addition by further
effects like spindle count, non-scaling of caches with factor
\begin_inset Formula $k$
\end_inset
, and many others.
\end_layout
\begin_layout Subsubsection
Pitfalls from
\emph on
too high
\emph default
IO Parallelism
\end_layout
\begin_layout Standard
You can tune the number of
\family typewriter
blkreplay
\family default
threads via the parameter
\family typewriter
threads=\SpecialChar \ldots{}
\family default
(see appendix
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Config-File-Parameters"
\end_inset
).
As a side effect, this will also influence the maximum number of outstanding
IO requests which can be
\begin_inset Quotes eld
\end_inset
on the fly
\begin_inset Quotes erd
\end_inset
in parallel, together with the replay parallelism (number of
\family typewriter
blkreplay
\family default
instances running in parallel on the same physical system).
\end_layout
\begin_layout Standard
Many people believe that an increasing number of outstanding IO requests
will improve overall throughput.
\end_layout
\begin_layout Standard
However, some devices / drivers / IO schedulers may respond in some
\emph on
counter-productive
\emph default
way when hammered with too many IO requests in parallel.
Sometimes, your throughput can even
\emph on
collapse
\emph default
to less than 1/10 of the maximum.
There are many possible reasons for this unexpected behaviour.
\end_layout
\begin_layout Standard
In most cases, you will notice collapsing effects only during
\series bold
overload situations
\series default
.
During normal operation, you will not notice anything, because the device
can
\emph on
catch up
\emph default
with the IO demands.
There are almost no queues, because the
\emph on
average service time
\emph default
is shorter than the average request rate.
\end_layout
\begin_layout Standard
As soon as the device gets overloaded, the situation will change rapidly.
Masses of
\family typewriter
blkreplay
\family default
threads are trying to fire off their requests in time, starting to overlap
now because the service time increases.
Suddenly, some queuing will take place, somewhere.
In mathematical theory, the queue lengths could even grow indefinitely
whenever the average request rate is higher than the average service time.
In practice, there will be some limits somewhere.
\end_layout
\begin_layout Standard
Imagine you are going shopping in a supermarket.
When the cashier girl cannot catch up with the demand from the customers,
their shopping trolleys will start to form a queue in front of her.
Now assume the following behaviour: the longer the queue, the
\emph on
slower
\emph default
she will work.
Imagine that! After a while, customer satisfaction will go down to zero,
because the queue will get longer and longer.
And the longer, the slower she will work.
And so on.
There is no escape, other than stopping to buy anything from there.
\end_layout
\begin_layout Standard
A similar behaviour can be observed in some IO systems.
Once a queue has formed, there is almost no escape from a behaviour similar
to
\series bold
traffic jam
\series default
.
If you cannot change the system, your only chance is to reduce load, or
even to remove the load at all.
\end_layout
\begin_layout Standard
Sometimes it is even hard to trigger such behaviour.
A systems appears to run smoothly for months, but suddenly it collapses.
\end_layout
\begin_layout Standard
If you have such a system (or cannot be sure to have one
\begin_inset Foot
status open
\begin_layout Plain Layout
To find out, we recommend the artifical loads
\family typewriter
*bursts*.load.gz
\family default
for creating anal kinds of overload.
\end_layout
\end_inset
) and want to simulate its behaviour in the laboratory, you should be carefully
tuning the
\family typewriter
threads
\family default
parameter.
In reality, the number of outstanding requests is often limited in some
way.
For example, a typical workstation load caused by a single user has often
some intrinsically limited IO parallelism, similar to a limited number
of shopping trolleys in the supermarket.
On the other hand, some server loads (such as those caused by Apache) can
\family typewriter
fork()
\family default
off a high number of threads.
\end_layout
\begin_layout Standard
So it is quite possible to observe some traffic jam behaviour in practice,
depending on your application.
It is quite possible that such behaviour is
\series bold
relevant
\series default
for IT operations, at least for
\emph on
some
\emph default
applications (but not for all).
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
filename images/MatieresToxiques.png
lyxscale 50
scale 17
\end_inset
Taking any of these
\series bold
non-linear
\series default
(and sometimes even
\emph on
binary collapsing
\emph default
) effects not seriously and/or using a wrong number of threads / replay
parallelism can easily lead to
\series bold
invalid results
\series default
, and in turn to failed invests!
\end_layout
\begin_layout Standard
Hint: a frequent case are distributed systems by
\emph on
themselves
\emph default
.
They tend to produce
\series bold
queues
\series default
at times and in places where you don't expect them, and they tend to produce
avalanche-like negative effects (self-amplifying), similar to suddenly
appearing
\series bold
traffic jams
\series default
where you cannot determine one single reason in isolation.
\end_layout
\begin_layout Section
Recommended Setup and Usage
\end_layout
\begin_layout Subsection
Planning Phase
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresToxiques.png
lyxscale 50
scale 17
\end_inset
Never try to plan a project without deep knowledge of the pitfalls described
in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:How-to-Avoid"
\end_inset
.
In addition, some experience with
\family typewriter
blkreplay
\family default
is helpful.
In order to gain such experience, consider a
\emph on
test project
\emph default
just for playing around, and for getting familiar with the pitfalls.
\end_layout
\begin_layout Subsubsection
Describe the Scope of Project
\end_layout
\begin_layout Standard
Before starting, you should get conscious with yourself.
\emph on
What exactly
\emph default
is the question you want to answer with help of
\family typewriter
blkreplay
\family default
?
\end_layout
\begin_layout Standard
Write down the question both as shortly, as well as precisely as possible.
Here are some examples:
\end_layout
\begin_layout Itemize
Compare hardware vendor
\family typewriter
A
\family default
with
\family typewriter
B
\family default
and
\family typewriter
C
\family default
for my production workload X.
\end_layout
\begin_layout Itemize
Compare hardware vendor
\family typewriter
A
\family default
with
\family typewriter
B
\family default
and
\family typewriter
C
\family default
in general.
\end_layout
\begin_layout Itemize
Debug kernel module
\family typewriter
xxx
\family default
.
\end_layout
\begin_layout Itemize
Compare iSCSI with Fibrechannel for my production workload X.
\end_layout
\begin_layout Itemize
\SpecialChar \ldots{}
\end_layout
\begin_layout Standard
Next, write down a precise description of your intended
\series bold
test environment
\series default
.
Best practice is to name hardware vendors, models, all components (including
intermediate gear like network switches), and so on.
\end_layout
\begin_layout Standard
Last step: describe all the parameters which know of, which
\emph on
could
\emph default
have an influence onto your test results.
Example:
\end_layout
\begin_layout Standard
\noindent
\align center
\size small
\begin_inset Tabular
|
\begin_inset Text
\begin_layout Plain Layout
\size small
Parameter
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\size small
Varying?
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
\size small
16 GB RAM in storage node
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\size small
no
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
\size small
10GBit vs 1 GBit network speed
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\size small
yes
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
\size small
Cheap 2TB SATA vs expensive 600GB raptor disk (model
\begin_inset Quotes eld
\end_inset
tyrannosaurus rex
\begin_inset Quotes erd
\end_inset
)
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\size small
yes
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
RAID-Level
\begin_inset Formula $\in\{1,5,6\}$
\end_inset
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
RAID Stripesize
\begin_inset Formula $\in\{16,32,64,128\}$
\end_inset
kB
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
yes
\end_layout
\end_inset
|
|
\begin_inset Text
\begin_layout Plain Layout
\size small
\SpecialChar \ldots{}
\end_layout
\end_inset
|
\begin_inset Text
\begin_layout Plain Layout
\size small
\SpecialChar \ldots{}
\end_layout
\end_inset
|
\end_inset
\end_layout
\begin_layout Subsubsection
Describe the Setup of your Experiment
\end_layout
\begin_layout Standard
In many cases, the parameters described in your table will make up a multi-dimen
sional problem space (cartesian product) which is too large to be explored
exhaustively.
As explained in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Replay"
\end_inset
, many of them will influence your results
\emph on
non-linearly
\emph default
.
\end_layout
\begin_layout Standard
Thus, you will have to consider the following general strategies:
\end_layout
\begin_layout Itemize
You may fix some of the parameters to particular values.
Although this saves time, you may miss an opportunity to find an optimal
solution.
\end_layout
\begin_layout Itemize
You may select some/enough random samples from your multi-dimensional problem
space and try them randomly (Monte Carlo methods).
\end_layout
\begin_layout Itemize
Stepwise refinement: explore the multidimensional space by varying exactly
\emph on
one
\emph default
paramenter at once.
This is slow, but you can be sure of the effects caused by this.
\end_layout
\begin_layout Subsubsection
Select
\family typewriter
blkreplay
\family default
Load
\begin_inset CommandInset label
LatexCommand label
name "sub:Select-blkreplay-Load"
\end_inset
\end_layout
\begin_layout Standard
Depending on your project, you should consider both artificial and natural
loads, but not too many of them.
Usually, more than three loads are impractical for an ambitious project
(unless you want to compare masses of loads on the same reference hardware).
\end_layout
\begin_layout Standard
If your project tries to answer a question for some specific workload X,
you should just record that workload if you can do so (see chapter
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:How-to-use"
\end_inset
).
\end_layout
\begin_layout Standard
If you cannot obtain your real workload in advance, you have to select one
from the
\family typewriter
blkreplay
\family default
project (or other sources) which comes as close as possible to your (intended)
natural workload.
\end_layout
\begin_layout Standard
For the sake of static comparison of workloads,
\family typewriter
cd
\family default
to a directory containing your
\family typewriter
*.load.gz
\family default
files and issue the following command:
\end_layout
\begin_layout Quotation
\family typewriter
/path/to/graph.sh --static myname.load.gz
\end_layout
\begin_layout Standard
This will produce some
\family typewriter
.png
\family default
graphics, describing the throughput and workingset behaviour of your load
(cf section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Workingset"
\end_inset
).
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
You need enough space in
\family typewriter
/tmp/
\family default
(or in another
\family typewriter
$TMPDIR
\family default
) for temporary intermediate files.
If your
\family typewriter
*.load.gz
\family default
file is very large (several hours or even days), you may need several gigabytes.
Please don't interrupt
\family typewriter
graph.sh
\family default
as it spawns lots of subprocesses and creates lots of temporary files.
Currently, there are no checks for free space in
\family typewriter
/tmp/
\family default
, so running out of space may produce wrong results
\emph on
silently
\emph default
.
As a countermeasure, run
\family typewriter
watch df /tmp/
\family default
in a separate window during your run of
\family typewriter
graph.sh
\family default
.
\end_layout
\begin_layout Standard
Hint: some
\family typewriter
blktrace
\family default
recordings (but not all) contain some timing information about the original
IO latencies as measured at the original site.
Use
\family typewriter
/path/to/graph.sh --dynamic myname.load.gz
\family default
to create some additional graphics about them.
\end_layout
\begin_layout Subsubsection
Selection of Replay Duration
\end_layout
\begin_layout Standard
This is a hairy problem, as already described in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Workingset"
\end_inset
.
Often, you cannot run 24h replays for many hundred times.
\end_layout
\begin_layout Standard
If you want to be sure that a particular load will run even under
\emph on
worst-case
\emph default
conditions, you should definitly select some appropriate time window around
\emph on
load peaks
\emph default
, measured both in throughput as well as in workingset size.
\end_layout
\begin_layout Standard
In addition, you can try the following strategies:
\end_layout
\begin_layout Itemize
For explorative phases in your project, such as determining the optimum
in your parameter space, you can try to minimize running times as much
as possible.
But not too much.
Otherwise you will be caught by the pitfalls described in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:How-to-Avoid"
\end_inset
.
\end_layout
\begin_layout Itemize
For final verification of your results, you should repeat benchmarks with
a longer window (at least 8h or 24h).
\end_layout
\begin_layout Subsubsection
Total Project Time
\end_layout
\begin_layout Standard
Working with huge parameter spaces is not all you have to consider.
Setup of different RAID levels, re-initialization after changes of stripe
sizes, filling LVs with random data, etc, may take a very long time, in
addition to the benchmark themselves.
Don't forget that! Your only chance are nights and weekends, if you manage
to run something unattendedly.
But predictions are sometimes wrong.
In addition, something may fail and then needs to be restarted.
Calculate some spare time for that!
\end_layout
\begin_layout Standard
If there is a high time pressure in the project timeline, you probably will
have to
\emph on
rework
\emph default
some parts of your project plan.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
Planning is crucial! When you find any discrepancies, try to re-think your
plan as a whole, not just some parts of it.
\end_layout
\begin_layout Subsection
Setup Phase
\end_layout
\begin_layout Subsubsection
Lab setup
\end_layout
\begin_layout Standard
Ensure that all your hard- and software components are ready in the lab
and operational.
\end_layout
\begin_layout Standard
In addition, you need some workstation (or server) where the
\family typewriter
blkreplay
\family default
suite is checked out.
Do the following steps:
\end_layout
\begin_layout Itemize
Ensure that
\family typewriter
gcc
\family default
,
\family typewriter
make
\family default
,
\family typewriter
gnuplot
\family default
, and some standard tools like
\family typewriter
grep
\family default
/
\family typewriter
gawk
\family default
are installed.
If you need multiple machine architectures (such as
\family typewriter
x86_64
\family default
and
\family typewriter
i386
\family default
) in parallel, ensure that
\family typewriter
gcc
\family default
can cross-compile via flags
\family typewriter
-m32
\family default
/
\family typewriter
-m64
\family default
and that the appropriate libs are installed.
\end_layout
\begin_layout Itemize
\family typewriter
git clone https://github.com/schoebel/blkreplay
\end_layout
\begin_layout Itemize
\family typewriter
cd blkreplay
\end_layout
\begin_layout Itemize
\family typewriter
./configure
\end_layout
\begin_layout Itemize
\family typewriter
make
\end_layout
\begin_layout Itemize
Attention!
\family typewriter
make install
\family default
is not yet supported.
Just leave everything in place.
You can either put
\family typewriter
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/path/to/blkreplay/scripts/
\end_layout
\end_inset
\family default
into your
\family typewriter
$PATH
\family default
, or call the scripts via hard path.
\end_layout
\begin_layout Standard
Ensure that all your test machines are reachable as
\family typewriter
root
\family default
via
\family typewriter
ssh
\family default
from your central workplace, without need for any password prompt.
In order to achieve that, you should consider
\family typewriter
ssh-agent
\family default
, in addition so some tweaking of
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/etc/ssh/ssh_config
\end_layout
\end_inset
(and probably
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/etc/ssh/sshd_config
\end_layout
\end_inset
on each of the target hosts).
\end_layout
\begin_layout Standard
On your workstation, you should have enough disk space to store your results.
Create a subdirectory there for your project.
Copy
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/path/to/blkreplay/example-run/default-main.conf
\end_layout
\end_inset
(and possibly other
\family typewriter
*.conf
\family default
files) to that new subdirectory, and finally
\family typewriter
cd
\family default
to it.
For the rest of your life, you will be working there ;)
\end_layout
\begin_layout Standard
You can now either call
\family typewriter
/path/to/blkreplay/scripts/
\emph on
something
\emph default
.sh
\family default
as hard paths as indicated in the following examples, or you may put
\family typewriter
/path/to/blkreplay/scripts/
\family default
into your
\family typewriter
$PATH
\family default
.
\end_layout
\begin_layout Standard
Customization of
\family typewriter
default-main.conf
\family default
is described in the following.
\end_layout
\begin_layout Subsubsection
Configuration Files
\end_layout
\begin_layout Standard
You should edit
\family typewriter
default-main.conf
\family default
to reflect the default setup for your project.
If you want to run multiple variants of your default setup, you can do
so by creating additional files like
\family typewriter
\emph on
something
\emph default
.conf
\family default
as well as a subdirectory
\family typewriter
\emph on
something
\emph default
/
\family default
(having the same name without suffix
\family typewriter
.conf
\family default
).
When you later start your benchmark, the values from
\family typewriter
\emph on
something
\emph default
.conf
\family default
will override those from
\family typewriter
default-main.conf
\family default
.
It is highly recommended to override only
\emph on
one
\emph default
parameter inside
\family typewriter
\emph on
something
\emph default
.conf
\family default
, otherwise it may become difficult to reveal the real impact of changed
parameters onto your test candidate.
\end_layout
\begin_layout Standard
In general, you may override
\emph on
any
\emph default
parameter from
\family typewriter
default-main.conf
\family default
, even hostnames, or input files
\family typewriter
*.load.gz
\family default
, or whatever.
\end_layout
\begin_layout Subsubsection
Meaning of the Config File Parameters
\end_layout
\begin_layout Standard
The meaning of the parameters is documented in the following places:
\end_layout
\begin_layout Enumerate
Comments inside
\family typewriter
default-main.conf
\family default
should provide enough information for experienced administrators, at least
for a quick start, and should guide you through the most basic steps.
\end_layout
\begin_layout Enumerate
The same information is available in appendix
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:Config-File-Parameters"
\end_inset
.
\end_layout
\begin_layout Enumerate
Last but not least: read the sources, if you are in doubt about anything.
\end_layout
\begin_layout Subsection
Benchmark Phase
\end_layout
\begin_layout Standard
The basic idea is simple: after customization of
\family typewriter
default-main.conf
\family default
(and probably other
\family typewriter
default-*.conf
\family default
files when using additional modules), you create a new subdirectory for
each benchmark run.
\end_layout
\begin_layout Standard
Whenever you call
\family typewriter
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/path/to/blkreplay/scripts/tree-replay.sh
\end_layout
\end_inset
\family default
(without parameters), a whole bundle of benchmarks will be started, one
for each
\emph on
leaf
\emph default
\begin_inset Foot
status open
\begin_layout Plain Layout
A leaf has no further subtree inside it.
\end_layout
\end_inset
subdirectory (starting from
\family typewriter
cwd
\family default
), provided that for each (intermedidate) subdirectory name
\family typewriter
\emph on
xxx
\family default
\emph default
there exists some
\family typewriter
\emph on
xxx
\emph default
.conf
\family default
in the current working directory or in one of its parents.
In the whole subdirectory structure, any directory ending with
\family typewriter
.old
\family default
, or including the substring
\family typewriter
ignore
\family default
, or containg a file
\family typewriter
skip
\family default
will be ignored.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
Example: you have created a subdirectory
\family typewriter
./short/
\family default
as well as two nested subdirectories
\family typewriter
./short/model1/
\family default
and
\family typewriter
./short/model2/
\family default
.
You further have prepared the config files
\family typewriter
short.conf
\family default
,
\family typewriter
model1.conf
\family default
and
\family typewriter
model2.conf
\family default
, existing in the current working directory or in some parent directory
down the
\family typewriter
../
\family default
chain.
Then exactly
\emph on
two
\emph default
benchmark runs will be started, namely in
\family typewriter
./short/model1/
\family default
and in
\family typewriter
./short/model2/
\family default
.
The benchmark running in
\family typewriter
./short/model1/
\family default
will include the following
\family typewriter
*.conf
\family default
files, in the following order:
\family typewriter
default-main.conf
\family default
,
\family typewriter
short.conf
\family default
, and finally
\family typewriter
model1.conf
\family default
.
Each of the specialized config files may override any previous setting,
but it is highly recommended to change only one parameter at a time and
to use short but expressive names.
Notice: the intermediate directory
\family typewriter
./short/
\family default
is no leaf (since it contains some subdirectories), therefore no benchmark
will be started inside it.
Later, you just need to create
\family typewriter
./long/
\family default
as well as
\family typewriter
./long/model1/
\family default
and
\family typewriter
./long/model2/
\family default
and some
\family typewriter
long.conf
\family default
in order to repeat the same benchmarks with a longer
\family typewriter
replay_duration
\family default
setting.
\end_layout
\begin_layout Standard
Hint: using
\begin_inset Quotes eld
\end_inset
intuitive
\begin_inset Quotes erd
\end_inset
names like
\family typewriter
short
\family default
and
\family typewriter
long
\family default
bears some danger.
A few years later, you will not remember what they exactly have meant.
Looking into
\family typewriter
*.conf
\family default
will not help other people, for example if you publish your benchmark results
somewhere.
Therefore it may be wise to use
\begin_inset Quotes eld
\end_inset
speaking
\begin_inset Quotes erd
\end_inset
names like
\family typewriter
duration_600
\family default
, at least if you have more than two variants.
On the other hand,
\begin_inset Quotes eld
\end_inset
intuitive
\begin_inset Quotes erd
\end_inset
names are better for presentation to some less-deeply involved audience.
Take some time for creating well-designed names for
\family typewriter
*.conf
\family default
and your directory hierarchy! Changing that names later is cumbersome.
Better to design your names in advance in a systematic (but simple) way.
\end_layout
\begin_layout Standard
On large investigation projects, deeply nested structures may be necessary,
involving different loads, different hardware, different hardware setup,
etc.
Not all of them are currently automated.
You can use the generic module mechanism to extend the default scripts
with further functionality, to push automation further.
\end_layout
\begin_layout Standard
However, not all setup tasks
\emph on
can
\emph default
be automated at all.
Some of them like forcing physical RAID degradation must be started by
physically removing a disk, which cannot be automated (other than buying
extremely expensive robots).
Therefore, you may include some human-readable dialogs inside your
\family typewriter
*.conf
\family default
files (in shell script syntax), or in some new modules you have written.
In any case, it is advisable to write some script code to
\emph on
check some preconditions
\emph default
(such as RAID status) in order to prevent wrong measurements.
\end_layout
\begin_layout Standard
In general,
\family typewriter
tree-replay.sh
\family default
will never repeat any benchmark which has already completed (i.e.
there exists an output file
\family typewriter
*.replay.gz
\family default
in that leaf directory).
This allows an incremental style of working.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
Continued
\begin_inset space ~
\end_inset
example: normally, you can call
\family typewriter
tree-replay.sh
\family default
again, but nothing happens, because all leaf directories already contain
some results.
However, then you find a problem with the results in
\family typewriter
./short/model1/
\family default
, so you want to repeat that benchmark without deleting your first (questionable
) results.
Now you create a subdirectory
\family typewriter
./short/model1/try1.old/
\family default
and move your results there.
As said above, any directory names ending with
\family typewriter
.old
\family default
or containing the substring
\family typewriter
ignore
\family default
are ignored, so the directory
\family typewriter
./short/model1/
\family default
will continue to count as a leaf (despite its newer subdirectory, which
is just ignored).
Since the
\family typewriter
*.replay.gz
\family default
files are now missing in
\family typewriter
./short/model1/
\family default
due to the
\family typewriter
mv
\family default
,
\family typewriter
tree-replay.sh
\family default
will repeat that single benchmark there.
\end_layout
\begin_layout Standard
Here are some useful hints:
\end_layout
\begin_layout Itemize
You may skip any directory by creating a file
\family typewriter
skip
\family default
inside it.
\family typewriter
touch ./short/model1/skip
\family default
will disable that directory.
Later, you can remove that file in order to fire off that benchmark.
\end_layout
\begin_layout Itemize
\family typewriter
skip
\family default
files are also working in intermediate directories like
\family typewriter
./short/
\family default
, disabling the whole subtree in one step.
\end_layout
\begin_layout Itemize
Design your
\family typewriter
*.conf
\family default
files such that arbitrary combinations are
\emph on
possible
\emph default
(cartesian product).
In contrast, your directory hierarchy need not (and, in many cases,
\emph on
will not
\emph default
) exploit the
\emph on
full
\emph default
cartesian product.
\end_layout
\begin_layout Itemize
You may create a new leaf directory (somewhere in the subtree) even in parallel
to an already running benchmark.
Whenever the currently running benchmark has completed,
\family typewriter
tree-replay.sh
\family default
will re-scan the subdirectory structure, find any freshly created leaf
directories, and determine which benchmark to start next.
All leaf directory names which have not yet completed are sorted alphabetically
, and the first name according to ASCII sort order is taken first.
\end_layout
\begin_layout Itemize
When 10 or more variants could appear somewhere (even after a while), use
leading zeros in any names like
\family typewriter
v001
\family default
,
\family typewriter
v002
\family default
etc to ensure that ASCII sort order is the same as numerical order.
\end_layout
\begin_layout Itemize
Sometimes the ASCII sort order of names like
\family typewriter
short
\family default
vs
\family typewriter
long
\family default
is boring, because you want to run the
\family typewriter
short
\family default
benchmarks first.
As a workaround in larger projects, add some numerical prefixes like in
\family typewriter
01_short
\family default
vs
\family typewriter
09_long
\family default
(leaving some numerical space such that you can later add
\family typewriter
05_medium
\family default
).
As a side effect, this also improves the ASCII sort order of your later
\family typewriter
*.png
\family default
graphics.
\end_layout
\begin_layout Itemize
You can rearrange the order in another way: just create an empty file
\family typewriter
prio-nnn
\family default
in a leaf directory, where
\family typewriter
nnn
\family default
is a number denoting a priority class.
Priority classes are overriding the global ASCII sort order.
Directories having the lowest priority class are run first, while directories
without any class are run last.
Inside of each class, the ASCII sort order is obeyed.
\end_layout
\begin_layout Itemize
When you design your
\family typewriter
*.conf
\family default
files systematically as a cartesian product, in theory it makes no difference
whether you permute some directory components (e.g
\family typewriter
./model1/short/
\family default
instead of
\family typewriter
./short/model1/
\family default
).
However, in practice it influences the ASCII sort order (taking the
\emph on
full
\emph default
path) and therefore the order in which your benchmarks are run.
\end_layout
\begin_layout Itemize
When some benchmark fails, just delete the corresponding output files.
On the next cycle,
\family typewriter
tree-replay.sh
\family default
will detect the missing files and just restart that benchmark (possibly
among others).
\end_layout
\begin_layout Itemize
\family typewriter
\emph on
xxx
\emph default
.conf
\family default
files can even reside in some parent directory of the current working directory.
This way, you need not copy your
\family typewriter
.conf
\family default
files inside a complex directory hierarchy (even spanning multiple projects).
However, only the
\family typewriter
\emph on
xxx
\emph default
.conf
\family default
files corresponding to directories reachable from the current working directory
will be included.
Example: if you go to a leaf of your subtree and start
\family typewriter
tree-replay.sh
\family default
there, no
\family typewriter
*.conf
\family default
file other than
\family typewriter
default-*.conf
\family default
will be included.
This may produce different results than expected.
Make sure you start your replays always in the same base directory!
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
It is easy to misconfigure almost anything by accident.
Check each step you make.
In particular, run tools like
\family typewriter
\size footnotesize
top
\family default
\size default
,
\family typewriter
\size footnotesize
xosview
\family default
\size default
,
\family typewriter
\size footnotesize
iostat
\family default
\size default
,
\family typewriter
\size footnotesize
watch df /tmp/
\family default
\size default
etc on
\emph on
all(!)
\emph default
your involved machines in order to get a chance for noticing when anything
goes wrong!
\emph on
Never
\emph default
, really
\emph on
never
\emph default
run
\family typewriter
\size footnotesize
tree-replay.sh
\family default
\size default
\series bold
blindly
\series default
!
\end_layout
\begin_layout Subsection
Visualization of Results
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
You need enough space in
\family typewriter
/tmp/
\family default
(or in another
\family typewriter
$TMPDIR
\family default
) for temporary intermediate files.
If your
\family typewriter
blkreplay
\family default
run was very long (several hours or even days), or if your replay had a
high degree of parallelism, you may need several gigabytes, in extreme
cases even several hundreds of gigabytes (as well as rather long running
times -- please don't interrupt
\family typewriter
graph.sh
\family default
as it spawns lots of subprocesses and creates lots of temporary files).
Currently, there are no checks for free space in
\family typewriter
/tmp/
\family default
, so running out of space may produce wrong results
\emph on
silently
\emph default
.
As a countermeasure, run
\family typewriter
watch df /tmp/
\family default
in a separate window during your run of
\family typewriter
graph.sh
\family default
.
\end_layout
\begin_layout Standard
If you have enabled the module
\family typewriter
graph
\family default
in the config file
\family typewriter
default-graph.conf
\family default
, the following steps will be carried out automatically for you.
Alternatively or in addition, the script
\family typewriter
tree-graph.sh
\family default
can be used to (re-)create all
\family typewriter
*.png
\family default
graphics in a whole directory hierarchy, analogously to
\family typewriter
tree-replay.sh
\family default
(and even sourcing the same
\family typewriter
*.conf
\family default
files in the same way).
In case you need some individual graphics, read on for details.
\end_layout
\begin_layout Standard
After a run of
\family typewriter
tree-replay.sh
\family default
,
\family typewriter
cd
\family default
to one of the subdirectories where your result files
\family typewriter
*.replay.gz
\family default
have been produced.
There should be as many
\family typewriter
*.replay.gz
\family default
files as there was replay parallelism (on multiple devices in parallel).
Check that.
In addition, check that no errors are inside them, for example by typing:
\end_layout
\begin_layout Quotation
\family typewriter
zgrep ERROR *.replay.gz
\end_layout
\begin_layout Standard
If all is right, issue the following command:
\end_layout
\begin_layout Quotation
\family typewriter
/path/to/graph.sh *.replay.gz
\end_layout
\begin_layout Standard
This will produce
\family typewriter
*.png
\family default
files, which you can inspect with any graphical viewer like
\family typewriter
eog
\family default
/
\family typewriter
konqueror
\family default
etc, print via
\family typewriter
lpr
\family default
, or even work on with graphical editors like
\family typewriter
gimp
\family default
.
\end_layout
\begin_layout Standard
All files
\family typewriter
*.replay.gz
\family default
will be taken together, to form a single result from contemporary replays
on several devices in parallel.
This means: where possible, results from multiple replay devices will be
merged together into a single graphics.
\end_layout
\begin_layout Description
Hint: If you want to view only a single particular device (or zoom into
it), just call
\family typewriter
\begin_inset Flex URL
status open
\begin_layout Plain Layout
/path/to/graph.sh
\end_layout
\end_inset
\family default
with a single argument.
\end_layout
\begin_layout Standard
As output, multiple kinds of graphics are produced.
Each one starts with the same prefix, but has another suffix.
For example,
\family typewriter
yourname.g01.latency.realtime.png
\family default
is a graphics file showing the measured latencies in realtime.
The numbering part
\family typewriter
.g01.
\family default
etc is for sorting in the shell, such that the
\begin_inset Quotes eld
\end_inset
most interesting
\begin_inset Foot
status open
\begin_layout Plain Layout
For many people; of course, there may be different needs.
Feel free to rename your result files as you like.
\end_layout
\end_inset
\begin_inset Quotes erd
\end_inset
graphics will come first.
\end_layout
\begin_layout Standard
By default, parameters matching
\family typewriter
*.load.gz
\family default
will produce graphics containing only
\emph on
static
\emph default
analyses.
When matching
\family typewriter
*.replay.gz
\family default
, only
\emph on
dynamic
\emph default
ones are produced.
In order additionally switch on any (or both), just add one of the options
\family typewriter
--static
\family default
or
\family typewriter
--dynamic
\family default
.
\end_layout
\begin_layout Standard
There is one exception: the throughput graphics are produced always, independent
ly from
\family typewriter
--static
\family default
or
\family typewriter
--dynamic
\family default
.
In the following, the meaning of different suffixes occurring in the output
filenames is described:
\end_layout
\begin_layout Description
\family typewriter
*.thrp.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The throughput is displayed on the y axis.
There is an overview variant, where the overall requested throughput is
depicted green, while the actual throughput (when running in dynamic mode)
is orange, for comparison.
When both are exactly identical (which occurs for example when you have
an extremely fast disk or a very low-demanding load), don't be confused
when the orange line completely covers the green one (a property of
\family typewriter
gnuplot
\family default
).
There are also more detailed variants where the read vs write throughput
is distinct.
\end_layout
\begin_layout Subsubsection
Static Analysis (
\family typewriter
--static
\family default
)
\end_layout
\begin_layout Description
\family typewriter
*.ws_log.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
On the y axis, the workingset size (cf section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Workingset"
\end_inset
) is displayed in logarithmic scale.
Multiple window sizes
\begin_inset Formula $\delta$
\end_inset
are displayed together in one graphics:
\family typewriter
001
\family default
means
\begin_inset Formula $\delta=1s$
\end_inset
,
\family typewriter
006
\family default
means 6 seconds,
\family typewriter
060
\family default
means 60 seconds, and
\family typewriter
000
\family default
means
\begin_inset Formula $\delta=\infty$
\end_inset
.
The latter is nothing but cumulation of
\emph on
all
\emph default
occurring sector numbers into one set
\begin_inset Foot
status open
\begin_layout Plain Layout
The slope of the
\family typewriter
000
\family default
line is an indicator for the
\begin_inset Quotes eld
\end_inset
repetitiveness
\begin_inset Quotes erd
\end_inset
of the workload.
\end_layout
\end_inset
.
\end_layout
\begin_layout Description
\family typewriter
*.ws_lin.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Like
\family typewriter
ws_log
\family default
, but the y axis is in linear scale.
May be useful for detecting more fine-grained behaviour in peaks.
\end_layout
\begin_layout Description
\family typewriter
*.sum_dist.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
On the y axis, the
\emph on
distance
\emph default
between the lowest and the highest sector number occurring in each workingset
window is displayed.
As a result, we can see something like a
\begin_inset Quotes eld
\end_inset
total seek distance
\begin_inset Quotes erd
\end_inset
, as if a disk elevator strategy
\emph on
would
\emph default
sort all requests inside a workingset window according to sector numbers,
in order to optimize throughput in favour of latency.
For experts, this can reveal some interesting internal property of the
workload.
\end_layout
\begin_layout Description
\family typewriter
*.avg_dist.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Same as
\family typewriter
sum_dist
\family default
, but the average distance is displayed (normalization against the workingset
size).
This results in something like an idealized
\begin_inset Quotes eld
\end_inset
average seek distance
\begin_inset Quotes erd
\end_inset
during each time window.
\end_layout
\begin_layout Description
\family typewriter
*.rqsize.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Displays a histogram of the request sizes (#sectors) occurring in the load,
in units of sectors.
\end_layout
\begin_layout Description
\family typewriter
*.rqpos.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Displays a histogram of the request position (sector#) occurring in the
load, in coarse units of whole GiB.
The coarse units are necessary to avoid too much jitter in the y axis.
\end_layout
\begin_layout Description
\family typewriter
*.turns.*.png
\family default
(both static and dynamic variants)
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
A turn is a request where the sector number is
\emph on
lower
\emph default
than the sector number of its predecessor.
For a mechanical hard disk, a turn means that it has to
\begin_inset Quotes eld
\end_inset
seek backwards
\begin_inset Quotes erd
\end_inset
, which is often more time-consuming than seeking forward.
Imagine an old-fashioned magnetic tape (e.g.
DAT), then you will get some feeling for the meaning of a turn.
Turns are an interesting measure for the locality behaviour of the load
(or the replay result).
There is no freak that some IO schedulers try to minimize the number of
turns (for example the famous sawtooth strategy).
In this graphics, the y axis shows the
\emph on
relative
\emph default
number of turns occurring during a time window of 1 seconds (in percent,
related to the total number of requests occurring during that time window).
\end_layout
\begin_layout Subsubsection
Dynamic Analysis (
\family typewriter
--dynamic
\family default
)
\end_layout
\begin_layout Description
\family typewriter
*.latency.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
On the y axis, latencies are displayed.
\end_layout
\begin_layout Description
\family typewriter
*.delay.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
On the y axis, the delays between the intended starting time and the real
starting time are displayed.
\end_layout
\begin_layout Description
\family typewriter
*.thrp.*.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
On the y axis, the troughput (IOPS) is displayed.
\end_layout
\begin_layout Description
\family typewriter
*.flying.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The y axis shows the total number of requests which are currently
\begin_inset Quotes eld
\end_inset
on the fly
\begin_inset Quotes erd
\end_inset
.
\begin_inset Newline newline
\end_inset
Examples:
\family typewriter
*.latency.flying.png
\family default
is derived from the latencies, and thus showing the number of requests
currently submitted to the device, while
\family typewriter
*.delay.flying.png
\family default
is derived from the delay, showing the number of requests currently waiting
in the logical
\begin_inset Quotes eld
\end_inset
delay queue
\begin_inset Quotes erd
\end_inset
.
\end_layout
\begin_layout Description
\family typewriter
*.smooth.*
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
This internal infix indicates that the y axis has been smoothed, in order
to be able to see anything in wildly jiggling data.
Example:
\family typewriter
*.smooth.latency.flying.png
\family default
.
\end_layout
\begin_layout Description
\family typewriter
*.realtime.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The x axis is ordered according to the real starting timestamps which have
actually occurred during replay, possibly containing any delays.
Notice: when multiple concurrent
\family typewriter
*.replay.gz
\family default
have been supplied as an argument, they are
\emph on
merged
\emph default
(since their timestamps are usually from the same range), similar to the
effect of overlay slides.
\end_layout
\begin_layout Description
\family typewriter
*.setpoint.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The x axis is ordered according to the
\emph on
intended
\emph default
request submission time (starting point), i.e.
when the request
\emph on
should have
\emph default
started (ignoring any delays).
As before, multiple
\family typewriter
*.replay.gz
\family default
are merged.
\end_layout
\begin_layout Description
\family typewriter
*.completed.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The x axis is ordered by the completion time of the requests.
Note that this may differ from the submission time.
\end_layout
\begin_layout Description
\family typewriter
*.points.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The x axis is ordered by
\emph on
requests
\emph default
, not timestamps.
As a result, requests on the x axis are
\emph on
equidistant
\emph default
, even in case of heavy throughput differences.
For
\family typewriter
*.replay.gz
\family default
, the order is the
\emph on
completion
\emph default
order.
This graphics is useful as a kind of
\begin_inset Quotes eld
\end_inset
looking glass
\begin_inset Quotes erd
\end_inset
, to reveal more details from performance hot spots.
Notice: multiple concurrent replays are
\emph on
not
\emph default
merged on the x axis (as is the case with
\family typewriter
*.realtime.png
\family default
,
\family typewriter
*.setpoint.png
\family default
and
\family typewriter
*.completed.png
\family default
).
Instead, the runs are just concatenated (pasted together sequentially)
in the same order as in the corresponding files.
\end_layout
\begin_layout Description
\family typewriter
*.bins.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The new y axis is now a
\emph on
histogram
\emph default
showing absolute frequency occurring at the former y axis, while the new
x axis now carries the role of the former y axis.
\begin_inset Newline newline
\end_inset
Examples:
\family typewriter
*.latency.bins.png
\family default
shows the latencies on the x axis (while the former
\family typewriter
*.latency.realtime.png
\family default
had it on the y axis), while the new y axis now shows the absolute frequency
of that latency (how often that latency occurs, independently from the
former replay timestamps).
\end_layout
\begin_layout Description
\family typewriter
*.xy.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The x axis as well as the y axis are displaying non-timely data.
This is useful for visualization of
\emph on
corrolations
\emph default
.
The following variants exist:
\end_layout
\begin_deeper
\begin_layout Description
\family typewriter
*.latency.xy.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
While latencies are displayed at the y axis, the x axis inidicates the number
of flying requests (request queue length).
Thus you can see the corrolation between request queue length and service
time.
\end_layout
\begin_layout Description
\family typewriter
*.delay.xy.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Similarly, but shows the corrolation between request queue length and delays
(the time needed to
\emph on
enter
\emph default
the request queue at all).
\end_layout
\begin_layout Description
\family typewriter
*.latency.delay.xy.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Immediate corrolation between latency and delay.
\end_layout
\end_deeper
\begin_layout Standard
Hint: the internal data format of
\family typewriter
*.replay.gz
\family default
is the same as
\family typewriter
*.load.gz
\family default
.
Thus you can use
\family typewriter
/path/to/graph.sh --static *.replay.gz
\family default
to additionally create the same static analysis graphics as described in
section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Select-blkreplay-Load"
\end_inset
.
In difference to analysis of
\family typewriter
*.load.gz
\family default
, this time you may have selected a different time window, and you may have
merged multiple replays together.
\end_layout
\begin_layout Section
Human Interpretation of Results
\end_layout
\begin_layout Standard
Many existing benchmark tools try to deliver a single number as a result,
such as
\begin_inset Formula $x$
\end_inset
\family typewriter
hyper-ops
\family default
or
\begin_inset Formula $x$
\end_inset
\family typewriter
hell-stones
\family default
.
Such numbers cannot describe the reality, which has multiple dimensions
and is much more complex.
Sometimes, new dimensions are even discovered to have an influence.
\end_layout
\begin_layout Standard
By default,
\family typewriter
blkreplay
\family default
does not produce any (single) number, but graphics, showing the behaviour
over time (or in other dimensions).
Although some of those graphics like
\family typewriter
*thrp*
\family default
could be used for production of numbers, most of them serve as input to
the human brain: the human
\begin_inset Quotes eld
\end_inset
neural network
\begin_inset Quotes erd
\end_inset
can be
\emph on
trained
\emph default
to detect
\series bold
hidden properties
\series default
in your measurement data, which cannot be (easily) detected by current
AI technology.
\end_layout
\begin_layout Subsection
Sonar Diagrams
\end_layout
\begin_layout Standard
The following example shows the replay latencies of a natural load over
real time.
Note that the y axis is logarithmic scale (otherwise you would'nt see too
much in the lower bands):
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename images/example1.g01.latency.realtime.png
lyxscale 20
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Reads are depicted as blue dots, while writes are colored red.
In case of
\family typewriter
--with-partial
\family default
, pushed-back requests are displayed additionally
\begin_inset Foot
status open
\begin_layout Plain Layout
Notice that the set of purple
\begin_inset Quotes eld
\end_inset
pushes
\begin_inset Quotes erd
\end_inset
is always a
\emph on
subset
\emph default
of the
\begin_inset Quotes eld
\end_inset
writes
\begin_inset Quotes erd
\end_inset
.
They are only visible because they are printed later (hiding the previous
red dots).
\end_layout
\end_inset
in purple.
Together, they form some kind of
\begin_inset Quotes eld
\end_inset
clouds
\begin_inset Quotes erd
\end_inset
, showing some
\begin_inset Quotes eld
\end_inset
density
\begin_inset Quotes erd
\end_inset
both in time scale, as well as in the
\begin_inset Quotes eld
\end_inset
latency scale
\begin_inset Quotes erd
\end_inset
.
\end_layout
\begin_layout Standard
It requires some experience to interpret sonar diagrams.
\end_layout
\begin_layout Itemize
\noindent
As you can see, there are two main
\series bold
\begin_inset Quotes eld
\end_inset
bands
\begin_inset Quotes erd
\end_inset
\series default
optically visible to the human interpretation.
These look similar to
\begin_inset Quotes eld
\end_inset
fish swarms
\begin_inset Quotes erd
\end_inset
occuring in sonar diagrams; hence the name
\begin_inset Quotes eld
\end_inset
sonar diagram
\begin_inset Quotes erd
\end_inset
.
\end_layout
\begin_layout Itemize
Most obviously,
\series bold
cache hierarchies
\series default
can be identified visually, by looking at the
\begin_inset Quotes eld
\end_inset
bands
\begin_inset Quotes erd
\end_inset
.
In addition, the natural fluctuation of the load can be seen.
Even better, it is quite interesting how the test candidate is responding
to the fluctuation, even over time.
\end_layout
\begin_layout Itemize
Any differences between read and write behaviour will immediately catch
your eye.
\end_layout
\begin_layout Itemize
The red
\begin_inset Quotes eld
\end_inset
needles
\begin_inset Quotes erd
\end_inset
are indications of queueing.
In difference to a single
\begin_inset Quotes eld
\end_inset
hang
\begin_inset Quotes erd
\end_inset
which would just
\begin_inset Quotes eld
\end_inset
fly above
\begin_inset Quotes erd
\end_inset
other requests, intermediate latencies from
\begin_inset Quotes eld
\end_inset
fellow requests
\begin_inset Quotes erd
\end_inset
are also present, spanning almost the full range from
\begin_inset Quotes eld
\end_inset
fast
\begin_inset Quotes erd
\end_inset
requests to the queued ones.
\end_layout
\begin_layout Itemize
In case of the strongest needle, you can even see a
\begin_inset Quotes eld
\end_inset
lash
\begin_inset Quotes erd
\end_inset
behind it.
There are no
\begin_inset Quotes eld
\end_inset
faster
\begin_inset Quotes erd
\end_inset
requests (not even reads) during this lash (i.e.
the space below it appears empty - even as if there were an interruption
in the time axis).
This is an indication of a
\emph on
global
\emph default
queuing strategy.
\end_layout
\begin_layout Standard
The same latencies are now displayed as a histogram.
A histogram is a non-timely survey of the absolute frequencies occurring
in the previous graphics.
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename images/example1.g06.latency.bins.png
lyxscale 20
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
You can identify the
\begin_inset Quotes eld
\end_inset
bands
\begin_inset Quotes erd
\end_inset
a second time by looking at the camel-like
\begin_inset Quotes eld
\end_inset
humps
\begin_inset Quotes erd
\end_inset
.
In addition to the sonar diagram, you can identify the
\begin_inset Quotes eld
\end_inset
hight
\begin_inset Quotes erd
\end_inset
and
\begin_inset Quotes eld
\end_inset
thickness
\begin_inset Quotes erd
\end_inset
of a hump more easily.
Some very small additional humps can also be found, probably an indicator
for further caches or probably for some scheduler timeouts.
\end_layout
\begin_layout Subsection
Delay Diagrams
\end_layout
\begin_layout Standard
As already explained, delays are the difference between the
\emph on
intended/demanded
\emph default
starting time of an IO requests, and its
\emph on
real
\emph default
starting time.
Huge delays are almost always an indication that the system cannot
\begin_inset Quotes eld
\end_inset
catch up
\begin_inset Quotes erd
\end_inset
with the application demands.
Here is an example:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename images/example1.g04.delay.setpoint.png
lyxscale 20
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Here are some interpretations:
\end_layout
\begin_layout Itemize
You will immediately notice the
\begin_inset Quotes eld
\end_inset
runaway tail
\begin_inset Quotes erd
\end_inset
after the purple indicator line, which is just a strong indication that
overall throughput is too low.
\end_layout
\begin_layout Itemize
Even before, you can see some
\begin_inset Quotes eld
\end_inset
needles
\begin_inset Quotes erd
\end_inset
with a
\begin_inset Quotes eld
\end_inset
lash
\begin_inset Quotes erd
\end_inset
behind.
Another indication, even occurring in a section with rather
\begin_inset Quotes eld
\end_inset
low
\begin_inset Quotes erd
\end_inset
throughput demands.
\end_layout
\begin_layout Standard
As before, there is also a histogram variant of delays:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename images/example1.g07.delay.bins.png
lyxscale 20
width 100col%
\end_inset
\end_layout
\begin_layout Subsection
Throughput Diagrams
\end_layout
\begin_layout Standard
\noindent
In order to compare this with the actual throughput, look at the following
picture, where demanded throughput is colored green, while actually delivered
throughput is orange:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename images/example1.g000.overview.thrp.png
lyxscale 20
width 100col%
\end_inset
\family typewriter
*.latency.flying
\end_layout
\begin_layout Standard
\noindent
This picture is a strong indication that single IOPS numbers are not sufficient
for really describing the actual IO throughput behaviour of a system.
Our timely plot can tell you a lot more about the system.
More examples can be found in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Overload-with-(Derived)"
\end_inset
.
\end_layout
\begin_layout Subsection
Flying Diagrams
\end_layout
\begin_layout Standard
These are useful to inspect the actual request queue length over time.
The following graphics exhibits the typical
\begin_inset Quotes eld
\end_inset
jitter
\begin_inset Quotes erd
\end_inset
of the queue length during replay of natural loads having a lot of conflicts,
and using
\family typewriter
--with-ordering
\family default
:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename images/example1.g08.latency.flying.png
lyxscale 20
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Notice: this is only the view of
\family typewriter
blkreplay
\family default
.
The physical device may be actually loaded with
\emph on
less
\emph default
requests in parallel, because intermediate layers (such as drivers, iSCSI,
etc) may
\series bold
limit
\series default
the IO parallelism.
When
\family typewriter
blkreplay
\family default
fires off a request, it need not hit the device immediately.
There are a lot of opportunities to be queued for some time in another
places.
\end_layout
\begin_layout Subsection
Corrolation Diagrams
\end_layout
\begin_layout Standard
Most of them are for advanced users and experts.
They can visualize internal relationships between interesting quantities,
such as observed request queue length versus observed latency:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename images/example1.g10.latency.xy.png
lyxscale 20
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Rough interpretation: under overload, write requests cannot be started in
time, and when they are started, they will take a long time.
Read requests are seemingly behaving better.
\end_layout
\begin_layout Section
Advanced Features
\end_layout
\begin_layout Subsection
Modules
\end_layout
\begin_layout Standard
The script
\family typewriter
tree-replay.sh
\family default
can be extended by plugins, aka modules.
Some modules are delivered with the
\family typewriter
blkreplay
\family default
suite, but you can add your own ones (and you should consider submitting
them to the opensource project
\family typewriter
blkreplay
\family default
at
\family typewriter
github.com
\family default
if they could be useful for a broader audience).
\end_layout
\begin_layout Standard
Here is a list of already existing modules:
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
iscsi_target_iet
\family default
When enabled, your
\family typewriter
iet
\family default
iSCSI target will be automatically configured for you, automatically generating
iSCSI
\family typewriter
IQN
\family default
names for you.
Usually this module is used in combination with
\family typewriter
iscsi_initiator
\family default
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
iscsi_initiator
\family default
When enabled, your
\family typewriter
blkreplay
\family default
runs will automatically act on iSCSI
\family typewriter
IQN
\family default
identifiers instead of on device names, transparently creating iSCSI connection
s for you.
It can be used standalone, for example to connect to commercial storage
boxes.
When combined with module
\family typewriter
iscsi_target_
\family default
*, even the
\family typewriter
IQN
\family default
name generation will be automated transparently (in this case you just
supply the device names present at the
\family typewriter
iscsi_target
\family default
host).
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
recreate_lvm
\family default
You just supply some physical devices (PVs) in
\family typewriter
$replay_device_list
\family default
, and this module will transparently (re-)create an LVM volume group (VG)
for you, together with some configurable number of logical volumes (LVs)
inside it, optionally even with LVM striping.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
create_lv
\family default
Here you can put in your code for re-creating logical volumes on an arbitrary
storage manager.
You
\series bold
need
\series default
this
\series bold
absolutely
\series default
whenever you are benchmarking virtualized storage; see section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Storage"
\end_inset
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
scheduler
\family default
By default (when enabled), the IO scheduler will be set to
\family typewriter
noop
\family default
(or another value) on all relevant target machines.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
wipe
\family default
When enabled, all your devices will be filled with random data before the
benchmark starts.
\series bold
Always
\series default
use this when benchmarking virtualized storage; see section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Storage"
\end_inset
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
bbu_megaraid
\family default
When enabled, your LSI megaraid conrollers will be programmed to enable/disable
BBU write caching.
The tool
\family typewriter
MegaCli
\family default
must be installed.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
graph
\family default
By default (when enabled), the script
\family typewriter
graph.sh
\family default
will be called automatically for you at the end of each
\family typewriter
blkreplay
\family default
run.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
pipe_*
\family default
When enabled, these filtering modules modify the
\family typewriter
*.load.gz
\family default
input at runtime before it reaches
\family typewriter
blkreplay
\family default
.
Following pipe filters are available:
\end_layout
\begin_deeper
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
pipe_repeat
\family default
Repeat the same input files forever, or until
\family typewriter
$replay_duration
\family default
is reached.
This is useful if your
\family typewriter
*.load.gz
\family default
is too short for a longer replay.
However keep in mind that the
\emph on
workingset
\emph default
will not increase, because exactly the
\emph on
same
\emph default
IO operations will be repeated after a while.
In order to circumvent that, combine this module with
\family typewriter
pipe_slip
\family default
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
pipe_slip
\family default
Add some offset to the sector# after a bunch of operations.
Use this in combination with
\family typewriter
pipe_repeat
\family default
to produce a
\begin_inset Quotes eld
\end_inset
slowly moving
\begin_inset Quotes erd
\end_inset
behaviour of the sector numbers, such that the next cycle will repeat the
operations at a slightly higher position.
Details see appendix
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:File-default-pipe_slip.conf"
\end_inset
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
pipe_subst
\family default
Replace
\family typewriter
R
\family default
[reads] with
\family typewriter
W
\family default
[writes] or vice versa.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
pipe_spread
\family default
Increase the region where the sector numbers will run about (corresponding
to the size of a LV) by some factor.
Details see appendix
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:File-default-pipe_spread.conf"
\end_inset
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
pipe_resize
\family default
Increase the request size (#sectors) by some factor, or clip it to some
bounds.
Details see appendix
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:File-default-pipe_resize.conf"
\end_inset
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
pipe_cmd
\family default
Insert arbitrary shell commands into the input pipeline.
\end_layout
\end_deeper
\begin_layout Standard
Detailed documentation of the module parameters can be found in
\family typewriter
default-*.conf
\family default
, or in appendix
\begin_inset CommandInset ref
LatexCommand ref
reference "sec:Module-Parameters"
\end_inset
.
\end_layout
\begin_layout Standard
In order to write your own module, just look at the existing ones and take
them as a kind of
\begin_inset Quotes eld
\end_inset
template
\begin_inset Quotes erd
\end_inset
.
Modules are simply
\family typewriter
/bin/bash
\family default
code residing in
\family typewriter
scripts/modules/
\family default
and starting with a number
\begin_inset Foot
status open
\begin_layout Plain Layout
The number is used to determine the ASCII sort order, and thus the order
in which modules are
\family typewriter
source
\family default
d.
\end_layout
\end_inset
, followed by an underscore, and finally
\family typewriter
\emph on
modulename
\emph default
.sh
\family default
.
Any existing module is automatically
\family typewriter
source
\family default
d by
\family typewriter
tree-replay.sh
\family default
, provided that some config file
\family typewriter
default-
\emph on
modulename
\emph default
.conf
\family default
exists in your working directory (or in any parent directory down the
\family typewriter
../
\family default
path).
\end_layout
\begin_layout Standard
Modules are sourced only once at the start of
\family typewriter
tree-replay.sh
\family default
.
Thus they should only define some shell
\family typewriter
function
\family default
s and change some list variables.
The shell functions will be called later for each leaf directory.
The names of your functions should obey the following conventions:
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
\emph on
modulename
\emph default
_prepare()
\family default
is called during the preparation phase.
Typically, you can set some variables here, initialize some controllers,
setup your network, create some iSCSI connections, or the like.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
\emph on
modulename
\emph default
_setup()
\family default
The setup phase starts after
\emph on
all
\emph default
prepare functions of all modules have been executed.
For example, some device setup operations are only possible after some
iSCSI connections have been already established.
Therefore, the plugin methodology offers you a separate phase in order
to guarantee some overall order on operations even from unknown foreign
modules.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
\emph on
modulename
\emph default
_run()
\family default
Typically, you won't do anything here, because the running phase is usually
the task of the
\family typewriter
main
\family default
module.
Only in very special cases, you would need to hook yourself into this phase.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
\emph on
modulename
\emph default
_cleanup()
\family default
Typically, you will de-configure something here, remove some temporary
files, check your results, create some statistics, etc.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
\emph on
modulename
\emph default
_finish()
\family default
Typically, network connections etc will be shutdown only here, because
some operations from the cleanup phase could be relying on them being active
yet.
\end_layout
\begin_layout Standard
For each phase, there exists a variable
\family typewriter
\emph on
phasename
\emph default
_list
\family default
determining the order in which the functions will be called.
In most cases, you will just
\emph on
append
\emph default
your function name to that list.
In some cases, you may want to
\emph on
pre
\emph default
pend instead.
Look into the existing code of small modules like
\family typewriter
graph
\family default
to get an impression, and read the
\family typewriter
main()
\family default
function in the
\family typewriter
main
\family default
module.
\end_layout
\begin_layout Section
Lowlevel Details and Expert Usage
\end_layout
\begin_layout Standard
Ordinary users should skip this section.
\end_layout
\begin_layout Subsection
Internal Overhead
\end_layout
\begin_layout Standard
Some new SSDs are promising IO rates of more than 100.000 IOPS when operated
over extremely fast IO channels.
When you try to achieve such high rates, you may probably stumble over
several difficulties.
\end_layout
\begin_layout Standard
For example, running too much threads / processes on the same file handle
or on the same pipe can lead to serious lock contention in the Linux kernel,
limiting overall throughput.
Although some people might argue that
\family typewriter
blkreplay
\family default
itself could produce an
\begin_inset Quotes eld
\end_inset
artificial
\begin_inset Quotes erd
\end_inset
bottleneck, please keep in mind that you need some
\emph on
real applications
\emph default
which were capable of producing / consuming such extremely high IO traffic.
Real applications usually don't use AIO, where such problems occur less
frequently.
No wonder that benchmark results published on the internet are often based
on AIO, which is far from practice.
\end_layout
\begin_layout Standard
Anyway, here is some advice how to deal with extremely high IO traffic.
For
\begin_inset Quotes eld
\end_inset
normal
\begin_inset Quotes erd
\end_inset
IO rates in the region of a few 10.000, the following is usually not needed.
Otherwise, you should be prepared to look into internals of the kernel,
compile your own high-speed custom kernels with some debugging options
switched off / better tuning of schedulers etc, and possibly use tools
like
\family typewriter
oprofile
\family default
to check what is really going on.
\end_layout
\begin_layout Standard
In order to determine the internal overhead of
\family typewriter
blkreplay
\family default
, play around with the following parameters:
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--threads=\SpecialChar \ldots{}
\family default
(or
\family typewriter
threads=\SpecialChar \ldots{}
\family default
as a shell variable in
\family typewriter
*.conf
\family default
) The most important setting.
Although counter-intuitive, too high numbers of threads may increase the
overall overhead and thus reduce throughput.
Be prepared to degrade this to rather low numbers in order to really achieve
extremely high IO throughput.
Usually, no more than 32 should be used.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--dry-run
\family default
(or, as a shell variable,
\family typewriter
dry_run
\family default
with underscore instead of minus) Omit any
\family typewriter
read()
\family default
and
\family typewriter
write()
\family default
operations, but continue to use
\family typewriter
lseek64()
\family default
on the file handles.
This way, you can determine the internal overhead in isolation.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--fake-io
\family default
Additionally omit even
\family typewriter
lseek64()
\family default
operations, as well as creation of header data.
Compare results with
\family typewriter
--dry-run
\family default
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--fan-out=\SpecialChar \ldots{}
\family default
In order to reduce parallelism on
\family typewriter
pipe()
\family default
channels,
\family typewriter
blkreplay
\family default
spawns internal distributor threads, forming a tree with limited fan-out
degree.
Too high numbers will increase pipe overhead, while too low numbers will
increase thread and scheduling overhead.
The optimum may depend on hardware (e.g.
NUMA) and kernel version.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--no-dispatcher
\family default
Omit distributor threads for the answer channels completely.
When combined with high numbers of threads, this may increase the kernel
overhead to almost unusable regions.
Use this for kernel tuning.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--bottleneck=\SpecialChar \ldots{}
\family default
Limit the number of flying IO requests.
Has a similar effect than
\family typewriter
--threads
\family default
, but is not the same.
Can be used for determining internal kernel and even hardware overheads
such as TLB thrashing (e.g.
by keeping
\family typewriter
--bottleneck
\family default
constant and increasing
\family typewriter
--threads
\family default
).
Could be useful for kernel and hardware tuning.
\begin_inset Newline newline
\end_inset
\series bold
Hint:
\series default
you can even set this to much higher numbers than
\family typewriter
--threads
\family default
, in order to pre-fill the communication pipes with some ahead requests
(minimizing the risk of waiting times).
However, when the number is way too high, a
\emph on
coprocess deadlock
\emph default
could occur (see the literature about coprocesses).
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\family typewriter
--speedup=\SpecialChar \ldots{}
\family default
A very dangerous option when used with natural loads.
Leads to heavy distortions of the relationship between timely and positionly
behaviour; in essence
\emph on
a natural load will not remain a natural load any more
\emph default
.
For artificial loads, however, such as
\family typewriter
*bursts*
\family default
, it is useful to reach top IOPS regions.
\end_layout
\begin_layout Standard
In any case, extremely high IOPS rates are likely to be reached only by
running multiple
\family typewriter
blkreplay
\family default
instances in parallel.
\end_layout
\begin_layout Chapter
How to use
\family typewriter
blktrace
\family default
for Recording of Natural Loads
\begin_inset CommandInset label
LatexCommand label
name "chap:How-to-use"
\end_inset
\end_layout
\begin_layout Standard
The tool
\family typewriter
blktrace
\family default
, as well as its friends like
\family typewriter
blkparse
\family default
, are described elsewhere.
Here are some additions from a practical viewpoint of IT operations:
\end_layout
\begin_layout Itemize
You will need a kernel with compile-time option
\family typewriter
CONFIG_BLK_DEV_IO_TRACE
\family default
enabled.
Although many newer Linux distros have it already enabled by default, you
may stumble over elder systems / distributions where it was missing (so
you may need to install and/or recompile a different kernel).
\end_layout
\begin_layout Itemize
Please note that some
\family typewriter
man
\family default
pages about
\family typewriter
blktrace
\family default
contain unclear / misleading information about the physical unit of the
\family typewriter
-b
\family default
/
\family typewriter
--buffer-size
\family default
option, which may lead to serious problems (including kernel hangups /
crashes) if you (accidentally) try to allocate more kernel memory than
physically available, and/or your production server is heavily loaded.
Unless you are recording extremely high-performance loads, the default
settings will succeed; so there will be no risk for IT operations.
\end_layout
\begin_layout Itemize
Tracing multiple devices in parallel is possible, but sometimes works differentl
y than (inconsistently) described in the
\family typewriter
man
\family default
, possibly leading to kernel problems (depending on versions etc).
Before recording blocktraces at production sites, we recommend to test
it somewhere in advance.
\end_layout
\begin_layout Standard
Please note that you need enough filesystem space for recording, residing
on a
\emph on
different
\emph default
partition than you want to record the IO traffic! Otherwise, your measurements
might be distorted.
\end_layout
\begin_layout Standard
You should record at least 8 hours of operation; better 24h.
\end_layout
\begin_layout Standard
Once
\family typewriter
blktrace
\family default
has finished, some files
\family typewriter
\emph on
yourname
\emph default
.blktrace.{0..7}
\family default
(or other numbers, corresponding to the number of CPUs) will exist in the
current working directory of your production machine.
Please copy all of them to your workstation before proceeding (never risk
more than necessary on production systems).
\end_layout
\begin_layout Standard
After copying, issue the following command:
\end_layout
\begin_layout Quote
\family typewriter
/path/to/blkreplay/scripts/conv_blktrace_to_load.sh
\emph on
yourname
\end_layout
\begin_layout Standard
This will produce an output file
\family typewriter
\emph on
yourname
\emph default
.load.gz
\family default
in your working directory.
You should seriously consider renaming it to
\family typewriter
\emph on
some_better_name
\emph default
.load.gz
\family default
.
\end_layout
\begin_layout Standard
The result file will contain as much IO requests as the script could find
in the
\family typewriter
blktrace
\family default
recording, but it will not contain any original timings (durations of the
requests, aka latencies).
The latency column is simply set to
\begin_inset Formula $0.0$
\end_inset
everywhere.
\end_layout
\begin_layout Standard
There is a reason for it.
Extracting timing information
\emph on
reliably
\emph default
from a
\family typewriter
blktrace
\family default
recording is difficult.
The problem is that each driver / subsystem in den kernel uses different
trace action characters to indicate the start of an IO request (as well
as for indicating various intermediate stages of request processing).
Request completion is
\emph on
another
\emph default
event denoted by character '
\family typewriter
C
\family default
'.
Depending on kernel version / driver, the completion events may be even
missing at all (some devices on very old kernels, which nevertheless occur
in practice).
More seriously, it is a hard task to determine pairs of events, i.e.
determining which completion event belongs to which submission event.
There is no concept of a
\begin_inset Quotes eld
\end_inset
request ID
\begin_inset Quotes erd
\end_inset
which would allow this easily.
You have to guess matching pairs from their sector numbers / length etc.
Guessing may be wrong due to ambiguities.
In general, a completion could sometimes even belong to multiple submission
events.
In addition, when you start and/or end your
\family typewriter
blktrace
\family default
recording at a heavily loaded
\emph on
production
\emph default
server (which is just the most interesting case), some pairs will very
likely remain incomplete at the start and at the end.
\end_layout
\begin_layout Standard
Even under
\begin_inset Quotes eld
\end_inset
normal
\begin_inset Quotes erd
\end_inset
circumstances, we found many cases where non-matching events led to an
incomplete
\family typewriter
*.load.gz
\family default
result.
Therefore we decided to omit timings from our default conversion script,
at least for now
\begin_inset Foot
status open
\begin_layout Plain Layout
Please submit any improvements to the author as patches / git pull requests.
\end_layout
\end_inset
.
However, there is another script which tries to
\emph on
guess
\emph default
the timings as best as it can:
\end_layout
\begin_layout Quote
\family typewriter
/path/to/blkreplay/scripts/conv_blktrace_to_load_with_guessed_timing.sh
\emph on
yourname
\end_layout
\begin_layout Standard
\noindent
The latter script will name its output
\family typewriter
\emph on
yourname
\emph default
.guessed.load.gz,
\family default
which is different from the former one.
There is a reason for it.
Please don't remove the
\family typewriter
*guessed*
\family default
infix when renaming that result file.
We found cases where the difference was considerable.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
Very often, some single guessed timings are obviously wrong.
You can see it at the sonar diagram.
\end_layout
\begin_layout Standard
Please use the
\family typewriter
*guessed*
\family default
version only for
\series bold
analysis
\series default
, such as producing
\family typewriter
*.latency.flying
\family default
or
\family typewriter
*.smooth.latency.flying
\family default
from
\family typewriter
graph.sh --static --dynamic
\family default
in order to determine the original IO parallelism and other interesting
properties.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresToxiques.png
lyxscale 50
scale 17
\end_inset
Only use the
\family typewriter
*.load.gz
\family default
output stemming from
\family typewriter
conv_blktrace_to_load.sh
\family default
as input for
\family typewriter
blkreplay
\family default
.
Never use
\family typewriter
*.guessed.load.gz
\family default
.
You may be missing some essential behaviour which could be crucial for
detection of incidents!
\end_layout
\begin_layout Standard
When your load contains some interesting effects and could be of broader
interest to a larger community, please contact the author of this paper
by Email in order to publish your load at
\family typewriter
www.blkreplay.org
\family default
.
In particular, unusual and novel application areas, heavy ordinary workloads,
and recordings of incidents, are of a broader interest.
\end_layout
\begin_layout Chapter
Experiences with some Setups and some Loads
\end_layout
\begin_layout Section
Overload Tests
\end_layout
\begin_layout Standard
Overload tests try to reveal the behaviour of a system under exceptionary
conditions, similar to a worst-case scenario.
\end_layout
\begin_layout Standard
Overload tests can (and should) be done with natural loads (such as
\begin_inset Flex URL
status open
\begin_layout Plain Layout
http://www.blkreplay.org/loads/natural/1and1/natural-derived/
\end_layout
\end_inset
).
In addition, it is one of the rare places where
\emph on
artificial
\emph default
loads can reach
\emph on
some
\emph default
justification.
\end_layout
\begin_layout Subsection
Overload with Artificial Bursts
\end_layout
\begin_layout Standard
The following examples are using the artificial loads from
\begin_inset Flex URL
status open
\begin_layout Plain Layout
http://www.blkreplay.org/loads/artificial/random.bursts/
\end_layout
\end_inset
, which are rather
\begin_inset Quotes eld
\end_inset
anal
\begin_inset Quotes erd
\end_inset
ones (see file
\family typewriter
00README
\family default
there).
The load is linearly increasing over time, until the test candidate will
\begin_inset Quotes eld
\end_inset
collapse
\begin_inset Quotes erd
\end_inset
under the overload.
\end_layout
\begin_layout Standard
We use a single
\family typewriter
blkreplay
\family default
instance, and compare between an old SATA RAID-6 and a newer RAID-6 with
faster SAS disks (15k RPM).
For now, we consistently use
\family typewriter
--threads=128
\family default
.
All results can be found in the directory
\family typewriter
blkreplay.git/example-run/
\family default
and subdirs.
\end_layout
\begin_layout Standard
Here is the sonar diagram for the SATA system.
It
\emph on
should
\emph default
have run 300s, but it actually run much longer in realtime:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
At the beginning, the load is rather low, and the system can catch up.
Soon, it reaches a kind of
\begin_inset Quotes eld
\end_inset
maximum throughput
\begin_inset Quotes erd
\end_inset
where the latencies go up to about 1 second.
You can see on the realtime x axis that the replay is delayed as a whole.
This can be better seen at the delay diagram:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g04.delay.setpoint.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Here the x axis shows the
\emph on
intended
\emph default
starting time of each request, which is deliberately concentrated at whole
seconds (with gaps inbetween), a property of of the
\family typewriter
*bursts*
\family default
loads.
The
\begin_inset Quotes eld
\end_inset
collapsing point
\begin_inset Quotes erd
\end_inset
is about 60s, which means that it roughly can deliver 600 random IOPS.
This can be even better seen at the overwiew
\family typewriter
thrp
\family default
comparing the actual with the demanded throughput, as well as the detailed
\family typewriter
thrp
\family default
diagram of the actual throughput ordered by completion time of the requests:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sata.bbu-1.g00.thrp.completed.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Now we turn to better hardware: the SAS system uses smaller disks with 15k
RPM, and thus needs more spindles to reach the same capacity.
If we would run the same test for 300s (as in the SATA case), we would
not notice any collapsing.
Thus we double
\family typewriter
replay_duration
\family default
to 600s in order to be able to see the collapsing point.
Please take this into account when looking at the following graphics:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Notice that the system is even able to reproduce the bursting behaviour
of the load, much better than the SATA system.
Only at higher request rates, some
\begin_inset Quotes eld
\end_inset
blurring clouds
\begin_inset Quotes erd
\end_inset
appear on the x axis, until it finally collapses under too high load.
It is amazing to see the
\begin_inset Quotes eld
\end_inset
binary
\begin_inset Quotes erd
\end_inset
behaviour at the collapsing point: as soon as the actual throughput rate
cannot catch up with the request rate, the internal queues will suddenly
grow up to some internal maximum, and thus lead to latencies around 0.1
seconds, which is better than SATA by an order of magnitude.
Of course, the same effect can be observed at the delay diagram and the
detailed throughput diagram:
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g03.latency.points.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\align center
\begin_inset Graphics
filename ../example-run/example-overload/random.bursts/with-conflicts/threads-128/test-sas/EXAMPLE.random.bursts.with-conflicts.threads-128.test-sas.g00.thrp.completed.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Subsection
Overload with (Derived) Natural Loads
\begin_inset CommandInset label
LatexCommand label
name "sub:Overload-with-(Derived)"
\end_inset
\end_layout
\begin_layout Standard
Overload tests with natural loads are probably the most interesting ones.
They can tell you a lot upon both the test candidate, as well as upon your
application.
\end_layout
\begin_layout Standard
In many cases, natural loads are varying very much over time.
Frequently, the variance in the IOPS rate can span an order of magnitude,
or even more.
Typical reasons are cron jobs or nightly backups on an otherwise (almost)
idle server.
\end_layout
\begin_layout Standard
In order to find the peaks in the load, you have to draw
\family typewriter
blktrace
\family default
s from production servers for at least 24h.
After conversion to
\family typewriter
*.load.gz
\family default
, it is rather cumbersome to find the load peaks by hand in order to get
suitable input data for
\emph on
overload
\emph default
tests.
Thus, we have developed the script
\family typewriter
create_derived_load.sh
\family default
which will do the following for you:
\end_layout
\begin_layout Enumerate
It splits the input file(s) into snippets of equal length in realtime (by
default, 5min length each).
\end_layout
\begin_layout Enumerate
It sorts the snippets according to IOPS in reverse order, i.e.
the
\begin_inset Quotes eld
\end_inset
heaviest
\begin_inset Quotes erd
\end_inset
snippet will come first.
\end_layout
\begin_layout Enumerate
It distributes the snippets to a number of output files in a round-robin
fashion (while adjusting the timestamps to the new order).
\end_layout
\begin_layout Standard
As a result, the output files will start with the heaviest IOPS load first,
and later declining.
This methodology has a number of advantages:
\end_layout
\begin_layout Itemize
The overload part is easily accessible at the beginning of each output file
(interesting part first).
\end_layout
\begin_layout Itemize
Thanks to the distribution to many output files, higher replay parallelism
is possible in an uncorrolated way.
\end_layout
\begin_layout Itemize
In case your test candidate should get overloaded: you can easily see its
\emph on
recovery behaviour
\emph default
from overload, thanks to the declination of the load over time.
Will it recover at all, and how fast?
\end_layout
\begin_layout Standard
The latter is an important property of any enterprise-grade storage system.
\end_layout
\begin_layout Standard
Here are some example comparisons between an old SATA system and a newer
SAS system, both equipped with hardware RAID-6, and loaded with a replay
parallelism of 4.
You can find the complete results with additional graphics as a tarball
at
\begin_inset Flex URL
status open
\begin_layout Plain Layout
http://www.blkreplay.org/examples/
\end_layout
\end_inset
.
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sas/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sas.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
It is interesting to observe the behaviour of the SATA system in contrast
to the high-performance SAS: at first, the real IOPS rate (orange) cannot
catch up with the demanded rate (green).
However, when the demanded rate decreases under the green line after some
time, the orange line becomes higher than the green one until the backlog
has been worked up.
Finally, the system can (almost) run in sync with the lower demands, and
the orange line is hiding the green one.
\end_layout
\begin_layout Standard
The same effects can also be seen in the corresponding delay diagrams, where
you can even notice multiple top lines stemming from overlay of replays
in parallel, and even watch their different fallback points:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sata.bbu-1.vg-test.g04.delay.setpoint.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/long/parallel-04/with-partial/threads-128/test-sas/EXAMPLE.natural-derived.long.parallel-04.with-partial.threads-128.test-sas.vg-test.g04.delay.setpoint.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Section
Influence of Replay Parameters
\end_layout
\begin_layout Standard
The following examples are run on the rather slow SATA system.
By default, we use
\family typewriter
threads=128
\family default
and 4
\family typewriter
blkreplay
\family default
instances, operating in parallel on 4 LVM devices created from the same
volume group.
Therefore, the total IO parallelism is
\begin_inset Formula $128*4=512$
\end_inset
.
Exceptions from that are denoted.
\end_layout
\begin_layout Subsection
Influence of Request Ordering
\begin_inset CommandInset label
LatexCommand label
name "sub:Influence-of-Request"
\end_inset
\end_layout
\begin_layout Standard
We start with the simplest case: mode
\family typewriter
--with-conflicts
\family default
fires up all requests without guaranteeing any order (besides that requests
are never started too early), which may lead to damaged IO.
Here is the sonar diagram as well as the throughput diagram:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
It is clear that the SATA cannot catch up with the high demands here.
But why? Is it the damaged IO?
\end_layout
\begin_layout Standard
A first hint is the rather
\begin_inset Quotes eld
\end_inset
thick
\begin_inset Quotes erd
\end_inset
red
\begin_inset Quotes eld
\end_inset
cloud carpet
\begin_inset Quotes erd
\end_inset
in the latency diagram near 1s, indicating some queueing behaviour.
Interestingly, this carpet falls a little down at the end, when the replay
parallelism decreases due to different termination of different
\family typewriter
blkreplay
\family default
instances.
\end_layout
\begin_layout Standard
Now we turn to
\family typewriter
--with-drop
\family default
.
Surprisingly, there is only a small difference, although damaged IO is
avoided at the cost of some lost requests:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
This is a clear indication that damaged IO cannot be the reason for the
delays, since it is avoided by
\family typewriter
--with-drop
\family default
.
\end_layout
\begin_layout Standard
Now we turn to
\family typewriter
--with-partial
\family default
.
Most of the time, there is almost the same behaviour as before.
Only the tail is different, because the actual running times are more diverging
between the four
\family typewriter
blkreplay
\family default
instances started in parallel:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
The tail may look disturbing to some people.
It has a rather
\emph on
low
\emph default
IOPS rate.
In case of delays,
\family typewriter
blkreplay
\family default
should try to catch up as fast as possible, shouldn't it? What the heck
is going on there? Is there a bug in
\family typewriter
blkreplay
\family default
?
\end_layout
\begin_layout Standard
Examination of
\family typewriter
*.lv-data3.replay.gz
\family default
from
\begin_inset Flex URL
status open
\begin_layout Plain Layout
http://www.blkreplay.org/examples/
\end_layout
\end_inset
tells us that the real last tail consists almost exclusively of pushed-back
requests.
In detail, there are lots of requests depending on each other
\emph on
transitively
\emph default
, reaching over many generations.
They have accumulated over a long time, because
\family typewriter
--with-partial
\family default
is
\emph on
defined
\emph default
to do so.
The tail is nothing but catch-up of pushed-back requests which could not
be run earlier, in order to obey the storage semantics.
Many of them are mutually hindering each other.
The ordinary non-conflicting requests form a different class than then
conflicting ones, leading to a 2-class society with different chances to
be processed (unfairness).
Thus one of the both classes finishes earlier than the other.
This is a disadvantage of
\family typewriter
--with-partial
\family default
you should know.
\end_layout
\begin_layout Standard
On the other hand, this can be turned into an advantage, because you can
use
\family typewriter
--with-partial
\family default
for detection of such a behaviour.
Think of the race between Achilles and the turtle.
If you find a much slower turtle, your test candidate has probably some
problems with that
\emph on
specific
\emph default
load, but not necessarily with other loads at the same IOPS level.
\end_layout
\begin_layout Standard
Hint: the 2-class society caused by
\family typewriter
--with-partial
\family default
is often visible in the delay diagrams.
Check out yourself!
\end_layout
\begin_layout Standard
Let us return to the comparion of request ordering modes: more prominent
differences can be found when we turn to
\family typewriter
--with-ordering
\family default
.
Now the system can catch up
\emph on
in average
\emph default
(but not in general at load peaks).
The sonar diagram shows that the
\begin_inset Quotes eld
\end_inset
thick clouds
\begin_inset Quotes erd
\end_inset
at about 1s have fallen down and have become thinner in densitiy:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
This rises the question for reasons.
Why should
\family typewriter
--with-ordering
\family default
be better than
\family typewriter
--with-partial
\family default
or even
\family typewriter
--with-drop
\family default
? Full ordering will usually
\emph on
reduce
\emph default
the IO parallelism when compared to partial ordering.
The full answer is in sections
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Number"
\end_inset
and
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Bottlenecks"
\end_inset
, but for now you can take the small
\begin_inset Quotes eld
\end_inset
gaps
\begin_inset Quotes erd
\end_inset
on the x axis as a hint that there must be something related to small micro-sta
lls, which seem to
\emph on
accelerate
\emph default
overall throughput in some
\series bold
counter-intuitive
\series default
way.
The micro-stalls are easily explainable by the submission delays caused
by
\family typewriter
--with-ordering
\family default
, reducing the IO parallelism in case of conflicts.
But why does that
\emph on
accelerate
\emph default
the overall throughput?
\end_layout
\begin_layout Standard
Before proceeding, we take a closer look at the micro-stall effect.
It is better visible in
\family typewriter
*.latency.flying
\family default
showing the
\emph on
actually achieved IO parallelism
\emph default
during the replay.
Here is the comparison between
\family typewriter
--with-conflicts
\family default
, and
\family typewriter
--with-drop
\family default
, and
\family typewriter
--with-
\family default
partial:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-drop/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-drop.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
The sum of both reads and writes (olive line) indicates that the RAID controller
is filled with a maximum of
\begin_inset Formula $128*4=512$
\end_inset
requests (caused by
\family typewriter
threads=128
\family default
and a replay parallelism of
\begin_inset Formula $4$
\end_inset
) most of the time, except at the tail.
\end_layout
\begin_layout Standard
And now, compare these with
\family typewriter
--with-ordering
\family default
:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-ordering/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-ordering.strong-1.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
It is clear that by definition of
\family typewriter
ordering
\family default
, some kind of stalls have to be introduced to guarantee some virtual
\begin_inset Quotes eld
\end_inset
strict storage semantics
\begin_inset Quotes erd
\end_inset
.
It is also clear that these micro-stalls will sometimes reduce the actual
IO parallelism, which is clearly visible in the graphics.
However, can these micro-stalls be made
\emph on
responsible
\emph default
for an
\emph on
increase
\emph default
of overall throughput?
\end_layout
\begin_layout Standard
In order to investigate that, we turn to the influence of the
\family typewriter
threads=
\family default
parameter.
\end_layout
\begin_layout Subsection
Influence of Number of Threads
\begin_inset CommandInset label
LatexCommand label
name "sub:Influence-of-Number"
\end_inset
\end_layout
\begin_layout Standard
The following throughput graphics are a direct comparison between 8, 16,
32, 64, 128, 256 and 512 threads on the SATA system using
\family typewriter
--with-conflicts
\family default
, which guarantees the best possible IO parallelism and avoids any non-linear
influences from any conflict-handling strategies:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-008/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-008.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-016/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-016.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-064/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-064.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-256/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-256.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-512/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-512.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
It is clear that a too low number of threads, such as 8, can hinder overall
throughput (even when in reality the total IO parallelism is
\begin_inset Formula $8*4=32$
\end_inset
by taking the number of
\family typewriter
blkreplay
\family default
instances into account).
It is therefore no wonder that 16 through 64 threads perform better than
8.
But, why is there a counter-productive break-down when increasing the number
of threads after that? The effect is rather strong: 512 threads will decrease
throughput almost by a factor of three when compared to the
\begin_inset Quotes eld
\end_inset
best
\begin_inset Quotes erd
\end_inset
setting.
This is no peanuts! What the hell is going on here?
\end_layout
\begin_layout Standard
The answer can only be buried in the internals of the hardware and/or its
firmware and/or its driver.
Typically, some systems have problems when filled with too many requests
in parallel.
Here, we see a typical
\series bold
non-linear behaviour
\series default
as explained in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Replay"
\end_inset
.
\end_layout
\begin_layout Standard
Consequences: the
\family typewriter
threads=
\family default
parameter can be very important.
And its
\begin_inset Quotes eld
\end_inset
optimum
\begin_inset Quotes erd
\end_inset
can non-linearly depend from the request ordering parameters.
Here, you see a proof for the warnings posed in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Pitfalls-from-Replay"
\end_inset
.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\begin_inset Graphics
filename images/MatieresToxiques.png
lyxscale 50
scale 17
\end_inset
\emph on
Never
\emph default
adjust the
\family typewriter
threads=
\family default
parameter to the value delivering the
\begin_inset Quotes eld
\end_inset
best
\begin_inset Quotes erd
\end_inset
performance when comparing different systems / vendors.
Doing so will be a
\emph on
lie(!)
\emph default
to your audience! Always use the
\family typewriter
threads=
\family default
setting which corresponds to your
\series bold
real application
\series default
based on
\series bold
knowledge
\series default
(by examining the submission IO parallelism at the original
\family typewriter
blktrace
\family default
recording site, e.g.
by looking at the
\family typewriter
*.flying
\family default
graphics produced from the original
\family typewriter
blktrace
\family default
recording), and
\series bold
not
\series default
based on any assumptions which could be faked in any direction! Be responsible
and tell your audience
\emph on
why
\emph default
you select a specific value, and what would happen if you selected a different
one! Or, provide a series of different
\family typewriter
threads=
\family default
settings telling the
\emph on
whole
\emph default
story!
\end_layout
\begin_layout Subsection
Influence of BBU units at RAID controllers
\end_layout
\begin_layout Standard
The following comparison is an example for the performace degradation resulting
from defective (or missing) BBU units:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-0/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-0.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
In essence, the missing BBU will slow down write requests.
This can be easily seen when comparing the sonar diagrams (just look at
the colors and where they appear):
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-0/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-0.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
More interesting differences can be found in the tarball at
\begin_inset Flex URL
status open
\begin_layout Plain Layout
http://www.blkreplay.org/examples/
\end_layout
\end_inset
.
\end_layout
\begin_layout Subsection
Influence of Bottlenecks
\begin_inset CommandInset label
LatexCommand label
name "sub:Influence-of-Bottlenecks"
\end_inset
\end_layout
\begin_layout Standard
The expert option
\family typewriter
--bottleneck=
\family default
can be used to limit the number of requests submitted to the worker threads,
which is usually higher than the number of threads.
In each individual pipeline connecting the main thread to each worker thread,
up to 8 requests may be queued in advance.
\end_layout
\begin_layout Standard
In the following example using
\family typewriter
--with-conflicts
\family default
and
\family typewriter
--threads=128
\family default
, we also limit the number of requests by
\family typewriter
--bottleneck=128
\family default
, which leads to almost the same effect as the above discussion regarding
\family typewriter
--with-ordering
\family default
:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
This result is unexpected.
In theory, it should not make any difference whether there is 1 request
in between the main thread and a worker, or many.
However, in practice it makes a difference: the
\emph on
communication latencies
\emph default
over the pipelines are avoided when the pipeline remains always filled
with some requests in advance.
\end_layout
\begin_layout Standard
When
\family typewriter
--bottleneck
\family default
has exactly the same value as
\family typewriter
--threads
\family default
, a
\begin_inset Quotes eld
\end_inset
logical request
\begin_inset Quotes erd
\end_inset
can be either working in a worker thread, or being processed by the main
thread, or underway over the pipes.
It cannot be present at more than one of these places at the same time.
\end_layout
\begin_layout Standard
An example, just for demonstration of the principle: Assume that the IO
system can catch up (at least at the beginning), and that the IO latencies
are
\emph on
constant
\emph default
.
Assume that the communication latencies are exactly the same constant.
What will be the effect? One half of the requests will always reside in
the IO system, but the other half will reside in the pipe communication
infrastructure, in average.
Did you get the point?
\end_layout
\begin_layout Standard
Now, assume that both the IO latencies as well as the communication latencies
are varying (e.g.
due to kernel scheduling etc).
Then the queue length might be staggering around, as observed in the following
graphics (first is the default bottleneck setting
\family typewriter
--bottleneck=$(( threads * 8 ))
\family default
, second is
\family typewriter
--bottleneck=${threads}
\family default
):
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
As before, the SATA system shows its counter-productive behaviour when too
many IO requests are fired up in parallel.
Once the number of requests has reduced, it works better.
But somewhen it staggers to the other extreme, flipping around.
And to the opposite.
\end_layout
\begin_layout Standard
Interesting side note: similar behaviour can be observed in real-life production
systems, because natural loads (as opposed to standard benchmark tests)
often vary by orders of magnitude.
When spuriously plagued by seemingly
\begin_inset Quotes eld
\end_inset
unmotivated
\begin_inset Quotes erd
\end_inset
incidents, look for places where non-linear and counter-productive behaviour
may cause unexpected (and possibly self-amplifying) effects!
\end_layout
\begin_layout Subsection
Influence of
\family typewriter
strong
\family default
Mode
\begin_inset CommandInset label
LatexCommand label
name "sub:Influence-of-strong"
\end_inset
\end_layout
\begin_layout Standard
Here is a direct comparison between all three
\family typewriter
--strong=
\family default
modes, when combined with
\family typewriter
--with-partial
\family default
.
The effect is highly dependable from the load, and usually rather small
(when compared to the drastic effects demonstrated in some of the previous
sections).
But don't take this as given! Better check for it.
First, we compare at
\family typewriter
--threads=128
\family default
:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-128/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-128.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
The
\begin_inset Quotes eld
\end_inset
long tail
\begin_inset Quotes erd
\end_inset
explained in section
\begin_inset CommandInset ref
LatexCommand ref
reference "sub:Influence-of-Request"
\end_inset
is disturbing.
The 2-class treatment inherent to
\family typewriter
--with-partial
\family default
leads to a high number of
\emph on
transitively
\emph default
pushed-back requests, racing against the ordinary requests.
This race is in some sense
\begin_inset Quotes eld
\end_inset
non-deterministic
\begin_inset Quotes erd
\end_inset
and may depend non-linearly from the
\family typewriter
--strong=
\family default
mode.
Therefore, we reduce the 2-class treatment by turning to
\family typewriter
--threads=32
\family default
:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g000.overview.thrp.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
Now the tail effect has almost vanished, because the better throughput leads
to less queueing, which in turn decrease the chances for a conflict to
actually occur during the time window
\family typewriter
--ahead_limit=
\family default
(default
\begin_inset Formula $1$
\end_inset
second).
In consequence, any self-amplifying
\begin_inset Quotes eld
\end_inset
traffic jam effects
\begin_inset Quotes erd
\end_inset
are much lower.
As expected, the actual throughput decreases with
\family typewriter
strong
\family default
er mode, but the effect is really tiny, probably below measurement tolerances.
In order to really see something, we take a closer look at the sonar diagrams:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g01.latency.realtime.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
As expected, the number of
\family typewriter
Write Pushbacks
\family default
is increasing with
\family typewriter
--strong=
\family default
mode, and
\family typewriter
Read Pushbacks
\family default
come also into play.
But the differences are minor.
A litte more impressive are the differences at
\family typewriter
*.latency.flying
\family default
, where the
\begin_inset Quotes eld
\end_inset
staggering effect
\begin_inset Quotes erd
\end_inset
is clearly increasing with growing
\family typewriter
--strong=
\family default
mode:
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-0/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-0.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-1/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-1.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-partial/strong-2/threads-032/test-sata/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-partial.strong-2.threads-032.test-sata.bbu-1.vg-test.g08.latency.flying.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
More details can be explored when looking at the purple
\begin_inset Quotes eld
\end_inset
Write Pushes
\begin_inset Quotes erd
\end_inset
/ deep-blue
\begin_inset Quotes eld
\end_inset
Read Pushes
\begin_inset Quotes erd
\end_inset
displayed in further graphics at
\begin_inset Flex URL
status open
\begin_layout Plain Layout
http://www.blkreplay.org/examples/
\end_layout
\end_inset
.
\end_layout
\begin_layout Chapter
\start_of_appendix
Config File Parameters
\begin_inset CommandInset label
LatexCommand label
name "chap:Config-File-Parameters"
\end_inset
\end_layout
\begin_layout Standard
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstset{ %
\end_layout
\begin_layout Plain Layout
language=bash,
\end_layout
\begin_layout Plain Layout
basicstyle=
\backslash
tt
\backslash
footnotesize,
\end_layout
\begin_layout Plain Layout
% numbers=left,
\end_layout
\begin_layout Plain Layout
% numberstyle=
\backslash
tiny
\backslash
color{gray},
\end_layout
\begin_layout Plain Layout
% stepnumber=2,
\end_layout
\begin_layout Plain Layout
% numbersep=5pt,
\end_layout
\begin_layout Plain Layout
%backgroundcolor=
\backslash
color{white},
\end_layout
\begin_layout Plain Layout
showspaces=false,
\end_layout
\begin_layout Plain Layout
showstringspaces=true,
\end_layout
\begin_layout Plain Layout
showtabs=true,
\end_layout
\begin_layout Plain Layout
%frame=single,
\end_layout
\begin_layout Plain Layout
%rulecolor=
\backslash
color{black},
\end_layout
\begin_layout Plain Layout
tabsize=8,
\end_layout
\begin_layout Plain Layout
captionpos=t,
\end_layout
\begin_layout Plain Layout
breaklines=true,
\end_layout
\begin_layout Plain Layout
breakatwhitespace=false,
\end_layout
\begin_layout Plain Layout
title=
\backslash
lstname,
\end_layout
\begin_layout Plain Layout
keywordstyle=
\backslash
color{brown},
\end_layout
\begin_layout Plain Layout
commentstyle=
\backslash
color{OliveGreen},
\end_layout
\begin_layout Plain Layout
}
\end_layout
\end_inset
\end_layout
\begin_layout Section
Basic Parameters
\end_layout
\begin_layout Subsection
File
\family typewriter
user_modules.conf
\family default
:
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/user_modules.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-main.conf
\family default
:
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-main.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Section
Ordinary Module Parameters
\begin_inset CommandInset label
LatexCommand label
name "sec:Module-Parameters"
\end_inset
\end_layout
\begin_layout Standard
The following list is in the order of activation.
\end_layout
\begin_layout Subsection
File
\family typewriter
default-recreate_lvm.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-recreate_lvm.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-create_lv.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-create_lv.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-iscsi_target_iet.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-iscsi_target_iet.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-iscsi_initiator.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-iscsi_initiator.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-scheduler.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-scheduler.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-wipe.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-wipe.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-bbu_megaraid.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-wipe.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-graph.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-graph.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Section
Pipe Module Parameters
\begin_inset CommandInset label
LatexCommand label
name "sec:Pipe-Module-Parameters"
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-pipe_repeat.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-pipe_repeat.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-pipe_slip.conf
\begin_inset CommandInset label
LatexCommand label
name "sub:File-default-pipe_slip.conf"
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-pipe_slip.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-pipe_subst.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-pipe_subst.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-pipe_spread.conf
\begin_inset CommandInset label
LatexCommand label
name "sub:File-default-pipe_spread.conf"
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-pipe_spread.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-pipe_resize.conf
\begin_inset CommandInset label
LatexCommand label
name "sub:File-default-pipe_resize.conf"
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-pipe_resize.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Subsection
File
\family typewriter
default-pipe_cmd.conf
\end_layout
\begin_layout Standard
\noindent
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting[language=bash]{example-run/default-pipe_cmd.conf}
\end_layout
\end_inset
\end_layout
\begin_layout Chapter
File Format
\end_layout
\begin_layout Standard
The
\family typewriter
*.load.gz
\family default
and
\family typewriter
*.replay.gz
\family default
files are in essence compressed CSV files, using
\family typewriter
\begin_inset Quotes eld
\end_inset
;
\begin_inset Quotes erd
\end_inset
\family default
as delimiter.
However, errors and comments can also occur in the file, violating the
classical CSV format.
\end_layout
\begin_layout Standard
In order to extract classical CSV, use the following command:
\end_layout
\begin_layout Quotation
\family typewriter
zgrep
\begin_inset Quotes eld
\end_inset
;
\begin_inset Quotes erd
\end_inset
\emph on
myname
\emph default
.replay.gz | less
\end_layout
\begin_layout Standard
The first line of the output contains the column names (and could be removed
by inserting
\family typewriter
| tail -n +2
\family default
into the pipeline).
In
\emph on
theory
\emph default
, you could even use MS Excel to work on the above output.
However, in practice the file will be so extremely large that interactive
workstation tools will likely start swapping your system to death, or even
bombing.
\end_layout
\begin_layout Standard
So the usual way to operate on the output is a
\begin_inset Quotes eld
\end_inset
pipe and filters style
\begin_inset Quotes erd
\end_inset
, using stream-processing tools like
\family typewriter
sed
\family default
,
\family typewriter
awk
\family default
,
\family typewriter
perl
\family default
, and so on.
\end_layout
\begin_layout Standard
Both
\family typewriter
*.load.gz
\family default
and
\family typewriter
*.replay.gz
\family default
use the same format.
However,
\family typewriter
*.load.gz
\family default
\series bold
\emph on
must
\series default
\emph default
be sorted numerically according to the timestamp, while
\family typewriter
*.replay.gz
\family default
is usually unsorted (roughly showing the order of request completion in
most cases, which is different from the submission timestamp ordering).
In order to re-use
\family typewriter
*.replay.gz
\family default
again as input for another run of
\family typewriter
blkreplay
\family default
, you must sort it with
\family typewriter
sort -n
\family default
.
In some cases, public
\family typewriter
blktrace
\family default
recordings at
\family typewriter
www.blktrace.org/loads/
\family default
will contain only constants 0.0 as delays and latencies.
\end_layout
\begin_layout Standard
Feel free to filter your results, or create your own artifical loads.
\end_layout
\begin_layout Standard
\noindent
Enjoy.
\end_layout
\begin_layout Chapter
Validation of the
\family typewriter
blkreplay
\family default
Tool
\end_layout
\begin_layout Standard
Some effort has been spent to check that
\family typewriter
blkreplay
\family default
really produces the intended kind of IO load on the target system, and
that its measurement results are likely to be valid.
However, there is no guarantee against hidden bugs, misconfigurations,
or other types of accidents.
See also the disclaimer of the GPL license.
\end_layout
\begin_layout Standard
The following tools try the
\emph on
help
\emph default
you by making
\family typewriter
blkreplay
\family default
as transparent to the public as possible, but cannot form an absolute guarantee
against all possible sources of errors.
\end_layout
\begin_layout Standard
Please report any bugs to the author, or, even better, send patches or git
pull requests.
\end_layout
\begin_layout Section
Running
\family typewriter
blktrace
\family default
during
\family typewriter
blkreplay
\end_layout
\begin_layout Standard
The timeliness of IO requests is already measured and displayed by the
\family typewriter
*.latency.*
\family default
graphics.
The following steps will thus concentrate on
\emph on
completeness
\emph default
.
\end_layout
\begin_layout Standard
In order to check that all IO requests present in a
\family typewriter
*.load.gz
\family default
file are actually applied to the test candidate, you can (and should) re-record
the IO load during an execution of
\family typewriter
blkreplay
\family default
via
\family typewriter
blktrace
\family default
.
The following steps are to be done manually:
\end_layout
\begin_layout Enumerate
Start
\family typewriter
blktrace
\family default
manually, as described in chapter
\begin_inset CommandInset ref
LatexCommand ref
reference "chap:How-to-use"
\end_inset
,
\emph on
before
\emph default
you start
\family typewriter
tree-replay.sh
\family default
.
Be sure to catch all involved devices on all involved hosts.
\end_layout
\begin_layout Enumerate
Either kill
\family typewriter
blktrace
\family default
with Ctrl-C
\emph on
after
\emph default
tree-replay.sh is done, or use the
\family typewriter
-w
\family default
option to specify a
\emph on
strictly longer
\emph default
(plus some safety margin) recording time.
\begin_inset Newline newline
\end_inset
Attention! some versions of
\family typewriter
blktrace
\family default
seem to loose some requests in some buffers, at least under some conditions.
Usually the number of lost events is rather low, about 10.
Keep that in mind if you later discover some purpoted discrepancies!
\end_layout
\begin_layout Enumerate
Copy the resulting
\family typewriter
*.blktrace.[0-9]+
\family default
file to your workstation.
\end_layout
\begin_layout Enumerate
Invoke
\family typewriter
/path/to/blkreplay/scripts/check_replay_against_blktrace.sh
\emph on
myresult
\emph default
.replay.gz
\emph on
mytrace-prefix
\family default
\emph default
(exactly two parameters).
\begin_inset Newline newline
\end_inset
Hint: the coincidence between
\family typewriter
\emph on
myresult
\emph default
.replay.gz
\family default
and
\family typewriter
\emph on
myload
\emph default
.load.gz
\family default
must be checked separately, using tools like
\family typewriter
cut
\family default
and
\family typewriter
diff
\family default
.
\end_layout
\begin_layout Enumerate
Enjoy the result in form of a wide side-by-side
\family typewriter
diff
\family default
output.
\end_layout
\begin_layout Standard
Hint: no output means that no differences have been found.
The timeliness as well as the order of requests is
\emph on
not
\emph default
checked.
Any difference means that a request is missing somewhere, or has been altered
in some way.
Expect some minor differences (usully less than 1%, but sometimes more)
from request splitting by some device drivers.
\end_layout
\begin_layout Standard
Keep in mind that sometimes
\family typewriter
blktrace
\family default
looses some events, leading to
\begin_inset Quotes eld
\end_inset
false positives
\begin_inset Quotes erd
\end_inset
.
You should take this as a validation of the validation tool ;)
\end_layout
\begin_layout Section
Verbosity Graphics
\end_layout
\begin_layout Standard
Experts only.
It is easy to draw wrong conclusions from this.
\end_layout
\begin_layout Standard
Enable the following parameters:
\family typewriter
verbose=10;
\family default
\family typewriter
enable_compress_ssh=1; graph_options="--static --verbose"
\family default
in your respective
\family typewriter
*.conf
\family default
files.
\end_layout
\begin_layout Standard
The resulting
\family typewriter
*.replay.gz
\family default
file will now contain
\emph on
huge masses
\emph default
of lines starting with
\family typewriter
INFO:
\family default
and carrying lots of internal variables, similar to debug / tracing output.
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
The sheer masses of data written to
\family typewriter
stdout
\family default
(as such) my lead to distortions of your measurements, e.g.
artificial delays or even hangs.
Always compare to
\family typewriter
--verbose=0
\family default
under otherwise identical conditions, to be sure! In order to check whether
\family typewriter
stdout
\family default
could have become a bottleneck, please search for the lines with
\family typewriter
flush_total=n.nnnnnnnnn
\family default
(some timing with nanosecond resolution).
In particular, the main thread (additionally indicated by
\family typewriter
role='main'
\family default
) should have spent much less time in
\family typewriter
fflush(stdout)
\family default
than total running time.
When the network is the bottleneck,
\family typewriter
enable_compress_ssh=1
\family default
may help; otherwise disabling could be better -- please check.
\end_layout
\begin_layout Standard
\noindent
In addition, the following types of internal visualizations will be produced:
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.submit_level.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Shows the actual filling level of the pipelines between the main thread
and the workers.
The x axis is in units of requests, and in
\emph on
completion order
\emph default
(different from the submission order).
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g50.submit_level.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.pushback_level.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Shows the current number of requests on the pushback list.
Only meaningful at
\family typewriter
--with-partial
\family default
.
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.submit_ahead.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Shows the time difference between submission to the pipelines (which is
usually ahead of the execution time, but not always) and the intended execution
timestamp.
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g52.submit_ahead.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
Don't draw wrong conclusions from negative values here! They
\emph on
can
\emph default
be a sign of
\begin_inset Quotes eld
\end_inset
too late
\begin_inset Quotes erd
\end_inset
, but the
\emph on
reason
\emph default
is almost always in the test candidate, just when ordinary delays are produced.
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.submit_lag.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The time difference between submission to the pipelines, and reception by
the worker thread (aka communication latency).
\end_layout
\begin_layout Standard
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g53.submit_lag.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.submit_lag_cumul.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The same, but cumulized over time.
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g53.submit_lag_cumul.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
Notice than cumulation of time lags belonging to different threads is
\begin_inset Quotes eld
\end_inset
unfair
\begin_inset Quotes erd
\end_inset
by concept.
Large values can regularly occur because of queueing in the submit pipelines.
Complaints about high values make only sense in case of
\family typewriter
$(( bottleneck <= threads )).
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.answer_lag.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The time difference between submission to the answer pipeline caused by
a worker thread, and reception by the main thread (aka communication latency).
\end_layout
\begin_layout Standard
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g54.answer_lag.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.answer_lag_cumul.png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
Dito, cumulated over time.
\end_layout
\begin_layout Standard
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g54.answer_lag_cumul.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Labeling
\labelwidthstring 00.00.0000
\noindent
\begin_inset Graphics
filename images/MatieresCorrosives.png
lyxscale 50
scale 17
\end_inset
Similar warning as before: high values occur regularly in
\family typewriter
--with-ordering
\family default
mode.
Only be startled in
\emph on
some
\emph default
other cases!
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.input_wait[_cumul].png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The time differences before and after
\family typewriter
fgets()
\family default
(aka stalls in
\family typewriter
stdin
\family default
).
When these numbers get too high, this time you should be alerted quite
rightly.
\end_layout
\begin_layout Standard
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g55.input_wait.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g55.input_wait_cumul.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Description
\noindent
\family typewriter
*.answer_wait[_cumul].png
\family default
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
strut
\end_layout
\end_inset
\begin_inset Newline newline
\end_inset
The time difference of the main thread it is spending for answer wait.
Not to be confused with communication latencies! High number will occur
regularly.
It is even a sign of good performance when the cumulation almost reaches
the total processing time.
\end_layout
\begin_layout Standard
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g56.answer_wait.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Standard
\noindent
\begin_inset Graphics
filename ../example-run/example-overload/natural-derived/short/parallel-04/with-conflicts/threads-128/bottleneck/test-sata/verbose/bbu-1/EXAMPLE.natural-derived.short.parallel-04.with-conflicts.threads-128.bottleneck.test-sata.verbose.bbu-1.vg-test.g56.answer_wait_cumul.png
lyxscale 50
width 100col%
\end_inset
\end_layout
\begin_layout Chapter
GNU Free Documentation License
\begin_inset CommandInset label
LatexCommand label
name "chap:GNU-FDL"
\end_inset
\end_layout
\begin_layout Standard
\noindent
\family typewriter
\size footnotesize
\begin_inset ERT
status open
\begin_layout Plain Layout
\backslash
lstinputlisting{fdl.txt}
\end_layout
\end_inset
\end_layout
\end_body
\end_document