Debian Codegraph @

examples/tuple_collections.cc

raw
1 //
2 // The "geners" package provides two templates for efficient I/O of
3 // tuple collections. These collections support the data processing
4 // mode in which precisely one tuple (or a part of one tuple) is
5 // needed in order to perform some meaningful calculation. The relevant
6 // templates are
7 //
8 // RowPacker -- Persistent tuple collection optimized for data
9 // readback row-by-row. Should be used when calculations
10 // use the whole row or most part of it.
11 //
12 // ColumnPacker -- Persistent tuple collection in which entries for
13 // each variable (each tuple member, or spreadsheet
14 // column) are collected together and saved as soon
15 // as the amount of data in this column exceeds
16 // certain buffer size. This often results in better
17 // overall compression for large collections than
18 // buffering data row-by-row, and allows for efficient
19 // data access in each column separately.
20 //
21 // Both of these collections are not single monolithic objects. Instead,
22 // they consist of a number of data buffers stored in an archive, with
23 // only a small number of buffers present in the computer memory. For
24 // example, RowPacker keeps one or two buffers at a time in memory
25 // (current read buffer and current write buffer which are sometimes
26 // the same), while ColumnPacker keeps in memory one or two buffers
27 // per column (i.e., tuple variable). In this manner, the tuple
28 // collections can easily grow very large, typically limited only by
29 // the available amount of disk space.
30 //
31 // This example illustrates the usage of these collections. It can be
32 // compiled only if C++11 support is enabled.
33 //
34 #include <cmath>
35 #include <cassert>
36 #include <iostream>
37 #include <cstdlib>
38 #include <numeric>
39
40 #include "geners/RowPacker.hh"
41 #include "geners/ColumnPacker.hh"
42 #include "geners/IOPtr.hh"
43 #include "geners/CP_column_iterator.hh"
44 #include "geners/StringArchive.hh"
45
46 using namespace gs;
47 using namespace std;
48
49 #ifdef CPP11_STD_AVAILABLE
50
51 // IOProxy is a useful object which is essentially a "named silent pointer":
52 // it works just like IOPtr for I/O purposes and it also has a string to
53 // store the name of the variable (or "column"). The macro below builds
54 // a pointer to a certain variable and gives it a name of that variable.
55 // Together with IOPtr, IOProxy is defined inside the "IOPtr.hh" header.
56 #define io_proxy(obj) gs::make_IOProxy( obj , #obj )
57
58
59 // Function which generates some data for the purposes of this example
60 static void generate_some_data(int* i, int* j, double* d,
61 double* e, float* f, float* g)
62 {
63 *i = static_cast<int>(rand()*100.0/RAND_MAX);
64 *j = static_cast<int>(rand()*(-200.0)/RAND_MAX);
65 *d = rand()*2.0/RAND_MAX;
66 *e = rand()*3.0/RAND_MAX;
67 *f = rand()*4.0/RAND_MAX;
68 *g = rand()*5.0/RAND_MAX;
69 }
70
71
72 // Function which illustrates the use of a RowPacker
73 //
74 static void row_packer_example(AbsArchive& ar)
75 {
76 // Pretend that we need the folowing variables to perform a certain
77 // calculation
78 int i(0), j(0);
79 double d(0), e(0);
80 float f(0), g(0);
81
82 // Make a tuple which organizes the I/O of these variables
83 auto t1 = make_tuple(
84 io_proxy(i),
85 io_proxy(j),
86 io_proxy(d),
87 io_proxy(e),
88 io_proxy(f),
89 io_proxy(g)
90 );
91
92 // Make a RowPacker capable of storing multiple entries of this tuple.
93 // Naturally, other ways to construct a RowPacker are available, see
94 // "RowPacker.hh" header for details.
95 typedef RowPacker<decltype(t1)> Packer1;
96 Packer1 packer1("title", ar, "packer1", "top", t1);
97
98 // To check that read/write operations of RowPacker work correctly,
99 // let's also keep the packed data in a vector of data-holding tuples
100 auto t1_data = std::tuple<int, int, double, double, float, float>();
101 std::vector<decltype(t1_data)> vec;
102
103 // Example cycle that generates and stores the packer data
104 const unsigned nEntries = 1000;
105 for (unsigned icycle=0; icycle<nEntries; ++icycle)
106 {
107 generate_some_data(&i, &j, &d, &e, &f, &g);
108
109 // Store the variables calculated above. The "fill" function
110 // returns "true" on success and "false" on failure.
111 assert(packer1.fill(t1));
112
113 // And make a copy as well for future data verification
114 std::get<0>(t1_data) = i;
115 std::get<1>(t1_data) = j;
116 std::get<2>(t1_data) = d;
117 std::get<3>(t1_data) = e;
118 std::get<4>(t1_data) = f;
119 std::get<5>(t1_data) = g;
120 vec.push_back(t1_data);
121 }
122
123 // Call the following function to ensure that all packer buffers
124 // were written into the archive. You don't have to call the "write"
125 // function explicitly if the packer goes out of scope -- in this
126 // case all of its data is automatically written out.
127 packer1.write();
128
129 // The "write()" method should be called only after all necessary
130 // data has been added. The "write()" method tags the collection as
131 // "read only", and it is no longer possible to fill it. Any new
132 // invocation of the "fill" method will fail (returning "false")
133 // and will not change the collection contents.
134 assert(!packer1.fill(t1));
135
136 // Retrieve the packed data from the archive. For large tuple
137 // collections, RowPacker data buffers could be spread out across
138 // the whole archive, sharing the archive space with many other
139 // objects. Because of this, we need to use a special reference
140 // to retrieve this data called "RPReference" (the standard
141 // "Reference" template only works for stand-alone records).
142 // RowPacker can only be retrieved onto the heap, using either
143 // unique_ptr (with "get" method of RPReference) or shared_ptr
144 // (with "getShared" method).
145 //
146 unique_ptr<Packer1> readback = RPReference<Packer1>(
147 ar, "packer1", "top").get(0);
148
149 // Verify integrity of the retrieved data
150 assert(readback->nRows() == nEntries);
151 for (unsigned icycle=0; icycle<nEntries; ++icycle)
152 {
153 // Retrieve the data produced in the given cycle
154 readback->rowContents(icycle, &t1);
155
156 // Get a reference to the alternative copy of the data
157 // and make sure these two data sources agree
158 #ifndef NDEBUG
159 const decltype(t1_data)& t2(vec[icycle]);
160 assert(std::get<0>(t2) == i);
161 assert(std::get<1>(t2) == j);
162 assert(std::get<2>(t2) == d);
163 assert(std::get<3>(t2) == e);
164 assert(std::get<4>(t2) == f);
165 assert(std::get<5>(t2) == g);
166 #endif
167 }
168 }
169
170
171 // Function which illustrates the use of a ColumnPacker
172 //
173 static void column_packer_example(AbsArchive& ar)
174 {
175 // Pretend that we need the folowing variables to perform a certain
176 // calculation
177 int i(0), j(0);
178 double d(0), e(0);
179 float f(0), g(0);
180
181 // Make a tuple which organizes the I/O of these variables
182 auto t1 = make_tuple(
183 io_proxy(i),
184 io_proxy(j),
185 io_proxy(d),
186 io_proxy(e),
187 io_proxy(f),
188 io_proxy(g)
189 );
190
191 // Make a ColumnPacker capable of storing multiple entries of this tuple
192 // Naturally, other ways to construct a ColumnPacker are available, see
193 // "ColumnPacker.hh" header for details.
194 typedef ColumnPacker<decltype(t1)> Packer1;
195 Packer1 packer1("title", ar, "packer1", "top", t1);
196
197 // Example cycle that generates and stores the packer data
198 const unsigned nEntries = 1000;
199 for (unsigned icycle=0; icycle<nEntries; ++icycle)
200 {
201 generate_some_data(&i, &j, &d, &e, &f, &g);
202 packer1.fill(t1);
203 }
204
205 // Call the "write" function to ensure that all packer buffers
206 // were written into the archive. Naturally, the packer writes out
207 // all of its data automatically when it goes out of scope. After
208 // calling this method, it is no longer possible to modify the
209 // collection.
210 packer1.write();
211
212 // Retrieve the packed data from the archive. For large tuple
213 // collections, ColumnPacker data buffers could be spread out across
214 // the whole archive, sharing the archive space with many other
215 // objects. Because of this, we need to use a special reference
216 // to retrieve this data called "CPReference". Just as RowPacker,
217 // ColumnPacker can only be retrieved on the heap, either into
218 // unique_ptr or into shared_ptr.
219 //
220 unique_ptr<Packer1> readback = CPReference<Packer1>(
221 ar, "packer1", "top").get(0);
222
223 // So far, this example proceeded more or less along the lines
224 // of RowPacker example. However, ColumnPacker has a number
225 // of unique interface features. These features are illustrated
226 // below.
227
228 // ColumnPacker has an appropriate iterator template class,
229 // CP_column_iterator, capable of iterating over column contents.
230 // For example, the code snippet below will calculate the sum of
231 // all entries in column 2 using variable "temp" to store fetched
232 // column values when the iterator is dereferenced. Naturally,
233 // with a column iterator it becomes easy to use a number of other
234 // STL algorithms which do not modify the container as well as to
235 // fetch column contents into an std::vector.
236 //
237 // Column iterator works only forward (no operator--) and it is
238 // effectively const. When dereferenced, it refers to a copy of
239 // the data and not to actual ColumnPacker contents.
240 //
241 double temp(0);
242 const double sum2 = std::accumulate(
243 CP_column_begin<2>(*readback, make_IOPtr(temp)),
244 CP_column_end<2>(*readback, make_IOPtr(temp)), 0.0);
245
246 // Print the sum and its expected value (note that column 2 values
247 // are random numbers uniformly distributed between 0 and 2)
248 cout << "Sum of column 2 contents is " << sum2 << ", expected "
249 << nEntries << " +- " << 2.0/sqrt(12.0)*sqrt(nEntries) << endl;
250
251 // Another interesting feature of ColumnPacker is that it is possible
252 // to cycle over ColumnPacker rows in such a manner that only those
253 // columns which are needed in subsequent calculations are read back
254 // from the archive (which, of course, speeds up the data retrieval).
255 // In order to do this, one needs to call a special CPReference
256 // constructor which refers only to a few selected columns. Suppose,
257 // in some calculation we only need to use the info which was
258 // available in the original variables "i" and "d". Then one can
259 // proceed as follows:
260 int i2;
261 double d2;
262 auto tup = make_tuple(
263 make_IOProxy(i2, "i"),
264 make_IOProxy(d2, "d")
265 );
266 typedef ColumnPacker<decltype(tup)> Packer2;
267 std::unique_ptr<Packer2> readback2 = CPReference<Packer2>(
268 tup, ar, "packer1", "top").get(0);
269
270 // Now, cycling over "readback2" rows will fill out variables
271 // "i2" and "d2" with the data that originally was in "i" and "d".
272 // For example, let's check that we can reproduce the sum of the
273 // column 2 contents calculated above.
274 double sum = 0.0;
275 for (unsigned icycle=0; icycle<nEntries; ++icycle)
276 {
277 readback2->rowContents(icycle, &tup);
278 sum += d2;
279 }
280 assert(sum == sum2);
281 }
282
283
284 int main(int, char**)
285 {
286 StringArchive ar;
287
288 row_packer_example(ar);
289 column_packer_example(ar);
290
291 return 0;
292 }
293
294 #else // CPP11_STD_AVAILABLE
295
296 int main(int, char**)
297 {
298 cout << "To run this example, please recompile "
299 << "the package with C++11 features enabled" << endl;
300 return 0;
301 }
302
303 #endif // CPP11_STD_AVAILABLE