Debian Codegraph @

tsdb/index/tsi1/log_file_test.go

raw
1 package tsi1_test
2
3 import (
4 "bytes"
5 "fmt"
6 "io/ioutil"
7 "math/rand"
8 "os"
9 "path/filepath"
10 "regexp"
11 "runtime/pprof"
12 "sort"
13 "testing"
14 "time"
15
16 "github.com/influxdata/influxdb/models"
17 "github.com/influxdata/influxdb/pkg/bloom"
18 "github.com/influxdata/influxdb/tsdb"
19 "github.com/influxdata/influxdb/tsdb/index/tsi1"
20 )
21
22 // Ensure log file can append series.
23 func TestLogFile_AddSeriesList(t *testing.T) {
24 sfile := MustOpenSeriesFile()
25 defer sfile.Close()
26
27 f := MustOpenLogFile(sfile.SeriesFile)
28 defer f.Close()
29 seriesSet := tsdb.NewSeriesIDSet()
30
31 // Add test data.
32 if err := f.AddSeriesList(seriesSet, [][]byte{
33 []byte("mem"),
34 []byte("cpu"),
35 []byte("cpu"),
36 }, []models.Tags{
37 {{Key: []byte("host"), Value: []byte("serverA")}},
38 {{Key: []byte("region"), Value: []byte("us-east")}},
39 {{Key: []byte("region"), Value: []byte("us-west")}},
40 }); err != nil {
41 t.Fatal(err)
42 }
43
44 // Verify data.
45 itr := f.MeasurementIterator()
46 if e := itr.Next(); e == nil || string(e.Name()) != "cpu" {
47 t.Fatalf("unexpected measurement: %#v", e)
48 } else if e := itr.Next(); e == nil || string(e.Name()) != "mem" {
49 t.Fatalf("unexpected measurement: %#v", e)
50 } else if e := itr.Next(); e != nil {
51 t.Fatalf("expected eof, got: %#v", e)
52 }
53
54 // Reopen file and re-verify.
55 if err := f.Reopen(); err != nil {
56 t.Fatal(err)
57 }
58
59 // Verify data.
60 itr = f.MeasurementIterator()
61 if e := itr.Next(); e == nil || string(e.Name()) != "cpu" {
62 t.Fatalf("unexpected measurement: %#v", e)
63 } else if e := itr.Next(); e == nil || string(e.Name()) != "mem" {
64 t.Fatalf("unexpected measurement: %#v", e)
65 } else if e := itr.Next(); e != nil {
66 t.Fatalf("expected eof, got: %#v", e)
67 }
68 }
69
70 func TestLogFile_SeriesStoredInOrder(t *testing.T) {
71 sfile := MustOpenSeriesFile()
72 defer sfile.Close()
73
74 f := MustOpenLogFile(sfile.SeriesFile)
75 defer f.Close()
76 seriesSet := tsdb.NewSeriesIDSet()
77
78 // Generate and add test data
79 tvm := make(map[string]struct{})
80 rand.Seed(time.Now().Unix())
81 for i := 0; i < 100; i++ {
82 tv := fmt.Sprintf("server-%d", rand.Intn(50)) // Encourage adding duplicate series.
83 tvm[tv] = struct{}{}
84
85 if err := f.AddSeriesList(seriesSet, [][]byte{
86 []byte("mem"),
87 []byte("cpu"),
88 }, []models.Tags{
89 {models.NewTag([]byte("host"), []byte(tv))},
90 {models.NewTag([]byte("host"), []byte(tv))},
91 }); err != nil {
92 t.Fatal(err)
93 }
94 }
95
96 // Sort the tag values so we know what order to expect.
97 tvs := make([]string, 0, len(tvm))
98 for tv := range tvm {
99 tvs = append(tvs, tv)
100 }
101 sort.Strings(tvs)
102
103 // Double the series values since we're adding them twice (two measurements)
104 tvs = append(tvs, tvs...)
105
106 // When we pull the series out via an iterator they should be in order.
107 itr := f.SeriesIDIterator()
108 if itr == nil {
109 t.Fatal("nil iterator")
110 }
111
112 var prevSeriesID uint64
113 for i := 0; i < len(tvs); i++ {
114 elem, err := itr.Next()
115 if err != nil {
116 t.Fatal(err)
117 } else if elem.SeriesID == 0 {
118 t.Fatal("got nil series")
119 } else if elem.SeriesID < prevSeriesID {
120 t.Fatalf("series out of order: %d !< %d ", elem.SeriesID, prevSeriesID)
121 }
122 prevSeriesID = elem.SeriesID
123 }
124 }
125
126 // Ensure log file can delete an existing measurement.
127 func TestLogFile_DeleteMeasurement(t *testing.T) {
128 sfile := MustOpenSeriesFile()
129 defer sfile.Close()
130
131 f := MustOpenLogFile(sfile.SeriesFile)
132 defer f.Close()
133 seriesSet := tsdb.NewSeriesIDSet()
134
135 // Add test data.
136 if err := f.AddSeriesList(seriesSet, [][]byte{
137 []byte("mem"),
138 []byte("cpu"),
139 []byte("cpu"),
140 }, []models.Tags{
141 {{Key: []byte("host"), Value: []byte("serverA")}},
142 {{Key: []byte("region"), Value: []byte("us-east")}},
143 {{Key: []byte("region"), Value: []byte("us-west")}},
144 }); err != nil {
145 t.Fatal(err)
146 }
147
148 // Remove measurement.
149 if err := f.DeleteMeasurement([]byte("cpu")); err != nil {
150 t.Fatal(err)
151 }
152
153 // Verify data.
154 itr := f.MeasurementIterator()
155 if e := itr.Next(); string(e.Name()) != "cpu" || !e.Deleted() {
156 t.Fatalf("unexpected measurement: %s/%v", e.Name(), e.Deleted())
157 } else if e := itr.Next(); string(e.Name()) != "mem" || e.Deleted() {
158 t.Fatalf("unexpected measurement: %s/%v", e.Name(), e.Deleted())
159 } else if e := itr.Next(); e != nil {
160 t.Fatalf("expected eof, got: %#v", e)
161 }
162 }
163
164 // Ensure log file can recover correctly.
165 func TestLogFile_Open(t *testing.T) {
166 t.Run("Truncate", func(t *testing.T) {
167 sfile := MustOpenSeriesFile()
168 defer sfile.Close()
169 seriesSet := tsdb.NewSeriesIDSet()
170
171 f := MustOpenLogFile(sfile.SeriesFile)
172 defer f.Close()
173
174 // Add test data & close.
175 if err := f.AddSeriesList(seriesSet, [][]byte{[]byte("cpu"), []byte("mem")}, []models.Tags{{{}}, {{}}}); err != nil {
176 t.Fatal(err)
177 } else if err := f.LogFile.Close(); err != nil {
178 t.Fatal(err)
179 }
180
181 // Truncate data & reopen.
182 if fi, err := os.Stat(f.LogFile.Path()); err != nil {
183 t.Fatal(err)
184 } else if err := os.Truncate(f.LogFile.Path(), fi.Size()-1); err != nil {
185 t.Fatal(err)
186 } else if err := f.LogFile.Open(); err != nil {
187 t.Fatal(err)
188 }
189
190 // Verify data.
191 itr := f.SeriesIDIterator()
192 if elem, err := itr.Next(); err != nil {
193 t.Fatal(err)
194 } else if name, tags := sfile.Series(elem.SeriesID); string(name) != `cpu` {
195 t.Fatalf("unexpected series: %s,%s", name, tags.String())
196 } else if elem, err := itr.Next(); err != nil {
197 t.Fatal(err)
198 } else if elem.SeriesID != 0 {
199 t.Fatalf("expected eof, got: %#v", elem)
200 }
201
202 // Add more data & reopen.
203 if err := f.AddSeriesList(seriesSet, [][]byte{[]byte("disk")}, []models.Tags{{{}}}); err != nil {
204 t.Fatal(err)
205 } else if err := f.Reopen(); err != nil {
206 t.Fatal(err)
207 }
208
209 // Verify new data.
210 itr = f.SeriesIDIterator()
211 if elem, err := itr.Next(); err != nil {
212 t.Fatal(err)
213 } else if name, tags := sfile.Series(elem.SeriesID); string(name) != `cpu` {
214 t.Fatalf("unexpected series: %s,%s", name, tags.String())
215 } else if elem, err := itr.Next(); err != nil {
216 t.Fatal(err)
217 } else if name, tags := sfile.Series(elem.SeriesID); string(name) != `disk` {
218 t.Fatalf("unexpected series: %s,%s", name, tags.String())
219 } else if elem, err := itr.Next(); err != nil {
220 t.Fatal(err)
221 } else if elem.SeriesID != 0 {
222 t.Fatalf("expected eof, got: %#v", elem)
223 }
224 })
225
226 t.Run("ChecksumMismatch", func(t *testing.T) {
227 sfile := MustOpenSeriesFile()
228 defer sfile.Close()
229 seriesSet := tsdb.NewSeriesIDSet()
230
231 f := MustOpenLogFile(sfile.SeriesFile)
232 defer f.Close()
233
234 // Add test data & close.
235 if err := f.AddSeriesList(seriesSet, [][]byte{[]byte("cpu"), []byte("mem")}, []models.Tags{{{}}, {{}}}); err != nil {
236 t.Fatal(err)
237 } else if err := f.LogFile.Close(); err != nil {
238 t.Fatal(err)
239 }
240
241 // Corrupt last entry.
242 buf, err := ioutil.ReadFile(f.LogFile.Path())
243 if err != nil {
244 t.Fatal(err)
245 }
246 buf[len(buf)-1] = 0
247
248 // Overwrite file with corrupt entry and reopen.
249 if err := ioutil.WriteFile(f.LogFile.Path(), buf, 0666); err != nil {
250 t.Fatal(err)
251 } else if err := f.LogFile.Open(); err != nil {
252 t.Fatal(err)
253 }
254
255 // Verify data.
256 itr := f.SeriesIDIterator()
257 if elem, err := itr.Next(); err != nil {
258 t.Fatal(err)
259 } else if name, tags := sfile.Series(elem.SeriesID); string(name) != `cpu` {
260 t.Fatalf("unexpected series: %s,%s", name, tags.String())
261 } else if elem, err := itr.Next(); err != nil {
262 t.Fatal(err)
263 } else if elem.SeriesID != 0 {
264 t.Fatalf("expected eof, got: %#v", elem)
265 }
266 })
267 }
268
269 // LogFile is a test wrapper for tsi1.LogFile.
270 type LogFile struct {
271 *tsi1.LogFile
272 }
273
274 // NewLogFile returns a new instance of LogFile with a temporary file path.
275 func NewLogFile(sfile *tsdb.SeriesFile) *LogFile {
276 file, err := ioutil.TempFile("", "tsi1-log-file-")
277 if err != nil {
278 panic(err)
279 }
280 file.Close()
281
282 return &LogFile{LogFile: tsi1.NewLogFile(sfile, file.Name())}
283 }
284
285 // MustOpenLogFile returns a new, open instance of LogFile. Panic on error.
286 func MustOpenLogFile(sfile *tsdb.SeriesFile) *LogFile {
287 f := NewLogFile(sfile)
288 if err := f.Open(); err != nil {
289 panic(err)
290 }
291 return f
292 }
293
294 // Close closes the log file and removes it from disk.
295 func (f *LogFile) Close() error {
296 defer os.Remove(f.Path())
297 return f.LogFile.Close()
298 }
299
300 // Reopen closes and reopens the file.
301 func (f *LogFile) Reopen() error {
302 if err := f.LogFile.Close(); err != nil {
303 return err
304 }
305 if err := f.LogFile.Open(); err != nil {
306 return err
307 }
308 return nil
309 }
310
311 // CreateLogFile creates a new temporary log file and adds a list of series.
312 func CreateLogFile(sfile *tsdb.SeriesFile, series []Series) (*LogFile, error) {
313 f := MustOpenLogFile(sfile)
314 seriesSet := tsdb.NewSeriesIDSet()
315 for _, serie := range series {
316 if err := f.AddSeriesList(seriesSet, [][]byte{serie.Name}, []models.Tags{serie.Tags}); err != nil {
317 return nil, err
318 }
319 }
320 return f, nil
321 }
322
323 // GenerateLogFile generates a log file from a set of series based on the count arguments.
324 // Total series returned will equal measurementN * tagN * valueN.
325 func GenerateLogFile(sfile *tsdb.SeriesFile, measurementN, tagN, valueN int) (*LogFile, error) {
326 tagValueN := pow(valueN, tagN)
327
328 f := MustOpenLogFile(sfile)
329 seriesSet := tsdb.NewSeriesIDSet()
330 for i := 0; i < measurementN; i++ {
331 name := []byte(fmt.Sprintf("measurement%d", i))
332
333 // Generate tag sets.
334 for j := 0; j < tagValueN; j++ {
335 var tags models.Tags
336 for k := 0; k < tagN; k++ {
337 key := []byte(fmt.Sprintf("key%d", k))
338 value := []byte(fmt.Sprintf("value%d", (j / pow(valueN, k) % valueN)))
339 tags = append(tags, models.NewTag(key, value))
340 }
341 if err := f.AddSeriesList(seriesSet, [][]byte{name}, []models.Tags{tags}); err != nil {
342 return nil, err
343 }
344 }
345 }
346 return f, nil
347 }
348
349 func benchmarkLogFile_AddSeries(b *testing.B, measurementN, seriesKeyN, seriesValueN int) {
350 sfile := MustOpenSeriesFile()
351 defer sfile.Close()
352
353 b.StopTimer()
354 f := MustOpenLogFile(sfile.SeriesFile)
355 seriesSet := tsdb.NewSeriesIDSet()
356
357 type Datum struct {
358 Name []byte
359 Tags models.Tags
360 }
361
362 // Pre-generate everything.
363 var (
364 data []Datum
365 series int
366 )
367
368 tagValueN := pow(seriesValueN, seriesKeyN)
369
370 for i := 0; i < measurementN; i++ {
371 name := []byte(fmt.Sprintf("measurement%d", i))
372 for j := 0; j < tagValueN; j++ {
373 var tags models.Tags
374 for k := 0; k < seriesKeyN; k++ {
375 key := []byte(fmt.Sprintf("key%d", k))
376 value := []byte(fmt.Sprintf("value%d", (j / pow(seriesValueN, k) % seriesValueN)))
377 tags = append(tags, models.NewTag(key, value))
378 }
379 data = append(data, Datum{Name: name, Tags: tags})
380 series += len(tags)
381 }
382 }
383 b.StartTimer()
384 b.ResetTimer()
385
386 for i := 0; i < b.N; i++ {
387 for _, d := range data {
388 if err := f.AddSeriesList(seriesSet, [][]byte{d.Name}, []models.Tags{d.Tags}); err != nil {
389 b.Fatal(err)
390 }
391 }
392 }
393 }
394
395 func BenchmarkLogFile_AddSeries_100_1_1(b *testing.B) { benchmarkLogFile_AddSeries(b, 100, 1, 1) } // 100 series
396 func BenchmarkLogFile_AddSeries_1000_1_1(b *testing.B) { benchmarkLogFile_AddSeries(b, 1000, 1, 1) } // 1000 series
397 func BenchmarkLogFile_AddSeries_10000_1_1(b *testing.B) { benchmarkLogFile_AddSeries(b, 10000, 1, 1) } // 10000 series
398 func BenchmarkLogFile_AddSeries_100_2_10(b *testing.B) { benchmarkLogFile_AddSeries(b, 100, 2, 10) } // ~20K series
399 func BenchmarkLogFile_AddSeries_100000_1_1(b *testing.B) { benchmarkLogFile_AddSeries(b, 100000, 1, 1) } // ~100K series
400 func BenchmarkLogFile_AddSeries_100_3_7(b *testing.B) { benchmarkLogFile_AddSeries(b, 100, 3, 7) } // ~100K series
401 func BenchmarkLogFile_AddSeries_200_3_7(b *testing.B) { benchmarkLogFile_AddSeries(b, 200, 3, 7) } // ~200K series
402 func BenchmarkLogFile_AddSeries_200_4_7(b *testing.B) { benchmarkLogFile_AddSeries(b, 200, 4, 7) } // ~1.9M series
403
404 func BenchmarkLogFile_WriteTo(b *testing.B) {
405 for _, seriesN := range []int{1000, 10000, 100000, 1000000} {
406 name := fmt.Sprintf("series=%d", seriesN)
407 b.Run(name, func(b *testing.B) {
408 sfile := MustOpenSeriesFile()
409 defer sfile.Close()
410
411 f := MustOpenLogFile(sfile.SeriesFile)
412 defer f.Close()
413 seriesSet := tsdb.NewSeriesIDSet()
414
415 // Estimate bloom filter size.
416 m, k := bloom.Estimate(uint64(seriesN), 0.02)
417
418 // Initialize log file with series data.
419 for i := 0; i < seriesN; i++ {
420 if err := f.AddSeriesList(
421 seriesSet,
422 [][]byte{[]byte("cpu")},
423 []models.Tags{{
424 {Key: []byte("host"), Value: []byte(fmt.Sprintf("server-%d", i))},
425 {Key: []byte("location"), Value: []byte("us-west")},
426 }},
427 ); err != nil {
428 b.Fatal(err)
429 }
430 }
431 b.ResetTimer()
432
433 // Create cpu profile for each subtest.
434 MustStartCPUProfile(name)
435 defer pprof.StopCPUProfile()
436
437 // Compact log file.
438 for i := 0; i < b.N; i++ {
439 buf := bytes.NewBuffer(make([]byte, 0, 150*seriesN))
440 if _, err := f.CompactTo(buf, m, k, nil); err != nil {
441 b.Fatal(err)
442 }
443 b.Logf("sz=%db", buf.Len())
444 }
445 })
446 }
447 }
448
449 // MustStartCPUProfile starts a cpu profile in a temporary path based on name.
450 func MustStartCPUProfile(name string) {
451 name = regexp.MustCompile(`\W+`).ReplaceAllString(name, "-")
452
453 // Open file and start pprof.
454 f, err := os.Create(filepath.Join("/tmp", fmt.Sprintf("cpu-%s.pprof", name)))
455 if err != nil {
456 panic(err)
457 }
458 if err := pprof.StartCPUProfile(f); err != nil {
459 panic(err)
460 }
461 }