Debian Codegraph @

tsdb/index/tsi1/index_test.go

raw
1 package tsi1_test
2
3 import (
4 "fmt"
5 "io/ioutil"
6 "os"
7 "path/filepath"
8 "reflect"
9 "regexp"
10 "testing"
11
12 "github.com/influxdata/influxdb/models"
13 "github.com/influxdata/influxdb/tsdb/index/tsi1"
14 )
15
16 // Bloom filter settings used in tests.
17 const M, K = 4096, 6
18
19 // Ensure index can iterate over all measurement names.
20 func TestIndex_ForEachMeasurementName(t *testing.T) {
21 idx := MustOpenIndex(1)
22 defer idx.Close()
23
24 // Add series to index.
25 if err := idx.CreateSeriesSliceIfNotExists([]Series{
26 {Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "east"})},
27 {Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "west"})},
28 {Name: []byte("mem"), Tags: models.NewTags(map[string]string{"region": "east"})},
29 }); err != nil {
30 t.Fatal(err)
31 }
32
33 // Verify measurements are returned.
34 idx.Run(t, func(t *testing.T) {
35 var names []string
36 if err := idx.ForEachMeasurementName(func(name []byte) error {
37 names = append(names, string(name))
38 return nil
39 }); err != nil {
40 t.Fatal(err)
41 }
42
43 if !reflect.DeepEqual(names, []string{"cpu", "mem"}) {
44 t.Fatalf("unexpected names: %#v", names)
45 }
46 })
47
48 // Add more series.
49 if err := idx.CreateSeriesSliceIfNotExists([]Series{
50 {Name: []byte("disk")},
51 {Name: []byte("mem")},
52 }); err != nil {
53 t.Fatal(err)
54 }
55
56 // Verify new measurements.
57 idx.Run(t, func(t *testing.T) {
58 var names []string
59 if err := idx.ForEachMeasurementName(func(name []byte) error {
60 names = append(names, string(name))
61 return nil
62 }); err != nil {
63 t.Fatal(err)
64 }
65
66 if !reflect.DeepEqual(names, []string{"cpu", "disk", "mem"}) {
67 t.Fatalf("unexpected names: %#v", names)
68 }
69 })
70 }
71
72 // Ensure index can return whether a measurement exists.
73 func TestIndex_MeasurementExists(t *testing.T) {
74 idx := MustOpenIndex(1)
75 defer idx.Close()
76
77 // Add series to index.
78 if err := idx.CreateSeriesSliceIfNotExists([]Series{
79 {Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "east"})},
80 {Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "west"})},
81 }); err != nil {
82 t.Fatal(err)
83 }
84
85 // Verify measurement exists.
86 idx.Run(t, func(t *testing.T) {
87 if v, err := idx.MeasurementExists([]byte("cpu")); err != nil {
88 t.Fatal(err)
89 } else if !v {
90 t.Fatal("expected measurement to exist")
91 }
92 })
93
94 name, tags := []byte("cpu"), models.NewTags(map[string]string{"region": "east"})
95 sid := idx.Index.SeriesFile().SeriesID(name, tags, nil)
96 if sid == 0 {
97 t.Fatalf("got 0 series id for %s/%v", name, tags)
98 }
99
100 // Delete one series.
101 if err := idx.DropSeries(sid, models.MakeKey(name, tags), true); err != nil {
102 t.Fatal(err)
103 }
104
105 // Verify measurement still exists.
106 idx.Run(t, func(t *testing.T) {
107 if v, err := idx.MeasurementExists([]byte("cpu")); err != nil {
108 t.Fatal(err)
109 } else if !v {
110 t.Fatal("expected measurement to still exist")
111 }
112 })
113
114 // Delete second series.
115 tags.Set([]byte("region"), []byte("west"))
116 sid = idx.Index.SeriesFile().SeriesID(name, tags, nil)
117 if sid == 0 {
118 t.Fatalf("got 0 series id for %s/%v", name, tags)
119 }
120 if err := idx.DropSeries(sid, models.MakeKey(name, tags), true); err != nil {
121 t.Fatal(err)
122 }
123
124 // Verify measurement is now deleted.
125 idx.Run(t, func(t *testing.T) {
126 if v, err := idx.MeasurementExists([]byte("cpu")); err != nil {
127 t.Fatal(err)
128 } else if v {
129 t.Fatal("expected measurement to be deleted")
130 }
131 })
132 }
133
134 // Ensure index can return a list of matching measurements.
135 func TestIndex_MeasurementNamesByRegex(t *testing.T) {
136 idx := MustOpenIndex(1)
137 defer idx.Close()
138
139 // Add series to index.
140 if err := idx.CreateSeriesSliceIfNotExists([]Series{
141 {Name: []byte("cpu")},
142 {Name: []byte("disk")},
143 {Name: []byte("mem")},
144 }); err != nil {
145 t.Fatal(err)
146 }
147
148 // Retrieve measurements by regex.
149 idx.Run(t, func(t *testing.T) {
150 names, err := idx.MeasurementNamesByRegex(regexp.MustCompile(`cpu|mem`))
151 if err != nil {
152 t.Fatal(err)
153 } else if !reflect.DeepEqual(names, [][]byte{[]byte("cpu"), []byte("mem")}) {
154 t.Fatalf("unexpected names: %v", names)
155 }
156 })
157 }
158
159 // Ensure index can delete a measurement and all related keys, values, & series.
160 func TestIndex_DropMeasurement(t *testing.T) {
161 idx := MustOpenIndex(1)
162 defer idx.Close()
163
164 // Add series to index.
165 if err := idx.CreateSeriesSliceIfNotExists([]Series{
166 {Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "east"})},
167 {Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "west"})},
168 {Name: []byte("disk"), Tags: models.NewTags(map[string]string{"region": "north"})},
169 {Name: []byte("mem"), Tags: models.NewTags(map[string]string{"region": "west", "country": "us"})},
170 }); err != nil {
171 t.Fatal(err)
172 }
173
174 // Drop measurement.
175 if err := idx.DropMeasurement([]byte("cpu")); err != nil {
176 t.Fatal(err)
177 }
178
179 // Verify data is gone in each stage.
180 idx.Run(t, func(t *testing.T) {
181 // Verify measurement is gone.
182 if v, err := idx.MeasurementExists([]byte("cpu")); err != nil {
183 t.Fatal(err)
184 } else if v {
185 t.Fatal("expected no measurement")
186 }
187
188 // Obtain file set to perform lower level checks.
189 fs, err := idx.PartitionAt(0).RetainFileSet()
190 if err != nil {
191 t.Fatal(err)
192 }
193 defer fs.Release()
194
195 // Verify tags & values are gone.
196 if e := fs.TagKeyIterator([]byte("cpu")).Next(); e != nil && !e.Deleted() {
197 t.Fatal("expected deleted tag key")
198 }
199 if itr := fs.TagValueIterator([]byte("cpu"), []byte("region")); itr != nil {
200 t.Fatal("expected nil tag value iterator")
201 }
202
203 })
204 }
205
206 func TestIndex_Open(t *testing.T) {
207 // Opening a fresh index should set the MANIFEST version to current version.
208 idx := NewIndex(tsi1.DefaultPartitionN)
209 t.Run("open new index", func(t *testing.T) {
210 if err := idx.Open(); err != nil {
211 t.Fatal(err)
212 }
213
214 // Check version set appropriately.
215 for i := 0; uint64(i) < tsi1.DefaultPartitionN; i++ {
216 partition := idx.PartitionAt(i)
217 if got, exp := partition.Manifest().Version, 1; got != exp {
218 t.Fatalf("got index version %d, expected %d", got, exp)
219 }
220 }
221 })
222
223 // Reopening an open index should return an error.
224 t.Run("reopen open index", func(t *testing.T) {
225 err := idx.Open()
226 if err == nil {
227 idx.Close()
228 t.Fatal("didn't get an error on reopen, but expected one")
229 }
230 idx.Close()
231 })
232
233 // Opening an incompatible index should return an error.
234 incompatibleVersions := []int{-1, 0, 2}
235 for _, v := range incompatibleVersions {
236 t.Run(fmt.Sprintf("incompatible index version: %d", v), func(t *testing.T) {
237 idx = NewIndex(tsi1.DefaultPartitionN)
238 // Manually create a MANIFEST file for an incompatible index version.
239 // under one of the partitions.
240 partitionPath := filepath.Join(idx.Path(), "2")
241 os.MkdirAll(partitionPath, 0777)
242
243 mpath := filepath.Join(partitionPath, tsi1.ManifestFileName)
244 m := tsi1.NewManifest(mpath)
245 m.Levels = nil
246 m.Version = v // Set example MANIFEST version.
247 if _, err := m.Write(); err != nil {
248 t.Fatal(err)
249 }
250
251 // Log the MANIFEST file.
252 data, err := ioutil.ReadFile(mpath)
253 if err != nil {
254 panic(err)
255 }
256 t.Logf("Incompatible MANIFEST: %s", data)
257
258 // Opening this index should return an error because the MANIFEST has an
259 // incompatible version.
260 err = idx.Open()
261 if err != tsi1.ErrIncompatibleVersion {
262 idx.Close()
263 t.Fatalf("got error %v, expected %v", err, tsi1.ErrIncompatibleVersion)
264 }
265 })
266 }
267 }
268
269 func TestIndex_Manifest(t *testing.T) {
270 t.Run("current MANIFEST", func(t *testing.T) {
271 idx := MustOpenIndex(tsi1.DefaultPartitionN)
272
273 // Check version set appropriately.
274 for i := 0; uint64(i) < tsi1.DefaultPartitionN; i++ {
275 partition := idx.PartitionAt(i)
276 if got, exp := partition.Manifest().Version, tsi1.Version; got != exp {
277 t.Fatalf("got MANIFEST version %d, expected %d", got, exp)
278 }
279 }
280 })
281 }
282
283 func TestIndex_DiskSizeBytes(t *testing.T) {
284 idx := MustOpenIndex(tsi1.DefaultPartitionN)
285 defer idx.Close()
286
287 // Add series to index.
288 if err := idx.CreateSeriesSliceIfNotExists([]Series{
289 {Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "east"})},
290 {Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "west"})},
291 {Name: []byte("disk"), Tags: models.NewTags(map[string]string{"region": "north"})},
292 {Name: []byte("mem"), Tags: models.NewTags(map[string]string{"region": "west", "country": "us"})},
293 }); err != nil {
294 t.Fatal(err)
295 }
296
297 // Verify on disk size is the same in each stage.
298 // Each series stores flag(1) + series(uvarint(2)) + len(name)(1) + len(key)(1) + len(value)(1) + checksum(4).
299 expSize := int64(4 * 9)
300
301 // Each MANIFEST file is 419 bytes and there are tsi1.DefaultPartitionN of them
302 expSize += int64(tsi1.DefaultPartitionN * 419)
303
304 idx.Run(t, func(t *testing.T) {
305 if got, exp := idx.DiskSizeBytes(), expSize; got != exp {
306 t.Fatalf("got %d bytes, expected %d", got, exp)
307 }
308 })
309 }
310
311 // Index is a test wrapper for tsi1.Index.
312 type Index struct {
313 *tsi1.Index
314 SeriesFile *SeriesFile
315 }
316
317 // NewIndex returns a new instance of Index at a temporary path.
318 func NewIndex(partitionN uint64) *Index {
319 idx := &Index{SeriesFile: NewSeriesFile()}
320 idx.Index = tsi1.NewIndex(idx.SeriesFile.SeriesFile, "db0", tsi1.WithPath(MustTempDir()))
321 idx.Index.PartitionN = partitionN
322 return idx
323 }
324
325 // MustOpenIndex returns a new, open index. Panic on error.
326 func MustOpenIndex(partitionN uint64) *Index {
327 idx := NewIndex(partitionN)
328 if err := idx.Open(); err != nil {
329 panic(err)
330 }
331 return idx
332 }
333
334 // Open opens the underlying tsi1.Index and tsdb.SeriesFile
335 func (idx Index) Open() error {
336 if err := idx.SeriesFile.Open(); err != nil {
337 return err
338 }
339 return idx.Index.Open()
340 }
341
342 // Close closes and removes the index directory.
343 func (idx *Index) Close() error {
344 defer os.RemoveAll(idx.Path())
345 if err := idx.SeriesFile.Close(); err != nil {
346 return err
347 }
348 return idx.Index.Close()
349 }
350
351 // Reopen closes and opens the index.
352 func (idx *Index) Reopen() error {
353 if err := idx.Index.Close(); err != nil {
354 return err
355 }
356
357 // Reopen the series file correctly, by initialising a new underlying series
358 // file using the same disk data.
359 if err := idx.SeriesFile.Reopen(); err != nil {
360 return err
361 }
362
363 partitionN := idx.Index.PartitionN // Remember how many partitions to use.
364 idx.Index = tsi1.NewIndex(idx.SeriesFile.SeriesFile, "db0", tsi1.WithPath(idx.Index.Path()))
365 idx.Index.PartitionN = partitionN
366 return idx.Open()
367 }
368
369 // Run executes a subtest for each of several different states:
370 //
371 // - Immediately
372 // - After reopen
373 // - After compaction
374 // - After reopen again
375 //
376 // The index should always respond in the same fashion regardless of
377 // how data is stored. This helper allows the index to be easily tested
378 // in all major states.
379 func (idx *Index) Run(t *testing.T, fn func(t *testing.T)) {
380 // Invoke immediately.
381 t.Run("state=initial", fn)
382
383 // Reopen and invoke again.
384 if err := idx.Reopen(); err != nil {
385 t.Fatalf("reopen error: %s", err)
386 }
387 t.Run("state=reopen", fn)
388
389 // TODO: Request a compaction.
390 // if err := idx.Compact(); err != nil {
391 // t.Fatalf("compact error: %s", err)
392 // }
393 // t.Run("state=post-compaction", fn)
394
395 // Reopen and invoke again.
396 if err := idx.Reopen(); err != nil {
397 t.Fatalf("post-compaction reopen error: %s", err)
398 }
399 t.Run("state=post-compaction-reopen", fn)
400 }
401
402 // CreateSeriesSliceIfNotExists creates multiple series at a time.
403 func (idx *Index) CreateSeriesSliceIfNotExists(a []Series) error {
404 keys := make([][]byte, 0, len(a))
405 names := make([][]byte, 0, len(a))
406 tags := make([]models.Tags, 0, len(a))
407 for _, s := range a {
408 keys = append(keys, models.MakeKey(s.Name, s.Tags))
409 names = append(names, s.Name)
410 tags = append(tags, s.Tags)
411 }
412 return idx.CreateSeriesListIfNotExists(keys, names, tags)
413 }