Debian Codegraph @

tsdb/index/tsi1/file_set.go

raw
1 package tsi1
2
3 import (
4 "bytes"
5 "fmt"
6 "regexp"
7 "sync"
8 "unsafe"
9
10 "github.com/influxdata/influxdb/pkg/estimator"
11 "github.com/influxdata/influxdb/pkg/estimator/hll"
12 "github.com/influxdata/influxdb/tsdb"
13 "github.com/influxdata/influxql"
14 )
15
16 // FileSet represents a collection of files.
17 type FileSet struct {
18 levels []CompactionLevel
19 sfile *tsdb.SeriesFile
20 files []File
21 manifestSize int64 // Size of the manifest file in bytes.
22 }
23
24 // NewFileSet returns a new instance of FileSet.
25 func NewFileSet(levels []CompactionLevel, sfile *tsdb.SeriesFile, files []File) (*FileSet, error) {
26 return &FileSet{
27 levels: levels,
28 sfile: sfile,
29 files: files,
30 }, nil
31 }
32
33 // bytes estimates the memory footprint of this FileSet, in bytes.
34 func (fs *FileSet) bytes() int {
35 var b int
36 for _, level := range fs.levels {
37 b += int(unsafe.Sizeof(level))
38 }
39 // Do not count SeriesFile because it belongs to the code that constructed this FileSet.
40 for _, file := range fs.files {
41 b += file.bytes()
42 }
43 b += int(unsafe.Sizeof(*fs))
44 return b
45 }
46
47 // Close closes all the files in the file set.
48 func (fs FileSet) Close() error {
49 var err error
50 for _, f := range fs.files {
51 if e := f.Close(); e != nil && err == nil {
52 err = e
53 }
54 }
55 return err
56 }
57
58 // Retain adds a reference count to all files.
59 func (fs *FileSet) Retain() {
60 for _, f := range fs.files {
61 f.Retain()
62 }
63 }
64
65 // Release removes a reference count from all files.
66 func (fs *FileSet) Release() {
67 for _, f := range fs.files {
68 f.Release()
69 }
70 }
71
72 // SeriesFile returns the attached series file.
73 func (fs *FileSet) SeriesFile() *tsdb.SeriesFile { return fs.sfile }
74
75 // PrependLogFile returns a new file set with f added at the beginning.
76 // Filters do not need to be rebuilt because log files have no bloom filter.
77 func (fs *FileSet) PrependLogFile(f *LogFile) *FileSet {
78 return &FileSet{
79 levels: fs.levels,
80 sfile: fs.sfile,
81 files: append([]File{f}, fs.files...),
82 }
83 }
84
85 // Size returns the on-disk size of the FileSet.
86 func (fs *FileSet) Size() int64 {
87 var total int64
88 for _, f := range fs.files {
89 total += f.Size()
90 }
91 return total + int64(fs.manifestSize)
92 }
93
94 // MustReplace swaps a list of files for a single file and returns a new file set.
95 // The caller should always guarantee that the files exist and are contiguous.
96 func (fs *FileSet) MustReplace(oldFiles []File, newFile File) *FileSet {
97 assert(len(oldFiles) > 0, "cannot replace empty files")
98
99 // Find index of first old file.
100 var i int
101 for ; i < len(fs.files); i++ {
102 if fs.files[i] == oldFiles[0] {
103 break
104 } else if i == len(fs.files)-1 {
105 panic("first replacement file not found")
106 }
107 }
108
109 // Ensure all old files are contiguous.
110 for j := range oldFiles {
111 if fs.files[i+j] != oldFiles[j] {
112 panic(fmt.Sprintf("cannot replace non-contiguous files: subset=%+v, fileset=%+v", Files(oldFiles).IDs(), Files(fs.files).IDs()))
113 }
114 }
115
116 // Copy to new fileset.
117 other := make([]File, len(fs.files)-len(oldFiles)+1)
118 copy(other[:i], fs.files[:i])
119 other[i] = newFile
120 copy(other[i+1:], fs.files[i+len(oldFiles):])
121
122 // Build new fileset and rebuild changed filters.
123 return &FileSet{
124 levels: fs.levels,
125 files: other,
126 }
127 }
128
129 // MaxID returns the highest file identifier.
130 func (fs *FileSet) MaxID() int {
131 var max int
132 for _, f := range fs.files {
133 if i := f.ID(); i > max {
134 max = i
135 }
136 }
137 return max
138 }
139
140 // Files returns all files in the set.
141 func (fs *FileSet) Files() []File {
142 return fs.files
143 }
144
145 // LogFiles returns all log files from the file set.
146 func (fs *FileSet) LogFiles() []*LogFile {
147 var a []*LogFile
148 for _, f := range fs.files {
149 if f, ok := f.(*LogFile); ok {
150 a = append(a, f)
151 }
152 }
153 return a
154 }
155
156 // IndexFiles returns all index files from the file set.
157 func (fs *FileSet) IndexFiles() []*IndexFile {
158 var a []*IndexFile
159 for _, f := range fs.files {
160 if f, ok := f.(*IndexFile); ok {
161 a = append(a, f)
162 }
163 }
164 return a
165 }
166
167 // LastContiguousIndexFilesByLevel returns the last contiguous files by level.
168 // These can be used by the compaction scheduler.
169 func (fs *FileSet) LastContiguousIndexFilesByLevel(level int) []*IndexFile {
170 if level == 0 {
171 return nil
172 }
173
174 var a []*IndexFile
175 for i := len(fs.files) - 1; i >= 0; i-- {
176 f := fs.files[i]
177
178 // Ignore files above level, stop on files below level.
179 if level < f.Level() {
180 continue
181 } else if level > f.Level() {
182 break
183 }
184
185 a = append([]*IndexFile{f.(*IndexFile)}, a...)
186 }
187 return a
188 }
189
190 /*
191 // SeriesIDIterator returns an iterator over all series in the index.
192 func (fs *FileSet) SeriesIDIterator() tsdb.SeriesIDIterator {
193 a := make([]tsdb.SeriesIDIterator, 0, len(fs.files))
194 for _, f := range fs.files {
195 itr := f.SeriesIDIterator()
196 if itr == nil {
197 continue
198 }
199 a = append(a, itr)
200 }
201 return FilterUndeletedSeriesIterator(MergeSeriesIterators(a...))
202 }
203 */
204
205 // Measurement returns a measurement by name.
206 func (fs *FileSet) Measurement(name []byte) MeasurementElem {
207 for _, f := range fs.files {
208 if e := f.Measurement(name); e == nil {
209 continue
210 } else if e.Deleted() {
211 return nil
212 } else {
213 return e
214 }
215 }
216 return nil
217 }
218
219 // MeasurementIterator returns an iterator over all measurements in the index.
220 func (fs *FileSet) MeasurementIterator() MeasurementIterator {
221 a := make([]MeasurementIterator, 0, len(fs.files))
222 for _, f := range fs.files {
223 itr := f.MeasurementIterator()
224 if itr != nil {
225 a = append(a, itr)
226 }
227 }
228 return MergeMeasurementIterators(a...)
229 }
230
231 // TagKeyIterator returns an iterator over all tag keys for a measurement.
232 func (fs *FileSet) TagKeyIterator(name []byte) TagKeyIterator {
233 a := make([]TagKeyIterator, 0, len(fs.files))
234 for _, f := range fs.files {
235 itr := f.TagKeyIterator(name)
236 if itr != nil {
237 a = append(a, itr)
238 }
239 }
240 return MergeTagKeyIterators(a...)
241 }
242
243 // MeasurementSeriesIDIterator returns a series iterator for a measurement.
244 func (fs *FileSet) MeasurementSeriesIDIterator(name []byte) tsdb.SeriesIDIterator {
245 a := make([]tsdb.SeriesIDIterator, 0, len(fs.files))
246 for _, f := range fs.files {
247 itr := f.MeasurementSeriesIDIterator(name)
248 if itr != nil {
249 a = append(a, itr)
250 }
251 }
252 return tsdb.MergeSeriesIDIterators(a...)
253 }
254
255 // MeasurementTagKeysByExpr extracts the tag keys wanted by the expression.
256 func (fs *FileSet) MeasurementTagKeysByExpr(name []byte, expr influxql.Expr) (map[string]struct{}, error) {
257 // Return all keys if no condition was passed in.
258 if expr == nil {
259 m := make(map[string]struct{})
260 if itr := fs.TagKeyIterator(name); itr != nil {
261 for e := itr.Next(); e != nil; e = itr.Next() {
262 m[string(e.Key())] = struct{}{}
263 }
264 }
265 return m, nil
266 }
267
268 switch e := expr.(type) {
269 case *influxql.BinaryExpr:
270 switch e.Op {
271 case influxql.EQ, influxql.NEQ, influxql.EQREGEX, influxql.NEQREGEX:
272 tag, ok := e.LHS.(*influxql.VarRef)
273 if !ok {
274 return nil, fmt.Errorf("left side of '%s' must be a tag key", e.Op.String())
275 } else if tag.Val != "_tagKey" {
276 return nil, nil
277 }
278
279 if influxql.IsRegexOp(e.Op) {
280 re, ok := e.RHS.(*influxql.RegexLiteral)
281 if !ok {
282 return nil, fmt.Errorf("right side of '%s' must be a regular expression", e.Op.String())
283 }
284 return fs.tagKeysByFilter(name, e.Op, nil, re.Val), nil
285 }
286
287 s, ok := e.RHS.(*influxql.StringLiteral)
288 if !ok {
289 return nil, fmt.Errorf("right side of '%s' must be a tag value string", e.Op.String())
290 }
291 return fs.tagKeysByFilter(name, e.Op, []byte(s.Val), nil), nil
292
293 case influxql.AND, influxql.OR:
294 lhs, err := fs.MeasurementTagKeysByExpr(name, e.LHS)
295 if err != nil {
296 return nil, err
297 }
298
299 rhs, err := fs.MeasurementTagKeysByExpr(name, e.RHS)
300 if err != nil {
301 return nil, err
302 }
303
304 if lhs != nil && rhs != nil {
305 if e.Op == influxql.OR {
306 return unionStringSets(lhs, rhs), nil
307 }
308 return intersectStringSets(lhs, rhs), nil
309 } else if lhs != nil {
310 return lhs, nil
311 } else if rhs != nil {
312 return rhs, nil
313 }
314 return nil, nil
315 default:
316 return nil, fmt.Errorf("invalid operator")
317 }
318
319 case *influxql.ParenExpr:
320 return fs.MeasurementTagKeysByExpr(name, e.Expr)
321 }
322
323 return nil, fmt.Errorf("%#v", expr)
324 }
325
326 // tagKeysByFilter will filter the tag keys for the measurement.
327 func (fs *FileSet) tagKeysByFilter(name []byte, op influxql.Token, val []byte, regex *regexp.Regexp) map[string]struct{} {
328 ss := make(map[string]struct{})
329 itr := fs.TagKeyIterator(name)
330 if itr != nil {
331 for e := itr.Next(); e != nil; e = itr.Next() {
332 var matched bool
333 switch op {
334 case influxql.EQ:
335 matched = bytes.Equal(e.Key(), val)
336 case influxql.NEQ:
337 matched = !bytes.Equal(e.Key(), val)
338 case influxql.EQREGEX:
339 matched = regex.Match(e.Key())
340 case influxql.NEQREGEX:
341 matched = !regex.Match(e.Key())
342 }
343
344 if !matched {
345 continue
346 }
347 ss[string(e.Key())] = struct{}{}
348 }
349 }
350 return ss
351 }
352
353 // TagKeySeriesIDIterator returns a series iterator for all values across a single key.
354 func (fs *FileSet) TagKeySeriesIDIterator(name, key []byte) tsdb.SeriesIDIterator {
355 a := make([]tsdb.SeriesIDIterator, 0, len(fs.files))
356 for _, f := range fs.files {
357 itr := f.TagKeySeriesIDIterator(name, key)
358 if itr != nil {
359 a = append(a, itr)
360 }
361 }
362 return tsdb.MergeSeriesIDIterators(a...)
363 }
364
365 // HasTagKey returns true if the tag key exists.
366 func (fs *FileSet) HasTagKey(name, key []byte) bool {
367 for _, f := range fs.files {
368 if e := f.TagKey(name, key); e != nil {
369 return !e.Deleted()
370 }
371 }
372 return false
373 }
374
375 // HasTagValue returns true if the tag value exists.
376 func (fs *FileSet) HasTagValue(name, key, value []byte) bool {
377 for _, f := range fs.files {
378 if e := f.TagValue(name, key, value); e != nil {
379 return !e.Deleted()
380 }
381 }
382 return false
383 }
384
385 // TagValueIterator returns a value iterator for a tag key.
386 func (fs *FileSet) TagValueIterator(name, key []byte) TagValueIterator {
387 a := make([]TagValueIterator, 0, len(fs.files))
388 for _, f := range fs.files {
389 itr := f.TagValueIterator(name, key)
390 if itr != nil {
391 a = append(a, itr)
392 }
393 }
394 return MergeTagValueIterators(a...)
395 }
396
397 // TagValueSeriesIDIterator returns a series iterator for a single tag value.
398 func (fs *FileSet) TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator {
399 a := make([]tsdb.SeriesIDIterator, 0, len(fs.files))
400 for _, f := range fs.files {
401 itr := f.TagValueSeriesIDIterator(name, key, value)
402 if itr != nil {
403 a = append(a, itr)
404 }
405 }
406 return tsdb.MergeSeriesIDIterators(a...)
407 }
408
409 // MeasurementsSketches returns the merged measurement sketches for the FileSet.
410 func (fs *FileSet) MeasurementsSketches() (estimator.Sketch, estimator.Sketch, error) {
411 sketch, tSketch := hll.NewDefaultPlus(), hll.NewDefaultPlus()
412 for _, f := range fs.files {
413 if s, t, err := f.MeasurementsSketches(); err != nil {
414 return nil, nil, err
415 } else if err := sketch.Merge(s); err != nil {
416 return nil, nil, err
417 } else if err := tSketch.Merge(t); err != nil {
418 return nil, nil, err
419 }
420 }
421 return sketch, tSketch, nil
422 }
423
424 // SeriesSketches returns the merged measurement sketches for the FileSet.
425 func (fs *FileSet) SeriesSketches() (estimator.Sketch, estimator.Sketch, error) {
426 sketch, tSketch := hll.NewDefaultPlus(), hll.NewDefaultPlus()
427 for _, f := range fs.files {
428 if s, t, err := f.SeriesSketches(); err != nil {
429 return nil, nil, err
430 } else if err := sketch.Merge(s); err != nil {
431 return nil, nil, err
432 } else if err := tSketch.Merge(t); err != nil {
433 return nil, nil, err
434 }
435 }
436 return sketch, tSketch, nil
437 }
438
439 // File represents a log or index file.
440 type File interface {
441 Close() error
442 Path() string
443
444 ID() int
445 Level() int
446
447 Measurement(name []byte) MeasurementElem
448 MeasurementIterator() MeasurementIterator
449 MeasurementHasSeries(ss *tsdb.SeriesIDSet, name []byte) bool
450
451 TagKey(name, key []byte) TagKeyElem
452 TagKeyIterator(name []byte) TagKeyIterator
453
454 TagValue(name, key, value []byte) TagValueElem
455 TagValueIterator(name, key []byte) TagValueIterator
456
457 // Series iteration.
458 MeasurementSeriesIDIterator(name []byte) tsdb.SeriesIDIterator
459 TagKeySeriesIDIterator(name, key []byte) tsdb.SeriesIDIterator
460 TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator
461
462 // Sketches for cardinality estimation
463 MeasurementsSketches() (s, t estimator.Sketch, err error)
464 SeriesSketches() (s, t estimator.Sketch, err error)
465
466 // Bitmap series existance.
467 SeriesIDSet() (*tsdb.SeriesIDSet, error)
468 TombstoneSeriesIDSet() (*tsdb.SeriesIDSet, error)
469
470 // Reference counting.
471 Retain()
472 Release()
473
474 // Size of file on disk
475 Size() int64
476
477 // Estimated memory footprint
478 bytes() int
479 }
480
481 type Files []File
482
483 func (a Files) IDs() []int {
484 ids := make([]int, len(a))
485 for i := range a {
486 ids[i] = a[i].ID()
487 }
488 return ids
489 }
490
491 // fileSetSeriesIDIterator attaches a fileset to an iterator that is released on close.
492 type fileSetSeriesIDIterator struct {
493 once sync.Once
494 fs *FileSet
495 itr tsdb.SeriesIDIterator
496 }
497
498 func newFileSetSeriesIDIterator(fs *FileSet, itr tsdb.SeriesIDIterator) tsdb.SeriesIDIterator {
499 if itr == nil {
500 fs.Release()
501 return nil
502 }
503 if itr, ok := itr.(tsdb.SeriesIDSetIterator); ok {
504 return &fileSetSeriesIDSetIterator{fs: fs, itr: itr}
505 }
506 return &fileSetSeriesIDIterator{fs: fs, itr: itr}
507 }
508
509 func (itr *fileSetSeriesIDIterator) Next() (tsdb.SeriesIDElem, error) {
510 return itr.itr.Next()
511 }
512
513 func (itr *fileSetSeriesIDIterator) Close() error {
514 itr.once.Do(func() { itr.fs.Release() })
515 return itr.itr.Close()
516 }
517
518 // fileSetSeriesIDSetIterator attaches a fileset to an iterator that is released on close.
519 type fileSetSeriesIDSetIterator struct {
520 once sync.Once
521 fs *FileSet
522 itr tsdb.SeriesIDSetIterator
523 }
524
525 func (itr *fileSetSeriesIDSetIterator) Next() (tsdb.SeriesIDElem, error) {
526 return itr.itr.Next()
527 }
528
529 func (itr *fileSetSeriesIDSetIterator) Close() error {
530 itr.once.Do(func() { itr.fs.Release() })
531 return itr.itr.Close()
532 }
533
534 func (itr *fileSetSeriesIDSetIterator) SeriesIDSet() *tsdb.SeriesIDSet {
535 return itr.itr.SeriesIDSet()
536 }
537
538 // fileSetMeasurementIterator attaches a fileset to an iterator that is released on close.
539 type fileSetMeasurementIterator struct {
540 once sync.Once
541 fs *FileSet
542 itr tsdb.MeasurementIterator
543 }
544
545 func newFileSetMeasurementIterator(fs *FileSet, itr tsdb.MeasurementIterator) *fileSetMeasurementIterator {
546 return &fileSetMeasurementIterator{fs: fs, itr: itr}
547 }
548
549 func (itr *fileSetMeasurementIterator) Next() ([]byte, error) {
550 return itr.itr.Next()
551 }
552
553 func (itr *fileSetMeasurementIterator) Close() error {
554 itr.once.Do(func() { itr.fs.Release() })
555 return itr.itr.Close()
556 }
557
558 // fileSetTagKeyIterator attaches a fileset to an iterator that is released on close.
559 type fileSetTagKeyIterator struct {
560 once sync.Once
561 fs *FileSet
562 itr tsdb.TagKeyIterator
563 }
564
565 func newFileSetTagKeyIterator(fs *FileSet, itr tsdb.TagKeyIterator) *fileSetTagKeyIterator {
566 return &fileSetTagKeyIterator{fs: fs, itr: itr}
567 }
568
569 func (itr *fileSetTagKeyIterator) Next() ([]byte, error) {
570 return itr.itr.Next()
571 }
572
573 func (itr *fileSetTagKeyIterator) Close() error {
574 itr.once.Do(func() { itr.fs.Release() })
575 return itr.itr.Close()
576 }
577
578 // fileSetTagValueIterator attaches a fileset to an iterator that is released on close.
579 type fileSetTagValueIterator struct {
580 once sync.Once
581 fs *FileSet
582 itr tsdb.TagValueIterator
583 }
584
585 func newFileSetTagValueIterator(fs *FileSet, itr tsdb.TagValueIterator) *fileSetTagValueIterator {
586 return &fileSetTagValueIterator{fs: fs, itr: itr}
587 }
588
589 func (itr *fileSetTagValueIterator) Next() ([]byte, error) {
590 return itr.itr.Next()
591 }
592
593 func (itr *fileSetTagValueIterator) Close() error {
594 itr.once.Do(func() { itr.fs.Release() })
595 return itr.itr.Close()
596 }