Debian Codegraph @

tsdb/index/tsi1/partition.go

raw
1 package tsi1
2
3 import (
4 "encoding/json"
5 "errors"
6 "fmt"
7 "io/ioutil"
8 "os"
9 "path/filepath"
10 "regexp"
11 "strconv"
12 "strings"
13 "sync"
14 "time"
15 "unsafe"
16
17 "github.com/influxdata/influxdb/logger"
18 "github.com/influxdata/influxdb/models"
19 "github.com/influxdata/influxdb/pkg/bytesutil"
20 "github.com/influxdata/influxdb/pkg/estimator"
21 "github.com/influxdata/influxdb/tsdb"
22 "github.com/influxdata/influxql"
23 "go.uber.org/zap"
24 )
25
26 // Version is the current version of the TSI index.
27 const Version = 1
28
29 // File extensions.
30 const (
31 LogFileExt = ".tsl"
32 IndexFileExt = ".tsi"
33
34 CompactingExt = ".compacting"
35 )
36
37 // ManifestFileName is the name of the index manifest file.
38 const ManifestFileName = "MANIFEST"
39
40 // Partition represents a collection of layered index files and WAL.
41 type Partition struct {
42 mu sync.RWMutex
43 opened bool
44
45 sfile *tsdb.SeriesFile // series lookup file
46 activeLogFile *LogFile // current log file
47 fileSet *FileSet // current file set
48 seq int // file id sequence
49
50 // Fast series lookup of series IDs in the series file that have been present
51 // in this partition. This set tracks both insertions and deletions of a series.
52 seriesIDSet *tsdb.SeriesIDSet
53
54 // Compaction management
55 levels []CompactionLevel // compaction levels
56 levelCompacting []bool // level compaction status
57
58 // Close management.
59 once sync.Once
60 closing chan struct{} // closing is used to inform iterators the partition is closing.
61 wg sync.WaitGroup
62
63 // Fieldset shared with engine.
64 fieldset *tsdb.MeasurementFieldSet
65
66 // Directory of the Partition's index files.
67 path string
68 id string // id portion of path.
69
70 // Log file compaction thresholds.
71 MaxLogFileSize int64
72 nosync bool // when true, flushing and syncing of LogFile will be disabled.
73 logbufferSize int // the LogFile's buffer is set to this value.
74
75 // Frequency of compaction checks.
76 compactionInterrupt chan struct{}
77 compactionsDisabled int
78
79 logger *zap.Logger
80
81 // Current size of MANIFEST. Used to determine partition size.
82 manifestSize int64
83
84 // Index's version.
85 version int
86 }
87
88 // NewPartition returns a new instance of Partition.
89 func NewPartition(sfile *tsdb.SeriesFile, path string) *Partition {
90 return &Partition{
91 closing: make(chan struct{}),
92 path: path,
93 sfile: sfile,
94 seriesIDSet: tsdb.NewSeriesIDSet(),
95
96 MaxLogFileSize: tsdb.DefaultMaxIndexLogFileSize,
97
98 // compactionEnabled: true,
99 compactionInterrupt: make(chan struct{}),
100
101 logger: zap.NewNop(),
102 version: Version,
103 }
104 }
105
106 // bytes estimates the memory footprint of this Partition, in bytes.
107 func (i *Partition) bytes() int {
108 var b int
109 b += 24 // mu RWMutex is 24 bytes
110 b += int(unsafe.Sizeof(i.opened))
111 // Do not count SeriesFile because it belongs to the code that constructed this Partition.
112 b += int(unsafe.Sizeof(i.activeLogFile)) + i.activeLogFile.bytes()
113 b += int(unsafe.Sizeof(i.fileSet)) + i.fileSet.bytes()
114 b += int(unsafe.Sizeof(i.seq))
115 b += int(unsafe.Sizeof(i.seriesIDSet)) + i.seriesIDSet.Bytes()
116 b += int(unsafe.Sizeof(i.levels))
117 for _, level := range i.levels {
118 b += int(unsafe.Sizeof(level))
119 }
120 b += int(unsafe.Sizeof(i.levelCompacting))
121 for _, levelCompacting := range i.levelCompacting {
122 b += int(unsafe.Sizeof(levelCompacting))
123 }
124 b += 12 // once sync.Once is 12 bytes
125 b += int(unsafe.Sizeof(i.closing))
126 b += 16 // wg sync.WaitGroup is 16 bytes
127 b += int(unsafe.Sizeof(i.fieldset)) + i.fieldset.Bytes()
128 b += int(unsafe.Sizeof(i.path)) + len(i.path)
129 b += int(unsafe.Sizeof(i.id)) + len(i.id)
130 b += int(unsafe.Sizeof(i.MaxLogFileSize))
131 b += int(unsafe.Sizeof(i.compactionInterrupt))
132 b += int(unsafe.Sizeof(i.compactionsDisabled))
133 b += int(unsafe.Sizeof(i.logger))
134 b += int(unsafe.Sizeof(i.manifestSize))
135 b += int(unsafe.Sizeof(i.version))
136 return b
137 }
138
139 // ErrIncompatibleVersion is returned when attempting to read from an
140 // incompatible tsi1 manifest file.
141 var ErrIncompatibleVersion = errors.New("incompatible tsi1 index MANIFEST")
142
143 // Open opens the partition.
144 func (i *Partition) Open() error {
145 i.mu.Lock()
146 defer i.mu.Unlock()
147
148 i.closing = make(chan struct{})
149
150 if i.opened {
151 return errors.New("index partition already open")
152 }
153
154 // Validate path is correct.
155 i.id = filepath.Base(i.path)
156 _, err := strconv.Atoi(i.id)
157 if err != nil {
158 return err
159 }
160
161 // Create directory if it doesn't exist.
162 if err := os.MkdirAll(i.path, 0777); err != nil {
163 return err
164 }
165
166 // Read manifest file.
167 m, manifestSize, err := ReadManifestFile(filepath.Join(i.path, ManifestFileName))
168 if os.IsNotExist(err) {
169 m = NewManifest(i.ManifestPath())
170 } else if err != nil {
171 return err
172 }
173 // Set manifest size on the partition
174 i.manifestSize = manifestSize
175
176 // Check to see if the MANIFEST file is compatible with the current Index.
177 if err := m.Validate(); err != nil {
178 return err
179 }
180
181 // Copy compaction levels to the index.
182 i.levels = make([]CompactionLevel, len(m.Levels))
183 copy(i.levels, m.Levels)
184
185 // Set up flags to track whether a level is compacting.
186 i.levelCompacting = make([]bool, len(i.levels))
187
188 // Open each file in the manifest.
189 var files []File
190 for _, filename := range m.Files {
191 switch filepath.Ext(filename) {
192 case LogFileExt:
193 f, err := i.openLogFile(filepath.Join(i.path, filename))
194 if err != nil {
195 return err
196 }
197 files = append(files, f)
198
199 // Make first log file active, if within threshold.
200 sz, _ := f.Stat()
201 if i.activeLogFile == nil && sz < i.MaxLogFileSize {
202 i.activeLogFile = f
203 }
204
205 case IndexFileExt:
206 f, err := i.openIndexFile(filepath.Join(i.path, filename))
207 if err != nil {
208 return err
209 }
210 files = append(files, f)
211 }
212 }
213 fs, err := NewFileSet(i.levels, i.sfile, files)
214 if err != nil {
215 return err
216 }
217 i.fileSet = fs
218
219 // Set initial sequence number.
220 i.seq = i.fileSet.MaxID()
221
222 // Delete any files not in the manifest.
223 if err := i.deleteNonManifestFiles(m); err != nil {
224 return err
225 }
226
227 // Ensure a log file exists.
228 if i.activeLogFile == nil {
229 if err := i.prependActiveLogFile(); err != nil {
230 return err
231 }
232 }
233
234 // Build series existance set.
235 if err := i.buildSeriesSet(); err != nil {
236 return err
237 }
238
239 // Mark opened.
240 i.opened = true
241
242 // Send a compaction request on start up.
243 i.compact()
244
245 return nil
246 }
247
248 // openLogFile opens a log file and appends it to the index.
249 func (i *Partition) openLogFile(path string) (*LogFile, error) {
250 f := NewLogFile(i.sfile, path)
251 f.nosync = i.nosync
252 f.bufferSize = i.logbufferSize
253
254 if err := f.Open(); err != nil {
255 return nil, err
256 }
257 return f, nil
258 }
259
260 // openIndexFile opens a log file and appends it to the index.
261 func (i *Partition) openIndexFile(path string) (*IndexFile, error) {
262 f := NewIndexFile(i.sfile)
263 f.SetPath(path)
264 if err := f.Open(); err != nil {
265 return nil, err
266 }
267 return f, nil
268 }
269
270 // deleteNonManifestFiles removes all files not in the manifest.
271 func (i *Partition) deleteNonManifestFiles(m *Manifest) error {
272 dir, err := os.Open(i.path)
273 if err != nil {
274 return err
275 }
276 defer dir.Close()
277
278 fis, err := dir.Readdir(-1)
279 if err != nil {
280 return err
281 }
282
283 // Loop over all files and remove any not in the manifest.
284 for _, fi := range fis {
285 filename := filepath.Base(fi.Name())
286 if filename == ManifestFileName || m.HasFile(filename) {
287 continue
288 }
289
290 if err := os.RemoveAll(filename); err != nil {
291 return err
292 }
293 }
294
295 return nil
296 }
297
298 func (p *Partition) buildSeriesSet() error {
299 fs := p.retainFileSet()
300 defer fs.Release()
301
302 p.seriesIDSet = tsdb.NewSeriesIDSet()
303
304 // Read series sets from files in reverse.
305 for i := len(fs.files) - 1; i >= 0; i-- {
306 f := fs.files[i]
307
308 // Delete anything that's been tombstoned.
309 ts, err := f.TombstoneSeriesIDSet()
310 if err != nil {
311 return err
312 }
313 p.seriesIDSet.Diff(ts)
314
315 // Add series created within the file.
316 ss, err := f.SeriesIDSet()
317 if err != nil {
318 return err
319 }
320 p.seriesIDSet.Merge(ss)
321 }
322 return nil
323 }
324
325 // Wait returns once outstanding compactions have finished.
326 func (i *Partition) Wait() {
327 i.wg.Wait()
328 }
329
330 // Close closes the index.
331 func (i *Partition) Close() error {
332 // Wait for goroutines to finish outstanding compactions.
333 i.once.Do(func() {
334 close(i.closing)
335 close(i.compactionInterrupt)
336 })
337 i.wg.Wait()
338
339 // Lock index and close remaining
340 i.mu.Lock()
341 defer i.mu.Unlock()
342
343 // Close log files.
344 for _, f := range i.fileSet.files {
345 f.Close()
346 }
347 i.fileSet.files = nil
348
349 return nil
350 }
351
352 // closing returns true if the partition is currently closing. It does not require
353 // a lock so will always return to callers.
354 func (p *Partition) isClosing() bool {
355 select {
356 case <-p.closing:
357 return true
358 default:
359 return false
360 }
361 }
362
363 // Path returns the path to the partition.
364 func (i *Partition) Path() string { return i.path }
365
366 // SeriesFile returns the attached series file.
367 func (i *Partition) SeriesFile() *tsdb.SeriesFile { return i.sfile }
368
369 // NextSequence returns the next file identifier.
370 func (i *Partition) NextSequence() int {
371 i.mu.Lock()
372 defer i.mu.Unlock()
373 return i.nextSequence()
374 }
375
376 func (i *Partition) nextSequence() int {
377 i.seq++
378 return i.seq
379 }
380
381 // ManifestPath returns the path to the index's manifest file.
382 func (i *Partition) ManifestPath() string {
383 return filepath.Join(i.path, ManifestFileName)
384 }
385
386 // Manifest returns a manifest for the index.
387 func (i *Partition) Manifest() *Manifest {
388 m := &Manifest{
389 Levels: i.levels,
390 Files: make([]string, len(i.fileSet.files)),
391 Version: i.version,
392 path: i.ManifestPath(),
393 }
394
395 for j, f := range i.fileSet.files {
396 m.Files[j] = filepath.Base(f.Path())
397 }
398
399 return m
400 }
401
402 // WithLogger sets the logger for the index.
403 func (i *Partition) WithLogger(logger *zap.Logger) {
404 i.logger = logger.With(zap.String("index", "tsi"))
405 }
406
407 // SetFieldSet sets a shared field set from the engine.
408 func (i *Partition) SetFieldSet(fs *tsdb.MeasurementFieldSet) {
409 i.mu.Lock()
410 i.fieldset = fs
411 i.mu.Unlock()
412 }
413
414 // FieldSet returns the fieldset.
415 func (i *Partition) FieldSet() *tsdb.MeasurementFieldSet {
416 i.mu.Lock()
417 fs := i.fieldset
418 i.mu.Unlock()
419 return fs
420 }
421
422 // RetainFileSet returns the current fileset and adds a reference count.
423 func (i *Partition) RetainFileSet() (*FileSet, error) {
424 select {
425 case <-i.closing:
426 return nil, tsdb.ErrIndexClosing
427 default:
428 i.mu.RLock()
429 defer i.mu.RUnlock()
430 return i.retainFileSet(), nil
431 }
432 }
433
434 func (i *Partition) retainFileSet() *FileSet {
435 fs := i.fileSet
436 fs.Retain()
437 return fs
438 }
439
440 // FileN returns the active files in the file set.
441 func (i *Partition) FileN() int { return len(i.fileSet.files) }
442
443 // prependActiveLogFile adds a new log file so that the current log file can be compacted.
444 func (i *Partition) prependActiveLogFile() error {
445 // Open file and insert it into the first position.
446 f, err := i.openLogFile(filepath.Join(i.path, FormatLogFileName(i.nextSequence())))
447 if err != nil {
448 return err
449 }
450 i.activeLogFile = f
451
452 // Prepend and generate new fileset.
453 i.fileSet = i.fileSet.PrependLogFile(f)
454
455 // Write new manifest.
456 manifestSize, err := i.Manifest().Write()
457 if err != nil {
458 // TODO: Close index if write fails.
459 return err
460 }
461 i.manifestSize = manifestSize
462 return nil
463 }
464
465 // ForEachMeasurementName iterates over all measurement names in the index.
466 func (i *Partition) ForEachMeasurementName(fn func(name []byte) error) error {
467 fs, err := i.RetainFileSet()
468 if err != nil {
469 return err
470 }
471 defer fs.Release()
472
473 itr := fs.MeasurementIterator()
474 if itr == nil {
475 return nil
476 }
477
478 for e := itr.Next(); e != nil; e = itr.Next() {
479 if err := fn(e.Name()); err != nil {
480 return err
481 }
482 }
483
484 return nil
485 }
486
487 // MeasurementHasSeries returns true if a measurement has at least one non-tombstoned series.
488 func (p *Partition) MeasurementHasSeries(name []byte) (bool, error) {
489 fs, err := p.RetainFileSet()
490 if err != nil {
491 return false, err
492 }
493 defer fs.Release()
494
495 for _, f := range fs.files {
496 if f.MeasurementHasSeries(p.seriesIDSet, name) {
497 return true, nil
498 }
499 }
500
501 return false, nil
502 }
503
504 // MeasurementIterator returns an iterator over all measurement names.
505 func (i *Partition) MeasurementIterator() (tsdb.MeasurementIterator, error) {
506 fs, err := i.RetainFileSet()
507 if err != nil {
508 return nil, err
509 }
510 itr := fs.MeasurementIterator()
511 if itr == nil {
512 fs.Release()
513 return nil, nil
514 }
515 return newFileSetMeasurementIterator(fs, NewTSDBMeasurementIteratorAdapter(itr)), nil
516 }
517
518 // MeasurementExists returns true if a measurement exists.
519 func (i *Partition) MeasurementExists(name []byte) (bool, error) {
520 fs, err := i.RetainFileSet()
521 if err != nil {
522 return false, err
523 }
524 defer fs.Release()
525 m := fs.Measurement(name)
526 return m != nil && !m.Deleted(), nil
527 }
528
529 func (i *Partition) MeasurementNamesByRegex(re *regexp.Regexp) ([][]byte, error) {
530 fs, err := i.RetainFileSet()
531 if err != nil {
532 return nil, err
533 }
534 defer fs.Release()
535
536 itr := fs.MeasurementIterator()
537 if itr == nil {
538 return nil, nil
539 }
540
541 var a [][]byte
542 for e := itr.Next(); e != nil; e = itr.Next() {
543 if re.Match(e.Name()) {
544 // Clone bytes since they will be used after the fileset is released.
545 a = append(a, bytesutil.Clone(e.Name()))
546 }
547 }
548 return a, nil
549 }
550
551 func (i *Partition) MeasurementSeriesIDIterator(name []byte) (tsdb.SeriesIDIterator, error) {
552 fs, err := i.RetainFileSet()
553 if err != nil {
554 return nil, err
555 }
556 return newFileSetSeriesIDIterator(fs, fs.MeasurementSeriesIDIterator(name)), nil
557 }
558
559 // DropMeasurement deletes a measurement from the index. DropMeasurement does
560 // not remove any series from the index directly.
561 func (i *Partition) DropMeasurement(name []byte) error {
562 fs, err := i.RetainFileSet()
563 if err != nil {
564 return err
565 }
566 defer fs.Release()
567
568 // Delete all keys and values.
569 if kitr := fs.TagKeyIterator(name); kitr != nil {
570 for k := kitr.Next(); k != nil; k = kitr.Next() {
571 // Delete key if not already deleted.
572 if !k.Deleted() {
573 if err := func() error {
574 i.mu.RLock()
575 defer i.mu.RUnlock()
576 return i.activeLogFile.DeleteTagKey(name, k.Key())
577 }(); err != nil {
578 return err
579 }
580 }
581
582 // Delete each value in key.
583 if vitr := k.TagValueIterator(); vitr != nil {
584 for v := vitr.Next(); v != nil; v = vitr.Next() {
585 if !v.Deleted() {
586 if err := func() error {
587 i.mu.RLock()
588 defer i.mu.RUnlock()
589 return i.activeLogFile.DeleteTagValue(name, k.Key(), v.Value())
590 }(); err != nil {
591 return err
592 }
593 }
594 }
595 }
596 }
597 }
598
599 // Delete all series.
600 if itr := fs.MeasurementSeriesIDIterator(name); itr != nil {
601 defer itr.Close()
602 for {
603 elem, err := itr.Next()
604 if err != nil {
605 return err
606 } else if elem.SeriesID == 0 {
607 break
608 }
609 if err := i.activeLogFile.DeleteSeriesID(elem.SeriesID); err != nil {
610 return err
611 }
612 }
613 }
614
615 // Mark measurement as deleted.
616 if err := func() error {
617 i.mu.RLock()
618 defer i.mu.RUnlock()
619 return i.activeLogFile.DeleteMeasurement(name)
620 }(); err != nil {
621 return err
622 }
623
624 // Check if the log file needs to be swapped.
625 if err := i.CheckLogFile(); err != nil {
626 return err
627 }
628
629 return nil
630 }
631
632 // createSeriesListIfNotExists creates a list of series if they doesn't exist in
633 // bulk.
634 func (i *Partition) createSeriesListIfNotExists(names [][]byte, tagsSlice []models.Tags) error {
635 // Is there anything to do? The partition may have been sent an empty batch.
636 if len(names) == 0 {
637 return nil
638 } else if len(names) != len(tagsSlice) {
639 return fmt.Errorf("uneven batch, partition %s sent %d names and %d tags", i.id, len(names), len(tagsSlice))
640 }
641
642 // Maintain reference count on files in file set.
643 fs, err := i.RetainFileSet()
644 if err != nil {
645 return err
646 }
647 defer fs.Release()
648
649 // Ensure fileset cannot change during insert.
650 i.mu.RLock()
651 if err := i.activeLogFile.AddSeriesList(i.seriesIDSet, names, tagsSlice); err != nil {
652 i.mu.RUnlock()
653 return err
654 }
655 i.mu.RUnlock()
656
657 return i.CheckLogFile()
658 }
659
660 func (i *Partition) DropSeries(seriesID uint64) error {
661 // Delete series from index.
662 if err := i.activeLogFile.DeleteSeriesID(seriesID); err != nil {
663 return err
664 }
665
666 i.seriesIDSet.Remove(seriesID)
667
668 // Swap log file, if necessary.
669 return i.CheckLogFile()
670 }
671
672 // MeasurementsSketches returns the two sketches for the partition by merging all
673 // instances of the type sketch types in all the index files.
674 func (i *Partition) MeasurementsSketches() (estimator.Sketch, estimator.Sketch, error) {
675 fs, err := i.RetainFileSet()
676 if err != nil {
677 return nil, nil, err
678 }
679 defer fs.Release()
680 return fs.MeasurementsSketches()
681 }
682
683 // SeriesSketches returns the two sketches for the partition by merging all
684 // instances of the type sketch types in all the index files.
685 func (i *Partition) SeriesSketches() (estimator.Sketch, estimator.Sketch, error) {
686 fs, err := i.RetainFileSet()
687 if err != nil {
688 return nil, nil, err
689 }
690 defer fs.Release()
691 return fs.SeriesSketches()
692 }
693
694 // HasTagKey returns true if tag key exists.
695 func (i *Partition) HasTagKey(name, key []byte) (bool, error) {
696 fs, err := i.RetainFileSet()
697 if err != nil {
698 return false, err
699 }
700 defer fs.Release()
701 return fs.HasTagKey(name, key), nil
702 }
703
704 // HasTagValue returns true if tag value exists.
705 func (i *Partition) HasTagValue(name, key, value []byte) (bool, error) {
706 fs, err := i.RetainFileSet()
707 if err != nil {
708 return false, err
709 }
710 defer fs.Release()
711 return fs.HasTagValue(name, key, value), nil
712 }
713
714 // TagKeyIterator returns an iterator for all keys across a single measurement.
715 func (i *Partition) TagKeyIterator(name []byte) tsdb.TagKeyIterator {
716 fs, err := i.RetainFileSet()
717 if err != nil {
718 return nil // TODO(edd): this should probably return an error.
719 }
720
721 itr := fs.TagKeyIterator(name)
722 if itr == nil {
723 fs.Release()
724 return nil
725 }
726 return newFileSetTagKeyIterator(fs, NewTSDBTagKeyIteratorAdapter(itr))
727 }
728
729 // TagValueIterator returns an iterator for all values across a single key.
730 func (i *Partition) TagValueIterator(name, key []byte) tsdb.TagValueIterator {
731 fs, err := i.RetainFileSet()
732 if err != nil {
733 return nil // TODO(edd): this should probably return an error.
734 }
735
736 itr := fs.TagValueIterator(name, key)
737 if itr == nil {
738 fs.Release()
739 return nil
740 }
741 return newFileSetTagValueIterator(fs, NewTSDBTagValueIteratorAdapter(itr))
742 }
743
744 // TagKeySeriesIDIterator returns a series iterator for all values across a single key.
745 func (i *Partition) TagKeySeriesIDIterator(name, key []byte) tsdb.SeriesIDIterator {
746 fs, err := i.RetainFileSet()
747 if err != nil {
748 return nil // TODO(edd): this should probably return an error.
749 }
750
751 itr := fs.TagKeySeriesIDIterator(name, key)
752 if itr == nil {
753 fs.Release()
754 return nil
755 }
756 return newFileSetSeriesIDIterator(fs, itr)
757 }
758
759 // TagValueSeriesIDIterator returns a series iterator for a single key value.
760 func (i *Partition) TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator {
761 fs, err := i.RetainFileSet()
762 if err != nil {
763 return nil // TODO(edd): this should probably return an error.
764 }
765
766 itr := fs.TagValueSeriesIDIterator(name, key, value)
767 if itr == nil {
768 fs.Release()
769 return nil
770 }
771 return newFileSetSeriesIDIterator(fs, itr)
772 }
773
774 // MeasurementTagKeysByExpr extracts the tag keys wanted by the expression.
775 func (i *Partition) MeasurementTagKeysByExpr(name []byte, expr influxql.Expr) (map[string]struct{}, error) {
776 fs, err := i.RetainFileSet()
777 if err != nil {
778 return nil, err
779 }
780 defer fs.Release()
781
782 return fs.MeasurementTagKeysByExpr(name, expr)
783 }
784
785 // ForEachMeasurementTagKey iterates over all tag keys in a measurement.
786 func (i *Partition) ForEachMeasurementTagKey(name []byte, fn func(key []byte) error) error {
787 fs, err := i.RetainFileSet()
788 if err != nil {
789 return err
790 }
791 defer fs.Release()
792
793 itr := fs.TagKeyIterator(name)
794 if itr == nil {
795 return nil
796 }
797
798 for e := itr.Next(); e != nil; e = itr.Next() {
799 if err := fn(e.Key()); err != nil {
800 return err
801 }
802 }
803
804 return nil
805 }
806
807 // TagKeyCardinality always returns zero.
808 // It is not possible to determine cardinality of tags across index files.
809 func (i *Partition) TagKeyCardinality(name, key []byte) int {
810 return 0
811 }
812
813 func (i *Partition) SetFieldName(measurement []byte, name string) {}
814 func (i *Partition) RemoveShard(shardID uint64) {}
815 func (i *Partition) AssignShard(k string, shardID uint64) {}
816
817 // Compact requests a compaction of log files.
818 func (i *Partition) Compact() {
819 i.mu.Lock()
820 defer i.mu.Unlock()
821 i.compact()
822 }
823
824 func (i *Partition) DisableCompactions() {
825 i.mu.Lock()
826 defer i.mu.Unlock()
827 i.compactionsDisabled++
828
829 select {
830 case <-i.closing:
831 return
832 default:
833 }
834
835 if i.compactionsDisabled == 0 {
836 close(i.compactionInterrupt)
837 i.compactionInterrupt = make(chan struct{})
838 }
839 }
840
841 func (i *Partition) EnableCompactions() {
842 i.mu.Lock()
843 defer i.mu.Unlock()
844
845 // Already enabled?
846 if i.compactionsEnabled() {
847 return
848 }
849 i.compactionsDisabled--
850 }
851
852 func (i *Partition) compactionsEnabled() bool {
853 return i.compactionsDisabled == 0
854 }
855
856 // compact compacts continguous groups of files that are not currently compacting.
857 func (i *Partition) compact() {
858 if i.isClosing() {
859 return
860 } else if !i.compactionsEnabled() {
861 return
862 }
863 interrupt := i.compactionInterrupt
864
865 fs := i.retainFileSet()
866 defer fs.Release()
867
868 // Iterate over each level we are going to compact.
869 // We skip the first level (0) because it is log files and they are compacted separately.
870 // We skip the last level because the files have no higher level to compact into.
871 minLevel, maxLevel := 1, len(i.levels)-2
872 for level := minLevel; level <= maxLevel; level++ {
873 // Skip level if it is currently compacting.
874 if i.levelCompacting[level] {
875 continue
876 }
877
878 // Collect contiguous files from the end of the level.
879 files := fs.LastContiguousIndexFilesByLevel(level)
880 if len(files) < 2 {
881 continue
882 } else if len(files) > MaxIndexMergeCount {
883 files = files[len(files)-MaxIndexMergeCount:]
884 }
885
886 // Retain files during compaction.
887 IndexFiles(files).Retain()
888
889 // Mark the level as compacting.
890 i.levelCompacting[level] = true
891
892 // Execute in closure to save reference to the group within the loop.
893 func(files []*IndexFile, level int) {
894 // Start compacting in a separate goroutine.
895 i.wg.Add(1)
896 go func() {
897 defer i.wg.Done()
898
899 // Compact to a new level.
900 i.compactToLevel(files, level+1, interrupt)
901
902 // Ensure compaction lock for the level is released.
903 i.mu.Lock()
904 i.levelCompacting[level] = false
905 i.mu.Unlock()
906
907 // Check for new compactions
908 i.Compact()
909 }()
910 }(files, level)
911 }
912 }
913
914 // compactToLevel compacts a set of files into a new file. Replaces old files with
915 // compacted file on successful completion. This runs in a separate goroutine.
916 func (i *Partition) compactToLevel(files []*IndexFile, level int, interrupt <-chan struct{}) {
917 assert(len(files) >= 2, "at least two index files are required for compaction")
918 assert(level > 0, "cannot compact level zero")
919
920 // Build a logger for this compaction.
921 log, logEnd := logger.NewOperation(i.logger, "TSI level compaction", "tsi1_compact_to_level", zap.Int("tsi1_level", level))
922 defer logEnd()
923
924 // Check for cancellation.
925 select {
926 case <-interrupt:
927 log.Error("Cannot begin compaction", zap.Error(ErrCompactionInterrupted))
928 return
929 default:
930 }
931
932 // Files have already been retained by caller.
933 // Ensure files are released only once.
934 var once sync.Once
935 defer once.Do(func() { IndexFiles(files).Release() })
936
937 // Track time to compact.
938 start := time.Now()
939
940 // Create new index file.
941 path := filepath.Join(i.path, FormatIndexFileName(i.NextSequence(), level))
942 f, err := os.Create(path)
943 if err != nil {
944 log.Error("Cannot create compaction files", zap.Error(err))
945 return
946 }
947 defer f.Close()
948
949 log.Info("Performing full compaction",
950 zap.String("src", joinIntSlice(IndexFiles(files).IDs(), ",")),
951 zap.String("dst", path),
952 )
953
954 // Compact all index files to new index file.
955 lvl := i.levels[level]
956 n, err := IndexFiles(files).CompactTo(f, i.sfile, lvl.M, lvl.K, interrupt)
957 if err != nil {
958 log.Error("Cannot compact index files", zap.Error(err))
959 return
960 }
961
962 // Close file.
963 if err := f.Close(); err != nil {
964 log.Error("Error closing index file", zap.Error(err))
965 return
966 }
967
968 // Reopen as an index file.
969 file := NewIndexFile(i.sfile)
970 file.SetPath(path)
971 if err := file.Open(); err != nil {
972 log.Error("Cannot open new index file", zap.Error(err))
973 return
974 }
975
976 // Obtain lock to swap in index file and write manifest.
977 if err := func() error {
978 i.mu.Lock()
979 defer i.mu.Unlock()
980
981 // Replace previous files with new index file.
982 i.fileSet = i.fileSet.MustReplace(IndexFiles(files).Files(), file)
983
984 // Write new manifest.
985 manifestSize, err := i.Manifest().Write()
986 if err != nil {
987 // TODO: Close index if write fails.
988 return err
989 }
990 i.manifestSize = manifestSize
991 return nil
992 }(); err != nil {
993 log.Error("Cannot write manifest", zap.Error(err))
994 return
995 }
996
997 elapsed := time.Since(start)
998 log.Info("Full compaction complete",
999 zap.String("path", path),
1000 logger.DurationLiteral("elapsed", elapsed),
1001 zap.Int64("bytes", n),
1002 zap.Int("kb_per_sec", int(float64(n)/elapsed.Seconds())/1024),
1003 )
1004
1005 // Release old files.
1006 once.Do(func() { IndexFiles(files).Release() })
1007
1008 // Close and delete all old index files.
1009 for _, f := range files {
1010 log.Info("Removing index file", zap.String("path", f.Path()))
1011
1012 if err := f.Close(); err != nil {
1013 log.Error("Cannot close index file", zap.Error(err))
1014 return
1015 } else if err := os.Remove(f.Path()); err != nil {
1016 log.Error("Cannot remove index file", zap.Error(err))
1017 return
1018 }
1019 }
1020 }
1021
1022 func (i *Partition) Rebuild() {}
1023
1024 func (i *Partition) CheckLogFile() error {
1025 // Check log file size under read lock.
1026 if size := func() int64 {
1027 i.mu.RLock()
1028 defer i.mu.RUnlock()
1029 return i.activeLogFile.Size()
1030 }(); size < i.MaxLogFileSize {
1031 return nil
1032 }
1033
1034 // If file size exceeded then recheck under write lock and swap files.
1035 i.mu.Lock()
1036 defer i.mu.Unlock()
1037 return i.checkLogFile()
1038 }
1039
1040 func (i *Partition) checkLogFile() error {
1041 if i.activeLogFile.Size() < i.MaxLogFileSize {
1042 return nil
1043 }
1044
1045 // Swap current log file.
1046 logFile := i.activeLogFile
1047
1048 // Open new log file and insert it into the first position.
1049 if err := i.prependActiveLogFile(); err != nil {
1050 return err
1051 }
1052
1053 // Begin compacting in a background goroutine.
1054 i.wg.Add(1)
1055 go func() {
1056 defer i.wg.Done()
1057 i.compactLogFile(logFile)
1058 i.Compact() // check for new compactions
1059 }()
1060
1061 return nil
1062 }
1063
1064 // compactLogFile compacts f into a tsi file. The new file will share the
1065 // same identifier but will have a ".tsi" extension. Once the log file is
1066 // compacted then the manifest is updated and the log file is discarded.
1067 func (i *Partition) compactLogFile(logFile *LogFile) {
1068 if i.isClosing() {
1069 return
1070 }
1071
1072 i.mu.Lock()
1073 interrupt := i.compactionInterrupt
1074 i.mu.Unlock()
1075
1076 start := time.Now()
1077
1078 // Retrieve identifier from current path.
1079 id := logFile.ID()
1080 assert(id != 0, "cannot parse log file id: %s", logFile.Path())
1081
1082 // Build a logger for this compaction.
1083 log, logEnd := logger.NewOperation(i.logger, "TSI log compaction", "tsi1_compact_log_file", zap.Int("tsi1_log_file_id", id))
1084 defer logEnd()
1085
1086 // Create new index file.
1087 path := filepath.Join(i.path, FormatIndexFileName(id, 1))
1088 f, err := os.Create(path)
1089 if err != nil {
1090 log.Error("Cannot create index file", zap.Error(err))
1091 return
1092 }
1093 defer f.Close()
1094
1095 // Compact log file to new index file.
1096 lvl := i.levels[1]
1097 n, err := logFile.CompactTo(f, lvl.M, lvl.K, interrupt)
1098 if err != nil {
1099 log.Error("Cannot compact log file", zap.Error(err), zap.String("path", logFile.Path()))
1100 return
1101 }
1102
1103 // Close file.
1104 if err := f.Close(); err != nil {
1105 log.Error("Cannot close log file", zap.Error(err))
1106 return
1107 }
1108
1109 // Reopen as an index file.
1110 file := NewIndexFile(i.sfile)
1111 file.SetPath(path)
1112 if err := file.Open(); err != nil {
1113 log.Error("Cannot open compacted index file", zap.Error(err), zap.String("path", file.Path()))
1114 return
1115 }
1116
1117 // Obtain lock to swap in index file and write manifest.
1118 if err := func() error {
1119 i.mu.Lock()
1120 defer i.mu.Unlock()
1121
1122 // Replace previous log file with index file.
1123 i.fileSet = i.fileSet.MustReplace([]File{logFile}, file)
1124
1125 // Write new manifest.
1126 manifestSize, err := i.Manifest().Write()
1127 if err != nil {
1128 // TODO: Close index if write fails.
1129 return err
1130 }
1131
1132 i.manifestSize = manifestSize
1133 return nil
1134 }(); err != nil {
1135 log.Error("Cannot update manifest", zap.Error(err))
1136 return
1137 }
1138
1139 elapsed := time.Since(start)
1140 log.Info("Log file compacted",
1141 logger.DurationLiteral("elapsed", elapsed),
1142 zap.Int64("bytes", n),
1143 zap.Int("kb_per_sec", int(float64(n)/elapsed.Seconds())/1024),
1144 )
1145
1146 // Closing the log file will automatically wait until the ref count is zero.
1147 if err := logFile.Close(); err != nil {
1148 log.Error("Cannot close log file", zap.Error(err))
1149 return
1150 } else if err := os.Remove(logFile.Path()); err != nil {
1151 log.Error("Cannot remove log file", zap.Error(err))
1152 return
1153 }
1154 }
1155
1156 // unionStringSets returns the union of two sets
1157 func unionStringSets(a, b map[string]struct{}) map[string]struct{} {
1158 other := make(map[string]struct{})
1159 for k := range a {
1160 other[k] = struct{}{}
1161 }
1162 for k := range b {
1163 other[k] = struct{}{}
1164 }
1165 return other
1166 }
1167
1168 // intersectStringSets returns the intersection of two sets.
1169 func intersectStringSets(a, b map[string]struct{}) map[string]struct{} {
1170 if len(a) < len(b) {
1171 a, b = b, a
1172 }
1173
1174 other := make(map[string]struct{})
1175 for k := range a {
1176 if _, ok := b[k]; ok {
1177 other[k] = struct{}{}
1178 }
1179 }
1180 return other
1181 }
1182
1183 var fileIDRegex = regexp.MustCompile(`^L(\d+)-(\d+)\..+$`)
1184
1185 // ParseFilename extracts the numeric id from a log or index file path.
1186 // Returns 0 if it cannot be parsed.
1187 func ParseFilename(name string) (level, id int) {
1188 a := fileIDRegex.FindStringSubmatch(filepath.Base(name))
1189 if a == nil {
1190 return 0, 0
1191 }
1192
1193 level, _ = strconv.Atoi(a[1])
1194 id, _ = strconv.Atoi(a[2])
1195 return id, level
1196 }
1197
1198 // Manifest represents the list of log & index files that make up the index.
1199 // The files are listed in time order, not necessarily ID order.
1200 type Manifest struct {
1201 Levels []CompactionLevel `json:"levels,omitempty"`
1202 Files []string `json:"files,omitempty"`
1203
1204 // Version should be updated whenever the TSI format has changed.
1205 Version int `json:"version,omitempty"`
1206
1207 path string // location on disk of the manifest.
1208 }
1209
1210 // NewManifest returns a new instance of Manifest with default compaction levels.
1211 func NewManifest(path string) *Manifest {
1212 m := &Manifest{
1213 Levels: make([]CompactionLevel, len(DefaultCompactionLevels)),
1214 Version: Version,
1215 path: path,
1216 }
1217 copy(m.Levels, DefaultCompactionLevels[:])
1218 return m
1219 }
1220
1221 // HasFile returns true if name is listed in the log files or index files.
1222 func (m *Manifest) HasFile(name string) bool {
1223 for _, filename := range m.Files {
1224 if filename == name {
1225 return true
1226 }
1227 }
1228 return false
1229 }
1230
1231 // Validate checks if the Manifest's version is compatible with this version
1232 // of the tsi1 index.
1233 func (m *Manifest) Validate() error {
1234 // If we don't have an explicit version in the manifest file then we know
1235 // it's not compatible with the latest tsi1 Index.
1236 if m.Version != Version {
1237 return ErrIncompatibleVersion
1238 }
1239 return nil
1240 }
1241
1242 // Write writes the manifest file to the provided path, returning the number of
1243 // bytes written and an error, if any.
1244 func (m *Manifest) Write() (int64, error) {
1245 buf, err := json.MarshalIndent(m, "", " ")
1246 if err != nil {
1247 return 0, err
1248 }
1249 buf = append(buf, '\n')
1250
1251 if err := ioutil.WriteFile(m.path, buf, 0666); err != nil {
1252 return 0, err
1253 }
1254 return int64(len(buf)), nil
1255 }
1256
1257 // ReadManifestFile reads a manifest from a file path and returns the Manifest,
1258 // the size of the manifest on disk, and any error if appropriate.
1259 func ReadManifestFile(path string) (*Manifest, int64, error) {
1260 buf, err := ioutil.ReadFile(path)
1261 if err != nil {
1262 return nil, 0, err
1263 }
1264
1265 // Decode manifest.
1266 var m Manifest
1267 if err := json.Unmarshal(buf, &m); err != nil {
1268 return nil, 0, err
1269 }
1270
1271 // Set the path of the manifest.
1272 m.path = path
1273 return &m, int64(len(buf)), nil
1274 }
1275
1276 func joinIntSlice(a []int, sep string) string {
1277 other := make([]string, len(a))
1278 for i := range a {
1279 other[i] = strconv.Itoa(a[i])
1280 }
1281 return strings.Join(other, sep)
1282 }
1283
1284 // CompactionLevel represents a grouping of index files based on bloom filter
1285 // settings. By having the same bloom filter settings, the filters
1286 // can be merged and evaluated at a higher level.
1287 type CompactionLevel struct {
1288 // Bloom filter bit size & hash count
1289 M uint64 `json:"m,omitempty"`
1290 K uint64 `json:"k,omitempty"`
1291 }
1292
1293 // DefaultCompactionLevels is the default settings used by the index.
1294 var DefaultCompactionLevels = []CompactionLevel{
1295 {M: 0, K: 0}, // L0: Log files, no filter.
1296 {M: 1 << 25, K: 6}, // L1: Initial compaction
1297 {M: 1 << 25, K: 6}, // L2
1298 {M: 1 << 26, K: 6}, // L3
1299 {M: 1 << 27, K: 6}, // L4
1300 {M: 1 << 28, K: 6}, // L5
1301 {M: 1 << 29, K: 6}, // L6
1302 {M: 1 << 30, K: 6}, // L7
1303 }
1304
1305 // MaxIndexMergeCount is the maximum number of files that can be merged together at once.
1306 const MaxIndexMergeCount = 2
1307
1308 // MaxIndexFileSize is the maximum expected size of an index file.
1309 const MaxIndexFileSize = 4 * (1 << 30)
1310
1311 // IsPartitionDir returns true if directory contains a MANIFEST file.
1312 func IsPartitionDir(path string) (bool, error) {
1313 if _, err := os.Stat(filepath.Join(path, ManifestFileName)); os.IsNotExist(err) {
1314 return false, nil
1315 } else if err != nil {
1316 return false, err
1317 }
1318 return true, nil
1319 }