Debian Codegraph @

tsdb/index/tsi1/index.go

raw
1 package tsi1
2
3 import (
4 "errors"
5 "fmt"
6 "io/ioutil"
7 "os"
8 "path/filepath"
9 "regexp"
10 "runtime"
11 "strconv"
12 "sync"
13 "sync/atomic"
14 "unsafe"
15
16 "github.com/cespare/xxhash"
17 "github.com/influxdata/influxdb/models"
18 "github.com/influxdata/influxdb/pkg/estimator"
19 "github.com/influxdata/influxdb/pkg/estimator/hll"
20 "github.com/influxdata/influxdb/pkg/slices"
21 "github.com/influxdata/influxdb/tsdb"
22 "github.com/influxdata/influxql"
23 "go.uber.org/zap"
24 )
25
26 // IndexName is the name of the index.
27 const IndexName = tsdb.TSI1IndexName
28
29 // ErrCompactionInterrupted is returned if compactions are disabled or
30 // an index is closed while a compaction is occurring.
31 var ErrCompactionInterrupted = errors.New("tsi1: compaction interrupted")
32
33 func init() {
34 // FIXME(edd): Remove this.
35 if os.Getenv("TSI_PARTITIONS") != "" {
36 i, err := strconv.Atoi(os.Getenv("TSI_PARTITIONS"))
37 if err != nil {
38 panic(err)
39 }
40 DefaultPartitionN = uint64(i)
41 }
42
43 tsdb.RegisterIndex(IndexName, func(_ uint64, db, path string, _ *tsdb.SeriesIDSet, sfile *tsdb.SeriesFile, opt tsdb.EngineOptions) tsdb.Index {
44 idx := NewIndex(sfile, db, WithPath(path), WithMaximumLogFileSize(int64(opt.Config.MaxIndexLogFileSize)))
45 return idx
46 })
47 }
48
49 // DefaultPartitionN determines how many shards the index will be partitioned into.
50 //
51 // NOTE: Currently, this must not be change once a database is created. Further,
52 // it must also be a power of 2.
53 //
54 var DefaultPartitionN uint64 = 8
55
56 // An IndexOption is a functional option for changing the configuration of
57 // an Index.
58 type IndexOption func(i *Index)
59
60 // WithPath sets the root path of the Index
61 var WithPath = func(path string) IndexOption {
62 return func(i *Index) {
63 i.path = path
64 }
65 }
66
67 // DisableCompactions disables compactions on the Index.
68 var DisableCompactions = func() IndexOption {
69 return func(i *Index) {
70 i.disableCompactions = true
71 }
72 }
73
74 // WithLogger sets the logger for the Index.
75 var WithLogger = func(l zap.Logger) IndexOption {
76 return func(i *Index) {
77 i.logger = l.With(zap.String("index", "tsi"))
78 }
79 }
80
81 // WithMaximumLogFileSize sets the maximum size of LogFiles before they're
82 // compacted into IndexFiles.
83 var WithMaximumLogFileSize = func(size int64) IndexOption {
84 return func(i *Index) {
85 i.maxLogFileSize = size
86 }
87 }
88
89 // DisableFsync disables flushing and syncing of underlying files. Primarily this
90 // impacts the LogFiles. This option can be set when working with the index in
91 // an offline manner, for cases where a hard failure can be overcome by re-running the tooling.
92 var DisableFsync = func() IndexOption {
93 return func(i *Index) {
94 i.disableFsync = true
95 }
96 }
97
98 // WithLogFileBufferSize sets the size of the buffer used within LogFiles.
99 // Typically appending an entry to a LogFile involves writing 11 or 12 bytes, so
100 // depending on how many new series are being created within a batch, it may
101 // be appropriate to set this.
102 var WithLogFileBufferSize = func(sz int) IndexOption {
103 return func(i *Index) {
104 if sz > 1<<17 { // 128K
105 sz = 1 << 17
106 } else if sz < 1<<12 {
107 sz = 1 << 12 // 4K (runtime default)
108 }
109 i.logfileBufferSize = sz
110 }
111 }
112
113 // Index represents a collection of layered index files and WAL.
114 type Index struct {
115 mu sync.RWMutex
116 partitions []*Partition
117 opened bool
118
119 // The following may be set when initializing an Index.
120 path string // Root directory of the index partitions.
121 disableCompactions bool // Initially disables compactions on the index.
122 maxLogFileSize int64 // Maximum size of a LogFile before it's compacted.
123 logfileBufferSize int // The size of the buffer used by the LogFile.
124 disableFsync bool // Disables flushing buffers and fsyning files. Used when working with indexes offline.
125 logger *zap.Logger // Index's logger.
126
127 // The following must be set when initializing an Index.
128 sfile *tsdb.SeriesFile // series lookup file
129 database string // Name of database.
130
131 // Cached sketches.
132 mSketch, mTSketch estimator.Sketch // Measurement sketches
133 sSketch, sTSketch estimator.Sketch // Series sketches
134
135 // Index's version.
136 version int
137
138 // Number of partitions used by the index.
139 PartitionN uint64
140 }
141
142 func (i *Index) UniqueReferenceID() uintptr {
143 return uintptr(unsafe.Pointer(i))
144 }
145
146 // NewIndex returns a new instance of Index.
147 func NewIndex(sfile *tsdb.SeriesFile, database string, options ...IndexOption) *Index {
148 idx := &Index{
149 maxLogFileSize: tsdb.DefaultMaxIndexLogFileSize,
150 logger: zap.NewNop(),
151 version: Version,
152 sfile: sfile,
153 database: database,
154 mSketch: hll.NewDefaultPlus(),
155 mTSketch: hll.NewDefaultPlus(),
156 sSketch: hll.NewDefaultPlus(),
157 sTSketch: hll.NewDefaultPlus(),
158 PartitionN: DefaultPartitionN,
159 }
160
161 for _, option := range options {
162 option(idx)
163 }
164
165 return idx
166 }
167
168 // Bytes estimates the memory footprint of this Index, in bytes.
169 func (i *Index) Bytes() int {
170 var b int
171 i.mu.RLock()
172 b += 24 // mu RWMutex is 24 bytes
173 b += int(unsafe.Sizeof(i.partitions))
174 for _, p := range i.partitions {
175 b += int(unsafe.Sizeof(p)) + p.bytes()
176 }
177 b += int(unsafe.Sizeof(i.opened))
178 b += int(unsafe.Sizeof(i.path)) + len(i.path)
179 b += int(unsafe.Sizeof(i.disableCompactions))
180 b += int(unsafe.Sizeof(i.maxLogFileSize))
181 b += int(unsafe.Sizeof(i.logger))
182 b += int(unsafe.Sizeof(i.sfile))
183 // Do not count SeriesFile because it belongs to the code that constructed this Index.
184 b += int(unsafe.Sizeof(i.mSketch)) + i.mSketch.Bytes()
185 b += int(unsafe.Sizeof(i.mTSketch)) + i.mTSketch.Bytes()
186 b += int(unsafe.Sizeof(i.sSketch)) + i.sSketch.Bytes()
187 b += int(unsafe.Sizeof(i.sTSketch)) + i.sTSketch.Bytes()
188 b += int(unsafe.Sizeof(i.database)) + len(i.database)
189 b += int(unsafe.Sizeof(i.version))
190 b += int(unsafe.Sizeof(i.PartitionN))
191 i.mu.RUnlock()
192 return b
193 }
194
195 // Database returns the name of the database the index was initialized with.
196 func (i *Index) Database() string {
197 return i.database
198 }
199
200 // WithLogger sets the logger on the index after it's been created.
201 //
202 // It's not safe to call WithLogger after the index has been opened, or before
203 // it has been closed.
204 func (i *Index) WithLogger(l *zap.Logger) {
205 i.mu.Lock()
206 defer i.mu.Unlock()
207 i.logger = l.With(zap.String("index", "tsi"))
208 }
209
210 // Type returns the type of Index this is.
211 func (i *Index) Type() string { return IndexName }
212
213 // SeriesFile returns the series file attached to the index.
214 func (i *Index) SeriesFile() *tsdb.SeriesFile { return i.sfile }
215
216 // SeriesIDSet returns the set of series ids associated with series in this
217 // index. Any series IDs for series no longer present in the index are filtered out.
218 func (i *Index) SeriesIDSet() *tsdb.SeriesIDSet {
219 seriesIDSet := tsdb.NewSeriesIDSet()
220 others := make([]*tsdb.SeriesIDSet, 0, i.PartitionN)
221 for _, p := range i.partitions {
222 others = append(others, p.seriesIDSet)
223 }
224 seriesIDSet.Merge(others...)
225 return seriesIDSet
226 }
227
228 // Open opens the index.
229 func (i *Index) Open() error {
230 i.mu.Lock()
231 defer i.mu.Unlock()
232
233 if i.opened {
234 return errors.New("index already open")
235 }
236
237 // Ensure root exists.
238 if err := os.MkdirAll(i.path, 0777); err != nil {
239 return err
240 }
241
242 // Initialize index partitions.
243 i.partitions = make([]*Partition, i.PartitionN)
244 for j := 0; j < len(i.partitions); j++ {
245 p := NewPartition(i.sfile, filepath.Join(i.path, fmt.Sprint(j)))
246 p.MaxLogFileSize = i.maxLogFileSize
247 p.nosync = i.disableFsync
248 p.logbufferSize = i.logfileBufferSize
249 p.logger = i.logger.With(zap.String("tsi1_partition", fmt.Sprint(j+1)))
250 i.partitions[j] = p
251 }
252
253 // Open all the Partitions in parallel.
254 partitionN := len(i.partitions)
255 n := i.availableThreads()
256
257 // Store results.
258 errC := make(chan error, partitionN)
259
260 // Run fn on each partition using a fixed number of goroutines.
261 var pidx uint32 // Index of maximum Partition being worked on.
262 for k := 0; k < n; k++ {
263 go func(k int) {
264 for {
265 idx := int(atomic.AddUint32(&pidx, 1) - 1) // Get next partition to work on.
266 if idx >= partitionN {
267 return // No more work.
268 }
269 err := i.partitions[idx].Open()
270 errC <- err
271 }
272 }(k)
273 }
274
275 // Check for error
276 for i := 0; i < partitionN; i++ {
277 if err := <-errC; err != nil {
278 return err
279 }
280 }
281
282 // Refresh cached sketches.
283 if err := i.updateSeriesSketches(); err != nil {
284 return err
285 } else if err := i.updateMeasurementSketches(); err != nil {
286 return err
287 }
288
289 // Mark opened.
290 i.opened = true
291 i.logger.Info(fmt.Sprintf("index opened with %d partitions", partitionN))
292 return nil
293 }
294
295 // Compact requests a compaction of partitions.
296 func (i *Index) Compact() {
297 i.mu.Lock()
298 defer i.mu.Unlock()
299 for _, p := range i.partitions {
300 p.Compact()
301 }
302 }
303
304 func (i *Index) EnableCompactions() {
305 for _, p := range i.partitions {
306 p.EnableCompactions()
307 }
308 }
309
310 func (i *Index) DisableCompactions() {
311 for _, p := range i.partitions {
312 p.DisableCompactions()
313 }
314 }
315
316 // Wait blocks until all outstanding compactions have completed.
317 func (i *Index) Wait() {
318 for _, p := range i.partitions {
319 p.Wait()
320 }
321 }
322
323 // Close closes the index.
324 func (i *Index) Close() error {
325 // Lock index and close partitions.
326 i.mu.Lock()
327 defer i.mu.Unlock()
328
329 for _, p := range i.partitions {
330 if err := p.Close(); err != nil {
331 return err
332 }
333 }
334
335 // Mark index as closed.
336 i.opened = false
337 return nil
338 }
339
340 // Path returns the path the index was opened with.
341 func (i *Index) Path() string { return i.path }
342
343 // PartitionAt returns the partition by index.
344 func (i *Index) PartitionAt(index int) *Partition {
345 return i.partitions[index]
346 }
347
348 // partition returns the appropriate Partition for a provided series key.
349 func (i *Index) partition(key []byte) *Partition {
350 return i.partitions[int(xxhash.Sum64(key)&(i.PartitionN-1))]
351 }
352
353 // partitionIdx returns the index of the partition that key belongs in.
354 func (i *Index) partitionIdx(key []byte) int {
355 return int(xxhash.Sum64(key) & (i.PartitionN - 1))
356 }
357
358 // availableThreads returns the minimum of GOMAXPROCS and the number of
359 // partitions in the Index.
360 func (i *Index) availableThreads() int {
361 n := runtime.GOMAXPROCS(0)
362 if len(i.partitions) < n {
363 return len(i.partitions)
364 }
365 return n
366 }
367
368 // updateMeasurementSketches rebuilds the cached measurement sketches.
369 func (i *Index) updateMeasurementSketches() error {
370 for j := 0; j < int(i.PartitionN); j++ {
371 if s, t, err := i.partitions[j].MeasurementsSketches(); err != nil {
372 return err
373 } else if i.mSketch.Merge(s); err != nil {
374 return err
375 } else if i.mTSketch.Merge(t); err != nil {
376 return err
377 }
378 }
379 return nil
380 }
381
382 // updateSeriesSketches rebuilds the cached series sketches.
383 func (i *Index) updateSeriesSketches() error {
384 for j := 0; j < int(i.PartitionN); j++ {
385 if s, t, err := i.partitions[j].SeriesSketches(); err != nil {
386 return err
387 } else if i.sSketch.Merge(s); err != nil {
388 return err
389 } else if i.sTSketch.Merge(t); err != nil {
390 return err
391 }
392 }
393 return nil
394 }
395
396 // SetFieldSet sets a shared field set from the engine.
397 func (i *Index) SetFieldSet(fs *tsdb.MeasurementFieldSet) {
398 for _, p := range i.partitions {
399 p.SetFieldSet(fs)
400 }
401 }
402
403 // FieldSet returns the assigned fieldset.
404 func (i *Index) FieldSet() *tsdb.MeasurementFieldSet {
405 if len(i.partitions) == 0 {
406 return nil
407 }
408 return i.partitions[0].FieldSet()
409 }
410
411 // ForEachMeasurementName iterates over all measurement names in the index,
412 // applying fn. It returns the first error encountered, if any.
413 //
414 // ForEachMeasurementName does not call fn on each partition concurrently so the
415 // call may provide a non-goroutine safe fn.
416 func (i *Index) ForEachMeasurementName(fn func(name []byte) error) error {
417 itr, err := i.MeasurementIterator()
418 if err != nil {
419 return err
420 } else if itr == nil {
421 return nil
422 }
423 defer itr.Close()
424
425 // Iterate over all measurements.
426 for {
427 e, err := itr.Next()
428 if err != nil {
429 return err
430 } else if e == nil {
431 break
432 }
433
434 if err := fn(e); err != nil {
435 return err
436 }
437 }
438 return nil
439 }
440
441 // MeasurementExists returns true if a measurement exists.
442 func (i *Index) MeasurementExists(name []byte) (bool, error) {
443 n := i.availableThreads()
444
445 // Store errors
446 var found uint32 // Use this to signal we found the measurement.
447 errC := make(chan error, i.PartitionN)
448
449 // Check each partition for the measurement concurrently.
450 var pidx uint32 // Index of maximum Partition being worked on.
451 for k := 0; k < n; k++ {
452 go func() {
453 for {
454 idx := int(atomic.AddUint32(&pidx, 1) - 1) // Get next partition to check
455 if idx >= len(i.partitions) {
456 return // No more work.
457 }
458
459 // Check if the measurement has been found. If it has don't
460 // need to check this partition and can just move on.
461 if atomic.LoadUint32(&found) == 1 {
462 errC <- nil
463 continue
464 }
465
466 b, err := i.partitions[idx].MeasurementExists(name)
467 if b {
468 atomic.StoreUint32(&found, 1)
469 }
470 errC <- err
471 }
472 }()
473 }
474
475 // Check for error
476 for i := 0; i < cap(errC); i++ {
477 if err := <-errC; err != nil {
478 return false, err
479 }
480 }
481
482 // Check if we found the measurement.
483 return atomic.LoadUint32(&found) == 1, nil
484 }
485
486 // MeasurementHasSeries returns true if a measurement has non-tombstoned series.
487 func (i *Index) MeasurementHasSeries(name []byte) (bool, error) {
488 for _, p := range i.partitions {
489 if v, err := p.MeasurementHasSeries(name); err != nil {
490 return false, err
491 } else if v {
492 return true, nil
493 }
494 }
495 return false, nil
496 }
497
498 // fetchByteValues is a helper for gathering values from each partition in the index,
499 // based on some criteria.
500 //
501 // fn is a function that works on partition idx and calls into some method on
502 // the partition that returns some ordered values.
503 func (i *Index) fetchByteValues(fn func(idx int) ([][]byte, error)) ([][]byte, error) {
504 n := i.availableThreads()
505
506 // Store results.
507 names := make([][][]byte, i.PartitionN)
508 errC := make(chan error, i.PartitionN)
509
510 var pidx uint32 // Index of maximum Partition being worked on.
511 for k := 0; k < n; k++ {
512 go func() {
513 for {
514 idx := int(atomic.AddUint32(&pidx, 1) - 1) // Get next partition to work on.
515 if idx >= len(i.partitions) {
516 return // No more work.
517 }
518
519 pnames, err := fn(idx)
520
521 // This is safe since there are no readers on names until all
522 // the writers are done.
523 names[idx] = pnames
524 errC <- err
525 }
526 }()
527 }
528
529 // Check for error
530 for i := 0; i < cap(errC); i++ {
531 if err := <-errC; err != nil {
532 return nil, err
533 }
534 }
535
536 // It's now safe to read from names.
537 return slices.MergeSortedBytes(names[:]...), nil
538 }
539
540 // MeasurementIterator returns an iterator over all measurements.
541 func (i *Index) MeasurementIterator() (tsdb.MeasurementIterator, error) {
542 itrs := make([]tsdb.MeasurementIterator, 0, len(i.partitions))
543 for _, p := range i.partitions {
544 itr, err := p.MeasurementIterator()
545 if err != nil {
546 tsdb.MeasurementIterators(itrs).Close()
547 return nil, err
548 } else if itr != nil {
549 itrs = append(itrs, itr)
550 }
551 }
552 return tsdb.MergeMeasurementIterators(itrs...), nil
553 }
554
555 // MeasurementSeriesIDIterator returns an iterator over all series in a measurement.
556 func (i *Index) MeasurementSeriesIDIterator(name []byte) (tsdb.SeriesIDIterator, error) {
557 itrs := make([]tsdb.SeriesIDIterator, 0, len(i.partitions))
558 for _, p := range i.partitions {
559 itr, err := p.MeasurementSeriesIDIterator(name)
560 if err != nil {
561 tsdb.SeriesIDIterators(itrs).Close()
562 return nil, err
563 } else if itr != nil {
564 itrs = append(itrs, itr)
565 }
566 }
567 return tsdb.MergeSeriesIDIterators(itrs...), nil
568 }
569
570 // MeasurementNamesByRegex returns measurement names for the provided regex.
571 func (i *Index) MeasurementNamesByRegex(re *regexp.Regexp) ([][]byte, error) {
572 return i.fetchByteValues(func(idx int) ([][]byte, error) {
573 return i.partitions[idx].MeasurementNamesByRegex(re)
574 })
575 }
576
577 // DropMeasurement deletes a measurement from the index. It returns the first
578 // error encountered, if any.
579 func (i *Index) DropMeasurement(name []byte) error {
580 n := i.availableThreads()
581
582 // Store results.
583 errC := make(chan error, i.PartitionN)
584
585 var pidx uint32 // Index of maximum Partition being worked on.
586 for k := 0; k < n; k++ {
587 go func() {
588 for {
589 idx := int(atomic.AddUint32(&pidx, 1) - 1) // Get next partition to work on.
590 if idx >= len(i.partitions) {
591 return // No more work.
592 }
593 errC <- i.partitions[idx].DropMeasurement(name)
594 }
595 }()
596 }
597
598 // Check for error
599 for i := 0; i < cap(errC); i++ {
600 if err := <-errC; err != nil {
601 return err
602 }
603 }
604
605 // Update sketches under lock.
606 i.mu.Lock()
607 defer i.mu.Unlock()
608
609 i.mTSketch.Add(name)
610 if err := i.updateSeriesSketches(); err != nil {
611 return err
612 }
613
614 return nil
615 }
616
617 // CreateSeriesListIfNotExists creates a list of series if they doesn't exist in bulk.
618 func (i *Index) CreateSeriesListIfNotExists(keys [][]byte, names [][]byte, tagsSlice []models.Tags) error {
619 // All slices must be of equal length.
620 if len(names) != len(tagsSlice) {
621 return errors.New("names/tags length mismatch in index")
622 }
623
624 // We need to move different series into collections for each partition
625 // to process.
626 pNames := make([][][]byte, i.PartitionN)
627 pTags := make([][]models.Tags, i.PartitionN)
628
629 // Determine partition for series using each series key.
630 for ki, key := range keys {
631 pidx := i.partitionIdx(key)
632 pNames[pidx] = append(pNames[pidx], names[ki])
633 pTags[pidx] = append(pTags[pidx], tagsSlice[ki])
634 }
635
636 // Process each subset of series on each partition.
637 n := i.availableThreads()
638
639 // Store errors.
640 errC := make(chan error, i.PartitionN)
641
642 var pidx uint32 // Index of maximum Partition being worked on.
643 for k := 0; k < n; k++ {
644 go func() {
645 for {
646 idx := int(atomic.AddUint32(&pidx, 1) - 1) // Get next partition to work on.
647 if idx >= len(i.partitions) {
648 return // No more work.
649 }
650 errC <- i.partitions[idx].createSeriesListIfNotExists(pNames[idx], pTags[idx])
651 }
652 }()
653 }
654
655 // Check for error
656 for i := 0; i < cap(errC); i++ {
657 if err := <-errC; err != nil {
658 return err
659 }
660 }
661
662 // Update sketches under lock.
663 i.mu.Lock()
664 defer i.mu.Unlock()
665
666 for _, key := range keys {
667 i.sSketch.Add(key)
668 }
669 for _, name := range names {
670 i.mSketch.Add(name)
671 }
672
673 return nil
674 }
675
676 // CreateSeriesIfNotExists creates a series if it doesn't exist or is deleted.
677 func (i *Index) CreateSeriesIfNotExists(key, name []byte, tags models.Tags) error {
678 if err := i.partition(key).createSeriesListIfNotExists([][]byte{name}, []models.Tags{tags}); err != nil {
679 return err
680 }
681
682 i.mu.Lock()
683 i.sSketch.Add(key)
684 i.mSketch.Add(name)
685 i.mu.Unlock()
686 return nil
687 }
688
689 // InitializeSeries is a no-op. This only applies to the in-memory index.
690 func (i *Index) InitializeSeries(keys, names [][]byte, tags []models.Tags) error {
691 return nil
692 }
693
694 // DropSeries drops the provided series from the index. If cascade is true
695 // and this is the last series to the measurement, the measurment will also be dropped.
696 func (i *Index) DropSeries(seriesID uint64, key []byte, cascade bool) error {
697 // Remove from partition.
698 if err := i.partition(key).DropSeries(seriesID); err != nil {
699 return err
700 }
701
702 // Add sketch tombstone.
703 i.mu.Lock()
704 i.sTSketch.Add(key)
705 i.mu.Unlock()
706
707 if !cascade {
708 return nil
709 }
710
711 // Extract measurement name.
712 name, _ := models.ParseKeyBytes(key)
713
714 // Check if that was the last series for the measurement in the entire index.
715 if ok, err := i.MeasurementHasSeries(name); err != nil {
716 return err
717 } else if ok {
718 return nil
719 }
720
721 // If no more series exist in the measurement then delete the measurement.
722 if err := i.DropMeasurement(name); err != nil {
723 return err
724 }
725 return nil
726 }
727
728 // DropSeriesGlobal is a no-op on the tsi1 index.
729 func (i *Index) DropSeriesGlobal(key []byte) error { return nil }
730
731 // DropMeasurementIfSeriesNotExist drops a measurement only if there are no more
732 // series for the measurment.
733 func (i *Index) DropMeasurementIfSeriesNotExist(name []byte) error {
734 // Check if that was the last series for the measurement in the entire index.
735 if ok, err := i.MeasurementHasSeries(name); err != nil {
736 return err
737 } else if ok {
738 return nil
739 }
740
741 // If no more series exist in the measurement then delete the measurement.
742 return i.DropMeasurement(name)
743 }
744
745 // MeasurementsSketches returns the two measurement sketches for the index.
746 func (i *Index) MeasurementsSketches() (estimator.Sketch, estimator.Sketch, error) {
747 i.mu.RLock()
748 defer i.mu.RUnlock()
749 return i.mSketch.Clone(), i.mTSketch.Clone(), nil
750 }
751
752 // SeriesSketches returns the two series sketches for the index.
753 func (i *Index) SeriesSketches() (estimator.Sketch, estimator.Sketch, error) {
754 i.mu.RLock()
755 defer i.mu.RUnlock()
756 return i.sSketch.Clone(), i.sTSketch.Clone(), nil
757 }
758
759 // Since indexes are not shared across shards, the count returned by SeriesN
760 // cannot be combined with other shard's results. If you need to count series
761 // across indexes then use either the database-wide series file, or merge the
762 // index-level bitsets or sketches.
763 func (i *Index) SeriesN() int64 {
764 return int64(i.SeriesIDSet().Cardinality())
765 }
766
767 // HasTagKey returns true if tag key exists. It returns the first error
768 // encountered if any.
769 func (i *Index) HasTagKey(name, key []byte) (bool, error) {
770 n := i.availableThreads()
771
772 // Store errors
773 var found uint32 // Use this to signal we found the tag key.
774 errC := make(chan error, i.PartitionN)
775
776 // Check each partition for the tag key concurrently.
777 var pidx uint32 // Index of maximum Partition being worked on.
778 for k := 0; k < n; k++ {
779 go func() {
780 for {
781 idx := int(atomic.AddUint32(&pidx, 1) - 1) // Get next partition to check
782 if idx >= len(i.partitions) {
783 return // No more work.
784 }
785
786 // Check if the tag key has already been found. If it has, we
787 // don't need to check this partition and can just move on.
788 if atomic.LoadUint32(&found) == 1 {
789 errC <- nil
790 continue
791 }
792
793 b, err := i.partitions[idx].HasTagKey(name, key)
794 if b {
795 atomic.StoreUint32(&found, 1)
796 }
797 errC <- err
798 }
799 }()
800 }
801
802 // Check for error
803 for i := 0; i < cap(errC); i++ {
804 if err := <-errC; err != nil {
805 return false, err
806 }
807 }
808
809 // Check if we found the tag key.
810 return atomic.LoadUint32(&found) == 1, nil
811 }
812
813 // HasTagValue returns true if tag value exists.
814 func (i *Index) HasTagValue(name, key, value []byte) (bool, error) {
815 n := i.availableThreads()
816
817 // Store errors
818 var found uint32 // Use this to signal we found the tag key.
819 errC := make(chan error, i.PartitionN)
820
821 // Check each partition for the tag key concurrently.
822 var pidx uint32 // Index of maximum Partition being worked on.
823 for k := 0; k < n; k++ {
824 go func() {
825 for {
826 idx := int(atomic.AddUint32(&pidx, 1) - 1) // Get next partition to check
827 if idx >= len(i.partitions) {
828 return // No more work.
829 }
830
831 // Check if the tag key has already been found. If it has, we
832 // don't need to check this partition and can just move on.
833 if atomic.LoadUint32(&found) == 1 {
834 errC <- nil
835 continue
836 }
837
838 b, err := i.partitions[idx].HasTagValue(name, key, value)
839 if b {
840 atomic.StoreUint32(&found, 1)
841 }
842 errC <- err
843 }
844 }()
845 }
846
847 // Check for error
848 for i := 0; i < cap(errC); i++ {
849 if err := <-errC; err != nil {
850 return false, err
851 }
852 }
853
854 // Check if we found the tag key.
855 return atomic.LoadUint32(&found) == 1, nil
856 }
857
858 // TagKeyIterator returns an iterator for all keys across a single measurement.
859 func (i *Index) TagKeyIterator(name []byte) (tsdb.TagKeyIterator, error) {
860 a := make([]tsdb.TagKeyIterator, 0, len(i.partitions))
861 for _, p := range i.partitions {
862 itr := p.TagKeyIterator(name)
863 if itr != nil {
864 a = append(a, itr)
865 }
866 }
867 return tsdb.MergeTagKeyIterators(a...), nil
868 }
869
870 // TagValueIterator returns an iterator for all values across a single key.
871 func (i *Index) TagValueIterator(name, key []byte) (tsdb.TagValueIterator, error) {
872 a := make([]tsdb.TagValueIterator, 0, len(i.partitions))
873 for _, p := range i.partitions {
874 itr := p.TagValueIterator(name, key)
875 if itr != nil {
876 a = append(a, itr)
877 }
878 }
879 return tsdb.MergeTagValueIterators(a...), nil
880 }
881
882 // TagKeySeriesIDIterator returns a series iterator for all values across a single key.
883 func (i *Index) TagKeySeriesIDIterator(name, key []byte) (tsdb.SeriesIDIterator, error) {
884 a := make([]tsdb.SeriesIDIterator, 0, len(i.partitions))
885 for _, p := range i.partitions {
886 itr := p.TagKeySeriesIDIterator(name, key)
887 if itr != nil {
888 a = append(a, itr)
889 }
890 }
891 return tsdb.MergeSeriesIDIterators(a...), nil
892 }
893
894 // TagValueSeriesIDIterator returns a series iterator for a single tag value.
895 func (i *Index) TagValueSeriesIDIterator(name, key, value []byte) (tsdb.SeriesIDIterator, error) {
896 a := make([]tsdb.SeriesIDIterator, 0, len(i.partitions))
897 for _, p := range i.partitions {
898 itr := p.TagValueSeriesIDIterator(name, key, value)
899 if itr != nil {
900 a = append(a, itr)
901 }
902 }
903 return tsdb.MergeSeriesIDIterators(a...), nil
904 }
905
906 // MeasurementTagKeysByExpr extracts the tag keys wanted by the expression.
907 func (i *Index) MeasurementTagKeysByExpr(name []byte, expr influxql.Expr) (map[string]struct{}, error) {
908 n := i.availableThreads()
909
910 // Store results.
911 keys := make([]map[string]struct{}, i.PartitionN)
912 errC := make(chan error, i.PartitionN)
913
914 var pidx uint32 // Index of maximum Partition being worked on.
915 for k := 0; k < n; k++ {
916 go func() {
917 for {
918 idx := int(atomic.AddUint32(&pidx, 1) - 1) // Get next partition to work on.
919 if idx >= len(i.partitions) {
920 return // No more work.
921 }
922
923 // This is safe since there are no readers on keys until all
924 // the writers are done.
925 tagKeys, err := i.partitions[idx].MeasurementTagKeysByExpr(name, expr)
926 keys[idx] = tagKeys
927 errC <- err
928 }
929 }()
930 }
931
932 // Check for error
933 for i := 0; i < cap(errC); i++ {
934 if err := <-errC; err != nil {
935 return nil, err
936 }
937 }
938
939 // Merge into single map.
940 result := keys[0]
941 for k := 1; k < len(i.partitions); k++ {
942 for k := range keys[k] {
943 result[k] = struct{}{}
944 }
945 }
946 return result, nil
947 }
948
949 // DiskSizeBytes returns the size of the index on disk.
950 func (i *Index) DiskSizeBytes() int64 {
951 fs, err := i.RetainFileSet()
952 if err != nil {
953 i.logger.Warn("Index is closing down")
954 return 0
955 }
956 defer fs.Release()
957
958 var manifestSize int64
959 // Get MANIFEST sizes from each partition.
960 for _, p := range i.partitions {
961 manifestSize += p.manifestSize
962 }
963 return fs.Size() + manifestSize
964 }
965
966 // TagKeyCardinality always returns zero.
967 // It is not possible to determine cardinality of tags across index files, and
968 // thus it cannot be done across partitions.
969 func (i *Index) TagKeyCardinality(name, key []byte) int {
970 return 0
971 }
972
973 // RetainFileSet returns the set of all files across all partitions.
974 // This is only needed when all files need to be retained for an operation.
975 func (i *Index) RetainFileSet() (*FileSet, error) {
976 i.mu.RLock()
977 defer i.mu.RUnlock()
978
979 fs, _ := NewFileSet(nil, i.sfile, nil)
980 for _, p := range i.partitions {
981 pfs, err := p.RetainFileSet()
982 if err != nil {
983 fs.Close()
984 return nil, err
985 }
986 fs.files = append(fs.files, pfs.files...)
987 }
988 return fs, nil
989 }
990
991 // SetFieldName is a no-op on this index.
992 func (i *Index) SetFieldName(measurement []byte, name string) {}
993
994 // Rebuild rebuilds an index. It's a no-op for this index.
995 func (i *Index) Rebuild() {}
996
997 // IsIndexDir returns true if directory contains at least one partition directory.
998 func IsIndexDir(path string) (bool, error) {
999 fis, err := ioutil.ReadDir(path)
1000 if err != nil {
1001 return false, err
1002 }
1003 for _, fi := range fis {
1004 if !fi.IsDir() {
1005 continue
1006 } else if ok, err := IsPartitionDir(filepath.Join(path, fi.Name())); err != nil {
1007 return false, err
1008 } else if ok {
1009 return true, nil
1010 }
1011 }
1012 return false, nil
1013 }