Debian Codegraph @

tsdb/index/tsi1/index_file.go

raw
1 package tsi1
2
3 import (
4 "bytes"
5 "encoding/binary"
6 "errors"
7 "fmt"
8 "io"
9 "sync"
10 "unsafe"
11
12 "github.com/influxdata/influxdb/models"
13 "github.com/influxdata/influxdb/pkg/estimator"
14 "github.com/influxdata/influxdb/pkg/estimator/hll"
15 "github.com/influxdata/influxdb/pkg/mmap"
16 "github.com/influxdata/influxdb/tsdb"
17 )
18
19 // IndexFileVersion is the current TSI1 index file version.
20 const IndexFileVersion = 1
21
22 // FileSignature represents a magic number at the header of the index file.
23 const FileSignature = "TSI1"
24
25 // IndexFile field size constants.
26 const (
27 // IndexFile trailer fields
28 IndexFileVersionSize = 2
29
30 // IndexFileTrailerSize is the size of the trailer. Currently 82 bytes.
31 IndexFileTrailerSize = IndexFileVersionSize +
32 8 + 8 + // measurement block offset + size
33 8 + 8 + // series id set offset + size
34 8 + 8 + // tombstone series id set offset + size
35 8 + 8 + // series sketch offset + size
36 8 + 8 + // tombstone series sketch offset + size
37 0
38 )
39
40 // IndexFile errors.
41 var (
42 ErrInvalidIndexFile = errors.New("invalid index file")
43 ErrUnsupportedIndexFileVersion = errors.New("unsupported index file version")
44 )
45
46 // IndexFile represents a collection of measurement, tag, and series data.
47 type IndexFile struct {
48 wg sync.WaitGroup // ref count
49 data []byte
50
51 // Components
52 sfile *tsdb.SeriesFile
53 tblks map[string]*TagBlock // tag blocks by measurement name
54 mblk MeasurementBlock
55
56 // Raw series set data.
57 seriesIDSetData []byte
58 tombstoneSeriesIDSetData []byte
59
60 // Series sketch data.
61 sketchData, tSketchData []byte
62
63 // Sortable identifier & filepath to the log file.
64 level int
65 id int
66
67 // Compaction tracking.
68 mu sync.RWMutex
69 compacting bool
70
71 // Path to data file.
72 path string
73 }
74
75 // NewIndexFile returns a new instance of IndexFile.
76 func NewIndexFile(sfile *tsdb.SeriesFile) *IndexFile {
77 return &IndexFile{
78 sfile: sfile,
79 }
80 }
81
82 // bytes estimates the memory footprint of this IndexFile, in bytes.
83 func (f *IndexFile) bytes() int {
84 var b int
85 f.wg.Add(1)
86 b += 16 // wg WaitGroup is 16 bytes
87 b += int(unsafe.Sizeof(f.data))
88 // Do not count f.data contents because it is mmap'd
89 b += int(unsafe.Sizeof(f.sfile))
90 // Do not count SeriesFile because it belongs to the code that constructed this IndexFile.
91 b += int(unsafe.Sizeof(f.tblks))
92 for k, v := range f.tblks {
93 // Do not count TagBlock contents, they all reference f.data
94 b += int(unsafe.Sizeof(k)) + len(k)
95 b += int(unsafe.Sizeof(*v))
96 }
97 b += int(unsafe.Sizeof(f.mblk)) + f.mblk.bytes()
98 b += int(unsafe.Sizeof(f.seriesIDSetData) + unsafe.Sizeof(f.tombstoneSeriesIDSetData))
99 // Do not count contents of seriesIDSetData or tombstoneSeriesIDSetData: references f.data
100 b += int(unsafe.Sizeof(f.level) + unsafe.Sizeof(f.id))
101 b += 24 // mu RWMutex is 24 bytes
102 b += int(unsafe.Sizeof(f.compacting))
103 b += int(unsafe.Sizeof(f.path)) + len(f.path)
104 f.wg.Done()
105 return b
106 }
107
108 // Open memory maps the data file at the file's path.
109 func (f *IndexFile) Open() error {
110 defer func() {
111 if err := recover(); err != nil {
112 err = fmt.Errorf("[Index file: %s] %v", f.path, err)
113 panic(err)
114 }
115 }()
116
117 // Extract identifier from path name.
118 f.id, f.level = ParseFilename(f.Path())
119
120 data, err := mmap.Map(f.Path(), 0)
121 if err != nil {
122 return err
123 }
124
125 return f.UnmarshalBinary(data)
126 }
127
128 // Close unmaps the data file.
129 func (f *IndexFile) Close() error {
130 // Wait until all references are released.
131 f.wg.Wait()
132
133 f.sfile = nil
134 f.tblks = nil
135 f.mblk = MeasurementBlock{}
136 return mmap.Unmap(f.data)
137 }
138
139 // ID returns the file sequence identifier.
140 func (f *IndexFile) ID() int { return f.id }
141
142 // Path returns the file path.
143 func (f *IndexFile) Path() string { return f.path }
144
145 // SetPath sets the file's path.
146 func (f *IndexFile) SetPath(path string) { f.path = path }
147
148 // Level returns the compaction level for the file.
149 func (f *IndexFile) Level() int { return f.level }
150
151 // Retain adds a reference count to the file.
152 func (f *IndexFile) Retain() { f.wg.Add(1) }
153
154 // Release removes a reference count from the file.
155 func (f *IndexFile) Release() { f.wg.Done() }
156
157 // Size returns the size of the index file, in bytes.
158 func (f *IndexFile) Size() int64 { return int64(len(f.data)) }
159
160 // Compacting returns true if the file is being compacted.
161 func (f *IndexFile) Compacting() bool {
162 f.mu.RLock()
163 v := f.compacting
164 f.mu.RUnlock()
165 return v
166 }
167
168 // UnmarshalBinary opens an index from data.
169 // The byte slice is retained so it must be kept open.
170 func (f *IndexFile) UnmarshalBinary(data []byte) error {
171 // Ensure magic number exists at the beginning.
172 if len(data) < len(FileSignature) {
173 return io.ErrShortBuffer
174 } else if !bytes.Equal(data[:len(FileSignature)], []byte(FileSignature)) {
175 return ErrInvalidIndexFile
176 }
177
178 // Read index file trailer.
179 t, err := ReadIndexFileTrailer(data)
180 if err != nil {
181 return err
182 }
183
184 // Slice series sketch data.
185 f.sketchData = data[t.SeriesSketch.Offset : t.SeriesSketch.Offset+t.SeriesSketch.Size]
186 f.tSketchData = data[t.TombstoneSeriesSketch.Offset : t.TombstoneSeriesSketch.Offset+t.TombstoneSeriesSketch.Size]
187
188 // Slice series set data.
189 f.seriesIDSetData = data[t.SeriesIDSet.Offset : t.SeriesIDSet.Offset+t.SeriesIDSet.Size]
190 f.tombstoneSeriesIDSetData = data[t.TombstoneSeriesIDSet.Offset : t.TombstoneSeriesIDSet.Offset+t.TombstoneSeriesIDSet.Size]
191
192 // Unmarshal measurement block.
193 if err := f.mblk.UnmarshalBinary(data[t.MeasurementBlock.Offset:][:t.MeasurementBlock.Size]); err != nil {
194 return err
195 }
196
197 // Unmarshal each tag block.
198 f.tblks = make(map[string]*TagBlock)
199 itr := f.mblk.Iterator()
200
201 for m := itr.Next(); m != nil; m = itr.Next() {
202 e := m.(*MeasurementBlockElem)
203
204 // Slice measurement block data.
205 buf := data[e.tagBlock.offset:]
206 buf = buf[:e.tagBlock.size]
207
208 // Unmarshal measurement block.
209 var tblk TagBlock
210 if err := tblk.UnmarshalBinary(buf); err != nil {
211 return err
212 }
213 f.tblks[string(e.name)] = &tblk
214 }
215
216 // Save reference to entire data block.
217 f.data = data
218
219 return nil
220 }
221
222 func (f *IndexFile) SeriesIDSet() (*tsdb.SeriesIDSet, error) {
223 ss := tsdb.NewSeriesIDSet()
224 if err := ss.UnmarshalBinary(f.seriesIDSetData); err != nil {
225 return nil, err
226 }
227 return ss, nil
228 }
229
230 func (f *IndexFile) TombstoneSeriesIDSet() (*tsdb.SeriesIDSet, error) {
231 ss := tsdb.NewSeriesIDSet()
232 if err := ss.UnmarshalBinary(f.tombstoneSeriesIDSetData); err != nil {
233 return nil, err
234 }
235 return ss, nil
236 }
237
238 // Measurement returns a measurement element.
239 func (f *IndexFile) Measurement(name []byte) MeasurementElem {
240 e, ok := f.mblk.Elem(name)
241 if !ok {
242 return nil
243 }
244 return &e
245 }
246
247 // MeasurementN returns the number of measurements in the file.
248 func (f *IndexFile) MeasurementN() (n uint64) {
249 mitr := f.mblk.Iterator()
250 for me := mitr.Next(); me != nil; me = mitr.Next() {
251 n++
252 }
253 return n
254 }
255
256 // MeasurementHasSeries returns true if a measurement has any non-tombstoned series.
257 func (f *IndexFile) MeasurementHasSeries(ss *tsdb.SeriesIDSet, name []byte) (ok bool) {
258 e, ok := f.mblk.Elem(name)
259 if !ok {
260 return false
261 }
262
263 var exists bool
264 e.ForEachSeriesID(func(id uint64) error {
265 if ss.Contains(id) {
266 exists = true
267 return errors.New("done")
268 }
269 return nil
270 })
271 return exists
272 }
273
274 // TagValueIterator returns a value iterator for a tag key and a flag
275 // indicating if a tombstone exists on the measurement or key.
276 func (f *IndexFile) TagValueIterator(name, key []byte) TagValueIterator {
277 tblk := f.tblks[string(name)]
278 if tblk == nil {
279 return nil
280 }
281
282 // Find key element.
283 ke := tblk.TagKeyElem(key)
284 if ke == nil {
285 return nil
286 }
287
288 // Merge all value series iterators together.
289 return ke.TagValueIterator()
290 }
291
292 // TagKeySeriesIDIterator returns a series iterator for a tag key and a flag
293 // indicating if a tombstone exists on the measurement or key.
294 func (f *IndexFile) TagKeySeriesIDIterator(name, key []byte) tsdb.SeriesIDIterator {
295 tblk := f.tblks[string(name)]
296 if tblk == nil {
297 return nil
298 }
299
300 // Find key element.
301 ke := tblk.TagKeyElem(key)
302 if ke == nil {
303 return nil
304 }
305
306 // Merge all value series iterators together.
307 vitr := ke.TagValueIterator()
308 var itrs []tsdb.SeriesIDIterator
309 for ve := vitr.Next(); ve != nil; ve = vitr.Next() {
310 sitr := &rawSeriesIDIterator{data: ve.(*TagBlockValueElem).series.data}
311 itrs = append(itrs, sitr)
312 }
313
314 return tsdb.MergeSeriesIDIterators(itrs...)
315 }
316
317 // TagValueSeriesIDIterator returns a series iterator for a tag value and a flag
318 // indicating if a tombstone exists on the measurement, key, or value.
319 func (f *IndexFile) TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator {
320 tblk := f.tblks[string(name)]
321 if tblk == nil {
322 return nil
323 }
324
325 // Find value element.
326 n, data := tblk.TagValueSeriesData(key, value)
327 if n == 0 {
328 return nil
329 }
330
331 // Create an iterator over value's series.
332 return &rawSeriesIDIterator{n: n, data: data}
333 }
334
335 // TagKey returns a tag key.
336 func (f *IndexFile) TagKey(name, key []byte) TagKeyElem {
337 tblk := f.tblks[string(name)]
338 if tblk == nil {
339 return nil
340 }
341 return tblk.TagKeyElem(key)
342 }
343
344 // TagValue returns a tag value.
345 func (f *IndexFile) TagValue(name, key, value []byte) TagValueElem {
346 tblk := f.tblks[string(name)]
347 if tblk == nil {
348 return nil
349 }
350 return tblk.TagValueElem(key, value)
351 }
352
353 // HasSeries returns flags indicating if the series exists and if it is tombstoned.
354 func (f *IndexFile) HasSeries(name []byte, tags models.Tags, buf []byte) (exists, tombstoned bool) {
355 return f.sfile.HasSeries(name, tags, buf), false // TODO(benbjohnson): series tombstone
356 }
357
358 // TagValueElem returns an element for a measurement/tag/value.
359 func (f *IndexFile) TagValueElem(name, key, value []byte) TagValueElem {
360 tblk, ok := f.tblks[string(name)]
361 if !ok {
362 return nil
363 }
364 return tblk.TagValueElem(key, value)
365 }
366
367 // MeasurementIterator returns an iterator over all measurements.
368 func (f *IndexFile) MeasurementIterator() MeasurementIterator {
369 return f.mblk.Iterator()
370 }
371
372 // TagKeyIterator returns an iterator over all tag keys for a measurement.
373 func (f *IndexFile) TagKeyIterator(name []byte) TagKeyIterator {
374 blk := f.tblks[string(name)]
375 if blk == nil {
376 return nil
377 }
378 return blk.TagKeyIterator()
379 }
380
381 // MeasurementSeriesIDIterator returns an iterator over a measurement's series.
382 func (f *IndexFile) MeasurementSeriesIDIterator(name []byte) tsdb.SeriesIDIterator {
383 return f.mblk.SeriesIDIterator(name)
384 }
385
386 // MeasurementsSketches returns existence and tombstone sketches for measurements.
387 func (f *IndexFile) MeasurementsSketches() (sketch, tSketch estimator.Sketch, err error) {
388 return f.mblk.Sketches()
389 }
390
391 // SeriesSketches returns existence and tombstone sketches for series.
392 func (f *IndexFile) SeriesSketches() (sketch, tSketch estimator.Sketch, err error) {
393 sketch = hll.NewDefaultPlus()
394 if err := sketch.UnmarshalBinary(f.sketchData); err != nil {
395 return nil, nil, err
396 }
397
398 tSketch = hll.NewDefaultPlus()
399 if err := tSketch.UnmarshalBinary(f.tSketchData); err != nil {
400 return nil, nil, err
401 }
402 return sketch, tSketch, nil
403 }
404
405 // ReadIndexFileTrailer returns the index file trailer from data.
406 func ReadIndexFileTrailer(data []byte) (IndexFileTrailer, error) {
407 var t IndexFileTrailer
408
409 // Read version.
410 t.Version = int(binary.BigEndian.Uint16(data[len(data)-IndexFileVersionSize:]))
411 if t.Version != IndexFileVersion {
412 return t, ErrUnsupportedIndexFileVersion
413 }
414
415 // Slice trailer data.
416 buf := data[len(data)-IndexFileTrailerSize:]
417
418 // Read measurement block info.
419 t.MeasurementBlock.Offset, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
420 t.MeasurementBlock.Size, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
421
422 // Read series id set info.
423 t.SeriesIDSet.Offset, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
424 t.SeriesIDSet.Size, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
425
426 // Read series tombstone id set info.
427 t.TombstoneSeriesIDSet.Offset, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
428 t.TombstoneSeriesIDSet.Size, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
429
430 // Read series sketch set info.
431 t.SeriesSketch.Offset, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
432 t.SeriesSketch.Size, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
433
434 // Read series tombstone sketch info.
435 t.TombstoneSeriesSketch.Offset, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
436 t.TombstoneSeriesSketch.Size, buf = int64(binary.BigEndian.Uint64(buf[0:8])), buf[8:]
437
438 if len(buf) != 2 { // Version field still in buffer.
439 return t, fmt.Errorf("unread %d bytes left unread in trailer", len(buf)-2)
440 }
441 return t, nil
442 }
443
444 // IndexFileTrailer represents meta data written to the end of the index file.
445 type IndexFileTrailer struct {
446 Version int
447
448 MeasurementBlock struct {
449 Offset int64
450 Size int64
451 }
452
453 SeriesIDSet struct {
454 Offset int64
455 Size int64
456 }
457
458 TombstoneSeriesIDSet struct {
459 Offset int64
460 Size int64
461 }
462
463 SeriesSketch struct {
464 Offset int64
465 Size int64
466 }
467
468 TombstoneSeriesSketch struct {
469 Offset int64
470 Size int64
471 }
472 }
473
474 // WriteTo writes the trailer to w.
475 func (t *IndexFileTrailer) WriteTo(w io.Writer) (n int64, err error) {
476 // Write measurement block info.
477 if err := writeUint64To(w, uint64(t.MeasurementBlock.Offset), &n); err != nil {
478 return n, err
479 } else if err := writeUint64To(w, uint64(t.MeasurementBlock.Size), &n); err != nil {
480 return n, err
481 }
482
483 // Write series id set info.
484 if err := writeUint64To(w, uint64(t.SeriesIDSet.Offset), &n); err != nil {
485 return n, err
486 } else if err := writeUint64To(w, uint64(t.SeriesIDSet.Size), &n); err != nil {
487 return n, err
488 }
489
490 // Write tombstone series id set info.
491 if err := writeUint64To(w, uint64(t.TombstoneSeriesIDSet.Offset), &n); err != nil {
492 return n, err
493 } else if err := writeUint64To(w, uint64(t.TombstoneSeriesIDSet.Size), &n); err != nil {
494 return n, err
495 }
496
497 // Write series sketch info.
498 if err := writeUint64To(w, uint64(t.SeriesSketch.Offset), &n); err != nil {
499 return n, err
500 } else if err := writeUint64To(w, uint64(t.SeriesSketch.Size), &n); err != nil {
501 return n, err
502 }
503
504 // Write series tombstone sketch info.
505 if err := writeUint64To(w, uint64(t.TombstoneSeriesSketch.Offset), &n); err != nil {
506 return n, err
507 } else if err := writeUint64To(w, uint64(t.TombstoneSeriesSketch.Size), &n); err != nil {
508 return n, err
509 }
510
511 // Write index file encoding version.
512 if err := writeUint16To(w, IndexFileVersion, &n); err != nil {
513 return n, err
514 }
515
516 return n, nil
517 }
518
519 // FormatIndexFileName generates an index filename for the given index.
520 func FormatIndexFileName(id, level int) string {
521 return fmt.Sprintf("L%d-%08d%s", level, id, IndexFileExt)
522 }