Debian Codegraph @

tsdb/index/tsi1/index_files.go

raw
1 package tsi1
2
3 import (
4 "bufio"
5 "io"
6 "os"
7 "sort"
8 "time"
9
10 "github.com/influxdata/influxdb/pkg/bytesutil"
11 "github.com/influxdata/influxdb/pkg/estimator/hll"
12 "github.com/influxdata/influxdb/tsdb"
13 )
14
15 // IndexFiles represents a layered set of index files.
16 type IndexFiles []*IndexFile
17
18 // IDs returns the ids for all index files.
19 func (p IndexFiles) IDs() []int {
20 a := make([]int, len(p))
21 for i, f := range p {
22 a[i] = f.ID()
23 }
24 return a
25 }
26
27 // Retain adds a reference count to all files.
28 func (p IndexFiles) Retain() {
29 for _, f := range p {
30 f.Retain()
31 }
32 }
33
34 // Release removes a reference count from all files.
35 func (p IndexFiles) Release() {
36 for _, f := range p {
37 f.Release()
38 }
39 }
40
41 // Files returns p as a list of File objects.
42 func (p IndexFiles) Files() []File {
43 other := make([]File, len(p))
44 for i, f := range p {
45 other[i] = f
46 }
47 return other
48 }
49
50 func (p IndexFiles) buildSeriesIDSets() (seriesIDSet, tombstoneSeriesIDSet *tsdb.SeriesIDSet, err error) {
51 if len(p) == 0 {
52 return tsdb.NewSeriesIDSet(), tsdb.NewSeriesIDSet(), nil
53 }
54
55 // Start with sets from last file.
56 if seriesIDSet, err = p[len(p)-1].SeriesIDSet(); err != nil {
57 return nil, nil, err
58 } else if tombstoneSeriesIDSet, err = p[len(p)-1].TombstoneSeriesIDSet(); err != nil {
59 return nil, nil, err
60 }
61
62 // Build sets in reverse order.
63 // This assumes that bits in both sets are mutually exclusive.
64 for i := len(p) - 2; i >= 0; i-- {
65 ss, err := p[i].SeriesIDSet()
66 if err != nil {
67 return nil, nil, err
68 }
69
70 ts, err := p[i].TombstoneSeriesIDSet()
71 if err != nil {
72 return nil, nil, err
73 }
74
75 // Add tombstones and remove from old series existence set.
76 seriesIDSet.Diff(ts)
77 tombstoneSeriesIDSet.Merge(ts)
78
79 // Add new series and remove from old series tombstone set.
80 tombstoneSeriesIDSet.Diff(ss)
81 seriesIDSet.Merge(ss)
82 }
83
84 return seriesIDSet, tombstoneSeriesIDSet, nil
85 }
86
87 // MeasurementNames returns a sorted list of all measurement names for all files.
88 func (p *IndexFiles) MeasurementNames() [][]byte {
89 itr := p.MeasurementIterator()
90 if itr == nil {
91 return nil
92 }
93
94 var names [][]byte
95 for e := itr.Next(); e != nil; e = itr.Next() {
96 names = append(names, bytesutil.Clone(e.Name()))
97 }
98 sort.Sort(byteSlices(names))
99 return names
100 }
101
102 // MeasurementIterator returns an iterator that merges measurements across all files.
103 func (p IndexFiles) MeasurementIterator() MeasurementIterator {
104 a := make([]MeasurementIterator, 0, len(p))
105 for i := range p {
106 itr := p[i].MeasurementIterator()
107 if itr == nil {
108 continue
109 }
110 a = append(a, itr)
111 }
112 return MergeMeasurementIterators(a...)
113 }
114
115 // TagKeyIterator returns an iterator that merges tag keys across all files.
116 func (p *IndexFiles) TagKeyIterator(name []byte) (TagKeyIterator, error) {
117 a := make([]TagKeyIterator, 0, len(*p))
118 for _, f := range *p {
119 itr := f.TagKeyIterator(name)
120 if itr == nil {
121 continue
122 }
123 a = append(a, itr)
124 }
125 return MergeTagKeyIterators(a...), nil
126 }
127
128 // MeasurementSeriesIDIterator returns an iterator that merges series across all files.
129 func (p IndexFiles) MeasurementSeriesIDIterator(name []byte) tsdb.SeriesIDIterator {
130 a := make([]tsdb.SeriesIDIterator, 0, len(p))
131 for _, f := range p {
132 itr := f.MeasurementSeriesIDIterator(name)
133 if itr == nil {
134 continue
135 }
136 a = append(a, itr)
137 }
138 return tsdb.MergeSeriesIDIterators(a...)
139 }
140
141 // TagValueSeriesIDIterator returns an iterator that merges series across all files.
142 func (p IndexFiles) TagValueSeriesIDIterator(name, key, value []byte) tsdb.SeriesIDIterator {
143 a := make([]tsdb.SeriesIDIterator, 0, len(p))
144
145 for i := range p {
146 itr := p[i].TagValueSeriesIDIterator(name, key, value)
147 if itr != nil {
148 a = append(a, itr)
149 }
150 }
151 return tsdb.MergeSeriesIDIterators(a...)
152 }
153
154 // CompactTo merges all index files and writes them to w.
155 func (p IndexFiles) CompactTo(w io.Writer, sfile *tsdb.SeriesFile, m, k uint64, cancel <-chan struct{}) (n int64, err error) {
156 var t IndexFileTrailer
157
158 // Check for cancellation.
159 select {
160 case <-cancel:
161 return n, ErrCompactionInterrupted
162 default:
163 }
164
165 // Wrap writer in buffered I/O.
166 bw := bufio.NewWriter(w)
167
168 // Setup context object to track shared data for this compaction.
169 var info indexCompactInfo
170 info.cancel = cancel
171 info.tagSets = make(map[string]indexTagSetPos)
172
173 // Write magic number.
174 if err := writeTo(bw, []byte(FileSignature), &n); err != nil {
175 return n, err
176 }
177
178 // Flush buffer before re-mapping.
179 if err := bw.Flush(); err != nil {
180 return n, err
181 }
182
183 // Write tagset blocks in measurement order.
184 if err := p.writeTagsetsTo(bw, &info, &n); err != nil {
185 return n, err
186 }
187
188 // Write measurement block.
189 t.MeasurementBlock.Offset = n
190 if err := p.writeMeasurementBlockTo(bw, &info, &n); err != nil {
191 return n, err
192 }
193 t.MeasurementBlock.Size = n - t.MeasurementBlock.Offset
194
195 // Build series sets.
196 seriesIDSet, tombstoneSeriesIDSet, err := p.buildSeriesIDSets()
197 if err != nil {
198 return n, err
199 }
200
201 // Generate sketches from series sets.
202 sketch := hll.NewDefaultPlus()
203 seriesIDSet.ForEach(func(id uint64) {
204 if key := sfile.SeriesKey(id); key != nil {
205 sketch.Add(key)
206 }
207 })
208
209 tSketch := hll.NewDefaultPlus()
210 tombstoneSeriesIDSet.ForEach(func(id uint64) {
211 if key := sfile.SeriesKey(id); key != nil {
212 tSketch.Add(key)
213 }
214 })
215
216 // Write series set.
217 t.SeriesIDSet.Offset = n
218 nn, err := seriesIDSet.WriteTo(bw)
219 if n += nn; err != nil {
220 return n, err
221 }
222 t.SeriesIDSet.Size = n - t.SeriesIDSet.Offset
223
224 // Write tombstone series set.
225 t.TombstoneSeriesIDSet.Offset = n
226 nn, err = tombstoneSeriesIDSet.WriteTo(bw)
227 if n += nn; err != nil {
228 return n, err
229 }
230 t.TombstoneSeriesIDSet.Size = n - t.TombstoneSeriesIDSet.Offset
231
232 // Write series sketches. TODO(edd): Implement WriterTo on HLL++.
233 t.SeriesSketch.Offset = n
234 data, err := sketch.MarshalBinary()
235 if err != nil {
236 return n, err
237 } else if _, err := bw.Write(data); err != nil {
238 return n, err
239 }
240 t.SeriesSketch.Size = int64(len(data))
241 n += t.SeriesSketch.Size
242
243 t.TombstoneSeriesSketch.Offset = n
244 if data, err = tSketch.MarshalBinary(); err != nil {
245 return n, err
246 } else if _, err := bw.Write(data); err != nil {
247 return n, err
248 }
249 t.TombstoneSeriesSketch.Size = int64(len(data))
250 n += t.TombstoneSeriesSketch.Size
251
252 // Write trailer.
253 nn, err = t.WriteTo(bw)
254 n += nn
255 if err != nil {
256 return n, err
257 }
258
259 // Flush file.
260 if err := bw.Flush(); err != nil {
261 return n, err
262 }
263
264 return n, nil
265 }
266
267 func (p IndexFiles) writeTagsetsTo(w io.Writer, info *indexCompactInfo, n *int64) error {
268 mitr := p.MeasurementIterator()
269 if mitr == nil {
270 return nil
271 }
272
273 for m := mitr.Next(); m != nil; m = mitr.Next() {
274 if err := p.writeTagsetTo(w, m.Name(), info, n); err != nil {
275 return err
276 }
277 }
278 return nil
279 }
280
281 // writeTagsetTo writes a single tagset to w and saves the tagset offset.
282 func (p IndexFiles) writeTagsetTo(w io.Writer, name []byte, info *indexCompactInfo, n *int64) error {
283 var seriesIDs []uint64
284
285 // Check for cancellation.
286 select {
287 case <-info.cancel:
288 return ErrCompactionInterrupted
289 default:
290 }
291
292 kitr, err := p.TagKeyIterator(name)
293 if err != nil {
294 return err
295 }
296
297 var seriesN int
298 enc := NewTagBlockEncoder(w)
299 for ke := kitr.Next(); ke != nil; ke = kitr.Next() {
300 // Encode key.
301 if err := enc.EncodeKey(ke.Key(), ke.Deleted()); err != nil {
302 return err
303 }
304
305 // Iterate over tag values.
306 vitr := ke.TagValueIterator()
307 for ve := vitr.Next(); ve != nil; ve = vitr.Next() {
308 seriesIDs = seriesIDs[:0]
309
310 // Merge all series together.
311 if err := func() error {
312 sitr := p.TagValueSeriesIDIterator(name, ke.Key(), ve.Value())
313 if sitr != nil {
314 defer sitr.Close()
315 for {
316 se, err := sitr.Next()
317 if err != nil {
318 return err
319 } else if se.SeriesID == 0 {
320 break
321 }
322 seriesIDs = append(seriesIDs, se.SeriesID)
323
324 // Check for cancellation periodically.
325 if seriesN++; seriesN%1000 == 0 {
326 select {
327 case <-info.cancel:
328 return ErrCompactionInterrupted
329 default:
330 }
331 }
332 }
333 }
334
335 // Encode value.
336 return enc.EncodeValue(ve.Value(), ve.Deleted(), seriesIDs)
337 }(); err != nil {
338 return nil
339 }
340 }
341 }
342
343 // Save tagset offset to measurement.
344 pos := info.tagSets[string(name)]
345 pos.offset = *n
346
347 // Flush data to writer.
348 err = enc.Close()
349 *n += enc.N()
350 if err != nil {
351 return err
352 }
353
354 // Save tagset size to measurement.
355 pos.size = *n - pos.offset
356
357 info.tagSets[string(name)] = pos
358
359 return nil
360 }
361
362 func (p IndexFiles) writeMeasurementBlockTo(w io.Writer, info *indexCompactInfo, n *int64) error {
363 mw := NewMeasurementBlockWriter()
364
365 // Check for cancellation.
366 select {
367 case <-info.cancel:
368 return ErrCompactionInterrupted
369 default:
370 }
371
372 // Add measurement data & compute sketches.
373 mitr := p.MeasurementIterator()
374 if mitr != nil {
375 var seriesN int
376 for m := mitr.Next(); m != nil; m = mitr.Next() {
377 name := m.Name()
378
379 // Look-up series ids.
380 if err := func() error {
381 itr := p.MeasurementSeriesIDIterator(name)
382 defer itr.Close()
383
384 var seriesIDs []uint64
385 for {
386 e, err := itr.Next()
387 if err != nil {
388 return err
389 } else if e.SeriesID == 0 {
390 break
391 }
392 seriesIDs = append(seriesIDs, e.SeriesID)
393
394 // Check for cancellation periodically.
395 if seriesN++; seriesN%1000 == 0 {
396 select {
397 case <-info.cancel:
398 return ErrCompactionInterrupted
399 default:
400 }
401 }
402 }
403 sort.Sort(uint64Slice(seriesIDs))
404
405 // Add measurement to writer.
406 pos := info.tagSets[string(name)]
407 mw.Add(name, m.Deleted(), pos.offset, pos.size, seriesIDs)
408
409 return nil
410 }(); err != nil {
411 return err
412 }
413 }
414 }
415
416 // Flush data to writer.
417 nn, err := mw.WriteTo(w)
418 *n += nn
419 return err
420 }
421
422 // Stat returns the max index file size and the total file size for all index files.
423 func (p IndexFiles) Stat() (*IndexFilesInfo, error) {
424 var info IndexFilesInfo
425 for _, f := range p {
426 fi, err := os.Stat(f.Path())
427 if os.IsNotExist(err) {
428 continue
429 } else if err != nil {
430 return nil, err
431 }
432
433 if fi.Size() > info.MaxSize {
434 info.MaxSize = fi.Size()
435 }
436 if fi.ModTime().After(info.ModTime) {
437 info.ModTime = fi.ModTime()
438 }
439
440 info.Size += fi.Size()
441 }
442 return &info, nil
443 }
444
445 type IndexFilesInfo struct {
446 MaxSize int64 // largest file size
447 Size int64 // total file size
448 ModTime time.Time // last modified
449 }
450
451 // indexCompactInfo is a context object used for tracking position information
452 // during the compaction of index files.
453 type indexCompactInfo struct {
454 cancel <-chan struct{}
455
456 // Tracks offset/size for each measurement's tagset.
457 tagSets map[string]indexTagSetPos
458 }
459
460 // indexTagSetPos stores the offset/size of tagsets.
461 type indexTagSetPos struct {
462 offset int64
463 size int64
464 }