| 1 |
package tsi1_test |
| 2 |
|
| 3 |
import ( |
| 4 |
"bytes" |
| 5 |
"fmt" |
| 6 |
"io/ioutil" |
| 7 |
"math/rand" |
| 8 |
"os" |
| 9 |
"path/filepath" |
| 10 |
"regexp" |
| 11 |
"runtime/pprof" |
| 12 |
"sort" |
| 13 |
"testing" |
| 14 |
"time" |
| 15 |
|
| 16 |
"github.com/influxdata/influxdb/models" |
| 17 |
"github.com/influxdata/influxdb/pkg/bloom" |
| 18 |
"github.com/influxdata/influxdb/tsdb" |
| 19 |
"github.com/influxdata/influxdb/tsdb/index/tsi1" |
| 20 |
) |
| 21 |
|
| 22 |
// Ensure log file can append series. |
| 23 |
func TestLogFile_AddSeriesList(t *testing.T) { |
| 24 |
sfile := MustOpenSeriesFile() |
| 25 |
defer sfile.Close() |
| 26 |
|
| 27 |
f := MustOpenLogFile(sfile.SeriesFile) |
| 28 |
defer f.Close() |
| 29 |
seriesSet := tsdb.NewSeriesIDSet() |
| 30 |
|
| 31 |
// Add test data. |
| 32 |
if err := f.AddSeriesList(seriesSet, [][]byte{ |
| 33 |
[]byte("mem"), |
| 34 |
[]byte("cpu"), |
| 35 |
[]byte("cpu"), |
| 36 |
}, []models.Tags{ |
| 37 |
{{Key: []byte("host"), Value: []byte("serverA")}}, |
| 38 |
{{Key: []byte("region"), Value: []byte("us-east")}}, |
| 39 |
{{Key: []byte("region"), Value: []byte("us-west")}}, |
| 40 |
}); err != nil { |
| 41 |
t.Fatal(err) |
| 42 |
} |
| 43 |
|
| 44 |
// Verify data. |
| 45 |
itr := f.MeasurementIterator() |
| 46 |
if e := itr.Next(); e == nil || string(e.Name()) != "cpu" { |
| 47 |
t.Fatalf("unexpected measurement: %#v", e) |
| 48 |
} else if e := itr.Next(); e == nil || string(e.Name()) != "mem" { |
| 49 |
t.Fatalf("unexpected measurement: %#v", e) |
| 50 |
} else if e := itr.Next(); e != nil { |
| 51 |
t.Fatalf("expected eof, got: %#v", e) |
| 52 |
} |
| 53 |
|
| 54 |
// Reopen file and re-verify. |
| 55 |
if err := f.Reopen(); err != nil { |
| 56 |
t.Fatal(err) |
| 57 |
} |
| 58 |
|
| 59 |
// Verify data. |
| 60 |
itr = f.MeasurementIterator() |
| 61 |
if e := itr.Next(); e == nil || string(e.Name()) != "cpu" { |
| 62 |
t.Fatalf("unexpected measurement: %#v", e) |
| 63 |
} else if e := itr.Next(); e == nil || string(e.Name()) != "mem" { |
| 64 |
t.Fatalf("unexpected measurement: %#v", e) |
| 65 |
} else if e := itr.Next(); e != nil { |
| 66 |
t.Fatalf("expected eof, got: %#v", e) |
| 67 |
} |
| 68 |
} |
| 69 |
|
| 70 |
func TestLogFile_SeriesStoredInOrder(t *testing.T) { |
| 71 |
sfile := MustOpenSeriesFile() |
| 72 |
defer sfile.Close() |
| 73 |
|
| 74 |
f := MustOpenLogFile(sfile.SeriesFile) |
| 75 |
defer f.Close() |
| 76 |
seriesSet := tsdb.NewSeriesIDSet() |
| 77 |
|
| 78 |
// Generate and add test data |
| 79 |
tvm := make(map[string]struct{}) |
| 80 |
rand.Seed(time.Now().Unix()) |
| 81 |
for i := 0; i < 100; i++ { |
| 82 |
tv := fmt.Sprintf("server-%d", rand.Intn(50)) // Encourage adding duplicate series. |
| 83 |
tvm[tv] = struct{}{} |
| 84 |
|
| 85 |
if err := f.AddSeriesList(seriesSet, [][]byte{ |
| 86 |
[]byte("mem"), |
| 87 |
[]byte("cpu"), |
| 88 |
}, []models.Tags{ |
| 89 |
{models.NewTag([]byte("host"), []byte(tv))}, |
| 90 |
{models.NewTag([]byte("host"), []byte(tv))}, |
| 91 |
}); err != nil { |
| 92 |
t.Fatal(err) |
| 93 |
} |
| 94 |
} |
| 95 |
|
| 96 |
// Sort the tag values so we know what order to expect. |
| 97 |
tvs := make([]string, 0, len(tvm)) |
| 98 |
for tv := range tvm { |
| 99 |
tvs = append(tvs, tv) |
| 100 |
} |
| 101 |
sort.Strings(tvs) |
| 102 |
|
| 103 |
// Double the series values since we're adding them twice (two measurements) |
| 104 |
tvs = append(tvs, tvs...) |
| 105 |
|
| 106 |
// When we pull the series out via an iterator they should be in order. |
| 107 |
itr := f.SeriesIDIterator() |
| 108 |
if itr == nil { |
| 109 |
t.Fatal("nil iterator") |
| 110 |
} |
| 111 |
|
| 112 |
var prevSeriesID uint64 |
| 113 |
for i := 0; i < len(tvs); i++ { |
| 114 |
elem, err := itr.Next() |
| 115 |
if err != nil { |
| 116 |
t.Fatal(err) |
| 117 |
} else if elem.SeriesID == 0 { |
| 118 |
t.Fatal("got nil series") |
| 119 |
} else if elem.SeriesID < prevSeriesID { |
| 120 |
t.Fatalf("series out of order: %d !< %d ", elem.SeriesID, prevSeriesID) |
| 121 |
} |
| 122 |
prevSeriesID = elem.SeriesID |
| 123 |
} |
| 124 |
} |
| 125 |
|
| 126 |
// Ensure log file can delete an existing measurement. |
| 127 |
func TestLogFile_DeleteMeasurement(t *testing.T) { |
| 128 |
sfile := MustOpenSeriesFile() |
| 129 |
defer sfile.Close() |
| 130 |
|
| 131 |
f := MustOpenLogFile(sfile.SeriesFile) |
| 132 |
defer f.Close() |
| 133 |
seriesSet := tsdb.NewSeriesIDSet() |
| 134 |
|
| 135 |
// Add test data. |
| 136 |
if err := f.AddSeriesList(seriesSet, [][]byte{ |
| 137 |
[]byte("mem"), |
| 138 |
[]byte("cpu"), |
| 139 |
[]byte("cpu"), |
| 140 |
}, []models.Tags{ |
| 141 |
{{Key: []byte("host"), Value: []byte("serverA")}}, |
| 142 |
{{Key: []byte("region"), Value: []byte("us-east")}}, |
| 143 |
{{Key: []byte("region"), Value: []byte("us-west")}}, |
| 144 |
}); err != nil { |
| 145 |
t.Fatal(err) |
| 146 |
} |
| 147 |
|
| 148 |
// Remove measurement. |
| 149 |
if err := f.DeleteMeasurement([]byte("cpu")); err != nil { |
| 150 |
t.Fatal(err) |
| 151 |
} |
| 152 |
|
| 153 |
// Verify data. |
| 154 |
itr := f.MeasurementIterator() |
| 155 |
if e := itr.Next(); string(e.Name()) != "cpu" || !e.Deleted() { |
| 156 |
t.Fatalf("unexpected measurement: %s/%v", e.Name(), e.Deleted()) |
| 157 |
} else if e := itr.Next(); string(e.Name()) != "mem" || e.Deleted() { |
| 158 |
t.Fatalf("unexpected measurement: %s/%v", e.Name(), e.Deleted()) |
| 159 |
} else if e := itr.Next(); e != nil { |
| 160 |
t.Fatalf("expected eof, got: %#v", e) |
| 161 |
} |
| 162 |
} |
| 163 |
|
| 164 |
// Ensure log file can recover correctly. |
| 165 |
func TestLogFile_Open(t *testing.T) { |
| 166 |
t.Run("Truncate", func(t *testing.T) { |
| 167 |
sfile := MustOpenSeriesFile() |
| 168 |
defer sfile.Close() |
| 169 |
seriesSet := tsdb.NewSeriesIDSet() |
| 170 |
|
| 171 |
f := MustOpenLogFile(sfile.SeriesFile) |
| 172 |
defer f.Close() |
| 173 |
|
| 174 |
// Add test data & close. |
| 175 |
if err := f.AddSeriesList(seriesSet, [][]byte{[]byte("cpu"), []byte("mem")}, []models.Tags{{{}}, {{}}}); err != nil { |
| 176 |
t.Fatal(err) |
| 177 |
} else if err := f.LogFile.Close(); err != nil { |
| 178 |
t.Fatal(err) |
| 179 |
} |
| 180 |
|
| 181 |
// Truncate data & reopen. |
| 182 |
if fi, err := os.Stat(f.LogFile.Path()); err != nil { |
| 183 |
t.Fatal(err) |
| 184 |
} else if err := os.Truncate(f.LogFile.Path(), fi.Size()-1); err != nil { |
| 185 |
t.Fatal(err) |
| 186 |
} else if err := f.LogFile.Open(); err != nil { |
| 187 |
t.Fatal(err) |
| 188 |
} |
| 189 |
|
| 190 |
// Verify data. |
| 191 |
itr := f.SeriesIDIterator() |
| 192 |
if elem, err := itr.Next(); err != nil { |
| 193 |
t.Fatal(err) |
| 194 |
} else if name, tags := sfile.Series(elem.SeriesID); string(name) != `cpu` { |
| 195 |
t.Fatalf("unexpected series: %s,%s", name, tags.String()) |
| 196 |
} else if elem, err := itr.Next(); err != nil { |
| 197 |
t.Fatal(err) |
| 198 |
} else if elem.SeriesID != 0 { |
| 199 |
t.Fatalf("expected eof, got: %#v", elem) |
| 200 |
} |
| 201 |
|
| 202 |
// Add more data & reopen. |
| 203 |
if err := f.AddSeriesList(seriesSet, [][]byte{[]byte("disk")}, []models.Tags{{{}}}); err != nil { |
| 204 |
t.Fatal(err) |
| 205 |
} else if err := f.Reopen(); err != nil { |
| 206 |
t.Fatal(err) |
| 207 |
} |
| 208 |
|
| 209 |
// Verify new data. |
| 210 |
itr = f.SeriesIDIterator() |
| 211 |
if elem, err := itr.Next(); err != nil { |
| 212 |
t.Fatal(err) |
| 213 |
} else if name, tags := sfile.Series(elem.SeriesID); string(name) != `cpu` { |
| 214 |
t.Fatalf("unexpected series: %s,%s", name, tags.String()) |
| 215 |
} else if elem, err := itr.Next(); err != nil { |
| 216 |
t.Fatal(err) |
| 217 |
} else if name, tags := sfile.Series(elem.SeriesID); string(name) != `disk` { |
| 218 |
t.Fatalf("unexpected series: %s,%s", name, tags.String()) |
| 219 |
} else if elem, err := itr.Next(); err != nil { |
| 220 |
t.Fatal(err) |
| 221 |
} else if elem.SeriesID != 0 { |
| 222 |
t.Fatalf("expected eof, got: %#v", elem) |
| 223 |
} |
| 224 |
}) |
| 225 |
|
| 226 |
t.Run("ChecksumMismatch", func(t *testing.T) { |
| 227 |
sfile := MustOpenSeriesFile() |
| 228 |
defer sfile.Close() |
| 229 |
seriesSet := tsdb.NewSeriesIDSet() |
| 230 |
|
| 231 |
f := MustOpenLogFile(sfile.SeriesFile) |
| 232 |
defer f.Close() |
| 233 |
|
| 234 |
// Add test data & close. |
| 235 |
if err := f.AddSeriesList(seriesSet, [][]byte{[]byte("cpu"), []byte("mem")}, []models.Tags{{{}}, {{}}}); err != nil { |
| 236 |
t.Fatal(err) |
| 237 |
} else if err := f.LogFile.Close(); err != nil { |
| 238 |
t.Fatal(err) |
| 239 |
} |
| 240 |
|
| 241 |
// Corrupt last entry. |
| 242 |
buf, err := ioutil.ReadFile(f.LogFile.Path()) |
| 243 |
if err != nil { |
| 244 |
t.Fatal(err) |
| 245 |
} |
| 246 |
buf[len(buf)-1] = 0 |
| 247 |
|
| 248 |
// Overwrite file with corrupt entry and reopen. |
| 249 |
if err := ioutil.WriteFile(f.LogFile.Path(), buf, 0666); err != nil { |
| 250 |
t.Fatal(err) |
| 251 |
} else if err := f.LogFile.Open(); err != nil { |
| 252 |
t.Fatal(err) |
| 253 |
} |
| 254 |
|
| 255 |
// Verify data. |
| 256 |
itr := f.SeriesIDIterator() |
| 257 |
if elem, err := itr.Next(); err != nil { |
| 258 |
t.Fatal(err) |
| 259 |
} else if name, tags := sfile.Series(elem.SeriesID); string(name) != `cpu` { |
| 260 |
t.Fatalf("unexpected series: %s,%s", name, tags.String()) |
| 261 |
} else if elem, err := itr.Next(); err != nil { |
| 262 |
t.Fatal(err) |
| 263 |
} else if elem.SeriesID != 0 { |
| 264 |
t.Fatalf("expected eof, got: %#v", elem) |
| 265 |
} |
| 266 |
}) |
| 267 |
} |
| 268 |
|
| 269 |
// LogFile is a test wrapper for tsi1.LogFile. |
| 270 |
type LogFile struct { |
| 271 |
*tsi1.LogFile |
| 272 |
} |
| 273 |
|
| 274 |
// NewLogFile returns a new instance of LogFile with a temporary file path. |
| 275 |
func NewLogFile(sfile *tsdb.SeriesFile) *LogFile { |
| 276 |
file, err := ioutil.TempFile("", "tsi1-log-file-") |
| 277 |
if err != nil { |
| 278 |
panic(err) |
| 279 |
} |
| 280 |
file.Close() |
| 281 |
|
| 282 |
return &LogFile{LogFile: tsi1.NewLogFile(sfile, file.Name())} |
| 283 |
} |
| 284 |
|
| 285 |
// MustOpenLogFile returns a new, open instance of LogFile. Panic on error. |
| 286 |
func MustOpenLogFile(sfile *tsdb.SeriesFile) *LogFile { |
| 287 |
f := NewLogFile(sfile) |
| 288 |
if err := f.Open(); err != nil { |
| 289 |
panic(err) |
| 290 |
} |
| 291 |
return f |
| 292 |
} |
| 293 |
|
| 294 |
// Close closes the log file and removes it from disk. |
| 295 |
func (f *LogFile) Close() error { |
| 296 |
defer os.Remove(f.Path()) |
| 297 |
return f.LogFile.Close() |
| 298 |
} |
| 299 |
|
| 300 |
// Reopen closes and reopens the file. |
| 301 |
func (f *LogFile) Reopen() error { |
| 302 |
if err := f.LogFile.Close(); err != nil { |
| 303 |
return err |
| 304 |
} |
| 305 |
if err := f.LogFile.Open(); err != nil { |
| 306 |
return err |
| 307 |
} |
| 308 |
return nil |
| 309 |
} |
| 310 |
|
| 311 |
// CreateLogFile creates a new temporary log file and adds a list of series. |
| 312 |
func CreateLogFile(sfile *tsdb.SeriesFile, series []Series) (*LogFile, error) { |
| 313 |
f := MustOpenLogFile(sfile) |
| 314 |
seriesSet := tsdb.NewSeriesIDSet() |
| 315 |
for _, serie := range series { |
| 316 |
if err := f.AddSeriesList(seriesSet, [][]byte{serie.Name}, []models.Tags{serie.Tags}); err != nil { |
| 317 |
return nil, err |
| 318 |
} |
| 319 |
} |
| 320 |
return f, nil |
| 321 |
} |
| 322 |
|
| 323 |
// GenerateLogFile generates a log file from a set of series based on the count arguments. |
| 324 |
// Total series returned will equal measurementN * tagN * valueN. |
| 325 |
func GenerateLogFile(sfile *tsdb.SeriesFile, measurementN, tagN, valueN int) (*LogFile, error) { |
| 326 |
tagValueN := pow(valueN, tagN) |
| 327 |
|
| 328 |
f := MustOpenLogFile(sfile) |
| 329 |
seriesSet := tsdb.NewSeriesIDSet() |
| 330 |
for i := 0; i < measurementN; i++ { |
| 331 |
name := []byte(fmt.Sprintf("measurement%d", i)) |
| 332 |
|
| 333 |
// Generate tag sets. |
| 334 |
for j := 0; j < tagValueN; j++ { |
| 335 |
var tags models.Tags |
| 336 |
for k := 0; k < tagN; k++ { |
| 337 |
key := []byte(fmt.Sprintf("key%d", k)) |
| 338 |
value := []byte(fmt.Sprintf("value%d", (j / pow(valueN, k) % valueN))) |
| 339 |
tags = append(tags, models.NewTag(key, value)) |
| 340 |
} |
| 341 |
if err := f.AddSeriesList(seriesSet, [][]byte{name}, []models.Tags{tags}); err != nil { |
| 342 |
return nil, err |
| 343 |
} |
| 344 |
} |
| 345 |
} |
| 346 |
return f, nil |
| 347 |
} |
| 348 |
|
| 349 |
func benchmarkLogFile_AddSeries(b *testing.B, measurementN, seriesKeyN, seriesValueN int) { |
| 350 |
sfile := MustOpenSeriesFile() |
| 351 |
defer sfile.Close() |
| 352 |
|
| 353 |
b.StopTimer() |
| 354 |
f := MustOpenLogFile(sfile.SeriesFile) |
| 355 |
seriesSet := tsdb.NewSeriesIDSet() |
| 356 |
|
| 357 |
type Datum struct { |
| 358 |
Name []byte |
| 359 |
Tags models.Tags |
| 360 |
} |
| 361 |
|
| 362 |
// Pre-generate everything. |
| 363 |
var ( |
| 364 |
data []Datum |
| 365 |
series int |
| 366 |
) |
| 367 |
|
| 368 |
tagValueN := pow(seriesValueN, seriesKeyN) |
| 369 |
|
| 370 |
for i := 0; i < measurementN; i++ { |
| 371 |
name := []byte(fmt.Sprintf("measurement%d", i)) |
| 372 |
for j := 0; j < tagValueN; j++ { |
| 373 |
var tags models.Tags |
| 374 |
for k := 0; k < seriesKeyN; k++ { |
| 375 |
key := []byte(fmt.Sprintf("key%d", k)) |
| 376 |
value := []byte(fmt.Sprintf("value%d", (j / pow(seriesValueN, k) % seriesValueN))) |
| 377 |
tags = append(tags, models.NewTag(key, value)) |
| 378 |
} |
| 379 |
data = append(data, Datum{Name: name, Tags: tags}) |
| 380 |
series += len(tags) |
| 381 |
} |
| 382 |
} |
| 383 |
b.StartTimer() |
| 384 |
b.ResetTimer() |
| 385 |
|
| 386 |
for i := 0; i < b.N; i++ { |
| 387 |
for _, d := range data { |
| 388 |
if err := f.AddSeriesList(seriesSet, [][]byte{d.Name}, []models.Tags{d.Tags}); err != nil { |
| 389 |
b.Fatal(err) |
| 390 |
} |
| 391 |
} |
| 392 |
} |
| 393 |
} |
| 394 |
|
| 395 |
func BenchmarkLogFile_AddSeries_100_1_1(b *testing.B) { benchmarkLogFile_AddSeries(b, 100, 1, 1) } // 100 series |
| 396 |
func BenchmarkLogFile_AddSeries_1000_1_1(b *testing.B) { benchmarkLogFile_AddSeries(b, 1000, 1, 1) } // 1000 series |
| 397 |
func BenchmarkLogFile_AddSeries_10000_1_1(b *testing.B) { benchmarkLogFile_AddSeries(b, 10000, 1, 1) } // 10000 series |
| 398 |
func BenchmarkLogFile_AddSeries_100_2_10(b *testing.B) { benchmarkLogFile_AddSeries(b, 100, 2, 10) } // ~20K series |
| 399 |
func BenchmarkLogFile_AddSeries_100000_1_1(b *testing.B) { benchmarkLogFile_AddSeries(b, 100000, 1, 1) } // ~100K series |
| 400 |
func BenchmarkLogFile_AddSeries_100_3_7(b *testing.B) { benchmarkLogFile_AddSeries(b, 100, 3, 7) } // ~100K series |
| 401 |
func BenchmarkLogFile_AddSeries_200_3_7(b *testing.B) { benchmarkLogFile_AddSeries(b, 200, 3, 7) } // ~200K series |
| 402 |
func BenchmarkLogFile_AddSeries_200_4_7(b *testing.B) { benchmarkLogFile_AddSeries(b, 200, 4, 7) } // ~1.9M series |
| 403 |
|
| 404 |
func BenchmarkLogFile_WriteTo(b *testing.B) { |
| 405 |
for _, seriesN := range []int{1000, 10000, 100000, 1000000} { |
| 406 |
name := fmt.Sprintf("series=%d", seriesN) |
| 407 |
b.Run(name, func(b *testing.B) { |
| 408 |
sfile := MustOpenSeriesFile() |
| 409 |
defer sfile.Close() |
| 410 |
|
| 411 |
f := MustOpenLogFile(sfile.SeriesFile) |
| 412 |
defer f.Close() |
| 413 |
seriesSet := tsdb.NewSeriesIDSet() |
| 414 |
|
| 415 |
// Estimate bloom filter size. |
| 416 |
m, k := bloom.Estimate(uint64(seriesN), 0.02) |
| 417 |
|
| 418 |
// Initialize log file with series data. |
| 419 |
for i := 0; i < seriesN; i++ { |
| 420 |
if err := f.AddSeriesList( |
| 421 |
seriesSet, |
| 422 |
[][]byte{[]byte("cpu")}, |
| 423 |
[]models.Tags{{ |
| 424 |
{Key: []byte("host"), Value: []byte(fmt.Sprintf("server-%d", i))}, |
| 425 |
{Key: []byte("location"), Value: []byte("us-west")}, |
| 426 |
}}, |
| 427 |
); err != nil { |
| 428 |
b.Fatal(err) |
| 429 |
} |
| 430 |
} |
| 431 |
b.ResetTimer() |
| 432 |
|
| 433 |
// Create cpu profile for each subtest. |
| 434 |
MustStartCPUProfile(name) |
| 435 |
defer pprof.StopCPUProfile() |
| 436 |
|
| 437 |
// Compact log file. |
| 438 |
for i := 0; i < b.N; i++ { |
| 439 |
buf := bytes.NewBuffer(make([]byte, 0, 150*seriesN)) |
| 440 |
if _, err := f.CompactTo(buf, m, k, nil); err != nil { |
| 441 |
b.Fatal(err) |
| 442 |
} |
| 443 |
b.Logf("sz=%db", buf.Len()) |
| 444 |
} |
| 445 |
}) |
| 446 |
} |
| 447 |
} |
| 448 |
|
| 449 |
// MustStartCPUProfile starts a cpu profile in a temporary path based on name. |
| 450 |
func MustStartCPUProfile(name string) { |
| 451 |
name = regexp.MustCompile(`\W+`).ReplaceAllString(name, "-") |
| 452 |
|
| 453 |
// Open file and start pprof. |
| 454 |
f, err := os.Create(filepath.Join("/tmp", fmt.Sprintf("cpu-%s.pprof", name))) |
| 455 |
if err != nil { |
| 456 |
panic(err) |
| 457 |
} |
| 458 |
if err := pprof.StartCPUProfile(f); err != nil { |
| 459 |
panic(err) |
| 460 |
} |
| 461 |
} |