| 1 |
package tsi1_test |
| 2 |
|
| 3 |
import ( |
| 4 |
"fmt" |
| 5 |
"io/ioutil" |
| 6 |
"os" |
| 7 |
"path/filepath" |
| 8 |
"reflect" |
| 9 |
"regexp" |
| 10 |
"testing" |
| 11 |
|
| 12 |
"github.com/influxdata/influxdb/models" |
| 13 |
"github.com/influxdata/influxdb/tsdb/index/tsi1" |
| 14 |
) |
| 15 |
|
| 16 |
// Bloom filter settings used in tests. |
| 17 |
const M, K = 4096, 6 |
| 18 |
|
| 19 |
// Ensure index can iterate over all measurement names. |
| 20 |
func TestIndex_ForEachMeasurementName(t *testing.T) { |
| 21 |
idx := MustOpenIndex(1) |
| 22 |
defer idx.Close() |
| 23 |
|
| 24 |
// Add series to index. |
| 25 |
if err := idx.CreateSeriesSliceIfNotExists([]Series{ |
| 26 |
{Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "east"})}, |
| 27 |
{Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "west"})}, |
| 28 |
{Name: []byte("mem"), Tags: models.NewTags(map[string]string{"region": "east"})}, |
| 29 |
}); err != nil { |
| 30 |
t.Fatal(err) |
| 31 |
} |
| 32 |
|
| 33 |
// Verify measurements are returned. |
| 34 |
idx.Run(t, func(t *testing.T) { |
| 35 |
var names []string |
| 36 |
if err := idx.ForEachMeasurementName(func(name []byte) error { |
| 37 |
names = append(names, string(name)) |
| 38 |
return nil |
| 39 |
}); err != nil { |
| 40 |
t.Fatal(err) |
| 41 |
} |
| 42 |
|
| 43 |
if !reflect.DeepEqual(names, []string{"cpu", "mem"}) { |
| 44 |
t.Fatalf("unexpected names: %#v", names) |
| 45 |
} |
| 46 |
}) |
| 47 |
|
| 48 |
// Add more series. |
| 49 |
if err := idx.CreateSeriesSliceIfNotExists([]Series{ |
| 50 |
{Name: []byte("disk")}, |
| 51 |
{Name: []byte("mem")}, |
| 52 |
}); err != nil { |
| 53 |
t.Fatal(err) |
| 54 |
} |
| 55 |
|
| 56 |
// Verify new measurements. |
| 57 |
idx.Run(t, func(t *testing.T) { |
| 58 |
var names []string |
| 59 |
if err := idx.ForEachMeasurementName(func(name []byte) error { |
| 60 |
names = append(names, string(name)) |
| 61 |
return nil |
| 62 |
}); err != nil { |
| 63 |
t.Fatal(err) |
| 64 |
} |
| 65 |
|
| 66 |
if !reflect.DeepEqual(names, []string{"cpu", "disk", "mem"}) { |
| 67 |
t.Fatalf("unexpected names: %#v", names) |
| 68 |
} |
| 69 |
}) |
| 70 |
} |
| 71 |
|
| 72 |
// Ensure index can return whether a measurement exists. |
| 73 |
func TestIndex_MeasurementExists(t *testing.T) { |
| 74 |
idx := MustOpenIndex(1) |
| 75 |
defer idx.Close() |
| 76 |
|
| 77 |
// Add series to index. |
| 78 |
if err := idx.CreateSeriesSliceIfNotExists([]Series{ |
| 79 |
{Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "east"})}, |
| 80 |
{Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "west"})}, |
| 81 |
}); err != nil { |
| 82 |
t.Fatal(err) |
| 83 |
} |
| 84 |
|
| 85 |
// Verify measurement exists. |
| 86 |
idx.Run(t, func(t *testing.T) { |
| 87 |
if v, err := idx.MeasurementExists([]byte("cpu")); err != nil { |
| 88 |
t.Fatal(err) |
| 89 |
} else if !v { |
| 90 |
t.Fatal("expected measurement to exist") |
| 91 |
} |
| 92 |
}) |
| 93 |
|
| 94 |
name, tags := []byte("cpu"), models.NewTags(map[string]string{"region": "east"}) |
| 95 |
sid := idx.Index.SeriesFile().SeriesID(name, tags, nil) |
| 96 |
if sid == 0 { |
| 97 |
t.Fatalf("got 0 series id for %s/%v", name, tags) |
| 98 |
} |
| 99 |
|
| 100 |
// Delete one series. |
| 101 |
if err := idx.DropSeries(sid, models.MakeKey(name, tags), true); err != nil { |
| 102 |
t.Fatal(err) |
| 103 |
} |
| 104 |
|
| 105 |
// Verify measurement still exists. |
| 106 |
idx.Run(t, func(t *testing.T) { |
| 107 |
if v, err := idx.MeasurementExists([]byte("cpu")); err != nil { |
| 108 |
t.Fatal(err) |
| 109 |
} else if !v { |
| 110 |
t.Fatal("expected measurement to still exist") |
| 111 |
} |
| 112 |
}) |
| 113 |
|
| 114 |
// Delete second series. |
| 115 |
tags.Set([]byte("region"), []byte("west")) |
| 116 |
sid = idx.Index.SeriesFile().SeriesID(name, tags, nil) |
| 117 |
if sid == 0 { |
| 118 |
t.Fatalf("got 0 series id for %s/%v", name, tags) |
| 119 |
} |
| 120 |
if err := idx.DropSeries(sid, models.MakeKey(name, tags), true); err != nil { |
| 121 |
t.Fatal(err) |
| 122 |
} |
| 123 |
|
| 124 |
// Verify measurement is now deleted. |
| 125 |
idx.Run(t, func(t *testing.T) { |
| 126 |
if v, err := idx.MeasurementExists([]byte("cpu")); err != nil { |
| 127 |
t.Fatal(err) |
| 128 |
} else if v { |
| 129 |
t.Fatal("expected measurement to be deleted") |
| 130 |
} |
| 131 |
}) |
| 132 |
} |
| 133 |
|
| 134 |
// Ensure index can return a list of matching measurements. |
| 135 |
func TestIndex_MeasurementNamesByRegex(t *testing.T) { |
| 136 |
idx := MustOpenIndex(1) |
| 137 |
defer idx.Close() |
| 138 |
|
| 139 |
// Add series to index. |
| 140 |
if err := idx.CreateSeriesSliceIfNotExists([]Series{ |
| 141 |
{Name: []byte("cpu")}, |
| 142 |
{Name: []byte("disk")}, |
| 143 |
{Name: []byte("mem")}, |
| 144 |
}); err != nil { |
| 145 |
t.Fatal(err) |
| 146 |
} |
| 147 |
|
| 148 |
// Retrieve measurements by regex. |
| 149 |
idx.Run(t, func(t *testing.T) { |
| 150 |
names, err := idx.MeasurementNamesByRegex(regexp.MustCompile(`cpu|mem`)) |
| 151 |
if err != nil { |
| 152 |
t.Fatal(err) |
| 153 |
} else if !reflect.DeepEqual(names, [][]byte{[]byte("cpu"), []byte("mem")}) { |
| 154 |
t.Fatalf("unexpected names: %v", names) |
| 155 |
} |
| 156 |
}) |
| 157 |
} |
| 158 |
|
| 159 |
// Ensure index can delete a measurement and all related keys, values, & series. |
| 160 |
func TestIndex_DropMeasurement(t *testing.T) { |
| 161 |
idx := MustOpenIndex(1) |
| 162 |
defer idx.Close() |
| 163 |
|
| 164 |
// Add series to index. |
| 165 |
if err := idx.CreateSeriesSliceIfNotExists([]Series{ |
| 166 |
{Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "east"})}, |
| 167 |
{Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "west"})}, |
| 168 |
{Name: []byte("disk"), Tags: models.NewTags(map[string]string{"region": "north"})}, |
| 169 |
{Name: []byte("mem"), Tags: models.NewTags(map[string]string{"region": "west", "country": "us"})}, |
| 170 |
}); err != nil { |
| 171 |
t.Fatal(err) |
| 172 |
} |
| 173 |
|
| 174 |
// Drop measurement. |
| 175 |
if err := idx.DropMeasurement([]byte("cpu")); err != nil { |
| 176 |
t.Fatal(err) |
| 177 |
} |
| 178 |
|
| 179 |
// Verify data is gone in each stage. |
| 180 |
idx.Run(t, func(t *testing.T) { |
| 181 |
// Verify measurement is gone. |
| 182 |
if v, err := idx.MeasurementExists([]byte("cpu")); err != nil { |
| 183 |
t.Fatal(err) |
| 184 |
} else if v { |
| 185 |
t.Fatal("expected no measurement") |
| 186 |
} |
| 187 |
|
| 188 |
// Obtain file set to perform lower level checks. |
| 189 |
fs, err := idx.PartitionAt(0).RetainFileSet() |
| 190 |
if err != nil { |
| 191 |
t.Fatal(err) |
| 192 |
} |
| 193 |
defer fs.Release() |
| 194 |
|
| 195 |
// Verify tags & values are gone. |
| 196 |
if e := fs.TagKeyIterator([]byte("cpu")).Next(); e != nil && !e.Deleted() { |
| 197 |
t.Fatal("expected deleted tag key") |
| 198 |
} |
| 199 |
if itr := fs.TagValueIterator([]byte("cpu"), []byte("region")); itr != nil { |
| 200 |
t.Fatal("expected nil tag value iterator") |
| 201 |
} |
| 202 |
|
| 203 |
}) |
| 204 |
} |
| 205 |
|
| 206 |
func TestIndex_Open(t *testing.T) { |
| 207 |
// Opening a fresh index should set the MANIFEST version to current version. |
| 208 |
idx := NewIndex(tsi1.DefaultPartitionN) |
| 209 |
t.Run("open new index", func(t *testing.T) { |
| 210 |
if err := idx.Open(); err != nil { |
| 211 |
t.Fatal(err) |
| 212 |
} |
| 213 |
|
| 214 |
// Check version set appropriately. |
| 215 |
for i := 0; uint64(i) < tsi1.DefaultPartitionN; i++ { |
| 216 |
partition := idx.PartitionAt(i) |
| 217 |
if got, exp := partition.Manifest().Version, 1; got != exp { |
| 218 |
t.Fatalf("got index version %d, expected %d", got, exp) |
| 219 |
} |
| 220 |
} |
| 221 |
}) |
| 222 |
|
| 223 |
// Reopening an open index should return an error. |
| 224 |
t.Run("reopen open index", func(t *testing.T) { |
| 225 |
err := idx.Open() |
| 226 |
if err == nil { |
| 227 |
idx.Close() |
| 228 |
t.Fatal("didn't get an error on reopen, but expected one") |
| 229 |
} |
| 230 |
idx.Close() |
| 231 |
}) |
| 232 |
|
| 233 |
// Opening an incompatible index should return an error. |
| 234 |
incompatibleVersions := []int{-1, 0, 2} |
| 235 |
for _, v := range incompatibleVersions { |
| 236 |
t.Run(fmt.Sprintf("incompatible index version: %d", v), func(t *testing.T) { |
| 237 |
idx = NewIndex(tsi1.DefaultPartitionN) |
| 238 |
// Manually create a MANIFEST file for an incompatible index version. |
| 239 |
// under one of the partitions. |
| 240 |
partitionPath := filepath.Join(idx.Path(), "2") |
| 241 |
os.MkdirAll(partitionPath, 0777) |
| 242 |
|
| 243 |
mpath := filepath.Join(partitionPath, tsi1.ManifestFileName) |
| 244 |
m := tsi1.NewManifest(mpath) |
| 245 |
m.Levels = nil |
| 246 |
m.Version = v // Set example MANIFEST version. |
| 247 |
if _, err := m.Write(); err != nil { |
| 248 |
t.Fatal(err) |
| 249 |
} |
| 250 |
|
| 251 |
// Log the MANIFEST file. |
| 252 |
data, err := ioutil.ReadFile(mpath) |
| 253 |
if err != nil { |
| 254 |
panic(err) |
| 255 |
} |
| 256 |
t.Logf("Incompatible MANIFEST: %s", data) |
| 257 |
|
| 258 |
// Opening this index should return an error because the MANIFEST has an |
| 259 |
// incompatible version. |
| 260 |
err = idx.Open() |
| 261 |
if err != tsi1.ErrIncompatibleVersion { |
| 262 |
idx.Close() |
| 263 |
t.Fatalf("got error %v, expected %v", err, tsi1.ErrIncompatibleVersion) |
| 264 |
} |
| 265 |
}) |
| 266 |
} |
| 267 |
} |
| 268 |
|
| 269 |
func TestIndex_Manifest(t *testing.T) { |
| 270 |
t.Run("current MANIFEST", func(t *testing.T) { |
| 271 |
idx := MustOpenIndex(tsi1.DefaultPartitionN) |
| 272 |
|
| 273 |
// Check version set appropriately. |
| 274 |
for i := 0; uint64(i) < tsi1.DefaultPartitionN; i++ { |
| 275 |
partition := idx.PartitionAt(i) |
| 276 |
if got, exp := partition.Manifest().Version, tsi1.Version; got != exp { |
| 277 |
t.Fatalf("got MANIFEST version %d, expected %d", got, exp) |
| 278 |
} |
| 279 |
} |
| 280 |
}) |
| 281 |
} |
| 282 |
|
| 283 |
func TestIndex_DiskSizeBytes(t *testing.T) { |
| 284 |
idx := MustOpenIndex(tsi1.DefaultPartitionN) |
| 285 |
defer idx.Close() |
| 286 |
|
| 287 |
// Add series to index. |
| 288 |
if err := idx.CreateSeriesSliceIfNotExists([]Series{ |
| 289 |
{Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "east"})}, |
| 290 |
{Name: []byte("cpu"), Tags: models.NewTags(map[string]string{"region": "west"})}, |
| 291 |
{Name: []byte("disk"), Tags: models.NewTags(map[string]string{"region": "north"})}, |
| 292 |
{Name: []byte("mem"), Tags: models.NewTags(map[string]string{"region": "west", "country": "us"})}, |
| 293 |
}); err != nil { |
| 294 |
t.Fatal(err) |
| 295 |
} |
| 296 |
|
| 297 |
// Verify on disk size is the same in each stage. |
| 298 |
// Each series stores flag(1) + series(uvarint(2)) + len(name)(1) + len(key)(1) + len(value)(1) + checksum(4). |
| 299 |
expSize := int64(4 * 9) |
| 300 |
|
| 301 |
// Each MANIFEST file is 419 bytes and there are tsi1.DefaultPartitionN of them |
| 302 |
expSize += int64(tsi1.DefaultPartitionN * 419) |
| 303 |
|
| 304 |
idx.Run(t, func(t *testing.T) { |
| 305 |
if got, exp := idx.DiskSizeBytes(), expSize; got != exp { |
| 306 |
t.Fatalf("got %d bytes, expected %d", got, exp) |
| 307 |
} |
| 308 |
}) |
| 309 |
} |
| 310 |
|
| 311 |
// Index is a test wrapper for tsi1.Index. |
| 312 |
type Index struct { |
| 313 |
*tsi1.Index |
| 314 |
SeriesFile *SeriesFile |
| 315 |
} |
| 316 |
|
| 317 |
// NewIndex returns a new instance of Index at a temporary path. |
| 318 |
func NewIndex(partitionN uint64) *Index { |
| 319 |
idx := &Index{SeriesFile: NewSeriesFile()} |
| 320 |
idx.Index = tsi1.NewIndex(idx.SeriesFile.SeriesFile, "db0", tsi1.WithPath(MustTempDir())) |
| 321 |
idx.Index.PartitionN = partitionN |
| 322 |
return idx |
| 323 |
} |
| 324 |
|
| 325 |
// MustOpenIndex returns a new, open index. Panic on error. |
| 326 |
func MustOpenIndex(partitionN uint64) *Index { |
| 327 |
idx := NewIndex(partitionN) |
| 328 |
if err := idx.Open(); err != nil { |
| 329 |
panic(err) |
| 330 |
} |
| 331 |
return idx |
| 332 |
} |
| 333 |
|
| 334 |
// Open opens the underlying tsi1.Index and tsdb.SeriesFile |
| 335 |
func (idx Index) Open() error { |
| 336 |
if err := idx.SeriesFile.Open(); err != nil { |
| 337 |
return err |
| 338 |
} |
| 339 |
return idx.Index.Open() |
| 340 |
} |
| 341 |
|
| 342 |
// Close closes and removes the index directory. |
| 343 |
func (idx *Index) Close() error { |
| 344 |
defer os.RemoveAll(idx.Path()) |
| 345 |
if err := idx.SeriesFile.Close(); err != nil { |
| 346 |
return err |
| 347 |
} |
| 348 |
return idx.Index.Close() |
| 349 |
} |
| 350 |
|
| 351 |
// Reopen closes and opens the index. |
| 352 |
func (idx *Index) Reopen() error { |
| 353 |
if err := idx.Index.Close(); err != nil { |
| 354 |
return err |
| 355 |
} |
| 356 |
|
| 357 |
// Reopen the series file correctly, by initialising a new underlying series |
| 358 |
// file using the same disk data. |
| 359 |
if err := idx.SeriesFile.Reopen(); err != nil { |
| 360 |
return err |
| 361 |
} |
| 362 |
|
| 363 |
partitionN := idx.Index.PartitionN // Remember how many partitions to use. |
| 364 |
idx.Index = tsi1.NewIndex(idx.SeriesFile.SeriesFile, "db0", tsi1.WithPath(idx.Index.Path())) |
| 365 |
idx.Index.PartitionN = partitionN |
| 366 |
return idx.Open() |
| 367 |
} |
| 368 |
|
| 369 |
// Run executes a subtest for each of several different states: |
| 370 |
// |
| 371 |
// - Immediately |
| 372 |
// - After reopen |
| 373 |
// - After compaction |
| 374 |
// - After reopen again |
| 375 |
// |
| 376 |
// The index should always respond in the same fashion regardless of |
| 377 |
// how data is stored. This helper allows the index to be easily tested |
| 378 |
// in all major states. |
| 379 |
func (idx *Index) Run(t *testing.T, fn func(t *testing.T)) { |
| 380 |
// Invoke immediately. |
| 381 |
t.Run("state=initial", fn) |
| 382 |
|
| 383 |
// Reopen and invoke again. |
| 384 |
if err := idx.Reopen(); err != nil { |
| 385 |
t.Fatalf("reopen error: %s", err) |
| 386 |
} |
| 387 |
t.Run("state=reopen", fn) |
| 388 |
|
| 389 |
// TODO: Request a compaction. |
| 390 |
// if err := idx.Compact(); err != nil { |
| 391 |
// t.Fatalf("compact error: %s", err) |
| 392 |
// } |
| 393 |
// t.Run("state=post-compaction", fn) |
| 394 |
|
| 395 |
// Reopen and invoke again. |
| 396 |
if err := idx.Reopen(); err != nil { |
| 397 |
t.Fatalf("post-compaction reopen error: %s", err) |
| 398 |
} |
| 399 |
t.Run("state=post-compaction-reopen", fn) |
| 400 |
} |
| 401 |
|
| 402 |
// CreateSeriesSliceIfNotExists creates multiple series at a time. |
| 403 |
func (idx *Index) CreateSeriesSliceIfNotExists(a []Series) error { |
| 404 |
keys := make([][]byte, 0, len(a)) |
| 405 |
names := make([][]byte, 0, len(a)) |
| 406 |
tags := make([]models.Tags, 0, len(a)) |
| 407 |
for _, s := range a { |
| 408 |
keys = append(keys, models.MakeKey(s.Name, s.Tags)) |
| 409 |
names = append(names, s.Name) |
| 410 |
tags = append(tags, s.Tags) |
| 411 |
} |
| 412 |
return idx.CreateSeriesListIfNotExists(keys, names, tags) |
| 413 |
} |