| 1 |
// Copyright The OpenTelemetry Authors |
| 2 |
// SPDX-License-Identifier: Apache-2.0 |
| 3 |
|
| 4 |
package trace |
| 5 |
|
| 6 |
import ( |
| 7 |
"context" |
| 8 |
"encoding/binary" |
| 9 |
"errors" |
| 10 |
"fmt" |
| 11 |
"runtime" |
| 12 |
"sync" |
| 13 |
"sync/atomic" |
| 14 |
"testing" |
| 15 |
"time" |
| 16 |
|
| 17 |
"github.com/stretchr/testify/assert" |
| 18 |
"github.com/stretchr/testify/require" |
| 19 |
|
| 20 |
"go.opentelemetry.io/otel" |
| 21 |
"go.opentelemetry.io/otel/attribute" |
| 22 |
"go.opentelemetry.io/otel/sdk" |
| 23 |
"go.opentelemetry.io/otel/sdk/instrumentation" |
| 24 |
sdkmetric "go.opentelemetry.io/otel/sdk/metric" |
| 25 |
"go.opentelemetry.io/otel/sdk/metric/metricdata" |
| 26 |
"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest" |
| 27 |
"go.opentelemetry.io/otel/sdk/trace/internal/env" |
| 28 |
"go.opentelemetry.io/otel/sdk/trace/internal/observ" |
| 29 |
semconv "go.opentelemetry.io/otel/semconv/v1.40.0" |
| 30 |
"go.opentelemetry.io/otel/semconv/v1.40.0/otelconv" |
| 31 |
"go.opentelemetry.io/otel/trace" |
| 32 |
) |
| 33 |
|
| 34 |
const componentID = 0 |
| 35 |
|
| 36 |
type testBatchExporter struct { |
| 37 |
mu sync.Mutex |
| 38 |
spans []ReadOnlySpan |
| 39 |
sizes []int |
| 40 |
batchCount int |
| 41 |
shutdownCount int |
| 42 |
errors []error |
| 43 |
droppedCount int |
| 44 |
idx int |
| 45 |
err error |
| 46 |
} |
| 47 |
|
| 48 |
func (t *testBatchExporter) ExportSpans(ctx context.Context, spans []ReadOnlySpan) error { |
| 49 |
t.mu.Lock() |
| 50 |
defer t.mu.Unlock() |
| 51 |
|
| 52 |
if t.idx < len(t.errors) { |
| 53 |
t.droppedCount += len(spans) |
| 54 |
err := t.errors[t.idx] |
| 55 |
t.idx++ |
| 56 |
return err |
| 57 |
} |
| 58 |
|
| 59 |
select { |
| 60 |
case <-ctx.Done(): |
| 61 |
t.err = ctx.Err() |
| 62 |
return ctx.Err() |
| 63 |
default: |
| 64 |
} |
| 65 |
|
| 66 |
t.spans = append(t.spans, spans...) |
| 67 |
t.sizes = append(t.sizes, len(spans)) |
| 68 |
t.batchCount++ |
| 69 |
return nil |
| 70 |
} |
| 71 |
|
| 72 |
func (t *testBatchExporter) Shutdown(context.Context) error { |
| 73 |
t.shutdownCount++ |
| 74 |
return nil |
| 75 |
} |
| 76 |
|
| 77 |
func (t *testBatchExporter) len() int { |
| 78 |
t.mu.Lock() |
| 79 |
defer t.mu.Unlock() |
| 80 |
return len(t.spans) |
| 81 |
} |
| 82 |
|
| 83 |
func (t *testBatchExporter) getBatchCount() int { |
| 84 |
t.mu.Lock() |
| 85 |
defer t.mu.Unlock() |
| 86 |
return t.batchCount |
| 87 |
} |
| 88 |
|
| 89 |
var _ SpanExporter = (*testBatchExporter)(nil) |
| 90 |
|
| 91 |
func TestNewBatchSpanProcessorWithNilExporter(t *testing.T) { |
| 92 |
tp := basicTracerProvider(t) |
| 93 |
bsp := NewBatchSpanProcessor(nil) |
| 94 |
tp.RegisterSpanProcessor(bsp) |
| 95 |
tr := tp.Tracer("NilExporter") |
| 96 |
|
| 97 |
_, span := tr.Start(t.Context(), "foo") |
| 98 |
span.End() |
| 99 |
|
| 100 |
// These should not panic. |
| 101 |
bsp.OnStart(t.Context(), span.(ReadWriteSpan)) |
| 102 |
bsp.OnEnd(span.(ReadOnlySpan)) |
| 103 |
if err := bsp.ForceFlush(t.Context()); err != nil { |
| 104 |
t.Errorf("failed to ForceFlush the BatchSpanProcessor: %v", err) |
| 105 |
} |
| 106 |
if err := bsp.Shutdown(t.Context()); err != nil { |
| 107 |
t.Errorf("failed to Shutdown the BatchSpanProcessor: %v", err) |
| 108 |
} |
| 109 |
} |
| 110 |
|
| 111 |
type testOption struct { |
| 112 |
name string |
| 113 |
o []BatchSpanProcessorOption |
| 114 |
wantNumSpans int |
| 115 |
wantBatchCount int |
| 116 |
genNumSpans int |
| 117 |
parallel bool |
| 118 |
envs map[string]string |
| 119 |
} |
| 120 |
|
| 121 |
func TestNewBatchSpanProcessorWithOptions(t *testing.T) { |
| 122 |
schDelay := 200 * time.Millisecond |
| 123 |
options := []testOption{ |
| 124 |
{ |
| 125 |
name: "default BatchSpanProcessorOptions", |
| 126 |
wantNumSpans: 2053, |
| 127 |
wantBatchCount: 4, |
| 128 |
genNumSpans: 2053, |
| 129 |
}, |
| 130 |
{ |
| 131 |
name: "non-default BatchTimeout", |
| 132 |
o: []BatchSpanProcessorOption{ |
| 133 |
WithBatchTimeout(schDelay), |
| 134 |
}, |
| 135 |
wantNumSpans: 2053, |
| 136 |
wantBatchCount: 4, |
| 137 |
genNumSpans: 2053, |
| 138 |
}, |
| 139 |
{ |
| 140 |
name: "non-default MaxQueueSize and BatchTimeout", |
| 141 |
o: []BatchSpanProcessorOption{ |
| 142 |
WithBatchTimeout(schDelay), |
| 143 |
WithMaxQueueSize(200), |
| 144 |
}, |
| 145 |
wantNumSpans: 205, |
| 146 |
wantBatchCount: 1, |
| 147 |
genNumSpans: 205, |
| 148 |
}, |
| 149 |
{ |
| 150 |
name: "non-default MaxQueueSize, BatchTimeout and MaxExportBatchSize", |
| 151 |
o: []BatchSpanProcessorOption{ |
| 152 |
WithBatchTimeout(schDelay), |
| 153 |
WithMaxQueueSize(205), |
| 154 |
WithMaxExportBatchSize(20), |
| 155 |
}, |
| 156 |
wantNumSpans: 210, |
| 157 |
wantBatchCount: 11, |
| 158 |
genNumSpans: 210, |
| 159 |
}, |
| 160 |
{ |
| 161 |
name: "blocking option", |
| 162 |
o: []BatchSpanProcessorOption{ |
| 163 |
WithBatchTimeout(schDelay), |
| 164 |
WithMaxQueueSize(200), |
| 165 |
WithMaxExportBatchSize(20), |
| 166 |
}, |
| 167 |
wantNumSpans: 205, |
| 168 |
wantBatchCount: 11, |
| 169 |
genNumSpans: 205, |
| 170 |
}, |
| 171 |
{ |
| 172 |
name: "parallel span generation", |
| 173 |
o: []BatchSpanProcessorOption{ |
| 174 |
WithBatchTimeout(schDelay), |
| 175 |
WithMaxQueueSize(200), |
| 176 |
}, |
| 177 |
wantNumSpans: 205, |
| 178 |
wantBatchCount: 1, |
| 179 |
genNumSpans: 205, |
| 180 |
parallel: true, |
| 181 |
}, |
| 182 |
{ |
| 183 |
name: "parallel span blocking", |
| 184 |
o: []BatchSpanProcessorOption{ |
| 185 |
WithBatchTimeout(schDelay), |
| 186 |
WithMaxExportBatchSize(200), |
| 187 |
}, |
| 188 |
wantNumSpans: 2000, |
| 189 |
wantBatchCount: 10, |
| 190 |
genNumSpans: 2000, |
| 191 |
parallel: true, |
| 192 |
}, |
| 193 |
} |
| 194 |
for _, option := range options { |
| 195 |
t.Run(option.name, func(t *testing.T) { |
| 196 |
te := testBatchExporter{} |
| 197 |
tp := basicTracerProvider(t) |
| 198 |
ssp := createAndRegisterBatchSP(option, &te) |
| 199 |
if ssp == nil { |
| 200 |
t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name) |
| 201 |
} |
| 202 |
tp.RegisterSpanProcessor(ssp) |
| 203 |
tr := tp.Tracer("BatchSpanProcessorWithOptions") |
| 204 |
|
| 205 |
if option.parallel { |
| 206 |
generateSpanParallel(t, tr, option) |
| 207 |
} else { |
| 208 |
generateSpan(t, tr, option) |
| 209 |
} |
| 210 |
|
| 211 |
tp.UnregisterSpanProcessor(ssp) |
| 212 |
|
| 213 |
gotNumOfSpans := te.len() |
| 214 |
if option.wantNumSpans > 0 && option.wantNumSpans != gotNumOfSpans { |
| 215 |
t.Errorf("number of exported span: got %+v, want %+v\n", |
| 216 |
gotNumOfSpans, option.wantNumSpans) |
| 217 |
} |
| 218 |
|
| 219 |
gotBatchCount := te.getBatchCount() |
| 220 |
if option.wantBatchCount > 0 && gotBatchCount < option.wantBatchCount { |
| 221 |
t.Errorf("number batches: got %+v, want >= %+v\n", |
| 222 |
gotBatchCount, option.wantBatchCount) |
| 223 |
t.Errorf("Batches %v\n", te.sizes) |
| 224 |
} |
| 225 |
}) |
| 226 |
} |
| 227 |
} |
| 228 |
|
| 229 |
func TestNewBatchSpanProcessorWithEnvOptions(t *testing.T) { |
| 230 |
options := []testOption{ |
| 231 |
{ |
| 232 |
name: "BatchSpanProcessorEnvOptions - Basic", |
| 233 |
wantNumSpans: 2053, |
| 234 |
wantBatchCount: 1, |
| 235 |
genNumSpans: 2053, |
| 236 |
envs: map[string]string{ |
| 237 |
env.BatchSpanProcessorMaxQueueSizeKey: "5000", |
| 238 |
env.BatchSpanProcessorMaxExportBatchSizeKey: "5000", |
| 239 |
}, |
| 240 |
}, |
| 241 |
{ |
| 242 |
name: "BatchSpanProcessorEnvOptions - A lager max export batch size than queue size", |
| 243 |
wantNumSpans: 2053, |
| 244 |
wantBatchCount: 4, |
| 245 |
genNumSpans: 2053, |
| 246 |
envs: map[string]string{ |
| 247 |
env.BatchSpanProcessorMaxQueueSizeKey: "5000", |
| 248 |
env.BatchSpanProcessorMaxExportBatchSizeKey: "10000", |
| 249 |
}, |
| 250 |
}, |
| 251 |
{ |
| 252 |
name: "BatchSpanProcessorEnvOptions - A lage max export batch size with a small queue size", |
| 253 |
wantNumSpans: 2053, |
| 254 |
wantBatchCount: 42, |
| 255 |
genNumSpans: 2053, |
| 256 |
envs: map[string]string{ |
| 257 |
env.BatchSpanProcessorMaxQueueSizeKey: "50", |
| 258 |
env.BatchSpanProcessorMaxExportBatchSizeKey: "10000", |
| 259 |
}, |
| 260 |
}, |
| 261 |
} |
| 262 |
|
| 263 |
for _, option := range options { |
| 264 |
t.Run(option.name, func(t *testing.T) { |
| 265 |
for k, v := range option.envs { |
| 266 |
t.Setenv(k, v) |
| 267 |
} |
| 268 |
|
| 269 |
te := testBatchExporter{} |
| 270 |
tp := basicTracerProvider(t) |
| 271 |
ssp := createAndRegisterBatchSP(option, &te) |
| 272 |
if ssp == nil { |
| 273 |
t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name) |
| 274 |
} |
| 275 |
tp.RegisterSpanProcessor(ssp) |
| 276 |
tr := tp.Tracer("BatchSpanProcessorWithOptions") |
| 277 |
|
| 278 |
if option.parallel { |
| 279 |
generateSpanParallel(t, tr, option) |
| 280 |
} else { |
| 281 |
generateSpan(t, tr, option) |
| 282 |
} |
| 283 |
|
| 284 |
tp.UnregisterSpanProcessor(ssp) |
| 285 |
|
| 286 |
gotNumOfSpans := te.len() |
| 287 |
if option.wantNumSpans > 0 && option.wantNumSpans != gotNumOfSpans { |
| 288 |
t.Errorf("number of exported span: got %+v, want %+v\n", |
| 289 |
gotNumOfSpans, option.wantNumSpans) |
| 290 |
} |
| 291 |
|
| 292 |
gotBatchCount := te.getBatchCount() |
| 293 |
if option.wantBatchCount > 0 && gotBatchCount < option.wantBatchCount { |
| 294 |
t.Errorf("number batches: got %+v, want >= %+v\n", |
| 295 |
gotBatchCount, option.wantBatchCount) |
| 296 |
t.Errorf("Batches %v\n", te.sizes) |
| 297 |
} |
| 298 |
}) |
| 299 |
} |
| 300 |
} |
| 301 |
|
| 302 |
type stuckExporter struct { |
| 303 |
testBatchExporter |
| 304 |
} |
| 305 |
|
| 306 |
// ExportSpans waits for ctx to expire and returns that error. |
| 307 |
func (e *stuckExporter) ExportSpans(ctx context.Context, _ []ReadOnlySpan) error { |
| 308 |
<-ctx.Done() |
| 309 |
e.err = ctx.Err() |
| 310 |
return ctx.Err() |
| 311 |
} |
| 312 |
|
| 313 |
func TestBatchSpanProcessorExportTimeout(t *testing.T) { |
| 314 |
exp := new(stuckExporter) |
| 315 |
bsp := NewBatchSpanProcessor( |
| 316 |
exp, |
| 317 |
// Set a non-zero export timeout so a deadline is set. |
| 318 |
WithExportTimeout(1*time.Microsecond), |
| 319 |
WithBlocking(), |
| 320 |
) |
| 321 |
tp := basicTracerProvider(t) |
| 322 |
tp.RegisterSpanProcessor(bsp) |
| 323 |
|
| 324 |
tr := tp.Tracer("BatchSpanProcessorExportTimeout") |
| 325 |
generateSpan(t, tr, testOption{genNumSpans: 1}) |
| 326 |
tp.UnregisterSpanProcessor(bsp) |
| 327 |
|
| 328 |
if !errors.Is(exp.err, context.DeadlineExceeded) { |
| 329 |
t.Errorf("context deadline error not returned: got %+v", exp.err) |
| 330 |
} |
| 331 |
} |
| 332 |
|
| 333 |
func createAndRegisterBatchSP(option testOption, te *testBatchExporter) SpanProcessor { |
| 334 |
// Always use blocking queue to avoid flaky tests. |
| 335 |
options := append(option.o, WithBlocking()) |
| 336 |
return NewBatchSpanProcessor(te, options...) |
| 337 |
} |
| 338 |
|
| 339 |
func generateSpan(t *testing.T, tr trace.Tracer, option testOption) { |
| 340 |
sc := getSpanContext() |
| 341 |
|
| 342 |
for i := 0; i < option.genNumSpans; i++ { |
| 343 |
tid := sc.TraceID() |
| 344 |
binary.BigEndian.PutUint64(tid[0:8], uint64(i+1)) |
| 345 |
newSc := sc.WithTraceID(tid) |
| 346 |
ctx := trace.ContextWithRemoteSpanContext(t.Context(), newSc) |
| 347 |
_, span := tr.Start(ctx, option.name) |
| 348 |
span.End() |
| 349 |
} |
| 350 |
} |
| 351 |
|
| 352 |
func generateSpanParallel(t *testing.T, tr trace.Tracer, option testOption) { |
| 353 |
sc := getSpanContext() |
| 354 |
|
| 355 |
wg := &sync.WaitGroup{} |
| 356 |
for i := 0; i < option.genNumSpans; i++ { |
| 357 |
tid := sc.TraceID() |
| 358 |
binary.BigEndian.PutUint64(tid[0:8], uint64(i+1)) |
| 359 |
|
| 360 |
wg.Add(1) |
| 361 |
go func(sc trace.SpanContext) { |
| 362 |
ctx := trace.ContextWithRemoteSpanContext(t.Context(), sc) |
| 363 |
_, span := tr.Start(ctx, option.name) |
| 364 |
span.End() |
| 365 |
wg.Done() |
| 366 |
}(sc.WithTraceID(tid)) |
| 367 |
} |
| 368 |
wg.Wait() |
| 369 |
} |
| 370 |
|
| 371 |
func getSpanContext() trace.SpanContext { |
| 372 |
tid, _ := trace.TraceIDFromHex("01020304050607080102040810203040") |
| 373 |
sid, _ := trace.SpanIDFromHex("0102040810203040") |
| 374 |
return trace.NewSpanContext(trace.SpanContextConfig{ |
| 375 |
TraceID: tid, |
| 376 |
SpanID: sid, |
| 377 |
TraceFlags: 0x1, |
| 378 |
}) |
| 379 |
} |
| 380 |
|
| 381 |
func TestBatchSpanProcessorShutdown(t *testing.T) { |
| 382 |
var bp testBatchExporter |
| 383 |
bsp := NewBatchSpanProcessor(&bp) |
| 384 |
|
| 385 |
err := bsp.Shutdown(t.Context()) |
| 386 |
if err != nil { |
| 387 |
t.Error("Error shutting the BatchSpanProcessor down\n") |
| 388 |
} |
| 389 |
assert.Equal(t, 1, bp.shutdownCount, "shutdown from span exporter not called") |
| 390 |
|
| 391 |
// Multiple call to Shutdown() should not panic. |
| 392 |
err = bsp.Shutdown(t.Context()) |
| 393 |
if err != nil { |
| 394 |
t.Error("Error shutting the BatchSpanProcessor down\n") |
| 395 |
} |
| 396 |
assert.Equal(t, 1, bp.shutdownCount) |
| 397 |
} |
| 398 |
|
| 399 |
func TestBatchSpanProcessorPostShutdown(t *testing.T) { |
| 400 |
tp := basicTracerProvider(t) |
| 401 |
be := testBatchExporter{} |
| 402 |
bsp := NewBatchSpanProcessor(&be) |
| 403 |
|
| 404 |
tp.RegisterSpanProcessor(bsp) |
| 405 |
tr := tp.Tracer("Normal") |
| 406 |
|
| 407 |
generateSpanParallel(t, tr, testOption{ |
| 408 |
o: []BatchSpanProcessorOption{ |
| 409 |
WithMaxExportBatchSize(50), |
| 410 |
}, |
| 411 |
genNumSpans: 60, |
| 412 |
}) |
| 413 |
|
| 414 |
require.NoError(t, bsp.Shutdown(t.Context()), "shutting down BatchSpanProcessor") |
| 415 |
lenJustAfterShutdown := be.len() |
| 416 |
|
| 417 |
_, span := tr.Start(t.Context(), "foo") |
| 418 |
span.End() |
| 419 |
assert.NoError(t, bsp.ForceFlush(t.Context()), "force flushing BatchSpanProcessor") |
| 420 |
|
| 421 |
assert.Equal(t, lenJustAfterShutdown, be.len(), "OnEnd and ForceFlush should have no effect after Shutdown") |
| 422 |
} |
| 423 |
|
| 424 |
func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) { |
| 425 |
te := testBatchExporter{} |
| 426 |
tp := basicTracerProvider(t) |
| 427 |
option := testOption{ |
| 428 |
name: "default BatchSpanProcessorOptions", |
| 429 |
o: []BatchSpanProcessorOption{ |
| 430 |
WithMaxQueueSize(0), |
| 431 |
WithMaxExportBatchSize(3000), |
| 432 |
}, |
| 433 |
wantNumSpans: 2053, |
| 434 |
wantBatchCount: 1, |
| 435 |
genNumSpans: 2053, |
| 436 |
} |
| 437 |
ssp := createAndRegisterBatchSP(option, &te) |
| 438 |
if ssp == nil { |
| 439 |
t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name) |
| 440 |
} |
| 441 |
tp.RegisterSpanProcessor(ssp) |
| 442 |
tr := tp.Tracer("BatchSpanProcessorWithOption") |
| 443 |
if option.parallel { |
| 444 |
generateSpanParallel(t, tr, option) |
| 445 |
} else { |
| 446 |
generateSpan(t, tr, option) |
| 447 |
} |
| 448 |
|
| 449 |
// Force flush any held span batches |
| 450 |
err := ssp.ForceFlush(t.Context()) |
| 451 |
|
| 452 |
assertMaxSpanDiff(t, te.len(), option.wantNumSpans, 10) |
| 453 |
|
| 454 |
gotBatchCount := te.getBatchCount() |
| 455 |
if gotBatchCount < option.wantBatchCount { |
| 456 |
t.Errorf("number batches: got %+v, want >= %+v\n", |
| 457 |
gotBatchCount, option.wantBatchCount) |
| 458 |
t.Errorf("Batches %v\n", te.sizes) |
| 459 |
} |
| 460 |
assert.NoError(t, err) |
| 461 |
} |
| 462 |
|
| 463 |
func TestBatchSpanProcessorDropBatchIfFailed(t *testing.T) { |
| 464 |
te := testBatchExporter{ |
| 465 |
errors: []error{errors.New("fail to export")}, |
| 466 |
} |
| 467 |
tp := basicTracerProvider(t) |
| 468 |
option := testOption{ |
| 469 |
o: []BatchSpanProcessorOption{ |
| 470 |
WithMaxQueueSize(0), |
| 471 |
WithMaxExportBatchSize(2000), |
| 472 |
}, |
| 473 |
wantNumSpans: 1000, |
| 474 |
wantBatchCount: 1, |
| 475 |
genNumSpans: 1000, |
| 476 |
} |
| 477 |
ssp := createAndRegisterBatchSP(option, &te) |
| 478 |
if ssp == nil { |
| 479 |
t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name) |
| 480 |
} |
| 481 |
tp.RegisterSpanProcessor(ssp) |
| 482 |
tr := tp.Tracer("BatchSpanProcessorWithOption") |
| 483 |
if option.parallel { |
| 484 |
generateSpanParallel(t, tr, option) |
| 485 |
} else { |
| 486 |
generateSpan(t, tr, option) |
| 487 |
} |
| 488 |
|
| 489 |
// Force flush any held span batches |
| 490 |
err := ssp.ForceFlush(t.Context()) |
| 491 |
assert.Error(t, err) |
| 492 |
assert.EqualError(t, err, "fail to export") |
| 493 |
|
| 494 |
// First flush will fail, nothing should be exported. |
| 495 |
assertMaxSpanDiff(t, te.droppedCount, option.wantNumSpans, 10) |
| 496 |
assert.Equal(t, 0, te.len()) |
| 497 |
assert.Equal(t, 0, te.getBatchCount()) |
| 498 |
|
| 499 |
// Generate a new batch, this will succeed |
| 500 |
if option.parallel { |
| 501 |
generateSpanParallel(t, tr, option) |
| 502 |
} else { |
| 503 |
generateSpan(t, tr, option) |
| 504 |
} |
| 505 |
|
| 506 |
// Force flush any held span batches |
| 507 |
err = ssp.ForceFlush(t.Context()) |
| 508 |
assert.NoError(t, err) |
| 509 |
|
| 510 |
assertMaxSpanDiff(t, te.len(), option.wantNumSpans, 10) |
| 511 |
gotBatchCount := te.getBatchCount() |
| 512 |
if gotBatchCount < option.wantBatchCount { |
| 513 |
t.Errorf("number batches: got %+v, want >= %+v\n", |
| 514 |
gotBatchCount, option.wantBatchCount) |
| 515 |
t.Errorf("Batches %v\n", te.sizes) |
| 516 |
} |
| 517 |
} |
| 518 |
|
| 519 |
func assertMaxSpanDiff(t *testing.T, want, got, maxDif int) { |
| 520 |
spanDifference := want - got |
| 521 |
if spanDifference < 0 { |
| 522 |
spanDifference *= -1 |
| 523 |
} |
| 524 |
if spanDifference > maxDif { |
| 525 |
t.Errorf("number of exported span not equal to or within %d less than: got %+v, want %+v\n", |
| 526 |
maxDif, got, want) |
| 527 |
} |
| 528 |
} |
| 529 |
|
| 530 |
type indefiniteExporter struct { |
| 531 |
stop chan struct{} |
| 532 |
} |
| 533 |
|
| 534 |
func newIndefiniteExporter(t *testing.T) indefiniteExporter { |
| 535 |
e := indefiniteExporter{stop: make(chan struct{})} |
| 536 |
t.Cleanup(func() { |
| 537 |
go close(e.stop) |
| 538 |
}) |
| 539 |
return e |
| 540 |
} |
| 541 |
|
| 542 |
func (indefiniteExporter) Shutdown(context.Context) error { |
| 543 |
return nil |
| 544 |
} |
| 545 |
|
| 546 |
func (e indefiniteExporter) ExportSpans(ctx context.Context, _ []ReadOnlySpan) error { |
| 547 |
<-e.stop |
| 548 |
return ctx.Err() |
| 549 |
} |
| 550 |
|
| 551 |
func TestBatchSpanProcessorForceFlushCancellation(t *testing.T) { |
| 552 |
ctx, cancel := context.WithCancel(t.Context()) |
| 553 |
// Cancel the context |
| 554 |
cancel() |
| 555 |
|
| 556 |
bsp := NewBatchSpanProcessor(newIndefiniteExporter(t)) |
| 557 |
t.Cleanup(func() { |
| 558 |
//nolint:usetesting // required to avoid getting a canceled context at cleanup. |
| 559 |
assert.NoError(t, bsp.Shutdown(context.Background())) |
| 560 |
}) |
| 561 |
|
| 562 |
if got, want := bsp.ForceFlush(ctx), context.Canceled; !errors.Is(got, want) { |
| 563 |
t.Errorf("expected %q error, got %v", want, got) |
| 564 |
} |
| 565 |
} |
| 566 |
|
| 567 |
func TestBatchSpanProcessorForceFlushTimeout(t *testing.T) { |
| 568 |
tp := basicTracerProvider(t) |
| 569 |
exp := newIndefiniteExporter(t) |
| 570 |
bsp := NewBatchSpanProcessor(exp) |
| 571 |
tp.RegisterSpanProcessor(bsp) |
| 572 |
tr := tp.Tracer(t.Name()) |
| 573 |
_, span := tr.Start(t.Context(), "foo") |
| 574 |
span.End() |
| 575 |
|
| 576 |
// Add timeout to context to test deadline |
| 577 |
ctx, cancel := context.WithTimeout(t.Context(), time.Millisecond) |
| 578 |
defer cancel() |
| 579 |
|
| 580 |
if got, want := bsp.ForceFlush(ctx), context.DeadlineExceeded; !errors.Is(got, want) { |
| 581 |
t.Errorf("expected %q error, got %v", want, got) |
| 582 |
} |
| 583 |
} |
| 584 |
|
| 585 |
func TestBatchSpanProcessorForceFlushQueuedSpans(t *testing.T) { |
| 586 |
ctx := t.Context() |
| 587 |
|
| 588 |
var bp testBatchExporter |
| 589 |
bsp := NewBatchSpanProcessor(&bp) |
| 590 |
tp := basicTracerProvider(t) |
| 591 |
tp.RegisterSpanProcessor(bsp) |
| 592 |
t.Cleanup(func() { |
| 593 |
//nolint:usetesting // required to avoid getting a canceled context at cleanup. |
| 594 |
assert.NoError(t, tp.Shutdown(context.Background())) |
| 595 |
}) |
| 596 |
|
| 597 |
tracer := tp.Tracer("tracer") |
| 598 |
|
| 599 |
for i := range 10 { |
| 600 |
_, span := tracer.Start(ctx, fmt.Sprintf("span%d", i)) |
| 601 |
span.End() |
| 602 |
|
| 603 |
err := tp.ForceFlush(ctx) |
| 604 |
assert.NoError(t, err) |
| 605 |
|
| 606 |
assert.Len(t, bp.spans, i+1) |
| 607 |
} |
| 608 |
} |
| 609 |
|
| 610 |
func TestBatchSpanProcessorConcurrentSafe(t *testing.T) { |
| 611 |
ctx := t.Context() |
| 612 |
var bp testBatchExporter |
| 613 |
bsp := NewBatchSpanProcessor(&bp) |
| 614 |
tp := basicTracerProvider(t) |
| 615 |
tp.RegisterSpanProcessor(bsp) |
| 616 |
tr := tp.Tracer(t.Name()) |
| 617 |
|
| 618 |
var wg sync.WaitGroup |
| 619 |
|
| 620 |
wg.Go(func() { |
| 621 |
generateSpan(t, tr, testOption{genNumSpans: 1}) |
| 622 |
}) |
| 623 |
|
| 624 |
wg.Go(func() { |
| 625 |
_ = bsp.ForceFlush(ctx) |
| 626 |
}) |
| 627 |
|
| 628 |
wg.Go(func() { |
| 629 |
_ = bsp.Shutdown(ctx) |
| 630 |
}) |
| 631 |
|
| 632 |
wg.Go(func() { |
| 633 |
_ = tp.ForceFlush(ctx) |
| 634 |
}) |
| 635 |
|
| 636 |
wg.Go(func() { |
| 637 |
_ = tp.Shutdown(ctx) |
| 638 |
}) |
| 639 |
|
| 640 |
wg.Wait() |
| 641 |
} |
| 642 |
|
| 643 |
// Drop metrics not being tested in this test. |
| 644 |
var dropSpanMetricsView = sdkmetric.NewView( |
| 645 |
sdkmetric.Instrument{ |
| 646 |
Name: "otel.sdk.span.*", |
| 647 |
}, |
| 648 |
sdkmetric.Stream{Aggregation: sdkmetric.AggregationDrop{}}, |
| 649 |
) |
| 650 |
|
| 651 |
func TestBatchSpanProcessorMetricsDisabled(t *testing.T) { |
| 652 |
t.Setenv("OTEL_GO_X_OBSERVABILITY", "false") |
| 653 |
tp := basicTracerProvider(t) |
| 654 |
reader := sdkmetric.NewManualReader() |
| 655 |
meterProvider := sdkmetric.NewMeterProvider( |
| 656 |
sdkmetric.WithReader(reader), |
| 657 |
sdkmetric.WithView(dropSpanMetricsView), |
| 658 |
) |
| 659 |
otel.SetMeterProvider(meterProvider) |
| 660 |
me := newBlockingExporter() |
| 661 |
t.Cleanup(func() { |
| 662 |
//nolint:usetesting // required to avoid getting a canceled context at cleanup. |
| 663 |
assert.NoError(t, me.Shutdown(context.Background())) |
| 664 |
}) |
| 665 |
bsp := NewBatchSpanProcessor( |
| 666 |
me, |
| 667 |
// Make sure timeout doesn't trigger during the test. |
| 668 |
WithBatchTimeout(time.Hour), |
| 669 |
WithMaxQueueSize(2), |
| 670 |
WithMaxExportBatchSize(2), |
| 671 |
) |
| 672 |
tp.RegisterSpanProcessor(bsp) |
| 673 |
|
| 674 |
tr := tp.Tracer("TestBatchSpanProcessorMetricsDisabled") |
| 675 |
// Generate 2 spans, which export and block during the export call. |
| 676 |
generateSpan(t, tr, testOption{genNumSpans: 2}) |
| 677 |
ctx, cancel := context.WithTimeout(t.Context(), time.Second) |
| 678 |
defer cancel() |
| 679 |
assert.NoError(t, me.waitForSpans(ctx, 2)) |
| 680 |
|
| 681 |
// Validate that there are no metrics produced. |
| 682 |
gotMetrics := new(metricdata.ResourceMetrics) |
| 683 |
assert.NoError(t, reader.Collect(t.Context(), gotMetrics)) |
| 684 |
require.Empty(t, gotMetrics.ScopeMetrics) |
| 685 |
// Generate 3 spans. 2 fill the queue, and 1 is dropped because the queue is full. |
| 686 |
generateSpan(t, tr, testOption{genNumSpans: 3}) |
| 687 |
// Validate that there are no metrics produced. |
| 688 |
gotMetrics = new(metricdata.ResourceMetrics) |
| 689 |
assert.NoError(t, reader.Collect(t.Context(), gotMetrics)) |
| 690 |
require.Empty(t, gotMetrics.ScopeMetrics) |
| 691 |
} |
| 692 |
|
| 693 |
func TestBatchSpanProcessorMetrics(t *testing.T) { |
| 694 |
// Reset for deterministic component ID. |
| 695 |
processorIDCounter.Store(componentID) |
| 696 |
|
| 697 |
t.Setenv("OTEL_GO_X_OBSERVABILITY", "true") |
| 698 |
tp := basicTracerProvider(t) |
| 699 |
reader := sdkmetric.NewManualReader() |
| 700 |
meterProvider := sdkmetric.NewMeterProvider( |
| 701 |
sdkmetric.WithReader(reader), |
| 702 |
sdkmetric.WithView(dropSpanMetricsView), |
| 703 |
) |
| 704 |
otel.SetMeterProvider(meterProvider) |
| 705 |
me := newBlockingExporter() |
| 706 |
t.Cleanup(func() { |
| 707 |
//nolint:usetesting // required to avoid getting a canceled context at cleanup. |
| 708 |
assert.NoError(t, me.Shutdown(context.Background())) |
| 709 |
}) |
| 710 |
bsp := NewBatchSpanProcessor( |
| 711 |
me, |
| 712 |
// Make sure timeout doesn't trigger during the test. |
| 713 |
WithBatchTimeout(time.Hour), |
| 714 |
WithMaxQueueSize(2), |
| 715 |
WithMaxExportBatchSize(2), |
| 716 |
) |
| 717 |
tp.RegisterSpanProcessor(bsp) |
| 718 |
|
| 719 |
tr := tp.Tracer("TestBatchSpanProcessorMetrics") |
| 720 |
// Generate 2 spans, which export and block during the export call. |
| 721 |
generateSpan(t, tr, testOption{genNumSpans: 2}) |
| 722 |
ctx, cancel := context.WithTimeout(t.Context(), time.Second) |
| 723 |
defer cancel() |
| 724 |
assert.NoError(t, me.waitForSpans(ctx, 2)) |
| 725 |
assertObsScopeMetrics(t, reader, expectMetrics{ |
| 726 |
queueCapacity: 2, |
| 727 |
queueSize: 0, |
| 728 |
successProcessed: 2, |
| 729 |
}) |
| 730 |
// Generate 3 spans. 2 fill the queue, and 1 is dropped because the queue is full. |
| 731 |
generateSpan(t, tr, testOption{genNumSpans: 3}) |
| 732 |
assertObsScopeMetrics(t, reader, expectMetrics{ |
| 733 |
queueCapacity: 2, |
| 734 |
queueSize: 2, |
| 735 |
queueFullProcessed: 1, |
| 736 |
successProcessed: 2, |
| 737 |
}) |
| 738 |
} |
| 739 |
|
| 740 |
func TestBatchSpanProcessorBlockingMetrics(t *testing.T) { |
| 741 |
// Reset for deterministic component ID. |
| 742 |
processorIDCounter.Store(componentID) |
| 743 |
|
| 744 |
t.Setenv("OTEL_GO_X_OBSERVABILITY", "true") |
| 745 |
tp := basicTracerProvider(t) |
| 746 |
reader := sdkmetric.NewManualReader() |
| 747 |
meterProvider := sdkmetric.NewMeterProvider( |
| 748 |
sdkmetric.WithReader(reader), |
| 749 |
sdkmetric.WithView(dropSpanMetricsView), |
| 750 |
) |
| 751 |
otel.SetMeterProvider(meterProvider) |
| 752 |
me := newBlockingExporter() |
| 753 |
t.Cleanup(func() { |
| 754 |
//nolint:usetesting // required to avoid getting a canceled context at cleanup. |
| 755 |
assert.NoError(t, me.Shutdown(context.Background())) |
| 756 |
}) |
| 757 |
bsp := NewBatchSpanProcessor( |
| 758 |
me, |
| 759 |
// Use WithBlocking so we can trigger a queueFull using ForceFlush. |
| 760 |
WithBlocking(), |
| 761 |
// Make sure timeout doesn't trigger during the test. |
| 762 |
WithBatchTimeout(time.Hour), |
| 763 |
WithMaxQueueSize(2), |
| 764 |
WithMaxExportBatchSize(2), |
| 765 |
) |
| 766 |
tp.RegisterSpanProcessor(bsp) |
| 767 |
|
| 768 |
tr := tp.Tracer("TestBatchSpanProcessorBlockingMetrics") |
| 769 |
// Generate 2 spans that are exported to the exporter, which blocks. |
| 770 |
generateSpan(t, tr, testOption{genNumSpans: 2}) |
| 771 |
ctx, cancel := context.WithTimeout(t.Context(), time.Second) |
| 772 |
defer cancel() |
| 773 |
assert.NoError(t, me.waitForSpans(ctx, 2)) |
| 774 |
assertObsScopeMetrics(t, reader, expectMetrics{ |
| 775 |
queueCapacity: 2, |
| 776 |
queueSize: 0, |
| 777 |
successProcessed: 2, |
| 778 |
}) |
| 779 |
// Generate 2 spans to fill the queue. |
| 780 |
generateSpan(t, tr, testOption{genNumSpans: 2}) |
| 781 |
go func() { |
| 782 |
// Generate a span which blocks because the queue is full. |
| 783 |
generateSpan(t, tr, testOption{genNumSpans: 1}) |
| 784 |
}() |
| 785 |
assertObsScopeMetrics(t, reader, expectMetrics{ |
| 786 |
queueCapacity: 2, |
| 787 |
queueSize: 2, |
| 788 |
successProcessed: 2, |
| 789 |
}) |
| 790 |
|
| 791 |
// Use ForceFlush to force the span that is blocking on the full queue to be dropped. |
| 792 |
ctx, cancel = context.WithTimeout(t.Context(), 10*time.Millisecond) |
| 793 |
defer cancel() |
| 794 |
assert.Error(t, tp.ForceFlush(ctx)) |
| 795 |
assertObsScopeMetrics(t, reader, expectMetrics{ |
| 796 |
queueCapacity: 2, |
| 797 |
queueSize: 2, |
| 798 |
queueFullProcessed: 1, |
| 799 |
successProcessed: 2, |
| 800 |
}) |
| 801 |
} |
| 802 |
|
| 803 |
type expectMetrics struct { |
| 804 |
queueCapacity int64 |
| 805 |
queueSize int64 |
| 806 |
successProcessed int64 |
| 807 |
queueFullProcessed int64 |
| 808 |
} |
| 809 |
|
| 810 |
func assertObsScopeMetrics( |
| 811 |
t *testing.T, |
| 812 |
reader sdkmetric.Reader, |
| 813 |
expectation expectMetrics, |
| 814 |
) { |
| 815 |
t.Helper() |
| 816 |
gotResourceMetrics := new(metricdata.ResourceMetrics) |
| 817 |
assert.NoError(t, reader.Collect(t.Context(), gotResourceMetrics)) |
| 818 |
|
| 819 |
componentNameAttr := observ.BSPComponentName(componentID) |
| 820 |
baseAttrs := attribute.NewSet( |
| 821 |
semconv.OTelComponentTypeBatchingSpanProcessor, |
| 822 |
componentNameAttr, |
| 823 |
) |
| 824 |
wantMetrics := []metricdata.Metrics{ |
| 825 |
{ |
| 826 |
Name: otelconv.SDKProcessorSpanQueueCapacity{}.Name(), |
| 827 |
Description: otelconv.SDKProcessorSpanQueueCapacity{}.Description(), |
| 828 |
Unit: otelconv.SDKProcessorSpanQueueCapacity{}.Unit(), |
| 829 |
Data: metricdata.Sum[int64]{ |
| 830 |
DataPoints: []metricdata.DataPoint[int64]{{Attributes: baseAttrs, Value: expectation.queueCapacity}}, |
| 831 |
Temporality: metricdata.CumulativeTemporality, |
| 832 |
IsMonotonic: false, |
| 833 |
}, |
| 834 |
}, |
| 835 |
{ |
| 836 |
Name: otelconv.SDKProcessorSpanQueueSize{}.Name(), |
| 837 |
Description: otelconv.SDKProcessorSpanQueueSize{}.Description(), |
| 838 |
Unit: otelconv.SDKProcessorSpanQueueSize{}.Unit(), |
| 839 |
Data: metricdata.Sum[int64]{ |
| 840 |
DataPoints: []metricdata.DataPoint[int64]{{Attributes: baseAttrs, Value: expectation.queueSize}}, |
| 841 |
Temporality: metricdata.CumulativeTemporality, |
| 842 |
IsMonotonic: false, |
| 843 |
}, |
| 844 |
}, |
| 845 |
} |
| 846 |
|
| 847 |
wantProcessedDataPoints := []metricdata.DataPoint[int64]{} |
| 848 |
if expectation.successProcessed > 0 { |
| 849 |
wantProcessedDataPoints = append(wantProcessedDataPoints, metricdata.DataPoint[int64]{ |
| 850 |
Value: expectation.successProcessed, |
| 851 |
Attributes: attribute.NewSet( |
| 852 |
semconv.OTelComponentTypeBatchingSpanProcessor, |
| 853 |
componentNameAttr, |
| 854 |
), |
| 855 |
}) |
| 856 |
} |
| 857 |
if expectation.queueFullProcessed > 0 { |
| 858 |
wantProcessedDataPoints = append(wantProcessedDataPoints, metricdata.DataPoint[int64]{ |
| 859 |
Value: expectation.queueFullProcessed, |
| 860 |
Attributes: attribute.NewSet( |
| 861 |
semconv.OTelComponentTypeBatchingSpanProcessor, |
| 862 |
componentNameAttr, |
| 863 |
observ.ErrQueueFull, |
| 864 |
), |
| 865 |
}) |
| 866 |
} |
| 867 |
|
| 868 |
if len(wantProcessedDataPoints) > 0 { |
| 869 |
wantMetrics = append(wantMetrics, |
| 870 |
metricdata.Metrics{ |
| 871 |
Name: otelconv.SDKProcessorSpanProcessed{}.Name(), |
| 872 |
Description: otelconv.SDKProcessorSpanProcessed{}.Description(), |
| 873 |
Unit: otelconv.SDKProcessorSpanProcessed{}.Unit(), |
| 874 |
Data: metricdata.Sum[int64]{ |
| 875 |
DataPoints: wantProcessedDataPoints, |
| 876 |
Temporality: metricdata.CumulativeTemporality, |
| 877 |
IsMonotonic: true, |
| 878 |
}, |
| 879 |
}, |
| 880 |
) |
| 881 |
} |
| 882 |
|
| 883 |
wantScopeMetric := metricdata.ScopeMetrics{ |
| 884 |
Scope: instrumentation.Scope{ |
| 885 |
Name: observ.ScopeName, |
| 886 |
Version: sdk.Version(), |
| 887 |
SchemaURL: observ.SchemaURL, |
| 888 |
}, |
| 889 |
Metrics: wantMetrics, |
| 890 |
} |
| 891 |
metricdatatest.AssertEqual(t, wantScopeMetric, gotResourceMetrics.ScopeMetrics[0], metricdatatest.IgnoreTimestamp()) |
| 892 |
} |
| 893 |
|
| 894 |
// blockingExporter blocks until the exported span is removed from the channel. |
| 895 |
type blockingExporter struct { |
| 896 |
shutdown chan struct{} |
| 897 |
total atomic.Int32 |
| 898 |
} |
| 899 |
|
| 900 |
func newBlockingExporter() *blockingExporter { |
| 901 |
e := &blockingExporter{shutdown: make(chan struct{})} |
| 902 |
return e |
| 903 |
} |
| 904 |
|
| 905 |
func (e *blockingExporter) Shutdown(ctx context.Context) error { |
| 906 |
select { |
| 907 |
case <-e.shutdown: |
| 908 |
default: |
| 909 |
close(e.shutdown) |
| 910 |
} |
| 911 |
return ctx.Err() |
| 912 |
} |
| 913 |
|
| 914 |
func (e *blockingExporter) ExportSpans(ctx context.Context, s []ReadOnlySpan) error { |
| 915 |
e.total.Add(int32(len(s))) |
| 916 |
<-e.shutdown |
| 917 |
return ctx.Err() |
| 918 |
} |
| 919 |
|
| 920 |
func (e *blockingExporter) waitForSpans(ctx context.Context, n int32) error { |
| 921 |
// Wait for all n spans to reach the export call |
| 922 |
for e.total.Load() < n { |
| 923 |
select { |
| 924 |
case <-ctx.Done(): |
| 925 |
return fmt.Errorf("timeout waiting for %d spans to be exported", n) |
| 926 |
default: |
| 927 |
// So the select will not block |
| 928 |
} |
| 929 |
runtime.Gosched() |
| 930 |
} |
| 931 |
return nil |
| 932 |
} |