Debian Codegraph @

sdk/trace/batch_span_processor_test.go

raw
1 // Copyright The OpenTelemetry Authors
2 // SPDX-License-Identifier: Apache-2.0
3
4 package trace
5
6 import (
7 "context"
8 "encoding/binary"
9 "errors"
10 "fmt"
11 "runtime"
12 "sync"
13 "sync/atomic"
14 "testing"
15 "time"
16
17 "github.com/stretchr/testify/assert"
18 "github.com/stretchr/testify/require"
19
20 "go.opentelemetry.io/otel"
21 "go.opentelemetry.io/otel/attribute"
22 "go.opentelemetry.io/otel/sdk"
23 "go.opentelemetry.io/otel/sdk/instrumentation"
24 sdkmetric "go.opentelemetry.io/otel/sdk/metric"
25 "go.opentelemetry.io/otel/sdk/metric/metricdata"
26 "go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
27 "go.opentelemetry.io/otel/sdk/trace/internal/env"
28 "go.opentelemetry.io/otel/sdk/trace/internal/observ"
29 semconv "go.opentelemetry.io/otel/semconv/v1.40.0"
30 "go.opentelemetry.io/otel/semconv/v1.40.0/otelconv"
31 "go.opentelemetry.io/otel/trace"
32 )
33
34 const componentID = 0
35
36 type testBatchExporter struct {
37 mu sync.Mutex
38 spans []ReadOnlySpan
39 sizes []int
40 batchCount int
41 shutdownCount int
42 errors []error
43 droppedCount int
44 idx int
45 err error
46 }
47
48 func (t *testBatchExporter) ExportSpans(ctx context.Context, spans []ReadOnlySpan) error {
49 t.mu.Lock()
50 defer t.mu.Unlock()
51
52 if t.idx < len(t.errors) {
53 t.droppedCount += len(spans)
54 err := t.errors[t.idx]
55 t.idx++
56 return err
57 }
58
59 select {
60 case <-ctx.Done():
61 t.err = ctx.Err()
62 return ctx.Err()
63 default:
64 }
65
66 t.spans = append(t.spans, spans...)
67 t.sizes = append(t.sizes, len(spans))
68 t.batchCount++
69 return nil
70 }
71
72 func (t *testBatchExporter) Shutdown(context.Context) error {
73 t.shutdownCount++
74 return nil
75 }
76
77 func (t *testBatchExporter) len() int {
78 t.mu.Lock()
79 defer t.mu.Unlock()
80 return len(t.spans)
81 }
82
83 func (t *testBatchExporter) getBatchCount() int {
84 t.mu.Lock()
85 defer t.mu.Unlock()
86 return t.batchCount
87 }
88
89 var _ SpanExporter = (*testBatchExporter)(nil)
90
91 func TestNewBatchSpanProcessorWithNilExporter(t *testing.T) {
92 tp := basicTracerProvider(t)
93 bsp := NewBatchSpanProcessor(nil)
94 tp.RegisterSpanProcessor(bsp)
95 tr := tp.Tracer("NilExporter")
96
97 _, span := tr.Start(t.Context(), "foo")
98 span.End()
99
100 // These should not panic.
101 bsp.OnStart(t.Context(), span.(ReadWriteSpan))
102 bsp.OnEnd(span.(ReadOnlySpan))
103 if err := bsp.ForceFlush(t.Context()); err != nil {
104 t.Errorf("failed to ForceFlush the BatchSpanProcessor: %v", err)
105 }
106 if err := bsp.Shutdown(t.Context()); err != nil {
107 t.Errorf("failed to Shutdown the BatchSpanProcessor: %v", err)
108 }
109 }
110
111 type testOption struct {
112 name string
113 o []BatchSpanProcessorOption
114 wantNumSpans int
115 wantBatchCount int
116 genNumSpans int
117 parallel bool
118 envs map[string]string
119 }
120
121 func TestNewBatchSpanProcessorWithOptions(t *testing.T) {
122 schDelay := 200 * time.Millisecond
123 options := []testOption{
124 {
125 name: "default BatchSpanProcessorOptions",
126 wantNumSpans: 2053,
127 wantBatchCount: 4,
128 genNumSpans: 2053,
129 },
130 {
131 name: "non-default BatchTimeout",
132 o: []BatchSpanProcessorOption{
133 WithBatchTimeout(schDelay),
134 },
135 wantNumSpans: 2053,
136 wantBatchCount: 4,
137 genNumSpans: 2053,
138 },
139 {
140 name: "non-default MaxQueueSize and BatchTimeout",
141 o: []BatchSpanProcessorOption{
142 WithBatchTimeout(schDelay),
143 WithMaxQueueSize(200),
144 },
145 wantNumSpans: 205,
146 wantBatchCount: 1,
147 genNumSpans: 205,
148 },
149 {
150 name: "non-default MaxQueueSize, BatchTimeout and MaxExportBatchSize",
151 o: []BatchSpanProcessorOption{
152 WithBatchTimeout(schDelay),
153 WithMaxQueueSize(205),
154 WithMaxExportBatchSize(20),
155 },
156 wantNumSpans: 210,
157 wantBatchCount: 11,
158 genNumSpans: 210,
159 },
160 {
161 name: "blocking option",
162 o: []BatchSpanProcessorOption{
163 WithBatchTimeout(schDelay),
164 WithMaxQueueSize(200),
165 WithMaxExportBatchSize(20),
166 },
167 wantNumSpans: 205,
168 wantBatchCount: 11,
169 genNumSpans: 205,
170 },
171 {
172 name: "parallel span generation",
173 o: []BatchSpanProcessorOption{
174 WithBatchTimeout(schDelay),
175 WithMaxQueueSize(200),
176 },
177 wantNumSpans: 205,
178 wantBatchCount: 1,
179 genNumSpans: 205,
180 parallel: true,
181 },
182 {
183 name: "parallel span blocking",
184 o: []BatchSpanProcessorOption{
185 WithBatchTimeout(schDelay),
186 WithMaxExportBatchSize(200),
187 },
188 wantNumSpans: 2000,
189 wantBatchCount: 10,
190 genNumSpans: 2000,
191 parallel: true,
192 },
193 }
194 for _, option := range options {
195 t.Run(option.name, func(t *testing.T) {
196 te := testBatchExporter{}
197 tp := basicTracerProvider(t)
198 ssp := createAndRegisterBatchSP(option, &te)
199 if ssp == nil {
200 t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name)
201 }
202 tp.RegisterSpanProcessor(ssp)
203 tr := tp.Tracer("BatchSpanProcessorWithOptions")
204
205 if option.parallel {
206 generateSpanParallel(t, tr, option)
207 } else {
208 generateSpan(t, tr, option)
209 }
210
211 tp.UnregisterSpanProcessor(ssp)
212
213 gotNumOfSpans := te.len()
214 if option.wantNumSpans > 0 && option.wantNumSpans != gotNumOfSpans {
215 t.Errorf("number of exported span: got %+v, want %+v\n",
216 gotNumOfSpans, option.wantNumSpans)
217 }
218
219 gotBatchCount := te.getBatchCount()
220 if option.wantBatchCount > 0 && gotBatchCount < option.wantBatchCount {
221 t.Errorf("number batches: got %+v, want >= %+v\n",
222 gotBatchCount, option.wantBatchCount)
223 t.Errorf("Batches %v\n", te.sizes)
224 }
225 })
226 }
227 }
228
229 func TestNewBatchSpanProcessorWithEnvOptions(t *testing.T) {
230 options := []testOption{
231 {
232 name: "BatchSpanProcessorEnvOptions - Basic",
233 wantNumSpans: 2053,
234 wantBatchCount: 1,
235 genNumSpans: 2053,
236 envs: map[string]string{
237 env.BatchSpanProcessorMaxQueueSizeKey: "5000",
238 env.BatchSpanProcessorMaxExportBatchSizeKey: "5000",
239 },
240 },
241 {
242 name: "BatchSpanProcessorEnvOptions - A lager max export batch size than queue size",
243 wantNumSpans: 2053,
244 wantBatchCount: 4,
245 genNumSpans: 2053,
246 envs: map[string]string{
247 env.BatchSpanProcessorMaxQueueSizeKey: "5000",
248 env.BatchSpanProcessorMaxExportBatchSizeKey: "10000",
249 },
250 },
251 {
252 name: "BatchSpanProcessorEnvOptions - A lage max export batch size with a small queue size",
253 wantNumSpans: 2053,
254 wantBatchCount: 42,
255 genNumSpans: 2053,
256 envs: map[string]string{
257 env.BatchSpanProcessorMaxQueueSizeKey: "50",
258 env.BatchSpanProcessorMaxExportBatchSizeKey: "10000",
259 },
260 },
261 }
262
263 for _, option := range options {
264 t.Run(option.name, func(t *testing.T) {
265 for k, v := range option.envs {
266 t.Setenv(k, v)
267 }
268
269 te := testBatchExporter{}
270 tp := basicTracerProvider(t)
271 ssp := createAndRegisterBatchSP(option, &te)
272 if ssp == nil {
273 t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name)
274 }
275 tp.RegisterSpanProcessor(ssp)
276 tr := tp.Tracer("BatchSpanProcessorWithOptions")
277
278 if option.parallel {
279 generateSpanParallel(t, tr, option)
280 } else {
281 generateSpan(t, tr, option)
282 }
283
284 tp.UnregisterSpanProcessor(ssp)
285
286 gotNumOfSpans := te.len()
287 if option.wantNumSpans > 0 && option.wantNumSpans != gotNumOfSpans {
288 t.Errorf("number of exported span: got %+v, want %+v\n",
289 gotNumOfSpans, option.wantNumSpans)
290 }
291
292 gotBatchCount := te.getBatchCount()
293 if option.wantBatchCount > 0 && gotBatchCount < option.wantBatchCount {
294 t.Errorf("number batches: got %+v, want >= %+v\n",
295 gotBatchCount, option.wantBatchCount)
296 t.Errorf("Batches %v\n", te.sizes)
297 }
298 })
299 }
300 }
301
302 type stuckExporter struct {
303 testBatchExporter
304 }
305
306 // ExportSpans waits for ctx to expire and returns that error.
307 func (e *stuckExporter) ExportSpans(ctx context.Context, _ []ReadOnlySpan) error {
308 <-ctx.Done()
309 e.err = ctx.Err()
310 return ctx.Err()
311 }
312
313 func TestBatchSpanProcessorExportTimeout(t *testing.T) {
314 exp := new(stuckExporter)
315 bsp := NewBatchSpanProcessor(
316 exp,
317 // Set a non-zero export timeout so a deadline is set.
318 WithExportTimeout(1*time.Microsecond),
319 WithBlocking(),
320 )
321 tp := basicTracerProvider(t)
322 tp.RegisterSpanProcessor(bsp)
323
324 tr := tp.Tracer("BatchSpanProcessorExportTimeout")
325 generateSpan(t, tr, testOption{genNumSpans: 1})
326 tp.UnregisterSpanProcessor(bsp)
327
328 if !errors.Is(exp.err, context.DeadlineExceeded) {
329 t.Errorf("context deadline error not returned: got %+v", exp.err)
330 }
331 }
332
333 func createAndRegisterBatchSP(option testOption, te *testBatchExporter) SpanProcessor {
334 // Always use blocking queue to avoid flaky tests.
335 options := append(option.o, WithBlocking())
336 return NewBatchSpanProcessor(te, options...)
337 }
338
339 func generateSpan(t *testing.T, tr trace.Tracer, option testOption) {
340 sc := getSpanContext()
341
342 for i := 0; i < option.genNumSpans; i++ {
343 tid := sc.TraceID()
344 binary.BigEndian.PutUint64(tid[0:8], uint64(i+1))
345 newSc := sc.WithTraceID(tid)
346 ctx := trace.ContextWithRemoteSpanContext(t.Context(), newSc)
347 _, span := tr.Start(ctx, option.name)
348 span.End()
349 }
350 }
351
352 func generateSpanParallel(t *testing.T, tr trace.Tracer, option testOption) {
353 sc := getSpanContext()
354
355 wg := &sync.WaitGroup{}
356 for i := 0; i < option.genNumSpans; i++ {
357 tid := sc.TraceID()
358 binary.BigEndian.PutUint64(tid[0:8], uint64(i+1))
359
360 wg.Add(1)
361 go func(sc trace.SpanContext) {
362 ctx := trace.ContextWithRemoteSpanContext(t.Context(), sc)
363 _, span := tr.Start(ctx, option.name)
364 span.End()
365 wg.Done()
366 }(sc.WithTraceID(tid))
367 }
368 wg.Wait()
369 }
370
371 func getSpanContext() trace.SpanContext {
372 tid, _ := trace.TraceIDFromHex("01020304050607080102040810203040")
373 sid, _ := trace.SpanIDFromHex("0102040810203040")
374 return trace.NewSpanContext(trace.SpanContextConfig{
375 TraceID: tid,
376 SpanID: sid,
377 TraceFlags: 0x1,
378 })
379 }
380
381 func TestBatchSpanProcessorShutdown(t *testing.T) {
382 var bp testBatchExporter
383 bsp := NewBatchSpanProcessor(&bp)
384
385 err := bsp.Shutdown(t.Context())
386 if err != nil {
387 t.Error("Error shutting the BatchSpanProcessor down\n")
388 }
389 assert.Equal(t, 1, bp.shutdownCount, "shutdown from span exporter not called")
390
391 // Multiple call to Shutdown() should not panic.
392 err = bsp.Shutdown(t.Context())
393 if err != nil {
394 t.Error("Error shutting the BatchSpanProcessor down\n")
395 }
396 assert.Equal(t, 1, bp.shutdownCount)
397 }
398
399 func TestBatchSpanProcessorPostShutdown(t *testing.T) {
400 tp := basicTracerProvider(t)
401 be := testBatchExporter{}
402 bsp := NewBatchSpanProcessor(&be)
403
404 tp.RegisterSpanProcessor(bsp)
405 tr := tp.Tracer("Normal")
406
407 generateSpanParallel(t, tr, testOption{
408 o: []BatchSpanProcessorOption{
409 WithMaxExportBatchSize(50),
410 },
411 genNumSpans: 60,
412 })
413
414 require.NoError(t, bsp.Shutdown(t.Context()), "shutting down BatchSpanProcessor")
415 lenJustAfterShutdown := be.len()
416
417 _, span := tr.Start(t.Context(), "foo")
418 span.End()
419 assert.NoError(t, bsp.ForceFlush(t.Context()), "force flushing BatchSpanProcessor")
420
421 assert.Equal(t, lenJustAfterShutdown, be.len(), "OnEnd and ForceFlush should have no effect after Shutdown")
422 }
423
424 func TestBatchSpanProcessorForceFlushSucceeds(t *testing.T) {
425 te := testBatchExporter{}
426 tp := basicTracerProvider(t)
427 option := testOption{
428 name: "default BatchSpanProcessorOptions",
429 o: []BatchSpanProcessorOption{
430 WithMaxQueueSize(0),
431 WithMaxExportBatchSize(3000),
432 },
433 wantNumSpans: 2053,
434 wantBatchCount: 1,
435 genNumSpans: 2053,
436 }
437 ssp := createAndRegisterBatchSP(option, &te)
438 if ssp == nil {
439 t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name)
440 }
441 tp.RegisterSpanProcessor(ssp)
442 tr := tp.Tracer("BatchSpanProcessorWithOption")
443 if option.parallel {
444 generateSpanParallel(t, tr, option)
445 } else {
446 generateSpan(t, tr, option)
447 }
448
449 // Force flush any held span batches
450 err := ssp.ForceFlush(t.Context())
451
452 assertMaxSpanDiff(t, te.len(), option.wantNumSpans, 10)
453
454 gotBatchCount := te.getBatchCount()
455 if gotBatchCount < option.wantBatchCount {
456 t.Errorf("number batches: got %+v, want >= %+v\n",
457 gotBatchCount, option.wantBatchCount)
458 t.Errorf("Batches %v\n", te.sizes)
459 }
460 assert.NoError(t, err)
461 }
462
463 func TestBatchSpanProcessorDropBatchIfFailed(t *testing.T) {
464 te := testBatchExporter{
465 errors: []error{errors.New("fail to export")},
466 }
467 tp := basicTracerProvider(t)
468 option := testOption{
469 o: []BatchSpanProcessorOption{
470 WithMaxQueueSize(0),
471 WithMaxExportBatchSize(2000),
472 },
473 wantNumSpans: 1000,
474 wantBatchCount: 1,
475 genNumSpans: 1000,
476 }
477 ssp := createAndRegisterBatchSP(option, &te)
478 if ssp == nil {
479 t.Fatalf("%s: Error creating new instance of BatchSpanProcessor\n", option.name)
480 }
481 tp.RegisterSpanProcessor(ssp)
482 tr := tp.Tracer("BatchSpanProcessorWithOption")
483 if option.parallel {
484 generateSpanParallel(t, tr, option)
485 } else {
486 generateSpan(t, tr, option)
487 }
488
489 // Force flush any held span batches
490 err := ssp.ForceFlush(t.Context())
491 assert.Error(t, err)
492 assert.EqualError(t, err, "fail to export")
493
494 // First flush will fail, nothing should be exported.
495 assertMaxSpanDiff(t, te.droppedCount, option.wantNumSpans, 10)
496 assert.Equal(t, 0, te.len())
497 assert.Equal(t, 0, te.getBatchCount())
498
499 // Generate a new batch, this will succeed
500 if option.parallel {
501 generateSpanParallel(t, tr, option)
502 } else {
503 generateSpan(t, tr, option)
504 }
505
506 // Force flush any held span batches
507 err = ssp.ForceFlush(t.Context())
508 assert.NoError(t, err)
509
510 assertMaxSpanDiff(t, te.len(), option.wantNumSpans, 10)
511 gotBatchCount := te.getBatchCount()
512 if gotBatchCount < option.wantBatchCount {
513 t.Errorf("number batches: got %+v, want >= %+v\n",
514 gotBatchCount, option.wantBatchCount)
515 t.Errorf("Batches %v\n", te.sizes)
516 }
517 }
518
519 func assertMaxSpanDiff(t *testing.T, want, got, maxDif int) {
520 spanDifference := want - got
521 if spanDifference < 0 {
522 spanDifference *= -1
523 }
524 if spanDifference > maxDif {
525 t.Errorf("number of exported span not equal to or within %d less than: got %+v, want %+v\n",
526 maxDif, got, want)
527 }
528 }
529
530 type indefiniteExporter struct {
531 stop chan struct{}
532 }
533
534 func newIndefiniteExporter(t *testing.T) indefiniteExporter {
535 e := indefiniteExporter{stop: make(chan struct{})}
536 t.Cleanup(func() {
537 go close(e.stop)
538 })
539 return e
540 }
541
542 func (indefiniteExporter) Shutdown(context.Context) error {
543 return nil
544 }
545
546 func (e indefiniteExporter) ExportSpans(ctx context.Context, _ []ReadOnlySpan) error {
547 <-e.stop
548 return ctx.Err()
549 }
550
551 func TestBatchSpanProcessorForceFlushCancellation(t *testing.T) {
552 ctx, cancel := context.WithCancel(t.Context())
553 // Cancel the context
554 cancel()
555
556 bsp := NewBatchSpanProcessor(newIndefiniteExporter(t))
557 t.Cleanup(func() {
558 //nolint:usetesting // required to avoid getting a canceled context at cleanup.
559 assert.NoError(t, bsp.Shutdown(context.Background()))
560 })
561
562 if got, want := bsp.ForceFlush(ctx), context.Canceled; !errors.Is(got, want) {
563 t.Errorf("expected %q error, got %v", want, got)
564 }
565 }
566
567 func TestBatchSpanProcessorForceFlushTimeout(t *testing.T) {
568 tp := basicTracerProvider(t)
569 exp := newIndefiniteExporter(t)
570 bsp := NewBatchSpanProcessor(exp)
571 tp.RegisterSpanProcessor(bsp)
572 tr := tp.Tracer(t.Name())
573 _, span := tr.Start(t.Context(), "foo")
574 span.End()
575
576 // Add timeout to context to test deadline
577 ctx, cancel := context.WithTimeout(t.Context(), time.Millisecond)
578 defer cancel()
579
580 if got, want := bsp.ForceFlush(ctx), context.DeadlineExceeded; !errors.Is(got, want) {
581 t.Errorf("expected %q error, got %v", want, got)
582 }
583 }
584
585 func TestBatchSpanProcessorForceFlushQueuedSpans(t *testing.T) {
586 ctx := t.Context()
587
588 var bp testBatchExporter
589 bsp := NewBatchSpanProcessor(&bp)
590 tp := basicTracerProvider(t)
591 tp.RegisterSpanProcessor(bsp)
592 t.Cleanup(func() {
593 //nolint:usetesting // required to avoid getting a canceled context at cleanup.
594 assert.NoError(t, tp.Shutdown(context.Background()))
595 })
596
597 tracer := tp.Tracer("tracer")
598
599 for i := range 10 {
600 _, span := tracer.Start(ctx, fmt.Sprintf("span%d", i))
601 span.End()
602
603 err := tp.ForceFlush(ctx)
604 assert.NoError(t, err)
605
606 assert.Len(t, bp.spans, i+1)
607 }
608 }
609
610 func TestBatchSpanProcessorConcurrentSafe(t *testing.T) {
611 ctx := t.Context()
612 var bp testBatchExporter
613 bsp := NewBatchSpanProcessor(&bp)
614 tp := basicTracerProvider(t)
615 tp.RegisterSpanProcessor(bsp)
616 tr := tp.Tracer(t.Name())
617
618 var wg sync.WaitGroup
619
620 wg.Go(func() {
621 generateSpan(t, tr, testOption{genNumSpans: 1})
622 })
623
624 wg.Go(func() {
625 _ = bsp.ForceFlush(ctx)
626 })
627
628 wg.Go(func() {
629 _ = bsp.Shutdown(ctx)
630 })
631
632 wg.Go(func() {
633 _ = tp.ForceFlush(ctx)
634 })
635
636 wg.Go(func() {
637 _ = tp.Shutdown(ctx)
638 })
639
640 wg.Wait()
641 }
642
643 // Drop metrics not being tested in this test.
644 var dropSpanMetricsView = sdkmetric.NewView(
645 sdkmetric.Instrument{
646 Name: "otel.sdk.span.*",
647 },
648 sdkmetric.Stream{Aggregation: sdkmetric.AggregationDrop{}},
649 )
650
651 func TestBatchSpanProcessorMetricsDisabled(t *testing.T) {
652 t.Setenv("OTEL_GO_X_OBSERVABILITY", "false")
653 tp := basicTracerProvider(t)
654 reader := sdkmetric.NewManualReader()
655 meterProvider := sdkmetric.NewMeterProvider(
656 sdkmetric.WithReader(reader),
657 sdkmetric.WithView(dropSpanMetricsView),
658 )
659 otel.SetMeterProvider(meterProvider)
660 me := newBlockingExporter()
661 t.Cleanup(func() {
662 //nolint:usetesting // required to avoid getting a canceled context at cleanup.
663 assert.NoError(t, me.Shutdown(context.Background()))
664 })
665 bsp := NewBatchSpanProcessor(
666 me,
667 // Make sure timeout doesn't trigger during the test.
668 WithBatchTimeout(time.Hour),
669 WithMaxQueueSize(2),
670 WithMaxExportBatchSize(2),
671 )
672 tp.RegisterSpanProcessor(bsp)
673
674 tr := tp.Tracer("TestBatchSpanProcessorMetricsDisabled")
675 // Generate 2 spans, which export and block during the export call.
676 generateSpan(t, tr, testOption{genNumSpans: 2})
677 ctx, cancel := context.WithTimeout(t.Context(), time.Second)
678 defer cancel()
679 assert.NoError(t, me.waitForSpans(ctx, 2))
680
681 // Validate that there are no metrics produced.
682 gotMetrics := new(metricdata.ResourceMetrics)
683 assert.NoError(t, reader.Collect(t.Context(), gotMetrics))
684 require.Empty(t, gotMetrics.ScopeMetrics)
685 // Generate 3 spans. 2 fill the queue, and 1 is dropped because the queue is full.
686 generateSpan(t, tr, testOption{genNumSpans: 3})
687 // Validate that there are no metrics produced.
688 gotMetrics = new(metricdata.ResourceMetrics)
689 assert.NoError(t, reader.Collect(t.Context(), gotMetrics))
690 require.Empty(t, gotMetrics.ScopeMetrics)
691 }
692
693 func TestBatchSpanProcessorMetrics(t *testing.T) {
694 // Reset for deterministic component ID.
695 processorIDCounter.Store(componentID)
696
697 t.Setenv("OTEL_GO_X_OBSERVABILITY", "true")
698 tp := basicTracerProvider(t)
699 reader := sdkmetric.NewManualReader()
700 meterProvider := sdkmetric.NewMeterProvider(
701 sdkmetric.WithReader(reader),
702 sdkmetric.WithView(dropSpanMetricsView),
703 )
704 otel.SetMeterProvider(meterProvider)
705 me := newBlockingExporter()
706 t.Cleanup(func() {
707 //nolint:usetesting // required to avoid getting a canceled context at cleanup.
708 assert.NoError(t, me.Shutdown(context.Background()))
709 })
710 bsp := NewBatchSpanProcessor(
711 me,
712 // Make sure timeout doesn't trigger during the test.
713 WithBatchTimeout(time.Hour),
714 WithMaxQueueSize(2),
715 WithMaxExportBatchSize(2),
716 )
717 tp.RegisterSpanProcessor(bsp)
718
719 tr := tp.Tracer("TestBatchSpanProcessorMetrics")
720 // Generate 2 spans, which export and block during the export call.
721 generateSpan(t, tr, testOption{genNumSpans: 2})
722 ctx, cancel := context.WithTimeout(t.Context(), time.Second)
723 defer cancel()
724 assert.NoError(t, me.waitForSpans(ctx, 2))
725 assertObsScopeMetrics(t, reader, expectMetrics{
726 queueCapacity: 2,
727 queueSize: 0,
728 successProcessed: 2,
729 })
730 // Generate 3 spans. 2 fill the queue, and 1 is dropped because the queue is full.
731 generateSpan(t, tr, testOption{genNumSpans: 3})
732 assertObsScopeMetrics(t, reader, expectMetrics{
733 queueCapacity: 2,
734 queueSize: 2,
735 queueFullProcessed: 1,
736 successProcessed: 2,
737 })
738 }
739
740 func TestBatchSpanProcessorBlockingMetrics(t *testing.T) {
741 // Reset for deterministic component ID.
742 processorIDCounter.Store(componentID)
743
744 t.Setenv("OTEL_GO_X_OBSERVABILITY", "true")
745 tp := basicTracerProvider(t)
746 reader := sdkmetric.NewManualReader()
747 meterProvider := sdkmetric.NewMeterProvider(
748 sdkmetric.WithReader(reader),
749 sdkmetric.WithView(dropSpanMetricsView),
750 )
751 otel.SetMeterProvider(meterProvider)
752 me := newBlockingExporter()
753 t.Cleanup(func() {
754 //nolint:usetesting // required to avoid getting a canceled context at cleanup.
755 assert.NoError(t, me.Shutdown(context.Background()))
756 })
757 bsp := NewBatchSpanProcessor(
758 me,
759 // Use WithBlocking so we can trigger a queueFull using ForceFlush.
760 WithBlocking(),
761 // Make sure timeout doesn't trigger during the test.
762 WithBatchTimeout(time.Hour),
763 WithMaxQueueSize(2),
764 WithMaxExportBatchSize(2),
765 )
766 tp.RegisterSpanProcessor(bsp)
767
768 tr := tp.Tracer("TestBatchSpanProcessorBlockingMetrics")
769 // Generate 2 spans that are exported to the exporter, which blocks.
770 generateSpan(t, tr, testOption{genNumSpans: 2})
771 ctx, cancel := context.WithTimeout(t.Context(), time.Second)
772 defer cancel()
773 assert.NoError(t, me.waitForSpans(ctx, 2))
774 assertObsScopeMetrics(t, reader, expectMetrics{
775 queueCapacity: 2,
776 queueSize: 0,
777 successProcessed: 2,
778 })
779 // Generate 2 spans to fill the queue.
780 generateSpan(t, tr, testOption{genNumSpans: 2})
781 go func() {
782 // Generate a span which blocks because the queue is full.
783 generateSpan(t, tr, testOption{genNumSpans: 1})
784 }()
785 assertObsScopeMetrics(t, reader, expectMetrics{
786 queueCapacity: 2,
787 queueSize: 2,
788 successProcessed: 2,
789 })
790
791 // Use ForceFlush to force the span that is blocking on the full queue to be dropped.
792 ctx, cancel = context.WithTimeout(t.Context(), 10*time.Millisecond)
793 defer cancel()
794 assert.Error(t, tp.ForceFlush(ctx))
795 assertObsScopeMetrics(t, reader, expectMetrics{
796 queueCapacity: 2,
797 queueSize: 2,
798 queueFullProcessed: 1,
799 successProcessed: 2,
800 })
801 }
802
803 type expectMetrics struct {
804 queueCapacity int64
805 queueSize int64
806 successProcessed int64
807 queueFullProcessed int64
808 }
809
810 func assertObsScopeMetrics(
811 t *testing.T,
812 reader sdkmetric.Reader,
813 expectation expectMetrics,
814 ) {
815 t.Helper()
816 gotResourceMetrics := new(metricdata.ResourceMetrics)
817 assert.NoError(t, reader.Collect(t.Context(), gotResourceMetrics))
818
819 componentNameAttr := observ.BSPComponentName(componentID)
820 baseAttrs := attribute.NewSet(
821 semconv.OTelComponentTypeBatchingSpanProcessor,
822 componentNameAttr,
823 )
824 wantMetrics := []metricdata.Metrics{
825 {
826 Name: otelconv.SDKProcessorSpanQueueCapacity{}.Name(),
827 Description: otelconv.SDKProcessorSpanQueueCapacity{}.Description(),
828 Unit: otelconv.SDKProcessorSpanQueueCapacity{}.Unit(),
829 Data: metricdata.Sum[int64]{
830 DataPoints: []metricdata.DataPoint[int64]{{Attributes: baseAttrs, Value: expectation.queueCapacity}},
831 Temporality: metricdata.CumulativeTemporality,
832 IsMonotonic: false,
833 },
834 },
835 {
836 Name: otelconv.SDKProcessorSpanQueueSize{}.Name(),
837 Description: otelconv.SDKProcessorSpanQueueSize{}.Description(),
838 Unit: otelconv.SDKProcessorSpanQueueSize{}.Unit(),
839 Data: metricdata.Sum[int64]{
840 DataPoints: []metricdata.DataPoint[int64]{{Attributes: baseAttrs, Value: expectation.queueSize}},
841 Temporality: metricdata.CumulativeTemporality,
842 IsMonotonic: false,
843 },
844 },
845 }
846
847 wantProcessedDataPoints := []metricdata.DataPoint[int64]{}
848 if expectation.successProcessed > 0 {
849 wantProcessedDataPoints = append(wantProcessedDataPoints, metricdata.DataPoint[int64]{
850 Value: expectation.successProcessed,
851 Attributes: attribute.NewSet(
852 semconv.OTelComponentTypeBatchingSpanProcessor,
853 componentNameAttr,
854 ),
855 })
856 }
857 if expectation.queueFullProcessed > 0 {
858 wantProcessedDataPoints = append(wantProcessedDataPoints, metricdata.DataPoint[int64]{
859 Value: expectation.queueFullProcessed,
860 Attributes: attribute.NewSet(
861 semconv.OTelComponentTypeBatchingSpanProcessor,
862 componentNameAttr,
863 observ.ErrQueueFull,
864 ),
865 })
866 }
867
868 if len(wantProcessedDataPoints) > 0 {
869 wantMetrics = append(wantMetrics,
870 metricdata.Metrics{
871 Name: otelconv.SDKProcessorSpanProcessed{}.Name(),
872 Description: otelconv.SDKProcessorSpanProcessed{}.Description(),
873 Unit: otelconv.SDKProcessorSpanProcessed{}.Unit(),
874 Data: metricdata.Sum[int64]{
875 DataPoints: wantProcessedDataPoints,
876 Temporality: metricdata.CumulativeTemporality,
877 IsMonotonic: true,
878 },
879 },
880 )
881 }
882
883 wantScopeMetric := metricdata.ScopeMetrics{
884 Scope: instrumentation.Scope{
885 Name: observ.ScopeName,
886 Version: sdk.Version(),
887 SchemaURL: observ.SchemaURL,
888 },
889 Metrics: wantMetrics,
890 }
891 metricdatatest.AssertEqual(t, wantScopeMetric, gotResourceMetrics.ScopeMetrics[0], metricdatatest.IgnoreTimestamp())
892 }
893
894 // blockingExporter blocks until the exported span is removed from the channel.
895 type blockingExporter struct {
896 shutdown chan struct{}
897 total atomic.Int32
898 }
899
900 func newBlockingExporter() *blockingExporter {
901 e := &blockingExporter{shutdown: make(chan struct{})}
902 return e
903 }
904
905 func (e *blockingExporter) Shutdown(ctx context.Context) error {
906 select {
907 case <-e.shutdown:
908 default:
909 close(e.shutdown)
910 }
911 return ctx.Err()
912 }
913
914 func (e *blockingExporter) ExportSpans(ctx context.Context, s []ReadOnlySpan) error {
915 e.total.Add(int32(len(s)))
916 <-e.shutdown
917 return ctx.Err()
918 }
919
920 func (e *blockingExporter) waitForSpans(ctx context.Context, n int32) error {
921 // Wait for all n spans to reach the export call
922 for e.total.Load() < n {
923 select {
924 case <-ctx.Done():
925 return fmt.Errorf("timeout waiting for %d spans to be exported", n)
926 default:
927 // So the select will not block
928 }
929 runtime.Gosched()
930 }
931 return nil
932 }