Debian Codegraph @

sdk/trace/batch_span_processor.go

raw
1 // Copyright The OpenTelemetry Authors
2 // SPDX-License-Identifier: Apache-2.0
3
4 package trace // import "go.opentelemetry.io/otel/sdk/trace"
5
6 import (
7 "context"
8 "errors"
9 "sync"
10 "sync/atomic"
11 "time"
12
13 "go.opentelemetry.io/otel"
14 "go.opentelemetry.io/otel/internal/global"
15 "go.opentelemetry.io/otel/sdk/trace/internal/env"
16 "go.opentelemetry.io/otel/sdk/trace/internal/observ"
17 "go.opentelemetry.io/otel/trace"
18 )
19
20 // Defaults for BatchSpanProcessorOptions.
21 const (
22 DefaultMaxQueueSize = 2048
23 // DefaultScheduleDelay is the delay interval between two consecutive exports, in milliseconds.
24 DefaultScheduleDelay = 5000
25 // DefaultExportTimeout is the duration after which an export is cancelled, in milliseconds.
26 DefaultExportTimeout = 30000
27 DefaultMaxExportBatchSize = 512
28 )
29
30 // BatchSpanProcessorOption configures a BatchSpanProcessor.
31 type BatchSpanProcessorOption func(o *BatchSpanProcessorOptions)
32
33 // BatchSpanProcessorOptions is configuration settings for a
34 // BatchSpanProcessor.
35 type BatchSpanProcessorOptions struct {
36 // MaxQueueSize is the maximum queue size to buffer spans for delayed processing. If the
37 // queue gets full it drops the spans. Use BlockOnQueueFull to change this behavior.
38 // The default value of MaxQueueSize is 2048.
39 MaxQueueSize int
40
41 // BatchTimeout is the maximum duration for constructing a batch. Processor
42 // forcefully sends available spans when timeout is reached.
43 // The default value of BatchTimeout is 5000 msec.
44 BatchTimeout time.Duration
45
46 // ExportTimeout specifies the maximum duration for exporting spans. If the timeout
47 // is reached, the export will be cancelled.
48 // The default value of ExportTimeout is 30000 msec.
49 ExportTimeout time.Duration
50
51 // MaxExportBatchSize is the maximum number of spans to process in a single batch.
52 // If there are more than one batch worth of spans then it processes multiple batches
53 // of spans one batch after the other without any delay.
54 // The default value of MaxExportBatchSize is 512.
55 MaxExportBatchSize int
56
57 // BlockOnQueueFull blocks onEnd() and onStart() method if the queue is full
58 // AND if BlockOnQueueFull is set to true.
59 // Blocking option should be used carefully as it can severely affect the performance of an
60 // application.
61 BlockOnQueueFull bool
62 }
63
64 // batchSpanProcessor is a SpanProcessor that batches asynchronously-received
65 // spans and sends them to a trace.Exporter when complete.
66 type batchSpanProcessor struct {
67 e SpanExporter
68 o BatchSpanProcessorOptions
69
70 queue chan ReadOnlySpan
71 dropped atomic.Uint32
72
73 inst *observ.BSP
74
75 batch []ReadOnlySpan
76 batchMutex sync.Mutex
77 timer *time.Timer
78 stopWait sync.WaitGroup
79 stopOnce sync.Once
80 stopCh chan struct{}
81 stopped atomic.Bool
82 }
83
84 var _ SpanProcessor = (*batchSpanProcessor)(nil)
85
86 // NewBatchSpanProcessor creates a new SpanProcessor that will send completed
87 // span batches to the exporter with the supplied options.
88 //
89 // If the exporter is nil, the span processor will perform no action.
90 func NewBatchSpanProcessor(exporter SpanExporter, options ...BatchSpanProcessorOption) SpanProcessor {
91 maxQueueSize := env.BatchSpanProcessorMaxQueueSize(DefaultMaxQueueSize)
92 maxExportBatchSize := env.BatchSpanProcessorMaxExportBatchSize(DefaultMaxExportBatchSize)
93
94 if maxExportBatchSize > maxQueueSize {
95 maxExportBatchSize = min(DefaultMaxExportBatchSize, maxQueueSize)
96 }
97
98 o := BatchSpanProcessorOptions{
99 BatchTimeout: time.Duration(env.BatchSpanProcessorScheduleDelay(DefaultScheduleDelay)) * time.Millisecond,
100 ExportTimeout: time.Duration(env.BatchSpanProcessorExportTimeout(DefaultExportTimeout)) * time.Millisecond,
101 MaxQueueSize: maxQueueSize,
102 MaxExportBatchSize: maxExportBatchSize,
103 }
104 for _, opt := range options {
105 opt(&o)
106 }
107 bsp := &batchSpanProcessor{
108 e: exporter,
109 o: o,
110 batch: make([]ReadOnlySpan, 0, o.MaxExportBatchSize),
111 timer: time.NewTimer(o.BatchTimeout),
112 queue: make(chan ReadOnlySpan, o.MaxQueueSize),
113 stopCh: make(chan struct{}),
114 }
115
116 var err error
117 bsp.inst, err = observ.NewBSP(
118 nextProcessorID(),
119 func() int64 { return int64(len(bsp.queue)) },
120 int64(bsp.o.MaxQueueSize),
121 )
122 if err != nil {
123 otel.Handle(err)
124 }
125
126 bsp.stopWait.Go(func() {
127 bsp.processQueue()
128 bsp.drainQueue()
129 })
130
131 return bsp
132 }
133
134 var processorIDCounter atomic.Int64
135
136 // nextProcessorID returns an identifier for this batch span processor,
137 // starting with 0 and incrementing by 1 each time it is called.
138 func nextProcessorID() int64 {
139 return processorIDCounter.Add(1) - 1
140 }
141
142 // OnStart method does nothing.
143 func (*batchSpanProcessor) OnStart(context.Context, ReadWriteSpan) {}
144
145 // OnEnd method enqueues a ReadOnlySpan for later processing.
146 func (bsp *batchSpanProcessor) OnEnd(s ReadOnlySpan) {
147 // Do not enqueue spans after Shutdown.
148 if bsp.stopped.Load() {
149 return
150 }
151
152 // Do not enqueue spans if we are just going to drop them.
153 if bsp.e == nil {
154 return
155 }
156 bsp.enqueue(s)
157 }
158
159 // Shutdown flushes the queue and waits until all spans are processed.
160 // It only executes once. Subsequent call does nothing.
161 func (bsp *batchSpanProcessor) Shutdown(ctx context.Context) error {
162 var err error
163 bsp.stopOnce.Do(func() {
164 bsp.stopped.Store(true)
165 wait := make(chan struct{})
166 go func() {
167 close(bsp.stopCh)
168 bsp.stopWait.Wait()
169 if bsp.e != nil {
170 if err := bsp.e.Shutdown(ctx); err != nil {
171 otel.Handle(err)
172 }
173 }
174 close(wait)
175 }()
176 // Wait until the wait group is done or the context is cancelled
177 select {
178 case <-wait:
179 case <-ctx.Done():
180 err = ctx.Err()
181 }
182 if bsp.inst != nil {
183 err = errors.Join(err, bsp.inst.Shutdown())
184 }
185 })
186 return err
187 }
188
189 type forceFlushSpan struct {
190 ReadOnlySpan
191 flushed chan struct{}
192 }
193
194 func (forceFlushSpan) SpanContext() trace.SpanContext {
195 return trace.NewSpanContext(trace.SpanContextConfig{TraceFlags: trace.FlagsSampled})
196 }
197
198 // ForceFlush exports all ended spans that have not yet been exported.
199 func (bsp *batchSpanProcessor) ForceFlush(ctx context.Context) error {
200 // Interrupt if context is already canceled.
201 if err := ctx.Err(); err != nil {
202 return err
203 }
204
205 // Do nothing after Shutdown.
206 if bsp.stopped.Load() {
207 return nil
208 }
209
210 var err error
211 if bsp.e != nil {
212 flushCh := make(chan struct{})
213 if bsp.enqueueBlockOnQueueFull(ctx, forceFlushSpan{flushed: flushCh}) {
214 select {
215 case <-bsp.stopCh:
216 // The batchSpanProcessor is Shutdown.
217 return nil
218 case <-flushCh:
219 // Processed any items in queue prior to ForceFlush being called
220 case <-ctx.Done():
221 return ctx.Err()
222 }
223 }
224
225 wait := make(chan error, 1)
226 go func() {
227 wait <- bsp.exportSpans(ctx)
228 }()
229 // Wait until the export is finished or the context is cancelled/timed out
230 select {
231 case err = <-wait:
232 case <-ctx.Done():
233 err = ctx.Err()
234 }
235 }
236 return err
237 }
238
239 // WithMaxQueueSize returns a BatchSpanProcessorOption that configures the
240 // maximum queue size allowed for a BatchSpanProcessor.
241 func WithMaxQueueSize(size int) BatchSpanProcessorOption {
242 return func(o *BatchSpanProcessorOptions) {
243 o.MaxQueueSize = size
244 }
245 }
246
247 // WithMaxExportBatchSize returns a BatchSpanProcessorOption that configures
248 // the maximum export batch size allowed for a BatchSpanProcessor.
249 func WithMaxExportBatchSize(size int) BatchSpanProcessorOption {
250 return func(o *BatchSpanProcessorOptions) {
251 o.MaxExportBatchSize = size
252 }
253 }
254
255 // WithBatchTimeout returns a BatchSpanProcessorOption that configures the
256 // maximum delay allowed for a BatchSpanProcessor before it will export any
257 // held span (whether the queue is full or not).
258 func WithBatchTimeout(delay time.Duration) BatchSpanProcessorOption {
259 return func(o *BatchSpanProcessorOptions) {
260 o.BatchTimeout = delay
261 }
262 }
263
264 // WithExportTimeout returns a BatchSpanProcessorOption that configures the
265 // amount of time a BatchSpanProcessor waits for an exporter to export before
266 // abandoning the export.
267 func WithExportTimeout(timeout time.Duration) BatchSpanProcessorOption {
268 return func(o *BatchSpanProcessorOptions) {
269 o.ExportTimeout = timeout
270 }
271 }
272
273 // WithBlocking returns a BatchSpanProcessorOption that configures a
274 // BatchSpanProcessor to wait for enqueue operations to succeed instead of
275 // dropping data when the queue is full.
276 func WithBlocking() BatchSpanProcessorOption {
277 return func(o *BatchSpanProcessorOptions) {
278 o.BlockOnQueueFull = true
279 }
280 }
281
282 // exportSpans is a subroutine of processing and draining the queue.
283 func (bsp *batchSpanProcessor) exportSpans(ctx context.Context) error {
284 bsp.timer.Reset(bsp.o.BatchTimeout)
285
286 bsp.batchMutex.Lock()
287 defer bsp.batchMutex.Unlock()
288
289 if bsp.o.ExportTimeout > 0 {
290 var cancel context.CancelFunc
291 ctx, cancel = context.WithTimeoutCause(ctx, bsp.o.ExportTimeout, errors.New("processor export timeout"))
292 defer cancel()
293 }
294
295 if l := len(bsp.batch); l > 0 {
296 global.Debug("exporting spans", "count", len(bsp.batch), "total_dropped", bsp.dropped.Load())
297 if bsp.inst != nil {
298 bsp.inst.Processed(ctx, int64(l))
299 }
300 err := bsp.e.ExportSpans(ctx, bsp.batch)
301
302 // A new batch is always created after exporting, even if the batch failed to be exported.
303 //
304 // It is up to the exporter to implement any type of retry logic if a batch is failing
305 // to be exported, since it is specific to the protocol and backend being sent to.
306 clear(bsp.batch) // Erase elements to let GC collect objects
307 bsp.batch = bsp.batch[:0]
308
309 if err != nil {
310 return err
311 }
312 }
313 return nil
314 }
315
316 // processQueue removes spans from the `queue` channel until processor
317 // is shut down. It calls the exporter in batches of up to MaxExportBatchSize
318 // waiting up to BatchTimeout to form a batch.
319 func (bsp *batchSpanProcessor) processQueue() {
320 defer bsp.timer.Stop()
321
322 ctx, cancel := context.WithCancel(context.Background())
323 defer cancel()
324 for {
325 select {
326 case <-bsp.stopCh:
327 return
328 case <-bsp.timer.C:
329 if err := bsp.exportSpans(ctx); err != nil {
330 otel.Handle(err)
331 }
332 case sd := <-bsp.queue:
333 if ffs, ok := sd.(forceFlushSpan); ok {
334 close(ffs.flushed)
335 continue
336 }
337 bsp.batchMutex.Lock()
338 bsp.batch = append(bsp.batch, sd)
339 shouldExport := len(bsp.batch) >= bsp.o.MaxExportBatchSize
340 bsp.batchMutex.Unlock()
341 if shouldExport {
342 if !bsp.timer.Stop() {
343 // Handle both GODEBUG=asynctimerchan=[0|1] properly.
344 select {
345 case <-bsp.timer.C:
346 default:
347 }
348 }
349 if err := bsp.exportSpans(ctx); err != nil {
350 otel.Handle(err)
351 }
352 }
353 }
354 }
355 }
356
357 // drainQueue awaits the any caller that had added to bsp.stopWait
358 // to finish the enqueue, then exports the final batch.
359 func (bsp *batchSpanProcessor) drainQueue() {
360 ctx, cancel := context.WithCancel(context.Background())
361 defer cancel()
362 for {
363 select {
364 case sd := <-bsp.queue:
365 if _, ok := sd.(forceFlushSpan); ok {
366 // Ignore flush requests as they are not valid spans.
367 continue
368 }
369
370 bsp.batchMutex.Lock()
371 bsp.batch = append(bsp.batch, sd)
372 shouldExport := len(bsp.batch) == bsp.o.MaxExportBatchSize
373 bsp.batchMutex.Unlock()
374
375 if shouldExport {
376 if err := bsp.exportSpans(ctx); err != nil {
377 otel.Handle(err)
378 }
379 }
380 default:
381 // There are no more enqueued spans. Make final export.
382 if err := bsp.exportSpans(ctx); err != nil {
383 otel.Handle(err)
384 }
385 return
386 }
387 }
388 }
389
390 func (bsp *batchSpanProcessor) enqueue(sd ReadOnlySpan) {
391 ctx := context.TODO()
392 if bsp.o.BlockOnQueueFull {
393 bsp.enqueueBlockOnQueueFull(ctx, sd)
394 } else {
395 bsp.enqueueDrop(ctx, sd)
396 }
397 }
398
399 func (bsp *batchSpanProcessor) enqueueBlockOnQueueFull(ctx context.Context, sd ReadOnlySpan) bool {
400 if !sd.SpanContext().IsSampled() {
401 return false
402 }
403
404 select {
405 case bsp.queue <- sd:
406 return true
407 case <-ctx.Done():
408 if bsp.inst != nil {
409 bsp.inst.ProcessedQueueFull(ctx, 1)
410 }
411 return false
412 }
413 }
414
415 func (bsp *batchSpanProcessor) enqueueDrop(ctx context.Context, sd ReadOnlySpan) bool {
416 if !sd.SpanContext().IsSampled() {
417 return false
418 }
419
420 select {
421 case bsp.queue <- sd:
422 return true
423 default:
424 bsp.dropped.Add(1)
425 if bsp.inst != nil {
426 bsp.inst.ProcessedQueueFull(ctx, 1)
427 }
428 }
429 return false
430 }
431
432 // MarshalLog is the marshaling function used by the logging system to represent this Span Processor.
433 func (bsp *batchSpanProcessor) MarshalLog() any {
434 return struct {
435 Type string
436 SpanExporter SpanExporter
437 Config BatchSpanProcessorOptions
438 }{
439 Type: "BatchSpanProcessor",
440 SpanExporter: bsp.e,
441 Config: bsp.o,
442 }
443 }