| 1 |
// Copyright The OpenTelemetry Authors |
| 2 |
// SPDX-License-Identifier: Apache-2.0 |
| 3 |
|
| 4 |
package trace // import "go.opentelemetry.io/otel/sdk/trace" |
| 5 |
|
| 6 |
import ( |
| 7 |
"context" |
| 8 |
"errors" |
| 9 |
"sync" |
| 10 |
"sync/atomic" |
| 11 |
"time" |
| 12 |
|
| 13 |
"go.opentelemetry.io/otel" |
| 14 |
"go.opentelemetry.io/otel/internal/global" |
| 15 |
"go.opentelemetry.io/otel/sdk/trace/internal/env" |
| 16 |
"go.opentelemetry.io/otel/sdk/trace/internal/observ" |
| 17 |
"go.opentelemetry.io/otel/trace" |
| 18 |
) |
| 19 |
|
| 20 |
// Defaults for BatchSpanProcessorOptions. |
| 21 |
const ( |
| 22 |
DefaultMaxQueueSize = 2048 |
| 23 |
// DefaultScheduleDelay is the delay interval between two consecutive exports, in milliseconds. |
| 24 |
DefaultScheduleDelay = 5000 |
| 25 |
// DefaultExportTimeout is the duration after which an export is cancelled, in milliseconds. |
| 26 |
DefaultExportTimeout = 30000 |
| 27 |
DefaultMaxExportBatchSize = 512 |
| 28 |
) |
| 29 |
|
| 30 |
// BatchSpanProcessorOption configures a BatchSpanProcessor. |
| 31 |
type BatchSpanProcessorOption func(o *BatchSpanProcessorOptions) |
| 32 |
|
| 33 |
// BatchSpanProcessorOptions is configuration settings for a |
| 34 |
// BatchSpanProcessor. |
| 35 |
type BatchSpanProcessorOptions struct { |
| 36 |
// MaxQueueSize is the maximum queue size to buffer spans for delayed processing. If the |
| 37 |
// queue gets full it drops the spans. Use BlockOnQueueFull to change this behavior. |
| 38 |
// The default value of MaxQueueSize is 2048. |
| 39 |
MaxQueueSize int |
| 40 |
|
| 41 |
// BatchTimeout is the maximum duration for constructing a batch. Processor |
| 42 |
// forcefully sends available spans when timeout is reached. |
| 43 |
// The default value of BatchTimeout is 5000 msec. |
| 44 |
BatchTimeout time.Duration |
| 45 |
|
| 46 |
// ExportTimeout specifies the maximum duration for exporting spans. If the timeout |
| 47 |
// is reached, the export will be cancelled. |
| 48 |
// The default value of ExportTimeout is 30000 msec. |
| 49 |
ExportTimeout time.Duration |
| 50 |
|
| 51 |
// MaxExportBatchSize is the maximum number of spans to process in a single batch. |
| 52 |
// If there are more than one batch worth of spans then it processes multiple batches |
| 53 |
// of spans one batch after the other without any delay. |
| 54 |
// The default value of MaxExportBatchSize is 512. |
| 55 |
MaxExportBatchSize int |
| 56 |
|
| 57 |
// BlockOnQueueFull blocks onEnd() and onStart() method if the queue is full |
| 58 |
// AND if BlockOnQueueFull is set to true. |
| 59 |
// Blocking option should be used carefully as it can severely affect the performance of an |
| 60 |
// application. |
| 61 |
BlockOnQueueFull bool |
| 62 |
} |
| 63 |
|
| 64 |
// batchSpanProcessor is a SpanProcessor that batches asynchronously-received |
| 65 |
// spans and sends them to a trace.Exporter when complete. |
| 66 |
type batchSpanProcessor struct { |
| 67 |
e SpanExporter |
| 68 |
o BatchSpanProcessorOptions |
| 69 |
|
| 70 |
queue chan ReadOnlySpan |
| 71 |
dropped atomic.Uint32 |
| 72 |
|
| 73 |
inst *observ.BSP |
| 74 |
|
| 75 |
batch []ReadOnlySpan |
| 76 |
batchMutex sync.Mutex |
| 77 |
timer *time.Timer |
| 78 |
stopWait sync.WaitGroup |
| 79 |
stopOnce sync.Once |
| 80 |
stopCh chan struct{} |
| 81 |
stopped atomic.Bool |
| 82 |
} |
| 83 |
|
| 84 |
var _ SpanProcessor = (*batchSpanProcessor)(nil) |
| 85 |
|
| 86 |
// NewBatchSpanProcessor creates a new SpanProcessor that will send completed |
| 87 |
// span batches to the exporter with the supplied options. |
| 88 |
// |
| 89 |
// If the exporter is nil, the span processor will perform no action. |
| 90 |
func NewBatchSpanProcessor(exporter SpanExporter, options ...BatchSpanProcessorOption) SpanProcessor { |
| 91 |
maxQueueSize := env.BatchSpanProcessorMaxQueueSize(DefaultMaxQueueSize) |
| 92 |
maxExportBatchSize := env.BatchSpanProcessorMaxExportBatchSize(DefaultMaxExportBatchSize) |
| 93 |
|
| 94 |
if maxExportBatchSize > maxQueueSize { |
| 95 |
maxExportBatchSize = min(DefaultMaxExportBatchSize, maxQueueSize) |
| 96 |
} |
| 97 |
|
| 98 |
o := BatchSpanProcessorOptions{ |
| 99 |
BatchTimeout: time.Duration(env.BatchSpanProcessorScheduleDelay(DefaultScheduleDelay)) * time.Millisecond, |
| 100 |
ExportTimeout: time.Duration(env.BatchSpanProcessorExportTimeout(DefaultExportTimeout)) * time.Millisecond, |
| 101 |
MaxQueueSize: maxQueueSize, |
| 102 |
MaxExportBatchSize: maxExportBatchSize, |
| 103 |
} |
| 104 |
for _, opt := range options { |
| 105 |
opt(&o) |
| 106 |
} |
| 107 |
bsp := &batchSpanProcessor{ |
| 108 |
e: exporter, |
| 109 |
o: o, |
| 110 |
batch: make([]ReadOnlySpan, 0, o.MaxExportBatchSize), |
| 111 |
timer: time.NewTimer(o.BatchTimeout), |
| 112 |
queue: make(chan ReadOnlySpan, o.MaxQueueSize), |
| 113 |
stopCh: make(chan struct{}), |
| 114 |
} |
| 115 |
|
| 116 |
var err error |
| 117 |
bsp.inst, err = observ.NewBSP( |
| 118 |
nextProcessorID(), |
| 119 |
func() int64 { return int64(len(bsp.queue)) }, |
| 120 |
int64(bsp.o.MaxQueueSize), |
| 121 |
) |
| 122 |
if err != nil { |
| 123 |
otel.Handle(err) |
| 124 |
} |
| 125 |
|
| 126 |
bsp.stopWait.Go(func() { |
| 127 |
bsp.processQueue() |
| 128 |
bsp.drainQueue() |
| 129 |
}) |
| 130 |
|
| 131 |
return bsp |
| 132 |
} |
| 133 |
|
| 134 |
var processorIDCounter atomic.Int64 |
| 135 |
|
| 136 |
// nextProcessorID returns an identifier for this batch span processor, |
| 137 |
// starting with 0 and incrementing by 1 each time it is called. |
| 138 |
func nextProcessorID() int64 { |
| 139 |
return processorIDCounter.Add(1) - 1 |
| 140 |
} |
| 141 |
|
| 142 |
// OnStart method does nothing. |
| 143 |
func (*batchSpanProcessor) OnStart(context.Context, ReadWriteSpan) {} |
| 144 |
|
| 145 |
// OnEnd method enqueues a ReadOnlySpan for later processing. |
| 146 |
func (bsp *batchSpanProcessor) OnEnd(s ReadOnlySpan) { |
| 147 |
// Do not enqueue spans after Shutdown. |
| 148 |
if bsp.stopped.Load() { |
| 149 |
return |
| 150 |
} |
| 151 |
|
| 152 |
// Do not enqueue spans if we are just going to drop them. |
| 153 |
if bsp.e == nil { |
| 154 |
return |
| 155 |
} |
| 156 |
bsp.enqueue(s) |
| 157 |
} |
| 158 |
|
| 159 |
// Shutdown flushes the queue and waits until all spans are processed. |
| 160 |
// It only executes once. Subsequent call does nothing. |
| 161 |
func (bsp *batchSpanProcessor) Shutdown(ctx context.Context) error { |
| 162 |
var err error |
| 163 |
bsp.stopOnce.Do(func() { |
| 164 |
bsp.stopped.Store(true) |
| 165 |
wait := make(chan struct{}) |
| 166 |
go func() { |
| 167 |
close(bsp.stopCh) |
| 168 |
bsp.stopWait.Wait() |
| 169 |
if bsp.e != nil { |
| 170 |
if err := bsp.e.Shutdown(ctx); err != nil { |
| 171 |
otel.Handle(err) |
| 172 |
} |
| 173 |
} |
| 174 |
close(wait) |
| 175 |
}() |
| 176 |
// Wait until the wait group is done or the context is cancelled |
| 177 |
select { |
| 178 |
case <-wait: |
| 179 |
case <-ctx.Done(): |
| 180 |
err = ctx.Err() |
| 181 |
} |
| 182 |
if bsp.inst != nil { |
| 183 |
err = errors.Join(err, bsp.inst.Shutdown()) |
| 184 |
} |
| 185 |
}) |
| 186 |
return err |
| 187 |
} |
| 188 |
|
| 189 |
type forceFlushSpan struct { |
| 190 |
ReadOnlySpan |
| 191 |
flushed chan struct{} |
| 192 |
} |
| 193 |
|
| 194 |
func (forceFlushSpan) SpanContext() trace.SpanContext { |
| 195 |
return trace.NewSpanContext(trace.SpanContextConfig{TraceFlags: trace.FlagsSampled}) |
| 196 |
} |
| 197 |
|
| 198 |
// ForceFlush exports all ended spans that have not yet been exported. |
| 199 |
func (bsp *batchSpanProcessor) ForceFlush(ctx context.Context) error { |
| 200 |
// Interrupt if context is already canceled. |
| 201 |
if err := ctx.Err(); err != nil { |
| 202 |
return err |
| 203 |
} |
| 204 |
|
| 205 |
// Do nothing after Shutdown. |
| 206 |
if bsp.stopped.Load() { |
| 207 |
return nil |
| 208 |
} |
| 209 |
|
| 210 |
var err error |
| 211 |
if bsp.e != nil { |
| 212 |
flushCh := make(chan struct{}) |
| 213 |
if bsp.enqueueBlockOnQueueFull(ctx, forceFlushSpan{flushed: flushCh}) { |
| 214 |
select { |
| 215 |
case <-bsp.stopCh: |
| 216 |
// The batchSpanProcessor is Shutdown. |
| 217 |
return nil |
| 218 |
case <-flushCh: |
| 219 |
// Processed any items in queue prior to ForceFlush being called |
| 220 |
case <-ctx.Done(): |
| 221 |
return ctx.Err() |
| 222 |
} |
| 223 |
} |
| 224 |
|
| 225 |
wait := make(chan error, 1) |
| 226 |
go func() { |
| 227 |
wait <- bsp.exportSpans(ctx) |
| 228 |
}() |
| 229 |
// Wait until the export is finished or the context is cancelled/timed out |
| 230 |
select { |
| 231 |
case err = <-wait: |
| 232 |
case <-ctx.Done(): |
| 233 |
err = ctx.Err() |
| 234 |
} |
| 235 |
} |
| 236 |
return err |
| 237 |
} |
| 238 |
|
| 239 |
// WithMaxQueueSize returns a BatchSpanProcessorOption that configures the |
| 240 |
// maximum queue size allowed for a BatchSpanProcessor. |
| 241 |
func WithMaxQueueSize(size int) BatchSpanProcessorOption { |
| 242 |
return func(o *BatchSpanProcessorOptions) { |
| 243 |
o.MaxQueueSize = size |
| 244 |
} |
| 245 |
} |
| 246 |
|
| 247 |
// WithMaxExportBatchSize returns a BatchSpanProcessorOption that configures |
| 248 |
// the maximum export batch size allowed for a BatchSpanProcessor. |
| 249 |
func WithMaxExportBatchSize(size int) BatchSpanProcessorOption { |
| 250 |
return func(o *BatchSpanProcessorOptions) { |
| 251 |
o.MaxExportBatchSize = size |
| 252 |
} |
| 253 |
} |
| 254 |
|
| 255 |
// WithBatchTimeout returns a BatchSpanProcessorOption that configures the |
| 256 |
// maximum delay allowed for a BatchSpanProcessor before it will export any |
| 257 |
// held span (whether the queue is full or not). |
| 258 |
func WithBatchTimeout(delay time.Duration) BatchSpanProcessorOption { |
| 259 |
return func(o *BatchSpanProcessorOptions) { |
| 260 |
o.BatchTimeout = delay |
| 261 |
} |
| 262 |
} |
| 263 |
|
| 264 |
// WithExportTimeout returns a BatchSpanProcessorOption that configures the |
| 265 |
// amount of time a BatchSpanProcessor waits for an exporter to export before |
| 266 |
// abandoning the export. |
| 267 |
func WithExportTimeout(timeout time.Duration) BatchSpanProcessorOption { |
| 268 |
return func(o *BatchSpanProcessorOptions) { |
| 269 |
o.ExportTimeout = timeout |
| 270 |
} |
| 271 |
} |
| 272 |
|
| 273 |
// WithBlocking returns a BatchSpanProcessorOption that configures a |
| 274 |
// BatchSpanProcessor to wait for enqueue operations to succeed instead of |
| 275 |
// dropping data when the queue is full. |
| 276 |
func WithBlocking() BatchSpanProcessorOption { |
| 277 |
return func(o *BatchSpanProcessorOptions) { |
| 278 |
o.BlockOnQueueFull = true |
| 279 |
} |
| 280 |
} |
| 281 |
|
| 282 |
// exportSpans is a subroutine of processing and draining the queue. |
| 283 |
func (bsp *batchSpanProcessor) exportSpans(ctx context.Context) error { |
| 284 |
bsp.timer.Reset(bsp.o.BatchTimeout) |
| 285 |
|
| 286 |
bsp.batchMutex.Lock() |
| 287 |
defer bsp.batchMutex.Unlock() |
| 288 |
|
| 289 |
if bsp.o.ExportTimeout > 0 { |
| 290 |
var cancel context.CancelFunc |
| 291 |
ctx, cancel = context.WithTimeoutCause(ctx, bsp.o.ExportTimeout, errors.New("processor export timeout")) |
| 292 |
defer cancel() |
| 293 |
} |
| 294 |
|
| 295 |
if l := len(bsp.batch); l > 0 { |
| 296 |
global.Debug("exporting spans", "count", len(bsp.batch), "total_dropped", bsp.dropped.Load()) |
| 297 |
if bsp.inst != nil { |
| 298 |
bsp.inst.Processed(ctx, int64(l)) |
| 299 |
} |
| 300 |
err := bsp.e.ExportSpans(ctx, bsp.batch) |
| 301 |
|
| 302 |
// A new batch is always created after exporting, even if the batch failed to be exported. |
| 303 |
// |
| 304 |
// It is up to the exporter to implement any type of retry logic if a batch is failing |
| 305 |
// to be exported, since it is specific to the protocol and backend being sent to. |
| 306 |
clear(bsp.batch) // Erase elements to let GC collect objects |
| 307 |
bsp.batch = bsp.batch[:0] |
| 308 |
|
| 309 |
if err != nil { |
| 310 |
return err |
| 311 |
} |
| 312 |
} |
| 313 |
return nil |
| 314 |
} |
| 315 |
|
| 316 |
// processQueue removes spans from the `queue` channel until processor |
| 317 |
// is shut down. It calls the exporter in batches of up to MaxExportBatchSize |
| 318 |
// waiting up to BatchTimeout to form a batch. |
| 319 |
func (bsp *batchSpanProcessor) processQueue() { |
| 320 |
defer bsp.timer.Stop() |
| 321 |
|
| 322 |
ctx, cancel := context.WithCancel(context.Background()) |
| 323 |
defer cancel() |
| 324 |
for { |
| 325 |
select { |
| 326 |
case <-bsp.stopCh: |
| 327 |
return |
| 328 |
case <-bsp.timer.C: |
| 329 |
if err := bsp.exportSpans(ctx); err != nil { |
| 330 |
otel.Handle(err) |
| 331 |
} |
| 332 |
case sd := <-bsp.queue: |
| 333 |
if ffs, ok := sd.(forceFlushSpan); ok { |
| 334 |
close(ffs.flushed) |
| 335 |
continue |
| 336 |
} |
| 337 |
bsp.batchMutex.Lock() |
| 338 |
bsp.batch = append(bsp.batch, sd) |
| 339 |
shouldExport := len(bsp.batch) >= bsp.o.MaxExportBatchSize |
| 340 |
bsp.batchMutex.Unlock() |
| 341 |
if shouldExport { |
| 342 |
if !bsp.timer.Stop() { |
| 343 |
// Handle both GODEBUG=asynctimerchan=[0|1] properly. |
| 344 |
select { |
| 345 |
case <-bsp.timer.C: |
| 346 |
default: |
| 347 |
} |
| 348 |
} |
| 349 |
if err := bsp.exportSpans(ctx); err != nil { |
| 350 |
otel.Handle(err) |
| 351 |
} |
| 352 |
} |
| 353 |
} |
| 354 |
} |
| 355 |
} |
| 356 |
|
| 357 |
// drainQueue awaits the any caller that had added to bsp.stopWait |
| 358 |
// to finish the enqueue, then exports the final batch. |
| 359 |
func (bsp *batchSpanProcessor) drainQueue() { |
| 360 |
ctx, cancel := context.WithCancel(context.Background()) |
| 361 |
defer cancel() |
| 362 |
for { |
| 363 |
select { |
| 364 |
case sd := <-bsp.queue: |
| 365 |
if _, ok := sd.(forceFlushSpan); ok { |
| 366 |
// Ignore flush requests as they are not valid spans. |
| 367 |
continue |
| 368 |
} |
| 369 |
|
| 370 |
bsp.batchMutex.Lock() |
| 371 |
bsp.batch = append(bsp.batch, sd) |
| 372 |
shouldExport := len(bsp.batch) == bsp.o.MaxExportBatchSize |
| 373 |
bsp.batchMutex.Unlock() |
| 374 |
|
| 375 |
if shouldExport { |
| 376 |
if err := bsp.exportSpans(ctx); err != nil { |
| 377 |
otel.Handle(err) |
| 378 |
} |
| 379 |
} |
| 380 |
default: |
| 381 |
// There are no more enqueued spans. Make final export. |
| 382 |
if err := bsp.exportSpans(ctx); err != nil { |
| 383 |
otel.Handle(err) |
| 384 |
} |
| 385 |
return |
| 386 |
} |
| 387 |
} |
| 388 |
} |
| 389 |
|
| 390 |
func (bsp *batchSpanProcessor) enqueue(sd ReadOnlySpan) { |
| 391 |
ctx := context.TODO() |
| 392 |
if bsp.o.BlockOnQueueFull { |
| 393 |
bsp.enqueueBlockOnQueueFull(ctx, sd) |
| 394 |
} else { |
| 395 |
bsp.enqueueDrop(ctx, sd) |
| 396 |
} |
| 397 |
} |
| 398 |
|
| 399 |
func (bsp *batchSpanProcessor) enqueueBlockOnQueueFull(ctx context.Context, sd ReadOnlySpan) bool { |
| 400 |
if !sd.SpanContext().IsSampled() { |
| 401 |
return false |
| 402 |
} |
| 403 |
|
| 404 |
select { |
| 405 |
case bsp.queue <- sd: |
| 406 |
return true |
| 407 |
case <-ctx.Done(): |
| 408 |
if bsp.inst != nil { |
| 409 |
bsp.inst.ProcessedQueueFull(ctx, 1) |
| 410 |
} |
| 411 |
return false |
| 412 |
} |
| 413 |
} |
| 414 |
|
| 415 |
func (bsp *batchSpanProcessor) enqueueDrop(ctx context.Context, sd ReadOnlySpan) bool { |
| 416 |
if !sd.SpanContext().IsSampled() { |
| 417 |
return false |
| 418 |
} |
| 419 |
|
| 420 |
select { |
| 421 |
case bsp.queue <- sd: |
| 422 |
return true |
| 423 |
default: |
| 424 |
bsp.dropped.Add(1) |
| 425 |
if bsp.inst != nil { |
| 426 |
bsp.inst.ProcessedQueueFull(ctx, 1) |
| 427 |
} |
| 428 |
} |
| 429 |
return false |
| 430 |
} |
| 431 |
|
| 432 |
// MarshalLog is the marshaling function used by the logging system to represent this Span Processor. |
| 433 |
func (bsp *batchSpanProcessor) MarshalLog() any { |
| 434 |
return struct { |
| 435 |
Type string |
| 436 |
SpanExporter SpanExporter |
| 437 |
Config BatchSpanProcessorOptions |
| 438 |
}{ |
| 439 |
Type: "BatchSpanProcessor", |
| 440 |
SpanExporter: bsp.e, |
| 441 |
Config: bsp.o, |
| 442 |
} |
| 443 |
} |