| 1 |
// Copyright The OpenTelemetry Authors |
| 2 |
// SPDX-License-Identifier: Apache-2.0 |
| 3 |
|
| 4 |
package log // import "go.opentelemetry.io/otel/sdk/log" |
| 5 |
|
| 6 |
import ( |
| 7 |
"context" |
| 8 |
"errors" |
| 9 |
"slices" |
| 10 |
"sync" |
| 11 |
"sync/atomic" |
| 12 |
"time" |
| 13 |
|
| 14 |
"go.opentelemetry.io/otel/internal/global" |
| 15 |
) |
| 16 |
|
| 17 |
const ( |
| 18 |
dfltMaxQSize = 2048 |
| 19 |
dfltExpInterval = time.Second |
| 20 |
dfltExpTimeout = 30 * time.Second |
| 21 |
dfltExpMaxBatchSize = 512 |
| 22 |
dfltExpBufferSize = 1 |
| 23 |
|
| 24 |
envarMaxQSize = "OTEL_BLRP_MAX_QUEUE_SIZE" |
| 25 |
envarExpInterval = "OTEL_BLRP_SCHEDULE_DELAY" |
| 26 |
envarExpTimeout = "OTEL_BLRP_EXPORT_TIMEOUT" |
| 27 |
envarExpMaxBatchSize = "OTEL_BLRP_MAX_EXPORT_BATCH_SIZE" |
| 28 |
) |
| 29 |
|
| 30 |
// Compile-time check BatchProcessor implements Processor. |
| 31 |
var _ Processor = (*BatchProcessor)(nil) |
| 32 |
|
| 33 |
// BatchProcessor is a processor that exports batches of log records. |
| 34 |
// |
| 35 |
// Use [NewBatchProcessor] to create a BatchProcessor. An empty BatchProcessor |
| 36 |
// is shut down by default, no records will be batched or exported. |
| 37 |
type BatchProcessor struct { |
| 38 |
// The BatchProcessor is designed to provide the highest throughput of |
| 39 |
// log records possible while being compatible with OpenTelemetry. The |
| 40 |
// entry point of log records is the OnEmit method. This method is designed |
| 41 |
// to receive records as fast as possible while still honoring shutdown |
| 42 |
// commands. All records received are enqueued to queue. |
| 43 |
// |
| 44 |
// In order to block OnEmit as little as possible, a separate "poll" |
| 45 |
// goroutine is spawned at the creation of a BatchProcessor. This |
| 46 |
// goroutine is responsible for batching the queue at regular polled |
| 47 |
// intervals, or when it is directly signaled to. |
| 48 |
// |
| 49 |
// To keep the polling goroutine from backing up, all batches it makes are |
| 50 |
// exported with a bufferedExporter. This exporter allows the poll |
| 51 |
// goroutine to enqueue an export payload that will be handled in a |
| 52 |
// separate goroutine dedicated to the export. This asynchronous behavior |
| 53 |
// allows the poll goroutine to maintain accurate interval polling. |
| 54 |
// |
| 55 |
// ___BatchProcessor____ __Poll Goroutine__ __Export Goroutine__ |
| 56 |
// || || || || || || |
| 57 |
// || ********** || || || || ********** || |
| 58 |
// || Records=>* OnEmit * || || | - ticker || || * export * || |
| 59 |
// || ********** || || | - trigger || || ********** || |
| 60 |
// || || || || | || || || || |
| 61 |
// || || || || | || || || || |
| 62 |
// || __________\/___ || || |*********** || || ______/\_______ || |
| 63 |
// || (____queue______)>=||=||===|* batch *===||=||=>[_export_buffer_] || |
| 64 |
// || || || |*********** || || || |
| 65 |
// ||_____________________|| ||__________________|| ||____________________|| |
| 66 |
// |
| 67 |
// |
| 68 |
// The "release valve" in this processing is the record queue. This queue |
| 69 |
// is a ring buffer. It will overwrite the oldest records first when writes |
| 70 |
// to OnEmit are made faster than the queue can be flushed. If batches |
| 71 |
// cannot be flushed to the export buffer, the records will remain in the |
| 72 |
// queue. |
| 73 |
|
| 74 |
// exporter is the bufferedExporter all batches are exported with. |
| 75 |
exporter *bufferExporter |
| 76 |
|
| 77 |
// q is the active queue of records that have not yet been exported. |
| 78 |
q *queue |
| 79 |
// batchSize is the minimum number of records needed before an export is |
| 80 |
// triggered (unless the interval expires). |
| 81 |
batchSize int |
| 82 |
|
| 83 |
// pollTrigger triggers the poll goroutine to flush a batch from the queue. |
| 84 |
// This is sent to when it is known that the queue contains at least one |
| 85 |
// complete batch. |
| 86 |
// |
| 87 |
// When a send is made to the channel, the poll loop will be reset after |
| 88 |
// the flush. If there is still enough records in the queue for another |
| 89 |
// batch the reset of the poll loop will automatically re-trigger itself. |
| 90 |
// There is no need for the original sender to monitor and resend. |
| 91 |
pollTrigger chan struct{} |
| 92 |
// pollKill kills the poll goroutine. This is only expected to be closed |
| 93 |
// once by the Shutdown method. |
| 94 |
pollKill chan struct{} |
| 95 |
// pollDone signals the poll goroutine has completed. |
| 96 |
pollDone chan struct{} |
| 97 |
|
| 98 |
// stopped holds the stopped state of the BatchProcessor. |
| 99 |
stopped atomic.Bool |
| 100 |
|
| 101 |
noCmp [0]func() //nolint: unused // This is indeed used. |
| 102 |
} |
| 103 |
|
| 104 |
// NewBatchProcessor decorates the provided exporter |
| 105 |
// so that the log records are batched before exporting. |
| 106 |
// |
| 107 |
// All of the exporter's methods are called synchronously. |
| 108 |
func NewBatchProcessor(exporter Exporter, opts ...BatchProcessorOption) *BatchProcessor { |
| 109 |
cfg := newBatchConfig(opts) |
| 110 |
if exporter == nil { |
| 111 |
// Do not panic on nil export. |
| 112 |
exporter = defaultNoopExporter |
| 113 |
} |
| 114 |
// Order is important here. Wrap the timeoutExporter with the chunkExporter |
| 115 |
// to ensure each export completes in timeout (instead of all chunked |
| 116 |
// exports). |
| 117 |
exporter = newTimeoutExporter(exporter, cfg.expTimeout.Value) |
| 118 |
// Use a chunkExporter to ensure ForceFlush and Shutdown calls are batched |
| 119 |
// appropriately on export. |
| 120 |
exporter = newChunkExporter(exporter, cfg.expMaxBatchSize.Value) |
| 121 |
|
| 122 |
b := &BatchProcessor{ |
| 123 |
exporter: newBufferExporter(exporter, cfg.expBufferSize.Value), |
| 124 |
|
| 125 |
q: newQueue(cfg.maxQSize.Value), |
| 126 |
batchSize: cfg.expMaxBatchSize.Value, |
| 127 |
pollTrigger: make(chan struct{}, 1), |
| 128 |
pollKill: make(chan struct{}), |
| 129 |
} |
| 130 |
b.pollDone = b.poll(cfg.expInterval.Value) |
| 131 |
return b |
| 132 |
} |
| 133 |
|
| 134 |
// poll spawns a goroutine to handle interval polling and batch exporting. The |
| 135 |
// returned done chan is closed when the spawned goroutine completes. |
| 136 |
func (b *BatchProcessor) poll(interval time.Duration) (done chan struct{}) { |
| 137 |
done = make(chan struct{}) |
| 138 |
|
| 139 |
ticker := time.NewTicker(interval) |
| 140 |
// TODO: investigate using a sync.Pool instead of cloning. |
| 141 |
buf := make([]Record, b.batchSize) |
| 142 |
go func() { |
| 143 |
defer close(done) |
| 144 |
defer ticker.Stop() |
| 145 |
|
| 146 |
for { |
| 147 |
select { |
| 148 |
case <-ticker.C: |
| 149 |
case <-b.pollTrigger: |
| 150 |
ticker.Reset(interval) |
| 151 |
case <-b.pollKill: |
| 152 |
return |
| 153 |
} |
| 154 |
|
| 155 |
if d := b.q.Dropped(); d > 0 { |
| 156 |
global.Warn("dropped log records", "dropped", d) |
| 157 |
} |
| 158 |
|
| 159 |
var qLen int |
| 160 |
// Don't copy data from queue unless exporter can accept more, it is very expensive. |
| 161 |
if b.exporter.Ready() { |
| 162 |
qLen = b.q.TryDequeue(buf, func(r []Record) bool { |
| 163 |
ok := b.exporter.EnqueueExport(r) |
| 164 |
if ok { |
| 165 |
buf = slices.Clone(buf) |
| 166 |
} |
| 167 |
return ok |
| 168 |
}) |
| 169 |
} else { |
| 170 |
qLen = b.q.Len() |
| 171 |
} |
| 172 |
|
| 173 |
if qLen >= b.batchSize { |
| 174 |
// There is another full batch ready. Immediately trigger |
| 175 |
// another export attempt. |
| 176 |
select { |
| 177 |
case b.pollTrigger <- struct{}{}: |
| 178 |
default: |
| 179 |
// Another flush signal already received. |
| 180 |
} |
| 181 |
} |
| 182 |
} |
| 183 |
}() |
| 184 |
return done |
| 185 |
} |
| 186 |
|
| 187 |
// Enabled returns true, indicating this Processor will process all records. |
| 188 |
func (*BatchProcessor) Enabled(context.Context, EnabledParameters) bool { |
| 189 |
return true |
| 190 |
} |
| 191 |
|
| 192 |
// OnEmit batches provided log record. |
| 193 |
func (b *BatchProcessor) OnEmit(_ context.Context, r *Record) error { |
| 194 |
if b.stopped.Load() || b.q == nil { |
| 195 |
return nil |
| 196 |
} |
| 197 |
// The record is cloned so that changes done by subsequent processors |
| 198 |
// are not going to lead to a data race. |
| 199 |
if n := b.q.Enqueue(r.Clone()); n >= b.batchSize { |
| 200 |
select { |
| 201 |
case b.pollTrigger <- struct{}{}: |
| 202 |
default: |
| 203 |
// Flush chan full. The poll goroutine will handle this by |
| 204 |
// re-sending any trigger until the queue has less than batchSize |
| 205 |
// records. |
| 206 |
} |
| 207 |
} |
| 208 |
return nil |
| 209 |
} |
| 210 |
|
| 211 |
// Shutdown flushes queued log records and shuts down the decorated exporter. |
| 212 |
func (b *BatchProcessor) Shutdown(ctx context.Context) error { |
| 213 |
if b.stopped.Swap(true) || b.q == nil { |
| 214 |
return nil |
| 215 |
} |
| 216 |
|
| 217 |
// Stop the poll goroutine. |
| 218 |
close(b.pollKill) |
| 219 |
select { |
| 220 |
case <-b.pollDone: |
| 221 |
case <-ctx.Done(): |
| 222 |
// Out of time. |
| 223 |
return errors.Join(ctx.Err(), b.exporter.Shutdown(ctx)) |
| 224 |
} |
| 225 |
|
| 226 |
// Flush remaining queued before exporter shutdown. |
| 227 |
err := b.exporter.Export(ctx, b.q.Flush()) |
| 228 |
return errors.Join(err, b.exporter.Shutdown(ctx)) |
| 229 |
} |
| 230 |
|
| 231 |
var errPartialFlush = errors.New("partial flush: export buffer full") |
| 232 |
|
| 233 |
// Used for testing. |
| 234 |
var ctxErr = func(ctx context.Context) error { |
| 235 |
return ctx.Err() |
| 236 |
} |
| 237 |
|
| 238 |
// ForceFlush flushes queued log records and flushes the decorated exporter. |
| 239 |
func (b *BatchProcessor) ForceFlush(ctx context.Context) error { |
| 240 |
if b.stopped.Load() || b.q == nil { |
| 241 |
return nil |
| 242 |
} |
| 243 |
|
| 244 |
buf := make([]Record, b.q.cap) |
| 245 |
notFlushed := func() bool { |
| 246 |
var flushed bool |
| 247 |
_ = b.q.TryDequeue(buf, func(r []Record) bool { |
| 248 |
flushed = b.exporter.EnqueueExport(r) |
| 249 |
return flushed |
| 250 |
}) |
| 251 |
return !flushed |
| 252 |
} |
| 253 |
var err error |
| 254 |
// For as long as ctx allows, try to make a single flush of the queue. |
| 255 |
for notFlushed() { |
| 256 |
// Use ctxErr instead of calling ctx.Err directly so we can test |
| 257 |
// the partial error return. |
| 258 |
if e := ctxErr(ctx); e != nil { |
| 259 |
err = errors.Join(e, errPartialFlush) |
| 260 |
break |
| 261 |
} |
| 262 |
} |
| 263 |
return errors.Join(err, b.exporter.ForceFlush(ctx)) |
| 264 |
} |
| 265 |
|
| 266 |
// queue holds a queue of logging records. |
| 267 |
// |
| 268 |
// When the queue becomes full, the oldest records in the queue are |
| 269 |
// overwritten. |
| 270 |
type queue struct { |
| 271 |
sync.Mutex |
| 272 |
|
| 273 |
dropped atomic.Uint64 |
| 274 |
cap, len int |
| 275 |
read, write *ring |
| 276 |
} |
| 277 |
|
| 278 |
func newQueue(size int) *queue { |
| 279 |
r := newRing(size) |
| 280 |
return &queue{ |
| 281 |
cap: size, |
| 282 |
read: r, |
| 283 |
write: r, |
| 284 |
} |
| 285 |
} |
| 286 |
|
| 287 |
func (q *queue) Len() int { |
| 288 |
q.Lock() |
| 289 |
defer q.Unlock() |
| 290 |
|
| 291 |
return q.len |
| 292 |
} |
| 293 |
|
| 294 |
// Dropped returns the number of Records dropped during enqueueing since the |
| 295 |
// last time Dropped was called. |
| 296 |
func (q *queue) Dropped() uint64 { |
| 297 |
return q.dropped.Swap(0) |
| 298 |
} |
| 299 |
|
| 300 |
// Enqueue adds r to the queue. The queue size, including the addition of r, is |
| 301 |
// returned. |
| 302 |
// |
| 303 |
// If enqueueing r will exceed the capacity of q, the oldest Record held in q |
| 304 |
// will be dropped and r retained. |
| 305 |
func (q *queue) Enqueue(r Record) int { |
| 306 |
q.Lock() |
| 307 |
defer q.Unlock() |
| 308 |
|
| 309 |
q.write.Value = r |
| 310 |
q.write = q.write.Next() |
| 311 |
|
| 312 |
q.len++ |
| 313 |
if q.len > q.cap { |
| 314 |
// Overflow. Advance read to be the new "oldest". |
| 315 |
q.len = q.cap |
| 316 |
q.read = q.read.Next() |
| 317 |
q.dropped.Add(1) |
| 318 |
} |
| 319 |
return q.len |
| 320 |
} |
| 321 |
|
| 322 |
// TryDequeue attempts to dequeue up to len(buf) Records. The available Records |
| 323 |
// will be assigned into buf and passed to write. If write fails, returning |
| 324 |
// false, the Records will not be removed from the queue. If write succeeds, |
| 325 |
// returning true, the dequeued Records are removed from the queue. The number |
| 326 |
// of Records remaining in the queue are returned. |
| 327 |
// |
| 328 |
// When write is called the lock of q is held. The write function must not call |
| 329 |
// other methods of this q that acquire the lock. |
| 330 |
func (q *queue) TryDequeue(buf []Record, write func([]Record) bool) int { |
| 331 |
q.Lock() |
| 332 |
defer q.Unlock() |
| 333 |
|
| 334 |
origRead := q.read |
| 335 |
|
| 336 |
n := min(len(buf), q.len) |
| 337 |
for i := range n { |
| 338 |
buf[i] = q.read.Value // nolint:gosec // n is bounded by len(buf) |
| 339 |
q.read = q.read.Next() |
| 340 |
} |
| 341 |
|
| 342 |
if write(buf[:n]) { |
| 343 |
q.len -= n |
| 344 |
} else { |
| 345 |
q.read = origRead |
| 346 |
} |
| 347 |
return q.len |
| 348 |
} |
| 349 |
|
| 350 |
// Flush returns all the Records held in the queue and resets it to be |
| 351 |
// empty. |
| 352 |
func (q *queue) Flush() []Record { |
| 353 |
q.Lock() |
| 354 |
defer q.Unlock() |
| 355 |
|
| 356 |
out := make([]Record, q.len) |
| 357 |
for i := range out { |
| 358 |
out[i] = q.read.Value |
| 359 |
q.read = q.read.Next() |
| 360 |
} |
| 361 |
q.len = 0 |
| 362 |
|
| 363 |
return out |
| 364 |
} |
| 365 |
|
| 366 |
type batchConfig struct { |
| 367 |
maxQSize setting[int] |
| 368 |
expInterval setting[time.Duration] |
| 369 |
expTimeout setting[time.Duration] |
| 370 |
expMaxBatchSize setting[int] |
| 371 |
expBufferSize setting[int] |
| 372 |
} |
| 373 |
|
| 374 |
func newBatchConfig(options []BatchProcessorOption) batchConfig { |
| 375 |
var c batchConfig |
| 376 |
for _, o := range options { |
| 377 |
c = o.apply(c) |
| 378 |
} |
| 379 |
|
| 380 |
c.maxQSize = c.maxQSize.Resolve( |
| 381 |
clearLessThanOne[int](), |
| 382 |
getenv[int](envarMaxQSize), |
| 383 |
clearLessThanOne[int](), // nolint:gocritic // the function argument is duplicated on purpose |
| 384 |
fallback[int](dfltMaxQSize), |
| 385 |
) |
| 386 |
c.expInterval = c.expInterval.Resolve( |
| 387 |
clearLessThanOne[time.Duration](), |
| 388 |
getenv[time.Duration](envarExpInterval), |
| 389 |
clearLessThanOne[time.Duration](), // nolint:gocritic // the function argument is duplicated on purpose |
| 390 |
fallback[time.Duration](dfltExpInterval), |
| 391 |
) |
| 392 |
c.expTimeout = c.expTimeout.Resolve( |
| 393 |
clearLessThanOne[time.Duration](), |
| 394 |
getenv[time.Duration](envarExpTimeout), |
| 395 |
clearLessThanOne[time.Duration](), // nolint:gocritic // the function argument is duplicated on purpose |
| 396 |
fallback[time.Duration](dfltExpTimeout), |
| 397 |
) |
| 398 |
c.expMaxBatchSize = c.expMaxBatchSize.Resolve( |
| 399 |
clearLessThanOne[int](), |
| 400 |
getenv[int](envarExpMaxBatchSize), |
| 401 |
clearLessThanOne[int](), // nolint:gocritic // the function argument is duplicated on purpose |
| 402 |
clampMax[int](c.maxQSize.Value), |
| 403 |
fallback[int](dfltExpMaxBatchSize), |
| 404 |
) |
| 405 |
c.expBufferSize = c.expBufferSize.Resolve( |
| 406 |
clearLessThanOne[int](), |
| 407 |
fallback[int](dfltExpBufferSize), |
| 408 |
) |
| 409 |
|
| 410 |
return c |
| 411 |
} |
| 412 |
|
| 413 |
// BatchProcessorOption applies a configuration to a [BatchProcessor]. |
| 414 |
type BatchProcessorOption interface { |
| 415 |
apply(batchConfig) batchConfig |
| 416 |
} |
| 417 |
|
| 418 |
type batchOptionFunc func(batchConfig) batchConfig |
| 419 |
|
| 420 |
func (fn batchOptionFunc) apply(c batchConfig) batchConfig { |
| 421 |
return fn(c) |
| 422 |
} |
| 423 |
|
| 424 |
// WithMaxQueueSize sets the maximum queue size used by the Batcher. |
| 425 |
// After the size is reached log records are dropped. |
| 426 |
// |
| 427 |
// If the OTEL_BLRP_MAX_QUEUE_SIZE environment variable is set, |
| 428 |
// and this option is not passed, that variable value will be used. |
| 429 |
// |
| 430 |
// By default, if an environment variable is not set, and this option is not |
| 431 |
// passed, 2048 will be used. |
| 432 |
// The default value is also used when the provided value is less than one. |
| 433 |
func WithMaxQueueSize(size int) BatchProcessorOption { |
| 434 |
return batchOptionFunc(func(cfg batchConfig) batchConfig { |
| 435 |
cfg.maxQSize = newSetting(size) |
| 436 |
return cfg |
| 437 |
}) |
| 438 |
} |
| 439 |
|
| 440 |
// WithExportInterval sets the maximum duration between batched exports. |
| 441 |
// |
| 442 |
// If the OTEL_BLRP_SCHEDULE_DELAY environment variable is set, |
| 443 |
// and this option is not passed, that variable value will be used. |
| 444 |
// |
| 445 |
// By default, if an environment variable is not set, and this option is not |
| 446 |
// passed, 1s will be used. |
| 447 |
// The default value is also used when the provided value is less than one. |
| 448 |
func WithExportInterval(d time.Duration) BatchProcessorOption { |
| 449 |
return batchOptionFunc(func(cfg batchConfig) batchConfig { |
| 450 |
cfg.expInterval = newSetting(d) |
| 451 |
return cfg |
| 452 |
}) |
| 453 |
} |
| 454 |
|
| 455 |
// WithExportTimeout sets the duration after which a batched export is canceled. |
| 456 |
// |
| 457 |
// If the OTEL_BLRP_EXPORT_TIMEOUT environment variable is set, |
| 458 |
// and this option is not passed, that variable value will be used. |
| 459 |
// |
| 460 |
// By default, if an environment variable is not set, and this option is not |
| 461 |
// passed, 30s will be used. |
| 462 |
// The default value is also used when the provided value is less than one. |
| 463 |
func WithExportTimeout(d time.Duration) BatchProcessorOption { |
| 464 |
return batchOptionFunc(func(cfg batchConfig) batchConfig { |
| 465 |
cfg.expTimeout = newSetting(d) |
| 466 |
return cfg |
| 467 |
}) |
| 468 |
} |
| 469 |
|
| 470 |
// WithExportMaxBatchSize sets the maximum batch size of every export. |
| 471 |
// A batch will be split into multiple exports to not exceed this size. |
| 472 |
// |
| 473 |
// If the OTEL_BLRP_MAX_EXPORT_BATCH_SIZE environment variable is set, |
| 474 |
// and this option is not passed, that variable value will be used. |
| 475 |
// |
| 476 |
// By default, if an environment variable is not set, and this option is not |
| 477 |
// passed, 512 will be used. |
| 478 |
// The default value is also used when the provided value is less than one. |
| 479 |
func WithExportMaxBatchSize(size int) BatchProcessorOption { |
| 480 |
return batchOptionFunc(func(cfg batchConfig) batchConfig { |
| 481 |
cfg.expMaxBatchSize = newSetting(size) |
| 482 |
return cfg |
| 483 |
}) |
| 484 |
} |
| 485 |
|
| 486 |
// WithExportBufferSize sets the batch buffer size. |
| 487 |
// Batches will be temporarily kept in a memory buffer until they are exported. |
| 488 |
// |
| 489 |
// By default, a value of 1 will be used. |
| 490 |
// The default value is also used when the provided value is less than one. |
| 491 |
func WithExportBufferSize(size int) BatchProcessorOption { |
| 492 |
return batchOptionFunc(func(cfg batchConfig) batchConfig { |
| 493 |
cfg.expBufferSize = newSetting(size) |
| 494 |
return cfg |
| 495 |
}) |
| 496 |
} |