Debian Codegraph @

sdk/log/batch.go

raw
1 // Copyright The OpenTelemetry Authors
2 // SPDX-License-Identifier: Apache-2.0
3
4 package log // import "go.opentelemetry.io/otel/sdk/log"
5
6 import (
7 "context"
8 "errors"
9 "slices"
10 "sync"
11 "sync/atomic"
12 "time"
13
14 "go.opentelemetry.io/otel/internal/global"
15 )
16
17 const (
18 dfltMaxQSize = 2048
19 dfltExpInterval = time.Second
20 dfltExpTimeout = 30 * time.Second
21 dfltExpMaxBatchSize = 512
22 dfltExpBufferSize = 1
23
24 envarMaxQSize = "OTEL_BLRP_MAX_QUEUE_SIZE"
25 envarExpInterval = "OTEL_BLRP_SCHEDULE_DELAY"
26 envarExpTimeout = "OTEL_BLRP_EXPORT_TIMEOUT"
27 envarExpMaxBatchSize = "OTEL_BLRP_MAX_EXPORT_BATCH_SIZE"
28 )
29
30 // Compile-time check BatchProcessor implements Processor.
31 var _ Processor = (*BatchProcessor)(nil)
32
33 // BatchProcessor is a processor that exports batches of log records.
34 //
35 // Use [NewBatchProcessor] to create a BatchProcessor. An empty BatchProcessor
36 // is shut down by default, no records will be batched or exported.
37 type BatchProcessor struct {
38 // The BatchProcessor is designed to provide the highest throughput of
39 // log records possible while being compatible with OpenTelemetry. The
40 // entry point of log records is the OnEmit method. This method is designed
41 // to receive records as fast as possible while still honoring shutdown
42 // commands. All records received are enqueued to queue.
43 //
44 // In order to block OnEmit as little as possible, a separate "poll"
45 // goroutine is spawned at the creation of a BatchProcessor. This
46 // goroutine is responsible for batching the queue at regular polled
47 // intervals, or when it is directly signaled to.
48 //
49 // To keep the polling goroutine from backing up, all batches it makes are
50 // exported with a bufferedExporter. This exporter allows the poll
51 // goroutine to enqueue an export payload that will be handled in a
52 // separate goroutine dedicated to the export. This asynchronous behavior
53 // allows the poll goroutine to maintain accurate interval polling.
54 //
55 // ___BatchProcessor____ __Poll Goroutine__ __Export Goroutine__
56 // || || || || || ||
57 // || ********** || || || || ********** ||
58 // || Records=>* OnEmit * || || | - ticker || || * export * ||
59 // || ********** || || | - trigger || || ********** ||
60 // || || || || | || || || ||
61 // || || || || | || || || ||
62 // || __________\/___ || || |*********** || || ______/\_______ ||
63 // || (____queue______)>=||=||===|* batch *===||=||=>[_export_buffer_] ||
64 // || || || |*********** || || ||
65 // ||_____________________|| ||__________________|| ||____________________||
66 //
67 //
68 // The "release valve" in this processing is the record queue. This queue
69 // is a ring buffer. It will overwrite the oldest records first when writes
70 // to OnEmit are made faster than the queue can be flushed. If batches
71 // cannot be flushed to the export buffer, the records will remain in the
72 // queue.
73
74 // exporter is the bufferedExporter all batches are exported with.
75 exporter *bufferExporter
76
77 // q is the active queue of records that have not yet been exported.
78 q *queue
79 // batchSize is the minimum number of records needed before an export is
80 // triggered (unless the interval expires).
81 batchSize int
82
83 // pollTrigger triggers the poll goroutine to flush a batch from the queue.
84 // This is sent to when it is known that the queue contains at least one
85 // complete batch.
86 //
87 // When a send is made to the channel, the poll loop will be reset after
88 // the flush. If there is still enough records in the queue for another
89 // batch the reset of the poll loop will automatically re-trigger itself.
90 // There is no need for the original sender to monitor and resend.
91 pollTrigger chan struct{}
92 // pollKill kills the poll goroutine. This is only expected to be closed
93 // once by the Shutdown method.
94 pollKill chan struct{}
95 // pollDone signals the poll goroutine has completed.
96 pollDone chan struct{}
97
98 // stopped holds the stopped state of the BatchProcessor.
99 stopped atomic.Bool
100
101 noCmp [0]func() //nolint: unused // This is indeed used.
102 }
103
104 // NewBatchProcessor decorates the provided exporter
105 // so that the log records are batched before exporting.
106 //
107 // All of the exporter's methods are called synchronously.
108 func NewBatchProcessor(exporter Exporter, opts ...BatchProcessorOption) *BatchProcessor {
109 cfg := newBatchConfig(opts)
110 if exporter == nil {
111 // Do not panic on nil export.
112 exporter = defaultNoopExporter
113 }
114 // Order is important here. Wrap the timeoutExporter with the chunkExporter
115 // to ensure each export completes in timeout (instead of all chunked
116 // exports).
117 exporter = newTimeoutExporter(exporter, cfg.expTimeout.Value)
118 // Use a chunkExporter to ensure ForceFlush and Shutdown calls are batched
119 // appropriately on export.
120 exporter = newChunkExporter(exporter, cfg.expMaxBatchSize.Value)
121
122 b := &BatchProcessor{
123 exporter: newBufferExporter(exporter, cfg.expBufferSize.Value),
124
125 q: newQueue(cfg.maxQSize.Value),
126 batchSize: cfg.expMaxBatchSize.Value,
127 pollTrigger: make(chan struct{}, 1),
128 pollKill: make(chan struct{}),
129 }
130 b.pollDone = b.poll(cfg.expInterval.Value)
131 return b
132 }
133
134 // poll spawns a goroutine to handle interval polling and batch exporting. The
135 // returned done chan is closed when the spawned goroutine completes.
136 func (b *BatchProcessor) poll(interval time.Duration) (done chan struct{}) {
137 done = make(chan struct{})
138
139 ticker := time.NewTicker(interval)
140 // TODO: investigate using a sync.Pool instead of cloning.
141 buf := make([]Record, b.batchSize)
142 go func() {
143 defer close(done)
144 defer ticker.Stop()
145
146 for {
147 select {
148 case <-ticker.C:
149 case <-b.pollTrigger:
150 ticker.Reset(interval)
151 case <-b.pollKill:
152 return
153 }
154
155 if d := b.q.Dropped(); d > 0 {
156 global.Warn("dropped log records", "dropped", d)
157 }
158
159 var qLen int
160 // Don't copy data from queue unless exporter can accept more, it is very expensive.
161 if b.exporter.Ready() {
162 qLen = b.q.TryDequeue(buf, func(r []Record) bool {
163 ok := b.exporter.EnqueueExport(r)
164 if ok {
165 buf = slices.Clone(buf)
166 }
167 return ok
168 })
169 } else {
170 qLen = b.q.Len()
171 }
172
173 if qLen >= b.batchSize {
174 // There is another full batch ready. Immediately trigger
175 // another export attempt.
176 select {
177 case b.pollTrigger <- struct{}{}:
178 default:
179 // Another flush signal already received.
180 }
181 }
182 }
183 }()
184 return done
185 }
186
187 // Enabled returns true, indicating this Processor will process all records.
188 func (*BatchProcessor) Enabled(context.Context, EnabledParameters) bool {
189 return true
190 }
191
192 // OnEmit batches provided log record.
193 func (b *BatchProcessor) OnEmit(_ context.Context, r *Record) error {
194 if b.stopped.Load() || b.q == nil {
195 return nil
196 }
197 // The record is cloned so that changes done by subsequent processors
198 // are not going to lead to a data race.
199 if n := b.q.Enqueue(r.Clone()); n >= b.batchSize {
200 select {
201 case b.pollTrigger <- struct{}{}:
202 default:
203 // Flush chan full. The poll goroutine will handle this by
204 // re-sending any trigger until the queue has less than batchSize
205 // records.
206 }
207 }
208 return nil
209 }
210
211 // Shutdown flushes queued log records and shuts down the decorated exporter.
212 func (b *BatchProcessor) Shutdown(ctx context.Context) error {
213 if b.stopped.Swap(true) || b.q == nil {
214 return nil
215 }
216
217 // Stop the poll goroutine.
218 close(b.pollKill)
219 select {
220 case <-b.pollDone:
221 case <-ctx.Done():
222 // Out of time.
223 return errors.Join(ctx.Err(), b.exporter.Shutdown(ctx))
224 }
225
226 // Flush remaining queued before exporter shutdown.
227 err := b.exporter.Export(ctx, b.q.Flush())
228 return errors.Join(err, b.exporter.Shutdown(ctx))
229 }
230
231 var errPartialFlush = errors.New("partial flush: export buffer full")
232
233 // Used for testing.
234 var ctxErr = func(ctx context.Context) error {
235 return ctx.Err()
236 }
237
238 // ForceFlush flushes queued log records and flushes the decorated exporter.
239 func (b *BatchProcessor) ForceFlush(ctx context.Context) error {
240 if b.stopped.Load() || b.q == nil {
241 return nil
242 }
243
244 buf := make([]Record, b.q.cap)
245 notFlushed := func() bool {
246 var flushed bool
247 _ = b.q.TryDequeue(buf, func(r []Record) bool {
248 flushed = b.exporter.EnqueueExport(r)
249 return flushed
250 })
251 return !flushed
252 }
253 var err error
254 // For as long as ctx allows, try to make a single flush of the queue.
255 for notFlushed() {
256 // Use ctxErr instead of calling ctx.Err directly so we can test
257 // the partial error return.
258 if e := ctxErr(ctx); e != nil {
259 err = errors.Join(e, errPartialFlush)
260 break
261 }
262 }
263 return errors.Join(err, b.exporter.ForceFlush(ctx))
264 }
265
266 // queue holds a queue of logging records.
267 //
268 // When the queue becomes full, the oldest records in the queue are
269 // overwritten.
270 type queue struct {
271 sync.Mutex
272
273 dropped atomic.Uint64
274 cap, len int
275 read, write *ring
276 }
277
278 func newQueue(size int) *queue {
279 r := newRing(size)
280 return &queue{
281 cap: size,
282 read: r,
283 write: r,
284 }
285 }
286
287 func (q *queue) Len() int {
288 q.Lock()
289 defer q.Unlock()
290
291 return q.len
292 }
293
294 // Dropped returns the number of Records dropped during enqueueing since the
295 // last time Dropped was called.
296 func (q *queue) Dropped() uint64 {
297 return q.dropped.Swap(0)
298 }
299
300 // Enqueue adds r to the queue. The queue size, including the addition of r, is
301 // returned.
302 //
303 // If enqueueing r will exceed the capacity of q, the oldest Record held in q
304 // will be dropped and r retained.
305 func (q *queue) Enqueue(r Record) int {
306 q.Lock()
307 defer q.Unlock()
308
309 q.write.Value = r
310 q.write = q.write.Next()
311
312 q.len++
313 if q.len > q.cap {
314 // Overflow. Advance read to be the new "oldest".
315 q.len = q.cap
316 q.read = q.read.Next()
317 q.dropped.Add(1)
318 }
319 return q.len
320 }
321
322 // TryDequeue attempts to dequeue up to len(buf) Records. The available Records
323 // will be assigned into buf and passed to write. If write fails, returning
324 // false, the Records will not be removed from the queue. If write succeeds,
325 // returning true, the dequeued Records are removed from the queue. The number
326 // of Records remaining in the queue are returned.
327 //
328 // When write is called the lock of q is held. The write function must not call
329 // other methods of this q that acquire the lock.
330 func (q *queue) TryDequeue(buf []Record, write func([]Record) bool) int {
331 q.Lock()
332 defer q.Unlock()
333
334 origRead := q.read
335
336 n := min(len(buf), q.len)
337 for i := range n {
338 buf[i] = q.read.Value // nolint:gosec // n is bounded by len(buf)
339 q.read = q.read.Next()
340 }
341
342 if write(buf[:n]) {
343 q.len -= n
344 } else {
345 q.read = origRead
346 }
347 return q.len
348 }
349
350 // Flush returns all the Records held in the queue and resets it to be
351 // empty.
352 func (q *queue) Flush() []Record {
353 q.Lock()
354 defer q.Unlock()
355
356 out := make([]Record, q.len)
357 for i := range out {
358 out[i] = q.read.Value
359 q.read = q.read.Next()
360 }
361 q.len = 0
362
363 return out
364 }
365
366 type batchConfig struct {
367 maxQSize setting[int]
368 expInterval setting[time.Duration]
369 expTimeout setting[time.Duration]
370 expMaxBatchSize setting[int]
371 expBufferSize setting[int]
372 }
373
374 func newBatchConfig(options []BatchProcessorOption) batchConfig {
375 var c batchConfig
376 for _, o := range options {
377 c = o.apply(c)
378 }
379
380 c.maxQSize = c.maxQSize.Resolve(
381 clearLessThanOne[int](),
382 getenv[int](envarMaxQSize),
383 clearLessThanOne[int](), // nolint:gocritic // the function argument is duplicated on purpose
384 fallback[int](dfltMaxQSize),
385 )
386 c.expInterval = c.expInterval.Resolve(
387 clearLessThanOne[time.Duration](),
388 getenv[time.Duration](envarExpInterval),
389 clearLessThanOne[time.Duration](), // nolint:gocritic // the function argument is duplicated on purpose
390 fallback[time.Duration](dfltExpInterval),
391 )
392 c.expTimeout = c.expTimeout.Resolve(
393 clearLessThanOne[time.Duration](),
394 getenv[time.Duration](envarExpTimeout),
395 clearLessThanOne[time.Duration](), // nolint:gocritic // the function argument is duplicated on purpose
396 fallback[time.Duration](dfltExpTimeout),
397 )
398 c.expMaxBatchSize = c.expMaxBatchSize.Resolve(
399 clearLessThanOne[int](),
400 getenv[int](envarExpMaxBatchSize),
401 clearLessThanOne[int](), // nolint:gocritic // the function argument is duplicated on purpose
402 clampMax[int](c.maxQSize.Value),
403 fallback[int](dfltExpMaxBatchSize),
404 )
405 c.expBufferSize = c.expBufferSize.Resolve(
406 clearLessThanOne[int](),
407 fallback[int](dfltExpBufferSize),
408 )
409
410 return c
411 }
412
413 // BatchProcessorOption applies a configuration to a [BatchProcessor].
414 type BatchProcessorOption interface {
415 apply(batchConfig) batchConfig
416 }
417
418 type batchOptionFunc func(batchConfig) batchConfig
419
420 func (fn batchOptionFunc) apply(c batchConfig) batchConfig {
421 return fn(c)
422 }
423
424 // WithMaxQueueSize sets the maximum queue size used by the Batcher.
425 // After the size is reached log records are dropped.
426 //
427 // If the OTEL_BLRP_MAX_QUEUE_SIZE environment variable is set,
428 // and this option is not passed, that variable value will be used.
429 //
430 // By default, if an environment variable is not set, and this option is not
431 // passed, 2048 will be used.
432 // The default value is also used when the provided value is less than one.
433 func WithMaxQueueSize(size int) BatchProcessorOption {
434 return batchOptionFunc(func(cfg batchConfig) batchConfig {
435 cfg.maxQSize = newSetting(size)
436 return cfg
437 })
438 }
439
440 // WithExportInterval sets the maximum duration between batched exports.
441 //
442 // If the OTEL_BLRP_SCHEDULE_DELAY environment variable is set,
443 // and this option is not passed, that variable value will be used.
444 //
445 // By default, if an environment variable is not set, and this option is not
446 // passed, 1s will be used.
447 // The default value is also used when the provided value is less than one.
448 func WithExportInterval(d time.Duration) BatchProcessorOption {
449 return batchOptionFunc(func(cfg batchConfig) batchConfig {
450 cfg.expInterval = newSetting(d)
451 return cfg
452 })
453 }
454
455 // WithExportTimeout sets the duration after which a batched export is canceled.
456 //
457 // If the OTEL_BLRP_EXPORT_TIMEOUT environment variable is set,
458 // and this option is not passed, that variable value will be used.
459 //
460 // By default, if an environment variable is not set, and this option is not
461 // passed, 30s will be used.
462 // The default value is also used when the provided value is less than one.
463 func WithExportTimeout(d time.Duration) BatchProcessorOption {
464 return batchOptionFunc(func(cfg batchConfig) batchConfig {
465 cfg.expTimeout = newSetting(d)
466 return cfg
467 })
468 }
469
470 // WithExportMaxBatchSize sets the maximum batch size of every export.
471 // A batch will be split into multiple exports to not exceed this size.
472 //
473 // If the OTEL_BLRP_MAX_EXPORT_BATCH_SIZE environment variable is set,
474 // and this option is not passed, that variable value will be used.
475 //
476 // By default, if an environment variable is not set, and this option is not
477 // passed, 512 will be used.
478 // The default value is also used when the provided value is less than one.
479 func WithExportMaxBatchSize(size int) BatchProcessorOption {
480 return batchOptionFunc(func(cfg batchConfig) batchConfig {
481 cfg.expMaxBatchSize = newSetting(size)
482 return cfg
483 })
484 }
485
486 // WithExportBufferSize sets the batch buffer size.
487 // Batches will be temporarily kept in a memory buffer until they are exported.
488 //
489 // By default, a value of 1 will be used.
490 // The default value is also used when the provided value is less than one.
491 func WithExportBufferSize(size int) BatchProcessorOption {
492 return batchOptionFunc(func(cfg batchConfig) batchConfig {
493 cfg.expBufferSize = newSetting(size)
494 return cfg
495 })
496 }