Debian Codegraph @

plugin/pkg/audit/buffered/buffered.go

raw
1 /*
2 Copyright 2018 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package buffered
18
19 import (
20 "fmt"
21 "sync"
22 "time"
23
24 "k8s.io/apimachinery/pkg/util/runtime"
25 "k8s.io/apimachinery/pkg/util/wait"
26 auditinternal "k8s.io/apiserver/pkg/apis/audit"
27 "k8s.io/apiserver/pkg/audit"
28 "k8s.io/client-go/util/flowcontrol"
29 )
30
31 // PluginName is the name reported in error metrics.
32 const PluginName = "buffered"
33
34 // BatchConfig represents batching delegate audit backend configuration.
35 type BatchConfig struct {
36 // BufferSize defines a size of the buffering queue.
37 BufferSize int
38 // MaxBatchSize defines maximum size of a batch.
39 MaxBatchSize int
40 // MaxBatchWait indicates the maximum interval between two batches.
41 MaxBatchWait time.Duration
42
43 // ThrottleEnable defines whether throttling will be applied to the batching process.
44 ThrottleEnable bool
45 // ThrottleQPS defines the allowed rate of batches per second sent to the delegate backend.
46 ThrottleQPS float32
47 // ThrottleBurst defines the maximum number of requests sent to the delegate backend at the same moment in case
48 // the capacity defined by ThrottleQPS was not utilized.
49 ThrottleBurst int
50
51 // Whether the delegate backend should be called asynchronously.
52 AsyncDelegate bool
53 }
54
55 type bufferedBackend struct {
56 // The delegate backend that actually exports events.
57 delegateBackend audit.Backend
58
59 // Channel to buffer events before sending to the delegate backend.
60 buffer chan *auditinternal.Event
61 // Maximum number of events in a batch sent to the delegate backend.
62 maxBatchSize int
63 // Amount of time to wait after sending a batch to the delegate backend before sending another one.
64 //
65 // Receiving maxBatchSize events will always trigger sending a batch, regardless of the amount of time passed.
66 maxBatchWait time.Duration
67
68 // Whether the delegate backend should be called asynchronously.
69 asyncDelegate bool
70
71 // Channel to signal that the batching routine has processed all remaining events and exited.
72 // Once `shutdownCh` is closed no new events will be sent to the delegate backend.
73 shutdownCh chan struct{}
74
75 // WaitGroup to control the concurrency of sending batches to the delegate backend.
76 // Worker routine calls Add before sending a batch and
77 // then spawns a routine that calls Done after batch was processed by the delegate backend.
78 // This WaitGroup is used to wait for all sending routines to finish before shutting down audit backend.
79 wg sync.WaitGroup
80
81 // Limits the number of batches sent to the delegate backend per second.
82 throttle flowcontrol.RateLimiter
83 }
84
85 var _ audit.Backend = &bufferedBackend{}
86
87 // NewBackend returns a buffered audit backend that wraps delegate backend.
88 // Buffered backend automatically runs and shuts down the delegate backend.
89 func NewBackend(delegate audit.Backend, config BatchConfig) audit.Backend {
90 var throttle flowcontrol.RateLimiter
91 if config.ThrottleEnable {
92 throttle = flowcontrol.NewTokenBucketRateLimiter(config.ThrottleQPS, config.ThrottleBurst)
93 }
94 return &bufferedBackend{
95 delegateBackend: delegate,
96 buffer: make(chan *auditinternal.Event, config.BufferSize),
97 maxBatchSize: config.MaxBatchSize,
98 maxBatchWait: config.MaxBatchWait,
99 asyncDelegate: config.AsyncDelegate,
100 shutdownCh: make(chan struct{}),
101 wg: sync.WaitGroup{},
102 throttle: throttle,
103 }
104 }
105
106 func (b *bufferedBackend) Run(stopCh <-chan struct{}) error {
107 go func() {
108 // Signal that the working routine has exited.
109 defer close(b.shutdownCh)
110
111 b.processIncomingEvents(stopCh)
112
113 // Handle the events that were received after the last buffer
114 // scraping and before this line. Since the buffer is closed, no new
115 // events will come through.
116 allEventsProcessed := false
117 timer := make(chan time.Time)
118 for !allEventsProcessed {
119 allEventsProcessed = func() bool {
120 // Recover from any panic in order to try to process all remaining events.
121 // Note, that in case of a panic, the return value will be false and
122 // the loop execution will continue.
123 defer runtime.HandleCrash()
124
125 events := b.collectEvents(timer, wait.NeverStop)
126 b.processEvents(events)
127 return len(events) == 0
128 }()
129 }
130 }()
131 return b.delegateBackend.Run(stopCh)
132 }
133
134 // Shutdown blocks until stopCh passed to the Run method is closed and all
135 // events added prior to that moment are batched and sent to the delegate backend.
136 func (b *bufferedBackend) Shutdown() {
137 // Wait until the routine spawned in Run method exits.
138 <-b.shutdownCh
139
140 // Wait until all sending routines exit.
141 //
142 // - When b.shutdownCh is closed, we know that the goroutine in Run has terminated.
143 // - This means that processIncomingEvents has terminated.
144 // - Which means that b.buffer is closed and cannot accept any new events anymore.
145 // - Because processEvents is called synchronously from the Run goroutine, the waitgroup has its final value.
146 // Hence wg.Wait will not miss any more outgoing batches.
147 b.wg.Wait()
148
149 b.delegateBackend.Shutdown()
150 }
151
152 // processIncomingEvents runs a loop that collects events from the buffer. When
153 // b.stopCh is closed, processIncomingEvents stops and closes the buffer.
154 func (b *bufferedBackend) processIncomingEvents(stopCh <-chan struct{}) {
155 defer close(b.buffer)
156
157 var (
158 maxWaitChan <-chan time.Time
159 maxWaitTimer *time.Timer
160 )
161 // Only use max wait batching if batching is enabled.
162 if b.maxBatchSize > 1 {
163 maxWaitTimer = time.NewTimer(b.maxBatchWait)
164 maxWaitChan = maxWaitTimer.C
165 defer maxWaitTimer.Stop()
166 }
167
168 for {
169 func() {
170 // Recover from any panics caused by this function so a panic in the
171 // goroutine can't bring down the main routine.
172 defer runtime.HandleCrash()
173
174 if b.maxBatchSize > 1 {
175 maxWaitTimer.Reset(b.maxBatchWait)
176 }
177 b.processEvents(b.collectEvents(maxWaitChan, stopCh))
178 }()
179
180 select {
181 case <-stopCh:
182 return
183 default:
184 }
185 }
186 }
187
188 // collectEvents attempts to collect some number of events in a batch.
189 //
190 // The following things can cause collectEvents to stop and return the list
191 // of events:
192 //
193 // - Maximum number of events for a batch.
194 // - Timer has passed.
195 // - Buffer channel is closed and empty.
196 // - stopCh is closed.
197 func (b *bufferedBackend) collectEvents(timer <-chan time.Time, stopCh <-chan struct{}) []*auditinternal.Event {
198 var events []*auditinternal.Event
199
200 L:
201 for i := 0; i < b.maxBatchSize; i++ {
202 select {
203 case ev, ok := <-b.buffer:
204 // Buffer channel was closed and no new events will follow.
205 if !ok {
206 break L
207 }
208 events = append(events, ev)
209 case <-timer:
210 // Timer has expired. Send currently accumulated batch.
211 break L
212 case <-stopCh:
213 // Backend has been stopped. Send currently accumulated batch.
214 break L
215 }
216 }
217
218 return events
219 }
220
221 // processEvents process the batch events in a goroutine using delegateBackend's ProcessEvents.
222 func (b *bufferedBackend) processEvents(events []*auditinternal.Event) {
223 if len(events) == 0 {
224 return
225 }
226
227 // TODO(audit): Should control the number of active goroutines
228 // if one goroutine takes 5 seconds to finish, the number of goroutines can be 5 * defaultBatchThrottleQPS
229 if b.throttle != nil {
230 b.throttle.Accept()
231 }
232
233 if b.asyncDelegate {
234 b.wg.Add(1)
235 go func() {
236 defer b.wg.Done()
237 defer runtime.HandleCrash()
238
239 // Execute the real processing in a goroutine to keep it from blocking.
240 // This lets the batching routine continue draining the queue immediately.
241 b.delegateBackend.ProcessEvents(events...)
242 }()
243 } else {
244 func() {
245 defer runtime.HandleCrash()
246
247 // Execute the real processing in a goroutine to keep it from blocking.
248 // This lets the batching routine continue draining the queue immediately.
249 b.delegateBackend.ProcessEvents(events...)
250 }()
251 }
252 }
253
254 func (b *bufferedBackend) ProcessEvents(ev ...*auditinternal.Event) bool {
255 // The following mechanism is in place to support the situation when audit
256 // events are still coming after the backend was stopped.
257 var sendErr error
258 var evIndex int
259
260 // If the delegateBackend was shutdown and the buffer channel was closed, an
261 // attempt to add an event to it will result in panic that we should
262 // recover from.
263 defer func() {
264 if err := recover(); err != nil {
265 sendErr = fmt.Errorf("audit backend shut down")
266 }
267 if sendErr != nil {
268 audit.HandlePluginError(PluginName, sendErr, ev[evIndex:]...)
269 }
270 }()
271
272 for i, e := range ev {
273 evIndex = i
274 // Per the audit.Backend interface these events are reused after being
275 // sent to the Sink. Deep copy and send the copy to the queue.
276 event := e.DeepCopy()
277
278 select {
279 case b.buffer <- event:
280 default:
281 sendErr = fmt.Errorf("audit buffer queue blocked")
282 return true
283 }
284 }
285 return true
286 }
287
288 func (b *bufferedBackend) String() string {
289 return fmt.Sprintf("%s<%s>", PluginName, b.delegateBackend)
290 }