Debian Codegraph @

pkg/endpoints/handlers/watch.go

raw
1 /*
2 Copyright 2014 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package handlers
18
19 import (
20 "context"
21 "fmt"
22 "io"
23 "net/http"
24 "time"
25
26 "golang.org/x/net/websocket"
27
28 "k8s.io/apimachinery/pkg/api/errors"
29 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30 "k8s.io/apimachinery/pkg/runtime"
31 "k8s.io/apimachinery/pkg/util/httpstream/wsstream"
32 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
33 "k8s.io/apimachinery/pkg/watch"
34 "k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
35 "k8s.io/apiserver/pkg/endpoints/metrics"
36 apirequest "k8s.io/apiserver/pkg/endpoints/request"
37 "k8s.io/apiserver/pkg/features"
38 "k8s.io/apiserver/pkg/storage"
39 utilfeature "k8s.io/apiserver/pkg/util/feature"
40 )
41
42 // nothing will ever be sent down this channel
43 var neverExitWatch <-chan time.Time = make(chan time.Time)
44
45 // timeoutFactory abstracts watch timeout logic for testing
46 type TimeoutFactory interface {
47 TimeoutCh() (<-chan time.Time, func() bool)
48 }
49
50 // realTimeoutFactory implements timeoutFactory
51 type realTimeoutFactory struct {
52 timeout time.Duration
53 }
54
55 // TimeoutCh returns a channel which will receive something when the watch times out,
56 // and a cleanup function to call when this happens.
57 func (w *realTimeoutFactory) TimeoutCh() (<-chan time.Time, func() bool) {
58 if w.timeout == 0 {
59 return neverExitWatch, func() bool { return false }
60 }
61 t := time.NewTimer(w.timeout)
62 return t.C, t.Stop
63 }
64
65 // serveWatchHandler returns a handle to serve a watch response.
66 // TODO: the functionality in this method and in WatchServer.Serve is not cleanly decoupled.
67 func serveWatchHandler(watcher watch.Interface, scope *RequestScope, mediaTypeOptions negotiation.MediaTypeOptions, req *http.Request, w http.ResponseWriter, timeout time.Duration, metricsScope string, initialEventsListBlueprint runtime.Object) (http.Handler, error) {
68 options, err := optionsForTransform(mediaTypeOptions, req)
69 if err != nil {
70 return nil, err
71 }
72
73 // negotiate for the stream serializer from the scope's serializer
74 serializer, err := negotiation.NegotiateOutputMediaTypeStream(req, scope.Serializer, scope)
75 if err != nil {
76 return nil, err
77 }
78 framer := serializer.StreamSerializer.Framer
79 var encoder runtime.Encoder
80 if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) {
81 encoder = scope.Serializer.EncoderForVersion(runtime.UseNondeterministicEncoding(serializer.StreamSerializer.Serializer), scope.Kind.GroupVersion())
82 } else {
83 encoder = scope.Serializer.EncoderForVersion(serializer.StreamSerializer.Serializer, scope.Kind.GroupVersion())
84 }
85 useTextFraming := serializer.EncodesAsText
86 if framer == nil {
87 return nil, fmt.Errorf("no framer defined for %q available for embedded encoding", serializer.MediaType)
88 }
89 // TODO: next step, get back mediaTypeOptions from negotiate and return the exact value here
90 mediaType := serializer.MediaType
91 switch mediaType {
92 case runtime.ContentTypeJSON:
93 // as-is
94 case runtime.ContentTypeCBOR:
95 // If a client indicated it accepts application/cbor (exactly one data item) on a
96 // watch request, set the conformant application/cbor-seq media type the watch
97 // response. RFC 9110 allows an origin server to deviate from the indicated
98 // preference rather than send a 406 (Not Acceptable) response (see
99 // https://www.rfc-editor.org/rfc/rfc9110.html#section-12.1-5).
100 mediaType = runtime.ContentTypeCBORSequence
101 default:
102 mediaType += ";stream=watch"
103 }
104
105 ctx := req.Context()
106
107 // locate the appropriate embedded encoder based on the transform
108 var negotiatedEncoder runtime.Encoder
109 contentKind, contentSerializer, transform := targetEncodingForTransform(scope, mediaTypeOptions, req)
110 if transform {
111 info, ok := runtime.SerializerInfoForMediaType(contentSerializer.SupportedMediaTypes(), serializer.MediaType)
112 if !ok {
113 return nil, fmt.Errorf("no encoder for %q exists in the requested target %#v", serializer.MediaType, contentSerializer)
114 }
115 if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) {
116 negotiatedEncoder = contentSerializer.EncoderForVersion(runtime.UseNondeterministicEncoding(info.Serializer), contentKind.GroupVersion())
117 } else {
118 negotiatedEncoder = contentSerializer.EncoderForVersion(info.Serializer, contentKind.GroupVersion())
119 }
120 } else {
121 if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) {
122 negotiatedEncoder = scope.Serializer.EncoderForVersion(runtime.UseNondeterministicEncoding(serializer.Serializer), contentKind.GroupVersion())
123 } else {
124 negotiatedEncoder = scope.Serializer.EncoderForVersion(serializer.Serializer, contentKind.GroupVersion())
125 }
126 }
127
128 var memoryAllocator runtime.MemoryAllocator
129
130 if encoderWithAllocator, supportsAllocator := negotiatedEncoder.(runtime.EncoderWithAllocator); supportsAllocator {
131 // don't put the allocator inside the embeddedEncodeFn as that would allocate memory on every call.
132 // instead, we allocate the buffer for the entire watch session and release it when we close the connection.
133 memoryAllocator = runtime.AllocatorPool.Get().(*runtime.Allocator)
134 negotiatedEncoder = runtime.NewEncoderWithAllocator(encoderWithAllocator, memoryAllocator)
135 }
136 var tableOptions *metav1.TableOptions
137 if options != nil {
138 if passedOptions, ok := options.(*metav1.TableOptions); ok {
139 tableOptions = passedOptions
140 } else {
141 return nil, fmt.Errorf("unexpected options type: %T", options)
142 }
143 }
144 embeddedEncoder := newWatchEmbeddedEncoder(ctx, negotiatedEncoder, mediaTypeOptions.Convert, tableOptions, scope)
145
146 if encoderWithAllocator, supportsAllocator := encoder.(runtime.EncoderWithAllocator); supportsAllocator {
147 if memoryAllocator == nil {
148 // don't put the allocator inside the embeddedEncodeFn as that would allocate memory on every call.
149 // instead, we allocate the buffer for the entire watch session and release it when we close the connection.
150 memoryAllocator = runtime.AllocatorPool.Get().(*runtime.Allocator)
151 }
152 encoder = runtime.NewEncoderWithAllocator(encoderWithAllocator, memoryAllocator)
153 }
154
155 var serverShuttingDownCh <-chan struct{}
156 if signals := apirequest.ServerShutdownSignalFrom(req.Context()); signals != nil {
157 serverShuttingDownCh = signals.ShuttingDown()
158 }
159
160 server := &WatchServer{
161 Watching: watcher,
162 Scope: scope,
163
164 UseTextFraming: useTextFraming,
165 MediaType: mediaType,
166 Framer: framer,
167 Encoder: encoder,
168 EmbeddedEncoder: embeddedEncoder,
169
170 watchListTransformerFn: newWatchListTransformer(initialEventsListBlueprint, mediaTypeOptions.Convert, negotiatedEncoder).transform,
171
172 MemoryAllocator: memoryAllocator,
173 TimeoutFactory: &realTimeoutFactory{timeout},
174 ServerShuttingDownCh: serverShuttingDownCh,
175
176 metricsScope: metricsScope,
177 }
178
179 if wsstream.IsWebSocketRequest(req) {
180 w.Header().Set("Content-Type", server.MediaType)
181 return websocket.Handler(server.HandleWS), nil
182 }
183 return http.HandlerFunc(server.HandleHTTP), nil
184 }
185
186 // WatchServer serves a watch.Interface over a websocket or vanilla HTTP.
187 type WatchServer struct {
188 Watching watch.Interface
189 Scope *RequestScope
190
191 // true if websocket messages should use text framing (as opposed to binary framing)
192 UseTextFraming bool
193 // the media type this watch is being served with
194 MediaType string
195 // used to frame the watch stream
196 Framer runtime.Framer
197 // used to encode the watch stream event itself
198 Encoder runtime.Encoder
199 // used to encode the nested object in the watch stream
200 EmbeddedEncoder runtime.Encoder
201 // watchListTransformerFn a function applied
202 // to watchlist bookmark events that transforms
203 // the embedded object before sending it to a client.
204 watchListTransformerFn watchListTransformerFunction
205
206 MemoryAllocator runtime.MemoryAllocator
207 TimeoutFactory TimeoutFactory
208 ServerShuttingDownCh <-chan struct{}
209
210 metricsScope string
211 }
212
213 // HandleHTTP serves a series of encoded events via HTTP with Transfer-Encoding: chunked.
214 // or over a websocket connection.
215 func (s *WatchServer) HandleHTTP(w http.ResponseWriter, req *http.Request) {
216 defer func() {
217 if s.MemoryAllocator != nil {
218 runtime.AllocatorPool.Put(s.MemoryAllocator)
219 }
220 }()
221
222 flusher, ok := w.(http.Flusher)
223 if !ok {
224 err := fmt.Errorf("unable to start watch - can't get http.Flusher: %#v", w)
225 utilruntime.HandleError(err)
226 s.Scope.err(errors.NewInternalError(err), w, req)
227 return
228 }
229
230 framer := s.Framer.NewFrameWriter(w)
231 if framer == nil {
232 // programmer error
233 err := fmt.Errorf("no stream framing support is available for media type %q", s.MediaType)
234 utilruntime.HandleError(err)
235 s.Scope.err(errors.NewBadRequest(err.Error()), w, req)
236 return
237 }
238
239 // ensure the connection times out
240 timeoutCh, cleanup := s.TimeoutFactory.TimeoutCh()
241 defer cleanup()
242
243 // begin the stream
244 w.Header().Set("Content-Type", s.MediaType)
245 w.Header().Set("Transfer-Encoding", "chunked")
246 w.WriteHeader(http.StatusOK)
247 flusher.Flush()
248
249 kind := s.Scope.Kind
250 watchEncoder := newWatchEncoder(req.Context(), kind, s.EmbeddedEncoder, s.Encoder, framer, s.watchListTransformerFn)
251 ch := s.Watching.ResultChan()
252 done := req.Context().Done()
253
254 for {
255 select {
256 case <-s.ServerShuttingDownCh:
257 // the server has signaled that it is shutting down (not accepting
258 // any new request), all active watch request(s) should return
259 // immediately here. The WithWatchTerminationDuringShutdown server
260 // filter will ensure that the response to the client is rate
261 // limited in order to avoid any thundering herd issue when the
262 // client(s) try to reestablish the WATCH on the other
263 // available apiserver instance(s).
264 return
265 case <-done:
266 return
267 case <-timeoutCh:
268 return
269 case event, ok := <-ch:
270 if !ok {
271 // End of results.
272 return
273 }
274 metrics.WatchEvents.WithContext(req.Context()).WithLabelValues(kind.Group, kind.Version, kind.Kind).Inc()
275 isWatchListLatencyRecordingRequired := shouldRecordWatchListLatency(event)
276
277 if err := watchEncoder.Encode(event); err != nil {
278 utilruntime.HandleError(err)
279 // client disconnect.
280 return
281 }
282
283 if len(ch) == 0 {
284 flusher.Flush()
285 }
286 if isWatchListLatencyRecordingRequired {
287 metrics.RecordWatchListLatency(req.Context(), s.Scope.Resource, s.metricsScope)
288 }
289 }
290 }
291 }
292
293 // HandleWS serves a series of encoded events over a websocket connection.
294 func (s *WatchServer) HandleWS(ws *websocket.Conn) {
295 defer func() {
296 if s.MemoryAllocator != nil {
297 runtime.AllocatorPool.Put(s.MemoryAllocator)
298 }
299 }()
300
301 defer ws.Close()
302 done := make(chan struct{})
303 // ensure the connection times out
304 timeoutCh, cleanup := s.TimeoutFactory.TimeoutCh()
305 defer cleanup()
306
307 go func() {
308 defer utilruntime.HandleCrash()
309 // This blocks until the connection is closed.
310 // Client should not send anything.
311 wsstream.IgnoreReceives(ws, 0)
312 // Once the client closes, we should also close
313 close(done)
314 }()
315
316 framer := newWebsocketFramer(ws, s.UseTextFraming)
317
318 kind := s.Scope.Kind
319 watchEncoder := newWatchEncoder(context.TODO(), kind, s.EmbeddedEncoder, s.Encoder, framer, s.watchListTransformerFn)
320 ch := s.Watching.ResultChan()
321
322 for {
323 select {
324 case <-done:
325 return
326 case <-timeoutCh:
327 return
328 case event, ok := <-ch:
329 if !ok {
330 // End of results.
331 return
332 }
333
334 if err := watchEncoder.Encode(event); err != nil {
335 utilruntime.HandleError(err)
336 // client disconnect.
337 return
338 }
339 }
340 }
341 }
342
343 type websocketFramer struct {
344 ws *websocket.Conn
345 useTextFraming bool
346 }
347
348 func newWebsocketFramer(ws *websocket.Conn, useTextFraming bool) io.Writer {
349 return &websocketFramer{
350 ws: ws,
351 useTextFraming: useTextFraming,
352 }
353 }
354
355 func (w *websocketFramer) Write(p []byte) (int, error) {
356 if w.useTextFraming {
357 // bytes.Buffer::String() has a special handling of nil value, but given
358 // we're writing serialized watch events, this will never happen here.
359 if err := websocket.Message.Send(w.ws, string(p)); err != nil {
360 return 0, err
361 }
362 return len(p), nil
363 }
364 if err := websocket.Message.Send(w.ws, p); err != nil {
365 return 0, err
366 }
367 return len(p), nil
368 }
369
370 var _ io.Writer = &websocketFramer{}
371
372 func shouldRecordWatchListLatency(event watch.Event) bool {
373 if event.Type != watch.Bookmark || !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
374 return false
375 }
376 // as of today the initial-events-end annotation is added only to a single event
377 // by the watch cache and only when certain conditions are met
378 //
379 // for more please read https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/3157-watch-list
380 hasAnnotation, err := storage.HasInitialEventsEndBookmarkAnnotation(event.Object)
381 if err != nil {
382 utilruntime.HandleError(fmt.Errorf("unable to determine if the obj has the required annotation for measuring watchlist latency, obj %T: %v", event.Object, err))
383 return false
384 }
385 return hasAnnotation
386 }