Debian Codegraph @

pkg/endpoints/handlers/responsewriters/writers.go

raw
1 /*
2 Copyright 2016 The Kubernetes Authors.
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 package responsewriters
18
19 import (
20 "compress/gzip"
21 "context"
22 "encoding/json"
23 "fmt"
24 "io"
25 "net/http"
26 "strconv"
27 "strings"
28 "sync"
29 "time"
30
31 "go.opentelemetry.io/otel/attribute"
32
33 "k8s.io/apiserver/pkg/features"
34
35 "k8s.io/apimachinery/pkg/runtime"
36 "k8s.io/apimachinery/pkg/runtime/schema"
37 "k8s.io/apimachinery/pkg/util/httpstream/wsstream"
38 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
39 "k8s.io/apiserver/pkg/audit"
40 "k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
41 "k8s.io/apiserver/pkg/endpoints/metrics"
42 "k8s.io/apiserver/pkg/endpoints/request"
43 "k8s.io/apiserver/pkg/registry/rest"
44 utilfeature "k8s.io/apiserver/pkg/util/feature"
45 "k8s.io/apiserver/pkg/util/flushwriter"
46 "k8s.io/component-base/tracing"
47 )
48
49 // StreamObject performs input stream negotiation from a ResourceStreamer and writes that to the response.
50 // If the client requests a websocket upgrade, negotiate for a websocket reader protocol (because many
51 // browser clients cannot easily handle binary streaming protocols).
52 func StreamObject(statusCode int, gv schema.GroupVersion, s runtime.NegotiatedSerializer, stream rest.ResourceStreamer, w http.ResponseWriter, req *http.Request) {
53 out, flush, contentType, err := stream.InputStream(req.Context(), gv.String(), req.Header.Get("Accept"))
54 if err != nil {
55 ErrorNegotiated(err, s, gv, w, req)
56 return
57 }
58 if out == nil {
59 // No output provided - return StatusNoContent
60 w.WriteHeader(http.StatusNoContent)
61 return
62 }
63 defer out.Close()
64
65 if wsstream.IsWebSocketRequest(req) {
66 r := wsstream.NewReader(out, true, wsstream.NewDefaultReaderProtocols())
67 if err := r.Copy(w, req); err != nil {
68 utilruntime.HandleError(fmt.Errorf("error encountered while streaming results via websocket: %v", err))
69 }
70 return
71 }
72
73 if len(contentType) == 0 {
74 contentType = "application/octet-stream"
75 }
76 w.Header().Set("Content-Type", contentType)
77 w.WriteHeader(statusCode)
78 // Flush headers, if possible
79 if flusher, ok := w.(http.Flusher); ok {
80 flusher.Flush()
81 }
82 writer := w.(io.Writer)
83 if flush {
84 writer = flushwriter.Wrap(w)
85 }
86 io.Copy(writer, out)
87 }
88
89 // SerializeObject renders an object in the content type negotiated by the client using the provided encoder.
90 // The context is optional and can be nil. This method will perform optional content compression if requested by
91 // a client and the feature gate for APIResponseCompression is enabled.
92 func SerializeObject(mediaType string, encoder runtime.Encoder, hw http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
93 ctx := req.Context()
94 ctx, span := tracing.Start(ctx, "SerializeObject",
95 attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
96 attribute.String("method", req.Method),
97 attribute.String("url", req.URL.Path),
98 attribute.String("protocol", req.Proto),
99 attribute.String("mediaType", mediaType),
100 attribute.String("encoder", string(encoder.Identifier())))
101 req = req.WithContext(ctx)
102 defer span.End(5 * time.Second)
103
104 w := &deferredResponseWriter{
105 mediaType: mediaType,
106 statusCode: statusCode,
107 contentEncoding: negotiateContentEncoding(req),
108 hw: hw,
109 ctx: ctx,
110 }
111
112 err := encoder.Encode(object, w)
113 if err == nil {
114 err = w.Close()
115 if err != nil {
116 // we cannot write an error to the writer anymore as the Encode call was successful.
117 utilruntime.HandleError(fmt.Errorf("apiserver was unable to close cleanly the response writer: %v", err))
118 }
119 return
120 }
121
122 // make a best effort to write the object if a failure is detected
123 utilruntime.HandleError(fmt.Errorf("apiserver was unable to write a JSON response: %v", err))
124 status := ErrorToAPIStatus(err)
125 candidateStatusCode := int(status.Code)
126 // if the current status code is successful, allow the error's status code to overwrite it
127 if statusCode >= http.StatusOK && statusCode < http.StatusBadRequest {
128 w.statusCode = candidateStatusCode
129 }
130 output, err := runtime.Encode(encoder, status)
131 if err != nil {
132 w.mediaType = "text/plain"
133 output = []byte(fmt.Sprintf("%s: %s", status.Reason, status.Message))
134 }
135 if _, err := w.Write(output); err != nil {
136 utilruntime.HandleError(fmt.Errorf("apiserver was unable to write a fallback JSON response: %v", err))
137 }
138 w.Close()
139 }
140
141 var gzipPool = &sync.Pool{
142 New: func() interface{} {
143 gw, err := gzip.NewWriterLevel(nil, defaultGzipContentEncodingLevel)
144 if err != nil {
145 panic(err)
146 }
147 return gw
148 },
149 }
150
151 const (
152 // defaultGzipContentEncodingLevel is set to 1 which uses least CPU compared to higher levels, yet offers
153 // similar compression ratios (off by at most 1.5x, but typically within 1.1x-1.3x). For further details see -
154 // https://github.com/kubernetes/kubernetes/issues/112296
155 defaultGzipContentEncodingLevel = 1
156 // defaultGzipThresholdBytes is compared to the size of the first write from the stream
157 // (usually the entire object), and if the size is smaller no gzipping will be performed
158 // if the client requests it.
159 defaultGzipThresholdBytes = 128 * 1024
160 // Use the length of the first write to recognize streaming implementations.
161 // When streaming JSON first write is "{", while Kubernetes protobuf starts unique 4 byte header.
162 firstWriteStreamingThresholdBytes = 4
163 )
164
165 // negotiateContentEncoding returns a supported client-requested content encoding for the
166 // provided request. It will return the empty string if no supported content encoding was
167 // found or if response compression is disabled.
168 func negotiateContentEncoding(req *http.Request) string {
169 encoding := req.Header.Get("Accept-Encoding")
170 if len(encoding) == 0 {
171 return ""
172 }
173 if !utilfeature.DefaultFeatureGate.Enabled(features.APIResponseCompression) {
174 return ""
175 }
176 for len(encoding) > 0 {
177 var token string
178 if next := strings.Index(encoding, ","); next != -1 {
179 token = encoding[:next]
180 encoding = encoding[next+1:]
181 } else {
182 token = encoding
183 encoding = ""
184 }
185 switch strings.TrimSpace(token) {
186 case "gzip":
187 return "gzip"
188 }
189 }
190 return ""
191 }
192
193 type deferredResponseWriter struct {
194 mediaType string
195 statusCode int
196 contentEncoding string
197
198 hasBuffered bool
199 buffer []byte
200 hasWritten bool
201 hw http.ResponseWriter
202 w io.Writer
203 // totalBytes is the number of bytes written to `w` and does not include buffered bytes
204 totalBytes int
205 // lastWriteErr holds the error result (if any) of the last write attempt to `w`
206 lastWriteErr error
207
208 ctx context.Context
209 }
210
211 func (w *deferredResponseWriter) Write(p []byte) (n int, err error) {
212 switch {
213 case w.hasWritten:
214 // already written, cannot buffer
215 return w.unbufferedWrite(p)
216
217 case w.contentEncoding != "gzip":
218 // non-gzip, no need to buffer
219 return w.unbufferedWrite(p)
220
221 case !w.hasBuffered && len(p) > defaultGzipThresholdBytes:
222 // not yet buffered, first write is long enough to trigger gzip, no need to buffer
223 return w.unbufferedWrite(p)
224
225 case !w.hasBuffered && len(p) > firstWriteStreamingThresholdBytes:
226 // not yet buffered, first write is longer than expected for streaming scenarios that would require buffering, no need to buffer
227 return w.unbufferedWrite(p)
228
229 default:
230 if !w.hasBuffered {
231 w.hasBuffered = true
232 // Start at 80 bytes to avoid rapid reallocation of the buffer.
233 // The minimum size of a 0-item serialized list object is 80 bytes:
234 // {"kind":"List","apiVersion":"v1","metadata":{"resourceVersion":"1"},"items":[]}\n
235 w.buffer = make([]byte, 0, max(80, len(p)))
236 }
237 w.buffer = append(w.buffer, p...)
238 var err error
239 if len(w.buffer) > defaultGzipThresholdBytes {
240 // we've accumulated enough to trigger gzip, write and clear buffer
241 _, err = w.unbufferedWrite(w.buffer)
242 w.buffer = nil
243 }
244 return len(p), err
245 }
246 }
247
248 func (w *deferredResponseWriter) unbufferedWrite(p []byte) (n int, err error) {
249 defer func() {
250 w.totalBytes += n
251 w.lastWriteErr = err
252 }()
253
254 if w.hasWritten {
255 return w.w.Write(p)
256 }
257 w.hasWritten = true
258
259 hw := w.hw
260 header := hw.Header()
261 switch {
262 case w.contentEncoding == "gzip" && len(p) > defaultGzipThresholdBytes:
263 header.Set("Content-Encoding", "gzip")
264 header.Add("Vary", "Accept-Encoding")
265
266 gw := gzipPool.Get().(*gzip.Writer)
267 gw.Reset(hw)
268
269 w.w = gw
270 default:
271 w.w = hw
272 }
273
274 span := tracing.SpanFromContext(w.ctx)
275 span.AddEvent("About to start writing response",
276 attribute.String("writer", fmt.Sprintf("%T", w.w)),
277 attribute.Int("size", len(p)),
278 )
279
280 header.Set("Content-Type", w.mediaType)
281 hw.WriteHeader(w.statusCode)
282 return w.w.Write(p)
283 }
284
285 func (w *deferredResponseWriter) Close() (err error) {
286 defer func() {
287 if !w.hasWritten {
288 return
289 }
290
291 span := tracing.SpanFromContext(w.ctx)
292
293 if w.lastWriteErr != nil {
294 span.AddEvent("Write call failed",
295 attribute.Int("size", w.totalBytes),
296 attribute.String("err", w.lastWriteErr.Error()))
297 } else {
298 span.AddEvent("Write call succeeded",
299 attribute.Int("size", w.totalBytes))
300 }
301 }()
302
303 if !w.hasWritten {
304 if !w.hasBuffered {
305 return nil
306 }
307 // never reached defaultGzipThresholdBytes, no need to do the gzip writer cleanup
308 _, err := w.unbufferedWrite(w.buffer)
309 w.buffer = nil
310 return err
311 }
312
313 switch t := w.w.(type) {
314 case *gzip.Writer:
315 err = t.Close()
316 t.Reset(nil)
317 gzipPool.Put(t)
318 }
319 return err
320 }
321
322 // WriteObjectNegotiated renders an object in the content type negotiated by the client.
323 func WriteObjectNegotiated(s runtime.NegotiatedSerializer, restrictions negotiation.EndpointRestrictions, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object, listGVKInContentType bool) {
324 stream, ok := object.(rest.ResourceStreamer)
325 if ok {
326 requestInfo, _ := request.RequestInfoFrom(req.Context())
327 metrics.RecordLongRunning(req, requestInfo, metrics.APIServerComponent, func() {
328 StreamObject(statusCode, gv, s, stream, w, req)
329 })
330 return
331 }
332
333 mediaType, serializer, err := negotiation.NegotiateOutputMediaType(req, s, restrictions)
334 if err != nil {
335 // if original statusCode was not successful we need to return the original error
336 // we cannot hide it behind negotiation problems
337 if statusCode < http.StatusOK || statusCode >= http.StatusBadRequest {
338 WriteRawJSON(int(statusCode), object, w)
339 return
340 }
341 status := ErrorToAPIStatus(err)
342 WriteRawJSON(int(status.Code), status, w)
343 return
344 }
345
346 audit.LogResponseObject(req.Context(), object, gv, s)
347
348 var encoder runtime.Encoder
349 if utilfeature.DefaultFeatureGate.Enabled(features.CBORServingAndStorage) {
350 encoder = s.EncoderForVersion(runtime.UseNondeterministicEncoding(serializer.Serializer), gv)
351 } else {
352 encoder = s.EncoderForVersion(serializer.Serializer, gv)
353 }
354 request.TrackSerializeResponseObjectLatency(req.Context(), func() {
355 if listGVKInContentType {
356 SerializeObject(generateMediaTypeWithGVK(serializer.MediaType, mediaType.Convert), encoder, w, req, statusCode, object)
357 } else {
358 SerializeObject(serializer.MediaType, encoder, w, req, statusCode, object)
359 }
360 })
361 }
362
363 func generateMediaTypeWithGVK(mediaType string, gvk *schema.GroupVersionKind) string {
364 if gvk == nil {
365 return mediaType
366 }
367 if gvk.Group != "" {
368 mediaType += ";g=" + gvk.Group
369 }
370 if gvk.Version != "" {
371 mediaType += ";v=" + gvk.Version
372 }
373 if gvk.Kind != "" {
374 mediaType += ";as=" + gvk.Kind
375 }
376 return mediaType
377 }
378
379 // ErrorNegotiated renders an error to the response. Returns the HTTP status code of the error.
380 // The context is optional and may be nil.
381 func ErrorNegotiated(err error, s runtime.NegotiatedSerializer, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request) int {
382 status := ErrorToAPIStatus(err)
383 code := int(status.Code)
384 // when writing an error, check to see if the status indicates a retry after period
385 if status.Details != nil && status.Details.RetryAfterSeconds > 0 {
386 delay := strconv.Itoa(int(status.Details.RetryAfterSeconds))
387 w.Header().Set("Retry-After", delay)
388 }
389
390 if code == http.StatusNoContent {
391 w.WriteHeader(code)
392 return code
393 }
394
395 WriteObjectNegotiated(s, negotiation.DefaultEndpointRestrictions, gv, w, req, code, status, false)
396 return code
397 }
398
399 // WriteRawJSON writes a non-API object in JSON.
400 func WriteRawJSON(statusCode int, object interface{}, w http.ResponseWriter) {
401 output, err := json.MarshalIndent(object, "", " ")
402 if err != nil {
403 http.Error(w, err.Error(), http.StatusInternalServerError)
404 return
405 }
406 w.Header().Set("Content-Type", "application/json")
407 w.WriteHeader(statusCode)
408 w.Write(output)
409 }