Debian Codegraph @

exporters/otlp/otlplog/otlploghttp/client.go

raw
1 // Copyright The OpenTelemetry Authors
2 // SPDX-License-Identifier: Apache-2.0
3
4 package otlploghttp // import "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
5
6 import (
7 "bytes"
8 "compress/gzip"
9 "context"
10 "errors"
11 "fmt"
12 "io"
13 "net"
14 "net/http"
15 "net/url"
16 "strconv"
17 "strings"
18 "sync"
19 "sync/atomic"
20 "time"
21
22 collogpb "go.opentelemetry.io/proto/otlp/collector/logs/v1"
23 logpb "go.opentelemetry.io/proto/otlp/logs/v1"
24 "google.golang.org/protobuf/proto"
25
26 "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp/internal"
27 "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp/internal/observ"
28 "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp/internal/retry"
29 )
30
31 type client struct {
32 uploadLogs func(context.Context, []*logpb.ResourceLogs) error
33 }
34
35 func (c *client) UploadLogs(ctx context.Context, rl []*logpb.ResourceLogs) error {
36 if c.uploadLogs != nil {
37 return c.uploadLogs(ctx, rl)
38 }
39 return nil
40 }
41
42 func newNoopClient() *client {
43 return &client{}
44 }
45
46 var exporterN atomic.Int64
47
48 var errInsecureEndpointWithTLS = errors.New("insecure HTTP endpoint cannot use TLS client configuration")
49
50 // maxResponseBodySize is the maximum number of bytes to read from a response
51 // body. It is set to 4 MiB per the OTLP specification recommendation to
52 // mitigate excessive memory usage caused by a misconfigured or malicious
53 // server. If exceeded, the response is treated as a not-retryable error.
54 // This is a variable to allow tests to override it.
55 var maxResponseBodySize int64 = 4 * 1024 * 1024
56
57 // nextExporterID returns the next unique ID for an exporter.
58 func nextExporterID() int64 {
59 const inc = 1
60 return exporterN.Add(inc) - inc
61 }
62
63 // newHTTPClient creates a new HTTP log client.
64 func newHTTPClient(ctx context.Context, cfg config) (*client, error) {
65 if cfg.insecure.Value && cfg.tlsCfg.Value != nil {
66 return nil, errInsecureEndpointWithTLS
67 }
68
69 hc := cfg.httpClient
70 if hc == nil {
71 hc = &http.Client{
72 Transport: ourTransport,
73 Timeout: cfg.timeout.Value,
74 }
75
76 if cfg.tlsCfg.Value != nil || cfg.proxy.Value != nil {
77 clonedTransport := ourTransport.Clone()
78 hc.Transport = clonedTransport
79
80 if cfg.tlsCfg.Value != nil {
81 clonedTransport.TLSClientConfig = cfg.tlsCfg.Value
82 }
83 if cfg.proxy.Value != nil {
84 clonedTransport.Proxy = cfg.proxy.Value
85 }
86 }
87 }
88
89 u := &url.URL{
90 Scheme: "https",
91 Host: cfg.endpoint.Value,
92 Path: cfg.path.Value,
93 }
94 if cfg.insecure.Value {
95 u.Scheme = "http"
96 }
97 // Body is set when this is cloned during upload.
98 req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), http.NoBody)
99 if err != nil {
100 return nil, err
101 }
102
103 userAgent := "OTel Go OTLP over HTTP/protobuf logs exporter/" + Version()
104 req.Header.Set("User-Agent", userAgent)
105
106 if n := len(cfg.headers.Value); n > 0 {
107 for k, v := range cfg.headers.Value {
108 req.Header.Set(k, v)
109 }
110 }
111 req.Header.Set("Content-Type", "application/x-protobuf")
112
113 c := &httpClient{
114 compression: cfg.compression.Value,
115 req: req,
116 requestFunc: cfg.retryCfg.Value.RequestFunc(evaluate),
117 client: hc,
118 }
119
120 id := nextExporterID()
121 c.inst, err = observ.NewInstrumentation(id, cfg.endpoint.Value)
122
123 return &client{uploadLogs: c.uploadLogs}, err
124 }
125
126 type httpClient struct {
127 // req is cloned for every upload the client makes.
128 req *http.Request
129 compression Compression
130 requestFunc retry.RequestFunc
131 client *http.Client
132
133 inst *observ.Instrumentation
134 }
135
136 // Keep it in sync with golang's DefaultTransport from net/http! We
137 // have our own copy to avoid handling a situation where the
138 // DefaultTransport is overwritten with some different implementation
139 // of http.RoundTripper or it's modified by another package.
140 var ourTransport = &http.Transport{
141 Proxy: http.ProxyFromEnvironment,
142 DialContext: (&net.Dialer{
143 Timeout: 30 * time.Second,
144 KeepAlive: 30 * time.Second,
145 }).DialContext,
146 ForceAttemptHTTP2: true,
147 MaxIdleConns: 100,
148 IdleConnTimeout: 90 * time.Second,
149 TLSHandshakeTimeout: 10 * time.Second,
150 ExpectContinueTimeout: 1 * time.Second,
151 }
152
153 func (c *httpClient) uploadLogs(ctx context.Context, data []*logpb.ResourceLogs) (uploadErr error) {
154 // The Exporter synchronizes access to client methods. This is not called
155 // after the Exporter is shutdown. Only thing to do here is send data.
156
157 pbRequest := &collogpb.ExportLogsServiceRequest{ResourceLogs: data}
158 body, err := proto.Marshal(pbRequest)
159 if err != nil {
160 return err
161 }
162 request, err := c.newRequest(ctx, body)
163 if err != nil {
164 return err
165 }
166
167 var statusCode int
168 if c.inst != nil {
169 op := c.inst.ExportLogs(ctx, int64(len(data)))
170 defer func() { op.End(uploadErr, statusCode) }()
171 }
172
173 return errors.Join(uploadErr, c.requestFunc(ctx, func(iCtx context.Context) error {
174 select {
175 case <-iCtx.Done():
176 return iCtx.Err()
177 default:
178 }
179
180 request.reset(iCtx)
181 // nolint:gosec // URL is constructed from validated OTLP endpoint configuration
182 resp, err := c.client.Do(request.Request)
183 var urlErr *url.Error
184 if errors.As(err, &urlErr) && urlErr.Temporary() {
185 return newResponseError(http.Header{}, err)
186 }
187 if err != nil {
188 return err
189 }
190 if resp != nil && resp.Body != nil {
191 defer func() {
192 if err := resp.Body.Close(); err != nil {
193 uploadErr = errors.Join(uploadErr, err)
194 }
195 }()
196 }
197
198 statusCode = resp.StatusCode
199 if sc := resp.StatusCode; sc >= 200 && sc <= 299 {
200 // Success, do not retry.
201
202 // Read the partial success message, if any.
203 var respData bytes.Buffer
204 if _, err := io.Copy(&respData, http.MaxBytesReader(nil, resp.Body, maxResponseBodySize)); err != nil {
205 var maxBytesErr *http.MaxBytesError
206 if errors.As(err, &maxBytesErr) {
207 return fmt.Errorf("response body too large: exceeded %d bytes", maxBytesErr.Limit)
208 }
209 return err
210 }
211 if respData.Len() == 0 {
212 return nil
213 }
214
215 if resp.Header.Get("Content-Type") == "application/x-protobuf" {
216 var respProto collogpb.ExportLogsServiceResponse
217 if err := proto.Unmarshal(respData.Bytes(), &respProto); err != nil {
218 return err
219 }
220
221 if respProto.PartialSuccess != nil {
222 msg := respProto.PartialSuccess.GetErrorMessage()
223 n := respProto.PartialSuccess.GetRejectedLogRecords()
224 if n != 0 || msg != "" {
225 err := internal.LogPartialSuccessError(n, msg)
226 uploadErr = errors.Join(uploadErr, err)
227 }
228 }
229 }
230 return nil
231 }
232 // Error cases.
233
234 // server may return a message with the response
235 // body, so we read it to include in the error
236 // message to be returned. It will help in
237 // debugging the actual issue.
238 var respData bytes.Buffer
239 if _, err := io.Copy(&respData, http.MaxBytesReader(nil, resp.Body, maxResponseBodySize)); err != nil {
240 var maxBytesErr *http.MaxBytesError
241 if errors.As(err, &maxBytesErr) {
242 return fmt.Errorf("response body too large: exceeded %d bytes", maxBytesErr.Limit)
243 }
244 return err
245 }
246 respStr := strings.TrimSpace(respData.String())
247 if respStr == "" {
248 respStr = "(empty)"
249 }
250 bodyErr := fmt.Errorf("body: %s", respStr)
251
252 switch resp.StatusCode {
253 case http.StatusTooManyRequests,
254 http.StatusBadGateway,
255 http.StatusServiceUnavailable,
256 http.StatusGatewayTimeout:
257 // Retryable failure.
258 return newResponseError(resp.Header, bodyErr)
259 default:
260 // Non-retryable failure.
261 return fmt.Errorf("failed to send logs to %s: %s (%w)", request.URL, resp.Status, bodyErr)
262 }
263 }))
264 }
265
266 var gzPool = sync.Pool{
267 New: func() any {
268 w := gzip.NewWriter(io.Discard)
269 return w
270 },
271 }
272
273 func (c *httpClient) newRequest(ctx context.Context, body []byte) (request, error) {
274 r := c.req.Clone(ctx)
275 req := request{Request: r}
276
277 switch c.compression {
278 case NoCompression:
279 r.ContentLength = int64(len(body))
280 req.bodyReader = bodyReader(body)
281 req.GetBody = bodyReaderErr(body)
282 case GzipCompression:
283 // Ensure the content length is not used.
284 r.ContentLength = -1
285 r.Header.Set("Content-Encoding", "gzip")
286
287 gz := gzPool.Get().(*gzip.Writer)
288 defer gzPool.Put(gz)
289
290 var b bytes.Buffer
291 gz.Reset(&b)
292
293 if _, err := gz.Write(body); err != nil {
294 return req, err
295 }
296 // Close needs to be called to ensure body is fully written.
297 if err := gz.Close(); err != nil {
298 return req, err
299 }
300
301 req.bodyReader = bodyReader(b.Bytes())
302 req.GetBody = bodyReaderErr(body)
303 }
304
305 return req, nil
306 }
307
308 // bodyReader returns a closure returning a new reader for buf.
309 func bodyReader(buf []byte) func() io.ReadCloser {
310 return func() io.ReadCloser {
311 return io.NopCloser(bytes.NewReader(buf))
312 }
313 }
314
315 // bodyReaderErr returns a closure returning a new reader for buf.
316 func bodyReaderErr(buf []byte) func() (io.ReadCloser, error) {
317 return func() (io.ReadCloser, error) {
318 return io.NopCloser(bytes.NewReader(buf)), nil
319 }
320 }
321
322 // request wraps an http.Request with a resettable body reader.
323 type request struct {
324 *http.Request
325
326 // bodyReader allows the same body to be used for multiple requests.
327 bodyReader func() io.ReadCloser
328 }
329
330 // reset reinitializes the request Body and uses ctx for the request.
331 func (r *request) reset(ctx context.Context) {
332 r.Body = r.bodyReader()
333 r.Request = r.WithContext(ctx)
334 }
335
336 // retryableError represents a request failure that can be retried.
337 type retryableError struct {
338 throttle int64
339 err error
340 }
341
342 // newResponseError returns a retryableError and will extract any explicit
343 // throttle delay contained in headers. The returned error wraps wrapped
344 // if it is not nil.
345 func newResponseError(header http.Header, wrapped error) error {
346 var rErr retryableError
347 if v := header.Get("Retry-After"); v != "" {
348 if t, err := strconv.ParseInt(v, 10, 64); err == nil {
349 rErr.throttle = t
350 }
351 }
352
353 rErr.err = wrapped
354 return rErr
355 }
356
357 func (e retryableError) Error() string {
358 if e.err != nil {
359 return fmt.Sprintf("retry-able request failure: %v", e.err.Error())
360 }
361
362 return "retry-able request failure"
363 }
364
365 func (e retryableError) Unwrap() error {
366 return e.err
367 }
368
369 func (e retryableError) As(target any) bool {
370 if e.err == nil {
371 return false
372 }
373
374 switch v := target.(type) {
375 case **retryableError:
376 *v = &e
377 return true
378 default:
379 return false
380 }
381 }
382
383 // evaluate returns if err is retry-able. If it is and it includes an explicit
384 // throttling delay, that delay is also returned.
385 func evaluate(err error) (bool, time.Duration) {
386 if err == nil {
387 return false, 0
388 }
389
390 // Do not use errors.As here, this should only be flattened one layer. If
391 // there are several chained errors, all the errors above it will be
392 // discarded if errors.As is used instead.
393 rErr, ok := err.(retryableError) //nolint:errorlint
394 if !ok {
395 return false, 0
396 }
397
398 return true, time.Duration(rErr.throttle)
399 }