// Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 package otlptracehttp // import "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" import ( "bytes" "compress/gzip" "context" "errors" "fmt" "io" "net" "net/http" "net/url" "strconv" "strings" "sync" "time" coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1" tracepb "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/counter" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/observ" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/otlpconfig" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp/internal/retry" ) const contentTypeProto = "application/x-protobuf" // maxResponseBodySize is the maximum number of bytes to read from a response // body. It is set to 4 MiB per the OTLP specification recommendation to // mitigate excessive memory usage caused by a misconfigured or malicious // server. If exceeded, the response is treated as a not-retryable error. // This is a variable to allow tests to override it. var maxResponseBodySize int64 = 4 * 1024 * 1024 var gzPool = sync.Pool{ New: func() any { w := gzip.NewWriter(io.Discard) return w }, } // Keep it in sync with golang's DefaultTransport from net/http! We // have our own copy to avoid handling a situation where the // DefaultTransport is overwritten with some different implementation // of http.RoundTripper or it's modified by other package. var ourTransport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, ForceAttemptHTTP2: true, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } var errInsecureEndpointWithTLS = errors.New("insecure HTTP endpoint cannot use TLS client configuration") type client struct { name string cfg otlpconfig.SignalConfig generalCfg otlpconfig.Config requestFunc retry.RequestFunc client *http.Client stopCh chan struct{} stopOnce sync.Once instID int64 inst *observ.Instrumentation } var _ otlptrace.Client = (*client)(nil) // NewClient creates a new HTTP trace client. func NewClient(opts ...Option) otlptrace.Client { cfg := otlpconfig.NewHTTPConfig(asHTTPOptions(opts)...) httpClient := cfg.Traces.HTTPClient if httpClient == nil { httpClient = &http.Client{ Transport: ourTransport, Timeout: cfg.Traces.Timeout, } if cfg.Traces.TLSCfg != nil || cfg.Traces.Proxy != nil { clonedTransport := ourTransport.Clone() httpClient.Transport = clonedTransport if cfg.Traces.TLSCfg != nil { clonedTransport.TLSClientConfig = cfg.Traces.TLSCfg } if cfg.Traces.Proxy != nil { clonedTransport.Proxy = cfg.Traces.Proxy } } } stopCh := make(chan struct{}) return &client{ name: "traces", cfg: cfg.Traces, generalCfg: cfg, requestFunc: cfg.RetryConfig.RequestFunc(evaluate), stopCh: stopCh, client: httpClient, instID: counter.NextExporterID(), } } // Start does nothing in a HTTP client. func (c *client) Start(ctx context.Context) error { if c.cfg.Insecure && c.cfg.TLSCfg != nil { return errInsecureEndpointWithTLS } // Initialize the instrumentation if not already done. // // Initialize here instead of NewClient to allow any errors to be passed // back to the caller and so that any setup of the environment variables to // enable instrumentation can be set via code. var err error if c.inst == nil { c.inst, err = observ.NewInstrumentation(c.instID, c.cfg.Endpoint) } // nothing to do select { case <-ctx.Done(): err = errors.Join(err, ctx.Err()) default: } return err } // Stop shuts down the client and interrupt any in-flight request. func (d *client) Stop(ctx context.Context) error { d.stopOnce.Do(func() { close(d.stopCh) }) select { case <-ctx.Done(): return ctx.Err() default: } return nil } // UploadTraces sends a batch of spans to the collector. func (d *client) UploadTraces(ctx context.Context, protoSpans []*tracepb.ResourceSpans) (uploadErr error) { pbRequest := &coltracepb.ExportTraceServiceRequest{ ResourceSpans: protoSpans, } rawRequest, err := proto.Marshal(pbRequest) if err != nil { return err } ctx, cancel := d.contextWithStop(ctx) defer cancel() request, err := d.newRequest(rawRequest) if err != nil { return err } var statusCode int if d.inst != nil { op := d.inst.ExportSpans(ctx, len(protoSpans)) defer func() { op.End(uploadErr, statusCode) }() } return errors.Join(uploadErr, d.requestFunc(ctx, func(ctx context.Context) error { select { case <-ctx.Done(): return ctx.Err() default: } request.reset(ctx) // nolint:gosec // URL is constructed from validated OTLP endpoint configuration resp, err := d.client.Do(request.Request) var urlErr *url.Error if errors.As(err, &urlErr) && urlErr.Temporary() { return newResponseError(http.Header{}, err) } if err != nil { return err } if resp != nil && resp.Body != nil { defer func() { if err := resp.Body.Close(); err != nil { uploadErr = errors.Join(uploadErr, err) } }() } statusCode = resp.StatusCode if statusCode >= 200 && statusCode <= 299 { // Success, do not retry. // Read the partial success message, if any. var respData bytes.Buffer if _, err := io.Copy(&respData, http.MaxBytesReader(nil, resp.Body, maxResponseBodySize)); err != nil { var maxBytesErr *http.MaxBytesError if errors.As(err, &maxBytesErr) { return fmt.Errorf("response body too large: exceeded %d bytes", maxBytesErr.Limit) } return err } if respData.Len() == 0 { return nil } if resp.Header.Get("Content-Type") == "application/x-protobuf" { var respProto coltracepb.ExportTraceServiceResponse if err := proto.Unmarshal(respData.Bytes(), &respProto); err != nil { return err } if respProto.PartialSuccess != nil { msg := respProto.PartialSuccess.GetErrorMessage() n := respProto.PartialSuccess.GetRejectedSpans() if n != 0 || msg != "" { err := internal.TracePartialSuccessError(n, msg) uploadErr = errors.Join(uploadErr, err) } } } return nil } // Error cases. // server may return a message with the response // body, so we read it to include in the error // message to be returned. It will help in // debugging the actual issue. var respData bytes.Buffer if _, err := io.Copy(&respData, http.MaxBytesReader(nil, resp.Body, maxResponseBodySize)); err != nil { var maxBytesErr *http.MaxBytesError if errors.As(err, &maxBytesErr) { return fmt.Errorf("response body too large: exceeded %d bytes", maxBytesErr.Limit) } return err } respStr := strings.TrimSpace(respData.String()) if respStr == "" { respStr = "(empty)" } bodyErr := fmt.Errorf("body: %s", respStr) switch statusCode { case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: // Retryable failure. return newResponseError(resp.Header, bodyErr) default: // Non-retryable failure. return fmt.Errorf("failed to send to %s: %s (%w)", request.URL, resp.Status, bodyErr) } })) } func (d *client) newRequest(body []byte) (request, error) { u := url.URL{Scheme: d.getScheme(), Host: d.cfg.Endpoint, Path: d.cfg.URLPath} r, err := http.NewRequestWithContext(context.Background(), http.MethodPost, u.String(), http.NoBody) if err != nil { return request{Request: r}, err } userAgent := "OTel OTLP Exporter Go/" + otlptrace.Version() r.Header.Set("User-Agent", userAgent) for k, v := range d.cfg.Headers { r.Header.Set(k, v) } r.Header.Set("Content-Type", contentTypeProto) req := request{Request: r} switch Compression(d.cfg.Compression) { case NoCompression: r.ContentLength = int64(len(body)) req.bodyReader = bodyReader(body) req.GetBody = bodyReaderErr(body) case GzipCompression: // Ensure the content length is not used. r.ContentLength = -1 r.Header.Set("Content-Encoding", "gzip") gz := gzPool.Get().(*gzip.Writer) defer gzPool.Put(gz) var b bytes.Buffer gz.Reset(&b) if _, err := gz.Write(body); err != nil { return req, err } // Close needs to be called to ensure body is fully written. if err := gz.Close(); err != nil { return req, err } req.bodyReader = bodyReader(b.Bytes()) req.GetBody = bodyReaderErr(b.Bytes()) } return req, nil } // MarshalLog is the marshaling function used by the logging system to represent this Client. func (d *client) MarshalLog() any { return struct { Type string Endpoint string Insecure bool }{ Type: "otlptracehttp", Endpoint: d.cfg.Endpoint, Insecure: d.cfg.Insecure, } } // bodyReader returns a closure returning a new reader for buf. func bodyReader(buf []byte) func() io.ReadCloser { return func() io.ReadCloser { return io.NopCloser(bytes.NewReader(buf)) } } // bodyReaderErr returns a closure returning a new reader for buf. func bodyReaderErr(buf []byte) func() (io.ReadCloser, error) { return func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf)), nil } } // request wraps an http.Request with a resettable body reader. type request struct { *http.Request // bodyReader allows the same body to be used for multiple requests. bodyReader func() io.ReadCloser } // reset reinitializes the request Body and uses ctx for the request. func (r *request) reset(ctx context.Context) { r.Body = r.bodyReader() r.Request = r.WithContext(ctx) } // retryableError represents a request failure that can be retried. type retryableError struct { throttle int64 err error } // newResponseError returns a retryableError and will extract any explicit // throttle delay contained in headers. The returned error wraps wrapped // if it is not nil. func newResponseError(header http.Header, wrapped error) error { var rErr retryableError if s, ok := header["Retry-After"]; ok { if t, err := strconv.ParseInt(s[0], 10, 64); err == nil { rErr.throttle = t } } rErr.err = wrapped return rErr } func (e retryableError) Error() string { if e.err != nil { return "retry-able request failure: " + e.err.Error() } return "retry-able request failure" } func (e retryableError) Unwrap() error { return e.err } func (e retryableError) As(target any) bool { if e.err == nil { return false } switch v := target.(type) { case **retryableError: *v = &e return true default: return false } } // evaluate returns if err is retry-able. If it is and it includes an explicit // throttling delay, that delay is also returned. func evaluate(err error) (bool, time.Duration) { if err == nil { return false, 0 } // Do not use errors.As here, this should only be flattened one layer. If // there are several chained errors, all the errors above it will be // discarded if errors.As is used instead. rErr, ok := err.(retryableError) //nolint:errorlint if !ok { return false, 0 } return true, time.Duration(rErr.throttle) } func (d *client) getScheme() string { if d.cfg.Insecure { return "http" } return "https" } func (d *client) contextWithStop(ctx context.Context) (context.Context, context.CancelFunc) { // Unify the parent context Done signal with the client's stop // channel. ctx, cancel := context.WithCancel(ctx) go func(ctx context.Context, cancel context.CancelFunc) { select { case <-ctx.Done(): // Nothing to do, either cancelled or deadline // happened. case <-d.stopCh: cancel() } }(ctx, cancel) return ctx, cancel }