Debian Codegraph @

storage/grpc_metrics.go

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 // Copyright 2024 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 package storage
16
17 import (
18 "context"
19 "errors"
20 "fmt"
21 "strings"
22 "time"
23
24 mexporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric"
25 "github.com/google/uuid"
26 "go.opentelemetry.io/contrib/detectors/gcp"
27 "go.opentelemetry.io/otel/attribute"
28 "go.opentelemetry.io/otel/sdk/metric"
29 "go.opentelemetry.io/otel/sdk/metric/metricdata"
30 "go.opentelemetry.io/otel/sdk/resource"
31 "google.golang.org/api/option"
32 "google.golang.org/grpc"
33 "google.golang.org/grpc/experimental/stats"
34 "google.golang.org/grpc/stats/opentelemetry"
35 )
36
37 const (
38 monitoredResourceName = "storage.googleapis.com/Client"
39 metricPrefix = "storage.googleapis.com/client/"
40 )
41
42 // Added to help with tests
43 type storageMonitoredResource struct {
44 project string
45 api string
46 location string
47 instance string
48 cloudPlatform string
49 host string
50 resource *resource.Resource
51 }
52
53 func (smr *storageMonitoredResource) exporter() (metric.Exporter, error) {
54 exporter, err := mexporter.New(
55 mexporter.WithProjectID(smr.project),
56 mexporter.WithMetricDescriptorTypeFormatter(metricFormatter),
57 mexporter.WithCreateServiceTimeSeries(),
58 mexporter.WithMonitoredResourceDescription(monitoredResourceName, []string{"project_id", "location", "cloud_platform", "host_id", "instance_id", "api"}),
59 )
60 if err != nil {
61 return nil, fmt.Errorf("storage: creating metrics exporter: %w", err)
62 }
63 return exporter, nil
64 }
65
66 func newStorageMonitoredResource(ctx context.Context, project, api string, opts ...resource.Option) (*storageMonitoredResource, error) {
67 detectedAttrs, err := resource.New(ctx, opts...)
68 if err != nil {
69 return nil, err
70 }
71 smr := &storageMonitoredResource{
72 instance: uuid.New().String(),
73 api: api,
74 project: project,
75 }
76 s := detectedAttrs.Set()
77 // Attempt to use resource detector project id if project id wasn't
78 // identified using ADC as a last resort. Otherwise metrics cannot be started.
79 if p, present := s.Value("cloud.account.id"); present && smr.project == "" {
80 smr.project = p.AsString()
81 } else if !present && smr.project == "" {
82 return nil, errors.New("google cloud project is required to start client-side metrics")
83 }
84 if v, ok := s.Value("cloud.region"); ok {
85 smr.location = v.AsString()
86 } else {
87 smr.location = "global"
88 }
89 if v, ok := s.Value("cloud.platform"); ok {
90 smr.cloudPlatform = v.AsString()
91 } else {
92 smr.cloudPlatform = "unknown"
93 }
94 if v, ok := s.Value("host.id"); ok {
95 smr.host = v.AsString()
96 } else if v, ok := s.Value("faas.id"); ok {
97 smr.host = v.AsString()
98 } else {
99 smr.host = "unknown"
100 }
101 smr.resource, err = resource.New(ctx, resource.WithAttributes([]attribute.KeyValue{
102 {Key: "gcp.resource_type", Value: attribute.StringValue(monitoredResourceName)},
103 {Key: "project_id", Value: attribute.StringValue(smr.project)},
104 {Key: "api", Value: attribute.StringValue(smr.api)},
105 {Key: "instance_id", Value: attribute.StringValue(smr.instance)},
106 {Key: "location", Value: attribute.StringValue(smr.location)},
107 {Key: "cloud_platform", Value: attribute.StringValue(smr.cloudPlatform)},
108 {Key: "host_id", Value: attribute.StringValue(smr.host)},
109 }...))
110 if err != nil {
111 return nil, err
112 }
113 return smr, nil
114 }
115
116 type metricsContext struct {
117 // client options passed to gRPC channels
118 clientOpts []option.ClientOption
119 // instance of metric reader used by gRPC client-side metrics
120 provider *metric.MeterProvider
121 // clean func to call when closing gRPC client
122 close func()
123 }
124
125 type metricsConfig struct {
126 project string
127 interval time.Duration
128 customExporter *metric.Exporter
129 meterProvider *metric.MeterProvider
130 manualReader *metric.ManualReader // used by tests
131 disableExporter bool // used by tests disables exports
132 resourceOpts []resource.Option // used by tests
133 }
134
135 func newGRPCMetricContext(ctx context.Context, cfg metricsConfig) (*metricsContext, error) {
136 var exporter metric.Exporter
137 meterOpts := []metric.Option{}
138 if cfg.customExporter == nil {
139 var ropts []resource.Option
140 if cfg.resourceOpts != nil {
141 ropts = cfg.resourceOpts
142 } else {
143 ropts = []resource.Option{resource.WithDetectors(gcp.NewDetector())}
144 }
145 smr, err := newStorageMonitoredResource(ctx, cfg.project, "grpc", ropts...)
146 if err != nil {
147 return nil, err
148 }
149 exporter, err = smr.exporter()
150 if err != nil {
151 return nil, err
152 }
153 meterOpts = append(meterOpts, metric.WithResource(smr.resource))
154 } else {
155 exporter = *cfg.customExporter
156 }
157 interval := time.Minute
158 if cfg.interval > 0 {
159 interval = cfg.interval
160 }
161 meterOpts = append(meterOpts,
162 // Metric views update histogram boundaries to be relevant to GCS
163 // otherwise default OTel histogram boundaries are used.
164 metric.WithView(
165 createHistogramView("grpc.client.attempt.duration", latencyHistogramBoundaries()),
166 createHistogramView("grpc.client.attempt.rcvd_total_compressed_message_size", sizeHistogramBoundaries()),
167 createHistogramView("grpc.client.attempt.sent_total_compressed_message_size", sizeHistogramBoundaries())),
168 )
169 if cfg.manualReader != nil {
170 meterOpts = append(meterOpts, metric.WithReader(cfg.manualReader))
171 }
172 if !cfg.disableExporter {
173 meterOpts = append(meterOpts, metric.WithReader(
174 metric.NewPeriodicReader(&exporterLogSuppressor{Exporter: exporter}, metric.WithInterval(interval))))
175 }
176 provider := cfg.meterProvider
177 if provider == nil {
178 provider = metric.NewMeterProvider(meterOpts...)
179 }
180 mo := opentelemetry.MetricsOptions{
181 MeterProvider: provider,
182 Metrics: stats.NewMetrics(
183 "grpc.client.attempt.started",
184 "grpc.client.attempt.duration",
185 "grpc.client.attempt.sent_total_compressed_message_size",
186 "grpc.client.attempt.rcvd_total_compressed_message_size",
187 "grpc.client.call.duration",
188 "grpc.lb.wrr.rr_fallback",
189 "grpc.lb.wrr.endpoint_weight_not_yet_usable",
190 "grpc.lb.wrr.endpoint_weight_stale",
191 "grpc.lb.wrr.endpoint_weights",
192 "grpc.lb.rls.cache_entries",
193 "grpc.lb.rls.cache_size",
194 "grpc.lb.rls.default_target_picks",
195 "grpc.lb.rls.target_picks",
196 "grpc.lb.rls.failed_picks",
197 ),
198 OptionalLabels: []string{"grpc.lb.locality"},
199 }
200 opts := []option.ClientOption{
201 option.WithGRPCDialOption(
202 opentelemetry.DialOption(opentelemetry.Options{MetricsOptions: mo})),
203 option.WithGRPCDialOption(
204 grpc.WithDefaultCallOptions(grpc.StaticMethodCallOption{})),
205 }
206 return &metricsContext{
207 clientOpts: opts,
208 provider: provider,
209 close: func() {
210 provider.Shutdown(ctx)
211 },
212 }, nil
213 }
214
215 // Silences permission errors after initial error is emitted to prevent
216 // chatty logs.
217 type exporterLogSuppressor struct {
218 metric.Exporter
219 emittedFailure bool
220 }
221
222 // Implements OTel SDK metric.Exporter interface to prevent noisy logs from
223 // lack of credentials after initial failure.
224 // https://pkg.go.dev/go.opentelemetry.io/otel/sdk/[email protected]#Exporter
225 func (e *exporterLogSuppressor) Export(ctx context.Context, rm *metricdata.ResourceMetrics) error {
226 if err := e.Exporter.Export(ctx, rm); err != nil && !e.emittedFailure {
227 if strings.Contains(err.Error(), "PermissionDenied") {
228 e.emittedFailure = true
229 return fmt.Errorf("gRPC metrics failed due permission issue: %w", err)
230 }
231 return err
232 }
233 return nil
234 }
235
236 func latencyHistogramBoundaries() []float64 {
237 boundaries := []float64{}
238 boundary := 0.0
239 increment := 0.002
240 // 2ms buckets for first 100ms, so we can have higher resolution for uploads and downloads in the 100 KiB range
241 for i := 0; i < 50; i++ {
242 boundaries = append(boundaries, boundary)
243 // increment by 2ms
244 boundary += increment
245 }
246 // For the remaining buckets do 10 10ms, 10 20ms, and so on, up until 5 minutes
247 for i := 0; i < 150 && boundary < 300; i++ {
248 boundaries = append(boundaries, boundary)
249 if i != 0 && i%10 == 0 {
250 increment *= 2
251 }
252 boundary += increment
253 }
254 return boundaries
255 }
256
257 func sizeHistogramBoundaries() []float64 {
258 kb := 1024.0
259 mb := 1024.0 * kb
260 gb := 1024.0 * mb
261 boundaries := []float64{}
262 boundary := 0.0
263 increment := 128 * kb
264 // 128 KiB increments up to 4MiB, then exponential growth
265 for len(boundaries) < 200 && boundary <= 16*gb {
266 boundaries = append(boundaries, boundary)
267 boundary += increment
268 if boundary >= 4*mb {
269 increment *= 2
270 }
271 }
272 return boundaries
273 }
274
275 func createHistogramView(name string, boundaries []float64) metric.View {
276 return metric.NewView(metric.Instrument{
277 Name: name,
278 Kind: metric.InstrumentKindHistogram,
279 }, metric.Stream{
280 Name: name,
281 Aggregation: metric.AggregationExplicitBucketHistogram{Boundaries: boundaries},
282 })
283 }
284
285 func metricFormatter(m metricdata.Metrics) string {
286 return metricPrefix + strings.ReplaceAll(string(m.Name), ".", "/")
287 }