Debian Codegraph @

statsd/statsd.go

raw
1 // Copyright 2013 Ooyala, Inc.
2
3 /*
4 Package statsd provides a Go dogstatsd client. Dogstatsd extends the popular statsd,
5 adding tags and histograms and pushing upstream to Datadog.
6
7 Refer to http://docs.datadoghq.com/guides/dogstatsd/ for information about DogStatsD.
8
9 Example Usage:
10
11 // Create the client
12 c, err := statsd.New("127.0.0.1:8125")
13 if err != nil {
14 log.Fatal(err)
15 }
16 // Prefix every metric with the app name
17 c.Namespace = "flubber."
18 // Send the EC2 availability zone as a tag with every metric
19 c.Tags = append(c.Tags, "us-east-1a")
20 err = c.Gauge("request.duration", 1.2, nil, 1)
21
22 statsd is based on go-statsd-client.
23 */
24 package statsd
25
26 import (
27 "bytes"
28 "errors"
29 "fmt"
30 "io"
31 "math/rand"
32 "strconv"
33 "strings"
34 "sync"
35 "time"
36 )
37
38 /*
39 OptimalPayloadSize defines the optimal payload size for a UDP datagram, 1432 bytes
40 is optimal for regular networks with an MTU of 1500 so datagrams don't get
41 fragmented. It's generally recommended not to fragment UDP datagrams as losing
42 a single fragment will cause the entire datagram to be lost.
43
44 This can be increased if your network has a greater MTU or you don't mind UDP
45 datagrams getting fragmented. The practical limit is MaxUDPPayloadSize
46 */
47 const OptimalPayloadSize = 1432
48
49 /*
50 MaxUDPPayloadSize defines the maximum payload size for a UDP datagram.
51 Its value comes from the calculation: 65535 bytes Max UDP datagram size -
52 8byte UDP header - 60byte max IP headers
53 any number greater than that will see frames being cut out.
54 */
55 const MaxUDPPayloadSize = 65467
56
57 /*
58 UnixAddressPrefix holds the prefix to use to enable Unix Domain Socket
59 traffic instead of UDP.
60 */
61 const UnixAddressPrefix = "unix://"
62
63 /*
64 Stat suffixes
65 */
66 var (
67 gaugeSuffix = []byte("|g")
68 countSuffix = []byte("|c")
69 histogramSuffix = []byte("|h")
70 distributionSuffix = []byte("|d")
71 decrSuffix = []byte("-1|c")
72 incrSuffix = []byte("1|c")
73 setSuffix = []byte("|s")
74 timingSuffix = []byte("|ms")
75 )
76
77 // A statsdWriter offers a standard interface regardless of the underlying
78 // protocol. For now UDS and UPD writers are available.
79 type statsdWriter interface {
80 Write(data []byte) (n int, err error)
81 SetWriteTimeout(time.Duration) error
82 Close() error
83 }
84
85 // A Client is a handle for sending messages to dogstatsd. It is safe to
86 // use one Client from multiple goroutines simultaneously.
87 type Client struct {
88 // Writer handles the underlying networking protocol
89 writer statsdWriter
90 // Namespace to prepend to all statsd calls
91 Namespace string
92 // Tags are global tags to be added to every statsd call
93 Tags []string
94 // skipErrors turns off error passing and allows UDS to emulate UDP behaviour
95 SkipErrors bool
96 // BufferLength is the length of the buffer in commands.
97 bufferLength int
98 flushTime time.Duration
99 commands []string
100 buffer bytes.Buffer
101 stop chan struct{}
102 sync.Mutex
103 }
104
105 // New returns a pointer to a new Client given an addr in the format "hostname:port" or
106 // "unix:///path/to/socket".
107 func New(addr string) (*Client, error) {
108 if strings.HasPrefix(addr, UnixAddressPrefix) {
109 w, err := newUdsWriter(addr[len(UnixAddressPrefix)-1:])
110 if err != nil {
111 return nil, err
112 }
113 return NewWithWriter(w)
114 }
115 w, err := newUDPWriter(addr)
116 if err != nil {
117 return nil, err
118 }
119 return NewWithWriter(w)
120 }
121
122 // NewWithWriter creates a new Client with given writer. Writer is a
123 // io.WriteCloser + SetWriteTimeout(time.Duration) error
124 func NewWithWriter(w statsdWriter) (*Client, error) {
125 client := &Client{writer: w, SkipErrors: false}
126 return client, nil
127 }
128
129 // NewBuffered returns a Client that buffers its output and sends it in chunks.
130 // Buflen is the length of the buffer in number of commands.
131 func NewBuffered(addr string, buflen int) (*Client, error) {
132 client, err := New(addr)
133 if err != nil {
134 return nil, err
135 }
136 client.bufferLength = buflen
137 client.commands = make([]string, 0, buflen)
138 client.flushTime = time.Millisecond * 100
139 client.stop = make(chan struct{}, 1)
140 go client.watch()
141 return client, nil
142 }
143
144 // format a message from its name, value, tags and rate. Also adds global
145 // namespace and tags.
146 func (c *Client) format(name string, value interface{}, suffix []byte, tags []string, rate float64) string {
147 var buf bytes.Buffer
148 if c.Namespace != "" {
149 buf.WriteString(c.Namespace)
150 }
151 buf.WriteString(name)
152 buf.WriteString(":")
153
154 switch val := value.(type) {
155 case float64:
156 buf.Write(strconv.AppendFloat([]byte{}, val, 'f', 6, 64))
157
158 case int64:
159 buf.Write(strconv.AppendInt([]byte{}, val, 10))
160
161 case string:
162 buf.WriteString(val)
163
164 default:
165 // do nothing
166 }
167 buf.Write(suffix)
168
169 if rate < 1 {
170 buf.WriteString(`|@`)
171 buf.WriteString(strconv.FormatFloat(rate, 'f', -1, 64))
172 }
173
174 writeTagString(&buf, c.Tags, tags)
175
176 return buf.String()
177 }
178
179 // SetWriteTimeout allows the user to set a custom UDS write timeout. Not supported for UDP.
180 func (c *Client) SetWriteTimeout(d time.Duration) error {
181 if c == nil {
182 return nil
183 }
184 return c.writer.SetWriteTimeout(d)
185 }
186
187 func (c *Client) watch() {
188 ticker := time.NewTicker(c.flushTime)
189
190 for {
191 select {
192 case <-ticker.C:
193 c.Lock()
194 if len(c.commands) > 0 {
195 // FIXME: eating error here
196 c.flushLocked()
197 }
198 c.Unlock()
199 case <-c.stop:
200 ticker.Stop()
201 return
202 }
203 }
204 }
205
206 func (c *Client) append(cmd string) error {
207 c.Lock()
208 defer c.Unlock()
209 c.commands = append(c.commands, cmd)
210 // if we should flush, lets do it
211 if len(c.commands) == c.bufferLength {
212 if err := c.flushLocked(); err != nil {
213 return err
214 }
215 }
216 return nil
217 }
218
219 func (c *Client) joinMaxSize(cmds []string, sep string, maxSize int) ([][]byte, []int) {
220 c.buffer.Reset() //clear buffer
221
222 var frames [][]byte
223 var ncmds []int
224 sepBytes := []byte(sep)
225 sepLen := len(sep)
226
227 elem := 0
228 for _, cmd := range cmds {
229 needed := len(cmd)
230
231 if elem != 0 {
232 needed = needed + sepLen
233 }
234
235 if c.buffer.Len()+needed <= maxSize {
236 if elem != 0 {
237 c.buffer.Write(sepBytes)
238 }
239 c.buffer.WriteString(cmd)
240 elem++
241 } else {
242 frames = append(frames, copyAndResetBuffer(&c.buffer))
243 ncmds = append(ncmds, elem)
244 // if cmd is bigger than maxSize it will get flushed on next loop
245 c.buffer.WriteString(cmd)
246 elem = 1
247 }
248 }
249
250 //add whatever is left! if there's actually something
251 if c.buffer.Len() > 0 {
252 frames = append(frames, copyAndResetBuffer(&c.buffer))
253 ncmds = append(ncmds, elem)
254 }
255
256 return frames, ncmds
257 }
258
259 func copyAndResetBuffer(buf *bytes.Buffer) []byte {
260 tmpBuf := make([]byte, buf.Len())
261 copy(tmpBuf, buf.Bytes())
262 buf.Reset()
263 return tmpBuf
264 }
265
266 // Flush forces a flush of the pending commands in the buffer
267 func (c *Client) Flush() error {
268 if c == nil {
269 return nil
270 }
271 c.Lock()
272 defer c.Unlock()
273 return c.flushLocked()
274 }
275
276 // flush the commands in the buffer. Lock must be held by caller.
277 func (c *Client) flushLocked() error {
278 frames, flushable := c.joinMaxSize(c.commands, "\n", OptimalPayloadSize)
279 var err error
280 cmdsFlushed := 0
281 for i, data := range frames {
282 _, e := c.writer.Write(data)
283 if e != nil {
284 err = e
285 break
286 }
287 cmdsFlushed += flushable[i]
288 }
289
290 // clear the slice with a slice op, doesn't realloc
291 if cmdsFlushed == len(c.commands) {
292 c.commands = c.commands[:0]
293 } else {
294 //this case will cause a future realloc...
295 // drop problematic command though (sorry).
296 c.commands = c.commands[cmdsFlushed+1:]
297 }
298 return err
299 }
300
301 func (c *Client) sendMsg(msg string) error {
302 // return an error if message is bigger than MaxUDPPayloadSize
303 if len(msg) > MaxUDPPayloadSize {
304 return errors.New("message size exceeds MaxUDPPayloadSize")
305 }
306
307 // if this client is buffered, then we'll just append this
308 if c.bufferLength > 0 {
309 return c.append(msg)
310 }
311
312 _, err := c.writer.Write([]byte(msg))
313
314 if c.SkipErrors {
315 return nil
316 }
317 return err
318 }
319
320 // send handles sampling and sends the message over UDP. It also adds global namespace prefixes and tags.
321 func (c *Client) send(name string, value interface{}, suffix []byte, tags []string, rate float64) error {
322 if c == nil {
323 return nil
324 }
325 if rate < 1 && rand.Float64() > rate {
326 return nil
327 }
328 data := c.format(name, value, suffix, tags, rate)
329 return c.sendMsg(data)
330 }
331
332 // Gauge measures the value of a metric at a particular time.
333 func (c *Client) Gauge(name string, value float64, tags []string, rate float64) error {
334 return c.send(name, value, gaugeSuffix, tags, rate)
335 }
336
337 // Count tracks how many times something happened per second.
338 func (c *Client) Count(name string, value int64, tags []string, rate float64) error {
339 return c.send(name, value, countSuffix, tags, rate)
340 }
341
342 // Histogram tracks the statistical distribution of a set of values on each host.
343 func (c *Client) Histogram(name string, value float64, tags []string, rate float64) error {
344 return c.send(name, value, histogramSuffix, tags, rate)
345 }
346
347 // Distribution tracks the statistical distribution of a set of values across your infrastructure.
348 func (c *Client) Distribution(name string, value float64, tags []string, rate float64) error {
349 return c.send(name, value, distributionSuffix, tags, rate)
350 }
351
352 // Decr is just Count of -1
353 func (c *Client) Decr(name string, tags []string, rate float64) error {
354 return c.send(name, nil, decrSuffix, tags, rate)
355 }
356
357 // Incr is just Count of 1
358 func (c *Client) Incr(name string, tags []string, rate float64) error {
359 return c.send(name, nil, incrSuffix, tags, rate)
360 }
361
362 // Set counts the number of unique elements in a group.
363 func (c *Client) Set(name string, value string, tags []string, rate float64) error {
364 return c.send(name, value, setSuffix, tags, rate)
365 }
366
367 // Timing sends timing information, it is an alias for TimeInMilliseconds
368 func (c *Client) Timing(name string, value time.Duration, tags []string, rate float64) error {
369 return c.TimeInMilliseconds(name, value.Seconds()*1000, tags, rate)
370 }
371
372 // TimeInMilliseconds sends timing information in milliseconds.
373 // It is flushed by statsd with percentiles, mean and other info (https://github.com/etsy/statsd/blob/master/docs/metric_types.md#timing)
374 func (c *Client) TimeInMilliseconds(name string, value float64, tags []string, rate float64) error {
375 return c.send(name, value, timingSuffix, tags, rate)
376 }
377
378 // Event sends the provided Event.
379 func (c *Client) Event(e *Event) error {
380 if c == nil {
381 return nil
382 }
383 stat, err := e.Encode(c.Tags...)
384 if err != nil {
385 return err
386 }
387 return c.sendMsg(stat)
388 }
389
390 // SimpleEvent sends an event with the provided title and text.
391 func (c *Client) SimpleEvent(title, text string) error {
392 e := NewEvent(title, text)
393 return c.Event(e)
394 }
395
396 // ServiceCheck sends the provided ServiceCheck.
397 func (c *Client) ServiceCheck(sc *ServiceCheck) error {
398 if c == nil {
399 return nil
400 }
401 stat, err := sc.Encode(c.Tags...)
402 if err != nil {
403 return err
404 }
405 return c.sendMsg(stat)
406 }
407
408 // SimpleServiceCheck sends an serviceCheck with the provided name and status.
409 func (c *Client) SimpleServiceCheck(name string, status ServiceCheckStatus) error {
410 sc := NewServiceCheck(name, status)
411 return c.ServiceCheck(sc)
412 }
413
414 // Close the client connection.
415 func (c *Client) Close() error {
416 if c == nil {
417 return nil
418 }
419 select {
420 case c.stop <- struct{}{}:
421 default:
422 }
423
424 // if this client is buffered, flush before closing the writer
425 if c.bufferLength > 0 {
426 if err := c.Flush(); err != nil {
427 return err
428 }
429 }
430
431 return c.writer.Close()
432 }
433
434 // Events support
435 // EventAlertType and EventAlertPriority became exported types after this issue was submitted: https://github.com/DataDog/datadog-go/issues/41
436 // The reason why they got exported is so that client code can directly use the types.
437
438 // EventAlertType is the alert type for events
439 type EventAlertType string
440
441 const (
442 // Info is the "info" AlertType for events
443 Info EventAlertType = "info"
444 // Error is the "error" AlertType for events
445 Error EventAlertType = "error"
446 // Warning is the "warning" AlertType for events
447 Warning EventAlertType = "warning"
448 // Success is the "success" AlertType for events
449 Success EventAlertType = "success"
450 )
451
452 // EventPriority is the event priority for events
453 type EventPriority string
454
455 const (
456 // Normal is the "normal" Priority for events
457 Normal EventPriority = "normal"
458 // Low is the "low" Priority for events
459 Low EventPriority = "low"
460 )
461
462 // An Event is an object that can be posted to your DataDog event stream.
463 type Event struct {
464 // Title of the event. Required.
465 Title string
466 // Text is the description of the event. Required.
467 Text string
468 // Timestamp is a timestamp for the event. If not provided, the dogstatsd
469 // server will set this to the current time.
470 Timestamp time.Time
471 // Hostname for the event.
472 Hostname string
473 // AggregationKey groups this event with others of the same key.
474 AggregationKey string
475 // Priority of the event. Can be statsd.Low or statsd.Normal.
476 Priority EventPriority
477 // SourceTypeName is a source type for the event.
478 SourceTypeName string
479 // AlertType can be statsd.Info, statsd.Error, statsd.Warning, or statsd.Success.
480 // If absent, the default value applied by the dogstatsd server is Info.
481 AlertType EventAlertType
482 // Tags for the event.
483 Tags []string
484 }
485
486 // NewEvent creates a new event with the given title and text. Error checking
487 // against these values is done at send-time, or upon running e.Check.
488 func NewEvent(title, text string) *Event {
489 return &Event{
490 Title: title,
491 Text: text,
492 }
493 }
494
495 // Check verifies that an event is valid.
496 func (e Event) Check() error {
497 if len(e.Title) == 0 {
498 return fmt.Errorf("statsd.Event title is required")
499 }
500 if len(e.Text) == 0 {
501 return fmt.Errorf("statsd.Event text is required")
502 }
503 return nil
504 }
505
506 // Encode returns the dogstatsd wire protocol representation for an event.
507 // Tags may be passed which will be added to the encoded output but not to
508 // the Event's list of tags, eg. for default tags.
509 func (e Event) Encode(tags ...string) (string, error) {
510 err := e.Check()
511 if err != nil {
512 return "", err
513 }
514 text := e.escapedText()
515
516 var buffer bytes.Buffer
517 buffer.WriteString("_e{")
518 buffer.WriteString(strconv.FormatInt(int64(len(e.Title)), 10))
519 buffer.WriteRune(',')
520 buffer.WriteString(strconv.FormatInt(int64(len(text)), 10))
521 buffer.WriteString("}:")
522 buffer.WriteString(e.Title)
523 buffer.WriteRune('|')
524 buffer.WriteString(text)
525
526 if !e.Timestamp.IsZero() {
527 buffer.WriteString("|d:")
528 buffer.WriteString(strconv.FormatInt(int64(e.Timestamp.Unix()), 10))
529 }
530
531 if len(e.Hostname) != 0 {
532 buffer.WriteString("|h:")
533 buffer.WriteString(e.Hostname)
534 }
535
536 if len(e.AggregationKey) != 0 {
537 buffer.WriteString("|k:")
538 buffer.WriteString(e.AggregationKey)
539
540 }
541
542 if len(e.Priority) != 0 {
543 buffer.WriteString("|p:")
544 buffer.WriteString(string(e.Priority))
545 }
546
547 if len(e.SourceTypeName) != 0 {
548 buffer.WriteString("|s:")
549 buffer.WriteString(e.SourceTypeName)
550 }
551
552 if len(e.AlertType) != 0 {
553 buffer.WriteString("|t:")
554 buffer.WriteString(string(e.AlertType))
555 }
556
557 writeTagString(&buffer, tags, e.Tags)
558
559 return buffer.String(), nil
560 }
561
562 // ServiceCheckStatus support
563 type ServiceCheckStatus byte
564
565 const (
566 // Ok is the "ok" ServiceCheck status
567 Ok ServiceCheckStatus = 0
568 // Warn is the "warning" ServiceCheck status
569 Warn ServiceCheckStatus = 1
570 // Critical is the "critical" ServiceCheck status
571 Critical ServiceCheckStatus = 2
572 // Unknown is the "unknown" ServiceCheck status
573 Unknown ServiceCheckStatus = 3
574 )
575
576 // An ServiceCheck is an object that contains status of DataDog service check.
577 type ServiceCheck struct {
578 // Name of the service check. Required.
579 Name string
580 // Status of service check. Required.
581 Status ServiceCheckStatus
582 // Timestamp is a timestamp for the serviceCheck. If not provided, the dogstatsd
583 // server will set this to the current time.
584 Timestamp time.Time
585 // Hostname for the serviceCheck.
586 Hostname string
587 // A message describing the current state of the serviceCheck.
588 Message string
589 // Tags for the serviceCheck.
590 Tags []string
591 }
592
593 // NewServiceCheck creates a new serviceCheck with the given name and status. Error checking
594 // against these values is done at send-time, or upon running sc.Check.
595 func NewServiceCheck(name string, status ServiceCheckStatus) *ServiceCheck {
596 return &ServiceCheck{
597 Name: name,
598 Status: status,
599 }
600 }
601
602 // Check verifies that an event is valid.
603 func (sc ServiceCheck) Check() error {
604 if len(sc.Name) == 0 {
605 return fmt.Errorf("statsd.ServiceCheck name is required")
606 }
607 if byte(sc.Status) < 0 || byte(sc.Status) > 3 {
608 return fmt.Errorf("statsd.ServiceCheck status has invalid value")
609 }
610 return nil
611 }
612
613 // Encode returns the dogstatsd wire protocol representation for an serviceCheck.
614 // Tags may be passed which will be added to the encoded output but not to
615 // the Event's list of tags, eg. for default tags.
616 func (sc ServiceCheck) Encode(tags ...string) (string, error) {
617 err := sc.Check()
618 if err != nil {
619 return "", err
620 }
621 message := sc.escapedMessage()
622
623 var buffer bytes.Buffer
624 buffer.WriteString("_sc|")
625 buffer.WriteString(sc.Name)
626 buffer.WriteRune('|')
627 buffer.WriteString(strconv.FormatInt(int64(sc.Status), 10))
628
629 if !sc.Timestamp.IsZero() {
630 buffer.WriteString("|d:")
631 buffer.WriteString(strconv.FormatInt(int64(sc.Timestamp.Unix()), 10))
632 }
633
634 if len(sc.Hostname) != 0 {
635 buffer.WriteString("|h:")
636 buffer.WriteString(sc.Hostname)
637 }
638
639 writeTagString(&buffer, tags, sc.Tags)
640
641 if len(message) != 0 {
642 buffer.WriteString("|m:")
643 buffer.WriteString(message)
644 }
645
646 return buffer.String(), nil
647 }
648
649 func (e Event) escapedText() string {
650 return strings.Replace(e.Text, "\n", "\\n", -1)
651 }
652
653 func (sc ServiceCheck) escapedMessage() string {
654 msg := strings.Replace(sc.Message, "\n", "\\n", -1)
655 return strings.Replace(msg, "m:", `m\:`, -1)
656 }
657
658 func removeNewlines(str string) string {
659 return strings.Replace(str, "\n", "", -1)
660 }
661
662 func writeTagString(w io.Writer, tagList1, tagList2 []string) {
663 // the tag lists may be shared with other callers, so we cannot modify
664 // them in any way (which means we cannot append to them either)
665 // therefore we must make an entirely separate copy just for this call
666 totalLen := len(tagList1) + len(tagList2)
667 if totalLen == 0 {
668 return
669 }
670 tags := make([]string, 0, totalLen)
671 tags = append(tags, tagList1...)
672 tags = append(tags, tagList2...)
673
674 io.WriteString(w, "|#")
675 io.WriteString(w, removeNewlines(tags[0]))
676 for _, tag := range tags[1:] {
677 io.WriteString(w, ",")
678 io.WriteString(w, removeNewlines(tag))
679 }
680 }