Debian Codegraph @

cmd/store/query/query.go

raw
1 package query
2
3 import (
4 "bufio"
5 "context"
6 "errors"
7 "flag"
8 "fmt"
9 "io"
10 "os"
11 "path/filepath"
12 "strconv"
13 "strings"
14 "time"
15
16 "github.com/gogo/protobuf/proto"
17 "github.com/influxdata/influxdb/models"
18 "github.com/influxdata/influxdb/services/storage"
19 "github.com/influxdata/influxql"
20 "github.com/influxdata/yarpc"
21 "go.uber.org/zap"
22 )
23
24 // Command represents the program execution for "store query".
25 type Command struct {
26 // Standard input/output, overridden for testing.
27 Stderr io.Writer
28 Stdout io.Writer
29 Logger *zap.Logger
30
31 addr string
32 cpuProfile string
33 memProfile string
34 database string
35 retentionPolicy string
36 startTime int64
37 endTime int64
38 limit int64
39 slimit int64
40 soffset int64
41 desc bool
42 silent bool
43 expr string
44 agg string
45 groupArg string
46 group storage.ReadRequest_Group
47 groupKeys string
48 keys []string
49 hintsArg string
50 hints storage.HintFlags
51
52 aggType storage.Aggregate_AggregateType
53
54 // response
55 integerSum int64
56 unsignedSum uint64
57 floatSum float64
58 pointCount uint64
59 }
60
61 // NewCommand returns a new instance of Command.
62 func NewCommand() *Command {
63 return &Command{
64 Stderr: os.Stderr,
65 Stdout: os.Stdout,
66 }
67 }
68
69 func parseTime(v string) (int64, error) {
70 if s, err := time.Parse(time.RFC3339, v); err == nil {
71 return s.UnixNano(), nil
72 }
73
74 if i, err := strconv.ParseInt(v, 10, 64); err == nil {
75 return i, nil
76 }
77
78 return 0, errors.New("invalid time")
79 }
80
81 // Run executes the command.
82 func (cmd *Command) Run(args ...string) error {
83 var start, end string
84 fs := flag.NewFlagSet("query", flag.ExitOnError)
85 fs.StringVar(&cmd.cpuProfile, "cpuprofile", "", "CPU profile name")
86 fs.StringVar(&cmd.memProfile, "memprofile", "", "memory profile name")
87 fs.StringVar(&cmd.addr, "addr", ":8082", "the RPC address")
88 fs.StringVar(&cmd.database, "database", "", "the database to query")
89 fs.StringVar(&cmd.retentionPolicy, "retention", "", "Optional: the retention policy to query")
90 fs.StringVar(&start, "start", "", "Optional: the start time to query (RFC3339 format)")
91 fs.StringVar(&end, "end", "", "Optional: the end time to query (RFC3339 format)")
92 fs.Int64Var(&cmd.slimit, "slimit", 0, "Optional: limit number of series")
93 fs.Int64Var(&cmd.soffset, "soffset", 0, "Optional: start offset for series")
94 fs.Int64Var(&cmd.limit, "limit", 0, "Optional: limit number of values per series (-1 to return series only)")
95 fs.BoolVar(&cmd.desc, "desc", false, "Optional: return results in descending order")
96 fs.BoolVar(&cmd.silent, "silent", false, "silence output")
97 fs.StringVar(&cmd.expr, "expr", "", "InfluxQL conditional expression")
98 fs.StringVar(&cmd.agg, "agg", "", "aggregate functions (sum, count)")
99 fs.StringVar(&cmd.groupArg, "group", "none", "group operation (none,all,by,except,disable)")
100 fs.StringVar(&cmd.groupKeys, "group-keys", "", "comma-separated list of tags to specify series order")
101 fs.StringVar(&cmd.hintsArg, "hints", "none", "comma-separated list of read hints (none,no_points,no_series)")
102
103 fs.SetOutput(cmd.Stdout)
104 fs.Usage = func() {
105 fmt.Fprintln(cmd.Stdout, "Query via RPC")
106 fmt.Fprintf(cmd.Stdout, "Usage: %s query [flags]\n\n", filepath.Base(os.Args[0]))
107 fs.PrintDefaults()
108 }
109
110 if err := fs.Parse(args); err != nil {
111 return err
112 }
113
114 // set defaults
115 if start != "" {
116 t, err := parseTime(start)
117 if err != nil {
118 return err
119 }
120 cmd.startTime = t
121
122 } else {
123 cmd.startTime = models.MinNanoTime
124 }
125 if end != "" {
126 t, err := parseTime(end)
127 if err != nil {
128 return err
129 }
130 cmd.endTime = t
131
132 } else {
133 // set end time to max if it is not set.
134 cmd.endTime = models.MaxNanoTime
135 }
136
137 if cmd.groupKeys != "" {
138 cmd.keys = strings.Split(cmd.groupKeys, ",")
139 }
140
141 if err := cmd.validate(); err != nil {
142 return err
143 }
144
145 conn, err := yarpc.Dial(cmd.addr)
146 if err != nil {
147 return err
148 }
149 defer conn.Close()
150
151 return cmd.query(storage.NewStorageClient(conn))
152 }
153
154 func (cmd *Command) validate() error {
155 if cmd.database == "" {
156 return fmt.Errorf("must specify a database")
157 }
158 if cmd.startTime != 0 && cmd.endTime != 0 && cmd.endTime < cmd.startTime {
159 return fmt.Errorf("end time before start time")
160 }
161
162 if cmd.agg != "" {
163 tm := proto.EnumValueMap("com.github.influxdata.influxdb.services.storage.Aggregate_AggregateType")
164 agg, ok := tm[strings.ToUpper(cmd.agg)]
165 if !ok {
166 return errors.New("invalid aggregate function: " + cmd.agg)
167 }
168 cmd.aggType = storage.Aggregate_AggregateType(agg)
169 }
170
171 enums := proto.EnumValueMap("com.github.influxdata.influxdb.services.storage.ReadRequest_Group")
172 group, ok := enums["GROUP_"+strings.ToUpper(cmd.groupArg)]
173 if !ok {
174 return errors.New("invalid group type: " + cmd.groupArg)
175 }
176 cmd.group = storage.ReadRequest_Group(group)
177
178 enums = proto.EnumValueMap("com.github.influxdata.influxdb.services.storage.ReadRequest_HintFlags")
179 for _, h := range strings.Split(cmd.hintsArg, ",") {
180 cmd.hints |= storage.HintFlags(enums["HINT_"+strings.ToUpper(h)])
181 }
182
183 return nil
184 }
185
186 func (cmd *Command) query(c storage.StorageClient) error {
187 var req storage.ReadRequest
188 req.Database = cmd.database
189 if cmd.retentionPolicy != "" {
190 req.Database += "/" + cmd.retentionPolicy
191 }
192
193 req.TimestampRange.Start = cmd.startTime
194 req.TimestampRange.End = cmd.endTime
195 req.SeriesLimit = cmd.slimit
196 req.SeriesOffset = cmd.soffset
197 req.PointsLimit = cmd.limit
198 req.Descending = cmd.desc
199 req.Group = cmd.group
200 req.GroupKeys = cmd.keys
201 req.Hints = cmd.hints
202
203 if cmd.aggType != storage.AggregateTypeNone {
204 req.Aggregate = &storage.Aggregate{Type: cmd.aggType}
205 }
206
207 if cmd.expr != "" {
208 expr, err := influxql.ParseExpr(cmd.expr)
209 if err != nil {
210 return nil
211 }
212 fmt.Fprintln(cmd.Stdout, expr)
213 var v exprToNodeVisitor
214 influxql.Walk(&v, expr)
215 if v.Err() != nil {
216 return v.Err()
217 }
218
219 req.Predicate = &storage.Predicate{Root: v.nodes[0]}
220 }
221
222 stream, err := c.Read(context.Background(), &req)
223 if err != nil {
224 fmt.Fprintln(cmd.Stdout, err)
225 return err
226 }
227
228 wr := bufio.NewWriter(os.Stdout)
229
230 now := time.Now()
231 defer func() {
232 dur := time.Since(now)
233 fmt.Fprintf(cmd.Stdout, "time: %v\n", dur)
234 }()
235
236 for {
237 var rep storage.ReadResponse
238
239 if err = stream.RecvMsg(&rep); err != nil {
240 if err == io.EOF {
241 break
242 }
243
244 return err
245 }
246
247 if cmd.silent {
248 cmd.processFramesSilent(rep.Frames)
249 } else {
250 cmd.processFrames(wr, rep.Frames)
251 }
252 }
253
254 fmt.Fprintln(cmd.Stdout)
255 fmt.Fprint(cmd.Stdout, "points(count): ", cmd.pointCount, ", sum(int64): ", cmd.integerSum, ", sum(uint64): ", cmd.unsignedSum, ", sum(float64): ", cmd.floatSum, "\n")
256
257 return nil
258 }
259
260 func (cmd *Command) processFramesSilent(frames []storage.ReadResponse_Frame) {
261 for _, frame := range frames {
262 switch f := frame.Data.(type) {
263 case *storage.ReadResponse_Frame_IntegerPoints:
264 for _, v := range f.IntegerPoints.Values {
265 cmd.integerSum += v
266 }
267 cmd.pointCount += uint64(len(f.IntegerPoints.Values))
268
269 case *storage.ReadResponse_Frame_UnsignedPoints:
270 for _, v := range f.UnsignedPoints.Values {
271 cmd.unsignedSum += v
272 }
273 cmd.pointCount += uint64(len(f.UnsignedPoints.Values))
274
275 case *storage.ReadResponse_Frame_FloatPoints:
276 for _, v := range f.FloatPoints.Values {
277 cmd.floatSum += v
278 }
279 cmd.pointCount += uint64(len(f.FloatPoints.Values))
280
281 case *storage.ReadResponse_Frame_StringPoints:
282 cmd.pointCount += uint64(len(f.StringPoints.Values))
283
284 case *storage.ReadResponse_Frame_BooleanPoints:
285 cmd.pointCount += uint64(len(f.BooleanPoints.Values))
286 }
287 }
288 }
289
290 func printByteSlice(wr *bufio.Writer, v [][]byte) {
291 wr.WriteString("[\033[36m")
292 first := true
293 for _, t := range v {
294 if !first {
295 wr.WriteByte(',')
296 } else {
297 first = false
298 }
299 wr.Write(t)
300 }
301 wr.WriteString("\033[0m]\n")
302 }
303
304 func (cmd *Command) processFrames(wr *bufio.Writer, frames []storage.ReadResponse_Frame) {
305 var buf [1024]byte
306 var line []byte
307
308 for _, frame := range frames {
309 switch f := frame.Data.(type) {
310 case *storage.ReadResponse_Frame_Group:
311 g := f.Group
312 wr.WriteString("partition values")
313 printByteSlice(wr, g.PartitionKeyVals)
314 wr.WriteString("group keys")
315 printByteSlice(wr, g.TagKeys)
316 wr.Flush()
317
318 case *storage.ReadResponse_Frame_Series:
319 s := f.Series
320 wr.WriteString("\033[36m")
321 first := true
322 for _, t := range s.Tags {
323 if !first {
324 wr.WriteByte(',')
325 } else {
326 first = false
327 }
328 wr.Write(t.Key)
329 wr.WriteByte(':')
330 wr.Write(t.Value)
331 }
332 wr.WriteString("\033[0m\n")
333 wr.Flush()
334
335 case *storage.ReadResponse_Frame_IntegerPoints:
336 p := f.IntegerPoints
337 for i := 0; i < len(p.Timestamps); i++ {
338 line = buf[:0]
339 wr.Write(strconv.AppendInt(line, p.Timestamps[i], 10))
340 wr.WriteByte(' ')
341
342 line = buf[:0]
343 wr.Write(strconv.AppendInt(line, p.Values[i], 10))
344 wr.WriteString("\n")
345 wr.Flush()
346
347 cmd.integerSum += p.Values[i]
348 }
349 cmd.pointCount += uint64(len(f.IntegerPoints.Values))
350
351 case *storage.ReadResponse_Frame_UnsignedPoints:
352 p := f.UnsignedPoints
353 for i := 0; i < len(p.Timestamps); i++ {
354 line = buf[:0]
355 wr.Write(strconv.AppendInt(line, p.Timestamps[i], 10))
356 wr.WriteByte(' ')
357
358 line = buf[:0]
359 wr.Write(strconv.AppendUint(line, p.Values[i], 10))
360 wr.WriteString("\n")
361 wr.Flush()
362
363 cmd.unsignedSum += p.Values[i]
364 }
365 cmd.pointCount += uint64(len(f.UnsignedPoints.Values))
366
367 case *storage.ReadResponse_Frame_FloatPoints:
368 p := f.FloatPoints
369 for i := 0; i < len(p.Timestamps); i++ {
370 line = buf[:0]
371 wr.Write(strconv.AppendInt(line, p.Timestamps[i], 10))
372 wr.WriteByte(' ')
373
374 line = buf[:0]
375 wr.Write(strconv.AppendFloat(line, p.Values[i], 'f', 10, 64))
376 wr.WriteString("\n")
377 wr.Flush()
378
379 cmd.floatSum += p.Values[i]
380 }
381 cmd.pointCount += uint64(len(f.FloatPoints.Values))
382
383 case *storage.ReadResponse_Frame_StringPoints:
384 p := f.StringPoints
385 for i := 0; i < len(p.Timestamps); i++ {
386 line = buf[:0]
387 wr.Write(strconv.AppendInt(line, p.Timestamps[i], 10))
388 wr.WriteByte(' ')
389
390 wr.WriteString(p.Values[i])
391 wr.WriteString("\n")
392 wr.Flush()
393 }
394 cmd.pointCount += uint64(len(f.StringPoints.Values))
395
396 case *storage.ReadResponse_Frame_BooleanPoints:
397 p := f.BooleanPoints
398 for i := 0; i < len(p.Timestamps); i++ {
399 line = buf[:0]
400 wr.Write(strconv.AppendInt(line, p.Timestamps[i], 10))
401 wr.WriteByte(' ')
402
403 if p.Values[i] {
404 wr.WriteString("true")
405 } else {
406 wr.WriteString("false")
407 }
408 wr.WriteString("\n")
409 wr.Flush()
410 }
411 cmd.pointCount += uint64(len(f.BooleanPoints.Values))
412 }
413 }
414 }
415
416 type exprToNodeVisitor struct {
417 nodes []*storage.Node
418 err error
419 }
420
421 func (v *exprToNodeVisitor) Err() error {
422 return v.err
423 }
424
425 func (v *exprToNodeVisitor) pop() (top *storage.Node) {
426 if len(v.nodes) < 1 {
427 panic("exprToNodeVisitor: stack empty")
428 }
429
430 top, v.nodes = v.nodes[len(v.nodes)-1], v.nodes[:len(v.nodes)-1]
431 return
432 }
433
434 func (v *exprToNodeVisitor) pop2() (lhs, rhs *storage.Node) {
435 if len(v.nodes) < 2 {
436 panic("exprToNodeVisitor: stack empty")
437 }
438
439 rhs = v.nodes[len(v.nodes)-1]
440 lhs = v.nodes[len(v.nodes)-2]
441 v.nodes = v.nodes[:len(v.nodes)-2]
442 return
443 }
444
445 func mapOpToComparison(op influxql.Token) storage.Node_Comparison {
446 switch op {
447 case influxql.EQ:
448 return storage.ComparisonEqual
449 case influxql.EQREGEX:
450 return storage.ComparisonRegex
451 case influxql.NEQ:
452 return storage.ComparisonNotEqual
453 case influxql.NEQREGEX:
454 return storage.ComparisonNotEqual
455 case influxql.LT:
456 return storage.ComparisonLess
457 case influxql.LTE:
458 return storage.ComparisonLessEqual
459 case influxql.GT:
460 return storage.ComparisonGreater
461 case influxql.GTE:
462 return storage.ComparisonGreaterEqual
463
464 default:
465 return -1
466 }
467 }
468
469 func (v *exprToNodeVisitor) Visit(node influxql.Node) influxql.Visitor {
470 switch n := node.(type) {
471 case *influxql.BinaryExpr:
472 if v.err != nil {
473 return nil
474 }
475
476 influxql.Walk(v, n.LHS)
477 if v.err != nil {
478 return nil
479 }
480
481 influxql.Walk(v, n.RHS)
482 if v.err != nil {
483 return nil
484 }
485
486 if comp := mapOpToComparison(n.Op); comp != -1 {
487 lhs, rhs := v.pop2()
488 v.nodes = append(v.nodes, &storage.Node{
489 NodeType: storage.NodeTypeComparisonExpression,
490 Value: &storage.Node_Comparison_{Comparison: comp},
491 Children: []*storage.Node{lhs, rhs},
492 })
493 } else if n.Op == influxql.AND || n.Op == influxql.OR {
494 var op storage.Node_Logical
495 if n.Op == influxql.AND {
496 op = storage.LogicalAnd
497 } else {
498 op = storage.LogicalOr
499 }
500
501 lhs, rhs := v.pop2()
502 v.nodes = append(v.nodes, &storage.Node{
503 NodeType: storage.NodeTypeLogicalExpression,
504 Value: &storage.Node_Logical_{Logical: op},
505 Children: []*storage.Node{lhs, rhs},
506 })
507 } else {
508 v.err = fmt.Errorf("unsupported operator, %s", n.Op)
509 }
510
511 return nil
512
513 case *influxql.ParenExpr:
514 influxql.Walk(v, n.Expr)
515 if v.err != nil {
516 return nil
517 }
518
519 v.nodes = append(v.nodes, &storage.Node{
520 NodeType: storage.NodeTypeParenExpression,
521 Children: []*storage.Node{v.pop()},
522 })
523 return nil
524
525 case *influxql.StringLiteral:
526 v.nodes = append(v.nodes, &storage.Node{
527 NodeType: storage.NodeTypeLiteral,
528 Value: &storage.Node_StringValue{StringValue: n.Val},
529 })
530 return nil
531
532 case *influxql.NumberLiteral:
533 v.nodes = append(v.nodes, &storage.Node{
534 NodeType: storage.NodeTypeLiteral,
535 Value: &storage.Node_FloatValue{FloatValue: n.Val},
536 })
537 return nil
538
539 case *influxql.IntegerLiteral:
540 v.nodes = append(v.nodes, &storage.Node{
541 NodeType: storage.NodeTypeLiteral,
542 Value: &storage.Node_IntegerValue{IntegerValue: n.Val},
543 })
544 return nil
545
546 case *influxql.UnsignedLiteral:
547 v.nodes = append(v.nodes, &storage.Node{
548 NodeType: storage.NodeTypeLiteral,
549 Value: &storage.Node_UnsignedValue{UnsignedValue: n.Val},
550 })
551 return nil
552
553 case *influxql.VarRef:
554 v.nodes = append(v.nodes, &storage.Node{
555 NodeType: storage.NodeTypeTagRef,
556 Value: &storage.Node_TagRefValue{TagRefValue: n.Val},
557 })
558 return nil
559
560 case *influxql.RegexLiteral:
561 v.nodes = append(v.nodes, &storage.Node{
562 NodeType: storage.NodeTypeLiteral,
563 Value: &storage.Node_RegexValue{RegexValue: n.Val.String()},
564 })
565 return nil
566 default:
567 v.err = fmt.Errorf("unsupported expression %T", n)
568 return nil
569 }
570 }