Debian Codegraph @

feeder/feeder_amqp.go

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 // balboa
2 // Copyright (c) 2018, DCSO GmbH
3
4 package feeder
5
6 import (
7 "bytes"
8 "compress/gzip"
9 "fmt"
10 "io"
11 "strings"
12 "sync"
13 "time"
14
15 "github.com/DCSO/balboa/format"
16 "github.com/DCSO/balboa/observation"
17
18 "github.com/NeowayLabs/wabbit"
19 "github.com/NeowayLabs/wabbit/amqp"
20 log "github.com/sirupsen/logrus"
21 origamqp "github.com/streadway/amqp"
22 )
23
24 // Consumer reads and processes messages from a fake RabbitMQ server.
25 type Consumer struct {
26 conn wabbit.Conn
27 channel wabbit.Channel
28 tag string
29 exchanges []string
30 URL string
31 done chan error
32 stop chan bool
33 deliveries <-chan wabbit.Delivery
34 Callback func(wabbit.Delivery)
35 StopReconnection chan bool
36 ChanMutex sync.Mutex
37 ConnMutex sync.Mutex
38 OutChan chan observation.InputObservation
39 MakeObservationFunc format.MakeObservationFunc
40 ErrorChan chan wabbit.Error
41 Reconnector func(string) (wabbit.Conn, string, error)
42 Connector func(*Consumer) error
43 }
44
45 func decompressBody(d wabbit.Delivery) ([]byte, error) {
46 var compressedBuffer bytes.Buffer
47 compressedBuffer.Write(d.Body())
48 compressedReader, err := gzip.NewReader(&compressedBuffer)
49 if err != nil {
50 return nil, err
51 }
52 decompressedWriter := new(bytes.Buffer)
53 io.Copy(decompressedWriter, compressedReader)
54 compressedReader.Close()
55 body := decompressedWriter.Bytes()
56 return body, nil
57 }
58
59 const amqpReconnDelay = 2 * time.Second
60
61 func reconnectOnFailure(s *Consumer) {
62 for {
63 select {
64 case <-s.StopReconnection:
65 return
66 case rabbitErr := <-s.ErrorChan:
67 if rabbitErr != nil {
68 log.Warnf("RabbitMQ connection failed: %s", rabbitErr.Reason())
69 close(s.stop)
70 for {
71 time.Sleep(amqpReconnDelay)
72 connErr := s.Connector(s)
73 if connErr != nil {
74 log.Warnf("RabbitMQ error: %s", connErr)
75 } else {
76 log.Infof("Reestablished connection to %s", s.URL)
77 s.ConnMutex.Lock()
78 s.conn.NotifyClose(s.ErrorChan)
79 s.ConnMutex.Unlock()
80 go handle(s.deliveries, s.stop, s.done, s.OutChan, s.MakeObservationFunc)
81 break
82 }
83 }
84 }
85 }
86 }
87 }
88
89 // NewConsumerWithReconnector creates a new consumer with the given properties. The callback
90 // function is called for each delivery accepted from a consumer channel.
91 func (f *AMQPFeeder) NewConsumerWithReconnector(amqpURI string, exchanges []string, exchangeType,
92 queueName, key, ctag string, out chan observation.InputObservation,
93 reconnector func(string) (wabbit.Conn, string, error)) (*Consumer, error) {
94 var err error
95 c := &Consumer{
96 conn: nil,
97 channel: nil,
98 exchanges: exchanges,
99 URL: amqpURI,
100 tag: ctag,
101 done: make(chan error),
102 stop: make(chan bool),
103 OutChan: out,
104 MakeObservationFunc: f.MakeObservationFunc,
105 Reconnector: reconnector,
106 StopReconnection: make(chan bool),
107 }
108
109 c.Connector = func(s *Consumer) error {
110 var err error
111 var exchangeType string
112
113 s.ConnMutex.Lock()
114 s.conn, exchangeType, err = s.Reconnector(s.URL)
115 s.ConnMutex.Unlock()
116 if err != nil {
117 return err
118 }
119 s.ChanMutex.Lock()
120 s.channel, err = s.conn.Channel()
121 s.ChanMutex.Unlock()
122 if err != nil {
123 s.ConnMutex.Lock()
124 s.conn.Close()
125 s.ConnMutex.Unlock()
126 return err
127 }
128
129 for _, exchange := range exchanges {
130 // We do not want to declare an exchange on non-default connection methods,
131 // as they may not support all exchange types. For instance amqptest does
132 // not support 'fanout'.
133 log.Debug("declaring exchange ", exchange)
134 err = s.channel.ExchangeDeclare(
135 exchange,
136 exchangeType,
137 wabbit.Option{
138 "durable": true,
139 "autoDelete": false,
140 "internal": false,
141 "noWait": false,
142 },
143 )
144 if err != nil {
145 log.Error(err)
146 s.ChanMutex.Lock()
147 s.channel.Close()
148 s.ChanMutex.Unlock()
149 s.ConnMutex.Lock()
150 s.conn.Close()
151 s.ConnMutex.Unlock()
152 return err
153 }
154 }
155 queueName := fmt.Sprintf("%s.%s", strings.Join(exchanges, "."), queueName)
156 queue, err := c.channel.QueueDeclare(
157 queueName,
158 wabbit.Option{
159 "durable": false,
160 "autoDelete": true,
161 "exclusive": true,
162 "noWait": false,
163 "args": origamqp.Table{
164 "x-message-ttl": int32(300000),
165 "x-max-length-bytes": int32(100 * 1024 * 1024),
166 },
167 },
168 )
169 if err != nil {
170 return fmt.Errorf("error queue declare: %s", err)
171 }
172 log.Debugf("declared Queue (%q %d messages, %d consumers), binding to Exchange (key %q)",
173 queue.Name(), queue.Messages(), queue.Consumers(), key)
174
175 for _, exchange := range exchanges {
176 log.Debug("binding to exchange ", exchange)
177 if err = c.channel.QueueBind(
178 queue.Name(),
179 key,
180 exchange,
181 wabbit.Option{
182 "noWait": false,
183 },
184 ); err != nil {
185 return fmt.Errorf("error queue bind: %s", err)
186 }
187 }
188
189 log.Debugf("queue bound to Exchange, starting Consume (consumer tag %q)", c.tag)
190 c.deliveries, err = c.channel.Consume(
191 queue.Name(),
192 c.tag,
193 wabbit.Option{
194 "autoAck": false,
195 "exclusive": false,
196 "noLocal": false,
197 "noWait": false,
198 },
199 )
200 if err != nil {
201 return fmt.Errorf("error queue consume: %s", err)
202 }
203
204 log.Debugf("consumer established connection to %s", s.URL)
205 s.stop = make(chan bool)
206 c.ErrorChan = make(chan wabbit.Error)
207
208 return nil
209 }
210
211 c.ErrorChan = make(chan wabbit.Error)
212 err = c.Connector(c)
213 if err != nil {
214 return nil, err
215 }
216 c.conn.NotifyClose(c.ErrorChan)
217
218 go reconnectOnFailure(c)
219 go handle(c.deliveries, c.stop, c.done, out, f.MakeObservationFunc)
220
221 return c, nil
222 }
223
224 func defaultReconnector(amqpURI string) (wabbit.Conn, string, error) {
225 conn, err := amqp.Dial(amqpURI)
226 if err != nil {
227 return nil, "fanout", err
228 }
229 return conn, "fanout", err
230 }
231
232 // NewConsumer returns a new Consumer.
233 func (f *AMQPFeeder) NewConsumer(amqpURI string, exchanges []string, exchangeType, queueName, key,
234 ctag string, out chan observation.InputObservation) (*Consumer, error) {
235 return f.NewConsumerWithReconnector(amqpURI, exchanges, exchangeType, queueName, key,
236 ctag, out, defaultReconnector)
237 }
238
239 // Shutdown shuts down a consumer, closing down its channels and connections.
240 func (c *Consumer) Shutdown() error {
241 // will close() the deliveries channel
242 if err := c.channel.Close(); err != nil {
243 return fmt.Errorf("channel close failed: %s", err)
244 }
245 if err := c.conn.Close(); err != nil {
246 return fmt.Errorf("AMQP connection close error: %s", err)
247 }
248 defer log.Debugf("AMQP shutdown OK")
249 // wait for handle() to exit
250 return <-c.done
251 }
252
253 func handle(deliveries <-chan wabbit.Delivery, stop chan bool, done chan error,
254 out chan observation.InputObservation, fn format.MakeObservationFunc) {
255 for {
256 select {
257 case <-stop:
258 done <- nil
259 return
260 case d := <-deliveries:
261 if d == nil {
262 done <- nil
263 return
264 }
265 log.Infof("got %d bytes via AMQP", len(d.Body()))
266 raw := d.Body()
267 if _, ok := d.Headers()["compressed"]; ok {
268 var err error
269 raw, err = decompressBody(d)
270 if err != nil {
271 log.Warn(err)
272 continue
273 }
274 }
275 var sensorID string
276 if _, ok := d.Headers()["sensor_id"]; ok {
277 sensorID = d.Headers()["sensor_id"].(string)
278 }
279 err := fn(raw, sensorID, out, stop)
280 if err != nil {
281 log.Warn(err)
282 }
283 d.Ack(true)
284 }
285 }
286
287 }
288
289 // AMQPFeeder is a Feeder that accepts input via AMQP queues.
290 type AMQPFeeder struct {
291 StopChan chan bool
292 StoppedChan chan bool
293 IsRunning bool
294 Consumer *Consumer
295 URL string
296 Exchanges []string
297 Queue string
298 MakeObservationFunc format.MakeObservationFunc
299 }
300
301 // MakeAMQPFeeder returns a new AMQPFeeder, connecting to the AMQP server at
302 // the given URL, creating a new queue with the given name bound to the
303 // provided exchanges.
304 func MakeAMQPFeeder(url string, exchanges []string, queue string) *AMQPFeeder {
305 return &AMQPFeeder{
306 IsRunning: false,
307 StopChan: make(chan bool),
308 Exchanges: exchanges,
309 URL: url,
310 Queue: queue,
311 MakeObservationFunc: format.MakeFeverAggregateInputObservations,
312 }
313 }
314
315 // SetInputDecoder states that the given MakeObservationFunc should be used to
316 // parse and decode data delivered to this feeder.
317 func (f *AMQPFeeder) SetInputDecoder(fn format.MakeObservationFunc) {
318 f.MakeObservationFunc = fn
319 }
320
321 // Run starts the feeder.
322 func (f *AMQPFeeder) Run(out chan observation.InputObservation) error {
323 var err error
324 f.Consumer, err = f.NewConsumer(f.URL, f.Exchanges, "fanout", f.Queue, "", "balboa", out)
325 if err != nil {
326 return err
327 }
328 f.IsRunning = true
329 return nil
330 }
331
332 // Stop causes the feeder to stop accepting deliveries and close all
333 // associated channels, including the passed notification channel.
334 func (f *AMQPFeeder) Stop(stopChan chan bool) {
335 close(stopChan)
336 if f.IsRunning {
337 f.Consumer.Shutdown()
338 }
339 }