Debian Codegraph @

lang/py3/avro/txipc.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 import io
22
23 from avro import io as avro_io
24 from avro import ipc
25
26 from zope.interface import implements
27
28 from twisted.internet.defer import maybeDeferred, Deferred
29 from twisted.internet.protocol import Protocol
30 from twisted.web import resource, server
31 from twisted.web.client import Agent
32 from twisted.web.http_headers import Headers
33 from twisted.web.iweb import IBodyProducer
34
35
36 class TwistedRequestor(ipc.BaseRequestor):
37 """A Twisted-compatible requestor. Returns a Deferred that will fire with the
38 returning value, instead of blocking until the request completes."""
39 def _process_handshake(self, call_response, message_name, request_datum):
40 # process the handshake and call response
41 buffer_decoder = avro_io.BinaryDecoder(io.StringIO(call_response))
42 call_response_exists = self.read_handshake_response(buffer_decoder)
43 if call_response_exists:
44 return self.read_call_response(message_name, buffer_decoder)
45 else:
46 return self.request(message_name, request_datum)
47
48 def issue_request(self, call_request, message_name, request_datum):
49 d = self.transceiver.transceive(call_request)
50 d.addCallback(self._process_handshake, message_name, request_datum)
51 return d
52
53 class RequestStreamingProducer(object):
54 """A streaming producer for issuing requests with the Twisted.web Agent."""
55 implements(IBodyProducer)
56
57 paused = False
58 stopped = False
59 started = False
60
61 def __init__(self, message):
62 self._message = message
63 self._length = len(message)
64 # We need a buffer length header for every buffer and an additional
65 # zero-length buffer as the message terminator
66 self._length += (self._length / ipc.BUFFER_SIZE + 2) \
67 * ipc.BUFFER_HEADER_LENGTH
68 self._total_bytes_sent = 0
69 self._deferred = Deferred()
70
71 # read-only properties
72 message = property(lambda self: self._message)
73 length = property(lambda self: self._length)
74 consumer = property(lambda self: self._consumer)
75 deferred = property(lambda self: self._deferred)
76
77 def _get_total_bytes_sent(self):
78 return self._total_bytes_sent
79
80 def _set_total_bytes_sent(self, bytes_sent):
81 self._total_bytes_sent = bytes_sent
82
83 total_bytes_sent = property(_get_total_bytes_sent, _set_total_bytes_sent)
84
85 def startProducing(self, consumer):
86 if self.started:
87 return
88
89 self.started = True
90 self._consumer = consumer
91 # Keep writing data to the consumer until we're finished,
92 # paused (pauseProducing()) or stopped (stopProducing())
93 while self.length - self.total_bytes_sent > 0 and \
94 not self.paused and not self.stopped:
95 self.write()
96 # self.write will fire this deferred once it has written
97 # the entire message to the consumer
98 return self.deferred
99
100 def resumeProducing(self):
101 self.paused = False
102 self.write(self)
103
104 def pauseProducing(self):
105 self.paused = True
106
107 def stopProducing(self):
108 self.stopped = True
109
110 def write(self):
111 if self.length - self.total_bytes_sent > ipc.BUFFER_SIZE:
112 buffer_length = ipc.BUFFER_SIZE
113 else:
114 buffer_length = self.length - self.total_bytes_sent
115 self.write_buffer(self.message[self.total_bytes_sent:
116 (self.total_bytes_sent + buffer_length)])
117 self.total_bytes_sent += buffer_length
118 # Make sure we wrote the entire message
119 if self.total_bytes_sent == self.length and not self.stopped:
120 self.stopProducing()
121 # A message is always terminated by a zero-length buffer.
122 self.write_buffer_length(0)
123 self.deferred.callback(None)
124
125 def write_buffer(self, chunk):
126 buffer_length = len(chunk)
127 self.write_buffer_length(buffer_length)
128 self.consumer.write(chunk)
129
130 def write_buffer_length(self, n):
131 self.consumer.write(ipc.BIG_ENDIAN_INT_STRUCT.pack(n))
132
133 class AvroProtocol(Protocol):
134
135 recvd = ''
136 done = False
137
138 def __init__(self, finished):
139 self.finished = finished
140 self.message = []
141
142 def dataReceived(self, data):
143 self.recvd = self.recvd + data
144 while len(self.recvd) >= ipc.BUFFER_HEADER_LENGTH:
145 buffer_length ,= ipc.BIG_ENDIAN_INT_STRUCT.unpack(
146 self.recvd[:ipc.BUFFER_HEADER_LENGTH])
147 if buffer_length == 0:
148 response = ''.join(self.message)
149 self.done = True
150 self.finished.callback(response)
151 break
152 if len(self.recvd) < buffer_length + ipc.BUFFER_HEADER_LENGTH:
153 break
154 buffer = self.recvd[ipc.BUFFER_HEADER_LENGTH:buffer_length + ipc.BUFFER_HEADER_LENGTH]
155 self.recvd = self.recvd[buffer_length + ipc.BUFFER_HEADER_LENGTH:]
156 self.message.append(buffer)
157
158 def connectionLost(self, reason):
159 if not self.done:
160 self.finished.errback(ipc.ConnectionClosedException("Reader read 0 bytes."))
161
162 class TwistedHTTPTransceiver(object):
163 """This transceiver uses the Agent class present in Twisted.web >= 9.0
164 for issuing requests to the remote endpoint."""
165 def __init__(self, host, port, remote_name=None, reactor=None):
166 self.url = "http://%s:%d/" % (host, port)
167
168 if remote_name is None:
169 # There's no easy way to get this peer's remote address
170 # in Twisted so I use a random UUID to identify ourselves
171 import uuid
172 self.remote_name = uuid.uuid4()
173
174 if reactor is None:
175 from twisted.internet import reactor
176 self.agent = Agent(reactor)
177
178 def read_framed_message(self, response):
179 finished = Deferred()
180 response.deliverBody(AvroProtocol(finished))
181 return finished
182
183 def transceive(self, request):
184 req_method = 'POST'
185 req_headers = {
186 'Content-Type': ['avro/binary'],
187 'Accept-Encoding': ['identity'],
188 }
189
190 body_producer = RequestStreamingProducer(request)
191 d = self.agent.request(
192 req_method,
193 self.url,
194 headers=Headers(req_headers),
195 bodyProducer=body_producer)
196 return d.addCallback(self.read_framed_message)
197
198 class AvroResponderResource(resource.Resource):
199 """This Twisted.web resource can be placed anywhere in a URL hierarchy
200 to provide an Avro endpoint. Different Avro protocols can be served
201 by the same web server as long as they are in different resources in
202 a URL hierarchy."""
203 isLeaf = True
204
205 def __init__(self, responder):
206 resource.Resource.__init__(self)
207 self.responder = responder
208
209 def cb_render_POST(self, resp_body, request):
210 request.setResponseCode(200)
211 request.setHeader('Content-Type', 'avro/binary')
212 resp_writer = ipc.FramedWriter(request)
213 resp_writer.write_framed_message(resp_body)
214 request.finish()
215
216 def render_POST(self, request):
217 # Unfortunately, Twisted.web doesn't support incoming
218 # streamed input yet, the whole payload must be kept in-memory
219 request.content.seek(0, 0)
220 call_request_reader = ipc.FramedReader(request.content)
221 call_request = call_request_reader.read_framed_message()
222 d = maybeDeferred(self.responder.respond, call_request)
223 d.addCallback(self.cb_render_POST, request)
224 return server.NOT_DONE_YET