| 1 |
#!/usr/bin/env python3 |
| 2 |
# -*- mode: python -*- |
| 3 |
# -*- coding: utf-8 -*- |
| 4 |
|
| 5 |
# Licensed to the Apache Software Foundation (ASF) under one |
| 6 |
# or more contributor license agreements. See the NOTICE file |
| 7 |
# distributed with this work for additional information |
| 8 |
# regarding copyright ownership. The ASF licenses this file |
| 9 |
# to you under the Apache License, Version 2.0 (the |
| 10 |
# "License"); you may not use this file except in compliance |
| 11 |
# with the License. You may obtain a copy of the License at |
| 12 |
# |
| 13 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 14 |
# |
| 15 |
# Unless required by applicable law or agreed to in writing, software |
| 16 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 17 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 18 |
# See the License for the specific language governing permissions and |
| 19 |
# limitations under the License. |
| 20 |
""" |
| 21 |
Protocol implementation. |
| 22 |
""" |
| 23 |
|
| 24 |
|
| 25 |
import hashlib |
| 26 |
import json |
| 27 |
import logging |
| 28 |
|
| 29 |
from avro import schema |
| 30 |
|
| 31 |
ImmutableDict = schema.ImmutableDict |
| 32 |
|
| 33 |
# ------------------------------------------------------------------------------ |
| 34 |
# Constants |
| 35 |
|
| 36 |
# Allowed top-level schemas in a protocol: |
| 37 |
VALID_TYPE_SCHEMA_TYPES = frozenset(['enum', 'record', 'error', 'fixed']) |
| 38 |
|
| 39 |
|
| 40 |
# ------------------------------------------------------------------------------ |
| 41 |
# Exceptions |
| 42 |
|
| 43 |
|
| 44 |
class ProtocolParseException(schema.AvroException): |
| 45 |
"""Error while parsing a JSON protocol descriptor.""" |
| 46 |
pass |
| 47 |
|
| 48 |
|
| 49 |
# ------------------------------------------------------------------------------ |
| 50 |
# Base Classes |
| 51 |
|
| 52 |
|
| 53 |
class Protocol(object): |
| 54 |
"""An application protocol.""" |
| 55 |
|
| 56 |
@staticmethod |
| 57 |
def _ParseTypeDesc(type_desc, names): |
| 58 |
type_schema = schema.SchemaFromJSONData(type_desc, names=names) |
| 59 |
if type_schema.type not in VALID_TYPE_SCHEMA_TYPES: |
| 60 |
raise ProtocolParseException( |
| 61 |
'Invalid type %r in protocol %r: ' |
| 62 |
'protocols can only declare types %s.' |
| 63 |
% (type_schema, avro_name, ','.join(VALID_TYPE_SCHEMA_TYPES))) |
| 64 |
return type_schema |
| 65 |
|
| 66 |
@staticmethod |
| 67 |
def _ParseMessageDesc(name, message_desc, names): |
| 68 |
"""Parses a protocol message descriptor. |
| 69 |
|
| 70 |
Args: |
| 71 |
name: Name of the message. |
| 72 |
message_desc: Descriptor of the message. |
| 73 |
names: Tracker of the named Avro schema. |
| 74 |
Returns: |
| 75 |
The parsed protocol message. |
| 76 |
Raises: |
| 77 |
ProtocolParseException: if the descriptor is invalid. |
| 78 |
""" |
| 79 |
request_desc = message_desc.get('request') |
| 80 |
if request_desc is None: |
| 81 |
raise ProtocolParseException( |
| 82 |
'Invalid message descriptor with no "request": %r.' % message_desc) |
| 83 |
request_schema = Message._ParseRequestFromJSONDesc( |
| 84 |
request_desc=request_desc, |
| 85 |
names=names, |
| 86 |
) |
| 87 |
|
| 88 |
response_desc = message_desc.get('response') |
| 89 |
if response_desc is None: |
| 90 |
raise ProtocolParseException( |
| 91 |
'Invalid message descriptor with no "response": %r.' % message_desc) |
| 92 |
response_schema = Message._ParseResponseFromJSONDesc( |
| 93 |
response_desc=response_desc, |
| 94 |
names=names, |
| 95 |
) |
| 96 |
|
| 97 |
# Errors are optional: |
| 98 |
errors_desc = message_desc.get('errors', tuple()) |
| 99 |
error_union_schema = Message._ParseErrorsFromJSONDesc( |
| 100 |
errors_desc=errors_desc, |
| 101 |
names=names, |
| 102 |
) |
| 103 |
|
| 104 |
return Message( |
| 105 |
name=name, |
| 106 |
request=request_schema, |
| 107 |
response=response_schema, |
| 108 |
errors=error_union_schema, |
| 109 |
) |
| 110 |
|
| 111 |
@staticmethod |
| 112 |
def _ParseMessageDescMap(message_desc_map, names): |
| 113 |
for name, message_desc in message_desc_map.items(): |
| 114 |
yield Protocol._ParseMessageDesc( |
| 115 |
name=name, |
| 116 |
message_desc=message_desc, |
| 117 |
names=names, |
| 118 |
) |
| 119 |
|
| 120 |
def __init__( |
| 121 |
self, |
| 122 |
name, |
| 123 |
namespace=None, |
| 124 |
types=tuple(), |
| 125 |
messages=tuple(), |
| 126 |
): |
| 127 |
"""Initializes a new protocol object. |
| 128 |
|
| 129 |
Args: |
| 130 |
name: Protocol name (absolute or relative). |
| 131 |
namespace: Optional explicit namespace (if name is relative). |
| 132 |
types: Collection of types in the protocol. |
| 133 |
messages: Collection of messages in the protocol. |
| 134 |
""" |
| 135 |
self._avro_name = schema.Name(name=name, namespace=namespace) |
| 136 |
self._fullname = self._avro_name.fullname |
| 137 |
self._name = self._avro_name.simple_name |
| 138 |
self._namespace = self._avro_name.namespace |
| 139 |
|
| 140 |
self._props = {} |
| 141 |
self._props['name'] = self._name |
| 142 |
if self._namespace: |
| 143 |
self._props['namespace'] = self._namespace |
| 144 |
|
| 145 |
self._names = schema.Names(default_namespace=self._namespace) |
| 146 |
|
| 147 |
self._types = tuple(types) |
| 148 |
# Map: type full name -> type schema |
| 149 |
self._type_map = ( |
| 150 |
ImmutableDict((type.fullname, type) for type in self._types)) |
| 151 |
# This assertion cannot fail unless we don't track named schemas properly: |
| 152 |
assert (len(self._types) == len(self._type_map)), ( |
| 153 |
'Type list %r does not match type map: %r' |
| 154 |
% (self._types, self._type_map)) |
| 155 |
# TODO: set props['types'] |
| 156 |
|
| 157 |
self._messages = tuple(messages) |
| 158 |
|
| 159 |
# Map: message name -> Message |
| 160 |
# Note that message names are simple names unique within the protocol. |
| 161 |
self._message_map = ImmutableDict( |
| 162 |
items=((message.name, message) for message in self._messages)) |
| 163 |
if len(self._messages) != len(self._message_map): |
| 164 |
raise ProtocolParseException( |
| 165 |
'Invalid protocol %s with duplicate message name: %r' |
| 166 |
% (self._avro_name, self._messages)) |
| 167 |
# TODO: set props['messages'] |
| 168 |
|
| 169 |
self._md5 = hashlib.md5(str(self).encode('utf-8')).digest() |
| 170 |
|
| 171 |
@property |
| 172 |
def name(self): |
| 173 |
"""Returns: the simple name of the protocol.""" |
| 174 |
return self._name |
| 175 |
|
| 176 |
@property |
| 177 |
def namespace(self): |
| 178 |
"""Returns: the namespace this protocol belongs to.""" |
| 179 |
return self._namespace |
| 180 |
|
| 181 |
@property |
| 182 |
def fullname(self): |
| 183 |
"""Returns: the fully qualified name of this protocol.""" |
| 184 |
return self._fullname |
| 185 |
|
| 186 |
@property |
| 187 |
def types(self): |
| 188 |
"""Returns: the collection of types declared in this protocol.""" |
| 189 |
return self._types |
| 190 |
|
| 191 |
@property |
| 192 |
def type_map(self): |
| 193 |
"""Returns: the map of types in this protocol, indexed by their full name.""" |
| 194 |
return self._type_map |
| 195 |
|
| 196 |
@property |
| 197 |
def messages(self): |
| 198 |
"""Returns: the collection of messages declared in this protocol.""" |
| 199 |
return self._messages |
| 200 |
|
| 201 |
@property |
| 202 |
def message_map(self): |
| 203 |
"""Returns: the map of messages in this protocol, indexed by their name.""" |
| 204 |
return self._message_map |
| 205 |
|
| 206 |
@property |
| 207 |
def md5(self): |
| 208 |
return self._md5 |
| 209 |
|
| 210 |
@property |
| 211 |
def props(self): |
| 212 |
return self._props |
| 213 |
|
| 214 |
def to_json(self): |
| 215 |
to_dump = {} |
| 216 |
to_dump['protocol'] = self.name |
| 217 |
names = schema.Names(default_namespace=self.namespace) |
| 218 |
if self.namespace: |
| 219 |
to_dump['namespace'] = self.namespace |
| 220 |
if self.types: |
| 221 |
to_dump['types'] = [ t.to_json(names) for t in self.types ] |
| 222 |
if self.messages: |
| 223 |
messages_dict = {} |
| 224 |
for name, body in self.message_map.items(): |
| 225 |
messages_dict[name] = body.to_json(names) |
| 226 |
to_dump['messages'] = messages_dict |
| 227 |
return to_dump |
| 228 |
|
| 229 |
def __str__(self): |
| 230 |
return json.dumps(self.to_json()) |
| 231 |
|
| 232 |
def __eq__(self, that): |
| 233 |
to_cmp = json.loads(str(self)) |
| 234 |
return to_cmp == json.loads(str(that)) |
| 235 |
|
| 236 |
|
| 237 |
# ------------------------------------------------------------------------------ |
| 238 |
|
| 239 |
|
| 240 |
class Message(object): |
| 241 |
"""A Protocol message.""" |
| 242 |
|
| 243 |
@staticmethod |
| 244 |
def _ParseRequestFromJSONDesc(request_desc, names): |
| 245 |
"""Parses the request descriptor of a protocol message. |
| 246 |
|
| 247 |
Args: |
| 248 |
request_desc: Descriptor of the message request. |
| 249 |
This is a list of fields that defines an unnamed record. |
| 250 |
names: Tracker for named Avro schemas. |
| 251 |
Returns: |
| 252 |
The parsed request schema, as an unnamed record. |
| 253 |
""" |
| 254 |
fields = schema.RecordSchema._MakeFieldList(request_desc, names=names) |
| 255 |
return schema.RecordSchema( |
| 256 |
name=None, |
| 257 |
namespace=None, |
| 258 |
fields=fields, |
| 259 |
names=names, |
| 260 |
record_type=schema.REQUEST, |
| 261 |
) |
| 262 |
|
| 263 |
@staticmethod |
| 264 |
def _ParseResponseFromJSONDesc(response_desc, names): |
| 265 |
"""Parses the response descriptor of a protocol message. |
| 266 |
|
| 267 |
Args: |
| 268 |
response_desc: Descriptor of the message response. |
| 269 |
This is an arbitrary Avro schema descriptor. |
| 270 |
Returns: |
| 271 |
The parsed response schema. |
| 272 |
""" |
| 273 |
return schema.SchemaFromJSONData(response_desc, names=names) |
| 274 |
|
| 275 |
@staticmethod |
| 276 |
def _ParseErrorsFromJSONDesc(errors_desc, names): |
| 277 |
"""Parses the errors descriptor of a protocol message. |
| 278 |
|
| 279 |
Args: |
| 280 |
errors_desc: Descriptor of the errors thrown by the protocol message. |
| 281 |
This is a list of error types understood as an implicit union. |
| 282 |
Each error type is an arbitrary Avro schema. |
| 283 |
names: Tracker for named Avro schemas. |
| 284 |
Returns: |
| 285 |
The parsed ErrorUnionSchema. |
| 286 |
""" |
| 287 |
error_union_desc = { |
| 288 |
'type': schema.ERROR_UNION, |
| 289 |
'declared_errors': errors_desc, |
| 290 |
} |
| 291 |
return schema.SchemaFromJSONData(error_union_desc, names=names) |
| 292 |
|
| 293 |
def __init__(self, name, request, response, errors=None): |
| 294 |
self._name = name |
| 295 |
|
| 296 |
self._props = {} |
| 297 |
# TODO: set properties |
| 298 |
self._request = request |
| 299 |
self._response = response |
| 300 |
self._errors = errors |
| 301 |
|
| 302 |
@property |
| 303 |
def name(self): |
| 304 |
return self._name |
| 305 |
|
| 306 |
@property |
| 307 |
def request(self): |
| 308 |
return self._request |
| 309 |
|
| 310 |
@property |
| 311 |
def response(self): |
| 312 |
return self._response |
| 313 |
|
| 314 |
@property |
| 315 |
def errors(self): |
| 316 |
return self._errors |
| 317 |
|
| 318 |
def props(self): |
| 319 |
return self._props |
| 320 |
|
| 321 |
def __str__(self): |
| 322 |
return json.dumps(self.to_json()) |
| 323 |
|
| 324 |
def to_json(self, names=None): |
| 325 |
if names is None: |
| 326 |
names = schema.Names() |
| 327 |
to_dump = {} |
| 328 |
to_dump['request'] = self.request.to_json(names) |
| 329 |
to_dump['response'] = self.response.to_json(names) |
| 330 |
if self.errors: |
| 331 |
to_dump['errors'] = self.errors.to_json(names) |
| 332 |
return to_dump |
| 333 |
|
| 334 |
def __eq__(self, that): |
| 335 |
return self.name == that.name and self.props == that.props |
| 336 |
|
| 337 |
|
| 338 |
# ------------------------------------------------------------------------------ |
| 339 |
|
| 340 |
|
| 341 |
def ProtocolFromJSONData(json_data): |
| 342 |
"""Builds an Avro Protocol from its JSON descriptor. |
| 343 |
|
| 344 |
Args: |
| 345 |
json_data: JSON data representing the descriptor of the Avro protocol. |
| 346 |
Returns: |
| 347 |
The Avro Protocol parsed from the JSON descriptor. |
| 348 |
Raises: |
| 349 |
ProtocolParseException: if the descriptor is invalid. |
| 350 |
""" |
| 351 |
if type(json_data) != dict: |
| 352 |
raise ProtocolParseException( |
| 353 |
'Invalid JSON descriptor for an Avro protocol: %r' % json_data) |
| 354 |
|
| 355 |
name = json_data.get('protocol') |
| 356 |
if name is None: |
| 357 |
raise ProtocolParseException( |
| 358 |
'Invalid protocol descriptor with no "name": %r' % json_data) |
| 359 |
|
| 360 |
# Namespace is optional |
| 361 |
namespace = json_data.get('namespace') |
| 362 |
|
| 363 |
avro_name = schema.Name(name=name, namespace=namespace) |
| 364 |
names = schema.Names(default_namespace=avro_name.namespace) |
| 365 |
|
| 366 |
type_desc_list = json_data.get('types', tuple()) |
| 367 |
types = tuple(map( |
| 368 |
lambda desc: Protocol._ParseTypeDesc(desc, names=names), |
| 369 |
type_desc_list)) |
| 370 |
|
| 371 |
message_desc_map = json_data.get('messages', dict()) |
| 372 |
messages = tuple(Protocol._ParseMessageDescMap(message_desc_map, names=names)) |
| 373 |
|
| 374 |
return Protocol( |
| 375 |
name=name, |
| 376 |
namespace=namespace, |
| 377 |
types=types, |
| 378 |
messages=messages, |
| 379 |
) |
| 380 |
|
| 381 |
|
| 382 |
def Parse(json_string): |
| 383 |
"""Constructs a Protocol from its JSON descriptor in text form. |
| 384 |
|
| 385 |
Args: |
| 386 |
json_string: String representation of the JSON descriptor of the protocol. |
| 387 |
Returns: |
| 388 |
The parsed protocol. |
| 389 |
Raises: |
| 390 |
ProtocolParseException: on JSON parsing error, |
| 391 |
or if the JSON descriptor is invalid. |
| 392 |
""" |
| 393 |
try: |
| 394 |
json_data = json.loads(json_string) |
| 395 |
except Exception as exn: |
| 396 |
raise ProtocolParseException( |
| 397 |
'Error parsing protocol from JSON: %r. ' |
| 398 |
'Error message: %r.' |
| 399 |
% (json_string, exn)) |
| 400 |
|
| 401 |
return ProtocolFromJSONData(json_data) |
| 402 |
|