Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add new including_default_value_without_presence_fields to Python JSON serializer.

Browse Source 
This flag has consistent behavior between proto2 and proto3 optionals (by not including either one), unlike including_default_value_fields which does include proto2 optional but excludes proto3 optionals.

including_default_value_fields is now deprecated and will be removed in an upcoming release.

PiperOrigin-RevId: 603156447
pull/15660/head
Protobuf Team Bot 10 months ago committed by Copybara-Service
parent f9ec860691
commit 25c6d34d4e
5 changed files with 1053 additions and 589 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1098
      python/google/protobuf/internal/json_format_test.py
  2. 3
      python/google/protobuf/internal/message_test.py
  3. 39
      python/google/protobuf/internal/test_proto2.proto
  4. 3
      python/google/protobuf/internal/test_proto3_optional.proto
  5. 499
      python/google/protobuf/json_format.py

1098
python/google/protobuf/internal/json_format_test.py
View File

File diff suppressed because it is too large Load Diff

3
python/google/protobuf/internal/message_test.py
Unescape View File

@ -1777,7 +1777,8 @@ class Proto3Test(unittest.TestCase):
# Test has presence: # Test has presence:
for field in test_proto3_optional_pb2.TestProto3Optional.DESCRIPTOR.fields: for field in test_proto3_optional_pb2.TestProto3Optional.DESCRIPTOR.fields:
self.assertTrue(field.has_presence) if field.name.startswith('optional_'):
self.assertTrue(field.has_presence)
for field in unittest_pb2.TestAllTypes.DESCRIPTOR.fields: for field in unittest_pb2.TestAllTypes.DESCRIPTOR.fields:
if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
self.assertFalse(field.has_presence) self.assertFalse(field.has_presence)

39
python/google/protobuf/internal/test_proto2.proto
Unescape View File

@ -0,0 +1,39 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd
syntax = "proto2";
package google.protobuf.python.internal;
message TestProto2 {
message NestedMessage {
// The field name "b" fails to compile in proto1 because it conflicts with
// a local variable named "b" in one of the generated methods. Doh.
// This file needs to compile in proto1 to test backwards-compatibility.
optional int32 bb = 1;
}
enum NestedEnum {
UNSPECIFIED = 0;
FOO = 1;
BAR = 2;
BAZ = 3;
NEG = -1; // Intentionally negative.
}
optional int32 optional_int32 = 1;
optional double optional_double = 12;
optional bool optional_bool = 13;
optional string optional_string = 14;
optional bytes optional_bytes = 15;
optional NestedMessage optional_nested_message = 18;
optional NestedEnum optional_nested_enum = 21;
repeated int32 repeated_int32 = 22;
repeated NestedMessage repeated_nested_message = 23;
}

3
python/google/protobuf/internal/test_proto3_optional.proto
Unescape View File

@ -44,4 +44,7 @@ message TestProto3Optional {
optional NestedMessage optional_nested_message = 18; optional NestedMessage optional_nested_message = 18;
optional NestedEnum optional_nested_enum = 21; optional NestedEnum optional_nested_enum = 21;
repeated int32 repeated_int32 = 22;
repeated NestedMessage repeated_nested_message = 23;
} }

499
python/google/protobuf/json_format.py
Unescape View File

@ -27,26 +27,33 @@ import math
from operator import methodcaller from operator import methodcaller
import re import re
from google.protobuf.internal import type_checkers
from google.protobuf import descriptor from google.protobuf import descriptor
from google.protobuf import message_factory from google.protobuf import message_factory
from google.protobuf import symbol_database from google.protobuf import symbol_database
from google.protobuf.internal import type_checkers
_INT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT32, _INT_TYPES = frozenset([
descriptor.FieldDescriptor.CPPTYPE_UINT32, descriptor.FieldDescriptor.CPPTYPE_INT32,
descriptor.FieldDescriptor.CPPTYPE_INT64, descriptor.FieldDescriptor.CPPTYPE_UINT32,
descriptor.FieldDescriptor.CPPTYPE_UINT64]) descriptor.FieldDescriptor.CPPTYPE_INT64,
_INT64_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_INT64, descriptor.FieldDescriptor.CPPTYPE_UINT64,
descriptor.FieldDescriptor.CPPTYPE_UINT64]) ])
_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT, _INT64_TYPES = frozenset([
descriptor.FieldDescriptor.CPPTYPE_DOUBLE]) descriptor.FieldDescriptor.CPPTYPE_INT64,
descriptor.FieldDescriptor.CPPTYPE_UINT64,
])
_FLOAT_TYPES = frozenset([
descriptor.FieldDescriptor.CPPTYPE_FLOAT,
descriptor.FieldDescriptor.CPPTYPE_DOUBLE,
])
_INFINITY = 'Infinity' _INFINITY = 'Infinity'
_NEG_INFINITY = '-Infinity' _NEG_INFINITY = '-Infinity'
_NAN = 'NaN' _NAN = 'NaN'
_UNPAIRED_SURROGATE_PATTERN = re.compile( _UNPAIRED_SURROGATE_PATTERN = re.compile(
u'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]') '[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]'
)
_VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$') _VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')
@ -72,28 +79,36 @@ def MessageToJson(
use_integers_for_enums=False, use_integers_for_enums=False,
descriptor_pool=None, descriptor_pool=None,
float_precision=None, float_precision=None,
ensure_ascii=True): ensure_ascii=True,
including_default_value_without_presence_fields=False,
):
"""Converts protobuf message to JSON format. """Converts protobuf message to JSON format.
Args: Args:
message: The protocol buffers message instance to serialize. message: The protocol buffers message instance to serialize.
including_default_value_fields: If True, singular primitive fields, including_default_value_fields: (DEPRECATED: use
repeated fields, and map fields will always be serialized. If including_default_value_without_presence_fields to correctly treats proto2
False, only serialize non-empty fields. Singular message fields and proto3 optional the same). If True, fields without presence (implicit
and oneof fields are not affected by this option. presence scalars, repeated fields, and map fields) and Proto2 optional
preserving_proto_field_name: If True, use the original proto field scalars will always be serialized. Singular message fields, oneof fields
names as defined in the .proto file. If False, convert the field and Proto3 optional scalars are not affected by this option.
names to lowerCamelCase. including_default_value_without_presence_fields: If True, fields without
indent: The JSON object will be pretty-printed with this indent level. presence (implicit presence scalars, repeated fields, and map fields) will
An indent level of 0 or negative will only insert newlines. If the always be serialized. Any field that supports presence is not affected by
indent level is None, no newlines will be inserted. this option (including singular message fields and oneof fields).
preserving_proto_field_name: If True, use the original proto field names as
defined in the .proto file. If False, convert the field names to
lowerCamelCase.
indent: The JSON object will be pretty-printed with this indent level. An
indent level of 0 or negative will only insert newlines. If the indent
level is None, no newlines will be inserted.
sort_keys: If True, then the output will be sorted by field names. sort_keys: If True, then the output will be sorted by field names.
use_integers_for_enums: If true, print integers instead of enum names. use_integers_for_enums: If true, print integers instead of enum names.
descriptor_pool: A Descriptor Pool for resolving types. If None use the descriptor_pool: A Descriptor Pool for resolving types. If None use the
default. default.
float_precision: If set, use this to specify float field valid digits. float_precision: If set, use this to specify float field valid digits.
ensure_ascii: If True, strings with non-ASCII characters are escaped. ensure_ascii: If True, strings with non-ASCII characters are escaped. If
If False, Unicode strings are returned unchanged. False, Unicode strings are returned unchanged.
Returns: Returns:
A string containing the JSON formatted protocol buffer message. A string containing the JSON formatted protocol buffer message.
@ -103,33 +118,43 @@ def MessageToJson(
preserving_proto_field_name, preserving_proto_field_name,
use_integers_for_enums, use_integers_for_enums,
descriptor_pool, descriptor_pool,
float_precision=float_precision) float_precision,
including_default_value_without_presence_fields
)
return printer.ToJsonString(message, indent, sort_keys, ensure_ascii) return printer.ToJsonString(message, indent, sort_keys, ensure_ascii)
def MessageToDict( def MessageToDict(
message, message,
including_default_value_fields=False, including_default_value_fields=False,
including_default_value_without_presence_fields=False,
preserving_proto_field_name=False, preserving_proto_field_name=False,
use_integers_for_enums=False, use_integers_for_enums=False,
descriptor_pool=None, descriptor_pool=None,
float_precision=None): float_precision=None,
):
"""Converts protobuf message to a dictionary. """Converts protobuf message to a dictionary.
When the dictionary is encoded to JSON, it conforms to proto3 JSON spec. When the dictionary is encoded to JSON, it conforms to proto3 JSON spec.
Args: Args:
message: The protocol buffers message instance to serialize. message: The protocol buffers message instance to serialize.
including_default_value_fields: If True, singular primitive fields, including_default_value_fields: (DEPRECATED: use
repeated fields, and map fields will always be serialized. If including_default_value_without_presence_fields to correctly treats proto2
False, only serialize non-empty fields. Singular message fields and proto3 optional the same). If True, fields without presence (implicit
and oneof fields are not affected by this option. presence scalars, repeated fields, and map fields) and Proto2 optional
preserving_proto_field_name: If True, use the original proto field scalars will always be serialized. Singular message fields, oneof fields
names as defined in the .proto file. If False, convert the field and Proto3 optional scalars are not affected by this option.
names to lowerCamelCase. including_default_value_without_presence_fields: If True, fields without
presence (implicit presence scalars, repeated fields, and map fields) will
always be serialized. Any field that supports presence is not affected by
this option (including singular message fields and oneof fields).
preserving_proto_field_name: If True, use the original proto field names as
defined in the .proto file. If False, convert the field names to
lowerCamelCase.
use_integers_for_enums: If true, print integers instead of enum names. use_integers_for_enums: If true, print integers instead of enum names.
descriptor_pool: A Descriptor Pool for resolving types. If None use the descriptor_pool: A Descriptor Pool for resolving types. If None use the
default. default.
float_precision: If set, use this to specify float field valid digits. float_precision: If set, use this to specify float field valid digits.
Returns: Returns:
@ -140,15 +165,19 @@ def MessageToDict(
preserving_proto_field_name, preserving_proto_field_name,
use_integers_for_enums, use_integers_for_enums,
descriptor_pool, descriptor_pool,
float_precision=float_precision) float_precision,
including_default_value_without_presence_fields,
)
# pylint: disable=protected-access # pylint: disable=protected-access
return printer._MessageToJsonObject(message) return printer._MessageToJsonObject(message)
def _IsMapEntry(field): def _IsMapEntry(field):
return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and return (
field.message_type.has_options and field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
field.message_type.GetOptions().map_entry) and field.message_type.has_options
and field.message_type.GetOptions().map_entry
)
class _Printer(object): class _Printer(object):
@ -160,8 +189,13 @@ class _Printer(object):
preserving_proto_field_name=False, preserving_proto_field_name=False,
use_integers_for_enums=False, use_integers_for_enums=False,
descriptor_pool=None, descriptor_pool=None,
float_precision=None): float_precision=None,
including_default_value_without_presence_fields=False,
):
self.including_default_value_fields = including_default_value_fields self.including_default_value_fields = including_default_value_fields
self.including_default_value_without_presence_fields = (
including_default_value_without_presence_fields
)
self.preserving_proto_field_name = preserving_proto_field_name self.preserving_proto_field_name = preserving_proto_field_name
self.use_integers_for_enums = use_integers_for_enums self.use_integers_for_enums = use_integers_for_enums
self.descriptor_pool = descriptor_pool self.descriptor_pool = descriptor_pool
@ -173,7 +207,8 @@ class _Printer(object):
def ToJsonString(self, message, indent, sort_keys, ensure_ascii): def ToJsonString(self, message, indent, sort_keys, ensure_ascii):
js = self._MessageToJsonObject(message) js = self._MessageToJsonObject(message)
return json.dumps( return json.dumps(
js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii) js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii
)
def _MessageToJsonObject(self, message): def _MessageToJsonObject(self, message):
"""Converts message to an object according to Proto3 JSON Specification.""" """Converts message to an object according to Proto3 JSON Specification."""
@ -208,13 +243,11 @@ class _Printer(object):
recorded_key = 'false' recorded_key = 'false'
else: else:
recorded_key = str(key) recorded_key = str(key)
js_map[recorded_key] = self._FieldToJsonObject( js_map[recorded_key] = self._FieldToJsonObject(v_field, value[key])
v_field, value[key])
js[name] = js_map js[name] = js_map
elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
# Convert a repeated field. # Convert a repeated field.
js[name] = [self._FieldToJsonObject(field, k) js[name] = [self._FieldToJsonObject(field, k) for k in value]
for k in value]
elif field.is_extension: elif field.is_extension:
name = '[%s]' % field.full_name name = '[%s]' % field.full_name
js[name] = self._FieldToJsonObject(field, value) js[name] = self._FieldToJsonObject(field, value)
@ -222,14 +255,33 @@ class _Printer(object):
js[name] = self._FieldToJsonObject(field, value) js[name] = self._FieldToJsonObject(field, value)
# Serialize default value if including_default_value_fields is True. # Serialize default value if including_default_value_fields is True.
if self.including_default_value_fields: if (
self.including_default_value_fields
or self.including_default_value_without_presence_fields
):
message_descriptor = message.DESCRIPTOR message_descriptor = message.DESCRIPTOR
for field in message_descriptor.fields: for field in message_descriptor.fields:
# Singular message fields and oneof fields will not be affected. # including_default_value doesn't apply to singular messages or any
if ((field.label != descriptor.FieldDescriptor.LABEL_REPEATED and # field with a containing oneof (including proto3 optionals which use
field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE) or # a synthetic oneof).
field.containing_oneof): if self.including_default_value_fields and (
(
field.label != descriptor.FieldDescriptor.LABEL_REPEATED
and field.cpp_type
== descriptor.FieldDescriptor.CPPTYPE_MESSAGE
)
or field.containing_oneof
):
continue
# including_default_value_without_presence_fields doesn't apply to
# any field which supports presence.
if (
self.including_default_value_without_presence_fields
and field.has_presence
):
continue continue
if self.preserving_proto_field_name: if self.preserving_proto_field_name:
name = field.name name = field.name
else: else:
@ -246,7 +298,8 @@ class _Printer(object):
except ValueError as e: except ValueError as e:
raise SerializeToJsonError( raise SerializeToJsonError(
'Failed to serialize {0} field: {1}.'.format(field.name, e)) from e 'Failed to serialize {0} field: {1}.'.format(field.name, e)
) from e
return js return js
@ -264,8 +317,10 @@ class _Printer(object):
return enum_value.name return enum_value.name
else: else:
if field.enum_type.is_closed: if field.enum_type.is_closed:
raise SerializeToJsonError('Enum field contains an integer value ' raise SerializeToJsonError(
'which can not mapped to an enum value.') 'Enum field contains an integer value '
'which can not mapped to an enum value.'
)
else: else:
return value return value
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
@ -310,8 +365,9 @@ class _Printer(object):
js['value'] = self._WrapperMessageToJsonObject(sub_message) js['value'] = self._WrapperMessageToJsonObject(sub_message)
return js return js
if full_name in _WKTJSONMETHODS: if full_name in _WKTJSONMETHODS:
js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0], js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0], sub_message)(
sub_message)(self) self
)
return js return js
return self._RegularMessageToJsonObject(sub_message, js) return self._RegularMessageToJsonObject(sub_message, js)
@ -333,11 +389,15 @@ class _Printer(object):
if which == 'number_value': if which == 'number_value':
value = message.number_value value = message.number_value
if math.isinf(value): if math.isinf(value):
raise ValueError('Fail to serialize Infinity for Value.number_value, ' raise ValueError(
'which would parse as string_value') 'Fail to serialize Infinity for Value.number_value, '
'which would parse as string_value'
)
if math.isnan(value): if math.isnan(value):
raise ValueError('Fail to serialize NaN for Value.number_value, ' raise ValueError(
'which would parse as string_value') 'Fail to serialize NaN for Value.number_value, '
'which would parse as string_value'
)
else: else:
value = getattr(message, which) value = getattr(message, which)
oneof_descriptor = message.DESCRIPTOR.fields_by_name[which] oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]
@ -345,8 +405,7 @@ class _Printer(object):
def _ListValueMessageToJsonObject(self, message): def _ListValueMessageToJsonObject(self, message):
"""Converts ListValue message according to Proto3 JSON Specification.""" """Converts ListValue message according to Proto3 JSON Specification."""
return [self._ValueMessageToJsonObject(value) return [self._ValueMessageToJsonObject(value) for value in message.values]
for value in message.values]
def _StructMessageToJsonObject(self, message): def _StructMessageToJsonObject(self, message):
"""Converts Struct message according to Proto3 JSON Specification.""" """Converts Struct message according to Proto3 JSON Specification."""
@ -358,7 +417,8 @@ class _Printer(object):
def _WrapperMessageToJsonObject(self, message): def _WrapperMessageToJsonObject(self, message):
return self._FieldToJsonObject( return self._FieldToJsonObject(
message.DESCRIPTOR.fields_by_name['value'], message.value) message.DESCRIPTOR.fields_by_name['value'], message.value
)
def _IsWrapperMessage(message_descriptor): def _IsWrapperMessage(message_descriptor):
@ -384,16 +444,18 @@ def _CreateMessageFromTypeUrl(type_url, descriptor_pool):
except KeyError as e: except KeyError as e:
raise TypeError( raise TypeError(
'Can not find message descriptor by type_url: {0}'.format(type_url) 'Can not find message descriptor by type_url: {0}'.format(type_url)
) from e ) from e
message_class = message_factory.GetMessageClass(message_descriptor) message_class = message_factory.GetMessageClass(message_descriptor)
return message_class() return message_class()
def Parse(text, def Parse(
message, text,
ignore_unknown_fields=False, message,
descriptor_pool=None, ignore_unknown_fields=False,
max_recursion_depth=100): descriptor_pool=None,
max_recursion_depth=100,
):
"""Parses a JSON representation of a protocol message into a message. """Parses a JSON representation of a protocol message into a message.
Args: Args:
@ -402,9 +464,9 @@ def Parse(text,
ignore_unknown_fields: If True, do not raise errors for unknown fields. ignore_unknown_fields: If True, do not raise errors for unknown fields.
descriptor_pool: A Descriptor Pool for resolving types. If None use the descriptor_pool: A Descriptor Pool for resolving types. If None use the
default. default.
max_recursion_depth: max recursion depth of JSON message to be max_recursion_depth: max recursion depth of JSON message to be deserialized.
deserialized. JSON messages over this depth will fail to be JSON messages over this depth will fail to be deserialized. Default value
deserialized. Default value is 100. is 100.
Returns: Returns:
The same message passed as argument. The same message passed as argument.
@ -418,15 +480,18 @@ def Parse(text,
js = json.loads(text, object_pairs_hook=_DuplicateChecker) js = json.loads(text, object_pairs_hook=_DuplicateChecker)
except ValueError as e: except ValueError as e:
raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e
return ParseDict(js, message, ignore_unknown_fields, descriptor_pool, return ParseDict(
max_recursion_depth) js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth
)
def ParseDict(js_dict, def ParseDict(
message, js_dict,
ignore_unknown_fields=False, message,
descriptor_pool=None, ignore_unknown_fields=False,
max_recursion_depth=100): descriptor_pool=None,
max_recursion_depth=100,
):
"""Parses a JSON dictionary representation into a message. """Parses a JSON dictionary representation into a message.
Args: Args:
@ -435,9 +500,9 @@ def ParseDict(js_dict,
ignore_unknown_fields: If True, do not raise errors for unknown fields. ignore_unknown_fields: If True, do not raise errors for unknown fields.
descriptor_pool: A Descriptor Pool for resolving types. If None use the descriptor_pool: A Descriptor Pool for resolving types. If None use the
default. default.
max_recursion_depth: max recursion depth of JSON message to be max_recursion_depth: max recursion depth of JSON message to be deserialized.
deserialized. JSON messages over this depth will fail to be JSON messages over this depth will fail to be deserialized. Default value
deserialized. Default value is 100. is 100.
Returns: Returns:
The same message passed as argument. The same message passed as argument.
@ -453,8 +518,9 @@ _INT_OR_FLOAT = (int, float)
class _Parser(object): class _Parser(object):
"""JSON format parser for protocol message.""" """JSON format parser for protocol message."""
def __init__(self, ignore_unknown_fields, descriptor_pool, def __init__(
max_recursion_depth): self, ignore_unknown_fields, descriptor_pool, max_recursion_depth
):
self.ignore_unknown_fields = ignore_unknown_fields self.ignore_unknown_fields = ignore_unknown_fields
self.descriptor_pool = descriptor_pool self.descriptor_pool = descriptor_pool
self.max_recursion_depth = max_recursion_depth self.max_recursion_depth = max_recursion_depth
@ -473,8 +539,11 @@ class _Parser(object):
""" """
self.recursion_depth += 1 self.recursion_depth += 1
if self.recursion_depth > self.max_recursion_depth: if self.recursion_depth > self.max_recursion_depth:
raise ParseError('Message too deep. Max recursion depth is {0}'.format( raise ParseError(
self.max_recursion_depth)) 'Message too deep. Max recursion depth is {0}'.format(
self.max_recursion_depth
)
)
message_descriptor = message.DESCRIPTOR message_descriptor = message.DESCRIPTOR
full_name = message_descriptor.full_name full_name = message_descriptor.full_name
if not path: if not path:
@ -500,8 +569,9 @@ class _Parser(object):
""" """
names = [] names = []
message_descriptor = message.DESCRIPTOR message_descriptor = message.DESCRIPTOR
fields_by_json_name = dict((f.json_name, f) fields_by_json_name = dict(
for f in message_descriptor.fields) (f.json_name, f) for f in message_descriptor.fields
)
for name in js: for name in js:
try: try:
field = fields_by_json_name.get(name, None) field = fields_by_json_name.get(name, None)
@ -511,7 +581,9 @@ class _Parser(object):
if not message_descriptor.is_extendable: if not message_descriptor.is_extendable:
raise ParseError( raise ParseError(
'Message type {0} does not have extensions at {1}'.format( 'Message type {0} does not have extensions at {1}'.format(
message_descriptor.full_name, path)) message_descriptor.full_name, path
)
)
identifier = name[1:-1] # strip [] brackets identifier = name[1:-1] # strip [] brackets
# pylint: disable=protected-access # pylint: disable=protected-access
field = message.Extensions._FindExtensionByName(identifier) field = message.Extensions._FindExtensionByName(identifier)
@ -527,33 +599,48 @@ class _Parser(object):
if self.ignore_unknown_fields: if self.ignore_unknown_fields:
continue continue
raise ParseError( raise ParseError(
('Message type "{0}" has no field named "{1}" at "{2}".\n' (
' Available Fields(except extensions): "{3}"').format( 'Message type "{0}" has no field named "{1}" at "{2}".\n'
message_descriptor.full_name, name, path, ' Available Fields(except extensions): "{3}"'
[f.json_name for f in message_descriptor.fields])) ).format(
message_descriptor.full_name,
name,
path,
[f.json_name for f in message_descriptor.fields],
)
)
if name in names: if name in names:
raise ParseError('Message type "{0}" should not have multiple ' raise ParseError(
'"{1}" fields at "{2}".'.format( 'Message type "{0}" should not have multiple '
message.DESCRIPTOR.full_name, name, path)) '"{1}" fields at "{2}".'.format(
message.DESCRIPTOR.full_name, name, path
)
)
names.append(name) names.append(name)
value = js[name] value = js[name]
# Check no other oneof field is parsed. # Check no other oneof field is parsed.
if field.containing_oneof is not None and value is not None: if field.containing_oneof is not None and value is not None:
oneof_name = field.containing_oneof.name oneof_name = field.containing_oneof.name
if oneof_name in names: if oneof_name in names:
raise ParseError('Message type "{0}" should not have multiple ' raise ParseError(
'"{1}" oneof fields at "{2}".'.format( 'Message type "{0}" should not have multiple '
message.DESCRIPTOR.full_name, oneof_name, '"{1}" oneof fields at "{2}".'.format(
path)) message.DESCRIPTOR.full_name, oneof_name, path
)
)
names.append(oneof_name) names.append(oneof_name)
if value is None: if value is None:
if (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE if (
and field.message_type.full_name == 'google.protobuf.Value'): field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE
and field.message_type.full_name == 'google.protobuf.Value'
):
sub_message = getattr(message, field.name) sub_message = getattr(message, field.name)
sub_message.null_value = 0 sub_message.null_value = 0
elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM elif (
and field.enum_type.full_name == 'google.protobuf.NullValue'): field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM
and field.enum_type.full_name == 'google.protobuf.NullValue'
):
setattr(message, field.name, 0) setattr(message, field.name, 0)
else: else:
message.ClearField(field.name) message.ClearField(field.name)
@ -562,35 +649,51 @@ class _Parser(object):
# Parse field value. # Parse field value.
if _IsMapEntry(field): if _IsMapEntry(field):
message.ClearField(field.name) message.ClearField(field.name)
self._ConvertMapFieldValue(value, message, field, self._ConvertMapFieldValue(
'{0}.{1}'.format(path, name)) value, message, field, '{0}.{1}'.format(path, name)
)
elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
message.ClearField(field.name) message.ClearField(field.name)
if not isinstance(value, list): if not isinstance(value, list):
raise ParseError('repeated field {0} must be in [] which is ' raise ParseError(
'{1} at {2}'.format(name, value, path)) 'repeated field {0} must be in [] which is {1} at {2}'.format(
name, value, path
)
)
if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
# Repeated message field. # Repeated message field.
for index, item in enumerate(value): for index, item in enumerate(value):
sub_message = getattr(message, field.name).add() sub_message = getattr(message, field.name).add()
# None is a null_value in Value. # None is a null_value in Value.
if (item is None and if (
sub_message.DESCRIPTOR.full_name != 'google.protobuf.Value'): item is None
raise ParseError('null is not allowed to be used as an element' and sub_message.DESCRIPTOR.full_name
' in a repeated field at {0}.{1}[{2}]'.format( != 'google.protobuf.Value'
path, name, index)) ):
self.ConvertMessage(item, sub_message, raise ParseError(
'{0}.{1}[{2}]'.format(path, name, index)) 'null is not allowed to be used as an element'
' in a repeated field at {0}.{1}[{2}]'.format(
path, name, index
)
)
self.ConvertMessage(
item, sub_message, '{0}.{1}[{2}]'.format(path, name, index)
)
else: else:
# Repeated scalar field. # Repeated scalar field.
for index, item in enumerate(value): for index, item in enumerate(value):
if item is None: if item is None:
raise ParseError('null is not allowed to be used as an element' raise ParseError(
' in a repeated field at {0}.{1}[{2}]'.format( 'null is not allowed to be used as an element'
path, name, index)) ' in a repeated field at {0}.{1}[{2}]'.format(
path, name, index
)
)
getattr(message, field.name).append( getattr(message, field.name).append(
_ConvertScalarFieldValue( _ConvertScalarFieldValue(
item, field, '{0}.{1}[{2}]'.format(path, name, index))) item, field, '{0}.{1}[{2}]'.format(path, name, index)
)
)
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
if field.is_extension: if field.is_extension:
sub_message = message.Extensions[field] sub_message = message.Extensions[field]
@ -601,26 +704,30 @@ class _Parser(object):
else: else:
if field.is_extension: if field.is_extension:
message.Extensions[field] = _ConvertScalarFieldValue( message.Extensions[field] = _ConvertScalarFieldValue(
value, field, '{0}.{1}'.format(path, name)) value, field, '{0}.{1}'.format(path, name)
)
else: else:
setattr( setattr(
message, field.name, message,
_ConvertScalarFieldValue(value, field, field.name,
'{0}.{1}'.format(path, name))) _ConvertScalarFieldValue(
value, field, '{0}.{1}'.format(path, name)
),
)
except ParseError as e: except ParseError as e:
if field and field.containing_oneof is None: if field and field.containing_oneof is None:
raise ParseError( raise ParseError(
'Failed to parse {0} field: {1}.'.format(name, e) 'Failed to parse {0} field: {1}.'.format(name, e)
) from e ) from e
else: else:
raise ParseError(str(e)) from e raise ParseError(str(e)) from e
except ValueError as e: except ValueError as e:
raise ParseError( raise ParseError(
'Failed to parse {0} field: {1}.'.format(name, e) 'Failed to parse {0} field: {1}.'.format(name, e)
) from e ) from e
except TypeError as e: except TypeError as e:
raise ParseError( raise ParseError(
'Failed to parse {0} field: {1}.'.format(name, e) 'Failed to parse {0} field: {1}.'.format(name, e)
) from e ) from e
def _ConvertAnyMessage(self, value, message, path): def _ConvertAnyMessage(self, value, message, path):
@ -631,7 +738,7 @@ class _Parser(object):
type_url = value['@type'] type_url = value['@type']
except KeyError as e: except KeyError as e:
raise ParseError( raise ParseError(
'@type is missing when parsing any message at {0}'.format(path) '@type is missing when parsing any message at {0}'.format(path)
) from e ) from e
try: try:
@ -641,12 +748,16 @@ class _Parser(object):
message_descriptor = sub_message.DESCRIPTOR message_descriptor = sub_message.DESCRIPTOR
full_name = message_descriptor.full_name full_name = message_descriptor.full_name
if _IsWrapperMessage(message_descriptor): if _IsWrapperMessage(message_descriptor):
self._ConvertWrapperMessage(value['value'], sub_message, self._ConvertWrapperMessage(
'{0}.value'.format(path)) value['value'], sub_message, '{0}.value'.format(path)
)
elif full_name in _WKTJSONMETHODS: elif full_name in _WKTJSONMETHODS:
methodcaller(_WKTJSONMETHODS[full_name][1], value['value'], sub_message, methodcaller(
'{0}.value'.format(path))( _WKTJSONMETHODS[full_name][1],
self) value['value'],
sub_message,
'{0}.value'.format(path),
)(self)
else: else:
del value['@type'] del value['@type']
self._ConvertFieldValuePair(value, sub_message, path) self._ConvertFieldValuePair(value, sub_message, path)
@ -679,38 +790,47 @@ class _Parser(object):
elif isinstance(value, _INT_OR_FLOAT): elif isinstance(value, _INT_OR_FLOAT):
message.number_value = value message.number_value = value
else: else:
raise ParseError('Value {0} has unexpected type {1} at {2}'.format( raise ParseError(
value, type(value), path)) 'Value {0} has unexpected type {1} at {2}'.format(
value, type(value), path
)
)
def _ConvertListValueMessage(self, value, message, path): def _ConvertListValueMessage(self, value, message, path):
"""Convert a JSON representation into ListValue message.""" """Convert a JSON representation into ListValue message."""
if not isinstance(value, list): if not isinstance(value, list):
raise ParseError('ListValue must be in [] which is {0} at {1}'.format( raise ParseError(
value, path)) 'ListValue must be in [] which is {0} at {1}'.format(value, path)
)
message.ClearField('values') message.ClearField('values')
for index, item in enumerate(value): for index, item in enumerate(value):
self._ConvertValueMessage(item, message.values.add(), self._ConvertValueMessage(
'{0}[{1}]'.format(path, index)) item, message.values.add(), '{0}[{1}]'.format(path, index)
)
def _ConvertStructMessage(self, value, message, path): def _ConvertStructMessage(self, value, message, path):
"""Convert a JSON representation into Struct message.""" """Convert a JSON representation into Struct message."""
if not isinstance(value, dict): if not isinstance(value, dict):
raise ParseError('Struct must be in a dict which is {0} at {1}'.format( raise ParseError(
value, path)) 'Struct must be in a dict which is {0} at {1}'.format(value, path)
)
# Clear will mark the struct as modified so it will be created even if # Clear will mark the struct as modified so it will be created even if
# there are no values. # there are no values.
message.Clear() message.Clear()
for key in value: for key in value:
self._ConvertValueMessage(value[key], message.fields[key], self._ConvertValueMessage(
'{0}.{1}'.format(path, key)) value[key], message.fields[key], '{0}.{1}'.format(path, key)
)
return return
def _ConvertWrapperMessage(self, value, message, path): def _ConvertWrapperMessage(self, value, message, path):
"""Convert a JSON representation into Wrapper message.""" """Convert a JSON representation into Wrapper message."""
field = message.DESCRIPTOR.fields_by_name['value'] field = message.DESCRIPTOR.fields_by_name['value']
setattr( setattr(
message, 'value', message,
_ConvertScalarFieldValue(value, field, path='{0}.value'.format(path))) 'value',
_ConvertScalarFieldValue(value, field, path='{0}.value'.format(path)),
)
def _ConvertMapFieldValue(self, value, message, field, path): def _ConvertMapFieldValue(self, value, message, field, path):
"""Convert map field value for a message map field. """Convert map field value for a message map field.
@ -727,19 +847,25 @@ class _Parser(object):
if not isinstance(value, dict): if not isinstance(value, dict):
raise ParseError( raise ParseError(
'Map field {0} must be in a dict which is {1} at {2}'.format( 'Map field {0} must be in a dict which is {1} at {2}'.format(
field.name, value, path)) field.name, value, path
)
)
key_field = field.message_type.fields_by_name['key'] key_field = field.message_type.fields_by_name['key']
value_field = field.message_type.fields_by_name['value'] value_field = field.message_type.fields_by_name['value']
for key in value: for key in value:
key_value = _ConvertScalarFieldValue(key, key_field, key_value = _ConvertScalarFieldValue(
'{0}.key'.format(path), True) key, key_field, '{0}.key'.format(path), True
)
if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
self.ConvertMessage(value[key], self.ConvertMessage(
getattr(message, field.name)[key_value], value[key],
'{0}[{1}]'.format(path, key_value)) getattr(message, field.name)[key_value],
'{0}[{1}]'.format(path, key_value),
)
else: else:
getattr(message, field.name)[key_value] = _ConvertScalarFieldValue( getattr(message, field.name)[key_value] = _ConvertScalarFieldValue(
value[key], value_field, path='{0}[{1}]'.format(path, key_value)) value[key], value_field, path='{0}[{1}]'.format(path, key_value)
)
def _ConvertScalarFieldValue(value, field, path, require_str=False): def _ConvertScalarFieldValue(value, field, path, require_str=False):
@ -787,12 +913,18 @@ def _ConvertScalarFieldValue(value, field, path, require_str=False):
number = int(value) number = int(value)
enum_value = field.enum_type.values_by_number.get(number, None) enum_value = field.enum_type.values_by_number.get(number, None)
except ValueError as e: except ValueError as e:
raise ParseError('Invalid enum value {0} for enum type {1}'.format( raise ParseError(
value, field.enum_type.full_name)) from e 'Invalid enum value {0} for enum type {1}'.format(
value, field.enum_type.full_name
)
) from e
if enum_value is None: if enum_value is None:
if field.enum_type.is_closed: if field.enum_type.is_closed:
raise ParseError('Invalid enum value {0} for enum type {1}'.format( raise ParseError(
value, field.enum_type.full_name)) 'Invalid enum value {0} for enum type {1}'.format(
value, field.enum_type.full_name
)
)
else: else:
return number return number
return enum_value.number return enum_value.number
@ -813,14 +945,15 @@ def _ConvertInteger(value):
ParseError: If an integer couldn't be consumed. ParseError: If an integer couldn't be consumed.
""" """
if isinstance(value, float) and not value.is_integer(): if isinstance(value, float) and not value.is_integer():
raise ParseError('Couldn\'t parse integer: {0}'.format(value)) raise ParseError("Couldn't parse integer: {0}".format(value))
if isinstance(value, str) and value.find(' ') != -1: if isinstance(value, str) and value.find(' ') != -1:
raise ParseError('Couldn\'t parse integer: "{0}"'.format(value)) raise ParseError('Couldn\'t parse integer: "{0}"'.format(value))
if isinstance(value, bool): if isinstance(value, bool):
raise ParseError('Bool value {0} is not acceptable for ' raise ParseError(
'integer field'.format(value)) 'Bool value {0} is not acceptable for integer field'.format(value)
)
return int(value) return int(value)
@ -832,11 +965,15 @@ def _ConvertFloat(value, field):
raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead') raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead')
if math.isinf(value): if math.isinf(value):
if value > 0: if value > 0:
raise ParseError('Couldn\'t parse Infinity or value too large, ' raise ParseError(
'use quoted "Infinity" instead') "Couldn't parse Infinity or value too large, "
'use quoted "Infinity" instead'
)
else: else:
raise ParseError('Couldn\'t parse -Infinity or value too small, ' raise ParseError(
'use quoted "-Infinity" instead') "Couldn't parse -Infinity or value too small, "
'use quoted "-Infinity" instead'
)
if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT: if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
# pylint: disable=protected-access # pylint: disable=protected-access
if value > type_checkers._FLOAT_MAX: if value > type_checkers._FLOAT_MAX:
@ -858,7 +995,7 @@ def _ConvertFloat(value, field):
elif value == _NAN: elif value == _NAN:
return float('nan') return float('nan')
else: else:
raise ParseError('Couldn\'t parse float: {0}'.format(value)) from e raise ParseError("Couldn't parse float: {0}".format(value)) from e
def _ConvertBool(value, require_str): def _ConvertBool(value, require_str):
@ -886,19 +1023,31 @@ def _ConvertBool(value, require_str):
raise ParseError('Expected true or false without quotes') raise ParseError('Expected true or false without quotes')
return value return value
_WKTJSONMETHODS = { _WKTJSONMETHODS = {
'google.protobuf.Any': ['_AnyMessageToJsonObject', 'google.protobuf.Any': ['_AnyMessageToJsonObject', '_ConvertAnyMessage'],
'_ConvertAnyMessage'], 'google.protobuf.Duration': [
'google.protobuf.Duration': ['_GenericMessageToJsonObject', '_GenericMessageToJsonObject',
'_ConvertGenericMessage'], '_ConvertGenericMessage',
'google.protobuf.FieldMask': ['_GenericMessageToJsonObject', ],
'_ConvertGenericMessage'], 'google.protobuf.FieldMask': [
'google.protobuf.ListValue': ['_ListValueMessageToJsonObject', '_GenericMessageToJsonObject',
'_ConvertListValueMessage'], '_ConvertGenericMessage',
'google.protobuf.Struct': ['_StructMessageToJsonObject', ],
'_ConvertStructMessage'], 'google.protobuf.ListValue': [
'google.protobuf.Timestamp': ['_GenericMessageToJsonObject', '_ListValueMessageToJsonObject',
'_ConvertGenericMessage'], '_ConvertListValueMessage',
'google.protobuf.Value': ['_ValueMessageToJsonObject', ],
'_ConvertValueMessage'] 'google.protobuf.Struct': [
'_StructMessageToJsonObject',
'_ConvertStructMessage',
],
'google.protobuf.Timestamp': [
'_GenericMessageToJsonObject',
'_ConvertGenericMessage',
],
'google.protobuf.Value': [
'_ValueMessageToJsonObject',
'_ConvertValueMessage',
],
} }

Write Preview
Loading…
Cancel
Save