down integrate python opensource to svn

pull/1/head
jieluo@google.com 11 years ago
parent d7339318a3
commit bde4a3254a
  1. 39
      Makefile.am
  2. 2
      python/README.txt
  3. 2
      python/ez_setup.py
  4. 178
      python/google/protobuf/descriptor.py
  5. 19
      python/google/protobuf/descriptor_database.py
  6. 394
      python/google/protobuf/descriptor_pool.py
  7. 139
      python/google/protobuf/internal/api_implementation.cc
  8. 52
      python/google/protobuf/internal/api_implementation.py
  9. 63
      python/google/protobuf/internal/api_implementation_default_test.py
  10. 22
      python/google/protobuf/internal/containers.py
  11. 2
      python/google/protobuf/internal/cpp_message.py
  12. 157
      python/google/protobuf/internal/decoder.py
  13. 58
      python/google/protobuf/internal/descriptor_cpp2_test.py
  14. 16
      python/google/protobuf/internal/descriptor_database_test.py
  15. 428
      python/google/protobuf/internal/descriptor_pool_test.py
  16. 94
      python/google/protobuf/internal/descriptor_pool_test1.proto
  17. 70
      python/google/protobuf/internal/descriptor_pool_test2.proto
  18. 54
      python/google/protobuf/internal/descriptor_python_test.py
  19. 152
      python/google/protobuf/internal/descriptor_test.py
  20. 43
      python/google/protobuf/internal/encoder.py
  21. 2
      python/google/protobuf/internal/factory_test1.proto
  22. 15
      python/google/protobuf/internal/factory_test2.proto
  23. 82
      python/google/protobuf/internal/generator_test.py
  24. 56
      python/google/protobuf/internal/message_factory_cpp2_test.py
  25. 54
      python/google/protobuf/internal/message_factory_python_test.py
  26. 52
      python/google/protobuf/internal/message_factory_test.py
  27. 21
      python/google/protobuf/internal/message_python_test.py
  28. 274
      python/google/protobuf/internal/message_test.py
  29. 50
      python/google/protobuf/internal/missing_enum_values.proto
  30. 151
      python/google/protobuf/internal/python_message.py
  31. 9
      python/google/protobuf/internal/reflection_cpp2_generated_test.py
  32. 419
      python/google/protobuf/internal/reflection_test.py
  33. 6
      python/google/protobuf/internal/service_reflection_test.py
  34. 120
      python/google/protobuf/internal/symbol_database_test.py
  35. 95
      python/google/protobuf/internal/test_util.py
  36. 68
      python/google/protobuf/internal/text_encoding_test.py
  37. 417
      python/google/protobuf/internal/text_format_test.py
  38. 72
      python/google/protobuf/internal/type_checkers.py
  39. 71
      python/google/protobuf/internal/unknown_fields_test.py
  40. 8
      python/google/protobuf/internal/wire_format_test.py
  41. 6
      python/google/protobuf/message.py
  42. 110
      python/google/protobuf/message_factory.py
  43. 6
      python/google/protobuf/pyext/README
  44. 61
      python/google/protobuf/pyext/cpp_message.py
  45. 357
      python/google/protobuf/pyext/descriptor.cc
  46. 96
      python/google/protobuf/pyext/descriptor.h
  47. 338
      python/google/protobuf/pyext/extension_dict.cc
  48. 123
      python/google/protobuf/pyext/extension_dict.h
  49. 2561
      python/google/protobuf/pyext/message.cc
  50. 305
      python/google/protobuf/pyext/message.h
  51. 38
      python/google/protobuf/pyext/proto2_api_test.proto
  52. 66
      python/google/protobuf/pyext/python.proto
  53. 57
      python/google/protobuf/pyext/python_protobuf.h
  54. 763
      python/google/protobuf/pyext/repeated_composite_container.cc
  55. 172
      python/google/protobuf/pyext/repeated_composite_container.h
  56. 825
      python/google/protobuf/pyext/repeated_scalar_container.cc
  57. 112
      python/google/protobuf/pyext/repeated_scalar_container.h
  58. 95
      python/google/protobuf/pyext/scoped_pyobject_ptr.h
  59. 46
      python/google/protobuf/reflection.py
  60. 185
      python/google/protobuf/symbol_database.py
  61. 110
      python/google/protobuf/text_encoding.py
  62. 322
      python/google/protobuf/text_format.py
  63. 149
      python/setup.py
  64. 4
      src/Makefile.am
  65. 34
      src/google/protobuf/compiler/python/python_generator.cc
  66. 129
      src/google/protobuf/testdata/text_format_unittest_data_pointy_oneof_implemented.txt

@ -168,37 +168,64 @@ EXTRA_DIST = \
java/src/test/java/com/google/protobuf/test_custom_options.proto \
java/pom.xml \
java/README.txt \
python/google/protobuf/internal/generator_test.py \
python/google/protobuf/internal/api_implementation.cc \
python/google/protobuf/internal/api_implementation.py \
python/google/protobuf/internal/api_implementation_default_test.py \
python/google/protobuf/internal/containers.py \
python/google/protobuf/internal/cpp_message.py \
python/google/protobuf/internal/decoder.py \
python/google/protobuf/internal/descriptor_cpp2_test.py \
python/google/protobuf/internal/descriptor_database_test.py \
python/google/protobuf/internal/descriptor_pool_test.py \
python/google/protobuf/internal/descriptor_pool_test1.proto \
python/google/protobuf/internal/descriptor_pool_test2.proto \
python/google/protobuf/internal/descriptor_python_test.py \
python/google/protobuf/internal/descriptor_test.py \
python/google/protobuf/internal/encoder.py \
python/google/protobuf/internal/enum_type_wrapper.py \
python/google/protobuf/internal/factory_test1.proto \
python/google/protobuf/internal/factory_test2.proto \
python/google/protobuf/internal/message_cpp_test.py \
python/google/protobuf/internal/generator_test.py \
python/google/protobuf/internal/message_factory_cpp2_test.py \
python/google/protobuf/internal/message_factory_python_test.py \
python/google/protobuf/internal/message_factory_test.py \
python/google/protobuf/internal/message_listener.py \
python/google/protobuf/internal/message_python_test.py \
python/google/protobuf/internal/message_test.py \
python/google/protobuf/internal/missing_enum_values.proto \
python/google/protobuf/internal/more_extensions.proto \
python/google/protobuf/internal/more_extensions_dynamic.proto \
python/google/protobuf/internal/more_messages.proto \
python/google/protobuf/internal/python_message.py \
python/google/protobuf/internal/cpp_message.py \
python/google/protobuf/internal/api_implementation.py \
python/google/protobuf/internal/reflection_cpp2_generated_test.py \
python/google/protobuf/internal/reflection_test.py \
python/google/protobuf/internal/reflection_cpp_generated_test.py \
python/google/protobuf/internal/service_reflection_test.py \
python/google/protobuf/internal/test_bad_identifiers.proto \
python/google/protobuf/internal/test_util.py \
python/google/protobuf/internal/text_encoding_test.py \
python/google/protobuf/internal/text_format_test.py \
python/google/protobuf/internal/type_checkers.py \
python/google/protobuf/internal/unknown_fields_test.py \
python/google/protobuf/internal/wire_format.py \
python/google/protobuf/internal/wire_format_test.py \
python/google/protobuf/internal/__init__.py \
python/google/protobuf/pyext/README \
python/google/protobuf/pyext/cpp_message.py \
python/google/protobuf/pyext/descriptor.h \
python/google/protobuf/pyext/descriptor.cc \
python/google/protobuf/pyext/extension_dict.h \
python/google/protobuf/pyext/extension_dict.cc \
python/google/protobuf/pyext/message.h \
python/google/protobuf/pyext/message.cc \
python/google/protobuf/pyext/proto2_api_test.proto \
python/google/protobuf/pyext/python.proto \
python/google/protobuf/pyext/python_protobuf.h \
python/google/protobuf/pyext/repeated_composite_container.h \
python/google/protobuf/pyext/repeated_composite_container.cc \
python/google/protobuf/pyext/repeated_scalar_container.h \
python/google/protobuf/pyext/repeated_scalar_container.cc \
python/google/protobuf/pyext/scoped_pyobject_ptr.h \
python/google/protobuf/pyext/__init__.py \
python/google/protobuf/descriptor.py \
python/google/protobuf/descriptor_database.py \
python/google/protobuf/descriptor_pool.py \
@ -207,6 +234,8 @@ EXTRA_DIST = \
python/google/protobuf/reflection.py \
python/google/protobuf/service.py \
python/google/protobuf/service_reflection.py \
python/google/protobuf/symbol_database.py \
python/google/protobuf/text_encoding.py \
python/google/protobuf/text_format.py \
python/google/protobuf/__init__.py \
python/google/__init__.py \

@ -47,6 +47,7 @@ Installation
$ python setup.py build
$ python setup.py test
$ python setup.py exttest # To test C++ implementation (see below).
If some tests fail, this library may not work correctly on your
system. Continue at your own risk.
@ -90,6 +91,7 @@ To use the C++ implementation, you need to:
2) Export an environment variable:
$ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
$ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2
You need to export this variable before running setup.py script to build and
install the extension. You must also set the variable at runtime, otherwise

@ -103,10 +103,12 @@ def use_setuptools(
sys.path.insert(0, egg)
import setuptools; setuptools.bootstrap_install_from = egg
try:
return do_download()
import pkg_resources
except ImportError:
return do_download()
try:
return do_download()
pkg_resources.require("setuptools>="+version); return
except pkg_resources.VersionConflict, e:
if was_imported:

@ -28,19 +28,26 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Needs to stay compatible with Python 2.5 due to GAE.
#
# Copyright 2007 Google Inc. All Rights Reserved.
"""Descriptors essentially contain exactly the information found in a .proto
file, in types that make this information accessible in Python.
"""
__author__ = 'robinson@google.com (Will Robinson)'
from google.protobuf.internal import api_implementation
if api_implementation.Type() == 'cpp':
# Used by MakeDescriptor in cpp mode
import os
import uuid
if api_implementation.Version() == 2:
from google.protobuf.internal.cpp import _message
from google.protobuf.pyext import _message
else:
from google.protobuf.internal import cpp_message
@ -220,13 +227,21 @@ class Descriptor(_NestedDescriptorBase):
options: (descriptor_pb2.MessageOptions) Protocol message options or None
to use default message options.
oneofs: (list of OneofDescriptor) The list of descriptors for oneof fields
in this message.
oneofs_by_name: (dict str -> OneofDescriptor) Same objects as in |oneofs|,
but indexed by "name" attribute.
file: (FileDescriptor) Reference to file descriptor.
"""
# NOTE(tmarek): The file argument redefining a builtin is nothing we can
# fix right now since we don't know how many clients already rely on the
# name of the argument.
def __init__(self, name, full_name, filename, containing_type, fields,
nested_types, enum_types, extensions, options=None,
is_extendable=True, extension_ranges=None, file=None,
serialized_start=None, serialized_end=None):
is_extendable=True, extension_ranges=None, oneofs=None,
file=None, serialized_start=None, serialized_end=None): # pylint:disable=redefined-builtin
"""Arguments to __init__() are as described in the description
of Descriptor fields above.
@ -236,7 +251,7 @@ class Descriptor(_NestedDescriptorBase):
super(Descriptor, self).__init__(
options, 'MessageOptions', name, full_name, file,
containing_type, serialized_start=serialized_start,
serialized_end=serialized_start)
serialized_end=serialized_end)
# We have fields in addition to fields_by_name and fields_by_number,
# so that:
@ -250,6 +265,8 @@ class Descriptor(_NestedDescriptorBase):
self.fields_by_name = dict((f.name, f) for f in fields)
self.nested_types = nested_types
for nested_type in nested_types:
nested_type.containing_type = self
self.nested_types_by_name = dict((t.name, t) for t in nested_types)
self.enum_types = enum_types
@ -265,9 +282,10 @@ class Descriptor(_NestedDescriptorBase):
self.extensions_by_name = dict((f.name, f) for f in extensions)
self.is_extendable = is_extendable
self.extension_ranges = extension_ranges
self._serialized_start = serialized_start
self._serialized_end = serialized_end
self.oneofs = oneofs if oneofs is not None else []
self.oneofs_by_name = dict((o.name, o) for o in self.oneofs)
for oneof in self.oneofs:
oneof.containing_type = self
def EnumValueName(self, enum, value):
"""Returns the string name of an enum value.
@ -353,6 +371,9 @@ class FieldDescriptor(DescriptorBase):
options: (descriptor_pb2.FieldOptions) Protocol message field options or
None to use default field options.
containing_oneof: (OneofDescriptor) If the field is a member of a oneof
union, contains its descriptor. Otherwise, None.
"""
# Must be consistent with C++ FieldDescriptor::Type enum in
@ -425,10 +446,16 @@ class FieldDescriptor(DescriptorBase):
LABEL_REPEATED = 3
MAX_LABEL = 3
# Must be consistent with C++ constants kMaxNumber, kFirstReservedNumber,
# and kLastReservedNumber in descriptor.h
MAX_FIELD_NUMBER = (1 << 29) - 1
FIRST_RESERVED_FIELD_NUMBER = 19000
LAST_RESERVED_FIELD_NUMBER = 19999
def __init__(self, name, full_name, index, number, type, cpp_type, label,
default_value, message_type, enum_type, containing_type,
is_extension, extension_scope, options=None,
has_default_value=True):
has_default_value=True, containing_oneof=None):
"""The arguments are as described in the description of FieldDescriptor
attributes above.
@ -451,15 +478,21 @@ class FieldDescriptor(DescriptorBase):
self.enum_type = enum_type
self.is_extension = is_extension
self.extension_scope = extension_scope
self.containing_oneof = containing_oneof
if api_implementation.Type() == 'cpp':
if is_extension:
if api_implementation.Version() == 2:
self._cdescriptor = _message.GetExtensionDescriptor(full_name)
# pylint: disable=protected-access
self._cdescriptor = (
_message.Message._GetExtensionDescriptor(full_name))
# pylint: enable=protected-access
else:
self._cdescriptor = cpp_message.GetExtensionDescriptor(full_name)
else:
if api_implementation.Version() == 2:
self._cdescriptor = _message.GetFieldDescriptor(full_name)
# pylint: disable=protected-access
self._cdescriptor = _message.Message._GetFieldDescriptor(full_name)
# pylint: enable=protected-access
else:
self._cdescriptor = cpp_message.GetFieldDescriptor(full_name)
else:
@ -522,7 +555,7 @@ class EnumDescriptor(_NestedDescriptorBase):
super(EnumDescriptor, self).__init__(
options, 'EnumOptions', name, full_name, file,
containing_type, serialized_start=serialized_start,
serialized_end=serialized_start)
serialized_end=serialized_end)
self.values = values
for value in self.values:
@ -530,9 +563,6 @@ class EnumDescriptor(_NestedDescriptorBase):
self.values_by_name = dict((v.name, v) for v in values)
self.values_by_number = dict((v.number, v) for v in values)
self._serialized_start = serialized_start
self._serialized_end = serialized_end
def CopyToProto(self, proto):
"""Copies this to a descriptor_pb2.EnumDescriptorProto.
@ -567,6 +597,29 @@ class EnumValueDescriptor(DescriptorBase):
self.type = type
class OneofDescriptor(object):
"""Descriptor for a oneof field.
name: (str) Name of the oneof field.
full_name: (str) Full name of the oneof field, including package name.
index: (int) 0-based index giving the order of the oneof field inside
its containing type.
containing_type: (Descriptor) Descriptor of the protocol message
type that contains this field. Set by the Descriptor constructor
if we're passed into one.
fields: (list of FieldDescriptor) The list of field descriptors this
oneof can contain.
"""
def __init__(self, name, full_name, index, containing_type, fields):
"""Arguments are as described in the attribute description above."""
self.name = name
self.full_name = full_name
self.index = index
self.containing_type = containing_type
self.fields = fields
class ServiceDescriptor(_NestedDescriptorBase):
"""Descriptor for a service.
@ -645,13 +698,22 @@ class MethodDescriptor(DescriptorBase):
class FileDescriptor(DescriptorBase):
"""Descriptor for a file. Mimics the descriptor_pb2.FileDescriptorProto.
Note that enum_types_by_name, extensions_by_name, and dependencies
fields are only set by the message_factory module, and not by the
generated proto code.
name: name of file, relative to root of source tree.
package: name of the package
serialized_pb: (str) Byte string of serialized
descriptor_pb2.FileDescriptorProto.
dependencies: List of other FileDescriptors this FileDescriptor depends on.
message_types_by_name: Dict of message names of their descriptors.
enum_types_by_name: Dict of enum names and their descriptors.
extensions_by_name: Dict of extension names and their descriptors.
"""
def __init__(self, name, package, options=None, serialized_pb=None):
def __init__(self, name, package, options=None, serialized_pb=None,
dependencies=None):
"""Constructor."""
super(FileDescriptor, self).__init__(options, 'FileOptions')
@ -659,10 +721,17 @@ class FileDescriptor(DescriptorBase):
self.name = name
self.package = package
self.serialized_pb = serialized_pb
self.enum_types_by_name = {}
self.extensions_by_name = {}
self.dependencies = (dependencies or [])
if (api_implementation.Type() == 'cpp' and
self.serialized_pb is not None):
if api_implementation.Version() == 2:
_message.BuildFile(self.serialized_pb)
# pylint: disable=protected-access
_message.Message._BuildFile(self.serialized_pb)
# pylint: enable=protected-access
else:
cpp_message.BuildFile(self.serialized_pb)
@ -685,29 +754,96 @@ def _ParseOptions(message, string):
return message
def MakeDescriptor(desc_proto, package=''):
def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
"""Make a protobuf Descriptor given a DescriptorProto protobuf.
Handles nested descriptors. Note that this is limited to the scope of defining
a message inside of another message. Composite fields can currently only be
resolved if the message is defined in the same scope as the field.
Args:
desc_proto: The descriptor_pb2.DescriptorProto protobuf message.
package: Optional package name for the new message Descriptor (string).
build_file_if_cpp: Update the C++ descriptor pool if api matches.
Set to False on recursion, so no duplicates are created.
Returns:
A Descriptor for protobuf messages.
"""
if api_implementation.Type() == 'cpp' and build_file_if_cpp:
# The C++ implementation requires all descriptors to be backed by the same
# definition in the C++ descriptor pool. To do this, we build a
# FileDescriptorProto with the same definition as this descriptor and build
# it into the pool.
from google.protobuf import descriptor_pb2
file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
file_descriptor_proto.message_type.add().MergeFrom(desc_proto)
# Generate a random name for this proto file to prevent conflicts with
# any imported ones. We need to specify a file name so BuildFile accepts
# our FileDescriptorProto, but it is not important what that file name
# is actually set to.
proto_name = str(uuid.uuid4())
if package:
file_descriptor_proto.name = os.path.join(package.replace('.', '/'),
proto_name + '.proto')
file_descriptor_proto.package = package
else:
file_descriptor_proto.name = proto_name + '.proto'
if api_implementation.Version() == 2:
# pylint: disable=protected-access
_message.Message._BuildFile(file_descriptor_proto.SerializeToString())
# pylint: enable=protected-access
else:
cpp_message.BuildFile(file_descriptor_proto.SerializeToString())
full_message_name = [desc_proto.name]
if package: full_message_name.insert(0, package)
# Create Descriptors for enum types
enum_types = {}
for enum_proto in desc_proto.enum_type:
full_name = '.'.join(full_message_name + [enum_proto.name])
enum_desc = EnumDescriptor(
enum_proto.name, full_name, None, [
EnumValueDescriptor(enum_val.name, ii, enum_val.number)
for ii, enum_val in enumerate(enum_proto.value)])
enum_types[full_name] = enum_desc
# Create Descriptors for nested types
nested_types = {}
for nested_proto in desc_proto.nested_type:
full_name = '.'.join(full_message_name + [nested_proto.name])
# Nested types are just those defined inside of the message, not all types
# used by fields in the message, so no loops are possible here.
nested_desc = MakeDescriptor(nested_proto,
package='.'.join(full_message_name),
build_file_if_cpp=False)
nested_types[full_name] = nested_desc
fields = []
for field_proto in desc_proto.field:
full_name = '.'.join(full_message_name + [field_proto.name])
enum_desc = None
nested_desc = None
if field_proto.HasField('type_name'):
type_name = field_proto.type_name
full_type_name = '.'.join(full_message_name +
[type_name[type_name.rfind('.')+1:]])
if full_type_name in nested_types:
nested_desc = nested_types[full_type_name]
elif full_type_name in enum_types:
enum_desc = enum_types[full_type_name]
# Else type_name references a non-local type, which isn't implemented
field = FieldDescriptor(
field_proto.name, full_name, field_proto.number - 1,
field_proto.number, field_proto.type,
FieldDescriptor.ProtoTypeToCppProtoType(field_proto.type),
field_proto.label, None, None, None, None, False, None,
field_proto.label, None, nested_desc, enum_desc, None, False, None,
has_default_value=False)
fields.append(field)
desc_name = '.'.join(full_message_name)
return Descriptor(desc_proto.name, desc_name, None, None, fields,
[], [], [])
nested_types.values(), enum_types.values(), [])

@ -33,6 +33,14 @@
__author__ = 'matthewtoia@google.com (Matt Toia)'
class Error(Exception):
pass
class DescriptorDatabaseConflictingDefinitionError(Error):
"""Raised when a proto is added with the same name & different descriptor."""
class DescriptorDatabase(object):
"""A container accepting FileDescriptorProtos and maps DescriptorProtos."""
@ -45,9 +53,18 @@ class DescriptorDatabase(object):
Args:
file_desc_proto: The FileDescriptorProto to add.
Raises:
DescriptorDatabaseException: if an attempt is made to add a proto
with the same name but different definition than an exisiting
proto in the database.
"""
proto_name = file_desc_proto.name
if proto_name not in self._file_desc_protos_by_file:
self._file_desc_protos_by_file[proto_name] = file_desc_proto
elif self._file_desc_protos_by_file[proto_name] != file_desc_proto:
raise DescriptorDatabaseConflictingDefinitionError(
'%s already added, but with different descriptor.' % proto_name)
self._file_desc_protos_by_file[file_desc_proto.name] = file_desc_proto
package = file_desc_proto.package
for message in file_desc_proto.message_type:
self._file_desc_protos_by_symbol.update(

@ -49,13 +49,34 @@ Below is a straightforward example on how to use this class:
The message descriptor can be used in conjunction with the message_factory
module in order to create a protocol buffer class that can be encoded and
decoded.
If you want to get a Python class for the specified proto, use the
helper functions inside google.protobuf.message_factory
directly instead of this class.
"""
__author__ = 'matthewtoia@google.com (Matt Toia)'
from google.protobuf import descriptor_pb2
import sys
from google.protobuf import descriptor
from google.protobuf import descriptor_database
from google.protobuf import text_encoding
def _NormalizeFullyQualifiedName(name):
"""Remove leading period from fully-qualified type name.
Due to b/13860351 in descriptor_database.py, types in the root namespace are
generated with a leading period. This function removes that prefix.
Args:
name: A str, the fully-qualified symbol name.
Returns:
A str, the normalized fully-qualified symbol name.
"""
return name.lstrip('.')
class DescriptorPool(object):
@ -89,6 +110,51 @@ class DescriptorPool(object):
self._internal_db.Add(file_desc_proto)
def AddDescriptor(self, desc):
"""Adds a Descriptor to the pool, non-recursively.
If the Descriptor contains nested messages or enums, the caller must
explicitly register them. This method also registers the FileDescriptor
associated with the message.
Args:
desc: A Descriptor.
"""
if not isinstance(desc, descriptor.Descriptor):
raise TypeError('Expected instance of descriptor.Descriptor.')
self._descriptors[desc.full_name] = desc
self.AddFileDescriptor(desc.file)
def AddEnumDescriptor(self, enum_desc):
"""Adds an EnumDescriptor to the pool.
This method also registers the FileDescriptor associated with the message.
Args:
enum_desc: An EnumDescriptor.
"""
if not isinstance(enum_desc, descriptor.EnumDescriptor):
raise TypeError('Expected instance of descriptor.EnumDescriptor.')
self._enum_descriptors[enum_desc.full_name] = enum_desc
self.AddFileDescriptor(enum_desc.file)
def AddFileDescriptor(self, file_desc):
"""Adds a FileDescriptor to the pool, non-recursively.
If the FileDescriptor contains messages or enums, the caller must explicitly
register them.
Args:
file_desc: A FileDescriptor.
"""
if not isinstance(file_desc, descriptor.FileDescriptor):
raise TypeError('Expected instance of descriptor.FileDescriptor.')
self._file_descriptors[file_desc.name] = file_desc
def FindFileByName(self, file_name):
"""Gets a FileDescriptor by file name.
@ -102,9 +168,15 @@ class DescriptorPool(object):
KeyError: if the file can not be found in the pool.
"""
try:
return self._file_descriptors[file_name]
except KeyError:
pass
try:
file_proto = self._internal_db.FindFileByName(file_name)
except KeyError as error:
except KeyError:
_, error, _ = sys.exc_info() #PY25 compatible for GAE.
if self._descriptor_db:
file_proto = self._descriptor_db.FindFileByName(file_name)
else:
@ -126,9 +198,21 @@ class DescriptorPool(object):
KeyError: if the file can not be found in the pool.
"""
symbol = _NormalizeFullyQualifiedName(symbol)
try:
return self._descriptors[symbol].file
except KeyError:
pass
try:
return self._enum_descriptors[symbol].file
except KeyError:
pass
try:
file_proto = self._internal_db.FindFileContainingSymbol(symbol)
except KeyError as error:
except KeyError:
_, error, _ = sys.exc_info() #PY25 compatible for GAE.
if self._descriptor_db:
file_proto = self._descriptor_db.FindFileContainingSymbol(symbol)
else:
@ -147,7 +231,7 @@ class DescriptorPool(object):
The descriptor for the named type.
"""
full_name = full_name.lstrip('.') # fix inconsistent qualified name formats
full_name = _NormalizeFullyQualifiedName(full_name)
if full_name not in self._descriptors:
self.FindFileContainingSymbol(full_name)
return self._descriptors[full_name]
@ -162,7 +246,7 @@ class DescriptorPool(object):
The enum descriptor for the named type.
"""
full_name = full_name.lstrip('.') # fix inconsistent qualified name formats
full_name = _NormalizeFullyQualifiedName(full_name)
if full_name not in self._enum_descriptors:
self.FindFileContainingSymbol(full_name)
return self._enum_descriptors[full_name]
@ -181,46 +265,56 @@ class DescriptorPool(object):
"""
if file_proto.name not in self._file_descriptors:
built_deps = list(self._GetDeps(file_proto.dependency))
direct_deps = [self.FindFileByName(n) for n in file_proto.dependency]
file_descriptor = descriptor.FileDescriptor(
name=file_proto.name,
package=file_proto.package,
options=file_proto.options,
serialized_pb=file_proto.SerializeToString())
serialized_pb=file_proto.SerializeToString(),
dependencies=direct_deps)
scope = {}
dependencies = list(self._GetDeps(file_proto))
for dependency in dependencies:
dep_desc = self.FindFileByName(dependency.name)
dep_proto = descriptor_pb2.FileDescriptorProto.FromString(
dep_desc.serialized_pb)
package = '.' + dep_proto.package
package_prefix = package + '.'
def _strip_package(symbol):
if symbol.startswith(package_prefix):
return symbol[len(package_prefix):]
return symbol
symbols = list(self._ExtractSymbols(dep_proto.message_type, package))
scope.update(symbols)
scope.update((_strip_package(k), v) for k, v in symbols)
symbols = list(self._ExtractEnums(dep_proto.enum_type, package))
scope.update(symbols)
scope.update((_strip_package(k), v) for k, v in symbols)
# This loop extracts all the message and enum types from all the
# dependencoes of the file_proto. This is necessary to create the
# scope of available message types when defining the passed in
# file proto.
for dependency in built_deps:
scope.update(self._ExtractSymbols(
dependency.message_types_by_name.values()))
scope.update((_PrefixWithDot(enum.full_name), enum)
for enum in dependency.enum_types_by_name.values())
for message_type in file_proto.message_type:
message_desc = self._ConvertMessageDescriptor(
message_type, file_proto.package, file_descriptor, scope)
file_descriptor.message_types_by_name[message_desc.name] = message_desc
for enum_type in file_proto.enum_type:
self._ConvertEnumDescriptor(enum_type, file_proto.package,
file_descriptor, None, scope)
for desc_proto in self._ExtractMessages(file_proto.message_type):
self._SetFieldTypes(desc_proto, scope)
file_descriptor.enum_types_by_name[enum_type.name] = (
self._ConvertEnumDescriptor(enum_type, file_proto.package,
file_descriptor, None, scope))
for index, extension_proto in enumerate(file_proto.extension):
extension_desc = self.MakeFieldDescriptor(
extension_proto, file_proto.package, index, is_extension=True)
extension_desc.containing_type = self._GetTypeFromScope(
file_descriptor.package, extension_proto.extendee, scope)
self.SetFieldType(extension_proto, extension_desc,
file_descriptor.package, scope)
file_descriptor.extensions_by_name[extension_desc.name] = extension_desc
for desc_proto in file_proto.message_type:
self.SetAllFieldTypes(file_proto.package, desc_proto, scope)
if file_proto.package:
desc_proto_prefix = _PrefixWithDot(file_proto.package)
else:
desc_proto_prefix = ''
for desc_proto in file_proto.message_type:
desc = scope[desc_proto.name]
desc = self._GetTypeFromScope(desc_proto_prefix, desc_proto.name, scope)
file_descriptor.message_types_by_name[desc_proto.name] = desc
self.Add(file_proto)
self._file_descriptors[file_proto.name] = file_descriptor
@ -260,10 +354,15 @@ class DescriptorPool(object):
enums = [
self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope)
for enum in desc_proto.enum_type]
fields = [self._MakeFieldDescriptor(field, desc_name, index)
fields = [self.MakeFieldDescriptor(field, desc_name, index)
for index, field in enumerate(desc_proto.field)]
extensions = [self._MakeFieldDescriptor(extension, desc_name, True)
for index, extension in enumerate(desc_proto.extension)]
extensions = [
self.MakeFieldDescriptor(extension, desc_name, index, is_extension=True)
for index, extension in enumerate(desc_proto.extension)]
oneofs = [
descriptor.OneofDescriptor(desc.name, '.'.join((desc_name, desc.name)),
index, None, [])
for index, desc in enumerate(desc_proto.oneof_decl)]
extension_ranges = [(r.start, r.end) for r in desc_proto.extension_range]
if extension_ranges:
is_extendable = True
@ -275,6 +374,7 @@ class DescriptorPool(object):
filename=file_name,
containing_type=None,
fields=fields,
oneofs=oneofs,
nested_types=nested,
enum_types=enums,
extensions=extensions,
@ -288,8 +388,13 @@ class DescriptorPool(object):
nested.containing_type = desc
for enum in desc.enum_types:
enum.containing_type = desc
scope[desc_proto.name] = desc
scope['.' + desc_name] = desc
for field_index, field_desc in enumerate(desc_proto.field):
if field_desc.HasField('oneof_index'):
oneof_index = field_desc.oneof_index
oneofs[oneof_index].fields.append(fields[field_index])
fields[field_index].containing_oneof = oneofs[oneof_index]
scope[_PrefixWithDot(desc_name)] = desc
self._descriptors[desc_name] = desc
return desc
@ -327,13 +432,12 @@ class DescriptorPool(object):
values=values,
containing_type=containing_type,
options=enum_proto.options)
scope[enum_proto.name] = desc
scope['.%s' % enum_name] = desc
self._enum_descriptors[enum_name] = desc
return desc
def _MakeFieldDescriptor(self, field_proto, message_name, index,
is_extension=False):
def MakeFieldDescriptor(self, field_proto, message_name, index,
is_extension=False):
"""Creates a field descriptor from a FieldDescriptorProto.
For message and enum type fields, this method will do a look up
@ -374,65 +478,93 @@ class DescriptorPool(object):
extension_scope=None,
options=field_proto.options)
def _SetFieldTypes(self, desc_proto, scope):
"""Sets the field's type, cpp_type, message_type and enum_type.
def SetAllFieldTypes(self, package, desc_proto, scope):
"""Sets all the descriptor's fields's types.
This method also sets the containing types on any extensions.
Args:
package: The current package of desc_proto.
desc_proto: The message descriptor to update.
scope: Enclosing scope of available types.
"""
desc = scope[desc_proto.name]
for field_proto, field_desc in zip(desc_proto.field, desc.fields):
if field_proto.type_name:
type_name = field_proto.type_name
if type_name not in scope:
type_name = '.' + type_name
desc = scope[type_name]
else:
desc = None
package = _PrefixWithDot(package)
if not field_proto.HasField('type'):
if isinstance(desc, descriptor.Descriptor):
field_proto.type = descriptor.FieldDescriptor.TYPE_MESSAGE
else:
field_proto.type = descriptor.FieldDescriptor.TYPE_ENUM
field_desc.cpp_type = descriptor.FieldDescriptor.ProtoTypeToCppProtoType(
field_proto.type)
if (field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE
or field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP):
field_desc.message_type = desc
if field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
field_desc.enum_type = desc
if field_proto.label == descriptor.FieldDescriptor.LABEL_REPEATED:
field_desc.has_default = False
field_desc.default_value = []
elif field_proto.HasField('default_value'):
field_desc.has_default = True
if (field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE or
field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT):
field_desc.default_value = float(field_proto.default_value)
elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING:
field_desc.default_value = field_proto.default_value
elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL:
field_desc.default_value = field_proto.default_value.lower() == 'true'
elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
field_desc.default_value = field_desc.enum_type.values_by_name[
field_proto.default_value].index
else:
field_desc.default_value = int(field_proto.default_value)
else:
field_desc.has_default = False
field_desc.default_value = None
main_desc = self._GetTypeFromScope(package, desc_proto.name, scope)
field_desc.type = field_proto.type
if package == '.':
nested_package = _PrefixWithDot(desc_proto.name)
else:
nested_package = '.'.join([package, desc_proto.name])
for field_proto, field_desc in zip(desc_proto.field, main_desc.fields):
self.SetFieldType(field_proto, field_desc, nested_package, scope)
for extension_proto, extension_desc in (
zip(desc_proto.extension, main_desc.extensions)):
extension_desc.containing_type = self._GetTypeFromScope(
nested_package, extension_proto.extendee, scope)
self.SetFieldType(extension_proto, extension_desc, nested_package, scope)
for nested_type in desc_proto.nested_type:
self._SetFieldTypes(nested_type, scope)
self.SetAllFieldTypes(nested_package, nested_type, scope)
def SetFieldType(self, field_proto, field_desc, package, scope):
"""Sets the field's type, cpp_type, message_type and enum_type.
Args:
field_proto: Data about the field in proto format.
field_desc: The descriptor to modiy.
package: The package the field's container is in.
scope: Enclosing scope of available types.
"""
if field_proto.type_name:
desc = self._GetTypeFromScope(package, field_proto.type_name, scope)
else:
desc = None
if not field_proto.HasField('type'):
if isinstance(desc, descriptor.Descriptor):
field_proto.type = descriptor.FieldDescriptor.TYPE_MESSAGE
else:
field_proto.type = descriptor.FieldDescriptor.TYPE_ENUM
field_desc.cpp_type = descriptor.FieldDescriptor.ProtoTypeToCppProtoType(
field_proto.type)
if (field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE
or field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP):
field_desc.message_type = desc
if field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
field_desc.enum_type = desc
if field_proto.label == descriptor.FieldDescriptor.LABEL_REPEATED:
field_desc.has_default_value = False
field_desc.default_value = []
elif field_proto.HasField('default_value'):
field_desc.has_default_value = True
if (field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE or
field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT):
field_desc.default_value = float(field_proto.default_value)
elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING:
field_desc.default_value = field_proto.default_value
elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL:
field_desc.default_value = field_proto.default_value.lower() == 'true'
elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
field_desc.default_value = field_desc.enum_type.values_by_name[
field_proto.default_value].index
elif field_proto.type == descriptor.FieldDescriptor.TYPE_BYTES:
field_desc.default_value = text_encoding.CUnescape(
field_proto.default_value)
else:
field_desc.default_value = int(field_proto.default_value)
else:
field_desc.has_default_value = False
field_desc.default_value = None
field_desc.type = field_proto.type
def _MakeEnumValueDescriptor(self, value_proto, index):
"""Creates a enum value descriptor object from a enum value proto.
@ -452,76 +584,60 @@ class DescriptorPool(object):
options=value_proto.options,
type=None)
def _ExtractSymbols(self, desc_protos, package):
def _ExtractSymbols(self, descriptors):
"""Pulls out all the symbols from descriptor protos.
Args:
desc_protos: The protos to extract symbols from.
package: The package containing the descriptor type.
descriptors: The messages to extract descriptors from.
Yields:
A two element tuple of the type name and descriptor object.
"""
for desc_proto in desc_protos:
if package:
message_name = '.'.join((package, desc_proto.name))
else:
message_name = desc_proto.name
message_desc = self.FindMessageTypeByName(message_name)
yield (message_name, message_desc)
for symbol in self._ExtractSymbols(desc_proto.nested_type, message_name):
yield symbol
for symbol in self._ExtractEnums(desc_proto.enum_type, message_name):
for desc in descriptors:
yield (_PrefixWithDot(desc.full_name), desc)
for symbol in self._ExtractSymbols(desc.nested_types):
yield symbol
for enum in desc.enum_types:
yield (_PrefixWithDot(enum.full_name), enum)
def _ExtractEnums(self, enum_protos, package):
"""Pulls out all the symbols from enum protos.
def _GetDeps(self, dependencies):
"""Recursively finds dependencies for file protos.
Args:
enum_protos: The protos to extract symbols from.
package: The package containing the enum type.
dependencies: The names of the files being depended on.
Yields:
A two element tuple of the type name and enum descriptor object.
Each direct and indirect dependency.
"""
for enum_proto in enum_protos:
if package:
enum_name = '.'.join((package, enum_proto.name))
else:
enum_name = enum_proto.name
enum_desc = self.FindEnumTypeByName(enum_name)
yield (enum_name, enum_desc)
for dependency in dependencies:
dep_desc = self.FindFileByName(dependency)
yield dep_desc
for parent_dep in dep_desc.dependencies:
yield parent_dep
def _ExtractMessages(self, desc_protos):
"""Pulls out all the message protos from descriptos.
def _GetTypeFromScope(self, package, type_name, scope):
"""Finds a given type name in the current scope.
Args:
desc_protos: The protos to extract symbols from.
package: The package the proto should be located in.
type_name: The name of the type to be found in the scope.
scope: Dict mapping short and full symbols to message and enum types.
Yields:
Descriptor protos.
Returns:
The descriptor for the requested type.
"""
if type_name not in scope:
components = _PrefixWithDot(package).split('.')
while components:
possible_match = '.'.join(components + [type_name])
if possible_match in scope:
type_name = possible_match
break
else:
components.pop(-1)
return scope[type_name]
for desc_proto in desc_protos:
yield desc_proto
for message in self._ExtractMessages(desc_proto.nested_type):
yield message
def _GetDeps(self, file_proto):
"""Recursively finds dependencies for file protos.
Args:
file_proto: The proto to get dependencies from.
Yields:
Each direct and indirect dependency.
"""
for dependency in file_proto.dependency:
dep_desc = self.FindFileByName(dependency)
dep_proto = descriptor_pb2.FileDescriptorProto.FromString(
dep_desc.serialized_pb)
yield dep_proto
for parent_dep in self._GetDeps(dep_proto):
yield parent_dep
def _PrefixWithDot(name):
return name if name.startswith('.') else '.%s' % name

@ -0,0 +1,139 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <Python.h>
namespace google {
namespace protobuf {
namespace python {
// Version constant.
// This is either 0 for python, 1 for CPP V1, 2 for CPP V2.
//
// 0 is default and is equivalent to
// PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
//
// 1 is set with -DPYTHON_PROTO2_CPP_IMPL_V1 and is equivalent to
// PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
// and
// PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=1
//
// 2 is set with -DPYTHON_PROTO2_CPP_IMPL_V2 and is equivalent to
// PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
// and
// PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2
#ifdef PYTHON_PROTO2_CPP_IMPL_V1
#if PY_MAJOR_VERSION >= 3
#error "PYTHON_PROTO2_CPP_IMPL_V1 is not supported under Python 3."
#endif
static int kImplVersion = 1;
#else
#ifdef PYTHON_PROTO2_CPP_IMPL_V2
static int kImplVersion = 2;
#else
#ifdef PYTHON_PROTO2_PYTHON_IMPL
static int kImplVersion = 0;
#else
// The defaults are set here. Python 3 uses the fast C++ APIv2 by default.
// Python 2 still uses the Python version by default until some compatibility
// issues can be worked around.
#if PY_MAJOR_VERSION >= 3
static int kImplVersion = 2;
#else
static int kImplVersion = 0;
#endif
#endif // PYTHON_PROTO2_PYTHON_IMPL
#endif // PYTHON_PROTO2_CPP_IMPL_V2
#endif // PYTHON_PROTO2_CPP_IMPL_V1
static const char* kImplVersionName = "api_version";
static const char* kModuleName = "_api_implementation";
static const char kModuleDocstring[] =
"_api_implementation is a module that exposes compile-time constants that\n"
"determine the default API implementation to use for Python proto2.\n"
"\n"
"It complements api_implementation.py by setting defaults using compile-time\n"
"constants defined in C, such that one can set defaults at compilation\n"
"(e.g. with blaze flag --copt=-DPYTHON_PROTO2_CPP_IMPL_V2).";
#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef _module = {
PyModuleDef_HEAD_INIT,
kModuleName,
kModuleDocstring,
-1,
NULL,
NULL,
NULL,
NULL,
NULL
};
#define INITFUNC PyInit__api_implementation
#define INITFUNC_ERRORVAL NULL
#else
#define INITFUNC init_api_implementation
#define INITFUNC_ERRORVAL
#endif
extern "C" {
PyMODINIT_FUNC INITFUNC() {
#if PY_MAJOR_VERSION >= 3
PyObject *module = PyModule_Create(&_module);
#else
PyObject *module = Py_InitModule3(
const_cast<char*>(kModuleName),
NULL,
const_cast<char*>(kModuleDocstring));
#endif
if (module == NULL) {
return INITFUNC_ERRORVAL;
}
// Adds the module variable "api_version".
if (PyModule_AddIntConstant(
module,
const_cast<char*>(kImplVersionName),
kImplVersion))
#if PY_MAJOR_VERSION < 3
return;
#else
{ Py_DECREF(module); return NULL; }
return module;
#endif
}
}
} // namespace python
} // namespace protobuf
} // namespace google

@ -28,41 +28,44 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Determine which implementation of the protobuf API is used in this process.
"""
This module is the central entity that determines which implementation of the
API is used.
"""
__author__ = 'petar@google.com (Petar Petrov)'
import os
import sys
try:
# pylint: disable=g-import-not-at-top
from google.protobuf.internal import _api_implementation
# The compile-time constants in the _api_implementation module can be used to
# switch to a certain implementation of the Python API at build time.
_api_version = _api_implementation.api_version
del _api_implementation
except ImportError:
_api_version = 0
_default_implementation_type = (
'python' if _api_version == 0 else 'cpp')
_default_version_str = (
'1' if _api_version <= 1 else '2')
# This environment variable can be used to switch to a certain implementation
# of the Python API. Right now only 'python' and 'cpp' are valid values. Any
# other value will be ignored.
# of the Python API, overriding the compile-time constants in the
# _api_implementation module. Right now only 'python' and 'cpp' are valid
# values. Any other value will be ignored.
_implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION',
'python')
_default_implementation_type)
if _implementation_type != 'python':
# For now, by default use the pure-Python implementation.
# The code below checks if the C extension is available and
# uses it if it is available.
_implementation_type = 'cpp'
## Determine automatically which implementation to use.
#try:
# from google.protobuf.internal import cpp_message
# _implementation_type = 'cpp'
#except ImportError, e:
# _implementation_type = 'python'
# This environment variable can be used to switch between the two
# 'cpp' implementations. Right now only 1 and 2 are valid values. Any
# other value will be ignored.
# 'cpp' implementations, overriding the compile-time constants in the
# _api_implementation module. Right now only 1 and 2 are valid values. Any other
# value will be ignored.
_implementation_version_str = os.getenv(
'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION',
'1')
_default_version_str)
if _implementation_version_str not in ('1', '2'):
raise ValueError(
@ -70,11 +73,9 @@ if _implementation_version_str not in ('1', '2'):
_implementation_version_str + "' (supported versions: 1, 2)"
)
_implementation_version = int(_implementation_version_str)
# Usage of this function is discouraged. Clients shouldn't care which
# implementation of the API is in use. Note that there is no guarantee
# that differences between APIs will be maintained.
@ -82,6 +83,7 @@ _implementation_version = int(_implementation_version_str)
def Type():
return _implementation_type
# See comment on 'Type' above.
def Version():
return _implementation_version

@ -0,0 +1,63 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Test that the api_implementation defaults are what we expect."""
import os
import sys
# Clear environment implementation settings before the google3 imports.
os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', None)
os.environ.pop('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', None)
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
class ApiImplementationDefaultTest(basetest.TestCase):
if sys.version_info.major <= 2:
def testThatPythonIsTheDefault(self):
"""If -DPYTHON_PROTO_*IMPL* was given at build time, this may fail."""
self.assertEqual('python', api_implementation.Type())
else:
def testThatCppApiV2IsTheDefault(self):
"""If -DPYTHON_PROTO_*IMPL* was given at build time, this may fail."""
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

@ -108,15 +108,13 @@ class RepeatedScalarFieldContainer(BaseContainer):
def append(self, value):
"""Appends an item to the list. Similar to list.append()."""
self._type_checker.CheckValue(value)
self._values.append(value)
self._values.append(self._type_checker.CheckValue(value))
if not self._message_listener.dirty:
self._message_listener.Modified()
def insert(self, key, value):
"""Inserts the item at the specified position. Similar to list.insert()."""
self._type_checker.CheckValue(value)
self._values.insert(key, value)
self._values.insert(key, self._type_checker.CheckValue(value))
if not self._message_listener.dirty:
self._message_listener.Modified()
@ -127,8 +125,7 @@ class RepeatedScalarFieldContainer(BaseContainer):
new_values = []
for elem in elem_seq:
self._type_checker.CheckValue(elem)
new_values.append(elem)
new_values.append(self._type_checker.CheckValue(elem))
self._values.extend(new_values)
self._message_listener.Modified()
@ -146,9 +143,13 @@ class RepeatedScalarFieldContainer(BaseContainer):
def __setitem__(self, key, value):
"""Sets the item on the specified position."""
self._type_checker.CheckValue(value)
self._values[key] = value
self._message_listener.Modified()
if isinstance(key, slice): # PY3
if key.step is not None:
raise ValueError('Extended slices not supported')
self.__setslice__(key.start, key.stop, value)
else:
self._values[key] = self._type_checker.CheckValue(value)
self._message_listener.Modified()
def __getslice__(self, start, stop):
"""Retrieves the subset of items from between the specified indices."""
@ -158,8 +159,7 @@ class RepeatedScalarFieldContainer(BaseContainer):
"""Sets the subset of items from between the specified indices."""
new_values = []
for value in values:
self._type_checker.CheckValue(value)
new_values.append(value)
new_values.append(self._type_checker.CheckValue(value))
self._values[start:stop] = new_values
self._message_listener.Modified()

@ -610,7 +610,7 @@ def _AddMessageMethods(message_descriptor, cls):
return self._cmsg.FindInitializationErrors()
def __str__(self):
return self._cmsg.DebugString()
return str(self._cmsg)
def __eq__(self, other):
if self is other:

@ -28,6 +28,10 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#PY25 compatible for GAE.
#
# Copyright 2009 Google Inc. All Rights Reserved.
"""Code for decoding protocol buffer primitives.
This code is very similar to encoder.py -- read the docs for that module first.
@ -81,6 +85,8 @@ we repeatedly read a tag, look up the corresponding decoder, and invoke it.
__author__ = 'kenton@google.com (Kenton Varda)'
import struct
import sys ##PY25
_PY2 = sys.version_info[0] < 3 ##PY25
from google.protobuf.internal import encoder
from google.protobuf.internal import wire_format
from google.protobuf import message
@ -98,7 +104,7 @@ _NAN = _POS_INF * 0
_DecodeError = message.DecodeError
def _VarintDecoder(mask):
def _VarintDecoder(mask, result_type):
"""Return an encoder for a basic varint value (does not include tag).
Decoded values will be bitwise-anded with the given mask before being
@ -109,15 +115,18 @@ def _VarintDecoder(mask):
"""
local_ord = ord
py2 = _PY2 ##PY25
##!PY25 py2 = str is bytes
def DecodeVarint(buffer, pos):
result = 0
shift = 0
while 1:
b = local_ord(buffer[pos])
b = local_ord(buffer[pos]) if py2 else buffer[pos]
result |= ((b & 0x7f) << shift)
pos += 1
if not (b & 0x80):
result &= mask
result = result_type(result)
return (result, pos)
shift += 7
if shift >= 64:
@ -125,15 +134,17 @@ def _VarintDecoder(mask):
return DecodeVarint
def _SignedVarintDecoder(mask):
def _SignedVarintDecoder(mask, result_type):
"""Like _VarintDecoder() but decodes signed values."""
local_ord = ord
py2 = _PY2 ##PY25
##!PY25 py2 = str is bytes
def DecodeVarint(buffer, pos):
result = 0
shift = 0
while 1:
b = local_ord(buffer[pos])
b = local_ord(buffer[pos]) if py2 else buffer[pos]
result |= ((b & 0x7f) << shift)
pos += 1
if not (b & 0x80):
@ -142,19 +153,23 @@ def _SignedVarintDecoder(mask):
result |= ~mask
else:
result &= mask
result = result_type(result)
return (result, pos)
shift += 7
if shift >= 64:
raise _DecodeError('Too many bytes when decoding varint.')
return DecodeVarint
# We force 32-bit values to int and 64-bit values to long to make
# alternate implementations where the distinction is more significant
# (e.g. the C++ implementation) simpler.
_DecodeVarint = _VarintDecoder((1 << 64) - 1)
_DecodeSignedVarint = _SignedVarintDecoder((1 << 64) - 1)
_DecodeVarint = _VarintDecoder((1 << 64) - 1, long)
_DecodeSignedVarint = _SignedVarintDecoder((1 << 64) - 1, long)
# Use these versions for values which must be limited to 32 bits.
_DecodeVarint32 = _VarintDecoder((1 << 32) - 1)
_DecodeSignedVarint32 = _SignedVarintDecoder((1 << 32) - 1)
_DecodeVarint32 = _VarintDecoder((1 << 32) - 1, int)
_DecodeSignedVarint32 = _SignedVarintDecoder((1 << 32) - 1, int)
def ReadTag(buffer, pos):
@ -168,8 +183,10 @@ def ReadTag(buffer, pos):
use that, but not in Python.
"""
py2 = _PY2 ##PY25
##!PY25 py2 = str is bytes
start = pos
while ord(buffer[pos]) & 0x80:
while (ord(buffer[pos]) if py2 else buffer[pos]) & 0x80:
pos += 1
pos += 1
return (buffer[start:pos], pos)
@ -284,6 +301,7 @@ def _FloatDecoder():
"""
local_unpack = struct.unpack
b = (lambda x:x) if _PY2 else lambda x:x.encode('latin1') ##PY25
def InnerDecode(buffer, pos):
# We expect a 32-bit value in little-endian byte order. Bit 1 is the sign
@ -294,13 +312,17 @@ def _FloatDecoder():
# If this value has all its exponent bits set, then it's non-finite.
# In Python 2.4, struct.unpack will convert it to a finite 64-bit value.
# To avoid that, we parse it specially.
if ((float_bytes[3] in '\x7F\xFF')
and (float_bytes[2] >= '\x80')):
if ((float_bytes[3:4] in b('\x7F\xFF')) ##PY25
##!PY25 if ((float_bytes[3:4] in b'\x7F\xFF')
and (float_bytes[2:3] >= b('\x80'))): ##PY25
##!PY25 and (float_bytes[2:3] >= b'\x80')):
# If at least one significand bit is set...
if float_bytes[0:3] != '\x00\x00\x80':
if float_bytes[0:3] != b('\x00\x00\x80'): ##PY25
##!PY25 if float_bytes[0:3] != b'\x00\x00\x80':
return (_NAN, new_pos)
# If sign bit is set...
if float_bytes[3] == '\xFF':
if float_bytes[3:4] == b('\xFF'): ##PY25
##!PY25 if float_bytes[3:4] == b'\xFF':
return (_NEG_INF, new_pos)
return (_POS_INF, new_pos)
@ -319,6 +341,7 @@ def _DoubleDecoder():
"""
local_unpack = struct.unpack
b = (lambda x:x) if _PY2 else lambda x:x.encode('latin1') ##PY25
def InnerDecode(buffer, pos):
# We expect a 64-bit value in little-endian byte order. Bit 1 is the sign
@ -329,9 +352,12 @@ def _DoubleDecoder():
# If this value has all its exponent bits set and at least one significand
# bit set, it's not a number. In Python 2.4, struct.unpack will treat it
# as inf or -inf. To avoid that, we treat it specially.
if ((double_bytes[7] in '\x7F\xFF')
and (double_bytes[6] >= '\xF0')
and (double_bytes[0:7] != '\x00\x00\x00\x00\x00\x00\xF0')):
##!PY25 if ((double_bytes[7:8] in b'\x7F\xFF')
##!PY25 and (double_bytes[6:7] >= b'\xF0')
##!PY25 and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')):
if ((double_bytes[7:8] in b('\x7F\xFF')) ##PY25
and (double_bytes[6:7] >= b('\xF0')) ##PY25
and (double_bytes[0:7] != b('\x00\x00\x00\x00\x00\x00\xF0'))): ##PY25
return (_NAN, new_pos)
# Note that we expect someone up-stack to catch struct.error and convert
@ -342,10 +368,86 @@ def _DoubleDecoder():
return _SimpleDecoder(wire_format.WIRETYPE_FIXED64, InnerDecode)
def EnumDecoder(field_number, is_repeated, is_packed, key, new_default):
enum_type = key.enum_type
if is_packed:
local_DecodeVarint = _DecodeVarint
def DecodePackedField(buffer, pos, end, message, field_dict):
value = field_dict.get(key)
if value is None:
value = field_dict.setdefault(key, new_default(message))
(endpoint, pos) = local_DecodeVarint(buffer, pos)
endpoint += pos
if endpoint > end:
raise _DecodeError('Truncated message.')
while pos < endpoint:
value_start_pos = pos
(element, pos) = _DecodeSignedVarint32(buffer, pos)
if element in enum_type.values_by_number:
value.append(element)
else:
if not message._unknown_fields:
message._unknown_fields = []
tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_VARINT)
message._unknown_fields.append(
(tag_bytes, buffer[value_start_pos:pos]))
if pos > endpoint:
if element in enum_type.values_by_number:
del value[-1] # Discard corrupt value.
else:
del message._unknown_fields[-1]
raise _DecodeError('Packed element was truncated.')
return pos
return DecodePackedField
elif is_repeated:
tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_VARINT)
tag_len = len(tag_bytes)
def DecodeRepeatedField(buffer, pos, end, message, field_dict):
value = field_dict.get(key)
if value is None:
value = field_dict.setdefault(key, new_default(message))
while 1:
(element, new_pos) = _DecodeSignedVarint32(buffer, pos)
if element in enum_type.values_by_number:
value.append(element)
else:
if not message._unknown_fields:
message._unknown_fields = []
message._unknown_fields.append(
(tag_bytes, buffer[pos:new_pos]))
# Predict that the next tag is another copy of the same repeated
# field.
pos = new_pos + tag_len
if buffer[new_pos:pos] != tag_bytes or new_pos >= end:
# Prediction failed. Return.
if new_pos > end:
raise _DecodeError('Truncated message.')
return new_pos
return DecodeRepeatedField
else:
def DecodeField(buffer, pos, end, message, field_dict):
value_start_pos = pos
(enum_value, pos) = _DecodeSignedVarint32(buffer, pos)
if pos > end:
raise _DecodeError('Truncated message.')
if enum_value in enum_type.values_by_number:
field_dict[key] = enum_value
else:
if not message._unknown_fields:
message._unknown_fields = []
tag_bytes = encoder.TagBytes(field_number,
wire_format.WIRETYPE_VARINT)
message._unknown_fields.append(
(tag_bytes, buffer[value_start_pos:pos]))
return pos
return DecodeField
# --------------------------------------------------------------------
Int32Decoder = EnumDecoder = _SimpleDecoder(
Int32Decoder = _SimpleDecoder(
wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32)
Int64Decoder = _SimpleDecoder(
@ -380,6 +482,14 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default):
local_DecodeVarint = _DecodeVarint
local_unicode = unicode
def _ConvertToUnicode(byte_str):
try:
return local_unicode(byte_str, 'utf-8')
except UnicodeDecodeError, e:
# add more information to the error message and re-raise it.
e.reason = '%s in field: %s' % (e, key.full_name)
raise
assert not is_packed
if is_repeated:
tag_bytes = encoder.TagBytes(field_number,
@ -394,7 +504,7 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default):
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated string.')
value.append(local_unicode(buffer[pos:new_pos], 'utf-8'))
value.append(_ConvertToUnicode(buffer[pos:new_pos]))
# Predict that the next tag is another copy of the same repeated field.
pos = new_pos + tag_len
if buffer[new_pos:pos] != tag_bytes or new_pos == end:
@ -407,7 +517,7 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default):
new_pos = pos + size
if new_pos > end:
raise _DecodeError('Truncated string.')
field_dict[key] = local_unicode(buffer[pos:new_pos], 'utf-8')
field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos])
return new_pos
return DecodeField
@ -631,8 +741,10 @@ def MessageSetItemDecoder(extensions_by_number):
def _SkipVarint(buffer, pos, end):
"""Skip a varint value. Returns the new position."""
while ord(buffer[pos]) & 0x80:
# Previously ord(buffer[pos]) raised IndexError when pos is out of range.
# With this code, ord(b'') raises TypeError. Both are handled in
# python_message.py to generate a 'Truncated message' error.
while ord(buffer[pos:pos+1]) & 0x80:
pos += 1
pos += 1
if pos > end:
@ -699,7 +811,6 @@ def _FieldSkipper():
]
wiretype_mask = wire_format.TAG_TYPE_MASK
local_ord = ord
def SkipField(buffer, pos, end, tag_bytes):
"""Skips a field with the specified tag.
@ -712,7 +823,7 @@ def _FieldSkipper():
"""
# The wire type is always in the first byte since varints are little-endian.
wire_type = local_ord(tag_bytes[0]) & wiretype_mask
wire_type = ord(tag_bytes[0:1]) & wiretype_mask
return WIRETYPE_TO_SKIPPER[wire_type](buffer, pos, end)
return SkipField

@ -0,0 +1,58 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.pyext behavior."""
__author__ = 'anuraag@google.com (Anuraag Agrawal)'
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.descriptor_test import *
class ConfirmCppApi2Test(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

@ -34,13 +34,13 @@
__author__ = 'matthewtoia@google.com (Matt Toia)'
import unittest
from google.apputils import basetest
from google.protobuf import descriptor_pb2
from google.protobuf.internal import factory_test2_pb2
from google.protobuf import descriptor_database
class DescriptorDatabaseTest(unittest.TestCase):
class DescriptorDatabaseTest(basetest.TestCase):
def testAdd(self):
db = descriptor_database.DescriptorDatabase()
@ -49,15 +49,15 @@ class DescriptorDatabaseTest(unittest.TestCase):
db.Add(file_desc_proto)
self.assertEquals(file_desc_proto, db.FindFileByName(
'net/proto2/python/internal/factory_test2.proto'))
'google/protobuf/internal/factory_test2.proto'))
self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
'net.proto2.python.internal.Factory2Message'))
'google.protobuf.python.internal.Factory2Message'))
self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
'net.proto2.python.internal.Factory2Message.NestedFactory2Message'))
'google.protobuf.python.internal.Factory2Message.NestedFactory2Message'))
self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
'net.proto2.python.internal.Factory2Enum'))
'google.protobuf.python.internal.Factory2Enum'))
self.assertEquals(file_desc_proto, db.FindFileContainingSymbol(
'net.proto2.python.internal.Factory2Message.NestedFactory2Enum'))
'google.protobuf.python.internal.Factory2Message.NestedFactory2Enum'))
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -34,8 +34,15 @@
__author__ = 'matthewtoia@google.com (Matt Toia)'
import os
import unittest
from google.apputils import basetest
from google.protobuf import unittest_pb2
from google.protobuf import descriptor_pb2
from google.protobuf.internal import api_implementation
from google.protobuf.internal import descriptor_pool_test1_pb2
from google.protobuf.internal import descriptor_pool_test2_pb2
from google.protobuf.internal import factory_test1_pb2
from google.protobuf.internal import factory_test2_pb2
from google.protobuf import descriptor
@ -43,7 +50,7 @@ from google.protobuf import descriptor_database
from google.protobuf import descriptor_pool
class DescriptorPoolTest(unittest.TestCase):
class DescriptorPoolTest(basetest.TestCase):
def setUp(self):
self.pool = descriptor_pool.DescriptorPool()
@ -55,57 +62,51 @@ class DescriptorPoolTest(unittest.TestCase):
self.pool.Add(self.factory_test2_fd)
def testFindFileByName(self):
name1 = 'net/proto2/python/internal/factory_test1.proto'
name1 = 'google/protobuf/internal/factory_test1.proto'
file_desc1 = self.pool.FindFileByName(name1)
self.assertIsInstance(file_desc1, descriptor.FileDescriptor)
self.assertEquals(name1, file_desc1.name)
self.assertEquals('net.proto2.python.internal', file_desc1.package)
self.assertEquals('google.protobuf.python.internal', file_desc1.package)
self.assertIn('Factory1Message', file_desc1.message_types_by_name)
name2 = 'net/proto2/python/internal/factory_test2.proto'
name2 = 'google/protobuf/internal/factory_test2.proto'
file_desc2 = self.pool.FindFileByName(name2)
self.assertIsInstance(file_desc2, descriptor.FileDescriptor)
self.assertEquals(name2, file_desc2.name)
self.assertEquals('net.proto2.python.internal', file_desc2.package)
self.assertEquals('google.protobuf.python.internal', file_desc2.package)
self.assertIn('Factory2Message', file_desc2.message_types_by_name)
def testFindFileByNameFailure(self):
try:
with self.assertRaises(KeyError):
self.pool.FindFileByName('Does not exist')
self.fail('Expected KeyError')
except KeyError:
pass
def testFindFileContainingSymbol(self):
file_desc1 = self.pool.FindFileContainingSymbol(
'net.proto2.python.internal.Factory1Message')
'google.protobuf.python.internal.Factory1Message')
self.assertIsInstance(file_desc1, descriptor.FileDescriptor)
self.assertEquals('net/proto2/python/internal/factory_test1.proto',
self.assertEquals('google/protobuf/internal/factory_test1.proto',
file_desc1.name)
self.assertEquals('net.proto2.python.internal', file_desc1.package)
self.assertEquals('google.protobuf.python.internal', file_desc1.package)
self.assertIn('Factory1Message', file_desc1.message_types_by_name)
file_desc2 = self.pool.FindFileContainingSymbol(
'net.proto2.python.internal.Factory2Message')
'google.protobuf.python.internal.Factory2Message')
self.assertIsInstance(file_desc2, descriptor.FileDescriptor)
self.assertEquals('net/proto2/python/internal/factory_test2.proto',
self.assertEquals('google/protobuf/internal/factory_test2.proto',
file_desc2.name)
self.assertEquals('net.proto2.python.internal', file_desc2.package)
self.assertEquals('google.protobuf.python.internal', file_desc2.package)
self.assertIn('Factory2Message', file_desc2.message_types_by_name)
def testFindFileContainingSymbolFailure(self):
try:
with self.assertRaises(KeyError):
self.pool.FindFileContainingSymbol('Does not exist')
self.fail('Expected KeyError')
except KeyError:
pass
def testFindMessageTypeByName(self):
msg1 = self.pool.FindMessageTypeByName(
'net.proto2.python.internal.Factory1Message')
'google.protobuf.python.internal.Factory1Message')
self.assertIsInstance(msg1, descriptor.Descriptor)
self.assertEquals('Factory1Message', msg1.name)
self.assertEquals('net.proto2.python.internal.Factory1Message',
self.assertEquals('google.protobuf.python.internal.Factory1Message',
msg1.full_name)
self.assertEquals(None, msg1.containing_type)
@ -123,10 +124,10 @@ class DescriptorPoolTest(unittest.TestCase):
'nested_factory_1_enum'].enum_type)
msg2 = self.pool.FindMessageTypeByName(
'net.proto2.python.internal.Factory2Message')
'google.protobuf.python.internal.Factory2Message')
self.assertIsInstance(msg2, descriptor.Descriptor)
self.assertEquals('Factory2Message', msg2.name)
self.assertEquals('net.proto2.python.internal.Factory2Message',
self.assertEquals('google.protobuf.python.internal.Factory2Message',
msg2.full_name)
self.assertIsNone(msg2.containing_type)
@ -143,45 +144,57 @@ class DescriptorPoolTest(unittest.TestCase):
self.assertEquals(nested_enum2, msg2.fields_by_name[
'nested_factory_2_enum'].enum_type)
self.assertTrue(msg2.fields_by_name['int_with_default'].has_default)
self.assertTrue(msg2.fields_by_name['int_with_default'].has_default_value)
self.assertEquals(
1776, msg2.fields_by_name['int_with_default'].default_value)
self.assertTrue(msg2.fields_by_name['double_with_default'].has_default)
self.assertTrue(
msg2.fields_by_name['double_with_default'].has_default_value)
self.assertEquals(
9.99, msg2.fields_by_name['double_with_default'].default_value)
self.assertTrue(msg2.fields_by_name['string_with_default'].has_default)
self.assertTrue(
msg2.fields_by_name['string_with_default'].has_default_value)
self.assertEquals(
'hello world', msg2.fields_by_name['string_with_default'].default_value)
self.assertTrue(msg2.fields_by_name['bool_with_default'].has_default)
self.assertTrue(msg2.fields_by_name['bool_with_default'].has_default_value)
self.assertFalse(msg2.fields_by_name['bool_with_default'].default_value)
self.assertTrue(msg2.fields_by_name['enum_with_default'].has_default)
self.assertTrue(msg2.fields_by_name['enum_with_default'].has_default_value)
self.assertEquals(
1, msg2.fields_by_name['enum_with_default'].default_value)
msg3 = self.pool.FindMessageTypeByName(
'net.proto2.python.internal.Factory2Message.NestedFactory2Message')
'google.protobuf.python.internal.Factory2Message.NestedFactory2Message')
self.assertEquals(nested_msg2, msg3)
self.assertTrue(msg2.fields_by_name['bytes_with_default'].has_default_value)
self.assertEquals(
b'a\xfb\x00c',
msg2.fields_by_name['bytes_with_default'].default_value)
self.assertEqual(1, len(msg2.oneofs))
self.assertEqual(1, len(msg2.oneofs_by_name))
self.assertEqual(2, len(msg2.oneofs[0].fields))
for name in ['oneof_int', 'oneof_string']:
self.assertEqual(msg2.oneofs[0],
msg2.fields_by_name[name].containing_oneof)
self.assertIn(msg2.fields_by_name[name], msg2.oneofs[0].fields)
def testFindMessageTypeByNameFailure(self):
try:
with self.assertRaises(KeyError):
self.pool.FindMessageTypeByName('Does not exist')
self.fail('Expected KeyError')
except KeyError:
pass
def testFindEnumTypeByName(self):
enum1 = self.pool.FindEnumTypeByName(
'net.proto2.python.internal.Factory1Enum')
'google.protobuf.python.internal.Factory1Enum')
self.assertIsInstance(enum1, descriptor.EnumDescriptor)
self.assertEquals(0, enum1.values_by_name['FACTORY_1_VALUE_0'].number)
self.assertEquals(1, enum1.values_by_name['FACTORY_1_VALUE_1'].number)
nested_enum1 = self.pool.FindEnumTypeByName(
'net.proto2.python.internal.Factory1Message.NestedFactory1Enum')
'google.protobuf.python.internal.Factory1Message.NestedFactory1Enum')
self.assertIsInstance(nested_enum1, descriptor.EnumDescriptor)
self.assertEquals(
0, nested_enum1.values_by_name['NESTED_FACTORY_1_VALUE_0'].number)
@ -189,13 +202,13 @@ class DescriptorPoolTest(unittest.TestCase):
1, nested_enum1.values_by_name['NESTED_FACTORY_1_VALUE_1'].number)
enum2 = self.pool.FindEnumTypeByName(
'net.proto2.python.internal.Factory2Enum')
'google.protobuf.python.internal.Factory2Enum')
self.assertIsInstance(enum2, descriptor.EnumDescriptor)
self.assertEquals(0, enum2.values_by_name['FACTORY_2_VALUE_0'].number)
self.assertEquals(1, enum2.values_by_name['FACTORY_2_VALUE_1'].number)
nested_enum2 = self.pool.FindEnumTypeByName(
'net.proto2.python.internal.Factory2Message.NestedFactory2Enum')
'google.protobuf.python.internal.Factory2Message.NestedFactory2Enum')
self.assertIsInstance(nested_enum2, descriptor.EnumDescriptor)
self.assertEquals(
0, nested_enum2.values_by_name['NESTED_FACTORY_2_VALUE_0'].number)
@ -203,11 +216,8 @@ class DescriptorPoolTest(unittest.TestCase):
1, nested_enum2.values_by_name['NESTED_FACTORY_2_VALUE_1'].number)
def testFindEnumTypeByNameFailure(self):
try:
with self.assertRaises(KeyError):
self.pool.FindEnumTypeByName('Does not exist')
self.fail('Expected KeyError')
except KeyError:
pass
def testUserDefinedDB(self):
db = descriptor_database.DescriptorDatabase()
@ -216,5 +226,339 @@ class DescriptorPoolTest(unittest.TestCase):
db.Add(self.factory_test2_fd)
self.testFindMessageTypeByName()
def testComplexNesting(self):
test1_desc = descriptor_pb2.FileDescriptorProto.FromString(
descriptor_pool_test1_pb2.DESCRIPTOR.serialized_pb)
test2_desc = descriptor_pb2.FileDescriptorProto.FromString(
descriptor_pool_test2_pb2.DESCRIPTOR.serialized_pb)
self.pool.Add(test1_desc)
self.pool.Add(test2_desc)
TEST1_FILE.CheckFile(self, self.pool)
TEST2_FILE.CheckFile(self, self.pool)
class ProtoFile(object):
def __init__(self, name, package, messages, dependencies=None):
self.name = name
self.package = package
self.messages = messages
self.dependencies = dependencies or []
def CheckFile(self, test, pool):
file_desc = pool.FindFileByName(self.name)
test.assertEquals(self.name, file_desc.name)
test.assertEquals(self.package, file_desc.package)
dependencies_names = [f.name for f in file_desc.dependencies]
test.assertEqual(self.dependencies, dependencies_names)
for name, msg_type in self.messages.items():
msg_type.CheckType(test, None, name, file_desc)
class EnumType(object):
def __init__(self, values):
self.values = values
def CheckType(self, test, msg_desc, name, file_desc):
enum_desc = msg_desc.enum_types_by_name[name]
test.assertEqual(name, enum_desc.name)
expected_enum_full_name = '.'.join([msg_desc.full_name, name])
test.assertEqual(expected_enum_full_name, enum_desc.full_name)
test.assertEqual(msg_desc, enum_desc.containing_type)
test.assertEqual(file_desc, enum_desc.file)
for index, (value, number) in enumerate(self.values):
value_desc = enum_desc.values_by_name[value]
test.assertEqual(value, value_desc.name)
test.assertEqual(index, value_desc.index)
test.assertEqual(number, value_desc.number)
test.assertEqual(enum_desc, value_desc.type)
test.assertIn(value, msg_desc.enum_values_by_name)
class MessageType(object):
def __init__(self, type_dict, field_list, is_extendable=False,
extensions=None):
self.type_dict = type_dict
self.field_list = field_list
self.is_extendable = is_extendable
self.extensions = extensions or []
def CheckType(self, test, containing_type_desc, name, file_desc):
if containing_type_desc is None:
desc = file_desc.message_types_by_name[name]
expected_full_name = '.'.join([file_desc.package, name])
else:
desc = containing_type_desc.nested_types_by_name[name]
expected_full_name = '.'.join([containing_type_desc.full_name, name])
test.assertEqual(name, desc.name)
test.assertEqual(expected_full_name, desc.full_name)
test.assertEqual(containing_type_desc, desc.containing_type)
test.assertEqual(desc.file, file_desc)
test.assertEqual(self.is_extendable, desc.is_extendable)
for name, subtype in self.type_dict.items():
subtype.CheckType(test, desc, name, file_desc)
for index, (name, field) in enumerate(self.field_list):
field.CheckField(test, desc, name, index)
for index, (name, field) in enumerate(self.extensions):
field.CheckField(test, desc, name, index)
class EnumField(object):
def __init__(self, number, type_name, default_value):
self.number = number
self.type_name = type_name
self.default_value = default_value
def CheckField(self, test, msg_desc, name, index):
field_desc = msg_desc.fields_by_name[name]
enum_desc = msg_desc.enum_types_by_name[self.type_name]
test.assertEqual(name, field_desc.name)
expected_field_full_name = '.'.join([msg_desc.full_name, name])
test.assertEqual(expected_field_full_name, field_desc.full_name)
test.assertEqual(index, field_desc.index)
test.assertEqual(self.number, field_desc.number)
test.assertEqual(descriptor.FieldDescriptor.TYPE_ENUM, field_desc.type)
test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_ENUM,
field_desc.cpp_type)
test.assertTrue(field_desc.has_default_value)
test.assertEqual(enum_desc.values_by_name[self.default_value].index,
field_desc.default_value)
test.assertEqual(msg_desc, field_desc.containing_type)
test.assertEqual(enum_desc, field_desc.enum_type)
class MessageField(object):
def __init__(self, number, type_name):
self.number = number
self.type_name = type_name
def CheckField(self, test, msg_desc, name, index):
field_desc = msg_desc.fields_by_name[name]
field_type_desc = msg_desc.nested_types_by_name[self.type_name]
test.assertEqual(name, field_desc.name)
expected_field_full_name = '.'.join([msg_desc.full_name, name])
test.assertEqual(expected_field_full_name, field_desc.full_name)
test.assertEqual(index, field_desc.index)
test.assertEqual(self.number, field_desc.number)
test.assertEqual(descriptor.FieldDescriptor.TYPE_MESSAGE, field_desc.type)
test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_MESSAGE,
field_desc.cpp_type)
test.assertFalse(field_desc.has_default_value)
test.assertEqual(msg_desc, field_desc.containing_type)
test.assertEqual(field_type_desc, field_desc.message_type)
class StringField(object):
def __init__(self, number, default_value):
self.number = number
self.default_value = default_value
def CheckField(self, test, msg_desc, name, index):
field_desc = msg_desc.fields_by_name[name]
test.assertEqual(name, field_desc.name)
expected_field_full_name = '.'.join([msg_desc.full_name, name])
test.assertEqual(expected_field_full_name, field_desc.full_name)
test.assertEqual(index, field_desc.index)
test.assertEqual(self.number, field_desc.number)
test.assertEqual(descriptor.FieldDescriptor.TYPE_STRING, field_desc.type)
test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_STRING,
field_desc.cpp_type)
test.assertTrue(field_desc.has_default_value)
test.assertEqual(self.default_value, field_desc.default_value)
class ExtensionField(object):
def __init__(self, number, extended_type):
self.number = number
self.extended_type = extended_type
def CheckField(self, test, msg_desc, name, index):
field_desc = msg_desc.extensions_by_name[name]
test.assertEqual(name, field_desc.name)
expected_field_full_name = '.'.join([msg_desc.full_name, name])
test.assertEqual(expected_field_full_name, field_desc.full_name)
test.assertEqual(self.number, field_desc.number)
test.assertEqual(index, field_desc.index)
test.assertEqual(descriptor.FieldDescriptor.TYPE_MESSAGE, field_desc.type)
test.assertEqual(descriptor.FieldDescriptor.CPPTYPE_MESSAGE,
field_desc.cpp_type)
test.assertFalse(field_desc.has_default_value)
test.assertTrue(field_desc.is_extension)
test.assertEqual(msg_desc, field_desc.extension_scope)
test.assertEqual(msg_desc, field_desc.message_type)
test.assertEqual(self.extended_type, field_desc.containing_type.name)
class AddDescriptorTest(basetest.TestCase):
def _TestMessage(self, prefix):
pool = descriptor_pool.DescriptorPool()
pool.AddDescriptor(unittest_pb2.TestAllTypes.DESCRIPTOR)
self.assertEquals(
'protobuf_unittest.TestAllTypes',
pool.FindMessageTypeByName(
prefix + 'protobuf_unittest.TestAllTypes').full_name)
# AddDescriptor is not recursive.
with self.assertRaises(KeyError):
pool.FindMessageTypeByName(
prefix + 'protobuf_unittest.TestAllTypes.NestedMessage')
pool.AddDescriptor(unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR)
self.assertEquals(
'protobuf_unittest.TestAllTypes.NestedMessage',
pool.FindMessageTypeByName(
prefix + 'protobuf_unittest.TestAllTypes.NestedMessage').full_name)
# Files are implicitly also indexed when messages are added.
self.assertEquals(
'google/protobuf/unittest.proto',
pool.FindFileByName(
'google/protobuf/unittest.proto').name)
self.assertEquals(
'google/protobuf/unittest.proto',
pool.FindFileContainingSymbol(
prefix + 'protobuf_unittest.TestAllTypes.NestedMessage').name)
def testMessage(self):
self._TestMessage('')
self._TestMessage('.')
def _TestEnum(self, prefix):
pool = descriptor_pool.DescriptorPool()
pool.AddEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR)
self.assertEquals(
'protobuf_unittest.ForeignEnum',
pool.FindEnumTypeByName(
prefix + 'protobuf_unittest.ForeignEnum').full_name)
# AddEnumDescriptor is not recursive.
with self.assertRaises(KeyError):
pool.FindEnumTypeByName(
prefix + 'protobuf_unittest.ForeignEnum.NestedEnum')
pool.AddEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR)
self.assertEquals(
'protobuf_unittest.TestAllTypes.NestedEnum',
pool.FindEnumTypeByName(
prefix + 'protobuf_unittest.TestAllTypes.NestedEnum').full_name)
# Files are implicitly also indexed when enums are added.
self.assertEquals(
'google/protobuf/unittest.proto',
pool.FindFileByName(
'google/protobuf/unittest.proto').name)
self.assertEquals(
'google/protobuf/unittest.proto',
pool.FindFileContainingSymbol(
prefix + 'protobuf_unittest.TestAllTypes.NestedEnum').name)
def testEnum(self):
self._TestEnum('')
self._TestEnum('.')
def testFile(self):
pool = descriptor_pool.DescriptorPool()
pool.AddFileDescriptor(unittest_pb2.DESCRIPTOR)
self.assertEquals(
'google/protobuf/unittest.proto',
pool.FindFileByName(
'google/protobuf/unittest.proto').name)
# AddFileDescriptor is not recursive; messages and enums within files must
# be explicitly registered.
with self.assertRaises(KeyError):
pool.FindFileContainingSymbol(
'protobuf_unittest.TestAllTypes')
TEST1_FILE = ProtoFile(
'google/protobuf/internal/descriptor_pool_test1.proto',
'google.protobuf.python.internal',
{
'DescriptorPoolTest1': MessageType({
'NestedEnum': EnumType([('ALPHA', 1), ('BETA', 2)]),
'NestedMessage': MessageType({
'NestedEnum': EnumType([('EPSILON', 5), ('ZETA', 6)]),
'DeepNestedMessage': MessageType({
'NestedEnum': EnumType([('ETA', 7), ('THETA', 8)]),
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'ETA')),
('nested_field', StringField(2, 'theta')),
]),
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'ZETA')),
('nested_field', StringField(2, 'beta')),
('deep_nested_message', MessageField(3, 'DeepNestedMessage')),
])
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'BETA')),
('nested_message', MessageField(2, 'NestedMessage')),
], is_extendable=True),
'DescriptorPoolTest2': MessageType({
'NestedEnum': EnumType([('GAMMA', 3), ('DELTA', 4)]),
'NestedMessage': MessageType({
'NestedEnum': EnumType([('IOTA', 9), ('KAPPA', 10)]),
'DeepNestedMessage': MessageType({
'NestedEnum': EnumType([('LAMBDA', 11), ('MU', 12)]),
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'MU')),
('nested_field', StringField(2, 'lambda')),
]),
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'IOTA')),
('nested_field', StringField(2, 'delta')),
('deep_nested_message', MessageField(3, 'DeepNestedMessage')),
])
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'GAMMA')),
('nested_message', MessageField(2, 'NestedMessage')),
]),
})
TEST2_FILE = ProtoFile(
'google/protobuf/internal/descriptor_pool_test2.proto',
'google.protobuf.python.internal',
{
'DescriptorPoolTest3': MessageType({
'NestedEnum': EnumType([('NU', 13), ('XI', 14)]),
'NestedMessage': MessageType({
'NestedEnum': EnumType([('OMICRON', 15), ('PI', 16)]),
'DeepNestedMessage': MessageType({
'NestedEnum': EnumType([('RHO', 17), ('SIGMA', 18)]),
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'RHO')),
('nested_field', StringField(2, 'sigma')),
]),
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'PI')),
('nested_field', StringField(2, 'nu')),
('deep_nested_message', MessageField(3, 'DeepNestedMessage')),
])
}, [
('nested_enum', EnumField(1, 'NestedEnum', 'XI')),
('nested_message', MessageField(2, 'NestedMessage')),
], extensions=[
('descriptor_pool_test',
ExtensionField(1001, 'DescriptorPoolTest1')),
]),
},
dependencies=['google/protobuf/internal/descriptor_pool_test1.proto'])
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -0,0 +1,94 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package google.protobuf.python.internal;
message DescriptorPoolTest1 {
extensions 1000 to max;
enum NestedEnum {
ALPHA = 1;
BETA = 2;
}
optional NestedEnum nested_enum = 1 [default = BETA];
message NestedMessage {
enum NestedEnum {
EPSILON = 5;
ZETA = 6;
}
optional NestedEnum nested_enum = 1 [default = ZETA];
optional string nested_field = 2 [default = "beta"];
optional DeepNestedMessage deep_nested_message = 3;
message DeepNestedMessage {
enum NestedEnum {
ETA = 7;
THETA = 8;
}
optional NestedEnum nested_enum = 1 [default = ETA];
optional string nested_field = 2 [default = "theta"];
}
}
optional NestedMessage nested_message = 2;
}
message DescriptorPoolTest2 {
enum NestedEnum {
GAMMA = 3;
DELTA = 4;
}
optional NestedEnum nested_enum = 1 [default = GAMMA];
message NestedMessage {
enum NestedEnum {
IOTA = 9;
KAPPA = 10;
}
optional NestedEnum nested_enum = 1 [default = IOTA];
optional string nested_field = 2 [default = "delta"];
optional DeepNestedMessage deep_nested_message = 3;
message DeepNestedMessage {
enum NestedEnum {
LAMBDA = 11;
MU = 12;
}
optional NestedEnum nested_enum = 1 [default = MU];
optional string nested_field = 2 [default = "lambda"];
}
}
optional NestedMessage nested_message = 2;
}

@ -0,0 +1,70 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package google.protobuf.python.internal;
import "google/protobuf/internal/descriptor_pool_test1.proto";
message DescriptorPoolTest3 {
extend DescriptorPoolTest1 {
optional DescriptorPoolTest3 descriptor_pool_test = 1001;
}
enum NestedEnum {
NU = 13;
XI = 14;
}
optional NestedEnum nested_enum = 1 [default = XI];
message NestedMessage {
enum NestedEnum {
OMICRON = 15;
PI = 16;
}
optional NestedEnum nested_enum = 1 [default = PI];
optional string nested_field = 2 [default = "nu"];
optional DeepNestedMessage deep_nested_message = 3;
message DeepNestedMessage {
enum NestedEnum {
RHO = 17;
SIGMA = 18;
}
optional NestedEnum nested_enum = 1 [default = RHO];
optional string nested_field = 2 [default = "sigma"];
}
}
optional NestedMessage nested_message = 2;
}

@ -0,0 +1,54 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Unittest for descriptor.py for the pure Python implementation."""
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.descriptor_test import *
class ConfirmPurePythonTest(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('python', api_implementation.Type())
if __name__ == '__main__':
basetest.main()

@ -34,7 +34,7 @@
__author__ = 'robinson@google.com (Will Robinson)'
import unittest
from google.apputils import basetest
from google.protobuf import unittest_custom_options_pb2
from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_pb2
@ -48,7 +48,7 @@ name: 'TestEmptyMessage'
"""
class DescriptorTest(unittest.TestCase):
class DescriptorTest(basetest.TestCase):
def setUp(self):
self.my_file = descriptor.FileDescriptor(
@ -244,7 +244,7 @@ class DescriptorTest(unittest.TestCase):
unittest_custom_options_pb2.double_opt])
self.assertEqual("Hello, \"World\"", message_options.Extensions[
unittest_custom_options_pb2.string_opt])
self.assertEqual("Hello\0World", message_options.Extensions[
self.assertEqual(b"Hello\0World", message_options.Extensions[
unittest_custom_options_pb2.bytes_opt])
dummy_enum = unittest_custom_options_pb2.DummyMessageContainingEnum
self.assertEqual(
@ -395,7 +395,7 @@ class DescriptorTest(unittest.TestCase):
self.assertEqual(self.my_file.package, 'protobuf_unittest')
class DescriptorCopyToProtoTest(unittest.TestCase):
class DescriptorCopyToProtoTest(basetest.TestCase):
"""Tests for CopyTo functions of Descriptor."""
def _AssertProtoEqual(self, actual_proto, expected_class, expected_ascii):
@ -530,47 +530,49 @@ class DescriptorCopyToProtoTest(unittest.TestCase):
descriptor_pb2.DescriptorProto,
TEST_MESSAGE_WITH_SEVERAL_EXTENSIONS_ASCII)
def testCopyToProto_FileDescriptor(self):
UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII = ("""
name: 'google/protobuf/unittest_import.proto'
package: 'protobuf_unittest_import'
dependency: 'google/protobuf/unittest_import_public.proto'
message_type: <
name: 'ImportMessage'
field: <
name: 'd'
number: 1
label: 1 # Optional
type: 5 # TYPE_INT32
>
>
""" +
"""enum_type: <
name: 'ImportEnum'
value: <
name: 'IMPORT_FOO'
number: 7
>
value: <
name: 'IMPORT_BAR'
number: 8
>
value: <
name: 'IMPORT_BAZ'
number: 9
>
>
options: <
java_package: 'com.google.protobuf.test'
optimize_for: 1 # SPEED
>
public_dependency: 0
""")
self._InternalTestCopyToProto(
unittest_import_pb2.DESCRIPTOR,
descriptor_pb2.FileDescriptorProto,
UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII)
# Disable this test so we can make changes to the proto file.
# TODO(xiaofeng): Enable this test after cl/55530659 is submitted.
#
# def testCopyToProto_FileDescriptor(self):
# UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII = ("""
# name: 'google/protobuf/unittest_import.proto'
# package: 'protobuf_unittest_import'
# dependency: 'google/protobuf/unittest_import_public.proto'
# message_type: <
# name: 'ImportMessage'
# field: <
# name: 'd'
# number: 1
# label: 1 # Optional
# type: 5 # TYPE_INT32
# >
# >
# """ +
# """enum_type: <
# name: 'ImportEnum'
# value: <
# name: 'IMPORT_FOO'
# number: 7
# >
# value: <
# name: 'IMPORT_BAR'
# number: 8
# >
# value: <
# name: 'IMPORT_BAZ'
# number: 9
# >
# >
# options: <
# java_package: 'com.google.protobuf.test'
# optimize_for: 1 # SPEED
# >
# public_dependency: 0
# """)
# self._InternalTestCopyToProto(
# unittest_import_pb2.DESCRIPTOR,
# descriptor_pb2.FileDescriptorProto,
# UNITTEST_IMPORT_FILE_DESCRIPTOR_ASCII)
def testCopyToProto_ServiceDescriptor(self):
TEST_SERVICE_ASCII = """
@ -586,28 +588,82 @@ class DescriptorCopyToProtoTest(unittest.TestCase):
output_type: '.protobuf_unittest.BarResponse'
>
"""
self._InternalTestCopyToProto(
unittest_pb2.TestService.DESCRIPTOR,
descriptor_pb2.ServiceDescriptorProto,
TEST_SERVICE_ASCII)
class MakeDescriptorTest(unittest.TestCase):
class MakeDescriptorTest(basetest.TestCase):
def testMakeDescriptorWithNestedFields(self):
file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
file_descriptor_proto.name = 'Foo2'
message_type = file_descriptor_proto.message_type.add()
message_type.name = file_descriptor_proto.name
nested_type = message_type.nested_type.add()
nested_type.name = 'Sub'
enum_type = nested_type.enum_type.add()
enum_type.name = 'FOO'
enum_type_val = enum_type.value.add()
enum_type_val.name = 'BAR'
enum_type_val.number = 3
field = message_type.field.add()
field.number = 1
field.name = 'uint64_field'
field.label = descriptor.FieldDescriptor.LABEL_REQUIRED
field.type = descriptor.FieldDescriptor.TYPE_UINT64
field = message_type.field.add()
field.number = 2
field.name = 'nested_message_field'
field.label = descriptor.FieldDescriptor.LABEL_REQUIRED
field.type = descriptor.FieldDescriptor.TYPE_MESSAGE
field.type_name = 'Sub'
enum_field = nested_type.field.add()
enum_field.number = 2
enum_field.name = 'bar_field'
enum_field.label = descriptor.FieldDescriptor.LABEL_REQUIRED
enum_field.type = descriptor.FieldDescriptor.TYPE_ENUM
enum_field.type_name = 'Foo2.Sub.FOO'
result = descriptor.MakeDescriptor(message_type)
self.assertEqual(result.fields[0].cpp_type,
descriptor.FieldDescriptor.CPPTYPE_UINT64)
self.assertEqual(result.fields[1].cpp_type,
descriptor.FieldDescriptor.CPPTYPE_MESSAGE)
self.assertEqual(result.fields[1].message_type.containing_type,
result)
self.assertEqual(result.nested_types[0].fields[0].full_name,
'Foo2.Sub.bar_field')
self.assertEqual(result.nested_types[0].fields[0].enum_type,
result.nested_types[0].enum_types[0])
def testMakeDescriptorWithUnsignedIntField(self):
file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
file_descriptor_proto.name = 'Foo'
message_type = file_descriptor_proto.message_type.add()
message_type.name = file_descriptor_proto.name
enum_type = message_type.enum_type.add()
enum_type.name = 'FOO'
enum_type_val = enum_type.value.add()
enum_type_val.name = 'BAR'
enum_type_val.number = 3
field = message_type.field.add()
field.number = 1
field.name = 'uint64_field'
field.label = descriptor.FieldDescriptor.LABEL_REQUIRED
field.type = descriptor.FieldDescriptor.TYPE_UINT64
enum_field = message_type.field.add()
enum_field.number = 2
enum_field.name = 'bar_field'
enum_field.label = descriptor.FieldDescriptor.LABEL_REQUIRED
enum_field.type = descriptor.FieldDescriptor.TYPE_ENUM
enum_field.type_name = 'Foo.FOO'
result = descriptor.MakeDescriptor(message_type)
self.assertEqual(result.fields[0].cpp_type,
descriptor.FieldDescriptor.CPPTYPE_UINT64)
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -28,6 +28,10 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#PY25 compatible for GAE.
#
# Copyright 2009 Google Inc. All Rights Reserved.
"""Code for encoding protocol message primitives.
Contains the logic for encoding every logical protocol field type
@ -67,6 +71,8 @@ sizer rather than when calling them. In particular:
__author__ = 'kenton@google.com (Kenton Varda)'
import struct
import sys ##PY25
_PY2 = sys.version_info[0] < 3 ##PY25
from google.protobuf.internal import wire_format
@ -340,7 +346,8 @@ def MessageSetItemSizer(field_number):
def _VarintEncoder():
"""Return an encoder for a basic varint value (does not include tag)."""
local_chr = chr
local_chr = _PY2 and chr or (lambda x: bytes((x,))) ##PY25
##!PY25 local_chr = chr if bytes is str else lambda x: bytes((x,))
def EncodeVarint(write, value):
bits = value & 0x7f
value >>= 7
@ -357,7 +364,8 @@ def _SignedVarintEncoder():
"""Return an encoder for a basic signed varint value (does not include
tag)."""
local_chr = chr
local_chr = _PY2 and chr or (lambda x: bytes((x,))) ##PY25
##!PY25 local_chr = chr if bytes is str else lambda x: bytes((x,))
def EncodeSignedVarint(write, value):
if value < 0:
value += (1 << 64)
@ -382,7 +390,8 @@ def _VarintBytes(value):
pieces = []
_EncodeVarint(pieces.append, value)
return "".join(pieces)
return "".encode("latin1").join(pieces) ##PY25
##!PY25 return b"".join(pieces)
def TagBytes(field_number, wire_type):
@ -520,26 +529,33 @@ def _FloatingPointEncoder(wire_type, format):
format: The format string to pass to struct.pack().
"""
b = _PY2 and (lambda x:x) or (lambda x:x.encode('latin1')) ##PY25
value_size = struct.calcsize(format)
if value_size == 4:
def EncodeNonFiniteOrRaise(write, value):
# Remember that the serialized form uses little-endian byte order.
if value == _POS_INF:
write('\x00\x00\x80\x7F')
write(b('\x00\x00\x80\x7F')) ##PY25
##!PY25 write(b'\x00\x00\x80\x7F')
elif value == _NEG_INF:
write('\x00\x00\x80\xFF')
write(b('\x00\x00\x80\xFF')) ##PY25
##!PY25 write(b'\x00\x00\x80\xFF')
elif value != value: # NaN
write('\x00\x00\xC0\x7F')
write(b('\x00\x00\xC0\x7F')) ##PY25
##!PY25 write(b'\x00\x00\xC0\x7F')
else:
raise
elif value_size == 8:
def EncodeNonFiniteOrRaise(write, value):
if value == _POS_INF:
write('\x00\x00\x00\x00\x00\x00\xF0\x7F')
write(b('\x00\x00\x00\x00\x00\x00\xF0\x7F')) ##PY25
##!PY25 write(b'\x00\x00\x00\x00\x00\x00\xF0\x7F')
elif value == _NEG_INF:
write('\x00\x00\x00\x00\x00\x00\xF0\xFF')
write(b('\x00\x00\x00\x00\x00\x00\xF0\xFF')) ##PY25
##!PY25 write(b'\x00\x00\x00\x00\x00\x00\xF0\xFF')
elif value != value: # NaN
write('\x00\x00\x00\x00\x00\x00\xF8\x7F')
write(b('\x00\x00\x00\x00\x00\x00\xF8\x7F')) ##PY25
##!PY25 write(b'\x00\x00\x00\x00\x00\x00\xF8\x7F')
else:
raise
else:
@ -615,8 +631,10 @@ DoubleEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, '<d')
def BoolEncoder(field_number, is_repeated, is_packed):
"""Returns an encoder for a boolean field."""
false_byte = chr(0)
true_byte = chr(1)
##!PY25 false_byte = b'\x00'
##!PY25 true_byte = b'\x01'
false_byte = '\x00'.encode('latin1') ##PY25
true_byte = '\x01'.encode('latin1') ##PY25
if is_packed:
tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
local_EncodeVarint = _EncodeVarint
@ -752,7 +770,8 @@ def MessageSetItemEncoder(field_number):
}
}
"""
start_bytes = "".join([
start_bytes = "".encode("latin1").join([ ##PY25
##!PY25 start_bytes = b"".join([
TagBytes(1, wire_format.WIRETYPE_START_GROUP),
TagBytes(2, wire_format.WIRETYPE_VARINT),
_VarintBytes(field_number),

@ -52,4 +52,6 @@ message Factory1Message {
optional NestedFactory1Message nested_factory_1_message = 3;
optional int32 scalar_value = 4;
repeated string list_value = 5;
extensions 1000 to max;
}

@ -70,8 +70,23 @@ message Factory2Message {
optional string string_with_default = 18 [default = "hello world"];
optional bool bool_with_default = 19 [default = false];
optional Factory2Enum enum_with_default = 20 [default = FACTORY_2_VALUE_1];
optional bytes bytes_with_default = 21 [default = "a\373\000c"];
extend Factory1Message {
optional string one_more_field = 1001;
}
oneof oneof_field {
int32 oneof_int = 22;
string oneof_string = 23;
}
}
message LoopMessage {
optional Factory2Message loop = 1;
}
extend Factory1Message {
optional string another_field = 1002;
}

@ -41,20 +41,21 @@ further ensures that we can use Python protocol message objects as we expect.
__author__ = 'robinson@google.com (Will Robinson)'
import unittest
from google.apputils import basetest
from google.protobuf.internal import test_bad_identifiers_pb2
from google.protobuf import unittest_custom_options_pb2
from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_import_public_pb2
from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2
from google.protobuf import unittest_no_generic_services_pb2
from google.protobuf import unittest_pb2
from google.protobuf import service
from google.protobuf import symbol_database
MAX_EXTENSION = 536870912
class GeneratorTest(unittest.TestCase):
class GeneratorTest(basetest.TestCase):
def testNestedMessageDescriptor(self):
field_name = 'optional_nested_message'
@ -217,6 +218,10 @@ class GeneratorTest(unittest.TestCase):
'google/protobuf/unittest.proto')
self.assertEqual(unittest_pb2.DESCRIPTOR.package, 'protobuf_unittest')
self.assertFalse(unittest_pb2.DESCRIPTOR.serialized_pb is None)
self.assertEqual(unittest_pb2.DESCRIPTOR.dependencies,
[unittest_import_pb2.DESCRIPTOR])
self.assertEqual(unittest_import_pb2.DESCRIPTOR.dependencies,
[unittest_import_public_pb2.DESCRIPTOR])
def testNoGenericServices(self):
self.assertTrue(hasattr(unittest_no_generic_services_pb2, "TestMessage"))
@ -241,6 +246,18 @@ class GeneratorTest(unittest.TestCase):
unittest_pb2._TESTALLTYPES_NESTEDMESSAGE.name in
file_type.message_types_by_name)
def testEnumTypesByName(self):
file_type = unittest_pb2.DESCRIPTOR
self.assertEqual(
unittest_pb2._FOREIGNENUM,
file_type.enum_types_by_name[unittest_pb2._FOREIGNENUM.name])
def testExtensionsByName(self):
file_type = unittest_pb2.DESCRIPTOR
self.assertEqual(
unittest_pb2.my_extension_string,
file_type.extensions_by_name[unittest_pb2.my_extension_string.name])
def testPublicImports(self):
# Test public imports as embedded message.
all_type_proto = unittest_pb2.TestAllTypes()
@ -265,5 +282,62 @@ class GeneratorTest(unittest.TestCase):
self.assertEqual(message.Extensions[test_bad_identifiers_pb2.service],
"qux")
def testOneof(self):
desc = unittest_pb2.TestAllTypes.DESCRIPTOR
self.assertEqual(1, len(desc.oneofs))
self.assertEqual('oneof_field', desc.oneofs[0].name)
self.assertEqual(0, desc.oneofs[0].index)
self.assertIs(desc, desc.oneofs[0].containing_type)
self.assertIs(desc.oneofs[0], desc.oneofs_by_name['oneof_field'])
nested_names = set(['oneof_uint32', 'oneof_nested_message',
'oneof_string', 'oneof_bytes'])
self.assertSameElements(
nested_names,
[field.name for field in desc.oneofs[0].fields])
for field_name, field_desc in desc.fields_by_name.iteritems():
if field_name in nested_names:
self.assertIs(desc.oneofs[0], field_desc.containing_oneof)
else:
self.assertIsNone(field_desc.containing_oneof)
class SymbolDatabaseRegistrationTest(basetest.TestCase):
"""Checks that messages, enums and files are correctly registered."""
def testGetSymbol(self):
self.assertEquals(
unittest_pb2.TestAllTypes, symbol_database.Default().GetSymbol(
'protobuf_unittest.TestAllTypes'))
self.assertEquals(
unittest_pb2.TestAllTypes.NestedMessage,
symbol_database.Default().GetSymbol(
'protobuf_unittest.TestAllTypes.NestedMessage'))
with self.assertRaises(KeyError):
symbol_database.Default().GetSymbol('protobuf_unittest.NestedMessage')
self.assertEquals(
unittest_pb2.TestAllTypes.OptionalGroup,
symbol_database.Default().GetSymbol(
'protobuf_unittest.TestAllTypes.OptionalGroup'))
self.assertEquals(
unittest_pb2.TestAllTypes.RepeatedGroup,
symbol_database.Default().GetSymbol(
'protobuf_unittest.TestAllTypes.RepeatedGroup'))
def testEnums(self):
self.assertEquals(
'protobuf_unittest.ForeignEnum',
symbol_database.Default().pool.FindEnumTypeByName(
'protobuf_unittest.ForeignEnum').full_name)
self.assertEquals(
'protobuf_unittest.TestAllTypes.NestedEnum',
symbol_database.Default().pool.FindEnumTypeByName(
'protobuf_unittest.TestAllTypes.NestedEnum').full_name)
def testFindFileByName(self):
self.assertEquals(
'google/protobuf/unittest.proto',
symbol_database.Default().pool.FindFileByName(
'google/protobuf/unittest.proto').name)
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -0,0 +1,56 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.message_factory."""
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.message_factory_test import *
class ConfirmCppApi2Test(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
if __name__ == '__main__':
basetest.main()

@ -0,0 +1,54 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for ..public.message_factory for the pure Python implementation."""
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.message_factory_test import *
class ConfirmPurePythonTest(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('python', api_implementation.Type())
if __name__ == '__main__':
basetest.main()

@ -34,7 +34,7 @@
__author__ = 'matthewtoia@google.com (Matt Toia)'
import unittest
from google.apputils import basetest
from google.protobuf import descriptor_pb2
from google.protobuf.internal import factory_test1_pb2
from google.protobuf.internal import factory_test2_pb2
@ -43,7 +43,7 @@ from google.protobuf import descriptor_pool
from google.protobuf import message_factory
class MessageFactoryTest(unittest.TestCase):
class MessageFactoryTest(basetest.TestCase):
def setUp(self):
self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString(
@ -61,8 +61,8 @@ class MessageFactoryTest(unittest.TestCase):
msg.factory_1_message.nested_factory_1_message.value = (
'nested message value')
msg.factory_1_message.scalar_value = 22
msg.factory_1_message.list_value.extend(['one', 'two', 'three'])
msg.factory_1_message.list_value.append('four')
msg.factory_1_message.list_value.extend([u'one', u'two', u'three'])
msg.factory_1_message.list_value.append(u'four')
msg.factory_1_enum = 1
msg.nested_factory_1_enum = 0
msg.nested_factory_1_message.value = 'nested message value'
@ -70,8 +70,8 @@ class MessageFactoryTest(unittest.TestCase):
msg.circular_message.circular_message.mandatory = 2
msg.circular_message.scalar_value = 'one deep'
msg.scalar_value = 'zero deep'
msg.list_value.extend(['four', 'three', 'two'])
msg.list_value.append('one')
msg.list_value.extend([u'four', u'three', u'two'])
msg.list_value.append(u'one')
msg.grouped.add()
msg.grouped[0].part_1 = 'hello'
msg.grouped[0].part_2 = 'world'
@ -92,22 +92,40 @@ class MessageFactoryTest(unittest.TestCase):
db.Add(self.factory_test2_fd)
factory = message_factory.MessageFactory()
cls = factory.GetPrototype(pool.FindMessageTypeByName(
'net.proto2.python.internal.Factory2Message'))
'google.protobuf.python.internal.Factory2Message'))
self.assertIsNot(cls, factory_test2_pb2.Factory2Message)
self._ExerciseDynamicClass(cls)
cls2 = factory.GetPrototype(pool.FindMessageTypeByName(
'net.proto2.python.internal.Factory2Message'))
'google.protobuf.python.internal.Factory2Message'))
self.assertIs(cls, cls2)
def testGetMessages(self):
messages = message_factory.GetMessages([self.factory_test2_fd,
self.factory_test1_fd])
self.assertContainsSubset(
['net.proto2.python.internal.Factory2Message',
'net.proto2.python.internal.Factory1Message'],
messages.keys())
self._ExerciseDynamicClass(
messages['net.proto2.python.internal.Factory2Message'])
# performed twice because multiple calls with the same input must be allowed
for _ in range(2):
messages = message_factory.GetMessages([self.factory_test2_fd,
self.factory_test1_fd])
self.assertContainsSubset(
['google.protobuf.python.internal.Factory2Message',
'google.protobuf.python.internal.Factory1Message'],
messages.keys())
self._ExerciseDynamicClass(
messages['google.protobuf.python.internal.Factory2Message'])
self.assertContainsSubset(
['google.protobuf.python.internal.Factory2Message.one_more_field',
'google.protobuf.python.internal.another_field'],
(messages['google.protobuf.python.internal.Factory1Message']
._extensions_by_name.keys()))
factory_msg1 = messages['google.protobuf.python.internal.Factory1Message']
msg1 = messages['google.protobuf.python.internal.Factory1Message']()
ext1 = factory_msg1._extensions_by_name[
'google.protobuf.python.internal.Factory2Message.one_more_field']
ext2 = factory_msg1._extensions_by_name[
'google.protobuf.python.internal.another_field']
msg1.Extensions[ext1] = 'test1'
msg1.Extensions[ext2] = 'test2'
self.assertEquals('test1', msg1.Extensions[ext1])
self.assertEquals('test2', msg1.Extensions[ext2])
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -30,16 +30,25 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.internal.message_cpp."""
__author__ = 'shahms@google.com (Shahms King)'
"""Tests for ..public.message for the pure Python implementation."""
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'
import unittest
# We must set the implementation version above before the google3 imports.
# pylint: disable=g-import-not-at-top
from google.apputils import basetest
from google.protobuf.internal import api_implementation
# Run all tests from the original module by putting them in our namespace.
# pylint: disable=wildcard-import
from google.protobuf.internal.message_test import *
class ConfirmPurePythonTest(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('python', api_implementation.Type())
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -47,9 +47,9 @@ import copy
import math
import operator
import pickle
import sys
import unittest
from google.protobuf import unittest_import_pb2
from google.apputils import basetest
from google.protobuf import unittest_pb2
from google.protobuf.internal import api_implementation
from google.protobuf.internal import test_util
@ -68,10 +68,23 @@ def IsPosInf(val):
def IsNegInf(val):
return isinf(val) and (val < 0)
class MessageTest(unittest.TestCase):
class MessageTest(basetest.TestCase):
def testBadUtf8String(self):
if api_implementation.Type() != 'python':
self.skipTest("Skipping testBadUtf8String, currently only the python "
"api implementation raises UnicodeDecodeError when a "
"string field contains bad utf-8.")
bad_utf8_data = test_util.GoldenFileData('bad_utf8_string')
with self.assertRaises(UnicodeDecodeError) as context:
unittest_pb2.TestAllTypes.FromString(bad_utf8_data)
self.assertIn('field: protobuf_unittest.TestAllTypes.optional_string',
str(context.exception))
def testGoldenMessage(self):
golden_data = test_util.GoldenFile('golden_message').read()
golden_data = test_util.GoldenFileData(
'golden_message_oneof_implemented')
golden_message = unittest_pb2.TestAllTypes()
golden_message.ParseFromString(golden_data)
test_util.ExpectAllFieldsSet(self, golden_message)
@ -80,7 +93,7 @@ class MessageTest(unittest.TestCase):
self.assertEqual(golden_data, golden_copy.SerializeToString())
def testGoldenExtensions(self):
golden_data = test_util.GoldenFile('golden_message').read()
golden_data = test_util.GoldenFileData('golden_message')
golden_message = unittest_pb2.TestAllExtensions()
golden_message.ParseFromString(golden_data)
all_set = unittest_pb2.TestAllExtensions()
@ -91,7 +104,7 @@ class MessageTest(unittest.TestCase):
self.assertEqual(golden_data, golden_copy.SerializeToString())
def testGoldenPackedMessage(self):
golden_data = test_util.GoldenFile('golden_packed_fields_message').read()
golden_data = test_util.GoldenFileData('golden_packed_fields_message')
golden_message = unittest_pb2.TestPackedTypes()
golden_message.ParseFromString(golden_data)
all_set = unittest_pb2.TestPackedTypes()
@ -102,7 +115,7 @@ class MessageTest(unittest.TestCase):
self.assertEqual(golden_data, golden_copy.SerializeToString())
def testGoldenPackedExtensions(self):
golden_data = test_util.GoldenFile('golden_packed_fields_message').read()
golden_data = test_util.GoldenFileData('golden_packed_fields_message')
golden_message = unittest_pb2.TestPackedExtensions()
golden_message.ParseFromString(golden_data)
all_set = unittest_pb2.TestPackedExtensions()
@ -113,7 +126,7 @@ class MessageTest(unittest.TestCase):
self.assertEqual(golden_data, golden_copy.SerializeToString())
def testPickleSupport(self):
golden_data = test_util.GoldenFile('golden_message').read()
golden_data = test_util.GoldenFileData('golden_message')
golden_message = unittest_pb2.TestAllTypes()
golden_message.ParseFromString(golden_data)
pickled_message = pickle.dumps(golden_message)
@ -121,6 +134,7 @@ class MessageTest(unittest.TestCase):
unpickled_message = pickle.loads(pickled_message)
self.assertEquals(unpickled_message, golden_message)
def testPickleIncompleteProto(self):
golden_message = unittest_pb2.TestRequired(a=1)
pickled_message = pickle.dumps(golden_message)
@ -132,10 +146,10 @@ class MessageTest(unittest.TestCase):
self.assertRaises(message.EncodeError, unpickled_message.SerializeToString)
def testPositiveInfinity(self):
golden_data = ('\x5D\x00\x00\x80\x7F'
'\x61\x00\x00\x00\x00\x00\x00\xF0\x7F'
'\xCD\x02\x00\x00\x80\x7F'
'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\x7F')
golden_data = (b'\x5D\x00\x00\x80\x7F'
b'\x61\x00\x00\x00\x00\x00\x00\xF0\x7F'
b'\xCD\x02\x00\x00\x80\x7F'
b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\x7F')
golden_message = unittest_pb2.TestAllTypes()
golden_message.ParseFromString(golden_data)
self.assertTrue(IsPosInf(golden_message.optional_float))
@ -145,10 +159,10 @@ class MessageTest(unittest.TestCase):
self.assertEqual(golden_data, golden_message.SerializeToString())
def testNegativeInfinity(self):
golden_data = ('\x5D\x00\x00\x80\xFF'
'\x61\x00\x00\x00\x00\x00\x00\xF0\xFF'
'\xCD\x02\x00\x00\x80\xFF'
'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\xFF')
golden_data = (b'\x5D\x00\x00\x80\xFF'
b'\x61\x00\x00\x00\x00\x00\x00\xF0\xFF'
b'\xCD\x02\x00\x00\x80\xFF'
b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF0\xFF')
golden_message = unittest_pb2.TestAllTypes()
golden_message.ParseFromString(golden_data)
self.assertTrue(IsNegInf(golden_message.optional_float))
@ -158,10 +172,10 @@ class MessageTest(unittest.TestCase):
self.assertEqual(golden_data, golden_message.SerializeToString())
def testNotANumber(self):
golden_data = ('\x5D\x00\x00\xC0\x7F'
'\x61\x00\x00\x00\x00\x00\x00\xF8\x7F'
'\xCD\x02\x00\x00\xC0\x7F'
'\xD1\x02\x00\x00\x00\x00\x00\x00\xF8\x7F')
golden_data = (b'\x5D\x00\x00\xC0\x7F'
b'\x61\x00\x00\x00\x00\x00\x00\xF8\x7F'
b'\xCD\x02\x00\x00\xC0\x7F'
b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF8\x7F')
golden_message = unittest_pb2.TestAllTypes()
golden_message.ParseFromString(golden_data)
self.assertTrue(isnan(golden_message.optional_float))
@ -182,8 +196,8 @@ class MessageTest(unittest.TestCase):
self.assertTrue(isnan(message.repeated_double[0]))
def testPositiveInfinityPacked(self):
golden_data = ('\xA2\x06\x04\x00\x00\x80\x7F'
'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\x7F')
golden_data = (b'\xA2\x06\x04\x00\x00\x80\x7F'
b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\x7F')
golden_message = unittest_pb2.TestPackedTypes()
golden_message.ParseFromString(golden_data)
self.assertTrue(IsPosInf(golden_message.packed_float[0]))
@ -191,8 +205,8 @@ class MessageTest(unittest.TestCase):
self.assertEqual(golden_data, golden_message.SerializeToString())
def testNegativeInfinityPacked(self):
golden_data = ('\xA2\x06\x04\x00\x00\x80\xFF'
'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\xFF')
golden_data = (b'\xA2\x06\x04\x00\x00\x80\xFF'
b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\xFF')
golden_message = unittest_pb2.TestPackedTypes()
golden_message.ParseFromString(golden_data)
self.assertTrue(IsNegInf(golden_message.packed_float[0]))
@ -200,8 +214,8 @@ class MessageTest(unittest.TestCase):
self.assertEqual(golden_data, golden_message.SerializeToString())
def testNotANumberPacked(self):
golden_data = ('\xA2\x06\x04\x00\x00\xC0\x7F'
'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF8\x7F')
golden_data = (b'\xA2\x06\x04\x00\x00\xC0\x7F'
b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF8\x7F')
golden_message = unittest_pb2.TestPackedTypes()
golden_message.ParseFromString(golden_data)
self.assertTrue(isnan(golden_message.packed_float[0]))
@ -303,6 +317,26 @@ class MessageTest(unittest.TestCase):
message.ParseFromString(message.SerializeToString())
self.assertTrue(message.optional_double == -kMostNegExponentOneSigBit)
def testFloatPrinting(self):
message = unittest_pb2.TestAllTypes()
message.optional_float = 2.0
self.assertEqual(str(message), 'optional_float: 2.0\n')
def testHighPrecisionFloatPrinting(self):
message = unittest_pb2.TestAllTypes()
message.optional_double = 0.12345678912345678
if sys.version_info.major >= 3:
self.assertEqual(str(message), 'optional_double: 0.12345678912345678\n')
else:
self.assertEqual(str(message), 'optional_double: 0.123456789123\n')
def testUnknownFieldPrinting(self):
populated = unittest_pb2.TestAllTypes()
test_util.SetAllNonLazyFields(populated)
empty = unittest_pb2.TestEmptyMessage()
empty.ParseFromString(populated.SerializeToString())
self.assertEqual(str(empty), '')
def testSortingRepeatedScalarFieldsDefaultComparator(self):
"""Check some different types with the default comparator."""
message = unittest_pb2.TestAllTypes()
@ -332,13 +366,13 @@ class MessageTest(unittest.TestCase):
self.assertEqual(message.repeated_string[1], 'b')
self.assertEqual(message.repeated_string[2], 'c')
message.repeated_bytes.append('a')
message.repeated_bytes.append('c')
message.repeated_bytes.append('b')
message.repeated_bytes.append(b'a')
message.repeated_bytes.append(b'c')
message.repeated_bytes.append(b'b')
message.repeated_bytes.sort()
self.assertEqual(message.repeated_bytes[0], 'a')
self.assertEqual(message.repeated_bytes[1], 'b')
self.assertEqual(message.repeated_bytes[2], 'c')
self.assertEqual(message.repeated_bytes[0], b'a')
self.assertEqual(message.repeated_bytes[1], b'b')
self.assertEqual(message.repeated_bytes[2], b'c')
def testSortingRepeatedScalarFieldsCustomComparator(self):
"""Check some different types with custom comparator."""
@ -347,7 +381,7 @@ class MessageTest(unittest.TestCase):
message.repeated_int32.append(-3)
message.repeated_int32.append(-2)
message.repeated_int32.append(-1)
message.repeated_int32.sort(lambda x,y: cmp(abs(x), abs(y)))
message.repeated_int32.sort(key=abs)
self.assertEqual(message.repeated_int32[0], -1)
self.assertEqual(message.repeated_int32[1], -2)
self.assertEqual(message.repeated_int32[2], -3)
@ -355,7 +389,7 @@ class MessageTest(unittest.TestCase):
message.repeated_string.append('aaa')
message.repeated_string.append('bb')
message.repeated_string.append('c')
message.repeated_string.sort(lambda x,y: cmp(len(x), len(y)))
message.repeated_string.sort(key=len)
self.assertEqual(message.repeated_string[0], 'c')
self.assertEqual(message.repeated_string[1], 'bb')
self.assertEqual(message.repeated_string[2], 'aaa')
@ -370,7 +404,7 @@ class MessageTest(unittest.TestCase):
message.repeated_nested_message.add().bb = 6
message.repeated_nested_message.add().bb = 5
message.repeated_nested_message.add().bb = 4
message.repeated_nested_message.sort(lambda x,y: cmp(x.bb, y.bb))
message.repeated_nested_message.sort(key=operator.attrgetter('bb'))
self.assertEqual(message.repeated_nested_message[0].bb, 1)
self.assertEqual(message.repeated_nested_message[1].bb, 2)
self.assertEqual(message.repeated_nested_message[2].bb, 3)
@ -396,6 +430,7 @@ class MessageTest(unittest.TestCase):
message.repeated_nested_message.sort(key=get_bb, reverse=True)
self.assertEqual([k.bb for k in message.repeated_nested_message],
[6, 5, 4, 3, 2, 1])
if sys.version_info.major >= 3: return # No cmp sorting in PY3.
message.repeated_nested_message.sort(sort_function=cmp_bb)
self.assertEqual([k.bb for k in message.repeated_nested_message],
[1, 2, 3, 4, 5, 6])
@ -407,7 +442,6 @@ class MessageTest(unittest.TestCase):
"""Check sorting a scalar field using list.sort() arguments."""
message = unittest_pb2.TestAllTypes()
abs_cmp = lambda a, b: cmp(abs(a), abs(b))
message.repeated_int32.append(-3)
message.repeated_int32.append(-2)
message.repeated_int32.append(-1)
@ -415,12 +449,13 @@ class MessageTest(unittest.TestCase):
self.assertEqual(list(message.repeated_int32), [-1, -2, -3])
message.repeated_int32.sort(key=abs, reverse=True)
self.assertEqual(list(message.repeated_int32), [-3, -2, -1])
message.repeated_int32.sort(sort_function=abs_cmp)
self.assertEqual(list(message.repeated_int32), [-1, -2, -3])
message.repeated_int32.sort(cmp=abs_cmp, reverse=True)
self.assertEqual(list(message.repeated_int32), [-3, -2, -1])
if sys.version_info.major < 3: # No cmp sorting in PY3.
abs_cmp = lambda a, b: cmp(abs(a), abs(b))
message.repeated_int32.sort(sort_function=abs_cmp)
self.assertEqual(list(message.repeated_int32), [-1, -2, -3])
message.repeated_int32.sort(cmp=abs_cmp, reverse=True)
self.assertEqual(list(message.repeated_int32), [-3, -2, -1])
len_cmp = lambda a, b: cmp(len(a), len(b))
message.repeated_string.append('aaa')
message.repeated_string.append('bb')
message.repeated_string.append('c')
@ -428,10 +463,47 @@ class MessageTest(unittest.TestCase):
self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa'])
message.repeated_string.sort(key=len, reverse=True)
self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c'])
message.repeated_string.sort(sort_function=len_cmp)
self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa'])
message.repeated_string.sort(cmp=len_cmp, reverse=True)
self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c'])
if sys.version_info.major < 3: # No cmp sorting in PY3.
len_cmp = lambda a, b: cmp(len(a), len(b))
message.repeated_string.sort(sort_function=len_cmp)
self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa'])
message.repeated_string.sort(cmp=len_cmp, reverse=True)
self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c'])
def testRepeatedFieldsComparable(self):
m1 = unittest_pb2.TestAllTypes()
m2 = unittest_pb2.TestAllTypes()
m1.repeated_int32.append(0)
m1.repeated_int32.append(1)
m1.repeated_int32.append(2)
m2.repeated_int32.append(0)
m2.repeated_int32.append(1)
m2.repeated_int32.append(2)
m1.repeated_nested_message.add().bb = 1
m1.repeated_nested_message.add().bb = 2
m1.repeated_nested_message.add().bb = 3
m2.repeated_nested_message.add().bb = 1
m2.repeated_nested_message.add().bb = 2
m2.repeated_nested_message.add().bb = 3
if sys.version_info.major >= 3: return # No cmp() in PY3.
# These comparisons should not raise errors.
_ = m1 < m2
_ = m1.repeated_nested_message < m2.repeated_nested_message
# Make sure cmp always works. If it wasn't defined, these would be
# id() comparisons and would all fail.
self.assertEqual(cmp(m1, m2), 0)
self.assertEqual(cmp(m1.repeated_int32, m2.repeated_int32), 0)
self.assertEqual(cmp(m1.repeated_int32, [0, 1, 2]), 0)
self.assertEqual(cmp(m1.repeated_nested_message,
m2.repeated_nested_message), 0)
with self.assertRaises(TypeError):
# Can't compare repeated composite containers to lists.
cmp(m1.repeated_nested_message, m2.repeated_nested_message[:])
# TODO(anuraag): Implement extensiondict comparison in C++ and then add test
def testParsingMerge(self):
"""Check the merge behavior when a required or optional field appears
@ -482,6 +554,87 @@ class MessageTest(unittest.TestCase):
self.assertEqual(len(parsing_merge.Extensions[
unittest_pb2.TestParsingMerge.repeated_ext]), 3)
def ensureNestedMessageExists(self, msg, attribute):
"""Make sure that a nested message object exists.
As soon as a nested message attribute is accessed, it will be present in the
_fields dict, without being marked as actually being set.
"""
getattr(msg, attribute)
self.assertFalse(msg.HasField(attribute))
def testOneofGetCaseNonexistingField(self):
m = unittest_pb2.TestAllTypes()
self.assertRaises(ValueError, m.WhichOneof, 'no_such_oneof_field')
def testOneofSemantics(self):
m = unittest_pb2.TestAllTypes()
self.assertIs(None, m.WhichOneof('oneof_field'))
m.oneof_uint32 = 11
self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field'))
self.assertTrue(m.HasField('oneof_uint32'))
m.oneof_string = u'foo'
self.assertEqual('oneof_string', m.WhichOneof('oneof_field'))
self.assertFalse(m.HasField('oneof_uint32'))
self.assertTrue(m.HasField('oneof_string'))
m.oneof_nested_message.bb = 11
self.assertEqual('oneof_nested_message', m.WhichOneof('oneof_field'))
self.assertFalse(m.HasField('oneof_string'))
self.assertTrue(m.HasField('oneof_nested_message'))
m.oneof_bytes = b'bb'
self.assertEqual('oneof_bytes', m.WhichOneof('oneof_field'))
self.assertFalse(m.HasField('oneof_nested_message'))
self.assertTrue(m.HasField('oneof_bytes'))
def testOneofCompositeFieldReadAccess(self):
m = unittest_pb2.TestAllTypes()
m.oneof_uint32 = 11
self.ensureNestedMessageExists(m, 'oneof_nested_message')
self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field'))
self.assertEqual(11, m.oneof_uint32)
def testOneofHasField(self):
m = unittest_pb2.TestAllTypes()
self.assertFalse(m.HasField('oneof_field'))
m.oneof_uint32 = 11
self.assertTrue(m.HasField('oneof_field'))
m.oneof_bytes = b'bb'
self.assertTrue(m.HasField('oneof_field'))
m.ClearField('oneof_bytes')
self.assertFalse(m.HasField('oneof_field'))
def testOneofClearField(self):
m = unittest_pb2.TestAllTypes()
m.oneof_uint32 = 11
m.ClearField('oneof_field')
self.assertFalse(m.HasField('oneof_field'))
self.assertFalse(m.HasField('oneof_uint32'))
self.assertIs(None, m.WhichOneof('oneof_field'))
def testOneofClearSetField(self):
m = unittest_pb2.TestAllTypes()
m.oneof_uint32 = 11
m.ClearField('oneof_uint32')
self.assertFalse(m.HasField('oneof_field'))
self.assertFalse(m.HasField('oneof_uint32'))
self.assertIs(None, m.WhichOneof('oneof_field'))
def testOneofClearUnsetField(self):
m = unittest_pb2.TestAllTypes()
m.oneof_uint32 = 11
self.ensureNestedMessageExists(m, 'oneof_nested_message')
m.ClearField('oneof_nested_message')
self.assertEqual(11, m.oneof_uint32)
self.assertTrue(m.HasField('oneof_field'))
self.assertTrue(m.HasField('oneof_uint32'))
self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field'))
def testSortEmptyRepeatedCompositeContainer(self):
"""Exercise a scenario that has led to segfaults in the past.
@ -489,6 +642,35 @@ class MessageTest(unittest.TestCase):
m = unittest_pb2.TestAllTypes()
m.repeated_nested_message.sort()
def testHasFieldOnRepeatedField(self):
"""Using HasField on a repeated field should raise an exception.
"""
m = unittest_pb2.TestAllTypes()
with self.assertRaises(ValueError) as _:
m.HasField('repeated_int32')
class ValidTypeNamesTest(basetest.TestCase):
def assertImportFromName(self, msg, base_name):
# Parse <type 'module.class_name'> to extra 'some.name' as a string.
tp_name = str(type(msg)).split("'")[1]
valid_names = ('Repeated%sContainer' % base_name,
'Repeated%sFieldContainer' % base_name)
self.assertTrue(any(tp_name.endswith(v) for v in valid_names),
'%r does end with any of %r' % (tp_name, valid_names))
parts = tp_name.split('.')
class_name = parts[-1]
module_name = '.'.join(parts[:-1])
__import__(module_name, fromlist=[class_name])
def testTypeNamesCanBeImported(self):
# If import doesn't work, pickling won't work either.
pb = unittest_pb2.TestAllTypes()
self.assertImportFromName(pb.repeated_int32, 'Scalar')
self.assertImportFromName(pb.repeated_nested_message, 'Composite')
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -0,0 +1,50 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package google.protobuf.python.internal;
message TestEnumValues {
enum NestedEnum {
ZERO = 0;
ONE = 1;
}
optional NestedEnum optional_nested_enum = 1;
repeated NestedEnum repeated_nested_enum = 2;
repeated NestedEnum packed_nested_enum = 3 [packed = true];
}
message TestMissingEnumValues {
enum NestedEnum {
TWO = 2;
}
optional NestedEnum optional_nested_enum = 1;
repeated NestedEnum repeated_nested_enum = 2;
repeated NestedEnum packed_nested_enum = 3 [packed = true];
}

@ -28,6 +28,10 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Keep it Python2.5 compatible for GAE.
#
# Copyright 2007 Google Inc. All Rights Reserved.
#
# This code is meant to work on Python 2.4 and above only.
#
# TODO(robinson): Helpers for verbose, common checks like seeing if a
@ -50,11 +54,16 @@ this file*.
__author__ = 'robinson@google.com (Will Robinson)'
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
import copy_reg
import sys
if sys.version_info[0] < 3:
try:
from cStringIO import StringIO as BytesIO
except ImportError:
from StringIO import StringIO as BytesIO
import copy_reg as copyreg
else:
from io import BytesIO
import copyreg
import struct
import weakref
@ -98,8 +107,8 @@ def InitMessage(descriptor, cls):
_AddPropertiesForExtensions(descriptor, cls)
_AddStaticMethods(cls)
_AddMessageMethods(descriptor, cls)
_AddPrivateHelperMethods(cls)
copy_reg.pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
_AddPrivateHelperMethods(descriptor, cls)
copyreg.pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
# Stateless helpers for GeneratedProtocolMessageType below.
@ -176,7 +185,8 @@ def _AddSlots(message_descriptor, dictionary):
'_is_present_in_parent',
'_listener',
'_listener_for_children',
'__weakref__']
'__weakref__',
'_oneofs']
def _IsMessageSetExtension(field):
@ -272,7 +282,7 @@ def _DefaultValueConstructorForField(field):
message._listener_for_children, field.message_type)
return MakeRepeatedMessageDefault
else:
type_checker = type_checkers.GetTypeChecker(field.cpp_type, field.type)
type_checker = type_checkers.GetTypeChecker(field)
def MakeRepeatedScalarDefault(message):
return containers.RepeatedScalarFieldContainer(
message._listener_for_children, type_checker)
@ -301,6 +311,10 @@ def _AddInitMethod(message_descriptor, cls):
self._cached_byte_size = 0
self._cached_byte_size_dirty = len(kwargs) > 0
self._fields = {}
# Contains a mapping from oneof field descriptors to the descriptor
# of the currently set field in that oneof field.
self._oneofs = {}
# _unknown_fields is () when empty for efficiency, and will be turned into
# a list if fields are added.
self._unknown_fields = ()
@ -440,7 +454,7 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
"""
proto_field_name = field.name
property_name = _PropertyName(proto_field_name)
type_checker = type_checkers.GetTypeChecker(field.cpp_type, field.type)
type_checker = type_checkers.GetTypeChecker(field)
default_value = field.default_value
valid_values = set()
@ -450,14 +464,21 @@ def _AddPropertiesForNonRepeatedScalarField(field, cls):
return self._fields.get(field, default_value)
getter.__module__ = None
getter.__doc__ = 'Getter for %s.' % proto_field_name
def setter(self, new_value):
type_checker.CheckValue(new_value)
self._fields[field] = new_value
def field_setter(self, new_value):
# pylint: disable=protected-access
self._fields[field] = type_checker.CheckValue(new_value)
# Check _cached_byte_size_dirty inline to improve performance, since scalar
# setters are called frequently.
if not self._cached_byte_size_dirty:
self._Modified()
if field.containing_oneof is not None:
def setter(self, new_value):
field_setter(self, new_value)
self._UpdateOneofState(field)
else:
setter = field_setter
setter.__module__ = None
setter.__doc__ = 'Setter for %s.' % proto_field_name
@ -493,7 +514,10 @@ def _AddPropertiesForNonRepeatedCompositeField(field, cls):
if field_value is None:
# Construct a new object to represent this field.
field_value = message_type._concrete_class() # use field.message_type?
field_value._SetListener(self._listener_for_children)
field_value._SetListener(
_OneofListener(self, field)
if field.containing_oneof is not None
else self._listener_for_children)
# Atomically check if another thread has preempted us and, if not, swap
# in the new object we just created. If someone has preempted us, we
@ -589,6 +613,9 @@ def _AddHasFieldMethod(message_descriptor, cls):
for field in message_descriptor.fields:
if field.label != _FieldDescriptor.LABEL_REPEATED:
singular_fields[field.name] = field
# Fields inside oneofs are never repeated (enforced by the compiler).
for field in message_descriptor.oneofs:
singular_fields[field.name] = field
def HasField(self, field_name):
try:
@ -597,11 +624,18 @@ def _AddHasFieldMethod(message_descriptor, cls):
raise ValueError(
'Protocol message has no singular "%s" field.' % field_name)
if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
value = self._fields.get(field)
return value is not None and value._is_present_in_parent
if isinstance(field, descriptor_mod.OneofDescriptor):
try:
return HasField(self, self._oneofs[field].name)
except KeyError:
return False
else:
return field in self._fields
if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
value = self._fields.get(field)
return value is not None and value._is_present_in_parent
else:
return field in self._fields
cls.HasField = HasField
@ -611,7 +645,14 @@ def _AddClearFieldMethod(message_descriptor, cls):
try:
field = message_descriptor.fields_by_name[field_name]
except KeyError:
raise ValueError('Protocol message has no "%s" field.' % field_name)
try:
field = message_descriptor.oneofs_by_name[field_name]
if field in self._oneofs:
field = self._oneofs[field]
else:
return
except KeyError:
raise ValueError('Protocol message has no "%s" field.' % field_name)
if field in self._fields:
# Note: If the field is a sub-message, its listener will still point
@ -619,6 +660,9 @@ def _AddClearFieldMethod(message_descriptor, cls):
# will call _Modified() and invalidate our byte size. Big deal.
del self._fields[field]
if self._oneofs.get(field.containing_oneof, None) is field:
del self._oneofs[field.containing_oneof]
# Always call _Modified() -- even if nothing was changed, this is
# a mutating method, and thus calling it should cause the field to become
# present in the parent message.
@ -773,7 +817,7 @@ def _AddSerializePartialToStringMethod(message_descriptor, cls):
"""Helper for _AddMessageMethods()."""
def SerializePartialToString(self):
out = StringIO()
out = BytesIO()
self._InternalSerialize(out.write)
return out.getvalue()
cls.SerializePartialToString = SerializePartialToString
@ -796,7 +840,8 @@ def _AddMergeFromStringMethod(message_descriptor, cls):
# The only reason _InternalParse would return early is if it
# encountered an end-group tag.
raise message_mod.DecodeError('Unexpected end-group tag.')
except IndexError:
except (IndexError, TypeError):
# Now ord(buf[p:p+1]) == ord('') gets TypeError.
raise message_mod.DecodeError('Truncated message.')
except struct.error, e:
raise message_mod.DecodeError(e)
@ -857,7 +902,7 @@ def _AddIsInitializedMethod(message_descriptor, cls):
errors.extend(self.FindInitializationErrors())
return False
for field, value in self._fields.iteritems():
for field, value in list(self._fields.items()): # dict can change size!
if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
if field.label == _FieldDescriptor.LABEL_REPEATED:
for element in value:
@ -953,6 +998,24 @@ def _AddMergeFromMethod(cls):
cls.MergeFrom = MergeFrom
def _AddWhichOneofMethod(message_descriptor, cls):
def WhichOneof(self, oneof_name):
"""Returns the name of the currently set field inside a oneof, or None."""
try:
field = message_descriptor.oneofs_by_name[oneof_name]
except KeyError:
raise ValueError(
'Protocol message has no oneof "%s" field.' % oneof_name)
nested_field = self._oneofs.get(field, None)
if nested_field is not None and self.HasField(nested_field.name):
return nested_field.name
else:
return None
cls.WhichOneof = WhichOneof
def _AddMessageMethods(message_descriptor, cls):
"""Adds implementations of all Message methods to cls."""
_AddListFieldsMethod(message_descriptor, cls)
@ -972,9 +1035,9 @@ def _AddMessageMethods(message_descriptor, cls):
_AddMergeFromStringMethod(message_descriptor, cls)
_AddIsInitializedMethod(message_descriptor, cls)
_AddMergeFromMethod(cls)
_AddWhichOneofMethod(message_descriptor, cls)
def _AddPrivateHelperMethods(cls):
def _AddPrivateHelperMethods(message_descriptor, cls):
"""Adds implementation of private helper methods to cls."""
def Modified(self):
@ -992,8 +1055,20 @@ def _AddPrivateHelperMethods(cls):
self._is_present_in_parent = True
self._listener.Modified()
def _UpdateOneofState(self, field):
"""Sets field as the active field in its containing oneof.
Will also delete currently active field in the oneof, if it is different
from the argument. Does not mark the message as modified.
"""
other_field = self._oneofs.setdefault(field.containing_oneof, field)
if other_field is not field:
del self._fields[other_field]
self._oneofs[field.containing_oneof] = field
cls._Modified = Modified
cls.SetInParent = Modified
cls._UpdateOneofState = _UpdateOneofState
class _Listener(object):
@ -1042,6 +1117,27 @@ class _Listener(object):
pass
class _OneofListener(_Listener):
"""Special listener implementation for setting composite oneof fields."""
def __init__(self, parent_message, field):
"""Args:
parent_message: The message whose _Modified() method we should call when
we receive Modified() messages.
field: The descriptor of the field being set in the parent message.
"""
super(_OneofListener, self).__init__(parent_message)
self._field = field
def Modified(self):
"""Also updates the state of the containing oneof in the parent message."""
try:
self._parent_message_weakref._UpdateOneofState(self._field)
super(_OneofListener, self).Modified()
except ReferenceError:
pass
# TODO(robinson): Move elsewhere? This file is getting pretty ridiculous...
# TODO(robinson): Unify error handling of "unknown extension" crap.
# TODO(robinson): Support iteritems()-style iteration over all
@ -1133,9 +1229,10 @@ class _ExtensionDict(object):
# It's slightly wasteful to lookup the type checker each time,
# but we expect this to be a vanishingly uncommon case anyway.
type_checker = type_checkers.GetTypeChecker(
extension_handle.cpp_type, extension_handle.type)
type_checker.CheckValue(value)
self._extended_message._fields[extension_handle] = value
extension_handle)
# pylint: disable=protected-access
self._extended_message._fields[extension_handle] = (
type_checker.CheckValue(value))
self._extended_message._Modified()
def _FindExtensionByName(self, name):

@ -37,17 +37,19 @@ __author__ = 'jasonh@google.com (Jason Hsueh)'
import os
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION'] = '2'
import unittest
from google.apputils import basetest
from google.protobuf.internal import api_implementation
from google.protobuf.internal import more_extensions_dynamic_pb2
from google.protobuf.internal import more_extensions_pb2
from google.protobuf.internal.reflection_test import *
class ReflectionCppTest(unittest.TestCase):
class ReflectionCppTest(basetest.TestCase):
def testImplementationSetting(self):
self.assertEqual('cpp', api_implementation.Type())
self.assertEqual(2, api_implementation.Version())
def testExtensionOfGeneratedTypeInDynamicFile(self):
"""Tests that a file built dynamically can extend a generated C++ type.
@ -87,5 +89,6 @@ class ReflectionCppTest(unittest.TestCase):
pb2.Extensions[more_extensions_dynamic_pb2.dynamic_message_extension].a)
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -37,11 +37,12 @@ pure-Python protocol compiler.
__author__ = 'robinson@google.com (Will Robinson)'
import copy
import gc
import operator
import struct
import unittest
from google.apputils import basetest
from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2
@ -49,6 +50,7 @@ from google.protobuf import descriptor_pb2
from google.protobuf import descriptor
from google.protobuf import message
from google.protobuf import reflection
from google.protobuf import text_format
from google.protobuf.internal import api_implementation
from google.protobuf.internal import more_extensions_pb2
from google.protobuf.internal import more_messages_pb2
@ -102,7 +104,7 @@ class _MiniDecoder(object):
return self._pos == len(self._bytes)
class ReflectionTest(unittest.TestCase):
class ReflectionTest(basetest.TestCase):
def assertListsEqual(self, values, others):
self.assertEqual(len(values), len(others))
@ -533,7 +535,7 @@ class ReflectionTest(unittest.TestCase):
self.assertEqual(0.0, proto.optional_double)
self.assertEqual(False, proto.optional_bool)
self.assertEqual('', proto.optional_string)
self.assertEqual('', proto.optional_bytes)
self.assertEqual(b'', proto.optional_bytes)
self.assertEqual(41, proto.default_int32)
self.assertEqual(42, proto.default_int64)
@ -549,7 +551,7 @@ class ReflectionTest(unittest.TestCase):
self.assertEqual(52e3, proto.default_double)
self.assertEqual(True, proto.default_bool)
self.assertEqual('hello', proto.default_string)
self.assertEqual('world', proto.default_bytes)
self.assertEqual(b'world', proto.default_bytes)
self.assertEqual(unittest_pb2.TestAllTypes.BAR, proto.default_nested_enum)
self.assertEqual(unittest_pb2.FOREIGN_BAR, proto.default_foreign_enum)
self.assertEqual(unittest_import_pb2.IMPORT_BAR,
@ -566,6 +568,17 @@ class ReflectionTest(unittest.TestCase):
proto = unittest_pb2.TestAllTypes()
self.assertRaises(ValueError, proto.ClearField, 'nonexistent_field')
def testClearRemovesChildren(self):
# Make sure there aren't any implementation bugs that are only partially
# clearing the message (which can happen in the more complex C++
# implementation which has parallel message lists).
proto = unittest_pb2.TestRequiredForeign()
for i in range(10):
proto.repeated_message.add()
proto2 = unittest_pb2.TestRequiredForeign()
proto.CopyFrom(proto2)
self.assertRaises(IndexError, lambda: proto.repeated_message[5])
def testDisallowedAssignments(self):
# It's illegal to assign values directly to repeated fields
# or to nonrepeated composite fields. Ensure that this fails.
@ -594,6 +607,30 @@ class ReflectionTest(unittest.TestCase):
self.assertRaises(TypeError, setattr, proto, 'optional_string', 10)
self.assertRaises(TypeError, setattr, proto, 'optional_bytes', 10)
def testIntegerTypes(self):
def TestGetAndDeserialize(field_name, value, expected_type):
proto = unittest_pb2.TestAllTypes()
setattr(proto, field_name, value)
self.assertTrue(isinstance(getattr(proto, field_name), expected_type))
proto2 = unittest_pb2.TestAllTypes()
proto2.ParseFromString(proto.SerializeToString())
self.assertTrue(isinstance(getattr(proto2, field_name), expected_type))
TestGetAndDeserialize('optional_int32', 1, int)
TestGetAndDeserialize('optional_int32', 1 << 30, int)
TestGetAndDeserialize('optional_uint32', 1 << 30, int)
if struct.calcsize('L') == 4:
# Python only has signed ints, so 32-bit python can't fit an uint32
# in an int.
TestGetAndDeserialize('optional_uint32', 1 << 31, long)
else:
# 64-bit python can fit uint32 inside an int
TestGetAndDeserialize('optional_uint32', 1 << 31, int)
TestGetAndDeserialize('optional_int64', 1 << 30, long)
TestGetAndDeserialize('optional_int64', 1 << 60, long)
TestGetAndDeserialize('optional_uint64', 1 << 30, long)
TestGetAndDeserialize('optional_uint64', 1 << 60, long)
def testSingleScalarBoundsChecking(self):
def TestMinAndMaxIntegers(field_name, expected_min, expected_max):
pb = unittest_pb2.TestAllTypes()
@ -613,29 +650,6 @@ class ReflectionTest(unittest.TestCase):
pb.optional_nested_enum = 1
self.assertEqual(1, pb.optional_nested_enum)
# Invalid enum values.
pb.optional_nested_enum = 0
self.assertEqual(0, pb.optional_nested_enum)
bytes_size_before = pb.ByteSize()
pb.optional_nested_enum = 4
self.assertEqual(4, pb.optional_nested_enum)
pb.optional_nested_enum = 0
self.assertEqual(0, pb.optional_nested_enum)
# Make sure that setting the same enum field doesn't just add unknown
# fields (but overwrites them).
self.assertEqual(bytes_size_before, pb.ByteSize())
# Is the invalid value preserved after serialization?
serialized = pb.SerializeToString()
pb2 = unittest_pb2.TestAllTypes()
pb2.ParseFromString(serialized)
self.assertEqual(0, pb2.optional_nested_enum)
self.assertEqual(pb, pb2)
def testRepeatedScalarTypeSafety(self):
proto = unittest_pb2.TestAllTypes()
self.assertRaises(TypeError, proto.repeated_int32.append, 1.1)
@ -749,9 +763,9 @@ class ReflectionTest(unittest.TestCase):
unittest_pb2.ForeignEnum.items())
proto = unittest_pb2.TestAllTypes()
self.assertEqual(['FOO', 'BAR', 'BAZ'], proto.NestedEnum.keys())
self.assertEqual([1, 2, 3], proto.NestedEnum.values())
self.assertEqual([('FOO', 1), ('BAR', 2), ('BAZ', 3)],
self.assertEqual(['FOO', 'BAR', 'BAZ', 'NEG'], proto.NestedEnum.keys())
self.assertEqual([1, 2, 3, -1], proto.NestedEnum.values())
self.assertEqual([('FOO', 1), ('BAR', 2), ('BAZ', 3), ('NEG', -1)],
proto.NestedEnum.items())
def testRepeatedScalars(self):
@ -1155,6 +1169,14 @@ class ReflectionTest(unittest.TestCase):
self.assertTrue(required is not extendee_proto.Extensions[extension])
self.assertTrue(not extendee_proto.HasExtension(extension))
def testRegisteredExtensions(self):
self.assertTrue('protobuf_unittest.optional_int32_extension' in
unittest_pb2.TestAllExtensions._extensions_by_name)
self.assertTrue(1 in unittest_pb2.TestAllExtensions._extensions_by_number)
# Make sure extensions haven't been registered into types that shouldn't
# have any.
self.assertEquals(0, len(unittest_pb2.TestAllTypes._extensions_by_name))
# If message A directly contains message B, and
# a.HasField('b') is currently False, then mutating any
# extension in B should change a.HasField('b') to True
@ -1451,6 +1473,19 @@ class ReflectionTest(unittest.TestCase):
proto2 = unittest_pb2.TestAllExtensions()
self.assertRaises(TypeError, proto1.CopyFrom, proto2)
def testDeepCopy(self):
proto1 = unittest_pb2.TestAllTypes()
proto1.optional_int32 = 1
proto2 = copy.deepcopy(proto1)
self.assertEqual(1, proto2.optional_int32)
proto1.repeated_int32.append(2)
proto1.repeated_int32.append(3)
container = copy.deepcopy(proto1.repeated_int32)
self.assertEqual([2, 3], container)
# TODO(anuraag): Implement deepcopy for repeated composite / extension dict
def testClear(self):
proto = unittest_pb2.TestAllTypes()
# C++ implementation does not support lazy fields right now so leave it
@ -1496,11 +1531,23 @@ class ReflectionTest(unittest.TestCase):
self.assertEqual(6, foreign.c)
nested.bb = 15
foreign.c = 16
self.assertTrue(not proto.HasField('optional_nested_message'))
self.assertFalse(proto.HasField('optional_nested_message'))
self.assertEqual(0, proto.optional_nested_message.bb)
self.assertTrue(not proto.HasField('optional_foreign_message'))
self.assertFalse(proto.HasField('optional_foreign_message'))
self.assertEqual(0, proto.optional_foreign_message.c)
def testOneOf(self):
proto = unittest_pb2.TestAllTypes()
proto.oneof_uint32 = 10
proto.oneof_nested_message.bb = 11
self.assertEqual(11, proto.oneof_nested_message.bb)
self.assertFalse(proto.HasField('oneof_uint32'))
nested = proto.oneof_nested_message
proto.oneof_string = 'abc'
self.assertEqual('abc', proto.oneof_string)
self.assertEqual(11, nested.bb)
self.assertFalse(proto.HasField('oneof_nested_message'))
def assertInitialized(self, proto):
self.assertTrue(proto.IsInitialized())
# Neither method should raise an exception.
@ -1571,6 +1618,40 @@ class ReflectionTest(unittest.TestCase):
self.assertFalse(proto.IsInitialized(errors))
self.assertEqual(errors, ['a', 'b', 'c'])
@basetest.unittest.skipIf(
api_implementation.Type() != 'cpp' or api_implementation.Version() != 2,
'Errors are only available from the most recent C++ implementation.')
def testFileDescriptorErrors(self):
file_name = 'test_file_descriptor_errors.proto'
package_name = 'test_file_descriptor_errors.proto'
file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
file_descriptor_proto.name = file_name
file_descriptor_proto.package = package_name
m1 = file_descriptor_proto.message_type.add()
m1.name = 'msg1'
# Compiles the proto into the C++ descriptor pool
descriptor.FileDescriptor(
file_name,
package_name,
serialized_pb=file_descriptor_proto.SerializeToString())
# Add a FileDescriptorProto that has duplicate symbols
another_file_name = 'another_test_file_descriptor_errors.proto'
file_descriptor_proto.name = another_file_name
m2 = file_descriptor_proto.message_type.add()
m2.name = 'msg2'
with self.assertRaises(TypeError) as cm:
descriptor.FileDescriptor(
another_file_name,
package_name,
serialized_pb=file_descriptor_proto.SerializeToString())
self.assertTrue(hasattr(cm, 'exception'), '%s not raised' %
getattr(cm.expected, '__name__', cm.expected))
self.assertIn('test_file_descriptor_errors.proto', str(cm.exception))
# Error message will say something about this definition being a
# duplicate, though we don't check the message exactly to avoid a
# dependency on the C++ logging code.
self.assertIn('test_file_descriptor_errors.msg1', str(cm.exception))
def testStringUTF8Encoding(self):
proto = unittest_pb2.TestAllTypes()
@ -1588,17 +1669,15 @@ class ReflectionTest(unittest.TestCase):
proto.optional_string = str('Testing')
self.assertEqual(proto.optional_string, unicode('Testing'))
if api_implementation.Type() == 'python':
# Values of type 'str' are also accepted as long as they can be
# encoded in UTF-8.
self.assertEqual(type(proto.optional_string), str)
# Try to assign a 'str' value which contains bytes that aren't 7-bit ASCII.
self.assertRaises(ValueError,
setattr, proto, 'optional_string', str('a\x80a'))
# Assign a 'str' object which contains a UTF-8 encoded string.
self.assertRaises(ValueError,
setattr, proto, 'optional_string', 'Тест')
setattr, proto, 'optional_string', b'a\x80a')
if str is bytes: # PY2
# Assign a 'str' object which contains a UTF-8 encoded string.
self.assertRaises(ValueError,
setattr, proto, 'optional_string', 'Тест')
else:
proto.optional_string = 'Тест'
# No exception thrown.
proto.optional_string = 'abc'
@ -1621,7 +1700,8 @@ class ReflectionTest(unittest.TestCase):
self.assertEqual(proto.ByteSize(), len(serialized))
raw = unittest_mset_pb2.RawMessageSet()
raw.MergeFromString(serialized)
bytes_read = raw.MergeFromString(serialized)
self.assertEqual(len(serialized), bytes_read)
message2 = unittest_mset_pb2.TestMessageSetExtension2()
@ -1632,7 +1712,8 @@ class ReflectionTest(unittest.TestCase):
# Check the actual bytes on the wire.
self.assertTrue(
raw.item[0].message.endswith(test_utf8_bytes))
message2.MergeFromString(raw.item[0].message)
bytes_read = message2.MergeFromString(raw.item[0].message)
self.assertEqual(len(raw.item[0].message), bytes_read)
self.assertEqual(type(message2.str), unicode)
self.assertEqual(message2.str, test_utf8)
@ -1643,17 +1724,22 @@ class ReflectionTest(unittest.TestCase):
# MergeFromString and thus has no way to throw the exception.
#
# The pure Python API always returns objects of type 'unicode' (UTF-8
# encoded), or 'str' (in 7 bit ASCII).
bytes = raw.item[0].message.replace(
test_utf8_bytes, len(test_utf8_bytes) * '\xff')
# encoded), or 'bytes' (in 7 bit ASCII).
badbytes = raw.item[0].message.replace(
test_utf8_bytes, len(test_utf8_bytes) * b'\xff')
unicode_decode_failed = False
try:
message2.MergeFromString(bytes)
except UnicodeDecodeError as e:
message2.MergeFromString(badbytes)
except UnicodeDecodeError:
unicode_decode_failed = True
string_field = message2.str
self.assertTrue(unicode_decode_failed or type(string_field) == str)
self.assertTrue(unicode_decode_failed or type(string_field) is bytes)
def testBytesInTextFormat(self):
proto = unittest_pb2.TestAllTypes(optional_bytes=b'\x00\x7f\x80\xff')
self.assertEqual(u'optional_bytes: "\\000\\177\\200\\377"\n',
unicode(proto))
def testEmptyNestedMessage(self):
proto = unittest_pb2.TestAllTypes()
@ -1667,16 +1753,19 @@ class ReflectionTest(unittest.TestCase):
self.assertTrue(proto.HasField('optional_nested_message'))
proto = unittest_pb2.TestAllTypes()
proto.optional_nested_message.MergeFromString('')
bytes_read = proto.optional_nested_message.MergeFromString(b'')
self.assertEqual(0, bytes_read)
self.assertTrue(proto.HasField('optional_nested_message'))
proto = unittest_pb2.TestAllTypes()
proto.optional_nested_message.ParseFromString('')
proto.optional_nested_message.ParseFromString(b'')
self.assertTrue(proto.HasField('optional_nested_message'))
serialized = proto.SerializeToString()
proto2 = unittest_pb2.TestAllTypes()
proto2.MergeFromString(serialized)
self.assertEqual(
len(serialized),
proto2.MergeFromString(serialized))
self.assertTrue(proto2.HasField('optional_nested_message'))
def testSetInParent(self):
@ -1690,7 +1779,7 @@ class ReflectionTest(unittest.TestCase):
# into separate TestCase classes.
class TestAllTypesEqualityTest(unittest.TestCase):
class TestAllTypesEqualityTest(basetest.TestCase):
def setUp(self):
self.first_proto = unittest_pb2.TestAllTypes()
@ -1706,7 +1795,7 @@ class TestAllTypesEqualityTest(unittest.TestCase):
self.assertEqual(self.first_proto, self.second_proto)
class FullProtosEqualityTest(unittest.TestCase):
class FullProtosEqualityTest(basetest.TestCase):
"""Equality tests using completely-full protos as a starting point."""
@ -1792,7 +1881,7 @@ class FullProtosEqualityTest(unittest.TestCase):
self.assertEqual(self.first_proto, self.second_proto)
class ExtensionEqualityTest(unittest.TestCase):
class ExtensionEqualityTest(basetest.TestCase):
def testExtensionEquality(self):
first_proto = unittest_pb2.TestAllExtensions()
@ -1825,7 +1914,7 @@ class ExtensionEqualityTest(unittest.TestCase):
self.assertEqual(first_proto, second_proto)
class MutualRecursionEqualityTest(unittest.TestCase):
class MutualRecursionEqualityTest(basetest.TestCase):
def testEqualityWithMutualRecursion(self):
first_proto = unittest_pb2.TestMutualRecursionA()
@ -1837,7 +1926,7 @@ class MutualRecursionEqualityTest(unittest.TestCase):
self.assertEqual(first_proto, second_proto)
class ByteSizeTest(unittest.TestCase):
class ByteSizeTest(basetest.TestCase):
def setUp(self):
self.proto = unittest_pb2.TestAllTypes()
@ -2133,14 +2222,16 @@ class ByteSizeTest(unittest.TestCase):
# * Handling of empty submessages (with and without "has"
# bits set).
class SerializationTest(unittest.TestCase):
class SerializationTest(basetest.TestCase):
def testSerializeEmtpyMessage(self):
first_proto = unittest_pb2.TestAllTypes()
second_proto = unittest_pb2.TestAllTypes()
serialized = first_proto.SerializeToString()
self.assertEqual(first_proto.ByteSize(), len(serialized))
second_proto.MergeFromString(serialized)
self.assertEqual(
len(serialized),
second_proto.MergeFromString(serialized))
self.assertEqual(first_proto, second_proto)
def testSerializeAllFields(self):
@ -2149,7 +2240,9 @@ class SerializationTest(unittest.TestCase):
test_util.SetAllFields(first_proto)
serialized = first_proto.SerializeToString()
self.assertEqual(first_proto.ByteSize(), len(serialized))
second_proto.MergeFromString(serialized)
self.assertEqual(
len(serialized),
second_proto.MergeFromString(serialized))
self.assertEqual(first_proto, second_proto)
def testSerializeAllExtensions(self):
@ -2157,7 +2250,19 @@ class SerializationTest(unittest.TestCase):
second_proto = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(first_proto)
serialized = first_proto.SerializeToString()
second_proto.MergeFromString(serialized)
self.assertEqual(
len(serialized),
second_proto.MergeFromString(serialized))
self.assertEqual(first_proto, second_proto)
def testSerializeWithOptionalGroup(self):
first_proto = unittest_pb2.TestAllTypes()
second_proto = unittest_pb2.TestAllTypes()
first_proto.optionalgroup.a = 242
serialized = first_proto.SerializeToString()
self.assertEqual(
len(serialized),
second_proto.MergeFromString(serialized))
self.assertEqual(first_proto, second_proto)
def testSerializeNegativeValues(self):
@ -2249,7 +2354,9 @@ class SerializationTest(unittest.TestCase):
second_proto.optional_int32 = 100
second_proto.optional_nested_message.bb = 999
second_proto.MergeFromString(serialized)
bytes_parsed = second_proto.MergeFromString(serialized)
self.assertEqual(len(serialized), bytes_parsed)
# Ensure that we append to repeated fields.
self.assertEqual(['baz', 'foobar'], list(second_proto.repeated_string))
# Ensure that we overwrite nonrepeatd scalars.
@ -2274,20 +2381,28 @@ class SerializationTest(unittest.TestCase):
raw = unittest_mset_pb2.RawMessageSet()
self.assertEqual(False,
raw.DESCRIPTOR.GetOptions().message_set_wire_format)
raw.MergeFromString(serialized)
self.assertEqual(
len(serialized),
raw.MergeFromString(serialized))
self.assertEqual(2, len(raw.item))
message1 = unittest_mset_pb2.TestMessageSetExtension1()
message1.MergeFromString(raw.item[0].message)
self.assertEqual(
len(raw.item[0].message),
message1.MergeFromString(raw.item[0].message))
self.assertEqual(123, message1.i)
message2 = unittest_mset_pb2.TestMessageSetExtension2()
message2.MergeFromString(raw.item[1].message)
self.assertEqual(
len(raw.item[1].message),
message2.MergeFromString(raw.item[1].message))
self.assertEqual('foo', message2.str)
# Deserialize using the MessageSet wire format.
proto2 = unittest_mset_pb2.TestMessageSet()
proto2.MergeFromString(serialized)
self.assertEqual(
len(serialized),
proto2.MergeFromString(serialized))
self.assertEqual(123, proto2.Extensions[extension1].i)
self.assertEqual('foo', proto2.Extensions[extension2].str)
@ -2327,7 +2442,9 @@ class SerializationTest(unittest.TestCase):
# Parse message using the message set wire format.
proto = unittest_mset_pb2.TestMessageSet()
proto.MergeFromString(serialized)
self.assertEqual(
len(serialized),
proto.MergeFromString(serialized))
# Check that the message parsed well.
extension_message1 = unittest_mset_pb2.TestMessageSetExtension1
@ -2345,7 +2462,9 @@ class SerializationTest(unittest.TestCase):
proto2 = unittest_pb2.TestEmptyMessage()
# Parsing this message should succeed.
proto2.MergeFromString(serialized)
self.assertEqual(
len(serialized),
proto2.MergeFromString(serialized))
# Now test with a int64 field set.
proto = unittest_pb2.TestAllTypes()
@ -2355,7 +2474,9 @@ class SerializationTest(unittest.TestCase):
# unknown.
proto2 = unittest_pb2.TestEmptyMessage()
# Parsing this message should succeed.
proto2.MergeFromString(serialized)
self.assertEqual(
len(serialized),
proto2.MergeFromString(serialized))
def _CheckRaises(self, exc_class, callable_obj, exception):
"""This method checks if the excpetion type and message are as expected."""
@ -2406,11 +2527,15 @@ class SerializationTest(unittest.TestCase):
partial = proto.SerializePartialToString()
proto2 = unittest_pb2.TestRequired()
proto2.MergeFromString(serialized)
self.assertEqual(
len(serialized),
proto2.MergeFromString(serialized))
self.assertEqual(1, proto2.a)
self.assertEqual(2, proto2.b)
self.assertEqual(3, proto2.c)
proto2.ParseFromString(partial)
self.assertEqual(
len(partial),
proto2.MergeFromString(partial))
self.assertEqual(1, proto2.a)
self.assertEqual(2, proto2.b)
self.assertEqual(3, proto2.c)
@ -2478,7 +2603,9 @@ class SerializationTest(unittest.TestCase):
second_proto.packed_double.extend([1.0, 2.0])
second_proto.packed_sint32.append(4)
second_proto.MergeFromString(serialized)
self.assertEqual(
len(serialized),
second_proto.MergeFromString(serialized))
self.assertEqual([3, 1, 2], second_proto.packed_int32)
self.assertEqual([1.0, 2.0, 3.0], second_proto.packed_double)
self.assertEqual([4], second_proto.packed_sint32)
@ -2511,7 +2638,10 @@ class SerializationTest(unittest.TestCase):
unpacked = unittest_pb2.TestUnpackedTypes()
test_util.SetAllUnpackedFields(unpacked)
packed = unittest_pb2.TestPackedTypes()
packed.MergeFromString(unpacked.SerializeToString())
serialized = unpacked.SerializeToString()
self.assertEqual(
len(serialized),
packed.MergeFromString(serialized))
expected = unittest_pb2.TestPackedTypes()
test_util.SetAllPackedFields(expected)
self.assertEqual(expected, packed)
@ -2520,7 +2650,10 @@ class SerializationTest(unittest.TestCase):
packed = unittest_pb2.TestPackedTypes()
test_util.SetAllPackedFields(packed)
unpacked = unittest_pb2.TestUnpackedTypes()
unpacked.MergeFromString(packed.SerializeToString())
serialized = packed.SerializeToString()
self.assertEqual(
len(serialized),
unpacked.MergeFromString(serialized))
expected = unittest_pb2.TestUnpackedTypes()
test_util.SetAllUnpackedFields(expected)
self.assertEqual(expected, unpacked)
@ -2572,7 +2705,7 @@ class SerializationTest(unittest.TestCase):
optional_int32=1,
optional_string='foo',
optional_bool=True,
optional_bytes='bar',
optional_bytes=b'bar',
optional_nested_message=unittest_pb2.TestAllTypes.NestedMessage(bb=1),
optional_foreign_message=unittest_pb2.ForeignMessage(c=1),
optional_nested_enum=unittest_pb2.TestAllTypes.FOO,
@ -2590,7 +2723,7 @@ class SerializationTest(unittest.TestCase):
self.assertEqual(1, proto.optional_int32)
self.assertEqual('foo', proto.optional_string)
self.assertEqual(True, proto.optional_bool)
self.assertEqual('bar', proto.optional_bytes)
self.assertEqual(b'bar', proto.optional_bytes)
self.assertEqual(1, proto.optional_nested_message.bb)
self.assertEqual(1, proto.optional_foreign_message.c)
self.assertEqual(unittest_pb2.TestAllTypes.FOO,
@ -2640,7 +2773,7 @@ class SerializationTest(unittest.TestCase):
self.assertEqual(3, proto.repeated_int32[2])
class OptionsTest(unittest.TestCase):
class OptionsTest(basetest.TestCase):
def testMessageOptions(self):
proto = unittest_mset_pb2.TestMessageSet()
@ -2667,5 +2800,135 @@ class OptionsTest(unittest.TestCase):
class ClassAPITest(basetest.TestCase):
def testMakeClassWithNestedDescriptor(self):
leaf_desc = descriptor.Descriptor('leaf', 'package.parent.child.leaf', '',
containing_type=None, fields=[],
nested_types=[], enum_types=[],
extensions=[])
child_desc = descriptor.Descriptor('child', 'package.parent.child', '',
containing_type=None, fields=[],
nested_types=[leaf_desc], enum_types=[],
extensions=[])
sibling_desc = descriptor.Descriptor('sibling', 'package.parent.sibling',
'', containing_type=None, fields=[],
nested_types=[], enum_types=[],
extensions=[])
parent_desc = descriptor.Descriptor('parent', 'package.parent', '',
containing_type=None, fields=[],
nested_types=[child_desc, sibling_desc],
enum_types=[], extensions=[])
message_class = reflection.MakeClass(parent_desc)
self.assertIn('child', message_class.__dict__)
self.assertIn('sibling', message_class.__dict__)
self.assertIn('leaf', message_class.child.__dict__)
def _GetSerializedFileDescriptor(self, name):
"""Get a serialized representation of a test FileDescriptorProto.
Args:
name: All calls to this must use a unique message name, to avoid
collisions in the cpp descriptor pool.
Returns:
A string containing the serialized form of a test FileDescriptorProto.
"""
file_descriptor_str = (
'message_type {'
' name: "' + name + '"'
' field {'
' name: "flat"'
' number: 1'
' label: LABEL_REPEATED'
' type: TYPE_UINT32'
' }'
' field {'
' name: "bar"'
' number: 2'
' label: LABEL_OPTIONAL'
' type: TYPE_MESSAGE'
' type_name: "Bar"'
' }'
' nested_type {'
' name: "Bar"'
' field {'
' name: "baz"'
' number: 3'
' label: LABEL_OPTIONAL'
' type: TYPE_MESSAGE'
' type_name: "Baz"'
' }'
' nested_type {'
' name: "Baz"'
' enum_type {'
' name: "deep_enum"'
' value {'
' name: "VALUE_A"'
' number: 0'
' }'
' }'
' field {'
' name: "deep"'
' number: 4'
' label: LABEL_OPTIONAL'
' type: TYPE_UINT32'
' }'
' }'
' }'
'}')
file_descriptor = descriptor_pb2.FileDescriptorProto()
text_format.Merge(file_descriptor_str, file_descriptor)
return file_descriptor.SerializeToString()
def testParsingFlatClassWithExplicitClassDeclaration(self):
"""Test that the generated class can parse a flat message."""
file_descriptor = descriptor_pb2.FileDescriptorProto()
file_descriptor.ParseFromString(self._GetSerializedFileDescriptor('A'))
msg_descriptor = descriptor.MakeDescriptor(
file_descriptor.message_type[0])
class MessageClass(message.Message):
__metaclass__ = reflection.GeneratedProtocolMessageType
DESCRIPTOR = msg_descriptor
msg = MessageClass()
msg_str = (
'flat: 0 '
'flat: 1 '
'flat: 2 ')
text_format.Merge(msg_str, msg)
self.assertEqual(msg.flat, [0, 1, 2])
def testParsingFlatClass(self):
"""Test that the generated class can parse a flat message."""
file_descriptor = descriptor_pb2.FileDescriptorProto()
file_descriptor.ParseFromString(self._GetSerializedFileDescriptor('B'))
msg_descriptor = descriptor.MakeDescriptor(
file_descriptor.message_type[0])
msg_class = reflection.MakeClass(msg_descriptor)
msg = msg_class()
msg_str = (
'flat: 0 '
'flat: 1 '
'flat: 2 ')
text_format.Merge(msg_str, msg)
self.assertEqual(msg.flat, [0, 1, 2])
def testParsingNestedClass(self):
"""Test that the generated class can parse a nested message."""
file_descriptor = descriptor_pb2.FileDescriptorProto()
file_descriptor.ParseFromString(self._GetSerializedFileDescriptor('C'))
msg_descriptor = descriptor.MakeDescriptor(
file_descriptor.message_type[0])
msg_class = reflection.MakeClass(msg_descriptor)
msg = msg_class()
msg_str = (
'bar {'
' baz {'
' deep: 4'
' }'
'}')
text_format.Merge(msg_str, msg)
self.assertEqual(msg.bar.baz.deep, 4)
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -34,13 +34,13 @@
__author__ = 'petar@google.com (Petar Petrov)'
import unittest
from google.apputils import basetest
from google.protobuf import unittest_pb2
from google.protobuf import service_reflection
from google.protobuf import service
class FooUnitTest(unittest.TestCase):
class FooUnitTest(basetest.TestCase):
def testService(self):
class MockRpcChannel(service.RpcChannel):
@ -133,4 +133,4 @@ class FooUnitTest(unittest.TestCase):
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -0,0 +1,120 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.symbol_database."""
from google.apputils import basetest
from google.protobuf import unittest_pb2
from google.protobuf import symbol_database
class SymbolDatabaseTest(basetest.TestCase):
def _Database(self):
db = symbol_database.SymbolDatabase()
# Register representative types from unittest_pb2.
db.RegisterFileDescriptor(unittest_pb2.DESCRIPTOR)
db.RegisterMessage(unittest_pb2.TestAllTypes)
db.RegisterMessage(unittest_pb2.TestAllTypes.NestedMessage)
db.RegisterMessage(unittest_pb2.TestAllTypes.OptionalGroup)
db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup)
db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR)
db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR)
return db
def testGetPrototype(self):
instance = self._Database().GetPrototype(
unittest_pb2.TestAllTypes.DESCRIPTOR)
self.assertTrue(instance is unittest_pb2.TestAllTypes)
def testGetMessages(self):
messages = self._Database().GetMessages(
['google/protobuf/unittest.proto'])
self.assertTrue(
unittest_pb2.TestAllTypes is
messages['protobuf_unittest.TestAllTypes'])
def testGetSymbol(self):
self.assertEquals(
unittest_pb2.TestAllTypes, self._Database().GetSymbol(
'protobuf_unittest.TestAllTypes'))
self.assertEquals(
unittest_pb2.TestAllTypes.NestedMessage, self._Database().GetSymbol(
'protobuf_unittest.TestAllTypes.NestedMessage'))
self.assertEquals(
unittest_pb2.TestAllTypes.OptionalGroup, self._Database().GetSymbol(
'protobuf_unittest.TestAllTypes.OptionalGroup'))
self.assertEquals(
unittest_pb2.TestAllTypes.RepeatedGroup, self._Database().GetSymbol(
'protobuf_unittest.TestAllTypes.RepeatedGroup'))
def testEnums(self):
# Check registration of types in the pool.
self.assertEquals(
'protobuf_unittest.ForeignEnum',
self._Database().pool.FindEnumTypeByName(
'protobuf_unittest.ForeignEnum').full_name)
self.assertEquals(
'protobuf_unittest.TestAllTypes.NestedEnum',
self._Database().pool.FindEnumTypeByName(
'protobuf_unittest.TestAllTypes.NestedEnum').full_name)
def testFindMessageTypeByName(self):
self.assertEquals(
'protobuf_unittest.TestAllTypes',
self._Database().pool.FindMessageTypeByName(
'protobuf_unittest.TestAllTypes').full_name)
self.assertEquals(
'protobuf_unittest.TestAllTypes.NestedMessage',
self._Database().pool.FindMessageTypeByName(
'protobuf_unittest.TestAllTypes.NestedMessage').full_name)
def testFindFindContainingSymbol(self):
# Lookup based on either enum or message.
self.assertEquals(
'google/protobuf/unittest.proto',
self._Database().pool.FindFileContainingSymbol(
'protobuf_unittest.TestAllTypes.NestedEnum').name)
self.assertEquals(
'google/protobuf/unittest.proto',
self._Database().pool.FindFileContainingSymbol(
'protobuf_unittest.TestAllTypes').name)
def testFindFileByName(self):
self.assertEquals(
'google/protobuf/unittest.proto',
self._Database().pool.FindFileByName(
'google/protobuf/unittest.proto').name)
if __name__ == '__main__':
basetest.main()

@ -66,14 +66,8 @@ def SetAllNonLazyFields(message):
message.optional_float = 111
message.optional_double = 112
message.optional_bool = True
# TODO(robinson): Firmly spec out and test how
# protos interact with unicode. One specific example:
# what happens if we change the literal below to
# u'115'? What *should* happen? Still some discussion
# to finish with Kenton about bytes vs. strings
# and forcing everything to be utf8. :-/
message.optional_string = '115'
message.optional_bytes = '116'
message.optional_string = u'115'
message.optional_bytes = b'116'
message.optionalgroup.a = 117
message.optional_nested_message.bb = 118
@ -85,8 +79,8 @@ def SetAllNonLazyFields(message):
message.optional_foreign_enum = unittest_pb2.FOREIGN_BAZ
message.optional_import_enum = unittest_import_pb2.IMPORT_BAZ
message.optional_string_piece = '124'
message.optional_cord = '125'
message.optional_string_piece = u'124'
message.optional_cord = u'125'
#
# Repeated fields.
@ -105,8 +99,8 @@ def SetAllNonLazyFields(message):
message.repeated_float.append(211)
message.repeated_double.append(212)
message.repeated_bool.append(True)
message.repeated_string.append('215')
message.repeated_bytes.append('216')
message.repeated_string.append(u'215')
message.repeated_bytes.append(b'216')
message.repeatedgroup.add().a = 217
message.repeated_nested_message.add().bb = 218
@ -118,8 +112,8 @@ def SetAllNonLazyFields(message):
message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAR)
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAR)
message.repeated_string_piece.append('224')
message.repeated_cord.append('225')
message.repeated_string_piece.append(u'224')
message.repeated_cord.append(u'225')
# Add a second one of each field.
message.repeated_int32.append(301)
@ -135,8 +129,8 @@ def SetAllNonLazyFields(message):
message.repeated_float.append(311)
message.repeated_double.append(312)
message.repeated_bool.append(False)
message.repeated_string.append('315')
message.repeated_bytes.append('316')
message.repeated_string.append(u'315')
message.repeated_bytes.append(b'316')
message.repeatedgroup.add().a = 317
message.repeated_nested_message.add().bb = 318
@ -148,8 +142,8 @@ def SetAllNonLazyFields(message):
message.repeated_foreign_enum.append(unittest_pb2.FOREIGN_BAZ)
message.repeated_import_enum.append(unittest_import_pb2.IMPORT_BAZ)
message.repeated_string_piece.append('324')
message.repeated_cord.append('325')
message.repeated_string_piece.append(u'324')
message.repeated_cord.append(u'325')
#
# Fields that have defaults.
@ -169,7 +163,7 @@ def SetAllNonLazyFields(message):
message.default_double = 412
message.default_bool = False
message.default_string = '415'
message.default_bytes = '416'
message.default_bytes = b'416'
message.default_nested_enum = unittest_pb2.TestAllTypes.FOO
message.default_foreign_enum = unittest_pb2.FOREIGN_FOO
@ -178,6 +172,11 @@ def SetAllNonLazyFields(message):
message.default_string_piece = '424'
message.default_cord = '425'
message.oneof_uint32 = 601
message.oneof_nested_message.bb = 602
message.oneof_string = '603'
message.oneof_bytes = b'604'
def SetAllFields(message):
SetAllNonLazyFields(message)
@ -212,8 +211,8 @@ def SetAllExtensions(message):
extensions[pb2.optional_float_extension] = 111
extensions[pb2.optional_double_extension] = 112
extensions[pb2.optional_bool_extension] = True
extensions[pb2.optional_string_extension] = '115'
extensions[pb2.optional_bytes_extension] = '116'
extensions[pb2.optional_string_extension] = u'115'
extensions[pb2.optional_bytes_extension] = b'116'
extensions[pb2.optionalgroup_extension].a = 117
extensions[pb2.optional_nested_message_extension].bb = 118
@ -227,8 +226,8 @@ def SetAllExtensions(message):
extensions[pb2.optional_foreign_enum_extension] = pb2.FOREIGN_BAZ
extensions[pb2.optional_import_enum_extension] = import_pb2.IMPORT_BAZ
extensions[pb2.optional_string_piece_extension] = '124'
extensions[pb2.optional_cord_extension] = '125'
extensions[pb2.optional_string_piece_extension] = u'124'
extensions[pb2.optional_cord_extension] = u'125'
#
# Repeated fields.
@ -247,8 +246,8 @@ def SetAllExtensions(message):
extensions[pb2.repeated_float_extension].append(211)
extensions[pb2.repeated_double_extension].append(212)
extensions[pb2.repeated_bool_extension].append(True)
extensions[pb2.repeated_string_extension].append('215')
extensions[pb2.repeated_bytes_extension].append('216')
extensions[pb2.repeated_string_extension].append(u'215')
extensions[pb2.repeated_bytes_extension].append(b'216')
extensions[pb2.repeatedgroup_extension].add().a = 217
extensions[pb2.repeated_nested_message_extension].add().bb = 218
@ -260,8 +259,8 @@ def SetAllExtensions(message):
extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAR)
extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAR)
extensions[pb2.repeated_string_piece_extension].append('224')
extensions[pb2.repeated_cord_extension].append('225')
extensions[pb2.repeated_string_piece_extension].append(u'224')
extensions[pb2.repeated_cord_extension].append(u'225')
# Append a second one of each field.
extensions[pb2.repeated_int32_extension].append(301)
@ -277,8 +276,8 @@ def SetAllExtensions(message):
extensions[pb2.repeated_float_extension].append(311)
extensions[pb2.repeated_double_extension].append(312)
extensions[pb2.repeated_bool_extension].append(False)
extensions[pb2.repeated_string_extension].append('315')
extensions[pb2.repeated_bytes_extension].append('316')
extensions[pb2.repeated_string_extension].append(u'315')
extensions[pb2.repeated_bytes_extension].append(b'316')
extensions[pb2.repeatedgroup_extension].add().a = 317
extensions[pb2.repeated_nested_message_extension].add().bb = 318
@ -290,8 +289,8 @@ def SetAllExtensions(message):
extensions[pb2.repeated_foreign_enum_extension].append(pb2.FOREIGN_BAZ)
extensions[pb2.repeated_import_enum_extension].append(import_pb2.IMPORT_BAZ)
extensions[pb2.repeated_string_piece_extension].append('324')
extensions[pb2.repeated_cord_extension].append('325')
extensions[pb2.repeated_string_piece_extension].append(u'324')
extensions[pb2.repeated_cord_extension].append(u'325')
#
# Fields with defaults.
@ -310,16 +309,21 @@ def SetAllExtensions(message):
extensions[pb2.default_float_extension] = 411
extensions[pb2.default_double_extension] = 412
extensions[pb2.default_bool_extension] = False
extensions[pb2.default_string_extension] = '415'
extensions[pb2.default_bytes_extension] = '416'
extensions[pb2.default_string_extension] = u'415'
extensions[pb2.default_bytes_extension] = b'416'
extensions[pb2.default_nested_enum_extension] = pb2.TestAllTypes.FOO
extensions[pb2.default_foreign_enum_extension] = pb2.FOREIGN_FOO
extensions[pb2.default_import_enum_extension] = import_pb2.IMPORT_FOO
extensions[pb2.default_string_piece_extension] = '424'
extensions[pb2.default_string_piece_extension] = u'424'
extensions[pb2.default_cord_extension] = '425'
extensions[pb2.oneof_uint32_extension] = 601
extensions[pb2.oneof_nested_message_extension].bb = 602
extensions[pb2.oneof_string_extension] = u'603'
extensions[pb2.oneof_bytes_extension] = b'604'
def SetAllFieldsAndExtensions(message):
"""Sets every field and extension in the message to a unique value.
@ -358,7 +362,7 @@ def ExpectAllFieldsAndExtensionsInOrder(serialized):
message.my_float = 1.0
expected_strings.append(message.SerializeToString())
message.Clear()
expected = ''.join(expected_strings)
expected = b''.join(expected_strings)
if expected != serialized:
raise ValueError('Expected %r, found %r' % (expected, serialized))
@ -413,7 +417,7 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(112, message.optional_double)
test_case.assertEqual(True, message.optional_bool)
test_case.assertEqual('115', message.optional_string)
test_case.assertEqual('116', message.optional_bytes)
test_case.assertEqual(b'116', message.optional_bytes)
test_case.assertEqual(117, message.optionalgroup.a)
test_case.assertEqual(118, message.optional_nested_message.bb)
@ -472,7 +476,7 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(212, message.repeated_double[0])
test_case.assertEqual(True, message.repeated_bool[0])
test_case.assertEqual('215', message.repeated_string[0])
test_case.assertEqual('216', message.repeated_bytes[0])
test_case.assertEqual(b'216', message.repeated_bytes[0])
test_case.assertEqual(217, message.repeatedgroup[0].a)
test_case.assertEqual(218, message.repeated_nested_message[0].bb)
@ -501,7 +505,7 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(312, message.repeated_double[1])
test_case.assertEqual(False, message.repeated_bool[1])
test_case.assertEqual('315', message.repeated_string[1])
test_case.assertEqual('316', message.repeated_bytes[1])
test_case.assertEqual(b'316', message.repeated_bytes[1])
test_case.assertEqual(317, message.repeatedgroup[1].a)
test_case.assertEqual(318, message.repeated_nested_message[1].bb)
@ -552,7 +556,7 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(412, message.default_double)
test_case.assertEqual(False, message.default_bool)
test_case.assertEqual('415', message.default_string)
test_case.assertEqual('416', message.default_bytes)
test_case.assertEqual(b'416', message.default_bytes)
test_case.assertEqual(unittest_pb2.TestAllTypes.FOO,
message.default_nested_enum)
@ -561,6 +565,7 @@ def ExpectAllFieldsSet(test_case, message):
test_case.assertEqual(unittest_import_pb2.IMPORT_FOO,
message.default_import_enum)
def GoldenFile(filename):
"""Finds the given golden file and returns a file object representing it."""
@ -574,9 +579,15 @@ def GoldenFile(filename):
path = os.path.join(path, '..')
raise RuntimeError(
'Could not find golden files. This test must be run from within the '
'protobuf source package so that it can read test data files from the '
'C++ source tree.')
'Could not find golden files. This test must be run from within the '
'protobuf source package so that it can read test data files from the '
'C++ source tree.')
def GoldenFileData(filename):
"""Finds the given golden file and returns its contents."""
with GoldenFile(filename) as f:
return f.read()
def SetAllPackedFields(message):

@ -0,0 +1,68 @@
#! /usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Tests for google.protobuf.text_encoding."""
from google.apputils import basetest
from google.protobuf import text_encoding
TEST_VALUES = [
("foo\\rbar\\nbaz\\t",
"foo\\rbar\\nbaz\\t",
b"foo\rbar\nbaz\t"),
("\\'full of \\\"sound\\\" and \\\"fury\\\"\\'",
"\\'full of \\\"sound\\\" and \\\"fury\\\"\\'",
b"'full of \"sound\" and \"fury\"'"),
("signi\\\\fying\\\\ nothing\\\\",
"signi\\\\fying\\\\ nothing\\\\",
b"signi\\fying\\ nothing\\"),
("\\010\\t\\n\\013\\014\\r",
"\x08\\t\\n\x0b\x0c\\r",
b"\010\011\012\013\014\015")]
class TextEncodingTestCase(basetest.TestCase):
def testCEscape(self):
for escaped, escaped_utf8, unescaped in TEST_VALUES:
self.assertEquals(escaped,
text_encoding.CEscape(unescaped, as_utf8=False))
self.assertEquals(escaped_utf8,
text_encoding.CEscape(unescaped, as_utf8=True))
def testCUnescape(self):
for escaped, escaped_utf8, unescaped in TEST_VALUES:
self.assertEquals(unescaped, text_encoding.CUnescape(escaped))
self.assertEquals(unescaped, text_encoding.CUnescape(escaped_utf8))
if __name__ == "__main__":
basetest.main()

@ -34,49 +34,71 @@
__author__ = 'kenton@google.com (Kenton Varda)'
import difflib
import re
import unittest
from google.apputils import basetest
from google.protobuf import text_format
from google.protobuf.internal import api_implementation
from google.protobuf.internal import test_util
from google.protobuf import unittest_pb2
from google.protobuf import unittest_mset_pb2
class TextFormatTest(basetest.TestCase):
class TextFormatTest(unittest.TestCase):
def ReadGolden(self, golden_filename):
f = test_util.GoldenFile(golden_filename)
golden_lines = f.readlines()
f.close()
return golden_lines
with test_util.GoldenFile(golden_filename) as f:
return (f.readlines() if str is bytes else # PY3
[golden_line.decode('utf-8') for golden_line in f])
def CompareToGoldenFile(self, text, golden_filename):
golden_lines = self.ReadGolden(golden_filename)
self.CompareToGoldenLines(text, golden_lines)
self.assertMultiLineEqual(text, ''.join(golden_lines))
def CompareToGoldenText(self, text, golden_text):
self.CompareToGoldenLines(text, golden_text.splitlines(1))
def CompareToGoldenLines(self, text, golden_lines):
actual_lines = text.splitlines(1)
self.assertEqual(golden_lines, actual_lines,
"Text doesn't match golden. Diff:\n" +
''.join(difflib.ndiff(golden_lines, actual_lines)))
self.assertMultiLineEqual(text, golden_text)
def testPrintAllFields(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_data.txt')
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_data_oneof_implemented.txt')
def testPrintInIndexOrder(self):
message = unittest_pb2.TestFieldOrderings()
message.my_string = '115'
message.my_int = 101
message.my_float = 111
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message, use_index_order=True)),
'my_string: \"115\"\nmy_int: 101\nmy_float: 111\n')
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(
message)), 'my_int: 101\nmy_string: \"115\"\nmy_float: 111\n')
def testPrintAllExtensions(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_extensions_data.txt')
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'text_format_unittest_extensions_data.txt')
def testPrintAllFieldsPointy(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(
text_format.MessageToString(message, pointy_brackets=True)),
'text_format_unittest_data_pointy_oneof_implemented.txt')
def testPrintAllExtensionsPointy(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.CompareToGoldenFile(
self.RemoveRedundantZeros(text_format.MessageToString(
message, pointy_brackets=True)),
'text_format_unittest_extensions_data_pointy.txt')
def testPrintMessageSet(self):
message = unittest_mset_pb2.TestMessageSetContainer()
@ -84,37 +106,16 @@ class TextFormatTest(unittest.TestCase):
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
message.message_set.Extensions[ext1].i = 23
message.message_set.Extensions[ext2].str = 'foo'
self.CompareToGoldenText(text_format.MessageToString(message),
'message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
def testPrintBadEnumValue(self):
message = unittest_pb2.TestAllTypes()
message.optional_nested_enum = 100
message.optional_foreign_enum = 101
message.optional_import_enum = 102
self.CompareToGoldenText(
text_format.MessageToString(message),
'optional_nested_enum: 100\n'
'optional_foreign_enum: 101\n'
'optional_import_enum: 102\n')
def testPrintBadEnumValueExtensions(self):
message = unittest_pb2.TestAllExtensions()
message.Extensions[unittest_pb2.optional_nested_enum_extension] = 100
message.Extensions[unittest_pb2.optional_foreign_enum_extension] = 101
message.Extensions[unittest_pb2.optional_import_enum_extension] = 102
self.CompareToGoldenText(
text_format.MessageToString(message),
'[protobuf_unittest.optional_nested_enum_extension]: 100\n'
'[protobuf_unittest.optional_foreign_enum_extension]: 101\n'
'[protobuf_unittest.optional_import_enum_extension]: 102\n')
'message_set {\n'
' [protobuf_unittest.TestMessageSetExtension1] {\n'
' i: 23\n'
' }\n'
' [protobuf_unittest.TestMessageSetExtension2] {\n'
' str: \"foo\"\n'
' }\n'
'}\n')
def testPrintExotic(self):
message = unittest_pb2.TestAllTypes()
@ -126,20 +127,29 @@ class TextFormatTest(unittest.TestCase):
message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"')
message.repeated_string.append(u'\u00fc\ua71f')
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'repeated_int64: -9223372036854775808\n'
'repeated_uint64: 18446744073709551615\n'
'repeated_double: 123.456\n'
'repeated_double: 1.23e+22\n'
'repeated_double: 1.23e-18\n'
'repeated_string: '
'"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n'
'repeated_string: "\\303\\274\\352\\234\\237"\n')
self.RemoveRedundantZeros(text_format.MessageToString(message)),
'repeated_int64: -9223372036854775808\n'
'repeated_uint64: 18446744073709551615\n'
'repeated_double: 123.456\n'
'repeated_double: 1.23e+22\n'
'repeated_double: 1.23e-18\n'
'repeated_string:'
' "\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""\n'
'repeated_string: "\\303\\274\\352\\234\\237"\n')
def testPrintExoticUnicodeSubclass(self):
class UnicodeSub(unicode):
pass
message = unittest_pb2.TestAllTypes()
message.repeated_string.append(UnicodeSub(u'\u00fc\ua71f'))
self.CompareToGoldenText(
text_format.MessageToString(message),
'repeated_string: "\\303\\274\\352\\234\\237"\n')
def testPrintNestedMessageAsOneLine(self):
message = unittest_pb2.TestAllTypes()
msg = message.repeated_nested_message.add()
msg.bb = 42;
msg.bb = 42
self.CompareToGoldenText(
text_format.MessageToString(message, as_one_line=True),
'repeated_nested_message { bb: 42 }')
@ -190,16 +200,16 @@ class TextFormatTest(unittest.TestCase):
message.repeated_string.append('\000\001\a\b\f\n\r\t\v\\\'"')
message.repeated_string.append(u'\u00fc\ua71f')
self.CompareToGoldenText(
self.RemoveRedundantZeros(
text_format.MessageToString(message, as_one_line=True)),
'repeated_int64: -9223372036854775808'
' repeated_uint64: 18446744073709551615'
' repeated_double: 123.456'
' repeated_double: 1.23e+22'
' repeated_double: 1.23e-18'
' repeated_string: '
'"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""'
' repeated_string: "\\303\\274\\352\\234\\237"')
self.RemoveRedundantZeros(
text_format.MessageToString(message, as_one_line=True)),
'repeated_int64: -9223372036854775808'
' repeated_uint64: 18446744073709551615'
' repeated_double: 123.456'
' repeated_double: 1.23e+22'
' repeated_double: 1.23e-18'
' repeated_string: '
'"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\""'
' repeated_string: "\\303\\274\\352\\234\\237"')
def testRoundTripExoticAsOneLine(self):
message = unittest_pb2.TestAllTypes()
@ -215,24 +225,60 @@ class TextFormatTest(unittest.TestCase):
wire_text = text_format.MessageToString(
message, as_one_line=True, as_utf8=False)
parsed_message = unittest_pb2.TestAllTypes()
text_format.Merge(wire_text, parsed_message)
r = text_format.Parse(wire_text, parsed_message)
self.assertIs(r, parsed_message)
self.assertEquals(message, parsed_message)
# Test as_utf8 = True.
wire_text = text_format.MessageToString(
message, as_one_line=True, as_utf8=True)
parsed_message = unittest_pb2.TestAllTypes()
text_format.Merge(wire_text, parsed_message)
self.assertEquals(message, parsed_message)
r = text_format.Parse(wire_text, parsed_message)
self.assertIs(r, parsed_message)
self.assertEquals(message, parsed_message,
'\n%s != %s' % (message, parsed_message))
def testPrintRawUtf8String(self):
message = unittest_pb2.TestAllTypes()
message.repeated_string.append(u'\u00fc\ua71f')
text = text_format.MessageToString(message, as_utf8 = True)
text = text_format.MessageToString(message, as_utf8=True)
self.CompareToGoldenText(text, 'repeated_string: "\303\274\352\234\237"\n')
parsed_message = unittest_pb2.TestAllTypes()
text_format.Merge(text, parsed_message)
self.assertEquals(message, parsed_message)
text_format.Parse(text, parsed_message)
self.assertEquals(message, parsed_message,
'\n%s != %s' % (message, parsed_message))
def testPrintFloatFormat(self):
# Check that float_format argument is passed to sub-message formatting.
message = unittest_pb2.NestedTestAllTypes()
# We use 1.25 as it is a round number in binary. The proto 32-bit float
# will not gain additional imprecise digits as a 64-bit Python float and
# show up in its str. 32-bit 1.2 is noisy when extended to 64-bit:
# >>> struct.unpack('f', struct.pack('f', 1.2))[0]
# 1.2000000476837158
# >>> struct.unpack('f', struct.pack('f', 1.25))[0]
# 1.25
message.payload.optional_float = 1.25
# Check rounding at 15 significant digits
message.payload.optional_double = -.000003456789012345678
# Check no decimal point.
message.payload.repeated_float.append(-5642)
# Check no trailing zeros.
message.payload.repeated_double.append(.000078900)
formatted_fields = ['optional_float: 1.25',
'optional_double: -3.45678901234568e-6',
'repeated_float: -5642',
'repeated_double: 7.89e-5']
text_message = text_format.MessageToString(message, float_format='.15g')
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_message),
'payload {{\n {}\n {}\n {}\n {}\n}}\n'.format(*formatted_fields))
# as_one_line=True is a separate code branch where float_format is passed.
text_message = text_format.MessageToString(message, as_one_line=True,
float_format='.15g')
self.CompareToGoldenText(
self.RemoveRedundantZeros(text_message),
'payload {{ {} {} {} {} }}'.format(*formatted_fields))
def testMessageToString(self):
message = unittest_pb2.ForeignMessage()
@ -249,49 +295,50 @@ class TextFormatTest(unittest.TestCase):
text = re.compile('\.0$', re.MULTILINE).sub('', text)
return text
def testMergeGolden(self):
def testParseGolden(self):
golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt'))
parsed_message = unittest_pb2.TestAllTypes()
text_format.Merge(golden_text, parsed_message)
r = text_format.Parse(golden_text, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEquals(message, parsed_message)
def testMergeGoldenExtensions(self):
def testParseGoldenExtensions(self):
golden_text = '\n'.join(self.ReadGolden(
'text_format_unittest_extensions_data.txt'))
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Merge(golden_text, parsed_message)
text_format.Parse(golden_text, parsed_message)
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
self.assertEquals(message, parsed_message)
def testMergeAllFields(self):
def testParseAllFields(self):
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
ascii_text = text_format.MessageToString(message)
parsed_message = unittest_pb2.TestAllTypes()
text_format.Merge(ascii_text, parsed_message)
text_format.Parse(ascii_text, parsed_message)
self.assertEqual(message, parsed_message)
test_util.ExpectAllFieldsSet(self, message)
def testMergeAllExtensions(self):
def testParseAllExtensions(self):
message = unittest_pb2.TestAllExtensions()
test_util.SetAllExtensions(message)
ascii_text = text_format.MessageToString(message)
parsed_message = unittest_pb2.TestAllExtensions()
text_format.Merge(ascii_text, parsed_message)
text_format.Parse(ascii_text, parsed_message)
self.assertEqual(message, parsed_message)
def testMergeMessageSet(self):
def testParseMessageSet(self):
message = unittest_pb2.TestAllTypes()
text = ('repeated_uint64: 1\n'
'repeated_uint64: 2\n')
text_format.Merge(text, message)
text_format.Parse(text, message)
self.assertEqual(1, message.repeated_uint64[0])
self.assertEqual(2, message.repeated_uint64[1])
@ -304,13 +351,13 @@ class TextFormatTest(unittest.TestCase):
' str: \"foo\"\n'
' }\n'
'}\n')
text_format.Merge(text, message)
text_format.Parse(text, message)
ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
self.assertEquals(23, message.message_set.Extensions[ext1].i)
self.assertEquals('foo', message.message_set.Extensions[ext2].str)
def testMergeExotic(self):
def testParseExotic(self):
message = unittest_pb2.TestAllTypes()
text = ('repeated_int64: -9223372036854775808\n'
'repeated_uint64: 18446744073709551615\n'
@ -323,7 +370,7 @@ class TextFormatTest(unittest.TestCase):
'repeated_string: "\\303\\274\\352\\234\\237"\n'
'repeated_string: "\\xc3\\xbc"\n'
'repeated_string: "\xc3\xbc"\n')
text_format.Merge(text, message)
text_format.Parse(text, message)
self.assertEqual(-9223372036854775808, message.repeated_int64[0])
self.assertEqual(18446744073709551615, message.repeated_uint64[0])
@ -336,100 +383,115 @@ class TextFormatTest(unittest.TestCase):
self.assertEqual(u'\u00fc\ua71f', message.repeated_string[2])
self.assertEqual(u'\u00fc', message.repeated_string[3])
def testMergeEmptyText(self):
def testParseTrailingCommas(self):
message = unittest_pb2.TestAllTypes()
text = ('repeated_int64: 100;\n'
'repeated_int64: 200;\n'
'repeated_int64: 300,\n'
'repeated_string: "one",\n'
'repeated_string: "two";\n')
text_format.Parse(text, message)
self.assertEqual(100, message.repeated_int64[0])
self.assertEqual(200, message.repeated_int64[1])
self.assertEqual(300, message.repeated_int64[2])
self.assertEqual(u'one', message.repeated_string[0])
self.assertEqual(u'two', message.repeated_string[1])
def testParseEmptyText(self):
message = unittest_pb2.TestAllTypes()
text = ''
text_format.Merge(text, message)
text_format.Parse(text, message)
self.assertEquals(unittest_pb2.TestAllTypes(), message)
def testMergeInvalidUtf8(self):
def testParseInvalidUtf8(self):
message = unittest_pb2.TestAllTypes()
text = 'repeated_string: "\\xc3\\xc3"'
self.assertRaises(text_format.ParseError, text_format.Merge, text, message)
self.assertRaises(text_format.ParseError, text_format.Parse, text, message)
def testMergeSingleWord(self):
def testParseSingleWord(self):
message = unittest_pb2.TestAllTypes()
text = 'foo'
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named '
'"foo".'),
text_format.Merge, text, message)
text_format.Parse, text, message)
def testMergeUnknownField(self):
def testParseUnknownField(self):
message = unittest_pb2.TestAllTypes()
text = 'unknown_field: 8\n'
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named '
'"unknown_field".'),
text_format.Merge, text, message)
text_format.Parse, text, message)
def testMergeBadExtension(self):
def testParseBadExtension(self):
message = unittest_pb2.TestAllExtensions()
text = '[unknown_extension]: 8\n'
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
'1:2 : Extension "unknown_extension" not registered.',
text_format.Merge, text, message)
text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes()
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
'extensions.'),
text_format.Merge, text, message)
text_format.Parse, text, message)
def testMergeGroupNotClosed(self):
def testParseGroupNotClosed(self):
message = unittest_pb2.TestAllTypes()
text = 'RepeatedGroup: <'
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError, '1:16 : Expected ">".',
text_format.Merge, text, message)
text_format.Parse, text, message)
text = 'RepeatedGroup: {'
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError, '1:16 : Expected "}".',
text_format.Merge, text, message)
text_format.Parse, text, message)
def testMergeEmptyGroup(self):
def testParseEmptyGroup(self):
message = unittest_pb2.TestAllTypes()
text = 'OptionalGroup: {}'
text_format.Merge(text, message)
text_format.Parse(text, message)
self.assertTrue(message.HasField('optionalgroup'))
message.Clear()
message = unittest_pb2.TestAllTypes()
text = 'OptionalGroup: <>'
text_format.Merge(text, message)
text_format.Parse(text, message)
self.assertTrue(message.HasField('optionalgroup'))
def testMergeBadEnumValue(self):
def testParseBadEnumValue(self):
message = unittest_pb2.TestAllTypes()
text = 'optional_nested_enum: BARR'
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
'has no value named BARR.'),
text_format.Merge, text, message)
text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes()
text = 'optional_nested_enum: 100'
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
'has no value with number 100.'),
text_format.Merge, text, message)
text_format.Parse, text, message)
def testMergeBadIntValue(self):
def testParseBadIntValue(self):
message = unittest_pb2.TestAllTypes()
text = 'optional_int32: bork'
self.assertRaisesWithMessage(
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:17 : Couldn\'t parse integer: bork'),
text_format.Merge, text, message)
text_format.Parse, text, message)
def testMergeStringFieldUnescape(self):
def testParseStringFieldUnescape(self):
message = unittest_pb2.TestAllTypes()
text = r'''repeated_string: "\xf\x62"
repeated_string: "\\xf\\x62"
@ -437,7 +499,7 @@ class TextFormatTest(unittest.TestCase):
repeated_string: "\\\\xf\\\\x62"
repeated_string: "\\\\\xf\\\\\x62"
repeated_string: "\x5cx20"'''
text_format.Merge(text, message)
text_format.Parse(text, message)
SLASH = '\\'
self.assertEqual('\x0fb', message.repeated_string[0])
@ -449,27 +511,84 @@ class TextFormatTest(unittest.TestCase):
message.repeated_string[4])
self.assertEqual(SLASH + 'x20', message.repeated_string[5])
def assertRaisesWithMessage(self, e_class, e, func, *args, **kwargs):
"""Same as assertRaises, but also compares the exception message."""
if hasattr(e_class, '__name__'):
exc_name = e_class.__name__
else:
exc_name = str(e_class)
def testMergeRepeatedScalars(self):
message = unittest_pb2.TestAllTypes()
text = ('optional_int32: 42 '
'optional_int32: 67')
r = text_format.Merge(text, message)
self.assertIs(r, message)
self.assertEqual(67, message.optional_int32)
def testParseRepeatedScalars(self):
message = unittest_pb2.TestAllTypes()
text = ('optional_int32: 42 '
'optional_int32: 67')
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:36 : Message type "protobuf_unittest.TestAllTypes" should not '
'have multiple "optional_int32" fields.'),
text_format.Parse, text, message)
def testMergeRepeatedNestedMessageScalars(self):
message = unittest_pb2.TestAllTypes()
text = ('optional_nested_message { bb: 1 } '
'optional_nested_message { bb: 2 }')
r = text_format.Merge(text, message)
self.assertTrue(r is message)
self.assertEqual(2, message.optional_nested_message.bb)
def testParseRepeatedNestedMessageScalars(self):
message = unittest_pb2.TestAllTypes()
text = ('optional_nested_message { bb: 1 } '
'optional_nested_message { bb: 2 }')
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:65 : Message type "protobuf_unittest.TestAllTypes.NestedMessage" '
'should not have multiple "bb" fields.'),
text_format.Parse, text, message)
def testMergeRepeatedExtensionScalars(self):
message = unittest_pb2.TestAllExtensions()
text = ('[protobuf_unittest.optional_int32_extension]: 42 '
'[protobuf_unittest.optional_int32_extension]: 67')
text_format.Merge(text, message)
self.assertEqual(
67,
message.Extensions[unittest_pb2.optional_int32_extension])
def testParseRepeatedExtensionScalars(self):
message = unittest_pb2.TestAllExtensions()
text = ('[protobuf_unittest.optional_int32_extension]: 42 '
'[protobuf_unittest.optional_int32_extension]: 67')
self.assertRaisesWithLiteralMatch(
text_format.ParseError,
('1:96 : Message type "protobuf_unittest.TestAllExtensions" '
'should not have multiple '
'"protobuf_unittest.optional_int32_extension" extensions.'),
text_format.Parse, text, message)
def testParseLinesGolden(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.ParseLines(opened, parsed_message)
self.assertIs(r, parsed_message)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEquals(message, parsed_message)
def testMergeLinesGolden(self):
opened = self.ReadGolden('text_format_unittest_data.txt')
parsed_message = unittest_pb2.TestAllTypes()
r = text_format.MergeLines(opened, parsed_message)
self.assertIs(r, parsed_message)
try:
func(*args, **kwargs)
except e_class as expr:
if str(expr) != e:
msg = '%s raised, but with wrong message: "%s" instead of "%s"'
raise self.failureException(msg % (exc_name,
str(expr).encode('string_escape'),
e.encode('string_escape')))
return
else:
raise self.failureException('%s not raised' % exc_name)
message = unittest_pb2.TestAllTypes()
test_util.SetAllFields(message)
self.assertEqual(message, parsed_message)
class TokenizerTest(unittest.TestCase):
class TokenizerTest(basetest.TestCase):
def testSimpleTokenCases(self):
text = ('identifier1:"string1"\n \n\n'
@ -478,8 +597,8 @@ class TokenizerTest(unittest.TestCase):
'ID7 : "aa\\"bb"\n\n\n\n ID8: {A:inf B:-inf C:true D:false}\n'
'ID9: 22 ID10: -111111111111111111 ID11: -22\n'
'ID12: 2222222222222222222 ID13: 1.23456f ID14: 1.2e+2f '
'false_bool: 0 true_BOOL:t \n true_bool1: 1 false_BOOL1:f ' )
tokenizer = text_format._Tokenizer(text)
'false_bool: 0 true_BOOL:t \n true_bool1: 1 false_BOOL1:f ')
tokenizer = text_format._Tokenizer(text.splitlines())
methods = [(tokenizer.ConsumeIdentifier, 'identifier1'),
':',
(tokenizer.ConsumeString, 'string1'),
@ -565,7 +684,7 @@ class TokenizerTest(unittest.TestCase):
int64_max = (1 << 63) - 1
uint32_max = (1 << 32) - 1
text = '-1 %d %d' % (uint32_max + 1, int64_max + 1)
tokenizer = text_format._Tokenizer(text)
tokenizer = text_format._Tokenizer(text.splitlines())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint64)
self.assertEqual(-1, tokenizer.ConsumeInt32())
@ -579,7 +698,7 @@ class TokenizerTest(unittest.TestCase):
self.assertTrue(tokenizer.AtEnd())
text = '-0 -0 0 0'
tokenizer = text_format._Tokenizer(text)
tokenizer = text_format._Tokenizer(text.splitlines())
self.assertEqual(0, tokenizer.ConsumeUint32())
self.assertEqual(0, tokenizer.ConsumeUint64())
self.assertEqual(0, tokenizer.ConsumeUint32())
@ -588,30 +707,30 @@ class TokenizerTest(unittest.TestCase):
def testConsumeByteString(self):
text = '"string1\''
tokenizer = text_format._Tokenizer(text)
tokenizer = text_format._Tokenizer(text.splitlines())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
text = 'string1"'
tokenizer = text_format._Tokenizer(text)
tokenizer = text_format._Tokenizer(text.splitlines())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
text = '\n"\\xt"'
tokenizer = text_format._Tokenizer(text)
tokenizer = text_format._Tokenizer(text.splitlines())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
text = '\n"\\"'
tokenizer = text_format._Tokenizer(text)
tokenizer = text_format._Tokenizer(text.splitlines())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
text = '\n"\\x"'
tokenizer = text_format._Tokenizer(text)
tokenizer = text_format._Tokenizer(text.splitlines())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
def testConsumeBool(self):
text = 'not-a-bool'
tokenizer = text_format._Tokenizer(text)
tokenizer = text_format._Tokenizer(text.splitlines())
self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool)
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -28,6 +28,10 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#PY25 compatible for GAE.
#
# Copyright 2008 Google Inc. All Rights Reserved.
"""Provides type checking routines.
This module defines type checking utilities in the forms of dictionaries:
@ -45,6 +49,9 @@ TYPE_TO_DESERIALIZE_METHOD: A dictionary with field types and deserialization
__author__ = 'robinson@google.com (Will Robinson)'
import sys ##PY25
if sys.version < '2.6': bytes = str ##PY25
from google.protobuf.internal import api_implementation
from google.protobuf.internal import decoder
from google.protobuf.internal import encoder
from google.protobuf.internal import wire_format
@ -53,21 +60,22 @@ from google.protobuf import descriptor
_FieldDescriptor = descriptor.FieldDescriptor
def GetTypeChecker(cpp_type, field_type):
def GetTypeChecker(field):
"""Returns a type checker for a message field of the specified types.
Args:
cpp_type: C++ type of the field (see descriptor.py).
field_type: Protocol message field type (see descriptor.py).
field: FieldDescriptor object for this field.
Returns:
An instance of TypeChecker which can be used to verify the types
of values assigned to a field of the specified type.
"""
if (cpp_type == _FieldDescriptor.CPPTYPE_STRING and
field_type == _FieldDescriptor.TYPE_STRING):
if (field.cpp_type == _FieldDescriptor.CPPTYPE_STRING and
field.type == _FieldDescriptor.TYPE_STRING):
return UnicodeValueChecker()
return _VALUE_CHECKERS[cpp_type]
if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM:
return EnumValueChecker(field.enum_type)
return _VALUE_CHECKERS[field.cpp_type]
# None of the typecheckers below make any attempt to guard against people
@ -85,10 +93,15 @@ class TypeChecker(object):
self._acceptable_types = acceptable_types
def CheckValue(self, proposed_value):
"""Type check the provided value and return it.
The returned value might have been normalized to another type.
"""
if not isinstance(proposed_value, self._acceptable_types):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), self._acceptable_types))
raise TypeError(message)
return proposed_value
# IntValueChecker and its subclasses perform integer type-checks
@ -104,28 +117,54 @@ class IntValueChecker(object):
raise TypeError(message)
if not self._MIN <= proposed_value <= self._MAX:
raise ValueError('Value out of range: %d' % proposed_value)
# We force 32-bit values to int and 64-bit values to long to make
# alternate implementations where the distinction is more significant
# (e.g. the C++ implementation) simpler.
proposed_value = self._TYPE(proposed_value)
return proposed_value
class EnumValueChecker(object):
"""Checker used for enum fields. Performs type-check and range check."""
def __init__(self, enum_type):
self._enum_type = enum_type
def CheckValue(self, proposed_value):
if not isinstance(proposed_value, (int, long)):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (int, long)))
raise TypeError(message)
if proposed_value not in self._enum_type.values_by_number:
raise ValueError('Unknown enum value: %d' % proposed_value)
return proposed_value
class UnicodeValueChecker(object):
"""Checker used for string fields."""
"""Checker used for string fields.
Always returns a unicode value, even if the input is of type str.
"""
def CheckValue(self, proposed_value):
if not isinstance(proposed_value, (str, unicode)):
if not isinstance(proposed_value, (bytes, unicode)):
message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (str, unicode)))
(proposed_value, type(proposed_value), (bytes, unicode)))
raise TypeError(message)
# If the value is of type 'str' make sure that it is in 7-bit ASCII
# If the value is of type 'bytes' make sure that it is in 7-bit ASCII
# encoding.
if isinstance(proposed_value, str):
if isinstance(proposed_value, bytes):
try:
unicode(proposed_value, 'ascii')
proposed_value = proposed_value.decode('ascii')
except UnicodeDecodeError:
raise ValueError('%.1024r has type str, but isn\'t in 7-bit ASCII '
raise ValueError('%.1024r has type bytes, but isn\'t in 7-bit ASCII '
'encoding. Non-ASCII strings must be converted to '
'unicode objects before being added.' %
(proposed_value))
return proposed_value
class Int32ValueChecker(IntValueChecker):
@ -133,21 +172,25 @@ class Int32ValueChecker(IntValueChecker):
# efficient.
_MIN = -2147483648
_MAX = 2147483647
_TYPE = int
class Uint32ValueChecker(IntValueChecker):
_MIN = 0
_MAX = (1 << 32) - 1
_TYPE = int
class Int64ValueChecker(IntValueChecker):
_MIN = -(1 << 63)
_MAX = (1 << 63) - 1
_TYPE = long
class Uint64ValueChecker(IntValueChecker):
_MIN = 0
_MAX = (1 << 64) - 1
_TYPE = long
# Type-checkers for all scalar CPPTYPEs.
@ -161,8 +204,7 @@ _VALUE_CHECKERS = {
_FieldDescriptor.CPPTYPE_FLOAT: TypeChecker(
float, int, long),
_FieldDescriptor.CPPTYPE_BOOL: TypeChecker(bool, int),
_FieldDescriptor.CPPTYPE_ENUM: Int32ValueChecker(),
_FieldDescriptor.CPPTYPE_STRING: TypeChecker(str),
_FieldDescriptor.CPPTYPE_STRING: TypeChecker(bytes),
}

@ -35,15 +35,16 @@
__author__ = 'bohdank@google.com (Bohdan Koval)'
import unittest
from google.apputils import basetest
from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2
from google.protobuf.internal import encoder
from google.protobuf.internal import missing_enum_values_pb2
from google.protobuf.internal import test_util
from google.protobuf.internal import type_checkers
class UnknownFieldsTest(unittest.TestCase):
class UnknownFieldsTest(basetest.TestCase):
def setUp(self):
self.descriptor = unittest_pb2.TestAllTypes.DESCRIPTOR
@ -58,12 +59,20 @@ class UnknownFieldsTest(unittest.TestCase):
field_descriptor = self.descriptor.fields_by_name[name]
wire_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type]
field_tag = encoder.TagBytes(field_descriptor.number, wire_type)
result_dict = {}
for tag_bytes, value in self.unknown_fields:
if tag_bytes == field_tag:
decoder = unittest_pb2.TestAllTypes._decoders_by_tag[tag_bytes]
result_dict = {}
decoder(value, 0, len(value), self.all_fields, result_dict)
return result_dict[field_descriptor]
return result_dict[field_descriptor]
def testEnum(self):
value = self.GetField('optional_nested_enum')
self.assertEqual(self.all_fields.optional_nested_enum, value)
def testRepeatedEnum(self):
value = self.GetField('repeated_nested_enum')
self.assertEqual(self.all_fields.repeated_nested_enum, value)
def testVarint(self):
value = self.GetField('optional_int32')
@ -166,5 +175,57 @@ class UnknownFieldsTest(unittest.TestCase):
self.assertNotEqual(self.empty_message, message)
class UnknownFieldsTest(basetest.TestCase):
def setUp(self):
self.descriptor = missing_enum_values_pb2.TestEnumValues.DESCRIPTOR
self.message = missing_enum_values_pb2.TestEnumValues()
self.message.optional_nested_enum = (
missing_enum_values_pb2.TestEnumValues.ZERO)
self.message.repeated_nested_enum.extend([
missing_enum_values_pb2.TestEnumValues.ZERO,
missing_enum_values_pb2.TestEnumValues.ONE,
])
self.message.packed_nested_enum.extend([
missing_enum_values_pb2.TestEnumValues.ZERO,
missing_enum_values_pb2.TestEnumValues.ONE,
])
self.message_data = self.message.SerializeToString()
self.missing_message = missing_enum_values_pb2.TestMissingEnumValues()
self.missing_message.ParseFromString(self.message_data)
self.unknown_fields = self.missing_message._unknown_fields
def GetField(self, name):
field_descriptor = self.descriptor.fields_by_name[name]
wire_type = type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type]
field_tag = encoder.TagBytes(field_descriptor.number, wire_type)
result_dict = {}
for tag_bytes, value in self.unknown_fields:
if tag_bytes == field_tag:
decoder = missing_enum_values_pb2.TestEnumValues._decoders_by_tag[
tag_bytes]
decoder(value, 0, len(value), self.message, result_dict)
return result_dict[field_descriptor]
def testUnknownEnumValue(self):
self.assertFalse(self.missing_message.HasField('optional_nested_enum'))
value = self.GetField('optional_nested_enum')
self.assertEqual(self.message.optional_nested_enum, value)
def testUnknownRepeatedEnumValue(self):
value = self.GetField('repeated_nested_enum')
self.assertEqual(self.message.repeated_nested_enum, value)
def testUnknownPackedEnumValue(self):
value = self.GetField('packed_nested_enum')
self.assertEqual(self.message.packed_nested_enum, value)
def testRoundTrip(self):
new_message = missing_enum_values_pb2.TestEnumValues()
new_message.ParseFromString(self.missing_message.SerializeToString())
self.assertEqual(self.message, new_message)
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -34,12 +34,12 @@
__author__ = 'robinson@google.com (Will Robinson)'
import unittest
from google.apputils import basetest
from google.protobuf import message
from google.protobuf.internal import wire_format
class WireFormatTest(unittest.TestCase):
class WireFormatTest(basetest.TestCase):
def testPackTag(self):
field_number = 0xabc
@ -195,7 +195,7 @@ class WireFormatTest(unittest.TestCase):
# Test UTF-8 string byte size calculation.
# 1 byte for tag, 1 byte for length, 8 bytes for content.
self.assertEqual(10, wire_format.StringByteSize(
5, unicode('\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82', 'utf-8')))
5, b'\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82'.decode('utf-8')))
class MockMessage(object):
def __init__(self, byte_size):
@ -250,4 +250,4 @@ class WireFormatTest(unittest.TestCase):
if __name__ == '__main__':
unittest.main()
basetest.main()

@ -177,7 +177,11 @@ class Message(object):
raise NotImplementedError
def ParseFromString(self, serialized):
"""Like MergeFromString(), except we clear the object first."""
"""Parse serialized protocol buffer data into this message.
Like MergeFromString(), except we clear the object first and
do not return the value that MergeFromString returns.
"""
self.Clear()
self.MergeFromString(serialized)

@ -28,10 +28,22 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Provides a factory class for generating dynamic messages."""
#PY25 compatible for GAE.
#
# Copyright 2012 Google Inc. All Rights Reserved.
"""Provides a factory class for generating dynamic messages.
The easiest way to use this class is if you have access to the FileDescriptor
protos containing the messages you want to create you can just do the following:
message_classes = message_factory.GetMessages(iterable_of_file_descriptors)
my_proto_instance = message_classes['some.proto.package.MessageName']()
"""
__author__ = 'matthewtoia@google.com (Matt Toia)'
import sys ##PY25
from google.protobuf import descriptor_database
from google.protobuf import descriptor_pool
from google.protobuf import message
@ -41,8 +53,12 @@ from google.protobuf import reflection
class MessageFactory(object):
"""Factory for creating Proto2 messages from descriptors in a pool."""
def __init__(self):
def __init__(self, pool=None):
"""Initializes a new factory."""
self.pool = (pool or descriptor_pool.DescriptorPool(
descriptor_database.DescriptorDatabase()))
# local cache of all classes built from protobuf descriptors
self._classes = {}
def GetPrototype(self, descriptor):
@ -57,21 +73,69 @@ class MessageFactory(object):
Returns:
A class describing the passed in descriptor.
"""
if descriptor.full_name not in self._classes:
descriptor_name = descriptor.name
if sys.version_info[0] < 3: ##PY25
##!PY25 if str is bytes: # PY2
descriptor_name = descriptor.name.encode('ascii', 'ignore')
result_class = reflection.GeneratedProtocolMessageType(
descriptor.name.encode('ascii', 'ignore'),
descriptor_name,
(message.Message,),
{'DESCRIPTOR': descriptor})
{'DESCRIPTOR': descriptor, '__module__': None})
# If module not set, it wrongly points to the reflection.py module.
self._classes[descriptor.full_name] = result_class
for field in descriptor.fields:
if field.message_type:
self.GetPrototype(field.message_type)
for extension in result_class.DESCRIPTOR.extensions:
if extension.containing_type.full_name not in self._classes:
self.GetPrototype(extension.containing_type)
extended_class = self._classes[extension.containing_type.full_name]
extended_class.RegisterExtension(extension)
return self._classes[descriptor.full_name]
def GetMessages(self, files):
"""Gets all the messages from a specified file.
This will find and resolve dependencies, failing if the descriptor
pool cannot satisfy them.
Args:
files: The file names to extract messages from.
Returns:
A dictionary mapping proto names to the message classes. This will include
any dependent messages as well as any messages defined in the same file as
a specified message.
"""
result = {}
for file_name in files:
file_desc = self.pool.FindFileByName(file_name)
for name, msg in file_desc.message_types_by_name.iteritems():
if file_desc.package:
full_name = '.'.join([file_desc.package, name])
else:
full_name = msg.name
result[full_name] = self.GetPrototype(
self.pool.FindMessageTypeByName(full_name))
# While the extension FieldDescriptors are created by the descriptor pool,
# the python classes created in the factory need them to be registered
# explicitly, which is done below.
#
# The call to RegisterExtension will specifically check if the
# extension was already registered on the object and either
# ignore the registration if the original was the same, or raise
# an error if they were different.
for name, extension in file_desc.extensions_by_name.iteritems():
if extension.containing_type.full_name not in self._classes:
self.GetPrototype(extension.containing_type)
extended_class = self._classes[extension.containing_type.full_name]
extended_class.RegisterExtension(extension)
return result
_DB = descriptor_database.DescriptorDatabase()
_POOL = descriptor_pool.DescriptorPool(_DB)
_FACTORY = MessageFactory()
@ -82,32 +146,10 @@ def GetMessages(file_protos):
file_protos: A sequence of file protos to build messages out of.
Returns:
A dictionary containing all the message types in the files mapping the
fully qualified name to a Message subclass for the descriptor.
A dictionary mapping proto names to the message classes. This will include
any dependent messages as well as any messages defined in the same file as
a specified message.
"""
result = {}
for file_proto in file_protos:
_DB.Add(file_proto)
for file_proto in file_protos:
for desc in _GetAllDescriptors(file_proto.message_type, file_proto.package):
result[desc.full_name] = _FACTORY.GetPrototype(desc)
return result
def _GetAllDescriptors(desc_protos, package):
"""Gets all levels of nested message types as a flattened list of descriptors.
Args:
desc_protos: The descriptor protos to process.
package: The package where the protos are defined.
Yields:
Each message descriptor for each nested type.
"""
for desc_proto in desc_protos:
name = '.'.join((package, desc_proto.name))
yield _POOL.FindMessageTypeByName(name)
for nested_desc in _GetAllDescriptors(desc_proto.nested_type, name):
yield nested_desc
_FACTORY.pool.Add(file_proto)
return _FACTORY.GetMessages([file_proto.name for file_proto in file_protos])

@ -0,0 +1,6 @@
This is the 'v2' C++ implementation for python proto2.
It is active when:
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2

@ -0,0 +1,61 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Protocol message implementation hooks for C++ implementation.
Contains helper functions used to create protocol message classes from
Descriptor objects at runtime backed by the protocol buffer C++ API.
"""
__author__ = 'tibell@google.com (Johan Tibell)'
from google.protobuf.pyext import _message
from google.protobuf import message
def NewMessage(bases, message_descriptor, dictionary):
"""Creates a new protocol message *class*."""
new_bases = []
for base in bases:
if base is message.Message:
# _message.Message must come before message.Message as it
# overrides methods in that class.
new_bases.append(_message.Message)
new_bases.append(base)
return tuple(new_bases)
def InitMessage(message_descriptor, cls):
"""Constructs a new message instance (called before instance's __init__)."""
def SubInit(self, **kwargs):
super(cls, self).__init__(message_descriptor, **kwargs)
cls.__init__ = SubInit
cls.AddDescriptors(message_descriptor)

@ -0,0 +1,357 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: petar@google.com (Petar Petrov)
#include <Python.h>
#include <string>
#include <google/protobuf/descriptor.pb.h>
#include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#define C(str) const_cast<char*>(str)
#if PY_MAJOR_VERSION >= 3
#define PyString_FromStringAndSize PyUnicode_FromStringAndSize
#define PyInt_FromLong PyLong_FromLong
#if PY_VERSION_HEX < 0x03030000
#error "Python 3.0 - 3.2 are not supported."
#else
#define PyString_AsString(ob) \
(PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob))
#endif
#endif
namespace google {
namespace protobuf {
namespace python {
#ifndef PyVarObject_HEAD_INIT
#define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size,
#endif
#ifndef Py_TYPE
#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type)
#endif
static google::protobuf::DescriptorPool* g_descriptor_pool = NULL;
namespace cfield_descriptor {
static void Dealloc(CFieldDescriptor* self) {
Py_CLEAR(self->descriptor_field);
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
static PyObject* GetFullName(CFieldDescriptor* self, void *closure) {
return PyString_FromStringAndSize(
self->descriptor->full_name().c_str(),
self->descriptor->full_name().size());
}
static PyObject* GetName(CFieldDescriptor *self, void *closure) {
return PyString_FromStringAndSize(
self->descriptor->name().c_str(),
self->descriptor->name().size());
}
static PyObject* GetCppType(CFieldDescriptor *self, void *closure) {
return PyInt_FromLong(self->descriptor->cpp_type());
}
static PyObject* GetLabel(CFieldDescriptor *self, void *closure) {
return PyInt_FromLong(self->descriptor->label());
}
static PyObject* GetID(CFieldDescriptor *self, void *closure) {
return PyLong_FromVoidPtr(self);
}
static PyGetSetDef Getters[] = {
{ C("full_name"), (getter)GetFullName, NULL, "Full name", NULL},
{ C("name"), (getter)GetName, NULL, "last name", NULL},
{ C("cpp_type"), (getter)GetCppType, NULL, "C++ Type", NULL},
{ C("label"), (getter)GetLabel, NULL, "Label", NULL},
{ C("id"), (getter)GetID, NULL, "ID", NULL},
{NULL}
};
} // namespace cfield_descriptor
PyTypeObject CFieldDescriptor_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
C("google.protobuf.internal."
"_net_proto2___python."
"CFieldDescriptor"), // tp_name
sizeof(CFieldDescriptor), // tp_basicsize
0, // tp_itemsize
(destructor)cfield_descriptor::Dealloc, // tp_dealloc
0, // tp_print
0, // tp_getattr
0, // tp_setattr
0, // tp_compare
0, // tp_repr
0, // tp_as_number
0, // tp_as_sequence
0, // tp_as_mapping
0, // tp_hash
0, // tp_call
0, // tp_str
0, // tp_getattro
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
C("A Field Descriptor"), // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
0, // tp_weaklistoffset
0, // tp_iter
0, // tp_iternext
0, // tp_methods
0, // tp_members
cfield_descriptor::Getters, // tp_getset
0, // tp_base
0, // tp_dict
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
0, // tp_init
PyType_GenericAlloc, // tp_alloc
PyType_GenericNew, // tp_new
PyObject_Del, // tp_free
};
namespace cdescriptor_pool {
static void Dealloc(CDescriptorPool* self) {
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
static PyObject* NewCDescriptor(
const google::protobuf::FieldDescriptor* field_descriptor) {
CFieldDescriptor* cfield_descriptor = PyObject_New(
CFieldDescriptor, &CFieldDescriptor_Type);
if (cfield_descriptor == NULL) {
return NULL;
}
cfield_descriptor->descriptor = field_descriptor;
cfield_descriptor->descriptor_field = NULL;
return reinterpret_cast<PyObject*>(cfield_descriptor);
}
PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name) {
const char* full_field_name = PyString_AsString(name);
if (full_field_name == NULL) {
return NULL;
}
const google::protobuf::FieldDescriptor* field_descriptor = NULL;
field_descriptor = self->pool->FindFieldByName(full_field_name);
if (field_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s",
full_field_name);
return NULL;
}
return NewCDescriptor(field_descriptor);
}
PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg) {
const char* full_field_name = PyString_AsString(arg);
if (full_field_name == NULL) {
return NULL;
}
const google::protobuf::FieldDescriptor* field_descriptor =
self->pool->FindExtensionByName(full_field_name);
if (field_descriptor == NULL) {
PyErr_Format(PyExc_TypeError, "Couldn't find field %.200s",
full_field_name);
return NULL;
}
return NewCDescriptor(field_descriptor);
}
static PyMethodDef Methods[] = {
{ C("FindFieldByName"),
(PyCFunction)FindFieldByName,
METH_O,
C("Searches for a field descriptor by full name.") },
{ C("FindExtensionByName"),
(PyCFunction)FindExtensionByName,
METH_O,
C("Searches for extension descriptor by full name.") },
{NULL}
};
} // namespace cdescriptor_pool
PyTypeObject CDescriptorPool_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
C("google.protobuf.internal."
"_net_proto2___python."
"CFieldDescriptor"), // tp_name
sizeof(CDescriptorPool), // tp_basicsize
0, // tp_itemsize
(destructor)cdescriptor_pool::Dealloc, // tp_dealloc
0, // tp_print
0, // tp_getattr
0, // tp_setattr
0, // tp_compare
0, // tp_repr
0, // tp_as_number
0, // tp_as_sequence
0, // tp_as_mapping
0, // tp_hash
0, // tp_call
0, // tp_str
0, // tp_getattro
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
C("A Descriptor Pool"), // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
0, // tp_weaklistoffset
0, // tp_iter
0, // tp_iternext
cdescriptor_pool::Methods, // tp_methods
0, // tp_members
0, // tp_getset
0, // tp_base
0, // tp_dict
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
0, // tp_init
PyType_GenericAlloc, // tp_alloc
PyType_GenericNew, // tp_new
PyObject_Del, // tp_free
};
google::protobuf::DescriptorPool* GetDescriptorPool() {
if (g_descriptor_pool == NULL) {
g_descriptor_pool = new google::protobuf::DescriptorPool(
google::protobuf::DescriptorPool::generated_pool());
}
return g_descriptor_pool;
}
PyObject* Python_NewCDescriptorPool(PyObject* ignored, PyObject* args) {
CDescriptorPool* cdescriptor_pool = PyObject_New(
CDescriptorPool, &CDescriptorPool_Type);
if (cdescriptor_pool == NULL) {
return NULL;
}
cdescriptor_pool->pool = GetDescriptorPool();
return reinterpret_cast<PyObject*>(cdescriptor_pool);
}
// Collects errors that occur during proto file building to allow them to be
// propagated in the python exception instead of only living in ERROR logs.
class BuildFileErrorCollector : public google::protobuf::DescriptorPool::ErrorCollector {
public:
BuildFileErrorCollector() : error_message(""), had_errors(false) {}
void AddError(const string& filename, const string& element_name,
const Message* descriptor, ErrorLocation location,
const string& message) {
// Replicates the logging behavior that happens in the C++ implementation
// when an error collector is not passed in.
if (!had_errors) {
error_message +=
("Invalid proto descriptor for file \"" + filename + "\":\n");
}
// As this only happens on failure and will result in the program not
// running at all, no effort is made to optimize this string manipulation.
error_message += (" " + element_name + ": " + message + "\n");
}
string error_message;
bool had_errors;
};
PyObject* Python_BuildFile(PyObject* ignored, PyObject* arg) {
char* message_type;
Py_ssize_t message_len;
if (PyBytes_AsStringAndSize(arg, &message_type, &message_len) < 0) {
return NULL;
}
google::protobuf::FileDescriptorProto file_proto;
if (!file_proto.ParseFromArray(message_type, message_len)) {
PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!");
return NULL;
}
if (google::protobuf::DescriptorPool::generated_pool()->FindFileByName(
file_proto.name()) != NULL) {
Py_RETURN_NONE;
}
BuildFileErrorCollector error_collector;
const google::protobuf::FileDescriptor* descriptor =
GetDescriptorPool()->BuildFileCollectingErrors(file_proto,
&error_collector);
if (descriptor == NULL) {
PyErr_Format(PyExc_TypeError,
"Couldn't build proto file into descriptor pool!\n%s",
error_collector.error_message.c_str());
return NULL;
}
Py_RETURN_NONE;
}
bool InitDescriptor() {
CFieldDescriptor_Type.tp_new = PyType_GenericNew;
if (PyType_Ready(&CFieldDescriptor_Type) < 0)
return false;
CDescriptorPool_Type.tp_new = PyType_GenericNew;
if (PyType_Ready(&CDescriptorPool_Type) < 0)
return false;
return true;
}
} // namespace python
} // namespace protobuf
} // namespace google

@ -0,0 +1,96 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: petar@google.com (Petar Petrov)
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__
#include <Python.h>
#include <structmember.h>
#include <google/protobuf/descriptor.h>
#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN)
typedef int Py_ssize_t;
#define PY_SSIZE_T_MAX INT_MAX
#define PY_SSIZE_T_MIN INT_MIN
#endif
namespace google {
namespace protobuf {
namespace python {
typedef struct CFieldDescriptor {
PyObject_HEAD
// The proto2 descriptor that this object represents.
const google::protobuf::FieldDescriptor* descriptor;
// Reference to the original field object in the Python DESCRIPTOR.
PyObject* descriptor_field;
} CFieldDescriptor;
typedef struct {
PyObject_HEAD
const google::protobuf::DescriptorPool* pool;
} CDescriptorPool;
extern PyTypeObject CFieldDescriptor_Type;
extern PyTypeObject CDescriptorPool_Type;
namespace cdescriptor_pool {
// Looks up a field by name. Returns a CDescriptor corresponding to
// the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindFieldByName(CDescriptorPool* self, PyObject* name);
// Looks up an extension by name. Returns a CDescriptor corresponding
// to the field on success, or NULL on failure.
//
// Returns a new reference.
PyObject* FindExtensionByName(CDescriptorPool* self, PyObject* arg);
} // namespace cdescriptor_pool
PyObject* Python_NewCDescriptorPool(PyObject* ignored, PyObject* args);
PyObject* Python_BuildFile(PyObject* ignored, PyObject* args);
bool InitDescriptor();
google::protobuf::DescriptorPool* GetDescriptorPool();
} // namespace python
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_CPP_DESCRIPTOR_H__

@ -0,0 +1,338 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: anuraag@google.com (Anuraag Agrawal)
// Author: tibell@google.com (Johan Tibell)
#include <google/protobuf/pyext/extension_dict.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/message.h>
#include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/repeated_composite_container.h>
#include <google/protobuf/pyext/repeated_scalar_container.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#include <google/protobuf/stubs/shared_ptr.h>
namespace google {
namespace protobuf {
namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace extension_dict {
// TODO(tibell): Always use self->message for clarity, just like in
// RepeatedCompositeContainer.
static google::protobuf::Message* GetMessage(ExtensionDict* self) {
if (self->parent != NULL) {
return self->parent->message;
} else {
return self->message;
}
}
CFieldDescriptor* InternalGetCDescriptorFromExtension(PyObject* extension) {
PyObject* cdescriptor = PyObject_GetAttrString(extension, "_cdescriptor");
if (cdescriptor == NULL) {
PyErr_SetString(PyExc_KeyError, "Unregistered extension.");
return NULL;
}
if (!PyObject_TypeCheck(cdescriptor, &CFieldDescriptor_Type)) {
PyErr_SetString(PyExc_TypeError, "Not a CFieldDescriptor");
Py_DECREF(cdescriptor);
return NULL;
}
CFieldDescriptor* descriptor =
reinterpret_cast<CFieldDescriptor*>(cdescriptor);
return descriptor;
}
PyObject* len(ExtensionDict* self) {
#if PY_MAJOR_VERSION >= 3
return PyLong_FromLong(PyDict_Size(self->values));
#else
return PyInt_FromLong(PyDict_Size(self->values));
#endif
}
// TODO(tibell): Use VisitCompositeField.
int ReleaseExtension(ExtensionDict* self,
PyObject* extension,
const google::protobuf::FieldDescriptor* descriptor) {
if (descriptor->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED) {
if (descriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
if (repeated_composite_container::Release(
reinterpret_cast<RepeatedCompositeContainer*>(
extension)) < 0) {
return -1;
}
} else {
if (repeated_scalar_container::Release(
reinterpret_cast<RepeatedScalarContainer*>(
extension)) < 0) {
return -1;
}
}
} else if (descriptor->cpp_type() ==
google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
if (cmessage::ReleaseSubMessage(
GetMessage(self), descriptor,
reinterpret_cast<CMessage*>(extension)) < 0) {
return -1;
}
}
return 0;
}
PyObject* subscript(ExtensionDict* self, PyObject* key) {
CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
key);
if (cdescriptor == NULL) {
return NULL;
}
ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor;
if (descriptor == NULL) {
return NULL;
}
if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
return cmessage::InternalGetScalar(self->parent, descriptor);
}
PyObject* value = PyDict_GetItem(self->values, key);
if (value != NULL) {
Py_INCREF(value);
return value;
}
if (descriptor->label() != FieldDescriptor::LABEL_REPEATED &&
descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
PyObject* sub_message = cmessage::InternalGetSubMessage(
self->parent, cdescriptor);
if (sub_message == NULL) {
return NULL;
}
PyDict_SetItem(self->values, key, sub_message);
return sub_message;
}
if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
// COPIED
PyObject* py_container = PyObject_CallObject(
reinterpret_cast<PyObject*>(&RepeatedCompositeContainer_Type),
NULL);
if (py_container == NULL) {
return NULL;
}
RepeatedCompositeContainer* container =
reinterpret_cast<RepeatedCompositeContainer*>(py_container);
PyObject* field = cdescriptor->descriptor_field;
PyObject* message_type = PyObject_GetAttrString(field, "message_type");
PyObject* concrete_class = PyObject_GetAttrString(message_type,
"_concrete_class");
container->owner = self->owner;
container->parent = self->parent;
container->message = self->parent->message;
container->parent_field = cdescriptor;
container->subclass_init = concrete_class;
Py_DECREF(message_type);
PyDict_SetItem(self->values, key, py_container);
return py_container;
} else {
// COPIED
ScopedPyObjectPtr init_args(PyTuple_Pack(2, self->parent, cdescriptor));
PyObject* py_container = PyObject_CallObject(
reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type),
init_args);
if (py_container == NULL) {
return NULL;
}
PyDict_SetItem(self->values, key, py_container);
return py_container;
}
}
PyErr_SetString(PyExc_ValueError, "control reached unexpected line");
return NULL;
}
int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value) {
CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
key);
if (cdescriptor == NULL) {
return -1;
}
ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
const google::protobuf::FieldDescriptor* descriptor = cdescriptor->descriptor;
if (descriptor->label() != FieldDescriptor::LABEL_OPTIONAL ||
descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
PyErr_SetString(PyExc_TypeError, "Extension is repeated and/or composite "
"type");
return -1;
}
cmessage::AssureWritable(self->parent);
if (cmessage::InternalSetScalar(self->parent, descriptor, value) < 0) {
return -1;
}
// TODO(tibell): We shouldn't write scalars to the cache.
PyDict_SetItem(self->values, key, value);
return 0;
}
PyObject* ClearExtension(ExtensionDict* self, PyObject* extension) {
CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
extension);
if (cdescriptor == NULL) {
return NULL;
}
ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
PyObject* value = PyDict_GetItem(self->values, extension);
if (value != NULL) {
if (ReleaseExtension(self, value, cdescriptor->descriptor) < 0) {
return NULL;
}
}
if (cmessage::ClearFieldByDescriptor(self->parent,
cdescriptor->descriptor) == NULL) {
return NULL;
}
if (PyDict_DelItem(self->values, extension) < 0) {
PyErr_Clear();
}
Py_RETURN_NONE;
}
PyObject* HasExtension(ExtensionDict* self, PyObject* extension) {
CFieldDescriptor* cdescriptor = InternalGetCDescriptorFromExtension(
extension);
if (cdescriptor == NULL) {
return NULL;
}
ScopedPyObjectPtr py_cdescriptor(reinterpret_cast<PyObject*>(cdescriptor));
PyObject* result = cmessage::HasFieldByDescriptor(
self->parent, cdescriptor->descriptor);
return result;
}
PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name) {
ScopedPyObjectPtr extensions_by_name(PyObject_GetAttrString(
reinterpret_cast<PyObject*>(self->parent), "_extensions_by_name"));
if (extensions_by_name == NULL) {
return NULL;
}
PyObject* result = PyDict_GetItem(extensions_by_name, name);
if (result == NULL) {
Py_RETURN_NONE;
} else {
Py_INCREF(result);
return result;
}
}
int init(ExtensionDict* self, PyObject* args, PyObject* kwargs) {
self->parent = NULL;
self->message = NULL;
self->values = PyDict_New();
return 0;
}
void dealloc(ExtensionDict* self) {
Py_CLEAR(self->values);
self->owner.reset();
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
static PyMappingMethods MpMethods = {
(lenfunc)len, /* mp_length */
(binaryfunc)subscript, /* mp_subscript */
(objobjargproc)ass_subscript,/* mp_ass_subscript */
};
#define EDMETHOD(name, args, doc) { #name, (PyCFunction)name, args, doc }
static PyMethodDef Methods[] = {
EDMETHOD(ClearExtension, METH_O, "Clears an extension from the object."),
EDMETHOD(HasExtension, METH_O, "Checks if the object has an extension."),
EDMETHOD(_FindExtensionByName, METH_O,
"Finds an extension by name."),
{ NULL, NULL }
};
} // namespace extension_dict
PyTypeObject ExtensionDict_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"google.protobuf.internal."
"cpp._message.ExtensionDict", // tp_name
sizeof(ExtensionDict), // tp_basicsize
0, // tp_itemsize
(destructor)extension_dict::dealloc, // tp_dealloc
0, // tp_print
0, // tp_getattr
0, // tp_setattr
0, // tp_compare
0, // tp_repr
0, // tp_as_number
0, // tp_as_sequence
&extension_dict::MpMethods, // tp_as_mapping
0, // tp_hash
0, // tp_call
0, // tp_str
0, // tp_getattro
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
"An extension dict", // tp_doc
0, // tp_traverse
0, // tp_clear
0, // tp_richcompare
0, // tp_weaklistoffset
0, // tp_iter
0, // tp_iternext
extension_dict::Methods, // tp_methods
0, // tp_members
0, // tp_getset
0, // tp_base
0, // tp_dict
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
(initproc)extension_dict::init, // tp_init
};
} // namespace python
} // namespace protobuf
} // namespace google

@ -0,0 +1,123 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: anuraag@google.com (Anuraag Agrawal)
// Author: tibell@google.com (Johan Tibell)
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__
#include <Python.h>
#include <memory>
#ifndef _SHARED_PTR_H
#include <google/protobuf/stubs/shared_ptr.h>
#endif
namespace google {
namespace protobuf {
class Message;
class FieldDescriptor;
using internal::shared_ptr;
namespace python {
struct CMessage;
struct CFieldDescriptor;
typedef struct ExtensionDict {
PyObject_HEAD;
shared_ptr<Message> owner;
CMessage* parent;
Message* message;
PyObject* values;
} ExtensionDict;
extern PyTypeObject ExtensionDict_Type;
namespace extension_dict {
// Gets the _cdescriptor reference to a CFieldDescriptor object given a
// python descriptor object.
//
// Returns a new reference.
CFieldDescriptor* InternalGetCDescriptorFromExtension(PyObject* extension);
// Gets the number of extension values in this ExtensionDict as a python object.
//
// Returns a new reference.
PyObject* len(ExtensionDict* self);
// Releases extensions referenced outside this dictionary to keep outside
// references alive.
//
// Returns 0 on success, -1 on failure.
int ReleaseExtension(ExtensionDict* self,
PyObject* extension,
const google::protobuf::FieldDescriptor* descriptor);
// Gets an extension from the dict for the given extension descriptor.
//
// Returns a new reference.
PyObject* subscript(ExtensionDict* self, PyObject* key);
// Assigns a value to an extension in the dict. Can only be used for singular
// simple types.
//
// Returns 0 on success, -1 on failure.
int ass_subscript(ExtensionDict* self, PyObject* key, PyObject* value);
// Clears an extension from the dict. Will release the extension if there
// is still an external reference left to it.
//
// Returns None on success.
PyObject* ClearExtension(ExtensionDict* self,
PyObject* extension);
// Checks if the dict has an extension.
//
// Returns a new python boolean reference.
PyObject* HasExtension(ExtensionDict* self, PyObject* extension);
// Gets an extension from the dict given the extension name as opposed to
// descriptor.
//
// Returns a new reference.
PyObject* _FindExtensionByName(ExtensionDict* self, PyObject* name);
} // namespace extension_dict
} // namespace python
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_CPP_EXTENSION_DICT_H__

File diff suppressed because it is too large Load Diff

@ -0,0 +1,305 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: anuraag@google.com (Anuraag Agrawal)
// Author: tibell@google.com (Johan Tibell)
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__
#include <Python.h>
#include <memory>
#ifndef _SHARED_PTR_H
#include <google/protobuf/stubs/shared_ptr.h>
#endif
#include <string>
namespace google {
namespace protobuf {
class Message;
class Reflection;
class FieldDescriptor;
using internal::shared_ptr;
namespace python {
struct CFieldDescriptor;
struct ExtensionDict;
typedef struct CMessage {
PyObject_HEAD;
// This is the top-level C++ Message object that owns the whole
// proto tree. Every Python CMessage holds a reference to it in
// order to keep it alive as long as there's a Python object that
// references any part of the tree.
shared_ptr<Message> owner;
// Weak reference to a parent CMessage object. This is NULL for any top-level
// message and is set for any child message (i.e. a child submessage or a
// part of a repeated composite field).
//
// Used to make sure all ancestors are also mutable when first modifying
// a child submessage (in other words, turning a default message instance
// into a mutable one).
//
// If a submessage is released (becomes a new top-level message), this field
// MUST be set to NULL. The parent may get deallocated and further attempts
// to use this pointer will result in a crash.
struct CMessage* parent;
// Weak reference to the parent's descriptor that describes this submessage.
// Used together with the parent's message when making a default message
// instance mutable.
// TODO(anuraag): With a bit of work on the Python/C++ layer, it should be
// possible to make this a direct pointer to a C++ FieldDescriptor, this would
// be easier if this implementation replaces upstream.
CFieldDescriptor* parent_field;
// Pointer to the C++ Message object for this CMessage. The
// CMessage does not own this pointer.
Message* message;
// Indicates this submessage is pointing to a default instance of a message.
// Submessages are always first created as read only messages and are then
// made writable, at which point this field is set to false.
bool read_only;
// A reference to a Python dictionary containing CMessage,
// RepeatedCompositeContainer, and RepeatedScalarContainer
// objects. Used as a cache to make sure we don't have to make a
// Python wrapper for the C++ Message objects on every access, or
// deal with the synchronization nightmare that could create.
PyObject* composite_fields;
// A reference to the dictionary containing the message's extensions.
// Similar to composite_fields, acting as a cache, but also contains the
// required extension dict logic.
ExtensionDict* extensions;
} CMessage;
extern PyTypeObject CMessage_Type;
namespace cmessage {
// Create a new empty message that can be populated by the parent.
PyObject* NewEmpty(PyObject* type);
// Release a submessage from its proto tree, making it a new top-level messgae.
// A new message will be created if this is a read-only default instance.
//
// Corresponds to reflection api method ReleaseMessage.
int ReleaseSubMessage(google::protobuf::Message* message,
const google::protobuf::FieldDescriptor* field_descriptor,
CMessage* child_cmessage);
// Initializes a new CMessage instance for a submessage. Only called once per
// submessage as the result is cached in composite_fields.
//
// Corresponds to reflection api method GetMessage.
PyObject* InternalGetSubMessage(CMessage* self,
CFieldDescriptor* cfield_descriptor);
// Deletes a range of C++ submessages in a repeated field (following a
// removal in a RepeatedCompositeContainer).
//
// Releases messages to the provided cmessage_list if it is not NULL rather
// than just removing them from the underlying proto. This cmessage_list must
// have a CMessage for each underlying submessage. The CMessages refered to
// by slice will be removed from cmessage_list by this function.
//
// Corresponds to reflection api method RemoveLast.
int InternalDeleteRepeatedField(google::protobuf::Message* message,
const google::protobuf::FieldDescriptor* field_descriptor,
PyObject* slice, PyObject* cmessage_list);
// Sets the specified scalar value to the message.
int InternalSetScalar(CMessage* self,
const google::protobuf::FieldDescriptor* field_descriptor,
PyObject* value);
// Retrieves the specified scalar value from the message.
//
// Returns a new python reference.
PyObject* InternalGetScalar(CMessage* self,
const google::protobuf::FieldDescriptor* field_descriptor);
// Clears the message, removing all contained data. Extension dictionary and
// submessages are released first if there are remaining external references.
//
// Corresponds to message api method Clear.
PyObject* Clear(CMessage* self);
// Clears the data described by the given descriptor. Used to clear extensions
// (which don't have names). Extension release is handled by ExtensionDict
// class, not this function.
// TODO(anuraag): Try to make this discrepancy in release semantics with
// ClearField less confusing.
//
// Corresponds to reflection api method ClearField.
PyObject* ClearFieldByDescriptor(
CMessage* self,
const google::protobuf::FieldDescriptor* descriptor);
// Clears the data for the given field name. The message is released if there
// are any external references.
//
// Corresponds to reflection api method ClearField.
PyObject* ClearField(CMessage* self, PyObject* arg);
// Checks if the message has the field described by the descriptor. Used for
// extensions (which have no name).
//
// Corresponds to reflection api method HasField
PyObject* HasFieldByDescriptor(
CMessage* self, const google::protobuf::FieldDescriptor* field_descriptor);
// Checks if the message has the named field.
//
// Corresponds to reflection api method HasField.
PyObject* HasField(CMessage* self, PyObject* arg);
// Initializes constants/enum values on a message. This is called by
// RepeatedCompositeContainer and ExtensionDict after calling the constructor.
// TODO(anuraag): Make it always called from within the constructor since it can
int InitAttributes(CMessage* self, PyObject* descriptor, PyObject* kwargs);
PyObject* MergeFrom(CMessage* self, PyObject* arg);
// Retrieves an attribute named 'name' from CMessage 'self'. Returns
// the attribute value on success, or NULL on failure.
//
// Returns a new reference.
PyObject* GetAttr(CMessage* self, PyObject* name);
// Set the value of the attribute named 'name', for CMessage 'self',
// to the value 'value'. Returns -1 on failure.
int SetAttr(CMessage* self, PyObject* name, PyObject* value);
PyObject* FindInitializationErrors(CMessage* self);
// Set the owner field of self and any children of self, recursively.
// Used when self is being released and thus has a new owner (the
// released Message.)
int SetOwner(CMessage* self, const shared_ptr<Message>& new_owner);
int AssureWritable(CMessage* self);
} // namespace cmessage
/* Is 64bit */
#define IS_64BIT (SIZEOF_LONG == 8)
#define FIELD_BELONGS_TO_MESSAGE(field_descriptor, message) \
((message)->GetDescriptor() == (field_descriptor)->containing_type())
#define FIELD_IS_REPEATED(field_descriptor) \
((field_descriptor)->label() == google::protobuf::FieldDescriptor::LABEL_REPEATED)
#define GOOGLE_CHECK_GET_INT32(arg, value, err) \
int32 value; \
if (!CheckAndGetInteger(arg, &value, kint32min_py, kint32max_py)) { \
return err; \
}
#define GOOGLE_CHECK_GET_INT64(arg, value, err) \
int64 value; \
if (!CheckAndGetInteger(arg, &value, kint64min_py, kint64max_py)) { \
return err; \
}
#define GOOGLE_CHECK_GET_UINT32(arg, value, err) \
uint32 value; \
if (!CheckAndGetInteger(arg, &value, kPythonZero, kuint32max_py)) { \
return err; \
}
#define GOOGLE_CHECK_GET_UINT64(arg, value, err) \
uint64 value; \
if (!CheckAndGetInteger(arg, &value, kPythonZero, kuint64max_py)) { \
return err; \
}
#define GOOGLE_CHECK_GET_FLOAT(arg, value, err) \
float value; \
if (!CheckAndGetFloat(arg, &value)) { \
return err; \
} \
#define GOOGLE_CHECK_GET_DOUBLE(arg, value, err) \
double value; \
if (!CheckAndGetDouble(arg, &value)) { \
return err; \
}
#define GOOGLE_CHECK_GET_BOOL(arg, value, err) \
bool value; \
if (!CheckAndGetBool(arg, &value)) { \
return err; \
}
extern PyObject* kPythonZero;
extern PyObject* kint32min_py;
extern PyObject* kint32max_py;
extern PyObject* kuint32max_py;
extern PyObject* kint64min_py;
extern PyObject* kint64max_py;
extern PyObject* kuint64max_py;
#define C(str) const_cast<char*>(str)
void FormatTypeError(PyObject* arg, char* expected_types);
template<class T>
bool CheckAndGetInteger(
PyObject* arg, T* value, PyObject* min, PyObject* max);
bool CheckAndGetDouble(PyObject* arg, double* value);
bool CheckAndGetFloat(PyObject* arg, float* value);
bool CheckAndGetBool(PyObject* arg, bool* value);
bool CheckAndSetString(
PyObject* arg, google::protobuf::Message* message,
const google::protobuf::FieldDescriptor* descriptor,
const google::protobuf::Reflection* reflection,
bool append,
int index);
PyObject* ToStringObject(
const google::protobuf::FieldDescriptor* descriptor, string value);
extern PyObject* PickleError_class;
} // namespace python
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_CPP_MESSAGE_H__

@ -0,0 +1,38 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import "google/protobuf/internal/cpp/proto1_api_test.proto";
package google.protobuf.python.internal;
message TestNestedProto1APIMessage {
optional int32 a = 1;
optional TestMessage.NestedMessage b = 2;
}

@ -0,0 +1,66 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: tibell@google.com (Johan Tibell)
//
// These message definitions are used to exercises known corner cases
// in the C++ implementation of the Python API.
package google.protobuf.python.internal;
// Protos optimized for SPEED use a strict superset of the generated code
// of equivalent ones optimized for CODE_SIZE, so we should optimize all our
// tests for speed unless explicitly testing code size optimization.
option optimize_for = SPEED;
message TestAllTypes {
message NestedMessage {
optional int32 bb = 1;
optional ForeignMessage cc = 2;
}
repeated NestedMessage repeated_nested_message = 1;
optional NestedMessage optional_nested_message = 2;
optional int32 optional_int32 = 3;
}
message ForeignMessage {
optional int32 c = 1;
repeated int32 d = 2;
}
message TestAllExtensions {
extensions 1 to max;
}
extend TestAllExtensions {
optional TestAllTypes.NestedMessage optional_nested_message_extension = 1;
}

@ -0,0 +1,57 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: qrczak@google.com (Marcin Kowalczyk)
//
// This module exposes the C proto inside the given Python proto, in
// case the Python proto is implemented with a C proto.
#ifndef GOOGLE_PROTOBUF_PYTHON_PYTHON_PROTOBUF_H__
#define GOOGLE_PROTOBUF_PYTHON_PYTHON_PROTOBUF_H__
#include <Python.h>
namespace google {
namespace protobuf {
class Message;
namespace python {
// Return the pointer to the C proto inside the given Python proto,
// or NULL when this is not a Python proto implemented with a C proto.
const Message* GetCProtoInsidePyProto(PyObject* msg);
Message* MutableCProtoInsidePyProto(PyObject* msg);
} // namespace python
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_PYTHON_PROTOBUF_H__

@ -0,0 +1,763 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: anuraag@google.com (Anuraag Agrawal)
// Author: tibell@google.com (Johan Tibell)
#include <google/protobuf/pyext/repeated_composite_container.h>
#include <memory>
#ifndef _SHARED_PTR_H
#include <google/protobuf/stubs/shared_ptr.h>
#endif
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/message.h>
#include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#if PY_MAJOR_VERSION >= 3
#define PyInt_Check PyLong_Check
#define PyInt_AsLong PyLong_AsLong
#define PyInt_FromLong PyLong_FromLong
#endif
namespace google {
namespace protobuf {
namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace repeated_composite_container {
// TODO(tibell): We might also want to check:
// GOOGLE_CHECK_NOTNULL((self)->owner.get());
#define GOOGLE_CHECK_ATTACHED(self) \
do { \
GOOGLE_CHECK_NOTNULL((self)->message); \
GOOGLE_CHECK_NOTNULL((self)->parent_field); \
} while (0);
#define GOOGLE_CHECK_RELEASED(self) \
do { \
GOOGLE_CHECK((self)->owner.get() == NULL); \
GOOGLE_CHECK((self)->message == NULL); \
GOOGLE_CHECK((self)->parent_field == NULL); \
GOOGLE_CHECK((self)->parent == NULL); \
} while (0);
// Returns a new reference.
static PyObject* GetKey(PyObject* x) {
// Just the identity function.
Py_INCREF(x);
return x;
}
#define GET_KEY(keyfunc, value) \
((keyfunc) == NULL ? \
GetKey((value)) : \
PyObject_CallFunctionObjArgs((keyfunc), (value), NULL))
// Converts a comparison function that returns -1, 0, or 1 into a
// less-than predicate.
//
// Returns -1 on error, 1 if x < y, 0 if x >= y.
static int islt(PyObject *x, PyObject *y, PyObject *compare) {
if (compare == NULL)
return PyObject_RichCompareBool(x, y, Py_LT);
ScopedPyObjectPtr res(PyObject_CallFunctionObjArgs(compare, x, y, NULL));
if (res == NULL)
return -1;
if (!PyInt_Check(res)) {
PyErr_Format(PyExc_TypeError,
"comparison function must return int, not %.200s",
Py_TYPE(res)->tp_name);
return -1;
}
return PyInt_AsLong(res) < 0;
}
// Copied from uarrsort.c but swaps memcpy swaps with protobuf/python swaps
// TODO(anuraag): Is there a better way to do this then reinventing the wheel?
static int InternalQuickSort(RepeatedCompositeContainer* self,
Py_ssize_t start,
Py_ssize_t limit,
PyObject* cmp,
PyObject* keyfunc) {
if (limit - start <= 1)
return 0; // Nothing to sort.
GOOGLE_CHECK_ATTACHED(self);
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
const google::protobuf::FieldDescriptor* descriptor = self->parent_field->descriptor;
Py_ssize_t left;
Py_ssize_t right;
PyObject* children = self->child_messages;
do {
left = start;
right = limit;
ScopedPyObjectPtr mid(
GET_KEY(keyfunc, PyList_GET_ITEM(children, (start + limit) / 2)));
do {
ScopedPyObjectPtr key(GET_KEY(keyfunc, PyList_GET_ITEM(children, left)));
int is_lt = islt(key, mid, cmp);
if (is_lt == -1)
return -1;
/* array[left]<x */
while (is_lt) {
++left;
ScopedPyObjectPtr key(GET_KEY(keyfunc,
PyList_GET_ITEM(children, left)));
is_lt = islt(key, mid, cmp);
if (is_lt == -1)
return -1;
}
key.reset(GET_KEY(keyfunc, PyList_GET_ITEM(children, right - 1)));
is_lt = islt(mid, key, cmp);
if (is_lt == -1)
return -1;
while (is_lt) {
--right;
ScopedPyObjectPtr key(GET_KEY(keyfunc,
PyList_GET_ITEM(children, right - 1)));
is_lt = islt(mid, key, cmp);
if (is_lt == -1)
return -1;
}
if (left < right) {
--right;
if (left < right) {
reflection->SwapElements(message, descriptor, left, right);
PyObject* tmp = PyList_GET_ITEM(children, left);
PyList_SET_ITEM(children, left, PyList_GET_ITEM(children, right));
PyList_SET_ITEM(children, right, tmp);
}
++left;
}
} while (left < right);
if ((right - start) < (limit - left)) {
/* sort [start..right[ */
if (start < (right - 1)) {
InternalQuickSort(self, start, right, cmp, keyfunc);
}
/* sort [left..limit[ */
start = left;
} else {
/* sort [left..limit[ */
if (left < (limit - 1)) {
InternalQuickSort(self, left, limit, cmp, keyfunc);
}
/* sort [start..right[ */
limit = right;
}
} while (start < (limit - 1));
return 0;
}
#undef GET_KEY
// ---------------------------------------------------------------------
// len()
static Py_ssize_t Length(RepeatedCompositeContainer* self) {
google::protobuf::Message* message = self->message;
if (message != NULL) {
return message->GetReflection()->FieldSize(*message,
self->parent_field->descriptor);
} else {
// The container has been released (i.e. by a call to Clear() or
// ClearField() on the parent) and thus there's no message.
return PyList_GET_SIZE(self->child_messages);
}
}
// Returns 0 if successful; returns -1 and sets an exception if
// unsuccessful.
static int UpdateChildMessages(RepeatedCompositeContainer* self) {
if (self->message == NULL)
return 0;
// A MergeFrom on a parent message could have caused extra messages to be
// added in the underlying protobuf so add them to our list. They can never
// be removed in such a way so there's no need to worry about that.
Py_ssize_t message_length = Length(self);
Py_ssize_t child_length = PyList_GET_SIZE(self->child_messages);
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
for (Py_ssize_t i = child_length; i < message_length; ++i) {
const Message& sub_message = reflection->GetRepeatedMessage(
*(self->message), self->parent_field->descriptor, i);
ScopedPyObjectPtr py_cmsg(cmessage::NewEmpty(self->subclass_init));
if (py_cmsg == NULL) {
return -1;
}
CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg.get());
cmsg->owner = self->owner;
cmsg->message = const_cast<google::protobuf::Message*>(&sub_message);
cmsg->parent = self->parent;
if (cmessage::InitAttributes(cmsg, NULL, NULL) < 0) {
return -1;
}
PyList_Append(self->child_messages, py_cmsg);
}
return 0;
}
// ---------------------------------------------------------------------
// add()
static PyObject* AddToAttached(RepeatedCompositeContainer* self,
PyObject* args,
PyObject* kwargs) {
GOOGLE_CHECK_ATTACHED(self);
if (UpdateChildMessages(self) < 0) {
return NULL;
}
if (cmessage::AssureWritable(self->parent) == -1)
return NULL;
google::protobuf::Message* message = self->message;
google::protobuf::Message* sub_message =
message->GetReflection()->AddMessage(message,
self->parent_field->descriptor);
PyObject* py_cmsg = cmessage::NewEmpty(self->subclass_init);
if (py_cmsg == NULL) {
return NULL;
}
CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
cmsg->owner = self->owner;
cmsg->message = sub_message;
cmsg->parent = self->parent;
// cmessage::InitAttributes must be called after cmsg->message has
// been set.
if (cmessage::InitAttributes(cmsg, NULL, kwargs) < 0) {
Py_DECREF(py_cmsg);
return NULL;
}
PyList_Append(self->child_messages, py_cmsg);
return py_cmsg;
}
static PyObject* AddToReleased(RepeatedCompositeContainer* self,
PyObject* args,
PyObject* kwargs) {
GOOGLE_CHECK_RELEASED(self);
// Create the CMessage
PyObject* py_cmsg = PyObject_CallObject(self->subclass_init, NULL);
if (py_cmsg == NULL)
return NULL;
CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
if (cmessage::InitAttributes(cmsg, NULL, kwargs) < 0) {
Py_DECREF(py_cmsg);
return NULL;
}
// The Message got created by the call to subclass_init above and
// it set self->owner to the newly allocated message.
PyList_Append(self->child_messages, py_cmsg);
return py_cmsg;
}
PyObject* Add(RepeatedCompositeContainer* self,
PyObject* args,
PyObject* kwargs) {
if (self->message == NULL)
return AddToReleased(self, args, kwargs);
else
return AddToAttached(self, args, kwargs);
}
// ---------------------------------------------------------------------
// extend()
PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value) {
cmessage::AssureWritable(self->parent);
if (UpdateChildMessages(self) < 0) {
return NULL;
}
ScopedPyObjectPtr iter(PyObject_GetIter(value));
if (iter == NULL) {
PyErr_SetString(PyExc_TypeError, "Value must be iterable");
return NULL;
}
ScopedPyObjectPtr next;
while ((next.reset(PyIter_Next(iter))) != NULL) {
if (!PyObject_TypeCheck(next, &CMessage_Type)) {
PyErr_SetString(PyExc_TypeError, "Not a cmessage");
return NULL;
}
ScopedPyObjectPtr new_message(Add(self, NULL, NULL));
if (new_message == NULL) {
return NULL;
}
CMessage* new_cmessage = reinterpret_cast<CMessage*>(new_message.get());
if (cmessage::MergeFrom(new_cmessage, next) == NULL) {
return NULL;
}
}
if (PyErr_Occurred()) {
return NULL;
}
Py_RETURN_NONE;
}
PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other) {
if (UpdateChildMessages(self) < 0) {
return NULL;
}
return Extend(self, other);
}
PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice) {
if (UpdateChildMessages(self) < 0) {
return NULL;
}
Py_ssize_t from;
Py_ssize_t to;
Py_ssize_t step;
Py_ssize_t length = Length(self);
Py_ssize_t slicelength;
if (PySlice_Check(slice)) {
#if PY_MAJOR_VERSION >= 3
if (PySlice_GetIndicesEx(slice,
#else
if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice),
#endif
length, &from, &to, &step, &slicelength) == -1) {
return NULL;
}
return PyList_GetSlice(self->child_messages, from, to);
} else if (PyInt_Check(slice) || PyLong_Check(slice)) {
from = to = PyLong_AsLong(slice);
if (from < 0) {
from = to = length + from;
}
PyObject* result = PyList_GetItem(self->child_messages, from);
if (result == NULL) {
return NULL;
}
Py_INCREF(result);
return result;
}
PyErr_SetString(PyExc_TypeError, "index must be an integer or slice");
return NULL;
}
int AssignSubscript(RepeatedCompositeContainer* self,
PyObject* slice,
PyObject* value) {
if (UpdateChildMessages(self) < 0) {
return -1;
}
if (value != NULL) {
PyErr_SetString(PyExc_TypeError, "does not support assignment");
return -1;
}
// Delete from the underlying Message, if any.
if (self->message != NULL) {
if (cmessage::InternalDeleteRepeatedField(self->message,
self->parent_field->descriptor,
slice,
self->child_messages) < 0) {
return -1;
}
} else {
Py_ssize_t from;
Py_ssize_t to;
Py_ssize_t step;
Py_ssize_t length = Length(self);
Py_ssize_t slicelength;
if (PySlice_Check(slice)) {
#if PY_MAJOR_VERSION >= 3
if (PySlice_GetIndicesEx(slice,
#else
if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice),
#endif
length, &from, &to, &step, &slicelength) == -1) {
return -1;
}
return PySequence_DelSlice(self->child_messages, from, to);
} else if (PyInt_Check(slice) || PyLong_Check(slice)) {
from = to = PyLong_AsLong(slice);
if (from < 0) {
from = to = length + from;
}
return PySequence_DelItem(self->child_messages, from);
}
}
return 0;
}
static PyObject* Remove(RepeatedCompositeContainer* self, PyObject* value) {
if (UpdateChildMessages(self) < 0) {
return NULL;
}
Py_ssize_t index = PySequence_Index(self->child_messages, value);
if (index == -1) {
return NULL;
}
ScopedPyObjectPtr py_index(PyLong_FromLong(index));
if (AssignSubscript(self, py_index, NULL) < 0) {
return NULL;
}
Py_RETURN_NONE;
}
static PyObject* RichCompare(RepeatedCompositeContainer* self,
PyObject* other,
int opid) {
if (UpdateChildMessages(self) < 0) {
return NULL;
}
if (!PyObject_TypeCheck(other, &RepeatedCompositeContainer_Type)) {
PyErr_SetString(PyExc_TypeError,
"Can only compare repeated composite fields "
"against other repeated composite fields.");
return NULL;
}
if (opid == Py_EQ || opid == Py_NE) {
// TODO(anuraag): Don't make new lists just for this...
ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
if (full_slice == NULL) {
return NULL;
}
ScopedPyObjectPtr list(Subscript(self, full_slice));
if (list == NULL) {
return NULL;
}
ScopedPyObjectPtr other_list(
Subscript(
reinterpret_cast<RepeatedCompositeContainer*>(other), full_slice));
if (other_list == NULL) {
return NULL;
}
return PyObject_RichCompare(list, other_list, opid);
} else {
Py_INCREF(Py_NotImplemented);
return Py_NotImplemented;
}
}
// ---------------------------------------------------------------------
// sort()
static PyObject* SortAttached(RepeatedCompositeContainer* self,
PyObject* args,
PyObject* kwds) {
// Sort the underlying Message array.
PyObject *compare = NULL;
int reverse = 0;
PyObject *keyfunc = NULL;
static char *kwlist[] = {"cmp", "key", "reverse", 0};
if (args != NULL) {
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi:sort",
kwlist, &compare, &keyfunc, &reverse))
return NULL;
}
if (compare == Py_None)
compare = NULL;
if (keyfunc == Py_None)
keyfunc = NULL;
const Py_ssize_t length = Length(self);
if (InternalQuickSort(self, 0, length, compare, keyfunc) < 0)
return NULL;
// Finally reverse the result if requested.
if (reverse) {
google::protobuf::Message* message = self->message;
const google::protobuf::Reflection* reflection = message->GetReflection();
const google::protobuf::FieldDescriptor* descriptor = self->parent_field->descriptor;
// Reverse the Message array.
for (int i = 0; i < length / 2; ++i)
reflection->SwapElements(message, descriptor, i, length - i - 1);
// Reverse the Python list.
ScopedPyObjectPtr res(PyObject_CallMethod(self->child_messages,
"reverse", NULL));
if (res == NULL)
return NULL;
}
Py_RETURN_NONE;
}
static PyObject* SortReleased(RepeatedCompositeContainer* self,
PyObject* args,
PyObject* kwds) {
ScopedPyObjectPtr m(PyObject_GetAttrString(self->child_messages, "sort"));
if (m == NULL)
return NULL;
if (PyObject_Call(m, args, kwds) == NULL)
return NULL;
Py_RETURN_NONE;
}
static PyObject* Sort(RepeatedCompositeContainer* self,
PyObject* args,
PyObject* kwds) {
// Support the old sort_function argument for backwards
// compatibility.
if (kwds != NULL) {
PyObject* sort_func = PyDict_GetItemString(kwds, "sort_function");
if (sort_func != NULL) {
// Must set before deleting as sort_func is a borrowed reference
// and kwds might be the only thing keeping it alive.
PyDict_SetItemString(kwds, "cmp", sort_func);
PyDict_DelItemString(kwds, "sort_function");
}
}
if (UpdateChildMessages(self) < 0)
return NULL;
if (self->message == NULL) {
return SortReleased(self, args, kwds);
} else {
return SortAttached(self, args, kwds);
}
}
// ---------------------------------------------------------------------
static PyObject* Item(RepeatedCompositeContainer* self, Py_ssize_t index) {
if (UpdateChildMessages(self) < 0) {
return NULL;
}
Py_ssize_t length = Length(self);
if (index < 0) {
index = length + index;
}
PyObject* item = PyList_GetItem(self->child_messages, index);
if (item == NULL) {
return NULL;
}
Py_INCREF(item);
return item;
}
// The caller takes ownership of the returned Message.
Message* ReleaseLast(const FieldDescriptor* field,
const Descriptor* type,
Message* message) {
GOOGLE_CHECK_NOTNULL(field);
GOOGLE_CHECK_NOTNULL(type);
GOOGLE_CHECK_NOTNULL(message);
Message* released_message = message->GetReflection()->ReleaseLast(
message, field);
// TODO(tibell): Deal with proto1.
// ReleaseMessage will return NULL which differs from
// child_cmessage->message, if the field does not exist. In this case,
// the latter points to the default instance via a const_cast<>, so we
// have to reset it to a new mutable object since we are taking ownership.
if (released_message == NULL) {
const Message* prototype = global_message_factory->GetPrototype(type);
GOOGLE_CHECK_NOTNULL(prototype);
return prototype->New();
} else {
return released_message;
}
}
// Release field of message and transfer the ownership to cmessage.
void ReleaseLastTo(const FieldDescriptor* field,
Message* message,
CMessage* cmessage) {
GOOGLE_CHECK_NOTNULL(field);
GOOGLE_CHECK_NOTNULL(message);
GOOGLE_CHECK_NOTNULL(cmessage);
shared_ptr<Message> released_message(
ReleaseLast(field, cmessage->message->GetDescriptor(), message));
cmessage->parent = NULL;
cmessage->parent_field = NULL;
cmessage->message = released_message.get();
cmessage->read_only = false;
cmessage::SetOwner(cmessage, released_message);
}
// Called to release a container using
// ClearField('container_field_name') on the parent.
int Release(RepeatedCompositeContainer* self) {
if (UpdateChildMessages(self) < 0) {
PyErr_WriteUnraisable(PyBytes_FromString("Failed to update released "
"messages"));
return -1;
}
Message* message = self->message;
const FieldDescriptor* field = self->parent_field->descriptor;
// The reflection API only lets us release the last message in a
// repeated field. Therefore we iterate through the children
// starting with the last one.
const Py_ssize_t size = PyList_GET_SIZE(self->child_messages);
GOOGLE_DCHECK_EQ(size, message->GetReflection()->FieldSize(*message, field));
for (Py_ssize_t i = size - 1; i >= 0; --i) {
CMessage* child_cmessage = reinterpret_cast<CMessage*>(
PyList_GET_ITEM(self->child_messages, i));
ReleaseLastTo(field, message, child_cmessage);
}
// Detach from containing message.
self->parent = NULL;
self->parent_field = NULL;
self->message = NULL;
self->owner.reset();
return 0;
}
int SetOwner(RepeatedCompositeContainer* self,
const shared_ptr<Message>& new_owner) {
GOOGLE_CHECK_ATTACHED(self);
self->owner = new_owner;
const Py_ssize_t n = PyList_GET_SIZE(self->child_messages);
for (Py_ssize_t i = 0; i < n; ++i) {
PyObject* msg = PyList_GET_ITEM(self->child_messages, i);
if (cmessage::SetOwner(reinterpret_cast<CMessage*>(msg), new_owner) == -1) {
return -1;
}
}
return 0;
}
static int Init(RepeatedCompositeContainer* self,
PyObject* args,
PyObject* kwargs) {
self->message = NULL;
self->parent = NULL;
self->parent_field = NULL;
self->subclass_init = NULL;
self->child_messages = PyList_New(0);
return 0;
}
static void Dealloc(RepeatedCompositeContainer* self) {
Py_CLEAR(self->child_messages);
// TODO(tibell): Do we need to call delete on these objects to make
// sure their destructors are called?
self->owner.reset();
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
static PySequenceMethods SqMethods = {
(lenfunc)Length, /* sq_length */
0, /* sq_concat */
0, /* sq_repeat */
(ssizeargfunc)Item /* sq_item */
};
static PyMappingMethods MpMethods = {
(lenfunc)Length, /* mp_length */
(binaryfunc)Subscript, /* mp_subscript */
(objobjargproc)AssignSubscript,/* mp_ass_subscript */
};
static PyMethodDef Methods[] = {
{ "add", (PyCFunction) Add, METH_VARARGS | METH_KEYWORDS,
"Adds an object to the repeated container." },
{ "extend", (PyCFunction) Extend, METH_O,
"Adds objects to the repeated container." },
{ "remove", (PyCFunction) Remove, METH_O,
"Removes an object from the repeated container." },
{ "sort", (PyCFunction) Sort, METH_VARARGS | METH_KEYWORDS,
"Sorts the repeated container." },
{ "MergeFrom", (PyCFunction) MergeFrom, METH_O,
"Adds objects to the repeated container." },
{ NULL, NULL }
};
} // namespace repeated_composite_container
PyTypeObject RepeatedCompositeContainer_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"google.protobuf.internal."
"cpp._message.RepeatedCompositeContainer", // tp_name
sizeof(RepeatedCompositeContainer), // tp_basicsize
0, // tp_itemsize
(destructor)repeated_composite_container::Dealloc, // tp_dealloc
0, // tp_print
0, // tp_getattr
0, // tp_setattr
0, // tp_compare
0, // tp_repr
0, // tp_as_number
&repeated_composite_container::SqMethods, // tp_as_sequence
&repeated_composite_container::MpMethods, // tp_as_mapping
0, // tp_hash
0, // tp_call
0, // tp_str
0, // tp_getattro
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
"A Repeated scalar container", // tp_doc
0, // tp_traverse
0, // tp_clear
(richcmpfunc)repeated_composite_container::RichCompare, // tp_richcompare
0, // tp_weaklistoffset
0, // tp_iter
0, // tp_iternext
repeated_composite_container::Methods, // tp_methods
0, // tp_members
0, // tp_getset
0, // tp_base
0, // tp_dict
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
(initproc)repeated_composite_container::Init, // tp_init
};
} // namespace python
} // namespace protobuf
} // namespace google

@ -0,0 +1,172 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: anuraag@google.com (Anuraag Agrawal)
// Author: tibell@google.com (Johan Tibell)
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__
#include <Python.h>
#include <memory>
#ifndef _SHARED_PTR_H
#include <google/protobuf/stubs/shared_ptr.h>
#endif
#include <string>
#include <vector>
namespace google {
namespace protobuf {
class FieldDescriptor;
class Message;
using internal::shared_ptr;
namespace python {
struct CMessage;
struct CFieldDescriptor;
// A RepeatedCompositeContainer can be in one of two states: attached
// or released.
//
// When in the attached state all modifications to the container are
// done both on the 'message' and on the 'child_messages'
// list. In this state all Messages refered to by the children in
// 'child_messages' are owner by the 'owner'.
//
// When in the released state 'message', 'owner', 'parent', and
// 'parent_field' are NULL.
typedef struct RepeatedCompositeContainer {
PyObject_HEAD;
// This is the top-level C++ Message object that owns the whole
// proto tree. Every Python RepeatedCompositeContainer holds a
// reference to it in order to keep it alive as long as there's a
// Python object that references any part of the tree.
shared_ptr<Message> owner;
// Weak reference to parent object. May be NULL. Used to make sure
// the parent is writable before modifying the
// RepeatedCompositeContainer.
CMessage* parent;
// A descriptor used to modify the underlying 'message'.
CFieldDescriptor* parent_field;
// Pointer to the C++ Message that contains this container. The
// RepeatedCompositeContainer does not own this pointer.
//
// If NULL, this message has been released from its parent (by
// calling Clear() or ClearField() on the parent.
Message* message;
// A callable that is used to create new child messages.
PyObject* subclass_init;
// A list of child messages.
PyObject* child_messages;
} RepeatedCompositeContainer;
extern PyTypeObject RepeatedCompositeContainer_Type;
namespace repeated_composite_container {
// Returns the number of items in this repeated composite container.
static Py_ssize_t Length(RepeatedCompositeContainer* self);
// Appends a new CMessage to the container and returns it. The
// CMessage is initialized using the content of kwargs.
//
// Returns a new reference if successful; returns NULL and sets an
// exception if unsuccessful.
PyObject* Add(RepeatedCompositeContainer* self,
PyObject* args,
PyObject* kwargs);
// Appends all the CMessages in the input iterator to the container.
//
// Returns None if successful; returns NULL and sets an exception if
// unsuccessful.
PyObject* Extend(RepeatedCompositeContainer* self, PyObject* value);
// Appends a new message to the container for each message in the
// input iterator, merging each data element in. Equivalent to extend.
//
// Returns None if successful; returns NULL and sets an exception if
// unsuccessful.
PyObject* MergeFrom(RepeatedCompositeContainer* self, PyObject* other);
// Accesses messages in the container.
//
// Returns a new reference to the message for an integer parameter.
// Returns a new reference to a list of messages for a slice.
PyObject* Subscript(RepeatedCompositeContainer* self, PyObject* slice);
// Deletes items from the container (cannot be used for assignment).
//
// Returns 0 on success, -1 on failure.
int AssignSubscript(RepeatedCompositeContainer* self,
PyObject* slice,
PyObject* value);
// Releases the messages in the container to the given message.
//
// Returns 0 on success, -1 on failure.
int ReleaseToMessage(RepeatedCompositeContainer* self,
google::protobuf::Message* new_message);
// Releases the messages in the container to a new message.
//
// Returns 0 on success, -1 on failure.
int Release(RepeatedCompositeContainer* self);
// Returns 0 on success, -1 on failure.
int SetOwner(RepeatedCompositeContainer* self,
const shared_ptr<Message>& new_owner);
// Removes the last element of the repeated message field 'field' on
// the Message 'message', and transfers the ownership of the released
// Message to 'cmessage'.
//
// Corresponds to reflection api method ReleaseMessage.
void ReleaseLastTo(const FieldDescriptor* field,
Message* message,
CMessage* cmessage);
} // namespace repeated_composite_container
} // namespace python
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_COMPOSITE_CONTAINER_H__

@ -0,0 +1,825 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: anuraag@google.com (Anuraag Agrawal)
// Author: tibell@google.com (Johan Tibell)
#include <google/protobuf/pyext/repeated_scalar_container.h>
#include <memory>
#ifndef _SHARED_PTR_H
#include <google/protobuf/stubs/shared_ptr.h>
#endif
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/message.h>
#include <google/protobuf/pyext/descriptor.h>
#include <google/protobuf/pyext/message.h>
#include <google/protobuf/pyext/scoped_pyobject_ptr.h>
#if PY_MAJOR_VERSION >= 3
#define PyInt_FromLong PyLong_FromLong
#if PY_VERSION_HEX < 0x03030000
#error "Python 3.0 - 3.2 are not supported."
#else
#define PyString_AsString(ob) \
(PyUnicode_Check(ob)? PyUnicode_AsUTF8(ob): PyBytes_AS_STRING(ob))
#endif
#endif
namespace google {
namespace protobuf {
namespace python {
extern google::protobuf::DynamicMessageFactory* global_message_factory;
namespace repeated_scalar_container {
static int InternalAssignRepeatedField(
RepeatedScalarContainer* self, PyObject* list) {
self->message->GetReflection()->ClearField(self->message,
self->parent_field->descriptor);
for (Py_ssize_t i = 0; i < PyList_GET_SIZE(list); ++i) {
PyObject* value = PyList_GET_ITEM(list, i);
if (Append(self, value) == NULL) {
return -1;
}
}
return 0;
}
static Py_ssize_t Len(RepeatedScalarContainer* self) {
google::protobuf::Message* message = self->message;
return message->GetReflection()->FieldSize(*message,
self->parent_field->descriptor);
}
static int AssignItem(RepeatedScalarContainer* self,
Py_ssize_t index,
PyObject* arg) {
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
self->parent_field->descriptor;
if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
PyErr_SetString(
PyExc_KeyError, "Field does not belong to message!");
return -1;
}
const google::protobuf::Reflection* reflection = message->GetReflection();
int field_size = reflection->FieldSize(*message, field_descriptor);
if (index < 0) {
index = field_size + index;
}
if (index < 0 || index >= field_size) {
PyErr_Format(PyExc_IndexError,
"list assignment index (%d) out of range",
static_cast<int>(index));
return -1;
}
if (arg == NULL) {
ScopedPyObjectPtr py_index(PyLong_FromLong(index));
return cmessage::InternalDeleteRepeatedField(message, field_descriptor,
py_index, NULL);
}
if (PySequence_Check(arg) && !(PyBytes_Check(arg) || PyUnicode_Check(arg))) {
PyErr_SetString(PyExc_TypeError, "Value must be scalar");
return -1;
}
switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: {
GOOGLE_CHECK_GET_INT32(arg, value, -1);
reflection->SetRepeatedInt32(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: {
GOOGLE_CHECK_GET_INT64(arg, value, -1);
reflection->SetRepeatedInt64(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: {
GOOGLE_CHECK_GET_UINT32(arg, value, -1);
reflection->SetRepeatedUInt32(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: {
GOOGLE_CHECK_GET_UINT64(arg, value, -1);
reflection->SetRepeatedUInt64(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: {
GOOGLE_CHECK_GET_FLOAT(arg, value, -1);
reflection->SetRepeatedFloat(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: {
GOOGLE_CHECK_GET_DOUBLE(arg, value, -1);
reflection->SetRepeatedDouble(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: {
GOOGLE_CHECK_GET_BOOL(arg, value, -1);
reflection->SetRepeatedBool(message, field_descriptor, index, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: {
if (!CheckAndSetString(
arg, message, field_descriptor, reflection, false, index)) {
return -1;
}
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: {
GOOGLE_CHECK_GET_INT32(arg, value, -1);
const google::protobuf::EnumDescriptor* enum_descriptor =
field_descriptor->enum_type();
const google::protobuf::EnumValueDescriptor* enum_value =
enum_descriptor->FindValueByNumber(value);
if (enum_value != NULL) {
reflection->SetRepeatedEnum(message, field_descriptor, index,
enum_value);
} else {
ScopedPyObjectPtr s(PyObject_Str(arg));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
PyString_AsString(s.get()));
}
return -1;
}
break;
}
default:
PyErr_Format(
PyExc_SystemError, "Adding value to a field of unknown type %d",
field_descriptor->cpp_type());
return -1;
}
return 0;
}
static PyObject* Item(RepeatedScalarContainer* self, Py_ssize_t index) {
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
self->parent_field->descriptor;
const google::protobuf::Reflection* reflection = message->GetReflection();
int field_size = reflection->FieldSize(*message, field_descriptor);
if (index < 0) {
index = field_size + index;
}
if (index < 0 || index >= field_size) {
PyErr_Format(PyExc_IndexError,
"list assignment index (%d) out of range",
static_cast<int>(index));
return NULL;
}
PyObject* result = NULL;
switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: {
int32 value = reflection->GetRepeatedInt32(
*message, field_descriptor, index);
result = PyInt_FromLong(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: {
int64 value = reflection->GetRepeatedInt64(
*message, field_descriptor, index);
result = PyLong_FromLongLong(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: {
uint32 value = reflection->GetRepeatedUInt32(
*message, field_descriptor, index);
result = PyLong_FromLongLong(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: {
uint64 value = reflection->GetRepeatedUInt64(
*message, field_descriptor, index);
result = PyLong_FromUnsignedLongLong(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: {
float value = reflection->GetRepeatedFloat(
*message, field_descriptor, index);
result = PyFloat_FromDouble(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: {
double value = reflection->GetRepeatedDouble(
*message, field_descriptor, index);
result = PyFloat_FromDouble(value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: {
bool value = reflection->GetRepeatedBool(
*message, field_descriptor, index);
result = PyBool_FromLong(value ? 1 : 0);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: {
const google::protobuf::EnumValueDescriptor* enum_value =
message->GetReflection()->GetRepeatedEnum(
*message, field_descriptor, index);
result = PyInt_FromLong(enum_value->number());
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: {
string value = reflection->GetRepeatedString(
*message, field_descriptor, index);
result = ToStringObject(field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE: {
PyObject* py_cmsg = PyObject_CallObject(reinterpret_cast<PyObject*>(
&CMessage_Type), NULL);
if (py_cmsg == NULL) {
return NULL;
}
CMessage* cmsg = reinterpret_cast<CMessage*>(py_cmsg);
const google::protobuf::Message& msg = reflection->GetRepeatedMessage(
*message, field_descriptor, index);
cmsg->owner = self->owner;
cmsg->parent = self->parent;
cmsg->message = const_cast<google::protobuf::Message*>(&msg);
cmsg->read_only = false;
result = reinterpret_cast<PyObject*>(py_cmsg);
break;
}
default:
PyErr_Format(
PyExc_SystemError,
"Getting value from a repeated field of unknown type %d",
field_descriptor->cpp_type());
}
return result;
}
static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) {
Py_ssize_t from;
Py_ssize_t to;
Py_ssize_t step;
Py_ssize_t length;
Py_ssize_t slicelength;
bool return_list = false;
#if PY_MAJOR_VERSION < 3
if (PyInt_Check(slice)) {
from = to = PyInt_AsLong(slice);
} else // NOLINT
#endif
if (PyLong_Check(slice)) {
from = to = PyLong_AsLong(slice);
} else if (PySlice_Check(slice)) {
length = Len(self);
#if PY_MAJOR_VERSION >= 3
if (PySlice_GetIndicesEx(slice,
#else
if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice),
#endif
length, &from, &to, &step, &slicelength) == -1) {
return NULL;
}
return_list = true;
} else {
PyErr_SetString(PyExc_TypeError, "list indices must be integers");
return NULL;
}
if (!return_list) {
return Item(self, from);
}
PyObject* list = PyList_New(0);
if (list == NULL) {
return NULL;
}
if (from <= to) {
if (step < 0) {
return list;
}
for (Py_ssize_t index = from; index < to; index += step) {
if (index < 0 || index >= length) {
break;
}
ScopedPyObjectPtr s(Item(self, index));
PyList_Append(list, s);
}
} else {
if (step > 0) {
return list;
}
for (Py_ssize_t index = from; index > to; index += step) {
if (index < 0 || index >= length) {
break;
}
ScopedPyObjectPtr s(Item(self, index));
PyList_Append(list, s);
}
}
return list;
}
PyObject* Append(RepeatedScalarContainer* self, PyObject* item) {
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
self->parent_field->descriptor;
if (!FIELD_BELONGS_TO_MESSAGE(field_descriptor, message)) {
PyErr_SetString(
PyExc_KeyError, "Field does not belong to message!");
return NULL;
}
const google::protobuf::Reflection* reflection = message->GetReflection();
switch (field_descriptor->cpp_type()) {
case google::protobuf::FieldDescriptor::CPPTYPE_INT32: {
GOOGLE_CHECK_GET_INT32(item, value, NULL);
reflection->AddInt32(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_INT64: {
GOOGLE_CHECK_GET_INT64(item, value, NULL);
reflection->AddInt64(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT32: {
GOOGLE_CHECK_GET_UINT32(item, value, NULL);
reflection->AddUInt32(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_UINT64: {
GOOGLE_CHECK_GET_UINT64(item, value, NULL);
reflection->AddUInt64(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT: {
GOOGLE_CHECK_GET_FLOAT(item, value, NULL);
reflection->AddFloat(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_DOUBLE: {
GOOGLE_CHECK_GET_DOUBLE(item, value, NULL);
reflection->AddDouble(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_BOOL: {
GOOGLE_CHECK_GET_BOOL(item, value, NULL);
reflection->AddBool(message, field_descriptor, value);
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_STRING: {
if (!CheckAndSetString(
item, message, field_descriptor, reflection, true, -1)) {
return NULL;
}
break;
}
case google::protobuf::FieldDescriptor::CPPTYPE_ENUM: {
GOOGLE_CHECK_GET_INT32(item, value, NULL);
const google::protobuf::EnumDescriptor* enum_descriptor =
field_descriptor->enum_type();
const google::protobuf::EnumValueDescriptor* enum_value =
enum_descriptor->FindValueByNumber(value);
if (enum_value != NULL) {
reflection->AddEnum(message, field_descriptor, enum_value);
} else {
ScopedPyObjectPtr s(PyObject_Str(item));
if (s != NULL) {
PyErr_Format(PyExc_ValueError, "Unknown enum value: %s",
PyString_AsString(s.get()));
}
return NULL;
}
break;
}
default:
PyErr_Format(
PyExc_SystemError, "Adding value to a field of unknown type %d",
field_descriptor->cpp_type());
return NULL;
}
Py_RETURN_NONE;
}
static int AssSubscript(RepeatedScalarContainer* self,
PyObject* slice,
PyObject* value) {
Py_ssize_t from;
Py_ssize_t to;
Py_ssize_t step;
Py_ssize_t length;
Py_ssize_t slicelength;
bool create_list = false;
cmessage::AssureWritable(self->parent);
google::protobuf::Message* message = self->message;
const google::protobuf::FieldDescriptor* field_descriptor =
self->parent_field->descriptor;
#if PY_MAJOR_VERSION < 3
if (PyInt_Check(slice)) {
from = to = PyInt_AsLong(slice);
} else
#endif
if (PyLong_Check(slice)) {
from = to = PyLong_AsLong(slice);
} else if (PySlice_Check(slice)) {
const google::protobuf::Reflection* reflection = message->GetReflection();
length = reflection->FieldSize(*message, field_descriptor);
#if PY_MAJOR_VERSION >= 3
if (PySlice_GetIndicesEx(slice,
#else
if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice),
#endif
length, &from, &to, &step, &slicelength) == -1) {
return -1;
}
create_list = true;
} else {
PyErr_SetString(PyExc_TypeError, "list indices must be integers");
return -1;
}
if (value == NULL) {
return cmessage::InternalDeleteRepeatedField(
message, field_descriptor, slice, NULL);
}
if (!create_list) {
return AssignItem(self, from, value);
}
ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
if (full_slice == NULL) {
return -1;
}
ScopedPyObjectPtr new_list(Subscript(self, full_slice));
if (new_list == NULL) {
return -1;
}
if (PySequence_SetSlice(new_list, from, to, value) < 0) {
return -1;
}
return InternalAssignRepeatedField(self, new_list);
}
PyObject* Extend(RepeatedScalarContainer* self, PyObject* value) {
cmessage::AssureWritable(self->parent);
if (PyObject_Not(value)) {
Py_RETURN_NONE;
}
ScopedPyObjectPtr iter(PyObject_GetIter(value));
if (iter == NULL) {
PyErr_SetString(PyExc_TypeError, "Value must be iterable");
return NULL;
}
ScopedPyObjectPtr next;
while ((next.reset(PyIter_Next(iter))) != NULL) {
if (Append(self, next) == NULL) {
return NULL;
}
}
if (PyErr_Occurred()) {
return NULL;
}
Py_RETURN_NONE;
}
static PyObject* Insert(RepeatedScalarContainer* self, PyObject* args) {
Py_ssize_t index;
PyObject* value;
if (!PyArg_ParseTuple(args, "lO", &index, &value)) {
return NULL;
}
ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
ScopedPyObjectPtr new_list(Subscript(self, full_slice));
if (PyList_Insert(new_list, index, value) < 0) {
return NULL;
}
int ret = InternalAssignRepeatedField(self, new_list);
if (ret < 0) {
return NULL;
}
Py_RETURN_NONE;
}
static PyObject* Remove(RepeatedScalarContainer* self, PyObject* value) {
Py_ssize_t match_index = -1;
for (Py_ssize_t i = 0; i < Len(self); ++i) {
ScopedPyObjectPtr elem(Item(self, i));
if (PyObject_RichCompareBool(elem, value, Py_EQ)) {
match_index = i;
break;
}
}
if (match_index == -1) {
PyErr_SetString(PyExc_ValueError, "remove(x): x not in container");
return NULL;
}
if (AssignItem(self, match_index, NULL) < 0) {
return NULL;
}
Py_RETURN_NONE;
}
static PyObject* RichCompare(RepeatedScalarContainer* self,
PyObject* other,
int opid) {
if (opid != Py_EQ && opid != Py_NE) {
Py_INCREF(Py_NotImplemented);
return Py_NotImplemented;
}
// Copy the contents of this repeated scalar container, and other if it is
// also a repeated scalar container, into Python lists so we can delegate
// to the list's compare method.
ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
if (full_slice == NULL) {
return NULL;
}
ScopedPyObjectPtr other_list_deleter;
if (PyObject_TypeCheck(other, &RepeatedScalarContainer_Type)) {
other_list_deleter.reset(Subscript(
reinterpret_cast<RepeatedScalarContainer*>(other), full_slice));
other = other_list_deleter.get();
}
ScopedPyObjectPtr list(Subscript(self, full_slice));
if (list == NULL) {
return NULL;
}
return PyObject_RichCompare(list, other, opid);
}
PyObject* Reduce(RepeatedScalarContainer* unused_self) {
PyErr_Format(
PickleError_class,
"can't pickle repeated message fields, convert to list first");
return NULL;
}
static PyObject* Sort(RepeatedScalarContainer* self,
PyObject* args,
PyObject* kwds) {
// Support the old sort_function argument for backwards
// compatibility.
if (kwds != NULL) {
PyObject* sort_func = PyDict_GetItemString(kwds, "sort_function");
if (sort_func != NULL) {
// Must set before deleting as sort_func is a borrowed reference
// and kwds might be the only thing keeping it alive.
if (PyDict_SetItemString(kwds, "cmp", sort_func) == -1)
return NULL;
if (PyDict_DelItemString(kwds, "sort_function") == -1)
return NULL;
}
}
ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
if (full_slice == NULL) {
return NULL;
}
ScopedPyObjectPtr list(Subscript(self, full_slice));
if (list == NULL) {
return NULL;
}
ScopedPyObjectPtr m(PyObject_GetAttrString(list, "sort"));
if (m == NULL) {
return NULL;
}
ScopedPyObjectPtr res(PyObject_Call(m, args, kwds));
if (res == NULL) {
return NULL;
}
int ret = InternalAssignRepeatedField(self, list);
if (ret < 0) {
return NULL;
}
Py_RETURN_NONE;
}
static int Init(RepeatedScalarContainer* self,
PyObject* args,
PyObject* kwargs) {
PyObject* py_parent;
PyObject* py_parent_field;
if (!PyArg_UnpackTuple(args, "__init__()", 2, 2, &py_parent,
&py_parent_field)) {
return -1;
}
if (!PyObject_TypeCheck(py_parent, &CMessage_Type)) {
PyErr_Format(PyExc_TypeError,
"expect %s, but got %s",
CMessage_Type.tp_name,
Py_TYPE(py_parent)->tp_name);
return -1;
}
if (!PyObject_TypeCheck(py_parent_field, &CFieldDescriptor_Type)) {
PyErr_Format(PyExc_TypeError,
"expect %s, but got %s",
CFieldDescriptor_Type.tp_name,
Py_TYPE(py_parent_field)->tp_name);
return -1;
}
CMessage* cmessage = reinterpret_cast<CMessage*>(py_parent);
CFieldDescriptor* cdescriptor = reinterpret_cast<CFieldDescriptor*>(
py_parent_field);
if (!FIELD_BELONGS_TO_MESSAGE(cdescriptor->descriptor, cmessage->message)) {
PyErr_SetString(
PyExc_KeyError, "Field does not belong to message!");
return -1;
}
self->message = cmessage->message;
self->parent = cmessage;
self->parent_field = cdescriptor;
self->owner = cmessage->owner;
return 0;
}
// Initializes the underlying Message object of "to" so it becomes a new parent
// repeated scalar, and copies all the values from "from" to it. A child scalar
// container can be released by passing it as both from and to (e.g. making it
// the recipient of the new parent message and copying the values from itself).
static int InitializeAndCopyToParentContainer(
RepeatedScalarContainer* from,
RepeatedScalarContainer* to) {
ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL));
if (full_slice == NULL) {
return -1;
}
ScopedPyObjectPtr values(Subscript(from, full_slice));
if (values == NULL) {
return -1;
}
google::protobuf::Message* new_message = global_message_factory->GetPrototype(
from->message->GetDescriptor())->New();
to->parent = NULL;
// TODO(anuraag): Document why it's OK to hang on to parent_field,
// even though it's a weak reference. It ought to be enough to
// hold on to the FieldDescriptor only.
to->parent_field = from->parent_field;
to->message = new_message;
to->owner.reset(new_message);
if (InternalAssignRepeatedField(to, values) < 0) {
return -1;
}
return 0;
}
int Release(RepeatedScalarContainer* self) {
return InitializeAndCopyToParentContainer(self, self);
}
PyObject* DeepCopy(RepeatedScalarContainer* self, PyObject* arg) {
ScopedPyObjectPtr init_args(
PyTuple_Pack(2, self->parent, self->parent_field));
PyObject* clone = PyObject_CallObject(
reinterpret_cast<PyObject*>(&RepeatedScalarContainer_Type), init_args);
if (clone == NULL) {
return NULL;
}
if (!PyObject_TypeCheck(clone, &RepeatedScalarContainer_Type)) {
Py_DECREF(clone);
return NULL;
}
if (InitializeAndCopyToParentContainer(
self, reinterpret_cast<RepeatedScalarContainer*>(clone)) < 0) {
Py_DECREF(clone);
return NULL;
}
return clone;
}
static void Dealloc(RepeatedScalarContainer* self) {
self->owner.reset();
Py_TYPE(self)->tp_free(reinterpret_cast<PyObject*>(self));
}
void SetOwner(RepeatedScalarContainer* self,
const shared_ptr<Message>& new_owner) {
self->owner = new_owner;
}
static PySequenceMethods SqMethods = {
(lenfunc)Len, /* sq_length */
0, /* sq_concat */
0, /* sq_repeat */
(ssizeargfunc)Item, /* sq_item */
0, /* sq_slice */
(ssizeobjargproc)AssignItem /* sq_ass_item */
};
static PyMappingMethods MpMethods = {
(lenfunc)Len, /* mp_length */
(binaryfunc)Subscript, /* mp_subscript */
(objobjargproc)AssSubscript, /* mp_ass_subscript */
};
static PyMethodDef Methods[] = {
{ "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
"Makes a deep copy of the class." },
{ "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
"Outputs picklable representation of the repeated field." },
{ "append", (PyCFunction)Append, METH_O,
"Appends an object to the repeated container." },
{ "extend", (PyCFunction)Extend, METH_O,
"Appends objects to the repeated container." },
{ "insert", (PyCFunction)Insert, METH_VARARGS,
"Appends objects to the repeated container." },
{ "remove", (PyCFunction)Remove, METH_O,
"Removes an object from the repeated container." },
{ "sort", (PyCFunction)Sort, METH_VARARGS | METH_KEYWORDS,
"Sorts the repeated container."},
{ NULL, NULL }
};
} // namespace repeated_scalar_container
PyTypeObject RepeatedScalarContainer_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"google.protobuf.internal."
"cpp._message.RepeatedScalarContainer", // tp_name
sizeof(RepeatedScalarContainer), // tp_basicsize
0, // tp_itemsize
(destructor)repeated_scalar_container::Dealloc, // tp_dealloc
0, // tp_print
0, // tp_getattr
0, // tp_setattr
0, // tp_compare
0, // tp_repr
0, // tp_as_number
&repeated_scalar_container::SqMethods, // tp_as_sequence
&repeated_scalar_container::MpMethods, // tp_as_mapping
0, // tp_hash
0, // tp_call
0, // tp_str
0, // tp_getattro
0, // tp_setattro
0, // tp_as_buffer
Py_TPFLAGS_DEFAULT, // tp_flags
"A Repeated scalar container", // tp_doc
0, // tp_traverse
0, // tp_clear
(richcmpfunc)repeated_scalar_container::RichCompare, // tp_richcompare
0, // tp_weaklistoffset
0, // tp_iter
0, // tp_iternext
repeated_scalar_container::Methods, // tp_methods
0, // tp_members
0, // tp_getset
0, // tp_base
0, // tp_dict
0, // tp_descr_get
0, // tp_descr_set
0, // tp_dictoffset
(initproc)repeated_scalar_container::Init, // tp_init
};
} // namespace python
} // namespace protobuf
} // namespace google

@ -0,0 +1,112 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: anuraag@google.com (Anuraag Agrawal)
// Author: tibell@google.com (Johan Tibell)
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__
#include <Python.h>
#include <memory>
#ifndef _SHARED_PTR_H
#include <google/protobuf/stubs/shared_ptr.h>
#endif
namespace google {
namespace protobuf {
class Message;
using internal::shared_ptr;
namespace python {
struct CFieldDescriptor;
struct CMessage;
typedef struct RepeatedScalarContainer {
PyObject_HEAD;
// This is the top-level C++ Message object that owns the whole
// proto tree. Every Python RepeatedScalarContainer holds a
// reference to it in order to keep it alive as long as there's a
// Python object that references any part of the tree.
shared_ptr<Message> owner;
// Pointer to the C++ Message that contains this container. The
// RepeatedScalarContainer does not own this pointer.
Message* message;
// Weak reference to a parent CMessage object (i.e. may be NULL.)
//
// Used to make sure all ancestors are also mutable when first
// modifying the container.
CMessage* parent;
// Weak reference to the parent's descriptor that describes this
// field. Used together with the parent's message when making a
// default message instance mutable.
CFieldDescriptor* parent_field;
} RepeatedScalarContainer;
extern PyTypeObject RepeatedScalarContainer_Type;
namespace repeated_scalar_container {
// Appends the scalar 'item' to the end of the container 'self'.
//
// Returns None if successful; returns NULL and sets an exception if
// unsuccessful.
PyObject* Append(RepeatedScalarContainer* self, PyObject* item);
// Releases the messages in the container to a new message.
//
// Returns 0 on success, -1 on failure.
int Release(RepeatedScalarContainer* self);
// Appends all the elements in the input iterator to the container.
//
// Returns None if successful; returns NULL and sets an exception if
// unsuccessful.
PyObject* Extend(RepeatedScalarContainer* self, PyObject* value);
// Set the owner field of self and any children of self.
void SetOwner(RepeatedScalarContainer* self,
const shared_ptr<Message>& new_owner);
} // namespace repeated_scalar_container
} // namespace python
} // namespace protobuf
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_CPP_REPEATED_SCALAR_CONTAINER_H__

@ -0,0 +1,95 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// http://code.google.com/p/protobuf/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: tibell@google.com (Johan Tibell)
#ifndef GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__
#define GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__
#include <Python.h>
namespace google {
class ScopedPyObjectPtr {
public:
// Constructor. Defaults to intializing with NULL.
// There is no way to create an uninitialized ScopedPyObjectPtr.
explicit ScopedPyObjectPtr(PyObject* p = NULL) : ptr_(p) { }
// Destructor. If there is a PyObject object, delete it.
~ScopedPyObjectPtr() {
Py_XDECREF(ptr_);
}
// Reset. Deletes the current owned object, if any.
// Then takes ownership of a new object, if given.
// this->reset(this->get()) works.
PyObject* reset(PyObject* p = NULL) {
if (p != ptr_) {
Py_XDECREF(ptr_);
ptr_ = p;
}
return ptr_;
}
// Releases ownership of the object.
PyObject* release() {
PyObject* p = ptr_;
ptr_ = NULL;
return p;
}
operator PyObject*() { return ptr_; }
PyObject* operator->() const {
assert(ptr_ != NULL);
return ptr_;
}
PyObject* get() const { return ptr_; }
Py_ssize_t refcnt() const { return Py_REFCNT(ptr_); }
void inc() const { Py_INCREF(ptr_); }
// Comparison operators.
// These return whether a ScopedPyObjectPtr and a raw pointer
// refer to the same object, not just to two different but equal
// objects.
bool operator==(const PyObject* p) const { return ptr_ == p; }
bool operator!=(const PyObject* p) const { return ptr_ != p; }
private:
PyObject* ptr_;
GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ScopedPyObjectPtr);
};
} // namespace google
#endif // GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__

@ -57,7 +57,7 @@ _FieldDescriptor = descriptor_mod.FieldDescriptor
if api_implementation.Type() == 'cpp':
if api_implementation.Version() == 2:
from google.protobuf.internal.cpp import cpp_message
from google.protobuf.pyext import cpp_message
_NewMessage = cpp_message.NewMessage
_InitMessage = cpp_message.InitMessage
else:
@ -91,6 +91,10 @@ class GeneratedProtocolMessageType(type):
myproto_instance = MyProtoClass()
myproto.foo_field = 23
...
The above example will not work for nested types. If you wish to include them,
use reflection.MakeClass() instead of manually instantiating the class in
order to create the appropriate class structure.
"""
# Must be consistent with the protocol-compiler code in
@ -159,11 +163,43 @@ def ParseMessage(descriptor, byte_str):
Returns:
Newly created protobuf Message object.
"""
result_class = MakeClass(descriptor)
new_msg = result_class()
new_msg.ParseFromString(byte_str)
return new_msg
class _ResultClass(message.Message):
def MakeClass(descriptor):
"""Construct a class object for a protobuf described by descriptor.
Composite descriptors are handled by defining the new class as a member of the
parent class, recursing as deep as necessary.
This is the dynamic equivalent to:
class Parent(message.Message):
__metaclass__ = GeneratedProtocolMessageType
DESCRIPTOR = descriptor
class Child(message.Message):
__metaclass__ = GeneratedProtocolMessageType
DESCRIPTOR = descriptor.nested_types[0]
new_msg = _ResultClass()
new_msg.ParseFromString(byte_str)
return new_msg
Sample usage:
file_descriptor = descriptor_pb2.FileDescriptorProto()
file_descriptor.ParseFromString(proto2_string)
msg_descriptor = descriptor.MakeDescriptor(file_descriptor.message_type[0])
msg_class = reflection.MakeClass(msg_descriptor)
msg = msg_class()
Args:
descriptor: A descriptor.Descriptor object describing the protobuf.
Returns:
The Message class object described by the descriptor.
"""
attributes = {}
for name, nested_type in descriptor.nested_types_by_name.items():
attributes[name] = MakeClass(nested_type)
attributes[GeneratedProtocolMessageType._DESCRIPTOR_KEY] = descriptor
return GeneratedProtocolMessageType(str(descriptor.name), (message.Message,),
attributes)

@ -0,0 +1,185 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""A database of Python protocol buffer generated symbols.
SymbolDatabase makes it easy to create new instances of a registered type, given
only the type's protocol buffer symbol name. Once all symbols are registered,
they can be accessed using either the MessageFactory interface which
SymbolDatabase exposes, or the DescriptorPool interface of the underlying
pool.
Example usage:
db = symbol_database.SymbolDatabase()
# Register symbols of interest, from one or multiple files.
db.RegisterFileDescriptor(my_proto_pb2.DESCRIPTOR)
db.RegisterMessage(my_proto_pb2.MyMessage)
db.RegisterEnumDescriptor(my_proto_pb2.MyEnum.DESCRIPTOR)
# The database can be used as a MessageFactory, to generate types based on
# their name:
types = db.GetMessages(['my_proto.proto'])
my_message_instance = types['MyMessage']()
# The database's underlying descriptor pool can be queried, so it's not
# necessary to know a type's filename to be able to generate it:
filename = db.pool.FindFileContainingSymbol('MyMessage')
my_message_instance = db.GetMessages([filename])['MyMessage']()
# This functionality is also provided directly via a convenience method:
my_message_instance = db.GetSymbol('MyMessage')()
"""
from google.protobuf import descriptor_pool
class SymbolDatabase(object):
"""A database of Python generated symbols.
SymbolDatabase also models message_factory.MessageFactory.
The symbol database can be used to keep a global registry of all protocol
buffer types used within a program.
"""
def __init__(self):
"""Constructor."""
self._symbols = {}
self._symbols_by_file = {}
self.pool = descriptor_pool.DescriptorPool()
def RegisterMessage(self, message):
"""Registers the given message type in the local database.
Args:
message: a message.Message, to be registered.
Returns:
The provided message.
"""
desc = message.DESCRIPTOR
self._symbols[desc.full_name] = message
if desc.file.name not in self._symbols_by_file:
self._symbols_by_file[desc.file.name] = {}
self._symbols_by_file[desc.file.name][desc.full_name] = message
self.pool.AddDescriptor(desc)
return message
def RegisterEnumDescriptor(self, enum_descriptor):
"""Registers the given enum descriptor in the local database.
Args:
enum_descriptor: a descriptor.EnumDescriptor.
Returns:
The provided descriptor.
"""
self.pool.AddEnumDescriptor(enum_descriptor)
return enum_descriptor
def RegisterFileDescriptor(self, file_descriptor):
"""Registers the given file descriptor in the local database.
Args:
file_descriptor: a descriptor.FileDescriptor.
Returns:
The provided descriptor.
"""
self.pool.AddFileDescriptor(file_descriptor)
def GetSymbol(self, symbol):
"""Tries to find a symbol in the local database.
Currently, this method only returns message.Message instances, however, if
may be extended in future to support other symbol types.
Args:
symbol: A str, a protocol buffer symbol.
Returns:
A Python class corresponding to the symbol.
Raises:
KeyError: if the symbol could not be found.
"""
return self._symbols[symbol]
def GetPrototype(self, descriptor):
"""Builds a proto2 message class based on the passed in descriptor.
Passing a descriptor with a fully qualified name matching a previous
invocation will cause the same class to be returned.
Args:
descriptor: The descriptor to build from.
Returns:
A class describing the passed in descriptor.
"""
return self.GetSymbol(descriptor.full_name)
def GetMessages(self, files):
"""Gets all the messages from a specified file.
This will find and resolve dependencies, failing if they are not registered
in the symbol database.
Args:
files: The file names to extract messages from.
Returns:
A dictionary mapping proto names to the message classes. This will include
any dependent messages as well as any messages defined in the same file as
a specified message.
Raises:
KeyError: if a file could not be found.
"""
result = {}
for f in files:
result.update(self._symbols_by_file[f])
return result
_DEFAULT = SymbolDatabase()
def Default():
"""Returns the default SymbolDatabase."""
return _DEFAULT

@ -0,0 +1,110 @@
# Protocol Buffers - Google's data interchange format
# Copyright 2008 Google Inc. All rights reserved.
# http://code.google.com/p/protobuf/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#PY25 compatible for GAE.
#
"""Encoding related utilities."""
import re
import sys ##PY25
# Lookup table for utf8
_cescape_utf8_to_str = [chr(i) for i in xrange(0, 256)]
_cescape_utf8_to_str[9] = r'\t' # optional escape
_cescape_utf8_to_str[10] = r'\n' # optional escape
_cescape_utf8_to_str[13] = r'\r' # optional escape
_cescape_utf8_to_str[39] = r"\'" # optional escape
_cescape_utf8_to_str[34] = r'\"' # necessary escape
_cescape_utf8_to_str[92] = r'\\' # necessary escape
# Lookup table for non-utf8, with necessary escapes at (o >= 127 or o < 32)
_cescape_byte_to_str = ([r'\%03o' % i for i in xrange(0, 32)] +
[chr(i) for i in xrange(32, 127)] +
[r'\%03o' % i for i in xrange(127, 256)])
_cescape_byte_to_str[9] = r'\t' # optional escape
_cescape_byte_to_str[10] = r'\n' # optional escape
_cescape_byte_to_str[13] = r'\r' # optional escape
_cescape_byte_to_str[39] = r"\'" # optional escape
_cescape_byte_to_str[34] = r'\"' # necessary escape
_cescape_byte_to_str[92] = r'\\' # necessary escape
def CEscape(text, as_utf8):
"""Escape a bytes string for use in an ascii protocol buffer.
text.encode('string_escape') does not seem to satisfy our needs as it
encodes unprintable characters using two-digit hex escapes whereas our
C++ unescaping function allows hex escapes to be any length. So,
"\0011".encode('string_escape') ends up being "\\x011", which will be
decoded in C++ as a single-character string with char code 0x11.
Args:
text: A byte string to be escaped
as_utf8: Specifies if result should be returned in UTF-8 encoding
Returns:
Escaped string
"""
# PY3 hack: make Ord work for str and bytes:
# //platforms/networking/data uses unicode here, hence basestring.
Ord = ord if isinstance(text, basestring) else lambda x: x
if as_utf8:
return ''.join(_cescape_utf8_to_str[Ord(c)] for c in text)
return ''.join(_cescape_byte_to_str[Ord(c)] for c in text)
_CUNESCAPE_HEX = re.compile(r'(\\+)x([0-9a-fA-F])(?![0-9a-fA-F])')
_cescape_highbit_to_str = ([chr(i) for i in range(0, 127)] +
[r'\%03o' % i for i in range(127, 256)])
def CUnescape(text):
"""Unescape a text string with C-style escape sequences to UTF-8 bytes."""
def ReplaceHex(m):
# Only replace the match if the number of leading back slashes is odd. i.e.
# the slash itself is not escaped.
if len(m.group(1)) & 1:
return m.group(1) + 'x0' + m.group(2)
return m.group(0)
# This is required because the 'string_escape' encoding doesn't
# allow single-digit hex escapes (like '\xf').
result = _CUNESCAPE_HEX.sub(ReplaceHex, text)
if sys.version_info[0] < 3: ##PY25
##!PY25 if str is bytes: # PY2
return result.decode('string_escape')
result = ''.join(_cescape_highbit_to_str[ord(c)] for c in result)
return (result.encode('ascii') # Make it bytes to allow decode.
.decode('unicode_escape')
# Make it bytes again to return the proper type.
.encode('raw_unicode_escape'))

@ -28,6 +28,10 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#PY25 compatible for GAE.
#
# Copyright 2007 Google Inc. All Rights Reserved.
"""Contains routines for printing protocol messages in text format."""
__author__ = 'kenton@google.com (Kenton Varda)'
@ -35,12 +39,12 @@ __author__ = 'kenton@google.com (Kenton Varda)'
import cStringIO
import re
from collections import deque
from google.protobuf.internal import type_checkers
from google.protobuf import descriptor
from google.protobuf import text_encoding
__all__ = [ 'MessageToString', 'PrintMessage', 'PrintField',
'PrintFieldValue', 'Merge' ]
__all__ = ['MessageToString', 'PrintMessage', 'PrintField',
'PrintFieldValue', 'Merge']
_INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
@ -49,15 +53,47 @@ _INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(),
type_checkers.Int64ValueChecker())
_FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE)
_FLOAT_NAN = re.compile('nanf?', re.IGNORECASE)
_FLOAT_TYPES = frozenset([descriptor.FieldDescriptor.CPPTYPE_FLOAT,
descriptor.FieldDescriptor.CPPTYPE_DOUBLE])
class Error(Exception):
"""Top-level module error for text_format."""
class ParseError(Exception):
class ParseError(Error):
"""Thrown in case of ASCII parsing error."""
def MessageToString(message, as_utf8=False, as_one_line=False):
def MessageToString(message, as_utf8=False, as_one_line=False,
pointy_brackets=False, use_index_order=False,
float_format=None):
"""Convert protobuf message to text format.
Floating point values can be formatted compactly with 15 digits of
precision (which is the most that IEEE 754 "double" can guarantee)
using float_format='.15g'.
Args:
message: The protocol buffers message.
as_utf8: Produce text output in UTF8 format.
as_one_line: Don't introduce newlines between fields.
pointy_brackets: If True, use angle brackets instead of curly braces for
nesting.
use_index_order: If True, print fields of a proto message using the order
defined in source code instead of the field number. By default, use the
field number order.
float_format: If set, use this to specify floating point number formatting
(per the "Format Specification Mini-Language"); otherwise, str() is used.
Returns:
A string of the text formatted protocol buffer message.
"""
out = cStringIO.StringIO()
PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line)
PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line,
pointy_brackets=pointy_brackets,
use_index_order=use_index_order,
float_format=float_format)
result = out.getvalue()
out.close()
if as_one_line:
@ -65,20 +101,30 @@ def MessageToString(message, as_utf8=False, as_one_line=False):
return result
def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False):
for field, value in message.ListFields():
def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False,
pointy_brackets=False, use_index_order=False,
float_format=None):
fields = message.ListFields()
if use_index_order:
fields.sort(key=lambda x: x[0].index)
for field, value in fields:
if field.label == descriptor.FieldDescriptor.LABEL_REPEATED:
for element in value:
PrintField(field, element, out, indent, as_utf8, as_one_line)
PrintField(field, element, out, indent, as_utf8, as_one_line,
pointy_brackets=pointy_brackets,
float_format=float_format)
else:
PrintField(field, value, out, indent, as_utf8, as_one_line)
PrintField(field, value, out, indent, as_utf8, as_one_line,
pointy_brackets=pointy_brackets,
float_format=float_format)
def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False):
def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False,
pointy_brackets=False, float_format=None):
"""Print a single field name/value pair. For repeated fields, the value
should be a single element."""
out.write(' ' * indent);
out.write(' ' * indent)
if field.is_extension:
out.write('[')
if (field.containing_type.GetOptions().message_set_wire_format and
@ -100,27 +146,41 @@ def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False):
# don't include it.
out.write(': ')
PrintFieldValue(field, value, out, indent, as_utf8, as_one_line)
PrintFieldValue(field, value, out, indent, as_utf8, as_one_line,
pointy_brackets=pointy_brackets,
float_format=float_format)
if as_one_line:
out.write(' ')
else:
out.write('\n')
def PrintFieldValue(field, value, out, indent=0,
as_utf8=False, as_one_line=False):
def PrintFieldValue(field, value, out, indent=0, as_utf8=False,
as_one_line=False, pointy_brackets=False,
float_format=None):
"""Print a single field value (not including name). For repeated fields,
the value should be a single element."""
if pointy_brackets:
openb = '<'
closeb = '>'
else:
openb = '{'
closeb = '}'
if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
if as_one_line:
out.write(' { ')
PrintMessage(value, out, indent, as_utf8, as_one_line)
out.write('}')
out.write(' %s ' % openb)
PrintMessage(value, out, indent, as_utf8, as_one_line,
pointy_brackets=pointy_brackets,
float_format=float_format)
out.write(closeb)
else:
out.write(' {\n')
PrintMessage(value, out, indent + 2, as_utf8, as_one_line)
out.write(' ' * indent + '}')
out.write(' %s\n' % openb)
PrintMessage(value, out, indent + 2, as_utf8, as_one_line,
pointy_brackets=pointy_brackets,
float_format=float_format)
out.write(' ' * indent + closeb)
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
enum_value = field.enum_type.values_by_number.get(value, None)
if enum_value is not None:
@ -129,41 +189,125 @@ def PrintFieldValue(field, value, out, indent=0,
out.write(str(value))
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
out.write('\"')
if type(value) is unicode:
out.write(_CEscape(value.encode('utf-8'), as_utf8))
if isinstance(value, unicode):
out_value = value.encode('utf-8')
else:
out_value = value
if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
# We need to escape non-UTF8 chars in TYPE_BYTES field.
out_as_utf8 = False
else:
out.write(_CEscape(value, as_utf8))
out_as_utf8 = as_utf8
out.write(text_encoding.CEscape(out_value, out_as_utf8))
out.write('\"')
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
if value:
out.write("true")
out.write('true')
else:
out.write("false")
out.write('false')
elif field.cpp_type in _FLOAT_TYPES and float_format is not None:
out.write('{1:{0}}'.format(float_format, value))
else:
out.write(str(value))
def _ParseOrMerge(lines, message, allow_multiple_scalars):
"""Converts an ASCII representation of a protocol message into a message.
Args:
lines: Lines of a message's ASCII representation.
message: A protocol buffer message to merge into.
allow_multiple_scalars: Determines if repeated values for a non-repeated
field are permitted, e.g., the string "foo: 1 foo: 2" for a
required/optional field named "foo".
Raises:
ParseError: On ASCII parsing problems.
"""
tokenizer = _Tokenizer(lines)
while not tokenizer.AtEnd():
_MergeField(tokenizer, message, allow_multiple_scalars)
def Parse(text, message):
"""Parses an ASCII representation of a protocol message into a message.
Args:
text: Message ASCII representation.
message: A protocol buffer message to merge into.
Returns:
The same message passed as argument.
Raises:
ParseError: On ASCII parsing problems.
"""
if not isinstance(text, str): text = text.decode('utf-8')
return ParseLines(text.split('\n'), message)
def Merge(text, message):
"""Merges an ASCII representation of a protocol message into a message.
"""Parses an ASCII representation of a protocol message into a message.
Like Parse(), but allows repeated values for a non-repeated field, and uses
the last one.
Args:
text: Message ASCII representation.
message: A protocol buffer message to merge into.
Returns:
The same message passed as argument.
Raises:
ParseError: On ASCII parsing problems.
"""
tokenizer = _Tokenizer(text)
while not tokenizer.AtEnd():
_MergeField(tokenizer, message)
return MergeLines(text.split('\n'), message)
def _MergeField(tokenizer, message):
def ParseLines(lines, message):
"""Parses an ASCII representation of a protocol message into a message.
Args:
lines: An iterable of lines of a message's ASCII representation.
message: A protocol buffer message to merge into.
Returns:
The same message passed as argument.
Raises:
ParseError: On ASCII parsing problems.
"""
_ParseOrMerge(lines, message, False)
return message
def MergeLines(lines, message):
"""Parses an ASCII representation of a protocol message into a message.
Args:
lines: An iterable of lines of a message's ASCII representation.
message: A protocol buffer message to merge into.
Returns:
The same message passed as argument.
Raises:
ParseError: On ASCII parsing problems.
"""
_ParseOrMerge(lines, message, True)
return message
def _MergeField(tokenizer, message, allow_multiple_scalars):
"""Merges a single protocol message field into a message.
Args:
tokenizer: A tokenizer to parse the field name and values.
message: A protocol message to record the data.
allow_multiple_scalars: Determines if repeated values for a non-repeated
field are permitted, e.g., the string "foo: 1 foo: 2" for a
required/optional field named "foo".
Raises:
ParseError: In case of ASCII parsing problems.
@ -179,7 +323,9 @@ def _MergeField(tokenizer, message):
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" does not have extensions.' %
message_descriptor.full_name)
# pylint: disable=protected-access
field = message.Extensions._FindExtensionByName(name)
# pylint: enable=protected-access
if not field:
raise tokenizer.ParseErrorPreviousToken(
'Extension "%s" not registered.' % name)
@ -233,18 +379,26 @@ def _MergeField(tokenizer, message):
while not tokenizer.TryConsume(end_token):
if tokenizer.AtEnd():
raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token))
_MergeField(tokenizer, sub_message)
_MergeField(tokenizer, sub_message, allow_multiple_scalars)
else:
_MergeScalarField(tokenizer, message, field)
_MergeScalarField(tokenizer, message, field, allow_multiple_scalars)
# For historical reasons, fields may optionally be separated by commas or
# semicolons.
if not tokenizer.TryConsume(','):
tokenizer.TryConsume(';')
def _MergeScalarField(tokenizer, message, field):
def _MergeScalarField(tokenizer, message, field, allow_multiple_scalars):
"""Merges a single protocol message scalar field into a message.
Args:
tokenizer: A tokenizer to parse the field value.
message: A protocol message to record the data.
field: The descriptor of the field to be merged.
allow_multiple_scalars: Determines if repeated values for a non-repeated
field are permitted, e.g., the string "foo: 1 foo: 2" for a
required/optional field named "foo".
Raises:
ParseError: In case of ASCII parsing problems.
@ -288,9 +442,19 @@ def _MergeScalarField(tokenizer, message, field):
getattr(message, field.name).append(value)
else:
if field.is_extension:
message.Extensions[field] = value
if not allow_multiple_scalars and message.HasExtension(field):
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" should not have multiple "%s" extensions.' %
(message.DESCRIPTOR.full_name, field.full_name))
else:
message.Extensions[field] = value
else:
setattr(message, field.name, value)
if not allow_multiple_scalars and message.HasField(field.name):
raise tokenizer.ParseErrorPreviousToken(
'Message type "%s" should not have multiple "%s" fields.' %
(message.DESCRIPTOR.full_name, field.name))
else:
setattr(message, field.name, value)
class _Tokenizer(object):
@ -308,20 +472,19 @@ class _Tokenizer(object):
'[0-9+-][0-9a-zA-Z_.+-]*|' # a number
'\"([^\"\n\\\\]|\\\\.)*(\"|\\\\?$)|' # a double-quoted string
'\'([^\'\n\\\\]|\\\\.)*(\'|\\\\?$)') # a single-quoted string
_IDENTIFIER = re.compile('\w+')
def __init__(self, text_message):
self._text_message = text_message
_IDENTIFIER = re.compile(r'\w+')
def __init__(self, lines):
self._position = 0
self._line = -1
self._column = 0
self._token_start = None
self.token = ''
self._lines = deque(text_message.split('\n'))
self._lines = iter(lines)
self._current_line = ''
self._previous_line = 0
self._previous_column = 0
self._more_lines = True
self._SkipWhitespace()
self.NextToken()
@ -331,16 +494,19 @@ class _Tokenizer(object):
Returns:
True iff the end was reached.
"""
return self.token == ''
return not self.token
def _PopLine(self):
while len(self._current_line) <= self._column:
if not self._lines:
try:
self._current_line = self._lines.next()
except StopIteration:
self._current_line = ''
self._more_lines = False
return
self._line += 1
self._column = 0
self._current_line = self._lines.popleft()
else:
self._line += 1
self._column = 0
def _SkipWhitespace(self):
while True:
@ -497,9 +663,9 @@ class _Tokenizer(object):
Raises:
ParseError: If a string value couldn't be consumed.
"""
bytes = self.ConsumeByteString()
the_bytes = self.ConsumeByteString()
try:
return unicode(bytes, 'utf-8')
return unicode(the_bytes, 'utf-8')
except UnicodeDecodeError, e:
raise self._StringParseError(e)
@ -512,10 +678,11 @@ class _Tokenizer(object):
Raises:
ParseError: If a byte array value couldn't be consumed.
"""
list = [self._ConsumeSingleByteString()]
while len(self.token) > 0 and self.token[0] in ('\'', '"'):
list.append(self._ConsumeSingleByteString())
return "".join(list)
the_list = [self._ConsumeSingleByteString()]
while self.token and self.token[0] in ('\'', '"'):
the_list.append(self._ConsumeSingleByteString())
return ''.encode('latin1').join(the_list) ##PY25
##!PY25 return b''.join(the_list)
def _ConsumeSingleByteString(self):
"""Consume one token of a string literal.
@ -532,7 +699,7 @@ class _Tokenizer(object):
raise self._ParseError('String missing ending quote.')
try:
result = _CUnescape(text[1:-1])
result = text_encoding.CUnescape(text[1:-1])
except ValueError, e:
raise self._ParseError(str(e))
self.NextToken()
@ -574,7 +741,7 @@ class _Tokenizer(object):
self._column += len(self.token)
self._SkipWhitespace()
if not self._lines and len(self._current_line) <= self._column:
if not self._more_lines:
self.token = ''
return
@ -586,45 +753,6 @@ class _Tokenizer(object):
self.token = self._current_line[self._column]
# text.encode('string_escape') does not seem to satisfy our needs as it
# encodes unprintable characters using two-digit hex escapes whereas our
# C++ unescaping function allows hex escapes to be any length. So,
# "\0011".encode('string_escape') ends up being "\\x011", which will be
# decoded in C++ as a single-character string with char code 0x11.
def _CEscape(text, as_utf8):
def escape(c):
o = ord(c)
if o == 10: return r"\n" # optional escape
if o == 13: return r"\r" # optional escape
if o == 9: return r"\t" # optional escape
if o == 39: return r"\'" # optional escape
if o == 34: return r'\"' # necessary escape
if o == 92: return r"\\" # necessary escape
# necessary escapes
if not as_utf8 and (o >= 127 or o < 32): return "\\%03o" % o
return c
return "".join([escape(c) for c in text])
_CUNESCAPE_HEX = re.compile(r'(\\+)x([0-9a-fA-F])(?![0-9a-fA-F])')
def _CUnescape(text):
def ReplaceHex(m):
# Only replace the match if the number of leading back slashes is odd. i.e.
# the slash itself is not escaped.
if len(m.group(1)) & 1:
return m.group(1) + 'x0' + m.group(2)
return m.group(0)
# This is required because the 'string_escape' encoding doesn't
# allow single-digit hex escapes (like '\xf').
result = _CUNESCAPE_HEX.sub(ReplaceHex, text)
return result.decode('string_escape')
def ParseInteger(text, is_signed=False, is_long=False):
"""Parses an integer.
@ -641,7 +769,13 @@ def ParseInteger(text, is_signed=False, is_long=False):
"""
# Do the actual parsing. Exception handling is propagated to caller.
try:
result = int(text, 0)
# We force 32-bit values to int and 64-bit values to long to make
# alternate implementations where the distinction is more significant
# (e.g. the C++ implementation) simpler.
if is_long:
result = long(text, 0)
else:
result = int(text, 0)
except ValueError:
raise ValueError('Couldn\'t parse integer: %s' % text)

@ -8,17 +8,14 @@ import subprocess
# We must use setuptools, not distutils, because we need to use the
# namespace_packages option for the "google" package.
try:
from setuptools import setup, Extension
from ez_setup import use_setuptools
use_setuptools()
from setuptools import setup, Extension, __version__
except ImportError:
try:
from ez_setup import use_setuptools
use_setuptools()
from setuptools import setup, Extension
except ImportError:
sys.stderr.write(
"Could not import setuptools; make sure you have setuptools or "
"ez_setup installed.\n")
raise
sys.stderr.write(
"Could not import setuptools; make sure you have setuptools or "
"ez_setup installed.\n")
raise
from distutils.command.clean import clean as _clean
from distutils.command.build_py import build_py as _build_py
from distutils.spawn import find_executable
@ -49,7 +46,7 @@ def generate_proto(source):
if (not os.path.exists(output) or
(os.path.exists(source) and
os.path.getmtime(source) > os.path.getmtime(output))):
print "Generating %s..." % output
print ("Generating %s..." % output)
if not os.path.exists(source):
sys.stderr.write("Can't find required file: %s\n" % source)
@ -72,7 +69,6 @@ def GenerateUnittestProtos():
generate_proto("../src/google/protobuf/unittest_import_public.proto")
generate_proto("../src/google/protobuf/unittest_mset.proto")
generate_proto("../src/google/protobuf/unittest_no_generic_services.proto")
generate_proto("google/protobuf/internal/compatibility_mode_test.proto")
generate_proto("google/protobuf/internal/descriptor_pool_test1.proto")
generate_proto("google/protobuf/internal/descriptor_pool_test2.proto")
generate_proto("google/protobuf/internal/test_bad_identifiers.proto")
@ -82,75 +78,7 @@ def GenerateUnittestProtos():
generate_proto("google/protobuf/internal/more_messages.proto")
generate_proto("google/protobuf/internal/factory_test1.proto")
generate_proto("google/protobuf/internal/factory_test2.proto")
def MakeTestSuite():
# This is apparently needed on some systems to make sure that the tests
# work even if a previous version is already installed.
if 'google' in sys.modules:
del sys.modules['google']
GenerateUnittestProtos()
import unittest
import google.protobuf.internal.generator_test as generator_test
import google.protobuf.internal.descriptor_test as descriptor_test
import google.protobuf.internal.reflection_test as reflection_test
import google.protobuf.internal.service_reflection_test \
as service_reflection_test
import google.protobuf.internal.text_format_test as text_format_test
import google.protobuf.internal.wire_format_test as wire_format_test
import google.protobuf.internal.unknown_fields_test as unknown_fields_test
import google.protobuf.internal.descriptor_database_test \
as descriptor_database_test
import google.protobuf.internal.descriptor_pool_test as descriptor_pool_test
import google.protobuf.internal.message_factory_test as message_factory_test
import google.protobuf.internal.message_cpp_test as message_cpp_test
import google.protobuf.internal.reflection_cpp_generated_test \
as reflection_cpp_generated_test
import google.protobuf.internal.api_implementation_default_test \
as api_implementation_default_test
import google.protobuf.internal.descriptor_cpp2_test as descriptor_cpp2_test
import google.protobuf.internal.descriptor_python_test \
as descriptor_python_test
import google.protobuf.internal.message_factory_cpp2_test \
as message_factory_cpp2_test
import google.protobuf.internal.message_factory_cpp_test \
as message_factory_cpp_test
import google.protobuf.internal.message_factory_python_test \
as message_factory_python_test
import google.protobuf.internal.message_python_test as message_python_test
import google.protobuf.internal.reflection_cpp2_generated_test \
as reflection_cpp2_generated_test
import google.protobuf.internal.symbol_database_test as symbol_database_test
import google.protobuf.internal.text_encoding_test as text_encoding_test
loader = unittest.defaultTestLoader
suite = unittest.TestSuite()
for test in [ generator_test,
descriptor_test,
reflection_test,
service_reflection_test,
text_format_test,
wire_format_test,
unknown_fields_test,
descriptor_pool_test,
message_factory_test,
message_cpp_test,
reflection_cpp_generated_test,
api_implementation_default_test,
descriptor_cpp2_test,
descriptor_python_test,
message_factory_cpp2_test,
message_factory_cpp_test,
message_factory_python_test,
message_python_test,
reflection_cpp2_generated_test,
symbol_database_test,
text_encoding_test ]:
suite.addTest(loader.loadTestsFromModule(test))
return suite
generate_proto("google/protobuf/pyext/python.proto")
class clean(_clean):
def run(self):
@ -170,31 +98,52 @@ class build_py(_build_py):
# Generate necessary .proto file if it doesn't exist.
generate_proto("../src/google/protobuf/descriptor.proto")
generate_proto("../src/google/protobuf/compiler/plugin.proto")
# Make sure google.protobuf.compiler is a valid package.
open('google/protobuf/compiler/__init__.py', 'a').close()
GenerateUnittestProtos()
# Make sure google.protobuf/** are valid packages.
for path in ['', 'internal/', 'compiler/', 'pyext/']:
try:
open('google/protobuf/%s__init__.py' % path, 'a').close()
except EnvironmentError:
pass
# _build_py is an old-style class, so super() doesn't work.
_build_py.run(self)
# TODO(mrovner): Subclass to run 2to3 on some files only.
# Tracing what https://wiki.python.org/moin/PortingPythonToPy3k's "Approach 2"
# section on how to get 2to3 to run on source files during install under
# Python 3. This class seems like a good place to put logic that calls
# python3's distutils.util.run_2to3 on the subset of the files we have in our
# release that are subject to conversion.
# See code reference in previous code review.
if __name__ == '__main__':
ext_module_list = []
if __name__ == '__main__':
print(__version__)
# C++ implementation extension
if os.getenv("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") == "cpp":
print "Using EXPERIMENTAL C++ Implmenetation."
ext_module_list.append(Extension(
"google.protobuf.internal._net_proto2___python",
[ "google/protobuf/pyext/python_descriptor.cc",
"google/protobuf/pyext/python_protobuf.cc",
"google/protobuf/pyext/python-proto2.cc" ],
include_dirs = [ "." ],
libraries = [ "protobuf" ]))
nocpp = '--nocpp_implementation'
if nocpp in sys.argv:
ext_module_list = []
sys.argv.remove(nocpp)
else:
nocpp = False
ext_module_list = [Extension(
"google.protobuf.pyext._message",
[ "google/protobuf/pyext/descriptor.cc",
"google/protobuf/pyext/message.cc",
"google/protobuf/pyext/extension_dict.cc",
"google/protobuf/pyext/repeated_scalar_container.cc",
"google/protobuf/pyext/repeated_composite_container.cc" ],
define_macros=[('GOOGLE_PROTOBUF_HAS_ONEOF', '1')],
include_dirs = [ ".", "../src" ],
libraries = [ "protobuf" ],
library_dirs = [ '../src/.libs' ],
)]
setup(name = 'protobuf',
version = '2.5.1-pre',
version = '2.6-pre',
packages = [ 'google' ],
namespace_packages = [ 'google' ],
test_suite = 'setup.MakeTestSuite',
google_test_dir = "google/protobuf/internal",
# Must list modules explicitly so that we don't install tests.
py_modules = [
'google.protobuf.internal.api_implementation',
@ -217,9 +166,12 @@ if __name__ == '__main__':
'google.protobuf.reflection',
'google.protobuf.service',
'google.protobuf.service_reflection',
'google.protobuf.text_format' ],
'google.protobuf.symbol_database',
'google.protobuf.text_encoding',
'google.protobuf.text_format'],
cmdclass = { 'clean': clean, 'build_py': build_py },
install_requires = ['setuptools'],
setup_requires = ['google-apputils'],
ext_modules = ext_module_list,
url = 'http://code.google.com/p/protobuf/',
maintainer = maintainer_email,
@ -228,4 +180,5 @@ if __name__ == '__main__':
description = 'Protocol Buffers',
long_description =
"Protocol Buffers are Google's data interchange format.",
use_2to3=True,
)

@ -101,6 +101,7 @@ libprotobuf_lite_la_SOURCES = \
google/protobuf/stubs/once.cc \
google/protobuf/stubs/hash.h \
google/protobuf/stubs/map_util.h \
google/protobuf/stubs/shared_ptr.h \
google/protobuf/stubs/stl_util.h \
google/protobuf/stubs/stringprintf.cc \
google/protobuf/stubs/stringprintf.h \
@ -246,7 +247,10 @@ EXTRA_DIST = \
google/protobuf/testdata/golden_message_oneof_implemented \
google/protobuf/testdata/golden_packed_fields_message \
google/protobuf/testdata/text_format_unittest_data_oneof_implemented.txt \
google/protobuf/testdata/text_format_unittest_data_pointy.txt \
google/protobuf/testdata/text_format_unittest_data_pointy_oneof_implemented.txt \
google/protobuf/testdata/text_format_unittest_extensions_data.txt \
google/protobuf/testdata/text_format_unittest_extensions_data_pointy.txt \
google/protobuf/package_info.h \
google/protobuf/io/package_info.h \
google/protobuf/compiler/package_info.h \

@ -642,7 +642,22 @@ void Generator::PrintDescriptor(const Descriptor& message_descriptor) const {
"end", SimpleItoa(range->end));
}
printer_->Print("],\n");
printer_->Print("oneofs=[\n");
printer_->Indent();
for (int i = 0; i < message_descriptor.oneof_decl_count(); ++i) {
const OneofDescriptor* desc = message_descriptor.oneof_decl(i);
map<string, string> m;
m["name"] = desc->name();
m["full_name"] = desc->full_name();
m["index"] = SimpleItoa(desc->index());
printer_->Print(
m,
"_descriptor.OneofDescriptor(\n"
" name='$name$', full_name='$full_name$',\n"
" index=$index$, containing_type=None, fields=[]),\n");
}
printer_->Outdent();
printer_->Print("],\n");
// Serialization of proto
DescriptorProto edp;
PrintSerializedPbInterval(message_descriptor, edp);
@ -743,6 +758,23 @@ void Generator::FixForeignFieldsInDescriptor(
const EnumDescriptor& enum_descriptor = *descriptor.enum_type(i);
FixContainingTypeInDescriptor(enum_descriptor, &descriptor);
}
for (int i = 0; i < descriptor.oneof_decl_count(); ++i) {
map<string, string> m;
const OneofDescriptor* oneof = descriptor.oneof_decl(i);
m["descriptor_name"] = ModuleLevelDescriptorName(descriptor);
m["oneof_name"] = oneof->name();
for (int j = 0; j < oneof->field_count(); ++j) {
m["field_name"] = oneof->field(j)->name();
printer_->Print(
m,
"$descriptor_name$.oneofs_by_name['$oneof_name$'].fields.append(\n"
" $descriptor_name$.fields_by_name['$field_name$'])\n");
printer_->Print(
m,
"$descriptor_name$.fields_by_name['$field_name$'].containing_oneof = "
"$descriptor_name$.oneofs_by_name['$oneof_name$']\n");
}
}
}
void Generator::AddMessageToFileDescriptor(const Descriptor& descriptor) const {

@ -0,0 +1,129 @@
optional_int32: 101
optional_int64: 102
optional_uint32: 103
optional_uint64: 104
optional_sint32: 105
optional_sint64: 106
optional_fixed32: 107
optional_fixed64: 108
optional_sfixed32: 109
optional_sfixed64: 110
optional_float: 111
optional_double: 112
optional_bool: true
optional_string: "115"
optional_bytes: "116"
OptionalGroup <
a: 117
>
optional_nested_message <
bb: 118
>
optional_foreign_message <
c: 119
>
optional_import_message <
d: 120
>
optional_nested_enum: BAZ
optional_foreign_enum: FOREIGN_BAZ
optional_import_enum: IMPORT_BAZ
optional_string_piece: "124"
optional_cord: "125"
optional_public_import_message <
e: 126
>
optional_lazy_message <
bb: 127
>
repeated_int32: 201
repeated_int32: 301
repeated_int64: 202
repeated_int64: 302
repeated_uint32: 203
repeated_uint32: 303
repeated_uint64: 204
repeated_uint64: 304
repeated_sint32: 205
repeated_sint32: 305
repeated_sint64: 206
repeated_sint64: 306
repeated_fixed32: 207
repeated_fixed32: 307
repeated_fixed64: 208
repeated_fixed64: 308
repeated_sfixed32: 209
repeated_sfixed32: 309
repeated_sfixed64: 210
repeated_sfixed64: 310
repeated_float: 211
repeated_float: 311
repeated_double: 212
repeated_double: 312
repeated_bool: true
repeated_bool: false
repeated_string: "215"
repeated_string: "315"
repeated_bytes: "216"
repeated_bytes: "316"
RepeatedGroup <
a: 217
>
RepeatedGroup <
a: 317
>
repeated_nested_message <
bb: 218
>
repeated_nested_message <
bb: 318
>
repeated_foreign_message <
c: 219
>
repeated_foreign_message <
c: 319
>
repeated_import_message <
d: 220
>
repeated_import_message <
d: 320
>
repeated_nested_enum: BAR
repeated_nested_enum: BAZ
repeated_foreign_enum: FOREIGN_BAR
repeated_foreign_enum: FOREIGN_BAZ
repeated_import_enum: IMPORT_BAR
repeated_import_enum: IMPORT_BAZ
repeated_string_piece: "224"
repeated_string_piece: "324"
repeated_cord: "225"
repeated_cord: "325"
repeated_lazy_message <
bb: 227
>
repeated_lazy_message <
bb: 327
>
default_int32: 401
default_int64: 402
default_uint32: 403
default_uint64: 404
default_sint32: 405
default_sint64: 406
default_fixed32: 407
default_fixed64: 408
default_sfixed32: 409
default_sfixed64: 410
default_float: 411
default_double: 412
default_bool: false
default_string: "415"
default_bytes: "416"
default_nested_enum: FOO
default_foreign_enum: FOREIGN_FOO
default_import_enum: IMPORT_FOO
default_string_piece: "424"
default_cord: "425"
oneof_bytes: "604"
Loading…
Cancel
Save