Removed Python 2.x support for Python.

pull/8961/head
Joshua Haberman 4 years ago
parent 32461877b7
commit 359594f4bb
  1. 2
      python/README.md
  2. 3
      python/google/protobuf/descriptor.py
  3. 4
      python/google/protobuf/internal/_parameterized.py
  4. 54
      python/google/protobuf/internal/decoder.py
  5. 5
      python/google/protobuf/internal/descriptor_database_test.py
  6. 6
      python/google/protobuf/internal/descriptor_pool_test.py
  7. 19
      python/google/protobuf/internal/descriptor_test.py
  8. 11
      python/google/protobuf/internal/encoder.py
  9. 4
      python/google/protobuf/internal/enum_type_wrapper.py
  10. 27
      python/google/protobuf/internal/json_format_test.py
  11. 5
      python/google/protobuf/internal/message_factory_test.py
  12. 263
      python/google/protobuf/internal/message_test.py
  13. 28
      python/google/protobuf/internal/python_message.py
  14. 35
      python/google/protobuf/internal/reflection_test.py
  15. 5
      python/google/protobuf/internal/service_reflection_test.py
  16. 5
      python/google/protobuf/internal/symbol_database_test.py
  17. 12
      python/google/protobuf/internal/testing_refleaks.py
  18. 5
      python/google/protobuf/internal/text_encoding_test.py
  19. 237
      python/google/protobuf/internal/text_format_test.py
  20. 34
      python/google/protobuf/internal/type_checkers.py
  21. 6
      python/google/protobuf/internal/unknown_fields_test.py
  22. 23
      python/google/protobuf/internal/well_known_types.py
  23. 14
      python/google/protobuf/internal/well_known_types_test.py
  24. 5
      python/google/protobuf/internal/wire_format_test.py
  25. 27
      python/google/protobuf/json_format.py
  26. 2
      python/google/protobuf/message_factory.py
  27. 5
      python/google/protobuf/proto_builder.py
  28. 19
      python/google/protobuf/text_encoding.py
  29. 55
      python/google/protobuf/text_format.py
  30. 2
      python/setup.py
  31. 7
      python/tox.ini

@ -26,7 +26,7 @@ use python c++ implementation.
Installation Installation
============ ============
1) Make sure you have Python 2.7 or newer. If in doubt, run: 1) Make sure you have Python 3.5 or newer. If in doubt, run:
$ python -V $ python -V

@ -36,7 +36,6 @@ __author__ = 'robinson@google.com (Will Robinson)'
import threading import threading
import warnings import warnings
import six
from google.protobuf.internal import api_implementation from google.protobuf.internal import api_implementation
@ -111,7 +110,7 @@ _Deprecated.count = 100
_internal_create_key = object() _internal_create_key = object()
class DescriptorBase(six.with_metaclass(DescriptorMetaclass)): class DescriptorBase(metaclass=DescriptorMetaclass):
"""Descriptors base class. """Descriptors base class.

@ -154,8 +154,6 @@ except ImportError:
import unittest import unittest
import uuid import uuid
import six
try: try:
# Since python 3 # Since python 3
import collections.abc as collections_abc import collections.abc as collections_abc
@ -181,7 +179,7 @@ def _StrClass(cls):
def _NonStringIterable(obj): def _NonStringIterable(obj):
return (isinstance(obj, collections_abc.Iterable) and not return (isinstance(obj, collections_abc.Iterable) and not
isinstance(obj, six.string_types)) isinstance(obj, str))
def _FormatParameterList(testcase_params): def _FormatParameterList(testcase_params):

@ -80,16 +80,8 @@ we repeatedly read a tag, look up the corresponding decoder, and invoke it.
__author__ = 'kenton@google.com (Kenton Varda)' __author__ = 'kenton@google.com (Kenton Varda)'
import math
import struct import struct
import sys
import six
_UCS2_MAXUNICODE = 65535
if six.PY3:
long = int
else:
import re # pylint: disable=g-import-not-at-top
_SURROGATE_PATTERN = re.compile(six.u(r'[\ud800-\udfff]'))
from google.protobuf.internal import containers from google.protobuf.internal import containers
from google.protobuf.internal import encoder from google.protobuf.internal import encoder
@ -97,13 +89,6 @@ from google.protobuf.internal import wire_format
from google.protobuf import message from google.protobuf import message
# This will overflow and thus become IEEE-754 "infinity". We would use
# "float('inf')" but it doesn't work on Windows pre-Python-2.6.
_POS_INF = 1e10000
_NEG_INF = -_POS_INF
_NAN = _POS_INF * 0
# This is not for optimization, but rather to avoid conflicts with local # This is not for optimization, but rather to avoid conflicts with local
# variables named "message". # variables named "message".
_DecodeError = message.DecodeError _DecodeError = message.DecodeError
@ -123,7 +108,7 @@ def _VarintDecoder(mask, result_type):
result = 0 result = 0
shift = 0 shift = 0
while 1: while 1:
b = six.indexbytes(buffer, pos) b = buffer[pos]
result |= ((b & 0x7f) << shift) result |= ((b & 0x7f) << shift)
pos += 1 pos += 1
if not (b & 0x80): if not (b & 0x80):
@ -146,7 +131,7 @@ def _SignedVarintDecoder(bits, result_type):
result = 0 result = 0
shift = 0 shift = 0
while 1: while 1:
b = six.indexbytes(buffer, pos) b = buffer[pos]
result |= ((b & 0x7f) << shift) result |= ((b & 0x7f) << shift)
pos += 1 pos += 1
if not (b & 0x80): if not (b & 0x80):
@ -159,12 +144,9 @@ def _SignedVarintDecoder(bits, result_type):
raise _DecodeError('Too many bytes when decoding varint.') raise _DecodeError('Too many bytes when decoding varint.')
return DecodeVarint return DecodeVarint
# We force 32-bit values to int and 64-bit values to long to make # All 32-bit and 64-bit values are represented as int.
# alternate implementations where the distinction is more significant _DecodeVarint = _VarintDecoder((1 << 64) - 1, int)
# (e.g. the C++ implementation) simpler. _DecodeSignedVarint = _SignedVarintDecoder(64, int)
_DecodeVarint = _VarintDecoder((1 << 64) - 1, long)
_DecodeSignedVarint = _SignedVarintDecoder(64, long)
# Use these versions for values which must be limited to 32 bits. # Use these versions for values which must be limited to 32 bits.
_DecodeVarint32 = _VarintDecoder((1 << 32) - 1, int) _DecodeVarint32 = _VarintDecoder((1 << 32) - 1, int)
@ -189,7 +171,7 @@ def ReadTag(buffer, pos):
Tuple[bytes, int] of the tag data and new position. Tuple[bytes, int] of the tag data and new position.
""" """
start = pos start = pos
while six.indexbytes(buffer, pos) & 0x80: while buffer[pos] & 0x80:
pos += 1 pos += 1
pos += 1 pos += 1
@ -333,11 +315,11 @@ def _FloatDecoder():
if (float_bytes[3:4] in b'\x7F\xFF' and float_bytes[2:3] >= b'\x80'): if (float_bytes[3:4] in b'\x7F\xFF' and float_bytes[2:3] >= b'\x80'):
# If at least one significand bit is set... # If at least one significand bit is set...
if float_bytes[0:3] != b'\x00\x00\x80': if float_bytes[0:3] != b'\x00\x00\x80':
return (_NAN, new_pos) return (math.nan, new_pos)
# If sign bit is set... # If sign bit is set...
if float_bytes[3:4] == b'\xFF': if float_bytes[3:4] == b'\xFF':
return (_NEG_INF, new_pos) return (-math.inf, new_pos)
return (_POS_INF, new_pos) return (math.inf, new_pos)
# Note that we expect someone up-stack to catch struct.error and convert # Note that we expect someone up-stack to catch struct.error and convert
# it to _DecodeError -- this way we don't have to set up exception- # it to _DecodeError -- this way we don't have to set up exception-
@ -377,7 +359,7 @@ def _DoubleDecoder():
if ((double_bytes[7:8] in b'\x7F\xFF') if ((double_bytes[7:8] in b'\x7F\xFF')
and (double_bytes[6:7] >= b'\xF0') and (double_bytes[6:7] >= b'\xF0')
and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')): and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')):
return (_NAN, new_pos) return (math.nan, new_pos)
# Note that we expect someone up-stack to catch struct.error and convert # Note that we expect someone up-stack to catch struct.error and convert
# it to _DecodeError -- this way we don't have to set up exception- # it to _DecodeError -- this way we don't have to set up exception-
@ -559,31 +541,21 @@ BoolDecoder = _ModifiedDecoder(
def StringDecoder(field_number, is_repeated, is_packed, key, new_default, def StringDecoder(field_number, is_repeated, is_packed, key, new_default,
is_strict_utf8=False, clear_if_default=False): clear_if_default=False):
"""Returns a decoder for a string field.""" """Returns a decoder for a string field."""
local_DecodeVarint = _DecodeVarint local_DecodeVarint = _DecodeVarint
local_unicode = six.text_type
def _ConvertToUnicode(memview): def _ConvertToUnicode(memview):
"""Convert byte to unicode.""" """Convert byte to unicode."""
byte_str = memview.tobytes() byte_str = memview.tobytes()
try: try:
value = local_unicode(byte_str, 'utf-8') value = str(byte_str, 'utf-8')
except UnicodeDecodeError as e: except UnicodeDecodeError as e:
# add more information to the error message and re-raise it. # add more information to the error message and re-raise it.
e.reason = '%s in field: %s' % (e, key.full_name) e.reason = '%s in field: %s' % (e, key.full_name)
raise raise
if is_strict_utf8 and six.PY2 and sys.maxunicode > _UCS2_MAXUNICODE:
# Only do the check for python2 ucs4 when is_strict_utf8 enabled
if _SURROGATE_PATTERN.search(value):
reason = ('String field %s contains invalid UTF-8 data when parsing'
'a protocol buffer: surrogates not allowed. Use'
'the bytes type if you intend to send raw bytes.') % (
key.full_name)
raise message.DecodeError(reason)
return value return value
assert not is_packed assert not is_packed

@ -34,10 +34,7 @@
__author__ = 'matthewtoia@google.com (Matt Toia)' __author__ = 'matthewtoia@google.com (Matt Toia)'
try: import unittest
import unittest2 as unittest #PY26
except ImportError:
import unittest
import warnings import warnings
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2

@ -36,13 +36,9 @@ __author__ = 'matthewtoia@google.com (Matt Toia)'
import copy import copy
import os import os
import unittest
import warnings import warnings
try:
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_import_public_pb2 from google.protobuf import unittest_import_public_pb2
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2

@ -34,14 +34,9 @@
__author__ = 'robinson@google.com (Will Robinson)' __author__ = 'robinson@google.com (Will Robinson)'
import sys import unittest
import warnings import warnings
try:
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import unittest_custom_options_pb2 from google.protobuf import unittest_custom_options_pb2
from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2
@ -582,10 +577,7 @@ class GeneratedDescriptorTest(unittest.TestCase):
self.assertEqual(mapping, mapping) self.assertEqual(mapping, mapping)
self.assertGreater(len(mapping), 0) # Sized self.assertGreater(len(mapping), 0) # Sized
self.assertEqual(len(mapping), len(excepted_dict)) # Iterable self.assertEqual(len(mapping), len(excepted_dict)) # Iterable
if sys.version_info >= (3,): key, item = next(iter(mapping.items()))
key, item = next(iter(mapping.items()))
else:
key, item = mapping.items()[0]
self.assertIn(key, mapping) # Container self.assertIn(key, mapping) # Container
self.assertEqual(mapping.get(key), item) self.assertEqual(mapping.get(key), item)
with self.assertRaises(TypeError): with self.assertRaises(TypeError):
@ -598,13 +590,6 @@ class GeneratedDescriptorTest(unittest.TestCase):
# keys(), iterkeys() &co # keys(), iterkeys() &co
item = (next(iter(mapping.keys())), next(iter(mapping.values()))) item = (next(iter(mapping.keys())), next(iter(mapping.values())))
self.assertEqual(item, next(iter(mapping.items()))) self.assertEqual(item, next(iter(mapping.items())))
if sys.version_info < (3,):
def CheckItems(seq, iterator):
self.assertEqual(next(iterator), seq[0])
self.assertEqual(list(iterator), seq[1:])
CheckItems(mapping.keys(), mapping.iterkeys())
CheckItems(mapping.values(), mapping.itervalues())
CheckItems(mapping.items(), mapping.iteritems())
excepted_dict[key] = 'change value' excepted_dict[key] = 'change value'
self.assertNotEqual(mapping, excepted_dict) self.assertNotEqual(mapping, excepted_dict)
del excepted_dict[key] del excepted_dict[key]

@ -68,8 +68,6 @@ __author__ = 'kenton@google.com (Kenton Varda)'
import struct import struct
import six
from google.protobuf.internal import wire_format from google.protobuf.internal import wire_format
@ -372,7 +370,8 @@ def MapSizer(field_descriptor, is_message_map):
def _VarintEncoder(): def _VarintEncoder():
"""Return an encoder for a basic varint value (does not include tag).""" """Return an encoder for a basic varint value (does not include tag)."""
local_int2byte = six.int2byte local_int2byte = struct.Struct('>B').pack
def EncodeVarint(write, value, unused_deterministic=None): def EncodeVarint(write, value, unused_deterministic=None):
bits = value & 0x7f bits = value & 0x7f
value >>= 7 value >>= 7
@ -389,7 +388,8 @@ def _SignedVarintEncoder():
"""Return an encoder for a basic signed varint value (does not include """Return an encoder for a basic signed varint value (does not include
tag).""" tag)."""
local_int2byte = six.int2byte local_int2byte = struct.Struct('>B').pack
def EncodeSignedVarint(write, value, unused_deterministic=None): def EncodeSignedVarint(write, value, unused_deterministic=None):
if value < 0: if value < 0:
value += (1 << 64) value += (1 << 64)
@ -420,8 +420,7 @@ def _VarintBytes(value):
def TagBytes(field_number, wire_type): def TagBytes(field_number, wire_type):
"""Encode the given tag and return the bytes. Only called at startup.""" """Encode the given tag and return the bytes. Only called at startup."""
return six.binary_type( return bytes(_VarintBytes(wire_format.PackTag(field_number, wire_type)))
_VarintBytes(wire_format.PackTag(field_number, wire_type)))
# -------------------------------------------------------------------- # --------------------------------------------------------------------
# As with sizers (see above), we have a number of common encoder # As with sizers (see above), we have a number of common encoder

@ -37,8 +37,6 @@ on proto classes. For usage, see:
__author__ = 'rabsatt@google.com (Kevin Rabsatt)' __author__ = 'rabsatt@google.com (Kevin Rabsatt)'
import six
class EnumTypeWrapper(object): class EnumTypeWrapper(object):
"""A utility for finding the names of enum values.""" """A utility for finding the names of enum values."""
@ -57,7 +55,7 @@ class EnumTypeWrapper(object):
except KeyError: except KeyError:
pass # fall out to break exception chaining pass # fall out to break exception chaining
if not isinstance(number, six.integer_types): if not isinstance(number, int):
raise TypeError( raise TypeError(
'Enum value for {} must be an int, but got {} {!r}.'.format( 'Enum value for {} must be an int, but got {} {!r}.'.format(
self._enum_type.name, type(number), number)) self._enum_type.name, type(number), number))

@ -37,12 +37,8 @@ __author__ = 'jieluo@google.com (Jie Luo)'
import json import json
import math import math
import struct import struct
import sys
try: import unittest
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import any_pb2 from google.protobuf import any_pb2
from google.protobuf import duration_pb2 from google.protobuf import duration_pb2
@ -296,11 +292,8 @@ class JsonFormatTest(JsonFormatBase):
def testJsonEscapeString(self): def testJsonEscapeString(self):
message = json_format_proto3_pb2.TestMessage() message = json_format_proto3_pb2.TestMessage()
if sys.version_info[0] < 3: message.string_value = '&\n<\"\r>\b\t\f\\\001/'
message.string_value = '&\n<\"\r>\b\t\f\\\001/\xe2\x80\xa8\xe2\x80\xa9' message.string_value += (b'\xe2\x80\xa8\xe2\x80\xa9').decode('utf-8')
else:
message.string_value = '&\n<\"\r>\b\t\f\\\001/'
message.string_value += (b'\xe2\x80\xa8\xe2\x80\xa9').decode('utf-8')
self.assertEqual( self.assertEqual(
json_format.MessageToJson(message), json_format.MessageToJson(message),
'{\n "stringValue": ' '{\n "stringValue": '
@ -1039,8 +1032,6 @@ class JsonFormatTest(JsonFormatBase):
json_format.ParseError, json_format.ParseError,
'Failed to parse boolMap field: Expected "true" or "false", not null.', 'Failed to parse boolMap field: Expected "true" or "false", not null.',
json_format.Parse, text, message) json_format.Parse, text, message)
if sys.version_info < (2, 7):
return
text = r'{"stringMap": {"a": 3, "\u0061": 2}}' text = r'{"stringMap": {"a": 3, "\u0061": 2}}'
self.assertRaisesRegexp( self.assertRaisesRegexp(
json_format.ParseError, json_format.ParseError,
@ -1124,24 +1115,24 @@ class JsonFormatTest(JsonFormatBase):
text = '{"value": {"foo": 123}}' text = '{"value": {"foo": 123}}'
self.assertRaisesRegexp( self.assertRaisesRegexp(
json_format.ParseError, json_format.ParseError,
r"Timestamp JSON value not a string: {u?'foo': 123}", r"Timestamp JSON value not a string: {u?'foo': 123}", json_format.Parse,
json_format.Parse, text, message) text, message)
def testDurationInvalidStringValue(self): def testDurationInvalidStringValue(self):
message = json_format_proto3_pb2.TestDuration() message = json_format_proto3_pb2.TestDuration()
text = '{"value": {"foo": 123}}' text = '{"value": {"foo": 123}}'
self.assertRaisesRegexp( self.assertRaisesRegexp(
json_format.ParseError, json_format.ParseError,
r"Duration JSON value not a string: {u?'foo': 123}", r"Duration JSON value not a string: {u?'foo': 123}", json_format.Parse,
json_format.Parse, text, message) text, message)
def testFieldMaskInvalidStringValue(self): def testFieldMaskInvalidStringValue(self):
message = json_format_proto3_pb2.TestFieldMask() message = json_format_proto3_pb2.TestFieldMask()
text = '{"value": {"foo": 123}}' text = '{"value": {"foo": 123}}'
self.assertRaisesRegexp( self.assertRaisesRegexp(
json_format.ParseError, json_format.ParseError,
r"FieldMask JSON value not a string: {u?'foo': 123}", r"FieldMask JSON value not a string: {u?'foo': 123}", json_format.Parse,
json_format.Parse, text, message) text, message)
def testInvalidAny(self): def testInvalidAny(self):
message = any_pb2.Any() message = any_pb2.Any()

@ -34,10 +34,7 @@
__author__ = 'matthewtoia@google.com (Matt Toia)' __author__ = 'matthewtoia@google.com (Matt Toia)'
try: import unittest
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pb2
from google.protobuf.internal import api_implementation from google.protobuf.internal import api_implementation

@ -44,31 +44,17 @@ abstract interface.
__author__ = 'gps@google.com (Gregory P. Smith)' __author__ = 'gps@google.com (Gregory P. Smith)'
import collections
import copy import copy
import math import math
import operator import operator
import pickle import pickle
import pydoc import pydoc
import six
import sys import sys
import unittest
import warnings import warnings
try: cmp = lambda x, y: (x > y) - (x < y)
# Since python 3
import collections.abc as collections_abc
except ImportError:
# Won't work after python 3.8
import collections as collections_abc
try:
import unittest2 as unittest # PY26
except ImportError:
import unittest
try:
cmp # Python 2
except NameError:
cmp = lambda x, y: (x > y) - (x < y) # Python 3
from google.protobuf import map_proto2_unittest_pb2 from google.protobuf import map_proto2_unittest_pb2
from google.protobuf import map_unittest_pb2 from google.protobuf import map_unittest_pb2
@ -89,22 +75,6 @@ from google.protobuf import message
from google.protobuf.internal import _parameterized from google.protobuf.internal import _parameterized
UCS2_MAXUNICODE = 65535 UCS2_MAXUNICODE = 65535
if six.PY3:
long = int
# Python pre-2.6 does not have isinf() or isnan() functions, so we have
# to provide our own.
def isnan(val):
# NaN is never equal to itself.
return val != val
def isinf(val):
# Infinity times zero equals NaN.
return not isnan(val) and isnan(val * 0)
def IsPosInf(val):
return isinf(val) and (val > 0)
def IsNegInf(val):
return isinf(val) and (val < 0)
warnings.simplefilter('error', DeprecationWarning) warnings.simplefilter('error', DeprecationWarning)
@ -243,10 +213,10 @@ class MessageTest(unittest.TestCase):
golden_message = message_module.TestAllTypes() golden_message = message_module.TestAllTypes()
golden_message.ParseFromString(golden_data) golden_message.ParseFromString(golden_data)
self.assertTrue(IsPosInf(golden_message.optional_float)) self.assertEqual(golden_message.optional_float, math.inf)
self.assertTrue(IsPosInf(golden_message.optional_double)) self.assertEqual(golden_message.optional_double, math.inf)
self.assertTrue(IsPosInf(golden_message.repeated_float[0])) self.assertEqual(golden_message.repeated_float[0], math.inf)
self.assertTrue(IsPosInf(golden_message.repeated_double[0])) self.assertEqual(golden_message.repeated_double[0], math.inf)
self.assertEqual(golden_data, golden_message.SerializeToString()) self.assertEqual(golden_data, golden_message.SerializeToString())
def testNegativeInfinity(self, message_module): def testNegativeInfinity(self, message_module):
@ -263,10 +233,10 @@ class MessageTest(unittest.TestCase):
golden_message = message_module.TestAllTypes() golden_message = message_module.TestAllTypes()
golden_message.ParseFromString(golden_data) golden_message.ParseFromString(golden_data)
self.assertTrue(IsNegInf(golden_message.optional_float)) self.assertEqual(golden_message.optional_float, -math.inf)
self.assertTrue(IsNegInf(golden_message.optional_double)) self.assertEqual(golden_message.optional_double, -math.inf)
self.assertTrue(IsNegInf(golden_message.repeated_float[0])) self.assertEqual(golden_message.repeated_float[0], -math.inf)
self.assertTrue(IsNegInf(golden_message.repeated_double[0])) self.assertEqual(golden_message.repeated_double[0], -math.inf)
self.assertEqual(golden_data, golden_message.SerializeToString()) self.assertEqual(golden_data, golden_message.SerializeToString())
def testNotANumber(self, message_module): def testNotANumber(self, message_module):
@ -276,10 +246,10 @@ class MessageTest(unittest.TestCase):
b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF8\x7F') b'\xD1\x02\x00\x00\x00\x00\x00\x00\xF8\x7F')
golden_message = message_module.TestAllTypes() golden_message = message_module.TestAllTypes()
golden_message.ParseFromString(golden_data) golden_message.ParseFromString(golden_data)
self.assertTrue(isnan(golden_message.optional_float)) self.assertTrue(math.isnan(golden_message.optional_float))
self.assertTrue(isnan(golden_message.optional_double)) self.assertTrue(math.isnan(golden_message.optional_double))
self.assertTrue(isnan(golden_message.repeated_float[0])) self.assertTrue(math.isnan(golden_message.repeated_float[0]))
self.assertTrue(isnan(golden_message.repeated_double[0])) self.assertTrue(math.isnan(golden_message.repeated_double[0]))
# The protocol buffer may serialize to any one of multiple different # The protocol buffer may serialize to any one of multiple different
# representations of a NaN. Rather than verify a specific representation, # representations of a NaN. Rather than verify a specific representation,
@ -288,18 +258,18 @@ class MessageTest(unittest.TestCase):
serialized = golden_message.SerializeToString() serialized = golden_message.SerializeToString()
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
message.ParseFromString(serialized) message.ParseFromString(serialized)
self.assertTrue(isnan(message.optional_float)) self.assertTrue(math.isnan(message.optional_float))
self.assertTrue(isnan(message.optional_double)) self.assertTrue(math.isnan(message.optional_double))
self.assertTrue(isnan(message.repeated_float[0])) self.assertTrue(math.isnan(message.repeated_float[0]))
self.assertTrue(isnan(message.repeated_double[0])) self.assertTrue(math.isnan(message.repeated_double[0]))
def testPositiveInfinityPacked(self, message_module): def testPositiveInfinityPacked(self, message_module):
golden_data = (b'\xA2\x06\x04\x00\x00\x80\x7F' golden_data = (b'\xA2\x06\x04\x00\x00\x80\x7F'
b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\x7F') b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\x7F')
golden_message = message_module.TestPackedTypes() golden_message = message_module.TestPackedTypes()
golden_message.ParseFromString(golden_data) golden_message.ParseFromString(golden_data)
self.assertTrue(IsPosInf(golden_message.packed_float[0])) self.assertEqual(golden_message.packed_float[0], math.inf)
self.assertTrue(IsPosInf(golden_message.packed_double[0])) self.assertEqual(golden_message.packed_double[0], math.inf)
self.assertEqual(golden_data, golden_message.SerializeToString()) self.assertEqual(golden_data, golden_message.SerializeToString())
def testNegativeInfinityPacked(self, message_module): def testNegativeInfinityPacked(self, message_module):
@ -307,8 +277,8 @@ class MessageTest(unittest.TestCase):
b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\xFF') b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF0\xFF')
golden_message = message_module.TestPackedTypes() golden_message = message_module.TestPackedTypes()
golden_message.ParseFromString(golden_data) golden_message.ParseFromString(golden_data)
self.assertTrue(IsNegInf(golden_message.packed_float[0])) self.assertEqual(golden_message.packed_float[0], -math.inf)
self.assertTrue(IsNegInf(golden_message.packed_double[0])) self.assertEqual(golden_message.packed_double[0], -math.inf)
self.assertEqual(golden_data, golden_message.SerializeToString()) self.assertEqual(golden_data, golden_message.SerializeToString())
def testNotANumberPacked(self, message_module): def testNotANumberPacked(self, message_module):
@ -316,14 +286,14 @@ class MessageTest(unittest.TestCase):
b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF8\x7F') b'\xAA\x06\x08\x00\x00\x00\x00\x00\x00\xF8\x7F')
golden_message = message_module.TestPackedTypes() golden_message = message_module.TestPackedTypes()
golden_message.ParseFromString(golden_data) golden_message.ParseFromString(golden_data)
self.assertTrue(isnan(golden_message.packed_float[0])) self.assertTrue(math.isnan(golden_message.packed_float[0]))
self.assertTrue(isnan(golden_message.packed_double[0])) self.assertTrue(math.isnan(golden_message.packed_double[0]))
serialized = golden_message.SerializeToString() serialized = golden_message.SerializeToString()
message = message_module.TestPackedTypes() message = message_module.TestPackedTypes()
message.ParseFromString(serialized) message.ParseFromString(serialized)
self.assertTrue(isnan(message.packed_float[0])) self.assertTrue(math.isnan(message.packed_float[0]))
self.assertTrue(isnan(message.packed_double[0])) self.assertTrue(math.isnan(message.packed_double[0]))
def testExtremeFloatValues(self, message_module): def testExtremeFloatValues(self, message_module):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
@ -451,10 +421,7 @@ class MessageTest(unittest.TestCase):
def testHighPrecisionDoublePrinting(self, message_module): def testHighPrecisionDoublePrinting(self, message_module):
msg = message_module.TestAllTypes() msg = message_module.TestAllTypes()
msg.optional_double = 0.12345678912345678 msg.optional_double = 0.12345678912345678
if sys.version_info >= (3,): self.assertEqual(str(msg), 'optional_double: 0.12345678912345678\n')
self.assertEqual(str(msg), 'optional_double: 0.12345678912345678\n')
else:
self.assertEqual(str(msg), 'optional_double: 0.123456789123\n')
def testUnknownFieldPrinting(self, message_module): def testUnknownFieldPrinting(self, message_module):
populated = message_module.TestAllTypes() populated = message_module.TestAllTypes()
@ -694,7 +661,6 @@ class MessageTest(unittest.TestCase):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
get_bb = operator.attrgetter('bb') get_bb = operator.attrgetter('bb')
cmp_bb = lambda a, b: cmp(a.bb, b.bb)
message.repeated_nested_message.add().bb = 1 message.repeated_nested_message.add().bb = 1
message.repeated_nested_message.add().bb = 3 message.repeated_nested_message.add().bb = 3
message.repeated_nested_message.add().bb = 2 message.repeated_nested_message.add().bb = 2
@ -707,13 +673,6 @@ class MessageTest(unittest.TestCase):
message.repeated_nested_message.sort(key=get_bb, reverse=True) message.repeated_nested_message.sort(key=get_bb, reverse=True)
self.assertEqual([k.bb for k in message.repeated_nested_message], self.assertEqual([k.bb for k in message.repeated_nested_message],
[6, 5, 4, 3, 2, 1]) [6, 5, 4, 3, 2, 1])
if sys.version_info >= (3,): return # No cmp sorting in PY3.
message.repeated_nested_message.sort(sort_function=cmp_bb)
self.assertEqual([k.bb for k in message.repeated_nested_message],
[1, 2, 3, 4, 5, 6])
message.repeated_nested_message.sort(cmp=cmp_bb, reverse=True)
self.assertEqual([k.bb for k in message.repeated_nested_message],
[6, 5, 4, 3, 2, 1])
def testRepeatedScalarFieldSortArguments(self, message_module): def testRepeatedScalarFieldSortArguments(self, message_module):
"""Check sorting a scalar field using list.sort() arguments.""" """Check sorting a scalar field using list.sort() arguments."""
@ -726,12 +685,6 @@ class MessageTest(unittest.TestCase):
self.assertEqual(list(message.repeated_int32), [-1, -2, -3]) self.assertEqual(list(message.repeated_int32), [-1, -2, -3])
message.repeated_int32.sort(key=abs, reverse=True) message.repeated_int32.sort(key=abs, reverse=True)
self.assertEqual(list(message.repeated_int32), [-3, -2, -1]) self.assertEqual(list(message.repeated_int32), [-3, -2, -1])
if sys.version_info < (3,): # No cmp sorting in PY3.
abs_cmp = lambda a, b: cmp(abs(a), abs(b))
message.repeated_int32.sort(sort_function=abs_cmp)
self.assertEqual(list(message.repeated_int32), [-1, -2, -3])
message.repeated_int32.sort(cmp=abs_cmp, reverse=True)
self.assertEqual(list(message.repeated_int32), [-3, -2, -1])
message.repeated_string.append('aaa') message.repeated_string.append('aaa')
message.repeated_string.append('bb') message.repeated_string.append('bb')
@ -740,12 +693,6 @@ class MessageTest(unittest.TestCase):
self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa']) self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa'])
message.repeated_string.sort(key=len, reverse=True) message.repeated_string.sort(key=len, reverse=True)
self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c']) self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c'])
if sys.version_info < (3,): # No cmp sorting in PY3.
len_cmp = lambda a, b: cmp(len(a), len(b))
message.repeated_string.sort(sort_function=len_cmp)
self.assertEqual(list(message.repeated_string), ['c', 'bb', 'aaa'])
message.repeated_string.sort(cmp=len_cmp, reverse=True)
self.assertEqual(list(message.repeated_string), ['aaa', 'bb', 'c'])
def testRepeatedFieldsComparable(self, message_module): def testRepeatedFieldsComparable(self, message_module):
m1 = message_module.TestAllTypes() m1 = message_module.TestAllTypes()
@ -763,30 +710,11 @@ class MessageTest(unittest.TestCase):
m2.repeated_nested_message.add().bb = 2 m2.repeated_nested_message.add().bb = 2
m2.repeated_nested_message.add().bb = 3 m2.repeated_nested_message.add().bb = 3
if sys.version_info >= (3,): return # No cmp() in PY3.
# These comparisons should not raise errors.
_ = m1 < m2
_ = m1.repeated_nested_message < m2.repeated_nested_message
# Make sure cmp always works. If it wasn't defined, these would be
# id() comparisons and would all fail.
self.assertEqual(cmp(m1, m2), 0)
self.assertEqual(cmp(m1.repeated_int32, m2.repeated_int32), 0)
self.assertEqual(cmp(m1.repeated_int32, [0, 1, 2]), 0)
self.assertEqual(cmp(m1.repeated_nested_message,
m2.repeated_nested_message), 0)
with self.assertRaises(TypeError):
# Can't compare repeated composite containers to lists.
cmp(m1.repeated_nested_message, m2.repeated_nested_message[:])
# TODO(anuraag): Implement extensiondict comparison in C++ and then add test
def testRepeatedFieldsAreSequences(self, message_module): def testRepeatedFieldsAreSequences(self, message_module):
m = message_module.TestAllTypes() m = message_module.TestAllTypes()
self.assertIsInstance(m.repeated_int32, collections_abc.MutableSequence) self.assertIsInstance(m.repeated_int32, collections.abc.MutableSequence)
self.assertIsInstance(m.repeated_nested_message, self.assertIsInstance(m.repeated_nested_message,
collections_abc.MutableSequence) collections.abc.MutableSequence)
def testRepeatedFieldsNotHashable(self, message_module): def testRepeatedFieldsNotHashable(self, message_module):
m = message_module.TestAllTypes() m = message_module.TestAllTypes()
@ -860,8 +788,8 @@ class MessageTest(unittest.TestCase):
# that a sub-sliced memoryview is not being used. # that a sub-sliced memoryview is not being used.
self.assertIsInstance(m1.optional_bytes, bytes) self.assertIsInstance(m1.optional_bytes, bytes)
self.assertIsInstance(m1.repeated_bytes[0], bytes) self.assertIsInstance(m1.repeated_bytes[0], bytes)
self.assertIsInstance(m1.optional_string, six.text_type) self.assertIsInstance(m1.optional_string, str)
self.assertIsInstance(m1.repeated_string[0], six.text_type) self.assertIsInstance(m1.repeated_string[0], str)
def testMergeFromEmpty(self, message_module): def testMergeFromEmpty(self, message_module):
m1 = message_module.TestAllTypes() m1 = message_module.TestAllTypes()
@ -1049,10 +977,10 @@ class MessageTest(unittest.TestCase):
in the value being converted to a Unicode string.""" in the value being converted to a Unicode string."""
m = message_module.TestAllTypes() m = message_module.TestAllTypes()
m.optional_string = str('') m.optional_string = str('')
self.assertIsInstance(m.optional_string, six.text_type) self.assertIsInstance(m.optional_string, str)
def testLongValuedSlice(self, message_module): def testLongValuedSlice(self, message_module):
"""It should be possible to use long-valued indices in slices. """It should be possible to use int-valued indices in slices.
This didn't used to work in the v2 C++ implementation. This didn't used to work in the v2 C++ implementation.
""" """
@ -1060,12 +988,12 @@ class MessageTest(unittest.TestCase):
# Repeated scalar # Repeated scalar
m.repeated_int32.append(1) m.repeated_int32.append(1)
sl = m.repeated_int32[long(0):long(len(m.repeated_int32))] sl = m.repeated_int32[int(0):int(len(m.repeated_int32))]
self.assertEqual(len(m.repeated_int32), len(sl)) self.assertEqual(len(m.repeated_int32), len(sl))
# Repeated composite # Repeated composite
m.repeated_nested_message.add().bb = 3 m.repeated_nested_message.add().bb = 3
sl = m.repeated_nested_message[long(0):long(len(m.repeated_nested_message))] sl = m.repeated_nested_message[int(0):int(len(m.repeated_nested_message))]
self.assertEqual(len(m.repeated_nested_message), len(sl)) self.assertEqual(len(m.repeated_nested_message), len(sl))
def testExtendShouldNotSwallowExceptions(self, message_module): def testExtendShouldNotSwallowExceptions(self, message_module):
@ -1794,7 +1722,7 @@ class Proto3Test(unittest.TestCase):
self.assertTrue(111 in msg.map_int32_bytes) self.assertTrue(111 in msg.map_int32_bytes)
self.assertTrue(888 in msg.map_int32_enum) self.assertTrue(888 in msg.map_int32_enum)
self.assertIsInstance(msg.map_string_string['abc'], six.text_type) self.assertIsInstance(msg.map_string_string['abc'], str)
# Accessing an unset key still throws TypeError if the type of the key # Accessing an unset key still throws TypeError if the type of the key
# is incorrect. # is incorrect.
@ -1909,8 +1837,8 @@ class Proto3Test(unittest.TestCase):
self.assertEqual(key, unicode_obj) self.assertEqual(key, unicode_obj)
self.assertEqual(value, unicode_obj) self.assertEqual(value, unicode_obj)
self.assertIsInstance(key, six.text_type) self.assertIsInstance(key, str)
self.assertIsInstance(value, six.text_type) self.assertIsInstance(value, str)
def testMessageMap(self): def testMessageMap(self):
msg = map_unittest_pb2.TestMap() msg = map_unittest_pb2.TestMap()
@ -2110,7 +2038,7 @@ class Proto3Test(unittest.TestCase):
with self.assertRaisesRegexp( with self.assertRaisesRegexp(
TypeError, TypeError,
r'Parameter to MergeFrom\(\) must be instance of same class: expected ' r'Parameter to MergeFrom\(\) must be instance of same class: expected '
r'.*TestMap got int\.'): r'.+TestMap got int\.'):
msg.MergeFrom(1) msg.MergeFrom(1)
def testCopyFromBadType(self): def testCopyFromBadType(self):
@ -2118,15 +2046,15 @@ class Proto3Test(unittest.TestCase):
with self.assertRaisesRegexp( with self.assertRaisesRegexp(
TypeError, TypeError,
r'Parameter to [A-Za-z]*From\(\) must be instance of same class: ' r'Parameter to [A-Za-z]*From\(\) must be instance of same class: '
r'expected .*TestMap got int\.'): r'expected .+TestMap got int\.'):
msg.CopyFrom(1) msg.CopyFrom(1)
def testIntegerMapWithLongs(self): def testIntegerMapWithLongs(self):
msg = map_unittest_pb2.TestMap() msg = map_unittest_pb2.TestMap()
msg.map_int32_int32[long(-123)] = long(-456) msg.map_int32_int32[int(-123)] = int(-456)
msg.map_int64_int64[long(-2**33)] = long(-2**34) msg.map_int64_int64[int(-2**33)] = int(-2**34)
msg.map_uint32_uint32[long(123)] = long(456) msg.map_uint32_uint32[int(123)] = int(456)
msg.map_uint64_uint64[long(2**33)] = long(2**34) msg.map_uint64_uint64[int(2**33)] = int(2**34)
serialized = msg.SerializeToString() serialized = msg.SerializeToString()
msg2 = map_unittest_pb2.TestMap() msg2 = map_unittest_pb2.TestMap()
@ -2238,54 +2166,6 @@ class Proto3Test(unittest.TestCase):
matching_dict = {2: 4, 3: 6, 4: 8} matching_dict = {2: 4, 3: 6, 4: 8}
self.assertMapIterEquals(msg.map_int32_int32.items(), matching_dict) self.assertMapIterEquals(msg.map_int32_int32.items(), matching_dict)
def testPython2Map(self):
if sys.version_info < (3,):
msg = map_unittest_pb2.TestMap()
msg.map_int32_int32[2] = 4
msg.map_int32_int32[3] = 6
msg.map_int32_int32[4] = 8
msg.map_int32_int32[5] = 10
map_int32 = msg.map_int32_int32
self.assertEqual(4, len(map_int32))
msg2 = map_unittest_pb2.TestMap()
msg2.ParseFromString(msg.SerializeToString())
def CheckItems(seq, iterator):
self.assertEqual(next(iterator), seq[0])
self.assertEqual(list(iterator), seq[1:])
CheckItems(map_int32.items(), map_int32.iteritems())
CheckItems(map_int32.keys(), map_int32.iterkeys())
CheckItems(map_int32.values(), map_int32.itervalues())
self.assertEqual(6, map_int32.get(3))
self.assertEqual(None, map_int32.get(999))
self.assertEqual(6, map_int32.pop(3))
self.assertEqual(0, map_int32.pop(3))
self.assertEqual(3, len(map_int32))
key, value = map_int32.popitem()
self.assertEqual(2 * key, value)
self.assertEqual(2, len(map_int32))
map_int32.clear()
self.assertEqual(0, len(map_int32))
with self.assertRaises(KeyError):
map_int32.popitem()
self.assertEqual(0, map_int32.setdefault(2))
self.assertEqual(1, len(map_int32))
map_int32.update(msg2.map_int32_int32)
self.assertEqual(4, len(map_int32))
with self.assertRaises(TypeError):
map_int32.update(msg2.map_int32_int32,
msg2.map_int32_int32)
with self.assertRaises(TypeError):
map_int32.update(0)
with self.assertRaises(TypeError):
map_int32.update(value=12)
def testMapItems(self): def testMapItems(self):
# Map items used to have strange behaviors when use c extension. Because # Map items used to have strange behaviors when use c extension. Because
# [] may reorder the map and invalidate any exsting iterators. # [] may reorder the map and invalidate any exsting iterators.
@ -2457,11 +2337,12 @@ class Proto3Test(unittest.TestCase):
def testMapsAreMapping(self): def testMapsAreMapping(self):
msg = map_unittest_pb2.TestMap() msg = map_unittest_pb2.TestMap()
self.assertIsInstance(msg.map_int32_int32, collections_abc.Mapping) self.assertIsInstance(msg.map_int32_int32, collections.abc.Mapping)
self.assertIsInstance(msg.map_int32_int32, collections_abc.MutableMapping) self.assertIsInstance(msg.map_int32_int32, collections.abc.MutableMapping)
self.assertIsInstance(msg.map_int32_foreign_message, collections_abc.Mapping) self.assertIsInstance(msg.map_int32_foreign_message,
collections.abc.Mapping)
self.assertIsInstance(msg.map_int32_foreign_message, self.assertIsInstance(msg.map_int32_foreign_message,
collections_abc.MutableMapping) collections.abc.MutableMapping)
def testMapsCompare(self): def testMapsCompare(self):
msg = map_unittest_pb2.TestMap() msg = map_unittest_pb2.TestMap()
@ -2501,19 +2382,7 @@ class Proto3Test(unittest.TestCase):
optional_string=u'\ud001') optional_string=u'\ud001')
self.assertEqual(msg.optional_string, u'\ud001') self.assertEqual(msg.optional_string, u'\ud001')
@unittest.skipIf(six.PY2, 'Surrogates are acceptable in python2')
def testSurrogatesInPython3(self): def testSurrogatesInPython3(self):
# Surrogates like U+D83D is an invalid unicode character, it is
# supported by Python2 only because in some builds, unicode strings
# use 2-bytes code units. Since Python 3.3, we don't have this problem.
#
# Surrogates are utf16 code units, in a unicode string they are invalid
# characters even when they appear in pairs like u'\ud801\udc01'. Protobuf
# Python3 reject such cases at setters and parsers. Python2 accpect it
# to keep same features with the language itself. 'Unpaired pairs'
# like u'\ud801' are rejected at parsers when strict utf8 check is enabled
# in proto3 to keep same behavior with c extension.
# Surrogates are rejected at setters in Python3. # Surrogates are rejected at setters in Python3.
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
unittest_proto3_arena_pb2.TestAllTypes( unittest_proto3_arena_pb2.TestAllTypes(
@ -2528,33 +2397,6 @@ class Proto3Test(unittest.TestCase):
unittest_proto3_arena_pb2.TestAllTypes( unittest_proto3_arena_pb2.TestAllTypes(
optional_string=u'\ud801\ud801') optional_string=u'\ud801\ud801')
@unittest.skipIf(six.PY3 or sys.maxunicode == UCS2_MAXUNICODE,
'Surrogates are rejected at setters in Python3')
def testSurrogatesInPython2(self):
# Test optional_string=u'\ud801\udc01'.
# surrogate pair is acceptable in python2.
msg = unittest_proto3_arena_pb2.TestAllTypes(
optional_string=u'\ud801\udc01')
# TODO(jieluo): Change pure python to have same behavior with c extension.
# Some build in python2 consider u'\ud801\udc01' and u'\U00010401' are
# equal, some are not equal.
if api_implementation.Type() == 'python':
self.assertEqual(msg.optional_string, u'\ud801\udc01')
else:
self.assertEqual(msg.optional_string, u'\U00010401')
serialized = msg.SerializeToString()
msg2 = unittest_proto3_arena_pb2.TestAllTypes()
msg2.MergeFromString(serialized)
self.assertEqual(msg2.optional_string, u'\U00010401')
# Python2 does not reject surrogates at setters.
msg = unittest_proto3_arena_pb2.TestAllTypes(
optional_string=b'\xed\xa0\x81')
unittest_proto3_arena_pb2.TestAllTypes(
optional_string=u'\ud801')
unittest_proto3_arena_pb2.TestAllTypes(
optional_string=u'\ud801\ud801')
@testing_refleaks.TestCase @testing_refleaks.TestCase
class ValidTypeNamesTest(unittest.TestCase): class ValidTypeNamesTest(unittest.TestCase):
@ -2636,9 +2478,8 @@ class PackedFieldTest(unittest.TestCase):
self.assertEqual(golden_data, message.SerializeToString()) self.assertEqual(golden_data, message.SerializeToString())
@unittest.skipIf(api_implementation.Type() != 'cpp' or @unittest.skipIf(api_implementation.Type() != 'cpp',
sys.version_info < (2, 7), 'explicit tests of the C++ implementation')
'explicit tests of the C++ implementation for PY27 and above')
@testing_refleaks.TestCase @testing_refleaks.TestCase
class OversizeProtosTest(unittest.TestCase): class OversizeProtosTest(unittest.TestCase):

@ -55,9 +55,6 @@ import struct
import sys import sys
import weakref import weakref
import six
from six.moves import range
# We use "as" to avoid name collisions with variables. # We use "as" to avoid name collisions with variables.
from google.protobuf.internal import api_implementation from google.protobuf.internal import api_implementation
from google.protobuf.internal import containers from google.protobuf.internal import containers
@ -284,13 +281,6 @@ def _IsMessageMapField(field):
return value_type.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE return value_type.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE
def _IsStrictUtf8Check(field):
if field.containing_type.syntax != 'proto3':
return False
enforce_utf8 = True
return enforce_utf8
def _AttachFieldHelpers(cls, field_descriptor): def _AttachFieldHelpers(cls, field_descriptor):
is_repeated = (field_descriptor.label == _FieldDescriptor.LABEL_REPEATED) is_repeated = (field_descriptor.label == _FieldDescriptor.LABEL_REPEATED)
is_packable = (is_repeated and is_packable = (is_repeated and
@ -348,11 +338,10 @@ def _AttachFieldHelpers(cls, field_descriptor):
field_descriptor, _GetInitializeDefaultForMap(field_descriptor), field_descriptor, _GetInitializeDefaultForMap(field_descriptor),
is_message_map) is_message_map)
elif decode_type == _FieldDescriptor.TYPE_STRING: elif decode_type == _FieldDescriptor.TYPE_STRING:
is_strict_utf8_check = _IsStrictUtf8Check(field_descriptor)
field_decoder = decoder.StringDecoder( field_decoder = decoder.StringDecoder(
field_descriptor.number, is_repeated, is_packed, field_descriptor.number, is_repeated, is_packed,
field_descriptor, field_descriptor._default_constructor, field_descriptor, field_descriptor._default_constructor,
is_strict_utf8_check, clear_if_default) clear_if_default)
elif field_descriptor.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: elif field_descriptor.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE:
field_decoder = type_checkers.TYPE_TO_DECODER[decode_type]( field_decoder = type_checkers.TYPE_TO_DECODER[decode_type](
field_descriptor.number, is_repeated, is_packed, field_descriptor.number, is_repeated, is_packed,
@ -485,7 +474,7 @@ def _ReraiseTypeErrorWithFieldName(message_name, field_name):
exc = TypeError('%s for field %s.%s' % (str(exc), message_name, field_name)) exc = TypeError('%s for field %s.%s' % (str(exc), message_name, field_name))
# re-raise possibly-amended exception with original traceback: # re-raise possibly-amended exception with original traceback:
six.reraise(type(exc), exc, sys.exc_info()[2]) raise exc.with_traceback(sys.exc_info()[2])
def _AddInitMethod(message_descriptor, cls): def _AddInitMethod(message_descriptor, cls):
@ -498,7 +487,7 @@ def _AddInitMethod(message_descriptor, cls):
enum_type with the same name. If the value is not a string, it's enum_type with the same name. If the value is not a string, it's
returned as-is. (No conversion or bounds-checking is done.) returned as-is. (No conversion or bounds-checking is done.)
""" """
if isinstance(value, six.string_types): if isinstance(value, str):
try: try:
return enum_type.values_by_name[value].number return enum_type.values_by_name[value].number
except KeyError: except KeyError:
@ -1305,6 +1294,14 @@ def _AddIsInitializedMethod(message_descriptor, cls):
cls.FindInitializationErrors = FindInitializationErrors cls.FindInitializationErrors = FindInitializationErrors
def _FullyQualifiedClassName(klass):
module = klass.__module__
name = getattr(klass, '__qualname__', klass.__name__)
if module in (None, 'builtins', '__builtin__'):
return name
return module + '.' + name
def _AddMergeFromMethod(cls): def _AddMergeFromMethod(cls):
LABEL_REPEATED = _FieldDescriptor.LABEL_REPEATED LABEL_REPEATED = _FieldDescriptor.LABEL_REPEATED
CPPTYPE_MESSAGE = _FieldDescriptor.CPPTYPE_MESSAGE CPPTYPE_MESSAGE = _FieldDescriptor.CPPTYPE_MESSAGE
@ -1313,7 +1310,8 @@ def _AddMergeFromMethod(cls):
if not isinstance(msg, cls): if not isinstance(msg, cls):
raise TypeError( raise TypeError(
'Parameter to MergeFrom() must be instance of same class: ' 'Parameter to MergeFrom() must be instance of same class: '
'expected %s got %s.' % (cls.__name__, msg.__class__.__name__)) 'expected %s got %s.' % (_FullyQualifiedClassName(cls),
_FullyQualifiedClassName(msg.__class__)))
assert msg is not self assert msg is not self
self._Modified() self._Modified()

@ -38,14 +38,9 @@ pure-Python protocol compiler.
import copy import copy
import gc import gc
import operator import operator
import six
import struct import struct
import warnings import warnings
import unittest
try:
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_import_pb2
from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_mset_pb2
@ -67,10 +62,6 @@ from google.protobuf.internal import decoder
from google.protobuf.internal import _parameterized from google.protobuf.internal import _parameterized
if six.PY3:
long = int # pylint: disable=redefined-builtin,invalid-name
warnings.simplefilter('error', DeprecationWarning) warnings.simplefilter('error', DeprecationWarning)
@ -411,7 +402,7 @@ class ReflectionTest(unittest.TestCase):
TestGetAndDeserialize('optional_int32', 1, int) TestGetAndDeserialize('optional_int32', 1, int)
TestGetAndDeserialize('optional_int32', 1 << 30, int) TestGetAndDeserialize('optional_int32', 1 << 30, int)
TestGetAndDeserialize('optional_uint32', 1 << 30, int) TestGetAndDeserialize('optional_uint32', 1 << 30, int)
integer_64 = long integer_64 = int
if struct.calcsize('L') == 4: if struct.calcsize('L') == 4:
# Python only has signed ints, so 32-bit python can't fit an uint32 # Python only has signed ints, so 32-bit python can't fit an uint32
# in an int. # in an int.
@ -852,14 +843,14 @@ class ReflectionTest(unittest.TestCase):
setattr, proto, 'optional_bytes', u'unicode object') setattr, proto, 'optional_bytes', u'unicode object')
# Check that the default value is of python's 'unicode' type. # Check that the default value is of python's 'unicode' type.
self.assertEqual(type(proto.optional_string), six.text_type) self.assertEqual(type(proto.optional_string), str)
proto.optional_string = six.text_type('Testing') proto.optional_string = str('Testing')
self.assertEqual(proto.optional_string, str('Testing')) self.assertEqual(proto.optional_string, str('Testing'))
# Assign a value of type 'str' which can be encoded in UTF-8. # Assign a value of type 'str' which can be encoded in UTF-8.
proto.optional_string = str('Testing') proto.optional_string = str('Testing')
self.assertEqual(proto.optional_string, six.text_type('Testing')) self.assertEqual(proto.optional_string, str('Testing'))
# Try to assign a 'bytes' object which contains non-UTF-8. # Try to assign a 'bytes' object which contains non-UTF-8.
self.assertRaises(ValueError, self.assertRaises(ValueError,
@ -874,8 +865,7 @@ class ReflectionTest(unittest.TestCase):
def testBytesInTextFormat(self, message_module): def testBytesInTextFormat(self, message_module):
proto = message_module.TestAllTypes(optional_bytes=b'\x00\x7f\x80\xff') proto = message_module.TestAllTypes(optional_bytes=b'\x00\x7f\x80\xff')
self.assertEqual(u'optional_bytes: "\\000\\177\\200\\377"\n', self.assertEqual(u'optional_bytes: "\\000\\177\\200\\377"\n', str(proto))
six.text_type(proto))
def testEmptyNestedMessage(self, message_module): def testEmptyNestedMessage(self, message_module):
proto = message_module.TestAllTypes() proto = message_module.TestAllTypes()
@ -1508,7 +1498,9 @@ class Proto2ReflectionTest(unittest.TestCase):
options=descriptor_pb2.MessageOptions(), options=descriptor_pb2.MessageOptions(),
# pylint: disable=protected-access # pylint: disable=protected-access
create_key=descriptor._internal_create_key) create_key=descriptor._internal_create_key)
class MyProtoClass(six.with_metaclass(reflection.GeneratedProtocolMessageType, message.Message)):
class MyProtoClass(
message.Message, metaclass=reflection.GeneratedProtocolMessageType):
DESCRIPTOR = mydescriptor DESCRIPTOR = mydescriptor
myproto_instance = MyProtoClass() myproto_instance = MyProtoClass()
self.assertEqual(0, myproto_instance.foo_field) self.assertEqual(0, myproto_instance.foo_field)
@ -1556,8 +1548,8 @@ class Proto2ReflectionTest(unittest.TestCase):
self.assertTrue('price' in desc.fields_by_name) self.assertTrue('price' in desc.fields_by_name)
self.assertTrue('owners' in desc.fields_by_name) self.assertTrue('owners' in desc.fields_by_name)
class CarMessage(six.with_metaclass(reflection.GeneratedProtocolMessageType, class CarMessage(
message.Message)): message.Message, metaclass=reflection.GeneratedProtocolMessageType):
DESCRIPTOR = desc DESCRIPTOR = desc
prius = CarMessage() prius = CarMessage()
@ -2094,7 +2086,7 @@ class Proto2ReflectionTest(unittest.TestCase):
bytes_read = message2.MergeFromString(raw.item[0].message) bytes_read = message2.MergeFromString(raw.item[0].message)
self.assertEqual(len(raw.item[0].message), bytes_read) self.assertEqual(len(raw.item[0].message), bytes_read)
self.assertEqual(type(message2.str), six.text_type) self.assertEqual(type(message2.str), str)
self.assertEqual(message2.str, test_utf8) self.assertEqual(message2.str, test_utf8)
# The pure Python API throws an exception on MergeFromString(), # The pure Python API throws an exception on MergeFromString(),
@ -3340,7 +3332,8 @@ class ClassAPITest(unittest.TestCase):
msg_descriptor = descriptor.MakeDescriptor( msg_descriptor = descriptor.MakeDescriptor(
file_descriptor.message_type[0]) file_descriptor.message_type[0])
class MessageClass(six.with_metaclass(reflection.GeneratedProtocolMessageType, message.Message)): class MessageClass(
message.Message, metaclass=reflection.GeneratedProtocolMessageType):
DESCRIPTOR = msg_descriptor DESCRIPTOR = msg_descriptor
msg = MessageClass() msg = MessageClass()
msg_str = ( msg_str = (

@ -35,10 +35,7 @@
__author__ = 'petar@google.com (Petar Petrov)' __author__ = 'petar@google.com (Petar Petrov)'
try: import unittest
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2
from google.protobuf import service_reflection from google.protobuf import service_reflection

@ -32,10 +32,7 @@
"""Tests for google.protobuf.symbol_database.""" """Tests for google.protobuf.symbol_database."""
try: import unittest
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2
from google.protobuf import descriptor from google.protobuf import descriptor

@ -38,18 +38,10 @@ If sys.gettotalrefcount() is not available (because Python was built without
the Py_DEBUG option), then this module is a no-op and tests will run normally. the Py_DEBUG option), then this module is a no-op and tests will run normally.
""" """
import copyreg
import gc import gc
import sys import sys
import unittest
try:
import copy_reg as copyreg #PY26
except ImportError:
import copyreg
try:
import unittest2 as unittest #PY26
except ImportError:
import unittest
class LocalTestResult(unittest.TestResult): class LocalTestResult(unittest.TestResult):

@ -32,10 +32,7 @@
"""Tests for google.protobuf.text_encoding.""" """Tests for google.protobuf.text_encoding."""
try: import unittest
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import text_encoding from google.protobuf import text_encoding

@ -39,13 +39,7 @@ import re
import string import string
import textwrap import textwrap
import six import unittest
# pylint: disable=g-import-not-at-top
try:
import unittest2 as unittest # PY26
except ImportError:
import unittest
from google.protobuf import any_pb2 from google.protobuf import any_pb2
from google.protobuf import any_test_pb2 from google.protobuf import any_test_pb2
@ -207,14 +201,10 @@ class TextFormatMessageToStringTests(TextFormatBase):
'repeated_double: 1.23456789\n' 'repeated_double: 1.23456789\n'
'repeated_double: 1.234567898\n' 'repeated_double: 1.234567898\n'
'repeated_double: 1.2345678987\n' 'repeated_double: 1.2345678987\n'
'repeated_double: 1.23456789876\n' + 'repeated_double: 1.23456789876\n'
('repeated_double: 1.23456789876\n' 'repeated_double: 1.234567898765\n'
'repeated_double: 1.23456789877\n' 'repeated_double: 1.2345678987654\n'
'repeated_double: 1.23456789877\n' 'repeated_double: 1.23456789876543\n'
if six.PY2 else
'repeated_double: 1.234567898765\n'
'repeated_double: 1.2345678987654\n'
'repeated_double: 1.23456789876543\n') +
'repeated_double: 1.2e+100\n' 'repeated_double: 1.2e+100\n'
'repeated_double: 1.23e+100\n' 'repeated_double: 1.23e+100\n'
'repeated_double: 1.234e+100\n' 'repeated_double: 1.234e+100\n'
@ -225,18 +215,14 @@ class TextFormatMessageToStringTests(TextFormatBase):
'repeated_double: 1.23456789e+100\n' 'repeated_double: 1.23456789e+100\n'
'repeated_double: 1.234567898e+100\n' 'repeated_double: 1.234567898e+100\n'
'repeated_double: 1.2345678987e+100\n' 'repeated_double: 1.2345678987e+100\n'
'repeated_double: 1.23456789876e+100\n' + 'repeated_double: 1.23456789876e+100\n'
('repeated_double: 1.23456789877e+100\n' 'repeated_double: 1.234567898765e+100\n'
'repeated_double: 1.23456789877e+100\n' 'repeated_double: 1.2345678987654e+100\n'
'repeated_double: 1.23456789877e+100\n' 'repeated_double: 1.23456789876543e+100\n')
if six.PY2 else
'repeated_double: 1.234567898765e+100\n'
'repeated_double: 1.2345678987654e+100\n'
'repeated_double: 1.23456789876543e+100\n'))
def testPrintExoticUnicodeSubclass(self, message_module): def testPrintExoticUnicodeSubclass(self, message_module):
class UnicodeSub(six.text_type): class UnicodeSub(str):
pass pass
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
@ -364,7 +350,7 @@ class TextFormatMessageToStringTests(TextFormatBase):
message.repeated_string.append(u'\u00fc\t\ua71f') message.repeated_string.append(u'\u00fc\t\ua71f')
text = text_format.MessageToString(message, as_utf8=True) text = text_format.MessageToString(message, as_utf8=True)
golden_unicode = u'repeated_string: "\u00fc\\t\ua71f"\n' golden_unicode = u'repeated_string: "\u00fc\\t\ua71f"\n'
golden_text = golden_unicode if six.PY3 else golden_unicode.encode('utf-8') golden_text = golden_unicode
# MessageToString always returns a native str. # MessageToString always returns a native str.
self.CompareToGoldenText(text, golden_text) self.CompareToGoldenText(text, golden_text)
parsed_message = message_module.TestAllTypes() parsed_message = message_module.TestAllTypes()
@ -777,16 +763,18 @@ class TextFormatParserTests(TextFormatBase):
def testParseSingleWord(self, message_module): def testParseSingleWord(self, message_module):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
text = 'foo' text = 'foo'
six.assertRaisesRegex(self, text_format.ParseError, ( self.assertRaisesRegex(
r'1:1 : Message type "\w+.TestAllTypes" has no field named ' text_format.ParseError,
r'"foo".'), text_format.Parse, text, message) (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
r'"foo".'), text_format.Parse, text, message)
def testParseUnknownField(self, message_module): def testParseUnknownField(self, message_module):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
text = 'unknown_field: 8\n' text = 'unknown_field: 8\n'
six.assertRaisesRegex(self, text_format.ParseError, ( self.assertRaisesRegex(
r'1:1 : Message type "\w+.TestAllTypes" has no field named ' text_format.ParseError,
r'"unknown_field".'), text_format.Parse, text, message) (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
r'"unknown_field".'), text_format.Parse, text, message)
text = ('optional_int32: 123\n' text = ('optional_int32: 123\n'
'unknown_field: 8\n' 'unknown_field: 8\n'
'optional_nested_message { bb: 45 }') 'optional_nested_message { bb: 45 }')
@ -797,19 +785,19 @@ class TextFormatParserTests(TextFormatBase):
def testParseBadEnumValue(self, message_module): def testParseBadEnumValue(self, message_module):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
text = 'optional_nested_enum: BARR' text = 'optional_nested_enum: BARR'
six.assertRaisesRegex(self, text_format.ParseError, self.assertRaisesRegex(text_format.ParseError,
(r'1:23 : \'optional_nested_enum: BARR\': ' (r'1:23 : \'optional_nested_enum: BARR\': '
r'Enum type "\w+.TestAllTypes.NestedEnum" ' r'Enum type "\w+.TestAllTypes.NestedEnum" '
r'has no value named BARR.'), text_format.Parse, r'has no value named BARR.'), text_format.Parse,
text, message) text, message)
def testParseBadIntValue(self, message_module): def testParseBadIntValue(self, message_module):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
text = 'optional_int32: bork' text = 'optional_int32: bork'
six.assertRaisesRegex(self, text_format.ParseError, self.assertRaisesRegex(text_format.ParseError,
('1:17 : \'optional_int32: bork\': ' ('1:17 : \'optional_int32: bork\': '
'Couldn\'t parse integer: bork'), 'Couldn\'t parse integer: bork'), text_format.Parse,
text_format.Parse, text, message) text, message)
def testParseStringFieldUnescape(self, message_module): def testParseStringFieldUnescape(self, message_module):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
@ -842,8 +830,8 @@ class TextFormatParserTests(TextFormatBase):
def testParseMultipleOneof(self, message_module): def testParseMultipleOneof(self, message_module):
m_string = '\n'.join(['oneof_uint32: 11', 'oneof_string: "foo"']) m_string = '\n'.join(['oneof_uint32: 11', 'oneof_string: "foo"'])
m2 = message_module.TestAllTypes() m2 = message_module.TestAllTypes()
with six.assertRaisesRegex(self, text_format.ParseError, with self.assertRaisesRegex(text_format.ParseError,
' is specified along with field '): ' is specified along with field '):
text_format.Parse(m_string, m2) text_format.Parse(m_string, m2)
# This example contains non-ASCII codepoint unicode data as literals # This example contains non-ASCII codepoint unicode data as literals
@ -922,27 +910,28 @@ class TextFormatParserTests(TextFormatBase):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
text = ('optional_nested_message { bb: 1 } ' text = ('optional_nested_message { bb: 1 } '
'optional_nested_message { bb: 2 }') 'optional_nested_message { bb: 2 }')
six.assertRaisesRegex(self, text_format.ParseError, ( self.assertRaisesRegex(
r'1:59 : Message type "\w+.TestAllTypes" ' text_format.ParseError,
r'should not have multiple "optional_nested_message" fields.'), (r'1:59 : Message type "\w+.TestAllTypes" '
text_format.Parse, text, r'should not have multiple "optional_nested_message" fields.'),
message) text_format.Parse, text, message)
def testParseDuplicateScalars(self, message_module): def testParseDuplicateScalars(self, message_module):
message = message_module.TestAllTypes() message = message_module.TestAllTypes()
text = ('optional_int32: 42 ' 'optional_int32: 67') text = ('optional_int32: 42 ' 'optional_int32: 67')
six.assertRaisesRegex(self, text_format.ParseError, ( self.assertRaisesRegex(
r'1:36 : Message type "\w+.TestAllTypes" should not ' text_format.ParseError,
r'have multiple "optional_int32" fields.'), text_format.Parse, text, (r'1:36 : Message type "\w+.TestAllTypes" should not '
message) r'have multiple "optional_int32" fields.'), text_format.Parse, text,
message)
def testParseExistingScalarInMessage(self, message_module): def testParseExistingScalarInMessage(self, message_module):
message = message_module.TestAllTypes(optional_int32=42) message = message_module.TestAllTypes(optional_int32=42)
text = 'optional_int32: 67' text = 'optional_int32: 67'
six.assertRaisesRegex(self, text_format.ParseError, self.assertRaisesRegex(text_format.ParseError,
(r'Message type "\w+.TestAllTypes" should not ' (r'Message type "\w+.TestAllTypes" should not '
r'have multiple "optional_int32" fields.'), r'have multiple "optional_int32" fields.'),
text_format.Parse, text, message) text_format.Parse, text, message)
@_parameterized.parameters(unittest_pb2, unittest_proto3_arena_pb2) @_parameterized.parameters(unittest_pb2, unittest_proto3_arena_pb2)
@ -1398,14 +1387,14 @@ class Proto2Tests(TextFormatBase):
# Can't parse field number without set allow_field_number=True. # Can't parse field number without set allow_field_number=True.
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()
text = '34:1\n' text = '34:1\n'
six.assertRaisesRegex(self, text_format.ParseError, ( self.assertRaisesRegex(
r'1:1 : Message type "\w+.TestAllTypes" has no field named ' text_format.ParseError,
r'"34".'), text_format.Parse, text, message) (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
r'"34".'), text_format.Parse, text, message)
# Can't parse if field number is not found. # Can't parse if field number is not found.
text = '1234:1\n' text = '1234:1\n'
six.assertRaisesRegex( self.assertRaisesRegex(
self,
text_format.ParseError, text_format.ParseError,
(r'1:1 : Message type "\w+.TestAllTypes" has no field named ' (r'1:1 : Message type "\w+.TestAllTypes" has no field named '
r'"1234".'), r'"1234".'),
@ -1492,13 +1481,13 @@ class Proto2Tests(TextFormatBase):
' i:\n' # Missing value. ' i:\n' # Missing value.
' }\n' ' }\n'
'}\n') '}\n')
six.assertRaisesRegex(self, self.assertRaisesRegex(
text_format.ParseError, text_format.ParseError,
'Invalid field value: }', 'Invalid field value: }',
text_format.Parse, text_format.Parse,
malformed, malformed,
message, message,
allow_unknown_extension=True) allow_unknown_extension=True)
message = unittest_mset_pb2.TestMessageSetContainer() message = unittest_mset_pb2.TestMessageSetContainer()
malformed = ('message_set {\n' malformed = ('message_set {\n'
@ -1506,13 +1495,13 @@ class Proto2Tests(TextFormatBase):
' str: "malformed string\n' # Missing closing quote. ' str: "malformed string\n' # Missing closing quote.
' }\n' ' }\n'
'}\n') '}\n')
six.assertRaisesRegex(self, self.assertRaisesRegex(
text_format.ParseError, text_format.ParseError,
'Invalid field value: "', 'Invalid field value: "',
text_format.Parse, text_format.Parse,
malformed, malformed,
message, message,
allow_unknown_extension=True) allow_unknown_extension=True)
message = unittest_mset_pb2.TestMessageSetContainer() message = unittest_mset_pb2.TestMessageSetContainer()
malformed = ('message_set {\n' malformed = ('message_set {\n'
@ -1520,13 +1509,13 @@ class Proto2Tests(TextFormatBase):
' str: "malformed\n multiline\n string\n' ' str: "malformed\n multiline\n string\n'
' }\n' ' }\n'
'}\n') '}\n')
six.assertRaisesRegex(self, self.assertRaisesRegex(
text_format.ParseError, text_format.ParseError,
'Invalid field value: "', 'Invalid field value: "',
text_format.Parse, text_format.Parse,
malformed, malformed,
message, message,
allow_unknown_extension=True) allow_unknown_extension=True)
message = unittest_mset_pb2.TestMessageSetContainer() message = unittest_mset_pb2.TestMessageSetContainer()
malformed = ('message_set {\n' malformed = ('message_set {\n'
@ -1534,28 +1523,28 @@ class Proto2Tests(TextFormatBase):
' i: -5\n' ' i: -5\n'
' \n' # Missing '>' here. ' \n' # Missing '>' here.
'}\n') '}\n')
six.assertRaisesRegex(self, self.assertRaisesRegex(
text_format.ParseError, text_format.ParseError,
'5:1 : \'}\': Expected ">".', '5:1 : \'}\': Expected ">".',
text_format.Parse, text_format.Parse,
malformed, malformed,
message, message,
allow_unknown_extension=True) allow_unknown_extension=True)
# Don't allow unknown fields with allow_unknown_extension=True. # Don't allow unknown fields with allow_unknown_extension=True.
message = unittest_mset_pb2.TestMessageSetContainer() message = unittest_mset_pb2.TestMessageSetContainer()
malformed = ('message_set {\n' malformed = ('message_set {\n'
' unknown_field: true\n' ' unknown_field: true\n'
'}\n') '}\n')
six.assertRaisesRegex(self, self.assertRaisesRegex(
text_format.ParseError, text_format.ParseError,
('2:3 : Message type ' ('2:3 : Message type '
'"proto2_wireformat_unittest.TestMessageSet" has no' '"proto2_wireformat_unittest.TestMessageSet" has no'
' field named "unknown_field".'), ' field named "unknown_field".'),
text_format.Parse, text_format.Parse,
malformed, malformed,
message, message,
allow_unknown_extension=True) allow_unknown_extension=True)
# Parse known extension correctly. # Parse known extension correctly.
message = unittest_mset_pb2.TestMessageSetContainer() message = unittest_mset_pb2.TestMessageSetContainer()
@ -1585,22 +1574,24 @@ class Proto2Tests(TextFormatBase):
def testParseBadExtension(self): def testParseBadExtension(self):
message = unittest_pb2.TestAllExtensions() message = unittest_pb2.TestAllExtensions()
text = '[unknown_extension]: 8\n' text = '[unknown_extension]: 8\n'
six.assertRaisesRegex(self, text_format.ParseError, self.assertRaisesRegex(
'1:2 : Extension "unknown_extension" not registered.', text_format.ParseError,
text_format.Parse, text, message) '1:2 : Extension "unknown_extension" not registered.',
text_format.Parse, text, message)
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()
six.assertRaisesRegex(self, text_format.ParseError, ( self.assertRaisesRegex(
'1:2 : Message type "protobuf_unittest.TestAllTypes" does not have ' text_format.ParseError,
'extensions.'), text_format.Parse, text, message) ('1:2 : Message type "protobuf_unittest.TestAllTypes" does not have '
'extensions.'), text_format.Parse, text, message)
def testParseNumericUnknownEnum(self): def testParseNumericUnknownEnum(self):
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()
text = 'optional_nested_enum: 100' text = 'optional_nested_enum: 100'
six.assertRaisesRegex(self, text_format.ParseError, self.assertRaisesRegex(text_format.ParseError,
(r'1:23 : \'optional_nested_enum: 100\': ' (r'1:23 : \'optional_nested_enum: 100\': '
r'Enum type "\w+.TestAllTypes.NestedEnum" ' r'Enum type "\w+.TestAllTypes.NestedEnum" '
r'has no value with number 100.'), text_format.Parse, r'has no value with number 100.'),
text, message) text_format.Parse, text, message)
def testMergeDuplicateExtensionScalars(self): def testMergeDuplicateExtensionScalars(self):
message = unittest_pb2.TestAllExtensions() message = unittest_pb2.TestAllExtensions()
@ -1614,30 +1605,32 @@ class Proto2Tests(TextFormatBase):
message = unittest_pb2.TestAllExtensions() message = unittest_pb2.TestAllExtensions()
text = ('[protobuf_unittest.optional_int32_extension]: 42 ' text = ('[protobuf_unittest.optional_int32_extension]: 42 '
'[protobuf_unittest.optional_int32_extension]: 67') '[protobuf_unittest.optional_int32_extension]: 67')
six.assertRaisesRegex(self, text_format.ParseError, ( self.assertRaisesRegex(
'1:96 : Message type "protobuf_unittest.TestAllExtensions" ' text_format.ParseError,
'should not have multiple ' ('1:96 : Message type "protobuf_unittest.TestAllExtensions" '
'"protobuf_unittest.optional_int32_extension" extensions.'), 'should not have multiple '
text_format.Parse, text, message) '"protobuf_unittest.optional_int32_extension" extensions.'),
text_format.Parse, text, message)
def testParseDuplicateExtensionMessages(self): def testParseDuplicateExtensionMessages(self):
message = unittest_pb2.TestAllExtensions() message = unittest_pb2.TestAllExtensions()
text = ('[protobuf_unittest.optional_nested_message_extension]: {} ' text = ('[protobuf_unittest.optional_nested_message_extension]: {} '
'[protobuf_unittest.optional_nested_message_extension]: {}') '[protobuf_unittest.optional_nested_message_extension]: {}')
six.assertRaisesRegex(self, text_format.ParseError, ( self.assertRaisesRegex(
'1:114 : Message type "protobuf_unittest.TestAllExtensions" ' text_format.ParseError,
'should not have multiple ' ('1:114 : Message type "protobuf_unittest.TestAllExtensions" '
'"protobuf_unittest.optional_nested_message_extension" extensions.'), 'should not have multiple '
text_format.Parse, text, message) '"protobuf_unittest.optional_nested_message_extension" extensions.'),
text_format.Parse, text, message)
def testParseGroupNotClosed(self): def testParseGroupNotClosed(self):
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()
text = 'RepeatedGroup: <' text = 'RepeatedGroup: <'
six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected ">".', self.assertRaisesRegex(text_format.ParseError, '1:16 : Expected ">".',
text_format.Parse, text, message) text_format.Parse, text, message)
text = 'RepeatedGroup: {' text = 'RepeatedGroup: {'
six.assertRaisesRegex(self, text_format.ParseError, '1:16 : Expected "}".', self.assertRaisesRegex(text_format.ParseError, '1:16 : Expected "}".',
text_format.Parse, text, message) text_format.Parse, text, message)
def testParseEmptyGroup(self): def testParseEmptyGroup(self):
message = unittest_pb2.TestAllTypes() message = unittest_pb2.TestAllTypes()

@ -45,16 +45,8 @@ TYPE_TO_DESERIALIZE_METHOD: A dictionary with field types and deserialization
__author__ = 'robinson@google.com (Will Robinson)' __author__ = 'robinson@google.com (Will Robinson)'
try: import ctypes
import ctypes
except Exception: # pylint: disable=broad-except
ctypes = None
import struct
import numbers import numbers
import six
if six.PY3:
long = int
from google.protobuf.internal import api_implementation from google.protobuf.internal import api_implementation
from google.protobuf.internal import decoder from google.protobuf.internal import decoder
@ -66,10 +58,7 @@ _FieldDescriptor = descriptor.FieldDescriptor
def TruncateToFourByteFloat(original): def TruncateToFourByteFloat(original):
if ctypes: return ctypes.c_float(original).value
return ctypes.c_float(original).value
else:
return struct.unpack('<f', struct.pack('<f', original))[0]
def ToShortestFloat(original): def ToShortestFloat(original):
@ -162,14 +151,13 @@ class IntValueChecker(object):
def CheckValue(self, proposed_value): def CheckValue(self, proposed_value):
if not isinstance(proposed_value, numbers.Integral): if not isinstance(proposed_value, numbers.Integral):
message = ('%.1024r has type %s, but expected one of: %s' % message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), six.integer_types)) (proposed_value, type(proposed_value), (int,)))
raise TypeError(message) raise TypeError(message)
if not self._MIN <= int(proposed_value) <= self._MAX: if not self._MIN <= int(proposed_value) <= self._MAX:
raise ValueError('Value out of range: %d' % proposed_value) raise ValueError('Value out of range: %d' % proposed_value)
# We force 32-bit values to int and 64-bit values to long to make # We force all values to int to make alternate implementations where the
# alternate implementations where the distinction is more significant # distinction is more significant (e.g. the C++ implementation) simpler.
# (e.g. the C++ implementation) simpler. proposed_value = int(proposed_value)
proposed_value = self._TYPE(proposed_value)
return proposed_value return proposed_value
def DefaultValue(self): def DefaultValue(self):
@ -186,7 +174,7 @@ class EnumValueChecker(object):
def CheckValue(self, proposed_value): def CheckValue(self, proposed_value):
if not isinstance(proposed_value, numbers.Integral): if not isinstance(proposed_value, numbers.Integral):
message = ('%.1024r has type %s, but expected one of: %s' % message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), six.integer_types)) (proposed_value, type(proposed_value), (int,)))
raise TypeError(message) raise TypeError(message)
if int(proposed_value) not in self._enum_type.values_by_number: if int(proposed_value) not in self._enum_type.values_by_number:
raise ValueError('Unknown enum value: %d' % proposed_value) raise ValueError('Unknown enum value: %d' % proposed_value)
@ -204,9 +192,9 @@ class UnicodeValueChecker(object):
""" """
def CheckValue(self, proposed_value): def CheckValue(self, proposed_value):
if not isinstance(proposed_value, (bytes, six.text_type)): if not isinstance(proposed_value, (bytes, str)):
message = ('%.1024r has type %s, but expected one of: %s' % message = ('%.1024r has type %s, but expected one of: %s' %
(proposed_value, type(proposed_value), (bytes, six.text_type))) (proposed_value, type(proposed_value), (bytes, str)))
raise TypeError(message) raise TypeError(message)
# If the value is of type 'bytes' make sure that it is valid UTF-8 data. # If the value is of type 'bytes' make sure that it is valid UTF-8 data.
@ -237,25 +225,21 @@ class Int32ValueChecker(IntValueChecker):
# efficient. # efficient.
_MIN = -2147483648 _MIN = -2147483648
_MAX = 2147483647 _MAX = 2147483647
_TYPE = int
class Uint32ValueChecker(IntValueChecker): class Uint32ValueChecker(IntValueChecker):
_MIN = 0 _MIN = 0
_MAX = (1 << 32) - 1 _MAX = (1 << 32) - 1
_TYPE = int
class Int64ValueChecker(IntValueChecker): class Int64ValueChecker(IntValueChecker):
_MIN = -(1 << 63) _MIN = -(1 << 63)
_MAX = (1 << 63) - 1 _MAX = (1 << 63) - 1
_TYPE = long
class Uint64ValueChecker(IntValueChecker): class Uint64ValueChecker(IntValueChecker):
_MIN = 0 _MIN = 0
_MAX = (1 << 64) - 1 _MAX = (1 << 64) - 1
_TYPE = long
# The max 4 bytes float is about 3.4028234663852886e+38 # The max 4 bytes float is about 3.4028234663852886e+38

@ -35,11 +35,9 @@
__author__ = 'bohdank@google.com (Bohdan Koval)' __author__ = 'bohdank@google.com (Bohdan Koval)'
try:
import unittest2 as unittest #PY26
except ImportError:
import unittest
import sys import sys
import unittest
from google.protobuf import map_unittest_pb2 from google.protobuf import map_unittest_pb2
from google.protobuf import unittest_mset_pb2 from google.protobuf import unittest_mset_pb2
from google.protobuf import unittest_pb2 from google.protobuf import unittest_pb2

@ -43,7 +43,6 @@ __author__ = 'jieluo@google.com (Jie Luo)'
import calendar import calendar
from datetime import datetime from datetime import datetime
from datetime import timedelta from datetime import timedelta
import six
try: try:
# Since python 3 # Since python 3
@ -143,9 +142,8 @@ class Timestamp(object):
Raises: Raises:
ValueError: On parsing problems. ValueError: On parsing problems.
""" """
if not isinstance(value, six.string_types): if not isinstance(value, str):
raise ValueError( raise ValueError('Timestamp JSON value not a string: {!r}'.format(value))
'Timestamp JSON value not a string: {!r}'.format(value))
timezone_offset = value.find('Z') timezone_offset = value.find('Z')
if timezone_offset == -1: if timezone_offset == -1:
timezone_offset = value.find('+') timezone_offset = value.find('+')
@ -306,9 +304,8 @@ class Duration(object):
Raises: Raises:
ValueError: On parsing problems. ValueError: On parsing problems.
""" """
if not isinstance(value, six.string_types): if not isinstance(value, str):
raise ValueError( raise ValueError('Duration JSON value not a string: {!r}'.format(value))
'Duration JSON value not a string: {!r}'.format(value))
if len(value) < 1 or value[-1] != 's': if len(value) < 1 or value[-1] != 's':
raise ValueError( raise ValueError(
'Duration must end with letter "s": {0}.'.format(value)) 'Duration must end with letter "s": {0}.'.format(value))
@ -434,9 +431,8 @@ class FieldMask(object):
def FromJsonString(self, value): def FromJsonString(self, value):
"""Converts string to FieldMask according to proto3 JSON spec.""" """Converts string to FieldMask according to proto3 JSON spec."""
if not isinstance(value, six.string_types): if not isinstance(value, str):
raise ValueError( raise ValueError('FieldMask JSON value not a string: {!r}'.format(value))
'FieldMask JSON value not a string: {!r}'.format(value))
self.Clear() self.Clear()
if value: if value:
for path in value.split(','): for path in value.split(','):
@ -721,9 +717,6 @@ def _AddFieldPaths(node, prefix, field_mask):
_AddFieldPaths(node[name], child_path, field_mask) _AddFieldPaths(node[name], child_path, field_mask)
_INT_OR_FLOAT = six.integer_types + (float,)
def _SetStructValue(struct_value, value): def _SetStructValue(struct_value, value):
if value is None: if value is None:
struct_value.null_value = 0 struct_value.null_value = 0
@ -731,9 +724,9 @@ def _SetStructValue(struct_value, value):
# Note: this check must come before the number check because in Python # Note: this check must come before the number check because in Python
# True and False are also considered numbers. # True and False are also considered numbers.
struct_value.bool_value = value struct_value.bool_value = value
elif isinstance(value, six.string_types): elif isinstance(value, str):
struct_value.string_value = value struct_value.string_value = value
elif isinstance(value, _INT_OR_FLOAT): elif isinstance(value, (int, float)):
struct_value.number_value = value struct_value.number_value = value
elif isinstance(value, (dict, Struct)): elif isinstance(value, (dict, Struct)):
struct_value.struct_value.Clear() struct_value.struct_value.Clear()

@ -34,19 +34,9 @@
__author__ = 'jieluo@google.com (Jie Luo)' __author__ = 'jieluo@google.com (Jie Luo)'
import collections.abc as collections_abc
import datetime import datetime
import unittest
try:
# Since python 3
import collections.abc as collections_abc
except ImportError:
# Won't work after python 3.8
import collections as collections_abc
try:
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import any_pb2 from google.protobuf import any_pb2
from google.protobuf import duration_pb2 from google.protobuf import duration_pb2

@ -34,10 +34,7 @@
__author__ = 'robinson@google.com (Will Robinson)' __author__ = 'robinson@google.com (Will Robinson)'
try: import unittest
import unittest2 as unittest #PY26
except ImportError:
import unittest
from google.protobuf import message from google.protobuf import message
from google.protobuf.internal import wire_format from google.protobuf.internal import wire_format

@ -42,24 +42,15 @@ Simple usage example:
__author__ = 'jieluo@google.com (Jie Luo)' __author__ = 'jieluo@google.com (Jie Luo)'
# pylint: disable=g-statement-before-imports,g-import-not-at-top
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict # PY26
# pylint: enable=g-statement-before-imports,g-import-not-at-top
import base64 import base64
from collections import OrderedDict
import json import json
import math import math
from operator import methodcaller from operator import methodcaller
import re import re
import sys import sys
import six
from google.protobuf.internal import type_checkers from google.protobuf.internal import type_checkers
from google.protobuf import descriptor from google.protobuf import descriptor
from google.protobuf import symbol_database from google.protobuf import symbol_database
@ -78,9 +69,8 @@ _INFINITY = 'Infinity'
_NEG_INFINITY = '-Infinity' _NEG_INFINITY = '-Infinity'
_NAN = 'NaN' _NAN = 'NaN'
_UNPAIRED_SURROGATE_PATTERN = re.compile(six.u( _UNPAIRED_SURROGATE_PATTERN = re.compile(
r'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]' u'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]')
))
_VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$') _VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')
@ -426,7 +416,8 @@ def Parse(text, message, ignore_unknown_fields=False, descriptor_pool=None):
Raises:: Raises::
ParseError: On JSON parsing problems. ParseError: On JSON parsing problems.
""" """
if not isinstance(text, six.text_type): text = text.decode('utf-8') if not isinstance(text, str):
text = text.decode('utf-8')
try: try:
js = json.loads(text, object_pairs_hook=_DuplicateChecker) js = json.loads(text, object_pairs_hook=_DuplicateChecker)
except ValueError as e: except ValueError as e:
@ -455,7 +446,7 @@ def ParseDict(js_dict,
return message return message
_INT_OR_FLOAT = six.integer_types + (float,) _INT_OR_FLOAT = (int, float)
class _Parser(object): class _Parser(object):
@ -646,7 +637,7 @@ class _Parser(object):
message.null_value = 0 message.null_value = 0
elif isinstance(value, bool): elif isinstance(value, bool):
message.bool_value = value message.bool_value = value
elif isinstance(value, six.string_types): elif isinstance(value, str):
message.string_value = value message.string_value = value
elif isinstance(value, _INT_OR_FLOAT): elif isinstance(value, _INT_OR_FLOAT):
message.number_value = value message.number_value = value
@ -729,7 +720,7 @@ def _ConvertScalarFieldValue(value, field, require_str=False):
return _ConvertBool(value, require_str) return _ConvertBool(value, require_str)
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
if field.type == descriptor.FieldDescriptor.TYPE_BYTES: if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
if isinstance(value, six.text_type): if isinstance(value, str):
encoded = value.encode('utf-8') encoded = value.encode('utf-8')
else: else:
encoded = value encoded = value
@ -776,7 +767,7 @@ def _ConvertInteger(value):
if isinstance(value, float) and not value.is_integer(): if isinstance(value, float) and not value.is_integer():
raise ParseError('Couldn\'t parse integer: {0}.'.format(value)) raise ParseError('Couldn\'t parse integer: {0}.'.format(value))
if isinstance(value, six.text_type) and value.find(' ') != -1: if isinstance(value, str) and value.find(' ') != -1:
raise ParseError('Couldn\'t parse integer: "{0}".'.format(value)) raise ParseError('Couldn\'t parse integer: "{0}".'.format(value))
if isinstance(value, bool): if isinstance(value, bool):

@ -98,8 +98,6 @@ class MessageFactory(object):
A class describing the passed in descriptor. A class describing the passed in descriptor.
""" """
descriptor_name = descriptor.name descriptor_name = descriptor.name
if str is bytes: # PY2
descriptor_name = descriptor.name.encode('ascii', 'ignore')
result_class = _GENERATED_PROTOCOL_MESSAGE_TYPE( result_class = _GENERATED_PROTOCOL_MESSAGE_TYPE(
descriptor_name, descriptor_name,
(message.Message,), (message.Message,),

@ -30,10 +30,7 @@
"""Dynamic Protobuf class creator.""" """Dynamic Protobuf class creator."""
try: from collections import OrderedDict
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict #PY26
import hashlib import hashlib
import os import os

@ -31,8 +31,6 @@
"""Encoding related utilities.""" """Encoding related utilities."""
import re import re
import six
_cescape_chr_to_symbol_map = {} _cescape_chr_to_symbol_map = {}
_cescape_chr_to_symbol_map[9] = r'\t' # optional escape _cescape_chr_to_symbol_map[9] = r'\t' # optional escape
_cescape_chr_to_symbol_map[10] = r'\n' # optional escape _cescape_chr_to_symbol_map[10] = r'\n' # optional escape
@ -72,14 +70,11 @@ def CEscape(text, as_utf8):
# escapes whereas our C++ unescaping function allows hex escapes to be any # escapes whereas our C++ unescaping function allows hex escapes to be any
# length. So, "\0011".encode('string_escape') ends up being "\\x011", which # length. So, "\0011".encode('string_escape') ends up being "\\x011", which
# will be decoded in C++ as a single-character string with char code 0x11. # will be decoded in C++ as a single-character string with char code 0x11.
if six.PY3: text_is_unicode = isinstance(text, str)
text_is_unicode = isinstance(text, str) if as_utf8 and text_is_unicode:
if as_utf8 and text_is_unicode: # We're already unicode, no processing beyond control char escapes.
# We're already unicode, no processing beyond control char escapes. return text.translate(_cescape_chr_to_symbol_map)
return text.translate(_cescape_chr_to_symbol_map) ord_ = ord if text_is_unicode else lambda x: x # bytes iterate as ints.
ord_ = ord if text_is_unicode else lambda x: x # bytes iterate as ints.
else:
ord_ = ord # PY2
if as_utf8: if as_utf8:
return ''.join(_cescape_unicode_to_str[ord_(c)] for c in text) return ''.join(_cescape_unicode_to_str[ord_(c)] for c in text)
return ''.join(_cescape_byte_to_str[ord_(c)] for c in text) return ''.join(_cescape_byte_to_str[ord_(c)] for c in text)
@ -109,9 +104,7 @@ def CUnescape(text):
# allow single-digit hex escapes (like '\xf'). # allow single-digit hex escapes (like '\xf').
result = _CUNESCAPE_HEX.sub(ReplaceHex, text) result = _CUNESCAPE_HEX.sub(ReplaceHex, text)
if six.PY2: return (result.encode('utf-8') # Make it bytes to allow decode.
return result.decode('string_escape')
return (result.encode('utf-8') # PY3: Make it bytes to allow decode.
.decode('unicode_escape') .decode('unicode_escape')
# Make it bytes again to return the proper type. # Make it bytes again to return the proper type.
.encode('raw_unicode_escape')) .encode('raw_unicode_escape'))

@ -48,16 +48,12 @@ import encodings.unicode_escape # pylint: disable=unused-import
import io import io
import math import math
import re import re
import six
from google.protobuf.internal import decoder from google.protobuf.internal import decoder
from google.protobuf.internal import type_checkers from google.protobuf.internal import type_checkers
from google.protobuf import descriptor from google.protobuf import descriptor
from google.protobuf import text_encoding from google.protobuf import text_encoding
if six.PY3:
long = int # pylint: disable=redefined-builtin,invalid-name
# pylint: disable=g-import-not-at-top # pylint: disable=g-import-not-at-top
__all__ = ['MessageToString', 'Parse', 'PrintMessage', 'PrintField', __all__ = ['MessageToString', 'Parse', 'PrintMessage', 'PrintField',
'PrintFieldValue', 'Merge', 'MessageToBytes'] 'PrintFieldValue', 'Merge', 'MessageToBytes']
@ -102,15 +98,9 @@ class ParseError(Error):
class TextWriter(object): class TextWriter(object):
def __init__(self, as_utf8): def __init__(self, as_utf8):
if six.PY2: self._writer = io.StringIO()
self._writer = io.BytesIO()
else:
self._writer = io.StringIO()
def write(self, val): def write(self, val):
if six.PY2:
if isinstance(val, six.text_type):
val = val.encode('utf-8')
return self._writer.write(val) return self._writer.write(val)
def close(self): def close(self):
@ -562,7 +552,7 @@ class _Printer(object):
# Note: this is called only when value has at least one element. # Note: this is called only when value has at least one element.
self._PrintFieldName(field) self._PrintFieldName(field)
self.out.write(' [') self.out.write(' [')
for i in six.moves.range(len(value) - 1): for i in range(len(value) - 1):
self.PrintFieldValue(field, value[i]) self.PrintFieldValue(field, value[i])
self.out.write(', ') self.out.write(', ')
self.PrintFieldValue(field, value[-1]) self.PrintFieldValue(field, value[-1])
@ -608,7 +598,7 @@ class _Printer(object):
out.write(str(value)) out.write(str(value))
elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
out.write('\"') out.write('\"')
if isinstance(value, six.text_type) and (six.PY2 or not self.as_utf8): if isinstance(value, str) and not self.as_utf8:
out_value = value.encode('utf-8') out_value = value.encode('utf-8')
else: else:
out_value = value out_value = value
@ -839,12 +829,9 @@ class _Parser(object):
ParseError: On text parsing problems. ParseError: On text parsing problems.
""" """
# Tokenize expects native str lines. # Tokenize expects native str lines.
if six.PY2: str_lines = (
str_lines = (line if isinstance(line, str) else line.encode('utf-8') line if isinstance(line, str) else line.decode('utf-8')
for line in lines) for line in lines)
else:
str_lines = (line if isinstance(line, str) else line.decode('utf-8')
for line in lines)
tokenizer = Tokenizer(str_lines) tokenizer = Tokenizer(str_lines)
while not tokenizer.AtEnd(): while not tokenizer.AtEnd():
self._MergeField(tokenizer, message) self._MergeField(tokenizer, message)
@ -1395,17 +1382,14 @@ class Tokenizer(object):
def TryConsumeInteger(self): def TryConsumeInteger(self):
try: try:
# Note: is_long only affects value type, not whether an error is raised.
self.ConsumeInteger() self.ConsumeInteger()
return True return True
except ParseError: except ParseError:
return False return False
def ConsumeInteger(self, is_long=False): def ConsumeInteger(self):
"""Consumes an integer number. """Consumes an integer number.
Args:
is_long: True if the value should be returned as a long integer.
Returns: Returns:
The integer parsed. The integer parsed.
@ -1413,7 +1397,7 @@ class Tokenizer(object):
ParseError: If an integer couldn't be consumed. ParseError: If an integer couldn't be consumed.
""" """
try: try:
result = _ParseAbstractInteger(self.token, is_long=is_long) result = _ParseAbstractInteger(self.token)
except ValueError as e: except ValueError as e:
raise self.ParseError(str(e)) raise self.ParseError(str(e))
self.NextToken() self.NextToken()
@ -1476,7 +1460,7 @@ class Tokenizer(object):
""" """
the_bytes = self.ConsumeByteString() the_bytes = self.ConsumeByteString()
try: try:
return six.text_type(the_bytes, 'utf-8') return str(the_bytes, 'utf-8')
except UnicodeDecodeError as e: except UnicodeDecodeError as e:
raise self._StringParseError(e) raise self._StringParseError(e)
@ -1650,14 +1634,6 @@ def _ConsumeUint64(tokenizer):
return _ConsumeInteger(tokenizer, is_signed=False, is_long=True) return _ConsumeInteger(tokenizer, is_signed=False, is_long=True)
def _TryConsumeInteger(tokenizer, is_signed=False, is_long=False):
try:
_ConsumeInteger(tokenizer, is_signed=is_signed, is_long=is_long)
return True
except ParseError:
return False
def _ConsumeInteger(tokenizer, is_signed=False, is_long=False): def _ConsumeInteger(tokenizer, is_signed=False, is_long=False):
"""Consumes an integer number from tokenizer. """Consumes an integer number from tokenizer.
@ -1695,7 +1671,7 @@ def ParseInteger(text, is_signed=False, is_long=False):
ValueError: Thrown Iff the text is not a valid integer. ValueError: Thrown Iff the text is not a valid integer.
""" """
# Do the actual parsing. Exception handling is propagated to caller. # Do the actual parsing. Exception handling is propagated to caller.
result = _ParseAbstractInteger(text, is_long=is_long) result = _ParseAbstractInteger(text)
# Check if the integer is sane. Exceptions handled by callers. # Check if the integer is sane. Exceptions handled by callers.
checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)] checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)]
@ -1703,12 +1679,11 @@ def ParseInteger(text, is_signed=False, is_long=False):
return result return result
def _ParseAbstractInteger(text, is_long=False): def _ParseAbstractInteger(text):
"""Parses an integer without checking size/signedness. """Parses an integer without checking size/signedness.
Args: Args:
text: The text to parse. text: The text to parse.
is_long: True if the value should be returned as a long integer.
Returns: Returns:
The integer value. The integer value.
@ -1724,13 +1699,7 @@ def _ParseAbstractInteger(text, is_long=False):
# we always use the '0o' prefix for multi-digit numbers starting with 0. # we always use the '0o' prefix for multi-digit numbers starting with 0.
text = c_octal_match.group(1) + '0o' + c_octal_match.group(2) text = c_octal_match.group(1) + '0o' + c_octal_match.group(2)
try: try:
# We force 32-bit values to int and 64-bit values to long to make return int(text, 0)
# alternate implementations where the distinction is more significant
# (e.g. the C++ implementation) simpler.
if is_long:
return long(text, 0)
else:
return int(text, 0)
except ValueError: except ValueError:
raise ValueError('Couldn\'t parse integer: %s' % orig_text) raise ValueError('Couldn\'t parse integer: %s' % orig_text)

@ -271,7 +271,7 @@ if __name__ == '__main__':
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
# Keep this list of dependencies in sync with tox.ini. # Keep this list of dependencies in sync with tox.ini.
install_requires = ['six>=1.9'] install_requires = []
setup( setup(
name='protobuf', name='protobuf',

@ -1,6 +1,6 @@
[tox] [tox]
envlist = envlist =
py{33,34,35,36,37,38,39}-{cpp,python} py{35,36,37,38,39}-{cpp,python}
[testenv] [testenv]
usedevelop=true usedevelop=true
@ -14,11 +14,8 @@ setenv =
commands = commands =
python setup.py -q build_py python setup.py -q build_py
python: python setup.py -q build python: python setup.py -q build
py{33,34,35,36,37,38,39}-cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension py{35,36,37,38,39}-cpp: python setup.py -q build --cpp_implementation --warnings_as_errors --compile_static_extension
python: python setup.py -q test -q python: python setup.py -q test -q
cpp: python setup.py -q test -q --cpp_implementation cpp: python setup.py -q test -q --cpp_implementation
python: python setup.py -q test_conformance python: python setup.py -q test_conformance
cpp: python setup.py -q test_conformance --cpp_implementation cpp: python setup.py -q test_conformance --cpp_implementation
deps =
# Keep this list of dependencies in sync with setup.py.
six>=1.9

Loading…
Cancel
Save