From fc80874adf84b4061d6540eff32b4be05541a973 Mon Sep 17 00:00:00 2001 From: Dan O'Reilly Date: Sat, 15 Aug 2015 10:11:28 -0400 Subject: [PATCH 1/2] Start work on getting text handled properly on PY3 --- python/google/protobuf/text_format.py | 34 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index 950bec16fb..7bdde33fc6 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -67,6 +67,29 @@ class Error(Exception): class ParseError(Error): """Thrown in case of ASCII parsing error.""" +class TextWriter(object): + def __init__(self, as_utf8): + self._utf8 = as_utf8 + if as_utf8: + self._writer = io.BytesIO() + else: + self._writer = io.StringIO() + + def write(self, val): + if self._utf8: + if isinstance(val, six.text_type): + val = val.encode('utf-8') + else: + if isinstance(val, bytes): + val = val.decode('utf-8') + return self._writer.write(val) + + def close(self): + return self._writer.close() + + def getvalue(self): + return self._writer.getvalue() + def MessageToString(message, as_utf8=False, as_one_line=False, pointy_brackets=False, use_index_order=False, @@ -92,7 +115,7 @@ def MessageToString(message, as_utf8=False, as_one_line=False, Returns: A string of the text formatted protocol buffer message. """ - out = io.BytesIO() + out = TextWriter(as_utf8) PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line, pointy_brackets=pointy_brackets, use_index_order=use_index_order, @@ -159,11 +182,7 @@ def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False, # For groups, use the capitalized name. out.write(field.message_type.name) else: - if isinstance(field.name, six.text_type): - name = field.name.encode('utf-8') - else: - name = field.name - out.write(name) + out.write(field.name) if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE: # The colon is optional in this case, but our cross-language golden files @@ -226,7 +245,8 @@ def PrintFieldValue(field, value, out, indent=0, as_utf8=False, out_as_utf8 = False else: out_as_utf8 = as_utf8 - out.write(text_encoding.CEscape(out_value, out_as_utf8)) + out_text = text_encoding.CEscape(out_value, out_as_utf8) + out.write(out_text) out.write('\"') elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: if value: From 38eef02aab2cf52bce537ef43870387ba30381c8 Mon Sep 17 00:00:00 2001 From: Dan O'Reilly Date: Sat, 22 Aug 2015 13:02:24 -0400 Subject: [PATCH 2/2] Fix metaclass issue on Python 3. Get text handling tests passing on Python 3. Signed-off-by: Dan O'Reilly --- python/google/protobuf/descriptor.py | 5 +++-- python/google/protobuf/text_format.py | 11 +++-------- python/tox.ini | 6 +++--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index 6840d1f4ba..95b703fceb 100755 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -36,6 +36,8 @@ file, in types that make this information accessible in Python. __author__ = 'robinson@google.com (Will Robinson)' +import six + from google.protobuf.internal import api_implementation @@ -73,7 +75,7 @@ else: DescriptorMetaclass = type -class DescriptorBase(object): +class DescriptorBase(six.with_metaclass(DescriptorMetaclass)): """Descriptors base class. @@ -88,7 +90,6 @@ class DescriptorBase(object): avoid some bootstrapping issues. """ - __metaclass__ = DescriptorMetaclass if _USE_C_DESCRIPTORS: # The class, or tuple of classes, that are considered as "virtual # subclasses" of this descriptor class. diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index 7bdde33fc6..145bb6566e 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -69,19 +69,15 @@ class ParseError(Error): class TextWriter(object): def __init__(self, as_utf8): - self._utf8 = as_utf8 - if as_utf8: + if six.PY2: self._writer = io.BytesIO() else: self._writer = io.StringIO() def write(self, val): - if self._utf8: + if six.PY2: if isinstance(val, six.text_type): val = val.encode('utf-8') - else: - if isinstance(val, bytes): - val = val.decode('utf-8') return self._writer.write(val) def close(self): @@ -245,8 +241,7 @@ def PrintFieldValue(field, value, out, indent=0, as_utf8=False, out_as_utf8 = False else: out_as_utf8 = as_utf8 - out_text = text_encoding.CEscape(out_value, out_as_utf8) - out.write(out_text) + out.write(text_encoding.CEscape(out_value, out_as_utf8)) out.write('\"') elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: if value: diff --git a/python/tox.ini b/python/tox.ini index 9a934d0a77..d010075810 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - # Py3 tests currently fail because of text handling issues, - # So only test py26/py27 for now. + # cpp implementation on py34 is currently broken due to + # changes introduced by http://bugs.python.org/issue22079. #py{26,27,33,34}-{cpp,python} - py{26,27}-{cpp,python} + py{26,27,33}-{cpp,python}, py34-{python} [testenv] usedevelop=true