diff --git a/.gitignore b/.gitignore index c21c20cfa9..6508eccb18 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ cpp_test*.pb.* *.pyc *.egg-info *_pb2.py +python/*.egg python/.eggs/ python/build/ python/google/protobuf/compiler/ @@ -82,3 +83,9 @@ vsprojects/Release # NuGet packages: we want the repository configuration, but not the # packages themselves. /csharp/src/packages/*/ + +# Directories created by opening the Objective C Xcode projects. +objectivec/ProtocolBuffers_OSX.xcodeproj/project.xcworkspace/xcuserdata/ +objectivec/ProtocolBuffers_OSX.xcodeproj/xcuserdata/ +objectivec/ProtocolBuffers_iOS.xcodeproj/project.xcworkspace/xcuserdata/ +objectivec/ProtocolBuffers_iOS.xcodeproj/xcuserdata/ diff --git a/.travis.yml b/.travis.yml index cbddac5864..354be896d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ script: - cd python && python setup.py build && python setup.py test && cd .. - export LD_LIBRARY_PATH=../src/.libs - cd python && python setup.py build --cpp_implementation && python setup.py test --cpp_implementation && cd .. + - cd conformance && make test_java && cd .. - make distcheck -j2 notifications: email: false diff --git a/Makefile.am b/Makefile.am index 4fd93f26b7..eecffa1f24 100644 --- a/Makefile.am +++ b/Makefile.am @@ -197,6 +197,128 @@ javanano_EXTRA_DIST= javanano/README.md \ javanano/pom.xml +objectivec_EXTRA_DIST= \ + objectivec/DevTools/generate_descriptors_proto.sh \ + objectivec/DevTools/pddm.py \ + objectivec/DevTools/pddm_tests.py \ + objectivec/google/protobuf/Descriptor.pbobjc.h \ + objectivec/google/protobuf/Descriptor.pbobjc.m \ + objectivec/google/protobuf/Duration.pbobjc.h \ + objectivec/google/protobuf/Duration.pbobjc.m \ + objectivec/google/protobuf/Timestamp.pbobjc.h \ + objectivec/google/protobuf/Timestamp.pbobjc.m \ + objectivec/GPBArray.h \ + objectivec/GPBArray.m \ + objectivec/GPBArray_PackagePrivate.h \ + objectivec/GPBBootstrap.h \ + objectivec/GPBCodedInputStream.h \ + objectivec/GPBCodedInputStream.m \ + objectivec/GPBCodedInputStream_PackagePrivate.h \ + objectivec/GPBCodedOutputStream.h \ + objectivec/GPBCodedOutputStream.m \ + objectivec/GPBDescriptor.h \ + objectivec/GPBDescriptor.m \ + objectivec/GPBDescriptor_PackagePrivate.h \ + objectivec/GPBDictionary.h \ + objectivec/GPBDictionary.m \ + objectivec/GPBDictionary_PackagePrivate.h \ + objectivec/GPBExtensionField.h \ + objectivec/GPBExtensionField.m \ + objectivec/GPBExtensionField_PackagePrivate.h \ + objectivec/GPBExtensionRegistry.h \ + objectivec/GPBExtensionRegistry.m \ + objectivec/GPBExtensionRegistry_PackagePrivate.h \ + objectivec/GPBField.h \ + objectivec/GPBField.m \ + objectivec/GPBField_PackagePrivate.h \ + objectivec/GPBMessage.h \ + objectivec/GPBMessage.m \ + objectivec/GPBMessage_PackagePrivate.h \ + objectivec/GPBProtocolBuffers.h \ + objectivec/GPBProtocolBuffers.m \ + objectivec/GPBProtocolBuffers_RuntimeSupport.h \ + objectivec/GPBRootObject.h \ + objectivec/GPBRootObject.m \ + objectivec/GPBRootObject_PackagePrivate.h \ + objectivec/GPBTypes.h \ + objectivec/GPBUnknownFieldSet.h \ + objectivec/GPBUnknownFieldSet.m \ + objectivec/GPBUnknownFieldSet_PackagePrivate.h \ + objectivec/GPBUtilities.h \ + objectivec/GPBUtilities.m \ + objectivec/GPBUtilities_PackagePrivate.h \ + objectivec/GPBWellKnownTypes.h \ + objectivec/GPBWellKnownTypes.m \ + objectivec/GPBWireFormat.h \ + objectivec/GPBWireFormat.m \ + objectivec/ProtocolBuffers_iOS.xcodeproj/project.pbxproj \ + objectivec/ProtocolBuffers_iOS.xcodeproj/project.xcworkspace/contents.xcworkspacedata \ + objectivec/ProtocolBuffers_iOS.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings \ + objectivec/ProtocolBuffers_iOS.xcodeproj/xcshareddata/xcbaselines/8BBEA4A5147C727100C4ADB7.xcbaseline/FFE465CA-0E74-40E8-9F09-500B66B7DCB2.plist \ + objectivec/ProtocolBuffers_iOS.xcodeproj/xcshareddata/xcbaselines/8BBEA4A5147C727100C4ADB7.xcbaseline/Info.plist \ + objectivec/ProtocolBuffers_iOS.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme \ + objectivec/ProtocolBuffers_iOS.xcodeproj/xcshareddata/xcschemes/ProtocolBuffers.xcscheme \ + objectivec/ProtocolBuffers_OSX.xcodeproj/project.pbxproj \ + objectivec/ProtocolBuffers_OSX.xcodeproj/project.xcworkspace/contents.xcworkspacedata \ + objectivec/ProtocolBuffers_OSX.xcodeproj/project.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings \ + objectivec/ProtocolBuffers_OSX.xcodeproj/xcshareddata/xcschemes/PerformanceTests.xcscheme \ + objectivec/ProtocolBuffers_OSX.xcodeproj/xcshareddata/xcschemes/ProtocolBuffers.xcscheme \ + objectivec/Tests/Filter1.txt \ + objectivec/Tests/Filter2.txt \ + objectivec/Tests/golden_message \ + objectivec/Tests/golden_packed_fields_message \ + objectivec/Tests/GPBARCUnittestProtos.m \ + objectivec/Tests/GPBArrayTests.m \ + objectivec/Tests/GPBCodedInputStreamTests.m \ + objectivec/Tests/GPBCodedOuputStreamTests.m \ + objectivec/Tests/GPBConcurrencyTests.m \ + objectivec/Tests/GPBDescriptorTests.m \ + objectivec/Tests/GPBDictionaryTests+Bool.m \ + objectivec/Tests/GPBDictionaryTests+Int32.m \ + objectivec/Tests/GPBDictionaryTests+Int64.m \ + objectivec/Tests/GPBDictionaryTests+String.m \ + objectivec/Tests/GPBDictionaryTests+UInt32.m \ + objectivec/Tests/GPBDictionaryTests+UInt64.m \ + objectivec/Tests/GPBDictionaryTests.pddm \ + objectivec/Tests/GPBFilteredMessageTests.m \ + objectivec/Tests/GPBMessageTests+Merge.m \ + objectivec/Tests/GPBMessageTests+Runtime.m \ + objectivec/Tests/GPBMessageTests+Serialization.m \ + objectivec/Tests/GPBMessageTests.m \ + objectivec/Tests/GPBPerfTests.m \ + objectivec/Tests/GPBStringTests.m \ + objectivec/Tests/GPBSwiftTests.swift \ + objectivec/Tests/GPBTestUtilities.h \ + objectivec/Tests/GPBTestUtilities.m \ + objectivec/Tests/GPBUnittestProtos.m \ + objectivec/Tests/GPBUnknownFieldSetTest.m \ + objectivec/Tests/GPBUtilitiesTests.m \ + objectivec/Tests/GPBWellKnownTypesTest.m \ + objectivec/Tests/GPBWireFormatTests.m \ + objectivec/Tests/iOSTestHarness/AppDelegate.m \ + objectivec/Tests/iOSTestHarness/en.lproj/InfoPlist.strings \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/Contents.json \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/iPad6.png \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/iPad6@2x.png \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/iPad7.png \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/iPad7@2x.png \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/iPhone6.png \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/iPhone6@2x.png \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/iPhone7@2x.png \ + objectivec/Tests/iOSTestHarness/Images.xcassets/AppIcon.appiconset/iPhone7@3x.png \ + objectivec/Tests/iOSTestHarness/Images.xcassets/LaunchImage.launchimage/Contents.json \ + objectivec/Tests/iOSTestHarness/Info.plist \ + objectivec/Tests/iOSTestHarness/LaunchScreen.xib \ + objectivec/Tests/text_format_map_unittest_data.txt \ + objectivec/Tests/text_format_unittest_data.txt \ + objectivec/Tests/unittest_cycle.proto \ + objectivec/Tests/unittest_filter.proto \ + objectivec/Tests/unittest_name_mangling.proto \ + objectivec/Tests/unittest_objc.proto \ + objectivec/Tests/unittest_runtime_proto2.proto \ + objectivec/Tests/unittest_runtime_proto3.proto \ + objectivec/Tests/UnitTests-Bridging-Header.h \ + objectivec/Tests/UnitTests-Info.plist python_EXTRA_DIST= \ python/google/protobuf/internal/api_implementation.cc \ @@ -301,7 +423,7 @@ ruby_EXTRA_DIST= \ ruby/tests/generated_code.rb \ ruby/tests/generated_code_test.rb -all_EXTRA_DIST=$(java_EXTRA_DIST) $(javanano_EXTRA_DIST) $(python_EXTRA_DIST) $(ruby_EXTRA_DIST) +all_EXTRA_DIST=$(java_EXTRA_DIST) $(javanano_EXTRA_DIST) $(objectivec_EXTRA_DIST) $(python_EXTRA_DIST) $(ruby_EXTRA_DIST) EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST) \ autogen.sh \ diff --git a/conformance/ConformanceJava.java b/conformance/ConformanceJava.java new file mode 100644 index 0000000000..c1a53141fc --- /dev/null +++ b/conformance/ConformanceJava.java @@ -0,0 +1,120 @@ + +import com.google.protobuf.conformance.Conformance; +import com.google.protobuf.InvalidProtocolBufferException; + +class ConformanceJava { + private int testCount = 0; + + private boolean readFromStdin(byte[] buf, int len) throws Exception { + int ofs = 0; + while (len > 0) { + int read = System.in.read(buf, ofs, len); + if (read == -1) { + return false; // EOF + } + ofs += read; + len -= read; + } + + return true; + } + + private void writeToStdout(byte[] buf) throws Exception { + System.out.write(buf); + } + + // Returns -1 on EOF (the actual values will always be positive). + private int readLittleEndianIntFromStdin() throws Exception { + byte[] buf = new byte[4]; + if (!readFromStdin(buf, 4)) { + return -1; + } + return buf[0] | (buf[1] << 1) | (buf[2] << 2) | (buf[3] << 3); + } + + private void writeLittleEndianIntToStdout(int val) throws Exception { + byte[] buf = new byte[4]; + buf[0] = (byte)val; + buf[1] = (byte)(val >> 8); + buf[2] = (byte)(val >> 16); + buf[3] = (byte)(val >> 24); + writeToStdout(buf); + } + + private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest request) { + Conformance.TestAllTypes testMessage; + + switch (request.getPayloadCase()) { + case PROTOBUF_PAYLOAD: { + try { + testMessage = Conformance.TestAllTypes.parseFrom(request.getProtobufPayload()); + } catch (InvalidProtocolBufferException e) { + return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build(); + } + break; + } + case JSON_PAYLOAD: { + return Conformance.ConformanceResponse.newBuilder().setRuntimeError("JSON not yet supported.").build(); + } + case PAYLOAD_NOT_SET: { + throw new RuntimeException("Request didn't have payload."); + } + + default: { + throw new RuntimeException("Unexpected payload case."); + } + } + + switch (request.getRequestedOutput()) { + case UNSPECIFIED: + throw new RuntimeException("Unspecified output format."); + + case PROTOBUF: + return Conformance.ConformanceResponse.newBuilder().setProtobufPayload(testMessage.toByteString()).build(); + + case JSON: + return Conformance.ConformanceResponse.newBuilder().setRuntimeError("JSON not yet supported.").build(); + + default: { + throw new RuntimeException("Unexpected request output."); + } + } + } + + private boolean doTestIo() throws Exception { + int bytes = readLittleEndianIntFromStdin(); + + if (bytes == -1) { + return false; // EOF + } + + byte[] serializedInput = new byte[bytes]; + + if (!readFromStdin(serializedInput, bytes)) { + throw new RuntimeException("Unexpected EOF from test program."); + } + + Conformance.ConformanceRequest request = + Conformance.ConformanceRequest.parseFrom(serializedInput); + Conformance.ConformanceResponse response = doTest(request); + byte[] serializedOutput = response.toByteArray(); + + writeLittleEndianIntToStdout(serializedOutput.length); + writeToStdout(serializedOutput); + + return true; + } + + public void run() throws Exception { + while (doTestIo()) { + // Empty. + } + + System.err.println("ConformanceJava: received EOF from test runner after " + + this.testCount + " tests"); + } + + public static void main(String[] args) throws Exception { + new ConformanceJava().run(); + } +} diff --git a/conformance/Makefile.am b/conformance/Makefile.am index 0c4eae758a..95343f140b 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -21,30 +21,43 @@ conformance_cpp_CPPFLAGS = -I$(top_srcdir)/src if USE_EXTERNAL_PROTOC -unittest_proto_middleman: $(protoc_inputs) - $(PROTOC) -I$(srcdir) --cpp_out=. $^ - touch unittest_proto_middleman +protoc_middleman: $(protoc_inputs) + $(PROTOC) -I$(srcdir) --cpp_out=. --java_out=. $^ + touch protoc_middleman else # We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is # relative to srcdir, which may not be the same as the current directory when # building out-of-tree. -unittest_proto_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. --cpp_out=$$oldpwd $(protoc_inputs) ) - touch unittest_proto_middleman +protoc_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. --cpp_out=$$oldpwd --java_out=$$oldpwd $(protoc_inputs) ) + touch protoc_middleman endif -$(protoc_outputs): unittest_proto_middleman +$(protoc_outputs): protoc_middleman BUILT_SOURCES = $(protoc_outputs) -CLEANFILES = $(protoc_outputs) unittest_proto_middleman +CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java MAINTAINERCLEANFILES = \ Makefile.in +javac_middleman: ConformanceJava.java protoc_middleman + javac -classpath ../java/target/classes ConformanceJava.java com/google/protobuf/conformance/Conformance.java + @touch javac_middleman + +conformance-java: javac_middleman + @echo "Writing shortcut script conformance-java..." + @echo '#! /bin/sh' > conformance-java + @echo 'java -classpath .:../java/target/classes ConformanceJava "$$@"' >> conformance-java + @chmod +x conformance-java + # Targets for actually running tests. -test_cpp: unittest_proto_middleman conformance-test-runner conformance-cpp +test_cpp: protoc_middleman conformance-test-runner conformance-cpp ./conformance-test-runner ./conformance-cpp + +test_java: protoc_middleman conformance-test-runner conformance-java + ./conformance-test-runner ./conformance-java diff --git a/conformance/conformance.proto b/conformance/conformance.proto index 0fb66cf932..7b67689851 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -30,6 +30,7 @@ syntax = "proto3"; package conformance; +option java_package = "com.google.protobuf.conformance"; // This defines the conformance testing protocol. This protocol exists between // the conformance test suite itself and the code being tested. For each test, diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc index 21277700bd..ca5877bde7 100644 --- a/conformance/conformance_test_runner.cc +++ b/conformance/conformance_test_runner.cc @@ -48,9 +48,9 @@ // Every test consists of a ConformanceRequest/ConformanceResponse // request/reply pair. The protocol on the pipe is simply: // -// 1. tester sends 4-byte length N +// 1. tester sends 4-byte length N (little endian) // 2. tester sends N bytes representing a ConformanceRequest proto -// 3. testee sends 4-byte length M +// 3. testee sends 4-byte length M (little endian) // 4. testee sends M bytes representing a ConformanceResponse proto #include diff --git a/csharp/src/ProtocolBuffers.Test/TestProtos/UnittestDropUnknownFields.cs b/csharp/src/ProtocolBuffers.Test/TestProtos/UnittestDropUnknownFields.cs index 3939c9a4de..02af2f5024 100644 --- a/csharp/src/ProtocolBuffers.Test/TestProtos/UnittestDropUnknownFields.cs +++ b/csharp/src/ProtocolBuffers.Test/TestProtos/UnittestDropUnknownFields.cs @@ -40,8 +40,8 @@ namespace Google.ProtocolBuffers.TestProtos { "X3ZhbHVlGAIgASgOMjsudW5pdHRlc3RfZHJvcF91bmtub3duX2ZpZWxkcy5G", "b29XaXRoRXh0cmFGaWVsZHMuTmVzdGVkRW51bRIZChFleHRyYV9pbnQzMl92", "YWx1ZRgDIAEoBSIwCgpOZXN0ZWRFbnVtEgcKA0ZPTxAAEgcKA0JBUhABEgcK", - "A0JBWhACEgcKA1FVWBADQiSqAiFHb29nbGUuUHJvdG9jb2xCdWZmZXJzLlRl", - "c3RQcm90b3NiBnByb3RvMw==")); + "A0JBWhACEgcKA1FVWBADQjOiAgxEcm9wVW5rbm93bnOqAiFHb29nbGUuUHJv", + "dG9jb2xCdWZmZXJzLlRlc3RQcm90b3NiBnByb3RvMw==")); pbd::FileDescriptor.InternalDescriptorAssigner assigner = delegate(pbd::FileDescriptor root) { descriptor = root; internal__static_unittest_drop_unknown_fields_Foo__Descriptor = Descriptor.MessageTypes[0]; diff --git a/examples/add_person.py b/examples/add_person.py index 78e569662d..5a7b968e4b 100755 --- a/examples/add_person.py +++ b/examples/add_person.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # See README.txt for information and build instructions. diff --git a/examples/list_people.py b/examples/list_people.py index f9f36b96b3..d470349a8b 100755 --- a/examples/list_people.py +++ b/examples/list_people.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # See README.txt for information and build instructions. diff --git a/generate_descriptor_proto.sh b/generate_descriptor_proto.sh index b25a3c6ad3..2dad36abc1 100755 --- a/generate_descriptor_proto.sh +++ b/generate_descriptor_proto.sh @@ -27,6 +27,7 @@ __EOF__ fi cd src +make $@ google/protobuf/stubs/pbconfig.h CORE_PROTO_IS_CORRECT=0 while [ $CORE_PROTO_IS_CORRECT -ne 1 ] do diff --git a/javanano/README.md b/javanano/README.md index 7d696aa7e8..e19b90b1c2 100644 --- a/javanano/README.md +++ b/javanano/README.md @@ -145,6 +145,7 @@ optional_field_style -> default or accessors enum_style -> c or java ignore_services -> true or false parcelable_messages -> true or false +generate_intdefs -> true or false ``` **java_package=\|\** (no default) @@ -302,6 +303,28 @@ parcelable_messages -> true or false Android-specific option to generate Parcelable messages. +**generate_intdefs={true,false}** (default: false) + Android-specific option to generate @IntDef annotations for enums. + + If turned on, an '@IntDef' annotation (a public @interface) will be + generated for each enum, and every integer parameter and return + value in the generated code meant for this enum will be annotated + with it. This interface is generated with the same name and at the + same place as the enum members' container interfaces described + above under 'enum_style=java', regardless of the enum_style option + used. When this is combined with enum_style=java, the interface + will be both the '@IntDef' annotation and the container of the enum + members; otherwise the interface has an empty body. + + Your app must declare a compile-time dependency on the + android-support-annotations library. + + For more information on how these @IntDef annotations help with + compile-time type safety, see: + https://sites.google.com/a/android.com/tools/tech-docs/support-annotations + and + https://developer.android.com/reference/android/support/annotation/IntDef.html + To use nano protobufs within the Android repo: ---------------------------------------------- diff --git a/javanano/pom.xml b/javanano/pom.xml index 7a2be9df24..3d8cfb9f1b 100644 --- a/javanano/pom.xml +++ b/javanano/pom.xml @@ -97,19 +97,19 @@ - + - + - + @@ -139,6 +139,15 @@ + + + + + target/generated-test-sources diff --git a/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java b/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java index 4b45c6d298..f399315511 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/CodedInputByteBufferNano.java @@ -236,6 +236,8 @@ public final class CodedInputByteBufferNano { System.arraycopy(buffer, bufferPos, result, 0, size); bufferPos += size; return result; + } else if (size == 0) { + return WireFormatNano.EMPTY_BYTES; } else { // Slow path: Build a byte array first then copy it. return readRawBytes(size); diff --git a/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java b/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java index 3590718308..b1b0c53a1e 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/CodedOutputByteBufferNano.java @@ -31,6 +31,9 @@ package com.google.protobuf.nano; import java.io.IOException; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.ReadOnlyBufferException; /** * Encodes and writes protocol message fields. @@ -47,15 +50,17 @@ import java.io.IOException; * @author kneton@google.com Kenton Varda */ public final class CodedOutputByteBufferNano { - private final byte[] buffer; - private final int limit; - private int position; + /* max bytes per java UTF-16 char in UTF-8 */ + private static final int MAX_UTF8_EXPANSION = 3; + private final ByteBuffer buffer; private CodedOutputByteBufferNano(final byte[] buffer, final int offset, final int length) { + this(ByteBuffer.wrap(buffer, offset, length)); + } + + private CodedOutputByteBufferNano(final ByteBuffer buffer) { this.buffer = buffer; - position = offset; - limit = offset + length; } /** @@ -287,14 +292,213 @@ public final class CodedOutputByteBufferNano { /** Write a {@code string} field to the stream. */ public void writeStringNoTag(final String value) throws IOException { - // Unfortunately there does not appear to be any way to tell Java to encode - // UTF-8 directly into our buffer, so we have to let it create its own byte - // array and then copy. - final byte[] bytes = value.getBytes(InternalNano.UTF_8); - writeRawVarint32(bytes.length); - writeRawBytes(bytes); + // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()), + // and at most 3 times of it. Optimize for the case where we know this length results in a + // constant varint length - saves measuring length of the string. + try { + final int minLengthVarIntSize = computeRawVarint32Size(value.length()); + final int maxLengthVarIntSize = computeRawVarint32Size(value.length() * MAX_UTF8_EXPANSION); + if (minLengthVarIntSize == maxLengthVarIntSize) { + int oldPosition = buffer.position(); + // Buffer.position, when passed a position that is past its limit, throws + // IllegalArgumentException, and this class is documented to throw + // OutOfSpaceException instead. + if (buffer.remaining() < minLengthVarIntSize) { + throw new OutOfSpaceException(oldPosition + minLengthVarIntSize, buffer.limit()); + } + buffer.position(oldPosition + minLengthVarIntSize); + encode(value, buffer); + int newPosition = buffer.position(); + buffer.position(oldPosition); + writeRawVarint32(newPosition - oldPosition - minLengthVarIntSize); + buffer.position(newPosition); + } else { + writeRawVarint32(encodedLength(value)); + encode(value, buffer); + } + } catch (BufferOverflowException e) { + final OutOfSpaceException outOfSpaceException = new OutOfSpaceException(buffer.position(), + buffer.limit()); + outOfSpaceException.initCause(e); + throw outOfSpaceException; + } + } + + // These UTF-8 handling methods are copied from Guava's Utf8 class. + /** + * Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string, + * this method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in + * both time and space. + * + * @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired + * surrogates) + */ + private static int encodedLength(CharSequence sequence) { + // Warning to maintainers: this implementation is highly optimized. + int utf16Length = sequence.length(); + int utf8Length = utf16Length; + int i = 0; + + // This loop optimizes for pure ASCII. + while (i < utf16Length && sequence.charAt(i) < 0x80) { + i++; + } + + // This loop optimizes for chars less than 0x800. + for (; i < utf16Length; i++) { + char c = sequence.charAt(i); + if (c < 0x800) { + utf8Length += ((0x7f - c) >>> 31); // branch free! + } else { + utf8Length += encodedLengthGeneral(sequence, i); + break; + } + } + + if (utf8Length < utf16Length) { + // Necessary and sufficient condition for overflow because of maximum 3x expansion + throw new IllegalArgumentException("UTF-8 length does not fit in int: " + + (utf8Length + (1L << 32))); + } + return utf8Length; + } + + private static int encodedLengthGeneral(CharSequence sequence, int start) { + int utf16Length = sequence.length(); + int utf8Length = 0; + for (int i = start; i < utf16Length; i++) { + char c = sequence.charAt(i); + if (c < 0x800) { + utf8Length += (0x7f - c) >>> 31; // branch free! + } else { + utf8Length += 2; + // jdk7+: if (Character.isSurrogate(c)) { + if (Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) { + // Check that we have a well-formed surrogate pair. + int cp = Character.codePointAt(sequence, i); + if (cp < Character.MIN_SUPPLEMENTARY_CODE_POINT) { + throw new IllegalArgumentException("Unpaired surrogate at index " + i); + } + i++; + } + } + } + return utf8Length; } + /** + * Encodes {@code sequence} into UTF-8, in {@code byteBuffer}. For a string, this method is + * equivalent to {@code buffer.put(string.getBytes(UTF_8))}, but is more efficient in both time + * and space. Bytes are written starting at the current position. This method requires paired + * surrogates, and therefore does not support chunking. + * + *

To ensure sufficient space in the output buffer, either call {@link #encodedLength} to + * compute the exact amount needed, or leave room for {@code 3 * sequence.length()}, which is the + * largest possible number of bytes that any input can be encoded to. + * + * @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired + * surrogates) + * @throws BufferOverflowException if {@code sequence} encoded in UTF-8 does not fit in + * {@code byteBuffer}'s remaining space. + * @throws ReadOnlyBufferException if {@code byteBuffer} is a read-only buffer. + */ + private static void encode(CharSequence sequence, ByteBuffer byteBuffer) { + if (byteBuffer.isReadOnly()) { + throw new ReadOnlyBufferException(); + } else if (byteBuffer.hasArray()) { + try { + int encoded = encode(sequence, + byteBuffer.array(), + byteBuffer.arrayOffset() + byteBuffer.position(), + byteBuffer.remaining()); + byteBuffer.position(encoded - byteBuffer.arrayOffset()); + } catch (ArrayIndexOutOfBoundsException e) { + BufferOverflowException boe = new BufferOverflowException(); + boe.initCause(e); + throw boe; + } + } else { + encodeDirect(sequence, byteBuffer); + } + } + + private static void encodeDirect(CharSequence sequence, ByteBuffer byteBuffer) { + int utf16Length = sequence.length(); + for (int i = 0; i < utf16Length; i++) { + final char c = sequence.charAt(i); + if (c < 0x80) { // ASCII + byteBuffer.put((byte) c); + } else if (c < 0x800) { // 11 bits, two UTF-8 bytes + byteBuffer.put((byte) ((0xF << 6) | (c >>> 6))); + byteBuffer.put((byte) (0x80 | (0x3F & c))); + } else if (c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) { + // Maximium single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + byteBuffer.put((byte) ((0xF << 5) | (c >>> 12))); + byteBuffer.put((byte) (0x80 | (0x3F & (c >>> 6)))); + byteBuffer.put((byte) (0x80 | (0x3F & c))); + } else { + final char low; + if (i + 1 == sequence.length() + || !Character.isSurrogatePair(c, (low = sequence.charAt(++i)))) { + throw new IllegalArgumentException("Unpaired surrogate at index " + (i - 1)); + } + int codePoint = Character.toCodePoint(c, low); + byteBuffer.put((byte) ((0xF << 4) | (codePoint >>> 18))); + byteBuffer.put((byte) (0x80 | (0x3F & (codePoint >>> 12)))); + byteBuffer.put((byte) (0x80 | (0x3F & (codePoint >>> 6)))); + byteBuffer.put((byte) (0x80 | (0x3F & codePoint))); + } + } + } + + private static int encode(CharSequence sequence, byte[] bytes, int offset, int length) { + int utf16Length = sequence.length(); + int j = offset; + int i = 0; + int limit = offset + length; + // Designed to take advantage of + // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination + for (char c; i < utf16Length && i + j < limit && (c = sequence.charAt(i)) < 0x80; i++) { + bytes[j + i] = (byte) c; + } + if (i == utf16Length) { + return j + utf16Length; + } + j += i; + for (char c; i < utf16Length; i++) { + c = sequence.charAt(i); + if (c < 0x80 && j < limit) { + bytes[j++] = (byte) c; + } else if (c < 0x800 && j <= limit - 2) { // 11 bits, two UTF-8 bytes + bytes[j++] = (byte) ((0xF << 6) | (c >>> 6)); + bytes[j++] = (byte) (0x80 | (0x3F & c)); + } else if ((c < Character.MIN_SURROGATE || Character.MAX_SURROGATE < c) && j <= limit - 3) { + // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes + bytes[j++] = (byte) ((0xF << 5) | (c >>> 12)); + bytes[j++] = (byte) (0x80 | (0x3F & (c >>> 6))); + bytes[j++] = (byte) (0x80 | (0x3F & c)); + } else if (j <= limit - 4) { + // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8 bytes + final char low; + if (i + 1 == sequence.length() + || !Character.isSurrogatePair(c, (low = sequence.charAt(++i)))) { + throw new IllegalArgumentException("Unpaired surrogate at index " + (i - 1)); + } + int codePoint = Character.toCodePoint(c, low); + bytes[j++] = (byte) ((0xF << 4) | (codePoint >>> 18)); + bytes[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 12))); + bytes[j++] = (byte) (0x80 | (0x3F & (codePoint >>> 6))); + bytes[j++] = (byte) (0x80 | (0x3F & codePoint)); + } else { + throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + j); + } + } + return j; + } + + // End guava UTF-8 methods + + /** Write a {@code group} field to the stream. */ public void writeGroupNoTag(final MessageNano value) throws IOException { value.writeTo(this); @@ -602,9 +806,8 @@ public final class CodedOutputByteBufferNano { * {@code string} field. */ public static int computeStringSizeNoTag(final String value) { - final byte[] bytes = value.getBytes(InternalNano.UTF_8); - return computeRawVarint32Size(bytes.length) + - bytes.length; + final int length = encodedLength(value); + return computeRawVarint32Size(length) + length; } /** @@ -687,7 +890,7 @@ public final class CodedOutputByteBufferNano { * Otherwise, throws {@code UnsupportedOperationException}. */ public int spaceLeft() { - return limit - position; + return buffer.remaining(); } /** @@ -704,6 +907,23 @@ public final class CodedOutputByteBufferNano { } } + /** + * Returns the position within the internal buffer. + */ + public int position() { + return buffer.position(); + } + + /** + * Resets the position within the internal buffer to zero. + * + * @see #position + * @see #spaceLeft + */ + public void reset() { + buffer.clear(); + } + /** * If you create a CodedOutputStream around a simple flat array, you must * not attempt to write more bytes than the array has space. Otherwise, @@ -720,12 +940,12 @@ public final class CodedOutputByteBufferNano { /** Write a single byte. */ public void writeRawByte(final byte value) throws IOException { - if (position == limit) { + if (!buffer.hasRemaining()) { // We're writing to a single buffer. - throw new OutOfSpaceException(position, limit); + throw new OutOfSpaceException(buffer.position(), buffer.limit()); } - buffer[position++] = value; + buffer.put(value); } /** Write a single byte, represented by an integer value. */ @@ -741,13 +961,11 @@ public final class CodedOutputByteBufferNano { /** Write part of an array of bytes. */ public void writeRawBytes(final byte[] value, int offset, int length) throws IOException { - if (limit - position >= length) { - // We have room in the current buffer. - System.arraycopy(value, offset, buffer, position, length); - position += length; + if (buffer.remaining() >= length) { + buffer.put(value, offset, length); } else { // We're writing to a single buffer. - throw new OutOfSpaceException(position, limit); + throw new OutOfSpaceException(buffer.position(), buffer.limit()); } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java b/javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java index b979390d68..87973d76f0 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/ExtendableMessageNano.java @@ -160,28 +160,10 @@ public abstract class ExtendableMessageNano> return true; } - /** - * Returns whether the stored unknown field data in this message is equivalent to that in the - * other message. - * - * @param other the other message. - * @return whether the two sets of unknown field data are equal. - */ - protected final boolean unknownFieldDataEquals(M other) { - if (unknownFieldData == null || unknownFieldData.isEmpty()) { - return other.unknownFieldData == null || other.unknownFieldData.isEmpty(); - } else { - return unknownFieldData.equals(other.unknownFieldData); - } - } - - /** - * Computes the hashcode representing the unknown field data stored in this message. - * - * @return the hashcode for the unknown field data. - */ - protected final int unknownFieldDataHashCode() { - return (unknownFieldData == null || unknownFieldData.isEmpty() - ? 0 : unknownFieldData.hashCode()); + @Override + public M clone() throws CloneNotSupportedException { + M cloned = (M) super.clone(); + InternalNano.cloneUnknownFieldData(this, cloned); + return cloned; } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/Extension.java b/javanano/src/main/java/com/google/protobuf/nano/Extension.java index c29b030fb6..c458f9b1fc 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/Extension.java +++ b/javanano/src/main/java/com/google/protobuf/nano/Extension.java @@ -79,12 +79,30 @@ public class Extension, T> { * Should be used by the generated code only. * * @param type {@link #TYPE_MESSAGE} or {@link #TYPE_GROUP} + * @deprecated use {@link #createMessageTyped(int, Class, long)} instead. */ + @Deprecated public static , T extends MessageNano> Extension createMessageTyped(int type, Class clazz, int tag) { return new Extension(type, clazz, tag, false); } + // Note: these create...() methods take a long for the tag parameter, + // because tags are represented as unsigned ints, and these values exist + // in generated code as long values. However, they can fit in 32-bits, so + // it's safe to cast them to int without loss of precision. + + /** + * Creates an {@code Extension} of the given message type and tag number. + * Should be used by the generated code only. + * + * @param type {@link #TYPE_MESSAGE} or {@link #TYPE_GROUP} + */ + public static , T extends MessageNano> + Extension createMessageTyped(int type, Class clazz, long tag) { + return new Extension(type, clazz, (int) tag, false); + } + /** * Creates a repeated {@code Extension} of the given message type and tag number. * Should be used by the generated code only. @@ -92,8 +110,8 @@ public class Extension, T> { * @param type {@link #TYPE_MESSAGE} or {@link #TYPE_GROUP} */ public static , T extends MessageNano> - Extension createRepeatedMessageTyped(int type, Class clazz, int tag) { - return new Extension(type, clazz, tag, true); + Extension createRepeatedMessageTyped(int type, Class clazz, long tag) { + return new Extension(type, clazz, (int) tag, true); } /** @@ -104,8 +122,8 @@ public class Extension, T> { * @param clazz the boxed Java type of this extension */ public static , T> - Extension createPrimitiveTyped(int type, Class clazz, int tag) { - return new PrimitiveExtension(type, clazz, tag, false, 0, 0); + Extension createPrimitiveTyped(int type, Class clazz, long tag) { + return new PrimitiveExtension(type, clazz, (int) tag, false, 0, 0); } /** @@ -117,8 +135,9 @@ public class Extension, T> { */ public static , T> Extension createRepeatedPrimitiveTyped( - int type, Class clazz, int tag, int nonPackedTag, int packedTag) { - return new PrimitiveExtension(type, clazz, tag, true, nonPackedTag, packedTag); + int type, Class clazz, long tag, long nonPackedTag, long packedTag) { + return new PrimitiveExtension(type, clazz, (int) tag, true, + (int) nonPackedTag, (int) packedTag); } /** @@ -136,7 +155,7 @@ public class Extension, T> { protected final Class clazz; /** - * Tag number of this extension. + * Tag number of this extension. The data should be viewed as an unsigned 32-bit value. */ public final int tag; diff --git a/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java b/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java index cdb66da200..eca9c0d9c1 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java +++ b/javanano/src/main/java/com/google/protobuf/nano/FieldArray.java @@ -35,9 +35,12 @@ package com.google.protobuf.nano; * A custom version of {@link android.util.SparseArray} with the minimal API * for storing {@link FieldData} objects. * + *

This class is an internal implementation detail of nano and should not + * be called directly by clients. + * * Based on {@link android.support.v4.util.SpareArrayCompat}. */ -class FieldArray { +public final class FieldArray implements Cloneable { private static final FieldData DELETED = new FieldData(); private boolean mGarbage = false; @@ -48,7 +51,7 @@ class FieldArray { /** * Creates a new FieldArray containing no fields. */ - public FieldArray() { + FieldArray() { this(10); } @@ -57,7 +60,7 @@ class FieldArray { * require any additional memory allocation to store the specified * number of mappings. */ - public FieldArray(int initialCapacity) { + FieldArray(int initialCapacity) { initialCapacity = idealIntArraySize(initialCapacity); mFieldNumbers = new int[initialCapacity]; mData = new FieldData[initialCapacity]; @@ -68,7 +71,7 @@ class FieldArray { * Gets the FieldData mapped from the specified fieldNumber, or null * if no such mapping has been made. */ - public FieldData get(int fieldNumber) { + FieldData get(int fieldNumber) { int i = binarySearch(fieldNumber); if (i < 0 || mData[i] == DELETED) { @@ -81,7 +84,7 @@ class FieldArray { /** * Removes the data from the specified fieldNumber, if there was any. */ - public void remove(int fieldNumber) { + void remove(int fieldNumber) { int i = binarySearch(fieldNumber); if (i >= 0 && mData[i] != DELETED) { @@ -118,7 +121,7 @@ class FieldArray { * Adds a mapping from the specified fieldNumber to the specified data, * replacing the previous mapping if there was one. */ - public void put(int fieldNumber, FieldData data) { + void put(int fieldNumber, FieldData data) { int i = binarySearch(fieldNumber); if (i >= 0) { @@ -167,7 +170,7 @@ class FieldArray { * Returns the number of key-value mappings that this FieldArray * currently stores. */ - public int size() { + int size() { if (mGarbage) { gc(); } @@ -184,7 +187,7 @@ class FieldArray { * the value from the indexth key-value mapping that this * FieldArray stores. */ - public FieldData dataAt(int index) { + FieldData dataAt(int index) { if (mGarbage) { gc(); } @@ -270,4 +273,19 @@ class FieldArray { } return true; } + + @Override + public final FieldArray clone() { + // Trigger GC so we compact and don't copy DELETED elements. + int size = size(); + FieldArray clone = new FieldArray(size); + System.arraycopy(mFieldNumbers, 0, clone.mFieldNumbers, 0, size); + for (int i = 0; i < size; i++) { + if (mData[i] != null) { + clone.mData[i] = mData[i].clone(); + } + } + clone.mSize = size; + return clone; + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/FieldData.java b/javanano/src/main/java/com/google/protobuf/nano/FieldData.java index 21ead88b90..ebebabc8e5 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/FieldData.java +++ b/javanano/src/main/java/com/google/protobuf/nano/FieldData.java @@ -39,7 +39,7 @@ import java.util.List; * Stores unknown fields. These might be extensions or fields that the generated API doesn't * know about yet. */ -class FieldData { +class FieldData implements Cloneable { private Extension cachedExtension; private Object value; /** The serialised values for this object. Will be cleared if getValue is called */ @@ -187,4 +187,54 @@ class FieldData { return result; } + @Override + public final FieldData clone() { + FieldData clone = new FieldData(); + try { + clone.cachedExtension = cachedExtension; + if (unknownFieldData == null) { + clone.unknownFieldData = null; + } else { + clone.unknownFieldData.addAll(unknownFieldData); + } + + // Whether we need to deep clone value depends on its type. Primitive reference types + // (e.g. Integer, Long etc.) are ok, since they're immutable. We need to clone arrays + // and messages. + if (value == null) { + // No cloning required. + } else if (value instanceof MessageNano) { + clone.value = ((MessageNano) value).clone(); + } else if (value instanceof byte[]) { + clone.value = ((byte[]) value).clone(); + } else if (value instanceof byte[][]) { + byte[][] valueArray = (byte[][]) value; + byte[][] cloneArray = new byte[valueArray.length][]; + clone.value = cloneArray; + for (int i = 0; i < valueArray.length; i++) { + cloneArray[i] = valueArray[i].clone(); + } + } else if (value instanceof boolean[]) { + clone.value = ((boolean[]) value).clone(); + } else if (value instanceof int[]) { + clone.value = ((int[]) value).clone(); + } else if (value instanceof long[]) { + clone.value = ((long[]) value).clone(); + } else if (value instanceof float[]) { + clone.value = ((float[]) value).clone(); + } else if (value instanceof double[]) { + clone.value = ((double[]) value).clone(); + } else if (value instanceof MessageNano[]) { + MessageNano[] valueArray = (MessageNano[]) value; + MessageNano[] cloneArray = new MessageNano[valueArray.length]; + clone.value = cloneArray; + for (int i = 0; i < valueArray.length; i++) { + cloneArray[i] = valueArray[i].clone(); + } + } + return clone; + } catch (CloneNotSupportedException e) { + throw new AssertionError(e); + } + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java b/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java index e7ba8d1262..f1263df580 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/InternalNano.java @@ -536,4 +536,12 @@ public final class InternalNano { } return o.hashCode(); } + + // This avoids having to make FieldArray public. + public static void cloneUnknownFieldData(ExtendableMessageNano original, + ExtendableMessageNano cloned) { + if (original.unknownFieldData != null) { + cloned.unknownFieldData = (FieldArray) original.unknownFieldData.clone(); + } + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/MessageNano.java b/javanano/src/main/java/com/google/protobuf/nano/MessageNano.java index 81e5857154..2347502702 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/MessageNano.java +++ b/javanano/src/main/java/com/google/protobuf/nano/MessageNano.java @@ -187,4 +187,12 @@ public abstract class MessageNano { public String toString() { return MessageNanoPrinter.print(this); } + + /** + * Provides support for cloning. This only works if you specify the generate_clone method. + */ + @Override + public MessageNano clone() throws CloneNotSupportedException { + return (MessageNano) super.clone(); + } } diff --git a/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java b/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java index c4b2ad3d0a..d9500bb948 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java +++ b/javanano/src/main/java/com/google/protobuf/nano/MessageNanoPrinter.java @@ -109,6 +109,10 @@ public final class MessageNanoPrinter { for (Field field : clazz.getFields()) { int modifiers = field.getModifiers(); String fieldName = field.getName(); + if ("cachedSize".equals(fieldName)) { + // TODO(bduff): perhaps cachedSize should have a more obscure name. + continue; + } if ((modifiers & Modifier.PUBLIC) == Modifier.PUBLIC && (modifiers & Modifier.STATIC) != Modifier.STATIC diff --git a/javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java b/javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java index a17fccf3a6..b1678d1b93 100644 --- a/javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java +++ b/javanano/src/main/java/com/google/protobuf/nano/UnknownFieldData.java @@ -42,6 +42,10 @@ import java.util.Arrays; final class UnknownFieldData { final int tag; + /** + * Important: this should be treated as immutable, even though it's possible + * to change the array values. + */ final byte[] bytes; UnknownFieldData(int tag, byte[] bytes) { diff --git a/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java b/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java index 7de84310c0..3a75777ac6 100644 --- a/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java +++ b/javanano/src/test/java/com/google/protobuf/nano/NanoTest.java @@ -31,11 +31,13 @@ package com.google.protobuf.nano; import com.google.protobuf.nano.MapTestProto.TestMap; +import com.google.protobuf.nano.CodedOutputByteBufferNano; import com.google.protobuf.nano.MapTestProto.TestMap.MessageValue; import com.google.protobuf.nano.NanoAccessorsOuterClass.TestNanoAccessors; import com.google.protobuf.nano.NanoHasOuterClass.TestAllTypesNanoHas; import com.google.protobuf.nano.NanoOuterClass.TestAllTypesNano; import com.google.protobuf.nano.UnittestRecursiveNano.RecursiveMessageNano; +import com.google.protobuf.nano.NanoReferenceTypesCompat; import com.google.protobuf.nano.UnittestSimpleNano.SimpleMessageNano; import com.google.protobuf.nano.UnittestSingleNano.SingleMessageNano; import com.google.protobuf.nano.testext.Extensions; @@ -2300,6 +2302,59 @@ public class NanoTest extends TestCase { } } + public void testDifferentStringLengthsNano() throws Exception { + // Test string serialization roundtrip using strings of the following lengths, + // with ASCII and Unicode characters requiring different UTF-8 byte counts per + // char, hence causing the length delimiter varint to sometimes require more + // bytes for the Unicode strings than the ASCII string of the same length. + int[] lengths = new int[] { + 0, + 1, + (1 << 4) - 1, // 1 byte for ASCII and Unicode + (1 << 7) - 1, // 1 byte for ASCII, 2 bytes for Unicode + (1 << 11) - 1, // 2 bytes for ASCII and Unicode + (1 << 14) - 1, // 2 bytes for ASCII, 3 bytes for Unicode + (1 << 17) - 1, // 3 bytes for ASCII and Unicode + }; + for (int i : lengths) { + testEncodingOfString('q', i); // 1 byte per char + testEncodingOfString('\u07FF', i); // 2 bytes per char + testEncodingOfString('\u0981', i); // 3 bytes per char + } + } + + /** Regression test for https://github.com/google/protobuf/issues/292 */ + public void testCorrectExceptionThrowWhenEncodingStringsWithoutEnoughSpace() throws Exception { + String testCase = "Foooooooo"; + assertEquals(CodedOutputByteBufferNano.computeRawVarint32Size(testCase.length()), + CodedOutputByteBufferNano.computeRawVarint32Size(testCase.length() * 3)); + assertEquals(11, CodedOutputByteBufferNano.computeStringSize(1, testCase)); + // Tag is one byte, varint describing string length is 1 byte, string length is 9 bytes. + // An array of size 1 will cause a failure when trying to write the varint. + for (int i = 0; i < 11; i++) { + CodedOutputByteBufferNano bufferNano = CodedOutputByteBufferNano.newInstance(new byte[i]); + try { + bufferNano.writeString(1, testCase); + fail("Should have thrown an out of space exception"); + } catch (CodedOutputByteBufferNano.OutOfSpaceException expected) {} + } + } + + private void testEncodingOfString(char c, int length) throws InvalidProtocolBufferNanoException { + TestAllTypesNano testAllTypesNano = new TestAllTypesNano(); + final String fullString = fullString(c, length); + testAllTypesNano.optionalString = fullString; + final TestAllTypesNano resultNano = new TestAllTypesNano(); + MessageNano.mergeFrom(resultNano, MessageNano.toByteArray(testAllTypesNano)); + assertEquals(fullString, resultNano.optionalString); + } + + private String fullString(char c, int length) { + char[] result = new char[length]; + Arrays.fill(result, c); + return new String(result); + } + public void testNanoWithHasParseFrom() throws Exception { TestAllTypesNanoHas msg = null; // Test false on creation, after clear and upon empty parse. @@ -2986,6 +3041,10 @@ public class NanoTest extends TestCase { assertTrue(Arrays.equals(floats, message.getExtension(RepeatedExtensions.repeatedFloat))); assertTrue(Arrays.equals(doubles, message.getExtension(RepeatedExtensions.repeatedDouble))); assertTrue(Arrays.equals(enums, message.getExtension(RepeatedExtensions.repeatedEnum))); + + // Clone the message and ensure it's still equal. + Extensions.ExtendableMessage clone = message.clone(); + assertEquals(clone, message); } public void testNullExtensions() throws Exception { @@ -4345,6 +4404,11 @@ public class NanoTest extends TestCase { assertMapSet(testMap.sfixed64ToSfixed64Field, int64Values, int64Values); } + public void testRepeatedFieldInitializedInReftypesCompatMode() { + NanoReferenceTypesCompat.TestAllTypesNano proto = new NanoReferenceTypesCompat.TestAllTypesNano(); + assertNotNull(proto.repeatedString); + } + private void assertRepeatedPackablesEqual( NanoRepeatedPackables.NonPacked nonPacked, NanoRepeatedPackables.Packed packed) { // Not using MessageNano.equals() -- that belongs to a separate test. @@ -4364,6 +4428,22 @@ public class NanoTest extends TestCase { assertTrue(Arrays.equals(nonPacked.enums, packed.enums)); } + public void testClone() throws Exception { + // A simple message. + AnotherMessage anotherMessage = new AnotherMessage(); + anotherMessage.string = "Hello"; + anotherMessage.value = true; + anotherMessage.integers = new int[] { 1, 2, 3 }; + + AnotherMessage clone = anotherMessage.clone(); + assertEquals(clone, anotherMessage); + + // Verify it was a deep clone - changes to the clone shouldn't affect the + // original. + clone.integers[1] = 100; + assertFalse(clone.equals(anotherMessage)); + } + private void assertHasWireData(MessageNano message, boolean expected) { byte[] bytes = MessageNano.toByteArray(message); int wireLength = bytes.length; diff --git a/javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto b/javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto index 2a678a8075..ca56b3dd45 100644 --- a/javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto +++ b/javanano/src/test/java/com/google/protobuf/nano/unittest_extension_nano.proto @@ -16,11 +16,15 @@ enum AnEnum { message AnotherMessage { optional string string = 1; optional bool value = 2; + repeated int32 integers = 3; } message ContainerMessage { extend ExtendableMessage { optional bool another_thing = 100; + // The largest permitted field number, per + // https://developers.google.com/protocol-buffers/docs/proto#simple + optional bool large_field_number = 536870911; } } diff --git a/objectivec/DevTools/check_version_stamps.sh b/objectivec/DevTools/check_version_stamps.sh new file mode 100755 index 0000000000..5de9ef146e --- /dev/null +++ b/objectivec/DevTools/check_version_stamps.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# This script checks that the runtime version number constant in the compiler +# source and in the runtime source is the same. +# +# A distro can be made of the protobuf sources with only a subset of the +# languages, so if the compiler depended on the Objective C runtime, those +# builds would break. At the same time, we don't want the runtime source +# depending on the compiler sources; so two copies of the constant are needed. + +set -eu + +readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") +readonly ProtoRootDir="${ScriptDir}/../.." + +die() { + echo "Error: $1" + exit 1 +} + +readonly ConstantName=GOOGLE_PROTOBUF_OBJC_GEN_VERSION + +# Collect version from plugin sources. + +readonly PluginSrc="${ProtoRootDir}/src/google/protobuf/compiler/objectivec/objectivec_file.cc" +readonly PluginVersion=$( \ + cat "${PluginSrc}" \ + | sed -n -e "s:const int32_t ${ConstantName} = \([0-9]*\);:\1:p" +) + +if [[ -z "${PluginVersion}" ]] ; then + die "Failed to fine ${ConstantName} in the plugin source (${PluginSrc})." +fi + +# Collect version from runtime sources. + +readonly RuntimeSrc="${ProtoRootDir}/objectivec/GPBBootstrap.h" +readonly RuntimeVersion=$( \ + cat "${RuntimeSrc}" \ + | sed -n -e "s:#define ${ConstantName} \([0-9]*\):\1:p" +) + +if [[ -z "${RuntimeVersion}" ]] ; then + die "Failed to fine ${ConstantName} in the runtime source (${RuntimeSrc})." +fi + +# Compare them. + +if [[ "${PluginVersion}" != "${RuntimeVersion}" ]] ; then + die "Versions don't match! + Plugin: ${PluginVersion} from ${PluginSrc} + Runtime: ${RuntimeVersion} from ${RuntimeSrc} +" +fi + +# Success diff --git a/objectivec/DevTools/generate_descriptors_proto.sh b/objectivec/DevTools/generate_descriptors_proto.sh new file mode 100755 index 0000000000..42502bfec5 --- /dev/null +++ b/objectivec/DevTools/generate_descriptors_proto.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This script will generate the common descriptors needed by the Objective C +# runtime. + +# HINT: Flags passed to generate_descriptor_proto.sh will be passed directly +# to make when building protoc. This is particularly useful for passing +# -j4 to run 4 jobs simultaneously. + +set -eu + +readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") +readonly ProtoRootDir="${ScriptDir}/../.." +readonly ProtoC="${ProtoRootDir}/src/protoc" + +pushd "${ProtoRootDir}" > /dev/null + +# Compiler build fails if config.h hasn't been made yet (even if configure/etc. +# have been run, so get that made first). +make $@ config.h + +# Make sure the compiler is current. +cd src +make $@ protoc + +# These really should only be run when the inputs or compiler are newer than +# the outputs. + +# Needed by the runtime. +./protoc --objc_out="${ProtoRootDir}/objectivec" google/protobuf/descriptor.proto + +# Well known types that the library provides helpers for. +./protoc --objc_out="${ProtoRootDir}/objectivec" google/protobuf/timestamp.proto +./protoc --objc_out="${ProtoRootDir}/objectivec" google/protobuf/duration.proto + +popd > /dev/null diff --git a/objectivec/DevTools/pddm.py b/objectivec/DevTools/pddm.py new file mode 100755 index 0000000000..d1b53f5a75 --- /dev/null +++ b/objectivec/DevTools/pddm.py @@ -0,0 +1,687 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2015 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""PDDM - Poor Developers' Debug-able Macros + +A simple markup that can be added in comments of source so they can then be +expanded out into code. Most of this could be done with CPP macros, but then +developers can't really step through them in the debugger, this way they are +expanded to the same code, but you can debug them. + +Any file can be processed, but the syntax is designed around a C based compiler. +Processed lines start with "//%". There are three types of sections you can +create: Text (left alone), Macro Definitions, and Macro Expansions. There is +no order required between definitions and expansions, all definitions are read +before any expansions are processed (thus, if desired, definitions can be put +at the end of the file to keep them out of the way of the code). + +Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines +afterwards that start with "//%" are included in the definition. Multiple +macros can be defined in one block by just using a another "//%PDDM-DEFINE" +line to start the next macro. Optionally, a macro can be ended with +"//%PDDM-DEFINE-END", this can be useful when you want to make it clear that +trailing blank lines are included in the macro. You can also end a definition +with an expansion. + +Macro Expansions are started by single lines containing +"//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another +expansions. All lines in-between are replaced by the result of the expansion. +The first line of the expansion is always a blank like just for readability. + +Expansion itself is pretty simple, one macro can invoke another macro, but +you cannot nest the invoke of a macro in another macro (i.e. - can't do +"foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo() +within its expansion. + +When macros are expanded, the arg references can also add "$O" suffix to the +name (i.e. - "NAME$O") to specify an option to be applied. The options are: + + $S - Replace each character in the value with a space. + $l - Lowercase the first letter of the value. + $L - Lowercase the whole value. + $u - Uppercase the first letter of the value. + $U - Uppercase the whole value. + +Within a macro you can use ## to cause things to get joined together after +expansion (i.e. - "a##b" within a macro will become "ab"). + +Example: + + int foo(MyEnum x) { + switch (x) { + //%PDDM-EXPAND case(Enum_Left, 1) + //%PDDM-EXPAND case(Enum_Center, 2) + //%PDDM-EXPAND case(Enum_Right, 3) + //%PDDM-EXPAND-END + } + + //%PDDM-DEFINE case(_A, _B) + //% case _A: + //% return _B; + + A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END + can be used to avoid adding extra blank lines/returns (or make it clear when + it is desired). + + One macro can invoke another by simply using its name NAME(ARGS). You cannot + nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported). + + Within a macro you can use ## to cause things to get joined together after + processing (i.e. - "a##b" within a macro will become "ab"). + + +""" + +import optparse +import os +import re +import sys + + +# Regex for macro definition. +_MACRO_RE = re.compile(r'(?P\w+)\((?P.*?)\)') +# Regex for macro's argument definition. +_MACRO_ARG_NAME_RE = re.compile(r'^\w+$') + +# Line inserted after each EXPAND. +_GENERATED_CODE_LINE = ( + '// This block of code is generated, do not edit it directly.' +) + + +def _MacroRefRe(macro_names): + # Takes in a list of macro names and makes a regex that will match invokes + # of those macros. + return re.compile(r'\b(?P(?P(%s))\((?P.*?)\))' % + '|'.join(macro_names)) + +def _MacroArgRefRe(macro_arg_names): + # Takes in a list of macro arg names and makes a regex that will match + # uses of those args. + return re.compile(r'\b(?P(%s))(\$(?P