From 4d98369f6d00dbdaa45ee2b91f1ec149a60ef905 Mon Sep 17 00:00:00 2001 From: Pradeep Gollakota Date: Fri, 18 Mar 2016 23:06:14 -0700 Subject: [PATCH 1/6] Allow custom URLs for Any in JsonFormat - Using custom URL for types in Any will no longer throw an InvalidProtocolBufferException in JsonFormat - Fixes #1128 --- .../src/main/java/com/google/protobuf/util/JsonFormat.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java index d13ff0ed04..c9a28c9e07 100644 --- a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java +++ b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java @@ -951,16 +951,15 @@ public class JsonFormat { } } - private static final String TYPE_URL_PREFIX = "type.googleapis.com"; - + private static String getTypeName(String typeUrl) throws InvalidProtocolBufferException { String[] parts = typeUrl.split("/"); - if (parts.length != 2 || !parts[0].equals(TYPE_URL_PREFIX)) { + if (parts.length == 1) { throw new InvalidProtocolBufferException( "Invalid type url found: " + typeUrl); } - return parts[1]; + return parts[parts.length - 1]; } private static class ParserImpl { From bd41a39f693d8307d407e42b634b315e075b6c8f Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Mon, 21 Mar 2016 11:11:14 -0400 Subject: [PATCH 2/6] Only create the readonlySemaphore on demand. This will lower the amount of dispatch_semaphores created per Message when the full object tree isn't walked in a way that would require them to be created. Uses a dispatch_once_t for one time init of the dispatch_semaphore. --- objectivec/GPBMessage.m | 9 ++++++--- objectivec/GPBMessage_PackagePrivate.h | 12 ++++++++++++ objectivec/GPBUtilities.m | 1 + 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 94d179bebf..3da85b5bfe 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -556,6 +556,7 @@ static id GetArrayIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { id array = GPBGetObjectIvarWithFieldNoAutocreate(self, field); if (!array) { // Check again after getting the lock. + GPBPrepareReadOnlySemaphore(self); dispatch_semaphore_wait(self->readOnlySemaphore_, DISPATCH_TIME_FOREVER); array = GPBGetObjectIvarWithFieldNoAutocreate(self, field); if (!array) { @@ -586,6 +587,7 @@ static id GetMapIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { id dict = GPBGetObjectIvarWithFieldNoAutocreate(self, field); if (!dict) { // Check again after getting the lock. + GPBPrepareReadOnlySemaphore(self); dispatch_semaphore_wait(self->readOnlySemaphore_, DISPATCH_TIME_FOREVER); dict = GPBGetObjectIvarWithFieldNoAutocreate(self, field); if (!dict) { @@ -791,8 +793,6 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { if ((self = [super init])) { messageStorage_ = (GPBMessage_StoragePtr)( ((uint8_t *)self) + class_getInstanceSize([self class])); - - readOnlySemaphore_ = dispatch_semaphore_create(1); } return self; @@ -868,7 +868,9 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { - (void)dealloc { [self internalClear:NO]; NSCAssert(!autocreator_, @"Autocreator was not cleared before dealloc."); - dispatch_release(readOnlySemaphore_); + if (readOnlySemaphore_) { + dispatch_release(readOnlySemaphore_); + } [super dealloc]; } @@ -1706,6 +1708,7 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { } // Check for an autocreated value. + GPBPrepareReadOnlySemaphore(self); dispatch_semaphore_wait(readOnlySemaphore_, DISPATCH_TIME_FOREVER); value = [autocreatedExtensionMap_ objectForKey:extension]; if (!value) { diff --git a/objectivec/GPBMessage_PackagePrivate.h b/objectivec/GPBMessage_PackagePrivate.h index b7e24fc933..478db2cf0f 100644 --- a/objectivec/GPBMessage_PackagePrivate.h +++ b/objectivec/GPBMessage_PackagePrivate.h @@ -67,6 +67,10 @@ typedef struct GPBMessage_Storage *GPBMessage_StoragePtr; // priority inversion: // http://mjtsai.com/blog/2015/12/16/osspinlock-is-unsafe/ // https://lists.swift.org/pipermail/swift-dev/Week-of-Mon-20151214/000372.html + // Use of readOnlySemaphore_ must be prefaced by a call to + // GPBPrepareReadOnlySemaphore to ensure it has been created. This allows + // readOnlySemaphore_ to be only created when actually needed. + dispatch_once_t readOnlySemaphoreCreationOnce_; dispatch_semaphore_t readOnlySemaphore_; } @@ -103,6 +107,14 @@ typedef struct GPBMessage_Storage *GPBMessage_StoragePtr; CF_EXTERN_C_BEGIN + +// Call this before using the readOnlySemaphore_. This ensures it is created only once. +NS_INLINE void GPBPrepareReadOnlySemaphore(GPBMessage *self) { + dispatch_once(&self->readOnlySemaphoreCreationOnce_, ^{ + self->readOnlySemaphore_ = dispatch_semaphore_create(1); + }); +} + // Returns a new instance that was automatically created by |autocreator| for // its field |field|. GPBMessage *GPBCreateMessageWithAutocreator(Class msgClass, diff --git a/objectivec/GPBUtilities.m b/objectivec/GPBUtilities.m index 3e9d11c01b..447c749aa7 100644 --- a/objectivec/GPBUtilities.m +++ b/objectivec/GPBUtilities.m @@ -412,6 +412,7 @@ id GPBGetObjectIvarWithField(GPBMessage *self, GPBFieldDescriptor *field) { return field.defaultValue.valueMessage; } + GPBPrepareReadOnlySemaphore(self); dispatch_semaphore_wait(self->readOnlySemaphore_, DISPATCH_TIME_FOREVER); GPBMessage *result = GPBGetObjectIvarWithFieldNoAutocreate(self, field); if (!result) { From 64dfb5f80a4c09ebf09d7f5c2bf0655c840e690b Mon Sep 17 00:00:00 2001 From: topillar Date: Tue, 22 Mar 2016 23:45:41 +0800 Subject: [PATCH 3/6] Update coded_stream.h fix warning treated as error prevents building on 64-bit windows. --- src/google/protobuf/io/coded_stream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index e37710035a..c81a33ac6b 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -1136,7 +1136,7 @@ inline void CodedOutputStream::WriteVarint32(uint32 value) { // this write won't cross the end, so we can skip the checks. uint8* target = buffer_; uint8* end = WriteVarint32ToArray(value, target); - int size = end - target; + int size = static_cast(end - target); Advance(size); } else { WriteVarint32SlowPath(value); From d5a573274df4fabd2afd34b5d98bc7ccf5cb5229 Mon Sep 17 00:00:00 2001 From: Steven Parkes Date: Tue, 22 Mar 2016 17:56:07 -0700 Subject: [PATCH 4/6] export well known protos --- BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/BUILD b/BUILD index 2cb96b20aa..e35a2e8e2b 100644 --- a/BUILD +++ b/BUILD @@ -194,6 +194,12 @@ RELATIVE_WELL_KNOWN_PROTOS = [ WELL_KNOWN_PROTOS = ["src/" + s for s in RELATIVE_WELL_KNOWN_PROTOS] +filegroup( + name = "well_known_protos", + srcs = WELL_KNOWN_PROTOS, + visibility = ["//visibility:public"], +) + cc_proto_library( name = "cc_wkt_protos", srcs = WELL_KNOWN_PROTOS, From 3f917447e7805fb470bac9487e2dcc19da3c545b Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Thu, 24 Mar 2016 15:25:21 -0400 Subject: [PATCH 5/6] The message was autoreleased, the -releases are an over release. --- objectivec/GPBMessage.m | 2 -- 1 file changed, 2 deletions(-) diff --git a/objectivec/GPBMessage.m b/objectivec/GPBMessage.m index 3da85b5bfe..0e1599dcc2 100644 --- a/objectivec/GPBMessage.m +++ b/objectivec/GPBMessage.m @@ -1920,7 +1920,6 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { } } @catch (NSException *exception) { - [message release]; message = nil; if (errorPtr) { *errorPtr = MessageErrorWithReason(GPBMessageErrorCodeMalformedData, @@ -1929,7 +1928,6 @@ static GPBUnknownFieldSet *GetOrMakeUnknownFields(GPBMessage *self) { } #ifdef DEBUG if (message && !message.initialized) { - [message release]; message = nil; if (errorPtr) { *errorPtr = MessageError(GPBMessageErrorCodeMissingRequiredField, nil); From 8d47d7873e7cf332d30e7a661a4ec9cc4d794ad8 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Tue, 29 Mar 2016 09:21:50 -0400 Subject: [PATCH 6/6] Mark iOS tests as able to fail. Travis updated their images to include an xctool that can randomly kill tests, so mark them as flaky to avoid turning things red. --- .travis.yml | 9 +++++++++ travis.sh | 8 +------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9bc4d88ff0..60c0427d6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,5 +71,14 @@ matrix: # we moved to an OS X image that is 10.11. - os: osx env: CONFIG=python_cpp + # xctool 0.2.8 seems to have a bug where it randomly kills tests saying + # they failed. + # https://github.com/facebook/xctool/issues/619 + # https://github.com/google/protobuf/issues/1232 + # travis updated their images to include 0.2.8: + # https://blog.travis-ci.com/2016-03-23-xcode-image-updates + # Mark the iOS test as flakey so these failures don't turn things red. + - os: osx + env: CONFIG=objectivec_ios notifications: email: false diff --git a/travis.sh b/travis.sh index ff5e99d586..f039b3921a 100755 --- a/travis.sh +++ b/travis.sh @@ -171,13 +171,7 @@ internal_objectivec_common () { # http://docs.travis-ci.com/user/osx-ci-environment/ # We don't use a before_install because we test multiple OSes. brew update - # xctool 0.2.8 seems to have a bug where it randomly kills tests saying - # they failed. Disabling the updates, but letting it report about being - # updates as a hint that this needs to eventually get re-enabled. - # https://github.com/facebook/xctool/issues/619 - # https://github.com/google/protobuf/issues/1232 - brew outdated xctool || true - #brew outdated xctool || brew upgrade xctool + brew outdated xctool || brew upgrade xctool # Reused the build script that takes care of configuring and ensuring things # are up to date. Xcode and conformance tests will be directly invoked. objectivec/DevTools/full_mac_build.sh \