Merge tag 'refs/tags/sync-piper' into sync-stage

pull/10196/head
theodorerose 2 years ago
commit 270111c472
  1. 1
      java/core/src/main/java/com/google/protobuf/Descriptors.java
  2. 18
      java/core/src/test/java/com/google/protobuf/DescriptorsTest.java
  3. 41
      java/kotlin-lite/src/test/kotlin/com/google/protobuf/Proto2LiteTest.kt
  4. 41
      java/kotlin/src/test/kotlin/com/google/protobuf/Proto2Test.kt
  5. 19
      java/kotlin/src/test/kotlin/com/google/protobuf/Proto3Test.kt
  6. 4
      java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto2.proto
  7. 2
      java/kotlin/src/test/proto/com/google/protobuf/evil_names_proto3.proto
  8. 33
      java/protoc/pom.xml
  9. 55
      protoc-artifacts/Dockerfile
  10. 146
      protoc-artifacts/README.md
  11. 294
      protoc-artifacts/build-protoc.sh
  12. 120
      protoc-artifacts/build-zip.sh
  13. 6
      python/google/protobuf/text_encoding.py
  14. 6
      python/google/protobuf/text_format.py
  15. 2
      src/google/protobuf/compiler/java/enum_field.cc
  16. 2
      src/google/protobuf/compiler/java/enum_field_lite.cc
  17. 2
      src/google/protobuf/compiler/java/file.cc
  18. 13
      src/google/protobuf/compiler/java/helpers.cc
  19. 3
      src/google/protobuf/compiler/java/helpers.h
  20. 33
      src/google/protobuf/compiler/java/message.cc
  21. 2
      src/google/protobuf/compiler/java/message_field.cc
  22. 2
      src/google/protobuf/compiler/java/message_field_lite.cc
  23. 36
      src/google/protobuf/compiler/java/message_lite.cc
  24. 16
      src/google/protobuf/compiler/objectivec/objectivec_enum.cc
  25. 7
      src/google/protobuf/stubs/strutil.cc
  26. 623
      src/google/protobuf/util/json_util_test.cc

@ -78,6 +78,7 @@ import java.util.logging.Logger;
*
* @author kenton@google.com Kenton Varda
*/
@CheckReturnValue
public final class Descriptors {
private static final Logger logger = Logger.getLogger(Descriptors.class.getName());
private static final int[] EMPTY_INT_ARRAY = new int[0];

@ -559,8 +559,10 @@ public class DescriptorsTest {
Descriptors.FileDescriptor.buildFrom(barProto, new FileDescriptor[] {fooFile});
// Items in the FileDescriptor array can be in any order.
Descriptors.FileDescriptor.buildFrom(bazProto, new FileDescriptor[] {fooFile, barFile});
Descriptors.FileDescriptor.buildFrom(bazProto, new FileDescriptor[] {barFile, fooFile});
FileDescriptor unused1 =
Descriptors.FileDescriptor.buildFrom(bazProto, new FileDescriptor[] {fooFile, barFile});
FileDescriptor unused2 =
Descriptors.FileDescriptor.buildFrom(bazProto, new FileDescriptor[] {barFile, fooFile});
}
@Test
@ -622,7 +624,8 @@ public class DescriptorsTest {
.setName("bar")
.setNumber(1)))
.build();
Descriptors.FileDescriptor.buildFrom(fooProto, new FileDescriptor[0], true);
FileDescriptor unused =
Descriptors.FileDescriptor.buildFrom(fooProto, new FileDescriptor[0], true);
}
@Test
@ -656,7 +659,8 @@ public class DescriptorsTest {
Descriptors.FileDescriptor.buildFrom(forwardProto, new FileDescriptor[] {barFile});
try {
Descriptors.FileDescriptor.buildFrom(fooProto, new FileDescriptor[] {forwardFile});
FileDescriptor unused =
Descriptors.FileDescriptor.buildFrom(fooProto, new FileDescriptor[] {forwardFile});
assertWithMessage("DescriptorValidationException expected").fail();
} catch (DescriptorValidationException e) {
assertThat(e).hasMessageThat().contains("Bar");
@ -694,7 +698,8 @@ public class DescriptorsTest {
FileDescriptor barFile = Descriptors.FileDescriptor.buildFrom(barProto, new FileDescriptor[0]);
FileDescriptor forwardFile =
Descriptors.FileDescriptor.buildFrom(forwardProto, new FileDescriptor[] {barFile});
Descriptors.FileDescriptor.buildFrom(fooProto, new FileDescriptor[] {forwardFile});
FileDescriptor unused =
Descriptors.FileDescriptor.buildFrom(fooProto, new FileDescriptor[] {forwardFile});
}
/** Tests the translate/crosslink for an example with a more complex namespace referencing. */
@ -825,7 +830,8 @@ public class DescriptorsTest {
.build())
.build())
.build();
Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, new FileDescriptor[0]);
FileDescriptor unused =
Descriptors.FileDescriptor.buildFrom(fileDescriptorProto, new FileDescriptor[0]);
}
@Test

@ -46,13 +46,6 @@ import com.google.protobuf.UnittestLite.TestEmptyMessageLite
import com.google.protobuf.UnittestLite.TestEmptyMessageWithExtensionsLite
import com.google.protobuf.copy
import com.google.protobuf.foreignMessageLite
import com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.EvilNamesProto2
import com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.HardKeywordsAllTypesProto2
import com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.Interface
import com.google.protobuf.kotlin.generator.HardKeywordsAllTypesProto2Kt
import com.google.protobuf.kotlin.generator.evilNamesProto2
import com.google.protobuf.kotlin.generator.hardKeywordsAllTypesProto2
import com.google.protobuf.kotlin.generator.interface_
import com.google.protobuf.optionalGroupExtensionLite
import com.google.protobuf.repeatedGroupExtensionLite
import com.google.protobuf.testAllExtensionsLite
@ -62,6 +55,13 @@ import com.google.protobuf.testEmptyMessageWithExtensionsLite
import protobuf_unittest.MapLiteUnittest.MapEnumLite
import protobuf_unittest.MapLiteUnittest.TestMapLite
import protobuf_unittest.testMapLite
import `in`.com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.EvilNamesProto2
import `in`.com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.HardKeywordsAllTypesProto2
import `in`.com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.Interface
import `in`.com.google.protobuf.kotlin.generator.HardKeywordsAllTypesProto2Kt
import `in`.com.google.protobuf.kotlin.generator.evilNamesProto2
import `in`.com.google.protobuf.kotlin.generator.hardKeywordsAllTypesProto2
import `in`.com.google.protobuf.kotlin.generator.interface_
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
@ -178,6 +178,7 @@ class Proto2LiteTest {
.isEqualTo(TestUtilLite.getAllLiteSetBuilder().build())
}
@Suppress("CheckResult")
@Test
fun testGetters() {
testAllTypesLite {
@ -198,6 +199,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testDefaultGetters() {
testAllTypesLite {
@ -208,6 +210,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testRepeatedGettersAndSetters() {
testAllTypesLite {
@ -298,6 +301,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testHazzers() {
testAllTypesLite {
@ -328,6 +332,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testClears() {
testAllTypesLite {
@ -520,6 +525,7 @@ class Proto2LiteTest {
.isEqualTo(TestUtilLite.getAllLiteExtensionsSet())
}
@Suppress("CheckResult")
@Test
fun testExtensionGetters() {
testAllExtensionsLite {
@ -543,6 +549,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testRepeatedExtensionGettersAndSetters() {
testAllExtensionsLite {
@ -632,6 +639,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testExtensionContains() {
testAllExtensionsLite {
@ -663,6 +671,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testExtensionClears() {
testAllExtensionsLite {
@ -751,6 +760,7 @@ class Proto2LiteTest {
)
}
@Suppress("CheckResult")
@Test
fun testMapGettersAndSetters() {
testMapLite {
@ -806,6 +816,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testMapRemove() {
testMapLite {
@ -831,6 +842,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testMapClear() {
testMapLite {
@ -877,7 +889,6 @@ class Proto2LiteTest {
boolean = 1L
sealed = "foo"
interface_ = 1F
in_ = 1
object_ = "foo"
cachedSize_ = "foo"
serializedSize_ = true
@ -903,7 +914,6 @@ class Proto2LiteTest {
.setBoolean(1L)
.setSealed("foo")
.setInterface(1F)
.setIn(1)
.setObject("foo")
.setCachedSize_("foo")
.setSerializedSize_(true)
@ -914,15 +924,13 @@ class Proto2LiteTest {
assertThat(interface_ {}).isEqualTo(Interface.newBuilder().build())
}
@Suppress("CheckResult")
@Test
fun testHardKeywordGettersAndSetters() {
hardKeywordsAllTypesProto2 {
as_ = 1
assertThat(as_).isEqualTo(1)
in_ = "foo"
assertThat(in_).isEqualTo("foo")
break_ = HardKeywordsAllTypesProto2.NestedEnum.FOO
assertThat(break_).isEqualTo(HardKeywordsAllTypesProto2.NestedEnum.FOO)
@ -946,15 +954,13 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testHardKeywordHazzers() {
hardKeywordsAllTypesProto2 {
as_ = 1
assertThat(hasAs_()).isTrue()
in_ = "foo"
assertThat(hasIn_()).isTrue()
break_ = HardKeywordsAllTypesProto2.NestedEnum.FOO
assertThat(hasBreak_()).isTrue()
@ -963,6 +969,7 @@ class Proto2LiteTest {
}
}
@Suppress("CheckResult")
@Test
fun testHardKeywordClears() {
hardKeywordsAllTypesProto2 {
@ -970,10 +977,6 @@ class Proto2LiteTest {
clearAs_()
assertThat(hasAs_()).isFalse()
in_ = "foo"
clearIn_()
assertThat(hasIn_()).isFalse()
break_ = HardKeywordsAllTypesProto2.NestedEnum.FOO
clearBreak_()
assertThat(hasBreak_()).isFalse()

@ -33,13 +33,6 @@ package com.google.protobuf.kotlin
import com.google.common.truth.Truth.assertThat
import com.google.protobuf.TestUtil
import com.google.protobuf.TestUtil.toBytes
import com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.EvilNamesProto2
import com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.HardKeywordsAllTypesProto2
import com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.Interface
import com.google.protobuf.kotlin.generator.HardKeywordsAllTypesProto2Kt
import com.google.protobuf.kotlin.generator.evilNamesProto2
import com.google.protobuf.kotlin.generator.hardKeywordsAllTypesProto2
import com.google.protobuf.kotlin.generator.interface_
import com.google.protobuf.test.UnittestImport.ImportEnum
import com.google.protobuf.test.UnittestImport.ImportMessage
import com.google.protobuf.test.UnittestImportPublic.PublicImportMessage
@ -67,6 +60,13 @@ import protobuf_unittest.testEmptyMessageWithExtensions
import protobuf_unittest.testEnumMap
import protobuf_unittest.testIntIntMap
import protobuf_unittest.testMaps
import `in`.com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.EvilNamesProto2
import `in`.com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.HardKeywordsAllTypesProto2
import `in`.com.google.protobuf.kotlin.generator.EvilNamesProto2OuterClass.Interface
import `in`.com.google.protobuf.kotlin.generator.HardKeywordsAllTypesProto2Kt
import `in`.com.google.protobuf.kotlin.generator.evilNamesProto2
import `in`.com.google.protobuf.kotlin.generator.hardKeywordsAllTypesProto2
import `in`.com.google.protobuf.kotlin.generator.interface_
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
@ -183,6 +183,7 @@ class Proto2Test {
.isEqualTo(TestUtil.getAllSetBuilder().build())
}
@Suppress("CheckResult")
@Test
fun testGetters() {
testAllTypes {
@ -203,6 +204,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testDefaultGetters() {
testAllTypes {
@ -213,6 +215,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testRepeatedGettersAndSetters() {
testAllTypes {
@ -294,6 +297,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testHazzers() {
testAllTypes {
@ -324,6 +328,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testClears() {
testAllTypes {
@ -512,6 +517,7 @@ class Proto2Test {
.isEqualTo(TestUtil.getAllExtensionsSet())
}
@Suppress("CheckResult")
@Test
fun testExtensionGetters() {
testAllExtensions {
@ -534,6 +540,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testRepeatedExtensionGettersAndSetters() {
testAllExtensions {
@ -614,6 +621,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testExtensionContains() {
testAllExtensions {
@ -644,6 +652,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testExtensionClears() {
testAllExtensions {
@ -725,6 +734,7 @@ class Proto2Test {
)
}
@Suppress("CheckResult")
@Test
fun testMapGettersAndSetters() {
val intMap = testIntIntMap {
@ -776,6 +786,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testMapRemove() {
val intMap = testIntIntMap {
@ -803,6 +814,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testMapClear() {
val intMap = testIntIntMap {
@ -851,7 +863,6 @@ class Proto2Test {
boolean = 1L
sealed = "foo"
interface_ = 1F
in_ = 1
object_ = "foo"
cachedSize_ = "foo"
serializedSize_ = true
@ -877,7 +888,6 @@ class Proto2Test {
.setBoolean(1L)
.setSealed("foo")
.setInterface(1F)
.setIn(1)
.setObject("foo")
.setCachedSize_("foo")
.setSerializedSize_(true)
@ -888,15 +898,13 @@ class Proto2Test {
assertThat(interface_ {}).isEqualTo(Interface.newBuilder().build())
}
@Suppress("CheckResult")
@Test
fun testHardKeywordGettersAndSetters() {
hardKeywordsAllTypesProto2 {
as_ = 1
assertThat(as_).isEqualTo(1)
in_ = "foo"
assertThat(in_).isEqualTo("foo")
break_ = HardKeywordsAllTypesProto2.NestedEnum.FOO
assertThat(break_).isEqualTo(HardKeywordsAllTypesProto2.NestedEnum.FOO)
@ -920,15 +928,13 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testHardKeywordHazzers() {
hardKeywordsAllTypesProto2 {
as_ = 1
assertThat(hasAs_()).isTrue()
in_ = "foo"
assertThat(hasIn_()).isTrue()
break_ = HardKeywordsAllTypesProto2.NestedEnum.FOO
assertThat(hasBreak_()).isTrue()
@ -937,6 +943,7 @@ class Proto2Test {
}
}
@Suppress("CheckResult")
@Test
fun testHardKeywordClears() {
hardKeywordsAllTypesProto2 {
@ -944,10 +951,6 @@ class Proto2Test {
clearAs_()
assertThat(hasAs_()).isFalse()
in_ = "foo"
clearIn_()
assertThat(hasIn_()).isFalse()
break_ = HardKeywordsAllTypesProto2.NestedEnum.FOO
clearBreak_()
assertThat(hasBreak_()).isFalse()

@ -31,13 +31,13 @@
package com.google.protobuf.kotlin
import com.google.common.truth.Truth.assertThat
import com.google.protobuf.kotlin.generator.EvilNamesProto3OuterClass.Class
import com.google.protobuf.kotlin.generator.EvilNamesProto3OuterClass.EvilNamesProto3
import com.google.protobuf.kotlin.generator.EvilNamesProto3OuterClass.HardKeywordsAllTypesProto3
import com.google.protobuf.kotlin.generator.HardKeywordsAllTypesProto3Kt
import com.google.protobuf.kotlin.generator.class_
import com.google.protobuf.kotlin.generator.evilNamesProto3
import com.google.protobuf.kotlin.generator.hardKeywordsAllTypesProto3
import com.google.protobuf.kotlin.generator.`in`.EvilNamesProto3OuterClass.Class
import com.google.protobuf.kotlin.generator.`in`.EvilNamesProto3OuterClass.EvilNamesProto3
import com.google.protobuf.kotlin.generator.`in`.EvilNamesProto3OuterClass.HardKeywordsAllTypesProto3
import com.google.protobuf.kotlin.generator.`in`.HardKeywordsAllTypesProto3Kt
import com.google.protobuf.kotlin.generator.`in`.class_
import com.google.protobuf.kotlin.generator.`in`.evilNamesProto3
import com.google.protobuf.kotlin.generator.`in`.hardKeywordsAllTypesProto3
import proto3_unittest.TestAllTypesKt
import proto3_unittest.TestAllTypesKt.nestedMessage
import proto3_unittest.UnittestProto3.TestAllTypes
@ -54,6 +54,7 @@ import org.junit.runners.JUnit4
@RunWith(JUnit4::class)
class Proto3Test {
@Suppress("CheckResult")
@Test
fun testGettersAndSetters() {
testAllTypes {
@ -70,6 +71,7 @@ class Proto3Test {
}
}
@Suppress("CheckResult")
@Test
fun testRepeatedGettersAndSetters() {
testAllTypes {
@ -259,6 +261,7 @@ class Proto3Test {
assertThat(class_ {}).isEqualTo(Class.newBuilder().build())
}
@Suppress("CheckResult")
@Test
fun testHardKeywordGettersAndSetters() {
hardKeywordsAllTypesProto3 {
@ -291,6 +294,7 @@ class Proto3Test {
}
}
@Suppress("CheckResult")
@Test
fun testHardKeywordHazzers() {
hardKeywordsAllTypesProto3 {
@ -308,6 +312,7 @@ class Proto3Test {
}
}
@Suppress("CheckResult")
@Test
fun testHardKeywordClears() {
hardKeywordsAllTypesProto3 {

@ -33,7 +33,7 @@ syntax = "proto2";
package protobuf.kotlin.generator;
option java_package = "com.google.protobuf.kotlin.generator";
option java_package = "in.com.google.protobuf.kotlin.generator";
message EvilNamesProto2 {
optional bool initialized = 1;
@ -61,7 +61,6 @@ message EvilNamesProto2 {
optional int64 boolean = 18;
optional string sealed = 19;
optional float interface = 20;
optional int32 in = 21;
optional string object = 22;
optional string cached_size = 23;
optional bool serialized_size = 24;
@ -79,7 +78,6 @@ message HardKeywordsAllTypesProto2 {
}
optional int32 as = 1;
optional string in = 2;
optional NestedEnum break = 3;
map<int32, int32> continue = 4;
optional NestedMessage do = 5;

@ -33,7 +33,7 @@ syntax = "proto3";
package protobuf.kotlin.generator;
option java_package = "com.google.protobuf.kotlin.generator";
option java_package = "com.google.protobuf.kotlin.generator.in";
message EvilNamesProto3 {
bool initialized = 1;

@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.google</groupId>
<artifactId>google</artifactId>
<version>1</version>
</parent>
<groupId>com.google.protobuf</groupId>
<artifactId>protoc</artifactId>
<version>3.21.2-rc1</version>
<packaging>pom</packaging>
<name>Protobuf Compiler</name>
<description>
Protobuf Compiler (protoc) is a compiler for .proto files. It generates
language-specific code for Protobuf messages and RPC interfaces.
</description>
<inceptionYear>2008</inceptionYear>
<url>https://developers.google.com/protocol-buffers/</url>
<licenses>
<license>
<name>BSD-3-Clause</name>
<url>https://opensource.org/licenses/BSD-3-Clause</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<url>https://github.com/protocolbuffers/protobuf</url>
<connection>
scm:git:https://github.com/protocolbuffers/protobuf.git
</connection>
</scm>
</project>

@ -1,55 +0,0 @@
FROM centos:6.9
RUN yum install -y git \
tar \
wget \
make \
autoconf \
curl-devel \
unzip \
automake \
libtool \
glibc-static.i686 \
glibc-devel \
glibc-devel.i686 \
&& \
yum clean all
# Install Java 8
RUN wget -q --no-cookies --no-check-certificate \
--header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \
"http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz" \
-O - | tar xz -C /var/local
ENV JAVA_HOME /var/local/jdk1.8.0_131
ENV PATH $JAVA_HOME/bin:$PATH
# Install Maven
RUN wget -q http://apache.cs.utah.edu/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -O - | \
tar xz -C /var/local
ENV PATH /var/local/apache-maven-3.3.9/bin:$PATH
# Install GCC 4.8 to support -static-libstdc++
RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d && \
bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' && \
bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" && \
sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo && \
rpm --rebuilddb
RUN yum install -y devtoolset-2-gcc \
devtoolset-2-gcc-c++ \
devtoolset-2-binutils \
devtoolset-2-libstdc++-devel \
devtoolset-2-libatomic-devel \
libatomic \
devtoolset-2-gcc.i686 \
devtoolset-2-gcc-c++.i686 \
devtoolset-2-binutils.i686 \
devtoolset-2-libstdc++-devel.i686 \
devtoolset-2-libatomic-devel.i686 \
libatomic.i686 && \
yum clean all
COPY scl-enable-devtoolset.sh /var/local/
# Start in devtoolset environment that uses GCC 4.7
ENTRYPOINT ["/var/local/scl-enable-devtoolset.sh"]

@ -2,20 +2,13 @@
``protoc`` is the compiler for ``.proto`` files. It generates language bindings
for the messages and/or RPC services from ``.proto`` files.
Because ``protoc`` is a native executable, the scripts under this directory
build and publish a ``protoc`` executable (a.k.a. artifact) to Maven
repositories. The artifact can be used by build automation tools so that users
would not need to compile and install ``protoc`` for their systems.
Because ``protoc`` is a native executable, we build and publish a ``protoc``
executable (a.k.a. artifact) to Maven repositories. The artifact can be used by
build automation tools so that users would not need to compile and install
``protoc`` for their systems.
If you would like us to publish protoc artifact for a new platform, please send
us a pull request to add support for the new platform. You would need to change
the following files:
* [build-protoc.sh](build-protoc.sh): script to cross-build the protoc for your
platform.
* [pom.xml](pom.xml): script to upload artifacts to maven.
* [build-zip.sh](build-zip.sh): script to package published maven artifacts in
our release page.
The [pom.xml](pom.xml) file specifies configuration details used by Maven to
publish the protoc binaries. This is only used internally for releases.
## Maven Location
The published protoc artifacts are available on Maven here:
@ -32,130 +25,3 @@ The name of a published ``protoc`` artifact is in the following format:
Note that artifacts for linux/macos also have the `.exe` suffix but they are
not windows binaries.
## System requirement
Install [Apache Maven](http://maven.apache.org/) if you don't have it.
The scripts only work under Unix-like environments, e.g., Linux, MacOSX, and
Cygwin or MinGW for Windows. Please see ``README.md`` of the Protobuf project
for how to set up the build environment.
## Building from a freshly checked-out source
If you just checked out the Protobuf source from github, you need to
generate the configure script.
Under the protobuf project directory:
```
$ ./autogen.sh
```
### Build the artifact for each platform
Run the build-protoc.sh script under this protoc-artifacts directory to build the protoc
artifact for each platform. For example:
```
$ cd protoc-artifacts
$ ./build-protoc.sh linux x86_64 protoc
```
The above command will produce a `target/linux/x86_64/protoc` binary under the
protoc-artifacts directory.
For a list of supported platforms, see the comments in the build-protoc.sh
script. We only use this script to build artifacts on Ubuntu and MacOS (both
with x86_64, and do cross-compilation for other platforms.
### Tips for building for Linux
We build on Centos 6.9 to provide a good compatibility for not very new
systems. We have provided a ``Dockerfile`` under this directory to build the
environment. It has been tested with Docker 1.6.1.
To build a image:
```
$ docker build -t protoc-artifacts .
```
To run the image:
```
$ docker run -it --rm=true protoc-artifacts bash
```
To checkout protobuf (run within the container):
```
$ # Replace v3.5.1 with the version you want
$ wget -O - https://github.com/protocolbuffers/protobuf/archive/v3.5.1.tar.gz | tar xvzp
```
### Windows build
We no longer use scripts in this directory to build windows artifacts. Instead,
we use Visual Studio 2015 to build our windows release artifacts. See our
[kokoro windows build scripts here](../kokoro/release/protoc/windows/build.bat).
To upload windows artifacts, copy the built binaries into this directory and
put it into the target/windows/(x86_64|x86_32) directory the same way as the
artifacts for other platforms. That will allow the maven script to find and
upload the artifacts to maven.
## To push artifacts to Maven Central
Before you can upload artifacts to Maven Central repository, make sure you have
read [this page](http://central.sonatype.org/pages/apache-maven.html) on how to
configure GPG and Sonatype account.
Before you do the deployment, make sure you have built the protoc artifacts for
every supported platform and put them under the target directory. Example
target directory layout:
+ pom.xml
+ target
+ linux
+ x86_64
protoc.exe
+ x86_32
protoc.exe
+ aarch_64
protoc.exe
+ ppcle_64
protoc.exe
+ s390_64
protoc.exe
+ osx
+ x86_64
protoc.exe
+ x86_32
protoc.exe
+ windows
+ x86_64
protoc.exe
+ x86_32
protoc.exe
You will need to build the artifacts on multiple machines and gather them
together into one place.
Use the following command to deploy artifacts for the host platform to a
staging repository.
```
$ mvn deploy -P release
```
It creates a new staging repository. Go to
https://oss.sonatype.org/#stagingRepositories and find the repository, usually
in the name like ``comgoogle-123``. Verify that the staging repository has all
the binaries, close and release this repository.
## Tested build environments
We have successfully built artifacts on the following environments:
- Linux x86_32 and x86_64:
- Centos 6.9 (within Docker 1.6.1)
- Ubuntu 14.04.5 64-bit
- Linux aarch_64: Cross compiled with `g++-aarch64-linux-gnu` on Ubuntu 14.04.5 64-bit
- Mac OS X x86_32 and x86_64: Mac OS X 10.9.5

@ -1,294 +0,0 @@
#!/bin/bash
# Builds protoc executable into target/<OS>/<ARCH>/protoc.exe; optionally builds
# protoc plugins into target/<OS>/<ARCH>/protoc-gen-*.exe
#
# Usage: ./build-protoc.sh <OS> <ARCH> <TARGET>
#
# <TARGET> can be "protoc" or "protoc-gen-javalite". Supported <OS> <ARCH>
# combinations:
# HOST <OS> <ARCH> <COMMENT>
# cygwin windows x86_32 Requires: i686-w64-mingw32-gcc
# cygwin windows x86_64 Requires: x86_64-w64-mingw32-gcc
# linux linux aarch_64 Requires: g++-aarch64-linux-gnu
# linux linux x86_32
# linux linux x86_64
# linux windows x86_32 Requires: i686-w64-mingw32-gcc
# linux windows x86_64 Requires: x86_64-w64-mingw32-gcc
# macos osx x86_32
# macos osx x86_64
# mingw windows x86_32
# mingw windows x86_64
#
# Before running this script, make sure you have generated the configure script
# in the parent directory (i.e., run ./autogen.sh there).
OS=$1
ARCH=$2
MAKE_TARGET=$3
if [[ $# < 3 ]]; then
echo "Not enough arguments provided."
exit 1
fi
case $MAKE_TARGET in
protoc-gen-javalite)
;;
protoc)
;;
*)
echo "Target ""$MAKE_TARGET"" invalid."
exit 1
esac
# Under Cygwin, bash doesn't have these in PATH when called from Maven which
# runs in Windows version of Java.
export PATH="/bin:/usr/bin:$PATH"
############################################################################
# Helper functions
############################################################################
E_PARAM_ERR=98
E_ASSERT_FAILED=99
# Usage:
fail()
{
echo "ERROR: $1"
echo
exit $E_ASSERT_FAILED
}
# Usage: assertEq VAL1 VAL2 $LINENO
assertEq ()
{
lineno=$3
if [ -z "$lineno" ]; then
echo "lineno not given"
exit $E_PARAM_ERR
fi
if [[ "$1" != "$2" ]]; then
echo "Assertion failed: \"$1\" == \"$2\""
echo "File \"$0\", line $lineno" # Give name of file and line number.
exit $E_ASSERT_FAILED
fi
}
# Checks the artifact is for the expected architecture
# Usage: checkArch <path-to-protoc>
checkArch ()
{
echo
echo "Checking file format ..."
if [[ "$OS" == windows || "$OS" == linux ]]; then
format="$(objdump -f "$1" | grep -o "file format .*$" | grep -o "[^ ]*$")"
echo Format=$format
if [[ "$OS" == linux ]]; then
host_machine="$(uname -m)";
if [[ "$ARCH" == x86_32 ]]; then
assertEq $format "elf32-i386" $LINENO
elif [[ "$ARCH" == x86_64 ]]; then
assertEq $format "elf64-x86-64" $LINENO
elif [[ "$ARCH" == aarch_64 ]]; then
assertEq $format "elf64-little" $LINENO
elif [[ "$ARCH" == s390_64 ]]; then
if [[ $host_machine == s390x ]];then
assertEq $format "elf64-s390" $LINENO
else
assertEq $format "elf64-big" $LINENO
fi
elif [[ "$ARCH" == ppcle_64 ]]; then
if [[ $host_machine == ppc64le ]];then
assertEq $format "elf64-powerpcle" $LINENO
else
assertEq $format "elf64-little" $LINENO
fi
else
fail "Unsupported arch: $ARCH"
fi
else
# $OS == windows
if [[ "$ARCH" == x86_32 ]]; then
assertEq $format "pei-i386" $LINENO
elif [[ "$ARCH" == x86_64 ]]; then
assertEq $format "pei-x86-64" $LINENO
else
fail "Unsupported arch: $ARCH"
fi
fi
elif [[ "$OS" == osx ]]; then
format="$(file -b "$1" | grep -o "[^ ]*$")"
echo Format=$format
if [[ "$ARCH" == x86_32 ]]; then
assertEq $format "i386" $LINENO
elif [[ "$ARCH" == x86_64 ]]; then
assertEq $format "x86_64" $LINENO
else
fail "Unsupported arch: $ARCH"
fi
else
fail "Unsupported system: $OS"
fi
echo
}
# Checks the dependencies of the artifact. Artifacts should only depend on
# system libraries.
# Usage: checkDependencies <path-to-protoc>
checkDependencies ()
{
if [[ "$OS" == windows ]]; then
dump_cmd='objdump -x '"$1"' | fgrep "DLL Name"'
white_list="KERNEL32\.dll\|msvcrt\.dll"
elif [[ "$OS" == linux ]]; then
host_machine="$(uname -m)";
dump_cmd='ldd '"$1"
if [[ "$ARCH" == x86_32 ]]; then
white_list="linux-gate\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux\.so\.2"
elif [[ "$ARCH" == x86_64 ]]; then
white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-x86-64\.so\.2"
elif [[ "$ARCH" == s390_64 ]]; then
if [[ $host_machine != s390x ]];then
dump_cmd='objdump -p '"$1"' | grep NEEDED'
fi
white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.1"
elif [[ "$ARCH" == ppcle_64 ]]; then
if [[ $host_machine != ppc64le ]];then
dump_cmd='objdump -p '"$1"' | grep NEEDED'
fi
white_list="linux-vdso64\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|libz\.so\.1\|ld64\.so\.2"
elif [[ "$ARCH" == aarch_64 ]]; then
dump_cmd='objdump -p '"$1"' | grep NEEDED'
white_list="libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1"
fi
elif [[ "$OS" == osx ]]; then
dump_cmd='otool -L '"$1"' | fgrep dylib'
white_list="libz\.1\.dylib\|libstdc++\.6\.dylib\|libSystem\.B\.dylib"
fi
if [[ -z "$white_list" || -z "$dump_cmd" ]]; then
fail "Unsupported platform $OS-$ARCH."
fi
echo "Checking for expected dependencies ..."
eval $dump_cmd | grep -i "$white_list" || fail "doesn't show any expected dependencies"
echo "Checking for unexpected dependencies ..."
eval $dump_cmd | grep -i -v "$white_list"
ret=$?
if [[ $ret == 0 ]]; then
fail "found unexpected dependencies (listed above)."
elif [[ $ret != 1 ]]; then
fail "Error when checking dependencies."
fi # grep returns 1 when "not found", which is what we expect
echo "Dependencies look good."
echo
}
############################################################################
echo "Building protoc, OS=$OS ARCH=$ARCH TARGET=$MAKE_TARGET"
CONFIGURE_ARGS="--disable-shared"
if [[ "$OS" == windows ]]; then
MAKE_TARGET="${MAKE_TARGET}.exe"
fi
# Override the default value set in configure.ac that has '-g' which produces
# huge binary.
CXXFLAGS="-DNDEBUG"
LDFLAGS=""
if [[ "$(uname)" == CYGWIN* ]]; then
assertEq "$OS" windows $LINENO
# Use mingw32 compilers because executables produced by Cygwin compiler
# always have dependency on Cygwin DLL.
if [[ "$ARCH" == x86_64 ]]; then
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32"
elif [[ "$ARCH" == x86_32 ]]; then
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-pc-mingw32"
else
fail "Unsupported arch by CYGWIN: $ARCH"
fi
elif [[ "$(uname)" == MINGW32* ]]; then
assertEq "$OS" windows $LINENO
assertEq "$ARCH" x86_32 $LINENO
elif [[ "$(uname)" == MINGW64* ]]; then
assertEq "$OS" windows $LINENO
assertEq "$ARCH" x86_64 $LINENO
elif [[ "$(uname)" == Linux* ]]; then
if [[ "$OS" == linux ]]; then
if [[ "$ARCH" == x86_64 ]]; then
CXXFLAGS="$CXXFLAGS -m64"
elif [[ "$ARCH" == x86_32 ]]; then
CXXFLAGS="$CXXFLAGS -m32"
elif [[ "$ARCH" == aarch_64 ]]; then
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=aarch64-linux-gnu"
elif [[ "$ARCH" == ppcle_64 ]]; then
CXXFLAGS="$CXXFLAGS -m64"
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=powerpc64le-linux-gnu"
elif [[ "$ARCH" == s390_64 ]]; then
CXXFLAGS="$CXXFLAGS -m64"
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=s390x-linux-gnu"
else
fail "Unsupported arch: $ARCH"
fi
elif [[ "$OS" == windows ]]; then
# Cross-compilation for Windows
CONFIGURE_ARGS="$CONFIGURE_ARGS"
if [[ "$ARCH" == x86_64 ]]; then
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=x86_64-w64-mingw32"
elif [[ "$ARCH" == x86_32 ]]; then
CONFIGURE_ARGS="$CONFIGURE_ARGS --host=i686-w64-mingw32"
else
fail "Unsupported arch: $ARCH"
fi
else
fail "Cannot build $OS on $(uname)"
fi
elif [[ "$(uname)" == Darwin* ]]; then
assertEq "$OS" osx $LINENO
# Make the binary compatible with OSX 10.7 and later
CXXFLAGS="$CXXFLAGS -mmacosx-version-min=10.7"
if [[ "$ARCH" == x86_64 ]]; then
CXXFLAGS="$CXXFLAGS -m64"
elif [[ "$ARCH" == x86_32 ]]; then
CXXFLAGS="$CXXFLAGS -m32"
else
fail "Unsupported arch: $ARCH"
fi
else
fail "Unsupported system: $(uname)"
fi
# Statically link libgcc and libstdc++.
# -s to produce stripped binary.
if [[ "$OS" == windows ]]; then
# Also static link libpthread required by mingw64
LDFLAGS="$LDFLAGS -static-libgcc -static-libstdc++ -Wl,-Bstatic -lstdc++ -lpthread -s"
elif [[ "$OS" != osx ]]; then
# And they don't work under Mac.
LDFLAGS="$LDFLAGS -static-libgcc -static-libstdc++ -s"
fi
export CXXFLAGS LDFLAGS
# Nested double quotes are unintuitive, but it works.
cd "$(dirname "$0")"
WORKING_DIR="$(pwd)"
BUILD_DIR="build/$OS/$ARCH"
TARGET_FILE="target/$OS/$ARCH/$MAKE_TARGET.exe"
mkdir -p "$BUILD_DIR" && cd "$BUILD_DIR" &&
../../../../configure $CONFIGURE_ARGS &&
cd src && make $MAKE_TARGET -j8 &&
cd "$WORKING_DIR" && mkdir -p $(dirname $TARGET_FILE) &&
cp $BUILD_DIR/src/$MAKE_TARGET $TARGET_FILE ||
exit 1
if [[ "$OS" == osx ]]; then
# Since Mac linker doesn't accept "-s", we need to run strip
strip $TARGET_FILE || exit 1
fi
checkArch $TARGET_FILE && checkDependencies $TARGET_FILE

@ -1,120 +0,0 @@
#!/bin/bash
if [ $# -ne 2 ]; then
cat <<EOF
Usage: $0 <TARGET> <VERSION_NUMBER>
TARGET: protoc | protoc-gen-javalite
Example:
$ $0 protoc 3.0.0
$ $0 protoc-gen-javalite 3.0.0
This script will download pre-built protoc or protoc plugin binaries from maven
repository and create .zip packages suitable to be included in the github
release page. If the target is protoc, well-known type .proto files will also be
included. Each invocation will create 8 zip packages:
dist/<TARGET>-<VERSION_NUMBER>-win32.zip
dist/<TARGET>-<VERSION_NUMBER>-win64.zip
dist/<TARGET>-<VERSION_NUMBER>-osx-aarch_64.zip
dist/<TARGET>-<VERSION_NUMBER>-osx-x86_64.zip
dist/<TARGET>-<VERSION_NUMBER>-linux-x86_32.zip
dist/<TARGET>-<VERSION_NUMBER>-linux-x86_64.zip
dist/<TARGET>-<VERSION_NUMBER>-linux-aarch_64.zip
dist/<TARGET>-<VERSION_NUMBER>-linux-ppcle_64.zip
dist/<TARGET>-<VERSION_NUMBER>-linux-s390_64.zip
EOF
exit 1
fi
TARGET=$1
VERSION_NUMBER=$2
# <zip file name> <binary file name> pairs.
declare -a FILE_NAMES=( \
win32.zip windows-x86_32.exe \
win64.zip windows-x86_64.exe \
osx-aarch_64.zip osx-aarch_64.exe \
osx-x86_64.zip osx-x86_64.exe \
linux-x86_32.zip linux-x86_32.exe \
linux-x86_64.zip linux-x86_64.exe \
linux-aarch_64.zip linux-aarch_64.exe \
linux-ppcle_64.zip linux-ppcle_64.exe \
linux-s390_64.zip linux-s390_64.exe \
)
# List of all well-known types to be included.
declare -a WELL_KNOWN_TYPES=( \
google/protobuf/descriptor.proto \
google/protobuf/any.proto \
google/protobuf/api.proto \
google/protobuf/duration.proto \
google/protobuf/empty.proto \
google/protobuf/field_mask.proto \
google/protobuf/source_context.proto \
google/protobuf/struct.proto \
google/protobuf/timestamp.proto \
google/protobuf/type.proto \
google/protobuf/wrappers.proto \
google/protobuf/compiler/plugin.proto \
)
set -e
# A temporary working directory to put all files.
DIR=$(mktemp -d)
# Copy over well-known types.
mkdir -p ${DIR}/include/google/protobuf/compiler
for PROTO in ${WELL_KNOWN_TYPES[@]}; do
cp -f ../src/${PROTO} ${DIR}/include/${PROTO}
done
# Create a readme file.
cat <<EOF > ${DIR}/readme.txt
Protocol Buffers - Google's data interchange format
Copyright 2008 Google Inc.
https://developers.google.com/protocol-buffers/
This package contains a precompiled binary version of the protocol buffer
compiler (protoc). This binary is intended for users who want to use Protocol
Buffers in languages other than C++ but do not want to compile protoc
themselves. To install, simply place this binary somewhere in your PATH.
If you intend to use the included well known types then don't forget to
copy the contents of the 'include' directory somewhere as well, for example
into '/usr/local/include/'.
Please refer to our official github site for more installation instructions:
https://github.com/protocolbuffers/protobuf
EOF
mkdir -p dist
mkdir -p ${DIR}/bin
# Create a zip file for each binary.
for((i=0;i<${#FILE_NAMES[@]};i+=2));do
ZIP_NAME=${FILE_NAMES[$i]}
if [ ${ZIP_NAME:0:3} = "win" ]; then
BINARY="$TARGET.exe"
else
BINARY="$TARGET"
fi
BINARY_NAME=${FILE_NAMES[$(($i+1))]}
BINARY_URL=https://repo1.maven.org/maven2/com/google/protobuf/$TARGET/${VERSION_NUMBER}/$TARGET-${VERSION_NUMBER}-${BINARY_NAME}
if ! wget ${BINARY_URL} -O ${DIR}/bin/$BINARY &> /dev/null; then
echo "[ERROR] Failed to download ${BINARY_URL}" >&2
echo "[ERROR] Skipped $TARGET-${VERSION_NAME}-${ZIP_NAME}" >&2
continue
fi
TARGET_ZIP_FILE=`pwd`/dist/$TARGET-${VERSION_NUMBER}-${ZIP_NAME}
pushd $DIR &> /dev/null
chmod +x bin/$BINARY
if [ "$TARGET" = "protoc" ]; then
zip -r ${TARGET_ZIP_FILE} include bin readme.txt &> /dev/null
else
zip -r ${TARGET_ZIP_FILE} bin &> /dev/null
fi
rm bin/$BINARY
popd &> /dev/null
echo "[INFO] Successfully created ${TARGET_ZIP_FILE}"
done

@ -53,8 +53,7 @@ for byte, string in _cescape_chr_to_symbol_map.items():
del byte, string
def CEscape(text, as_utf8):
# type: (...) -> str
def CEscape(text, as_utf8) -> str:
"""Escape a bytes string for use in an text protocol buffer.
Args:
@ -83,8 +82,7 @@ def CEscape(text, as_utf8):
_CUNESCAPE_HEX = re.compile(r'(\\+)x([0-9a-fA-F])(?![0-9a-fA-F])')
def CUnescape(text):
# type: (str) -> bytes
def CUnescape(text: str) -> bytes:
"""Unescape a text string with C-style escape sequences to UTF-8 bytes.
Args:

@ -125,8 +125,7 @@ def MessageToString(
indent=0,
message_formatter=None,
print_unknown_fields=False,
force_colon=False):
# type: (...) -> str
force_colon=False) -> str:
"""Convert protobuf message to text format.
Double values can be formatted compactly with 15 digits of
@ -191,8 +190,7 @@ def MessageToString(
return result
def MessageToBytes(message, **kwargs):
# type: (...) -> bytes
def MessageToBytes(message, **kwargs) -> bytes:
"""Convert protobuf message to encoded text format. See MessageToString."""
text = MessageToString(message, **kwargs)
if isinstance(text, bytes):

@ -65,7 +65,7 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex,
(*variables)["type"] =
name_resolver->GetImmutableClassName(descriptor->enum_type());
(*variables)["kt_type"] = (*variables)["type"];
(*variables)["kt_type"] = EscapeKotlinKeywords((*variables)["type"]);
(*variables)["mutable_type"] =
name_resolver->GetMutableClassName(descriptor->enum_type());
(*variables)["default"] = ImmutableDefaultValue(descriptor, name_resolver);

@ -73,7 +73,7 @@ void SetEnumVariables(const FieldDescriptor* descriptor, int messageBitIndex,
(*variables)["type"] =
name_resolver->GetImmutableClassName(descriptor->enum_type());
(*variables)["kt_type"] = (*variables)["type"];
(*variables)["kt_type"] = EscapeKotlinKeywords((*variables)["type"]);
(*variables)["mutable_type"] =
name_resolver->GetMutableClassName(descriptor->enum_type());
(*variables)["default"] = ImmutableDefaultValue(descriptor, name_resolver);

@ -712,7 +712,7 @@ void FileGenerator::GenerateKotlinSiblings(
printer.Print(
"package $package$;\n"
"\n",
"package", java_package_);
"package", EscapeKotlinKeywords(java_package_));
}
generator->GenerateKotlinMembers(&printer);

@ -279,6 +279,19 @@ bool IsForbiddenKotlin(const std::string& field_name) {
kKotlinForbiddenNames->end();
}
std::string EscapeKotlinKeywords(std::string name) {
std::vector<std::string> escaped_packages;
std::vector<std::string> packages = Split(name, "."); // NOLINT
for (const std::string& package : packages) {
if (IsForbiddenKotlin(package)) {
escaped_packages.push_back("`" + package + "`");
} else {
escaped_packages.push_back(package);
}
}
return Join(escaped_packages, ".");
}
std::string UniqueFileScopeIdentifier(const Descriptor* descriptor) {
return "static_" + StringReplace(descriptor->full_name(), ".", "_", true);
}

@ -118,6 +118,9 @@ std::string FileJavaPackage(const FileDescriptor* file, bool immutable);
// Returns output directory for the given package name.
std::string JavaPackageToDir(std::string package_name);
// Returns the name with Kotlin keywords enclosed in backticks
std::string EscapeKotlinKeywords(std::string name);
// Comma-separate list of option-specified interfaces implemented by the
// Message, to follow the "implements" declaration of the Message definition.
std::string ExtraMessageInterfaces(const Descriptor* descriptor);

@ -1405,7 +1405,8 @@ void ImmutableMessageGenerator::GenerateKotlinDsl(io::Printer* printer) const {
" @kotlin.jvm.JvmSynthetic\n"
" @kotlin.PublishedApi\n"
" internal fun _build(): $message$ = _builder.build()\n",
"message", name_resolver_->GetClassName(descriptor_, true));
"message",
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)));
printer->Indent();
@ -1426,7 +1427,7 @@ void ImmutableMessageGenerator::GenerateKotlinDsl(io::Printer* printer) const {
"oneof_name", context_->GetOneofGeneratorInfo(oneof)->name,
"oneof_capitalized_name",
context_->GetOneofGeneratorInfo(oneof)->capitalized_name, "message",
name_resolver_->GetClassName(descriptor_, true));
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)));
}
if (descriptor_->extension_range_count() > 0) {
@ -1448,8 +1449,11 @@ void ImmutableMessageGenerator::GenerateKotlinMembers(
" $message_kt$.Dsl._create($message$.newBuilder()).apply { block() "
"}._build()\n",
"camelcase_name", name_resolver_->GetKotlinFactoryName(descriptor_),
"message_kt", name_resolver_->GetKotlinExtensionsClassName(descriptor_),
"message", name_resolver_->GetClassName(descriptor_, true));
"message_kt",
EscapeKotlinKeywords(
name_resolver_->GetKotlinExtensionsClassName(descriptor_)),
"message",
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)));
WriteMessageDocComment(printer, descriptor_, /* kdoc */ true);
printer->Print("object $name$Kt {\n", "name", descriptor_->name());
@ -1469,12 +1473,14 @@ void ImmutableMessageGenerator::GenerateTopLevelKotlinMembers(
printer->Print(
"@kotlin.jvm.JvmSynthetic\n"
"inline fun $message$.copy(block: $message_kt$.Dsl.() -> "
"kotlin.Unit): "
"$message$ =\n"
"kotlin.Unit): $message$ =\n"
" $message_kt$.Dsl._create(this.toBuilder()).apply { block() "
"}._build()\n\n",
"message", name_resolver_->GetClassName(descriptor_, true), "message_kt",
name_resolver_->GetKotlinExtensionsClassName(descriptor_));
"message",
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)),
"message_kt",
EscapeKotlinKeywords(
name_resolver_->GetKotlinExtensionsClassName(descriptor_)));
for (int i = 0; i < descriptor_->nested_type_count(); i++) {
if (IsMapEntry(descriptor_->nested_type(i))) continue;
@ -1492,18 +1498,21 @@ void ImmutableMessageGenerator::GenerateKotlinOrNull(io::Printer* printer) const
printer->Print(
"val $full_classname$OrBuilder.$camelcase_name$OrNull: $full_name$?\n"
" get() = if (has$name$()) get$name$() else null\n\n",
"full_classname", name_resolver_->GetClassName(descriptor_, true),
"full_classname",
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)),
"camelcase_name", context_->GetFieldGeneratorInfo(field)->name,
"full_name",
name_resolver_->GetImmutableClassName(field->message_type()), "name",
context_->GetFieldGeneratorInfo(field)->capitalized_name);
EscapeKotlinKeywords(
name_resolver_->GetImmutableClassName(field->message_type())),
"name", context_->GetFieldGeneratorInfo(field)->capitalized_name);
}
}
}
void ImmutableMessageGenerator::GenerateKotlinExtensions(
io::Printer* printer) const {
std::string message_name = name_resolver_->GetClassName(descriptor_, true);
std::string message_name =
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true));
printer->Print(
"@Suppress(\"UNCHECKED_CAST\")\n"

@ -64,7 +64,7 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex,
(*variables)["type"] =
name_resolver->GetImmutableClassName(descriptor->message_type());
(*variables)["kt_type"] = (*variables)["type"];
(*variables)["kt_type"] = EscapeKotlinKeywords((*variables)["type"]);
(*variables)["mutable_type"] =
name_resolver->GetMutableClassName(descriptor->message_type());
(*variables)["group_or_message"] =

@ -64,7 +64,7 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex,
(*variables)["type"] =
name_resolver->GetImmutableClassName(descriptor->message_type());
(*variables)["kt_type"] = (*variables)["type"];
(*variables)["kt_type"] = EscapeKotlinKeywords((*variables)["type"]);
(*variables)["mutable_type"] =
name_resolver->GetMutableClassName(descriptor->message_type());
(*variables)["group_or_message"] =

@ -747,7 +747,8 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl(
" @kotlin.jvm.JvmSynthetic\n"
" @kotlin.PublishedApi\n"
" internal fun _build(): $message$ = _builder.build()\n",
"message", name_resolver_->GetClassName(descriptor_, true));
"message",
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)));
printer->Indent();
@ -768,7 +769,7 @@ void ImmutableMessageLiteGenerator::GenerateKotlinDsl(
"oneof_name", context_->GetOneofGeneratorInfo(oneof)->name,
"oneof_capitalized_name",
context_->GetOneofGeneratorInfo(oneof)->capitalized_name, "message",
name_resolver_->GetClassName(descriptor_, true));
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)));
}
if (descriptor_->extension_range_count() > 0) {
@ -784,13 +785,15 @@ void ImmutableMessageLiteGenerator::GenerateKotlinMembers(
printer->Print(
"@kotlin.jvm.JvmName(\"-initialize$camelcase_name$\")\n"
"inline fun $camelcase_name$(block: $message_kt$.Dsl.() -> "
"kotlin.Unit): "
"$message$ =\n"
"kotlin.Unit): $message$ =\n"
" $message_kt$.Dsl._create($message$.newBuilder()).apply { block() "
"}._build()\n",
"camelcase_name", name_resolver_->GetKotlinFactoryName(descriptor_),
"message_kt", name_resolver_->GetKotlinExtensionsClassName(descriptor_),
"message", name_resolver_->GetClassName(descriptor_, true));
"message_kt",
EscapeKotlinKeywords(
name_resolver_->GetKotlinExtensionsClassName(descriptor_)),
"message",
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)));
WriteMessageDocComment(printer, descriptor_, /* kdoc */ true);
printer->Print("object $name$Kt {\n", "name", descriptor_->name());
@ -809,12 +812,14 @@ void ImmutableMessageLiteGenerator::GenerateTopLevelKotlinMembers(
io::Printer* printer) const {
printer->Print(
"inline fun $message$.copy(block: $message_kt$.Dsl.() -> "
"kotlin.Unit): "
"$message$ =\n"
"kotlin.Unit): $message$ =\n"
" $message_kt$.Dsl._create(this.toBuilder()).apply { block() "
"}._build()\n\n",
"message", name_resolver_->GetClassName(descriptor_, true), "message_kt",
name_resolver_->GetKotlinExtensionsClassName(descriptor_));
"message",
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)),
"message_kt",
EscapeKotlinKeywords(
name_resolver_->GetKotlinExtensionsClassName(descriptor_)));
for (int i = 0; i < descriptor_->nested_type_count(); i++) {
if (IsMapEntry(descriptor_->nested_type(i))) continue;
@ -834,18 +839,21 @@ void ImmutableMessageLiteGenerator::GenerateKotlinOrNull(io::Printer* printer) c
"val $full_classname$OrBuilder.$camelcase_name$OrNull: "
"$full_name$?\n"
" get() = if (has$name$()) get$name$() else null\n\n",
"full_classname", name_resolver_->GetClassName(descriptor_, true),
"full_classname",
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true)),
"camelcase_name", context_->GetFieldGeneratorInfo(field)->name,
"full_name",
name_resolver_->GetImmutableClassName(field->message_type()), "name",
context_->GetFieldGeneratorInfo(field)->capitalized_name);
EscapeKotlinKeywords(
name_resolver_->GetImmutableClassName(field->message_type())),
"name", context_->GetFieldGeneratorInfo(field)->capitalized_name);
}
}
}
void ImmutableMessageLiteGenerator::GenerateKotlinExtensions(
io::Printer* printer) const {
std::string message_name = name_resolver_->GetClassName(descriptor_, true);
std::string message_name =
EscapeKotlinKeywords(name_resolver_->GetClassName(descriptor_, true));
printer->Print(
"@Suppress(\"UNCHECKED_CAST\")\n"

@ -28,6 +28,8 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <algorithm>
#include <limits>
#include <map>
#include <string>
@ -35,12 +37,22 @@
#include <google/protobuf/compiler/objectivec/objectivec_helpers.h>
#include <google/protobuf/io/printer.h>
#include <google/protobuf/stubs/strutil.h>
#include <algorithm> // std::find()
namespace google {
namespace protobuf {
namespace compiler {
namespace objectivec {
namespace {
std::string SafelyPrintIntToCode(int v) {
if (v == std::numeric_limits<int>::min()) {
// Some compilers try to parse -2147483648 as two tokens and then get spicy
// about the fact that +2147483648 cannot be represented as an int.
return "-2147483647 - 1";
} else {
return StrCat(v);
}
}
} // namespace
EnumGenerator::EnumGenerator(const EnumDescriptor* descriptor)
: descriptor_(descriptor),
@ -141,7 +153,7 @@ void EnumGenerator::GenerateHeader(io::Printer* printer) {
"$name$$deprecated_attribute$ = $value$,\n",
"name", EnumValueName(all_values_[i]),
"deprecated_attribute", GetOptionalDeprecatedAttribute(all_values_[i]),
"value", StrCat(all_values_[i]->number()));
"value", SafelyPrintIntToCode(all_values_[i]->number()));
}
printer->Outdent();
printer->Print(

@ -1567,9 +1567,10 @@ std::string StrCat(const AlphaNum &a, const AlphaNum &b, const AlphaNum &c,
// the string we're appending to. However the results of this are random.
// Therefore, check for this in debug mode. Use unsigned math so we only have
// to do one comparison.
#define GOOGLE_DCHECK_NO_OVERLAP(dest, src) \
GOOGLE_DCHECK_GT(uintptr_t((src).data() - (dest).data()), \
uintptr_t((dest).size()))
#define GOOGLE_DCHECK_NO_OVERLAP(dest, src) \
GOOGLE_DCHECK(((src).size() == 0) || \
uintptr_t((src).data() - (dest).data()) > \
uintptr_t((dest).size()))
void StrAppend(std::string *result, const AlphaNum &a) {
GOOGLE_DCHECK_NO_OVERLAP(*result, a);

@ -49,6 +49,7 @@
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <google/protobuf/util/internal/testdata/maps.pb.h>
#include <google/protobuf/util/json_format.pb.h>
#include <google/protobuf/util/json_format.pb.h>
#include <google/protobuf/util/json_format_proto3.pb.h>
#include <google/protobuf/util/json_format_proto3.pb.h>
#include <google/protobuf/util/type_resolver.h>
@ -94,25 +95,72 @@ MATCHER_P(StatusIs, status,
#define EXPECT_OK(x) EXPECT_THAT(x, StatusIs(util::StatusCode::kOk))
#define ASSERT_OK(x) ASSERT_THAT(x, StatusIs(util::StatusCode::kOk))
// As functions defined in json_util.h are just thin wrappers around the
// JSON conversion code in //net/proto2/util/converter, in this test we
// only cover some very basic cases to make sure the wrappers have forwarded
// parameters to the underlying implementation correctly. More detailed
// tests are contained in the //net/proto2/util/converter directory.
util::StatusOr<std::string> ToJson(const Message& message,
const JsonPrintOptions& options = {}) {
std::string result;
RETURN_IF_ERROR(MessageToJsonString(message, &result, options));
return result;
}
enum class Codec {
kReflective,
kResolver,
};
util::Status FromJson(StringPiece json, Message* message,
const JsonParseOptions& options = {}) {
return JsonStringToMessage(json, message, options);
}
class JsonTest : public testing::TestWithParam<Codec> {
protected:
util::StatusOr<std::string> ToJson(const Message& proto,
JsonPrintOptions options = {}) {
if (GetParam() == Codec::kReflective) {
std::string result;
RETURN_IF_ERROR(MessageToJsonString(proto, &result, options));
return result;
}
std::string proto_data = proto.SerializeAsString();
io::ArrayInputStream in(proto_data.data(), proto_data.size());
std::string result;
io::StringOutputStream out(&result);
RETURN_IF_ERROR(BinaryToJsonStream(
resolver_.get(),
StrCat("type.googleapis.com/", proto.GetTypeName()), &in, &out,
options));
return result;
}
// The out parameter comes first since `json` tends to be a very long string,
// and clang-format does a poor job if it is not the last parameter.
util::Status ToProto(Message& proto, StringPiece json,
JsonParseOptions options = {}) {
if (GetParam() == Codec::kReflective) {
return JsonStringToMessage(json, &proto, options);
}
io::ArrayInputStream in(json.data(), json.size());
std::string result;
io::StringOutputStream out(&result);
RETURN_IF_ERROR(JsonToBinaryStream(
resolver_.get(),
StrCat("type.googleapis.com/", proto.GetTypeName()), &in, &out,
options));
if (!proto.ParseFromString(result)) {
return util::InternalError("wire format parse failed");
}
return util::OkStatus();
}
TEST(JsonUtilTest, TestWhitespaces) {
template <typename Proto>
util::StatusOr<Proto> ToProto(StringPiece json,
JsonParseOptions options = {}) {
Proto proto;
RETURN_IF_ERROR(JsonStringToMessage(json, &proto, options));
return proto;
}
std::unique_ptr<TypeResolver> resolver_{NewTypeResolverForDescriptorPool(
"type.googleapis.com", DescriptorPool::generated_pool())};
};
INSTANTIATE_TEST_SUITE_P(JsonTestSuite, JsonTest,
testing::Values(Codec::kReflective, Codec::kResolver));
TEST_P(JsonTest, TestWhitespaces) {
TestMessage m;
m.mutable_message_value();
@ -125,7 +173,7 @@ TEST(JsonUtilTest, TestWhitespaces) {
"}\n"));
}
TEST(JsonUtilTest, TestDefaultValues) {
TEST_P(JsonTest, TestDefaultValues) {
TestMessage m;
EXPECT_THAT(ToJson(m), IsOkAndHolds("{}"));
@ -211,7 +259,11 @@ TEST(JsonUtilTest, TestDefaultValues) {
"}"));
}
TEST(JsonUtilTest, TestPreserveProtoFieldNames) {
TEST_P(JsonTest, TestPreserveProtoFieldNames) {
if (GetParam() == Codec::kResolver) {
GTEST_SKIP();
}
TestMessage m;
m.mutable_message_value();
@ -220,7 +272,7 @@ TEST(JsonUtilTest, TestPreserveProtoFieldNames) {
EXPECT_THAT(ToJson(m, options), IsOkAndHolds("{\"message_value\":{}}"));
}
TEST(JsonUtilTest, TestAlwaysPrintEnumsAsInts) {
TEST_P(JsonTest, TestAlwaysPrintEnumsAsInts) {
TestMessage orig;
orig.set_enum_value(proto3::BAR);
orig.add_repeated_enum_value(proto3::FOO);
@ -233,16 +285,16 @@ TEST(JsonUtilTest, TestAlwaysPrintEnumsAsInts) {
ASSERT_THAT(printed,
IsOkAndHolds("{\"enumValue\":1,\"repeatedEnumValue\":[0,1]}"));
TestMessage parsed;
ASSERT_OK(FromJson(*printed, &parsed));
auto parsed = ToProto<TestMessage>(*printed);
ASSERT_OK(parsed);
EXPECT_EQ(parsed.enum_value(), proto3::BAR);
EXPECT_EQ(parsed.repeated_enum_value_size(), 2);
EXPECT_EQ(parsed.repeated_enum_value(0), proto3::FOO);
EXPECT_EQ(parsed.repeated_enum_value(1), proto3::BAR);
EXPECT_EQ(parsed->enum_value(), proto3::BAR);
EXPECT_EQ(parsed->repeated_enum_value_size(), 2);
EXPECT_EQ(parsed->repeated_enum_value(0), proto3::FOO);
EXPECT_EQ(parsed->repeated_enum_value(1), proto3::BAR);
}
TEST(JsonUtilTest, TestPrintEnumsAsIntsWithDefaultValue) {
TEST_P(JsonTest, TestPrintEnumsAsIntsWithDefaultValue) {
TestEnumValue orig;
// orig.set_enum_value1(proto3::FOO)
orig.set_enum_value2(proto3::FOO);
@ -257,122 +309,164 @@ TEST(JsonUtilTest, TestPrintEnumsAsIntsWithDefaultValue) {
printed,
IsOkAndHolds("{\"enumValue1\":0,\"enumValue2\":0,\"enumValue3\":1}"));
TestEnumValue parsed;
ASSERT_OK(FromJson(*printed, &parsed));
auto parsed = ToProto<TestEnumValue>(*printed);
EXPECT_EQ(parsed.enum_value1(), proto3::FOO);
EXPECT_EQ(parsed.enum_value2(), proto3::FOO);
EXPECT_EQ(parsed.enum_value3(), proto3::BAR);
EXPECT_EQ(parsed->enum_value1(), proto3::FOO);
EXPECT_EQ(parsed->enum_value2(), proto3::FOO);
EXPECT_EQ(parsed->enum_value3(), proto3::BAR);
}
TEST(JsonUtilTest, TestPrintProto2EnumAsIntWithDefaultValue) {
TEST_P(JsonTest, DISABLED_TestPrintProto2EnumAsIntWithDefaultValue) {
protobuf_unittest::TestDefaultEnumValue orig;
JsonPrintOptions print_options;
// use enum as int
print_options.always_print_enums_as_ints = true;
print_options.always_print_primitive_fields = true;
// result should be int rather than string
auto printed = ToJson(orig, print_options);
ASSERT_THAT(printed, IsOkAndHolds("{\"enumValue\":2}"));
protobuf_unittest::TestDefaultEnumValue parsed;
ASSERT_OK(FromJson(*printed, &parsed));
auto parsed = ToProto<protobuf_unittest::TestDefaultEnumValue>(*printed);
ASSERT_OK(parsed);
EXPECT_EQ(parsed.enum_value(), protobuf_unittest::DEFAULT);
EXPECT_EQ(parsed->enum_value(), protobuf_unittest::DEFAULT);
}
TEST(JsonUtilTest, ParseMessage) {
// Some random message but good enough to verify that the parsing wrapper
// functions are working properly.
TestMessage m;
ASSERT_OK(FromJson(R"json(
TEST_P(JsonTest, ParseMessage) {
auto m = ToProto<TestMessage>(R"json(
{
"boolValue": true,
"int32Value": 1234567891,
"int64Value": 5302428716536692736,
"floatValue": 3.40282002e+38,
"repeatedInt32Value": [1, 2],
"int64Value": -5302428716536692736,
"uint32Value": 42,
"uint64Value": 530242871653669,
"floatValue": 3.4e+38,
"doubleValue": -55.5,
"stringValue": "foo bar baz",
"enumValue": "BAR",
"messageValue": {
"value": 2048
},
"repeatedBoolValue": [true],
"repeatedInt32Value": [0, -42],
"repeatedUint64Value": [1, 2],
"repeatedDoubleValue": [1.5, -2],
"repeatedStringValue": ["foo", "bar ", ""],
"repeatedEnumValue": [1, "FOO"],
"repeatedMessageValue": [
{"value": 40},
{"value": 96}
]
}
)json",
&m));
EXPECT_EQ(m.int32_value(), 1234567891);
EXPECT_EQ(m.int64_value(), 5302428716536692736);
EXPECT_EQ(m.float_value(), 3.40282002e+38f);
ASSERT_EQ(m.repeated_int32_value_size(), 2);
EXPECT_EQ(m.repeated_int32_value(0), 1);
EXPECT_EQ(m.repeated_int32_value(1), 2);
EXPECT_EQ(m.message_value().value(), 2048);
ASSERT_EQ(m.repeated_message_value_size(), 2);
EXPECT_EQ(m.repeated_message_value(0).value(), 40);
EXPECT_EQ(m.repeated_message_value(1).value(), 96);
)json");
ASSERT_OK(m);
EXPECT_TRUE(m->bool_value());
EXPECT_EQ(m->int32_value(), 1234567891);
EXPECT_EQ(m->int64_value(), -5302428716536692736);
EXPECT_EQ(m->uint32_value(), 42);
EXPECT_EQ(m->uint64_value(), 530242871653669);
EXPECT_EQ(m->float_value(), 3.4e+38f);
EXPECT_EQ(m->double_value(), -55.5);
EXPECT_EQ(m->string_value(), "foo bar baz");
EXPECT_EQ(m->enum_value(), proto3::EnumType::BAR);
EXPECT_EQ(m->message_value().value(), 2048);
ASSERT_EQ(m->repeated_bool_value_size(), 1);
EXPECT_TRUE(m->repeated_bool_value(0));
ASSERT_EQ(m->repeated_int32_value_size(), 2);
EXPECT_EQ(m->repeated_int32_value(0), 0);
EXPECT_EQ(m->repeated_int32_value(1), -42);
ASSERT_EQ(m->repeated_uint64_value_size(), 2);
EXPECT_EQ(m->repeated_uint64_value(0), 1);
EXPECT_EQ(m->repeated_uint64_value(1), 2);
ASSERT_EQ(m->repeated_double_value_size(), 2);
EXPECT_EQ(m->repeated_double_value(0), 1.5);
EXPECT_EQ(m->repeated_double_value(1), -2);
ASSERT_EQ(m->repeated_string_value_size(), 3);
EXPECT_EQ(m->repeated_string_value(0), "foo");
EXPECT_EQ(m->repeated_string_value(1), "bar ");
EXPECT_EQ(m->repeated_string_value(2), "");
ASSERT_EQ(m->repeated_enum_value_size(), 2);
EXPECT_EQ(m->repeated_enum_value(0), proto3::EnumType::BAR);
EXPECT_EQ(m->repeated_enum_value(1), proto3::EnumType::FOO);
ASSERT_EQ(m->repeated_message_value_size(), 2);
EXPECT_EQ(m->repeated_message_value(0).value(), 40);
EXPECT_EQ(m->repeated_message_value(1).value(), 96);
EXPECT_THAT(
ToJson(*m),
IsOkAndHolds(
R"({"boolValue":true,"int32Value":1234567891,"int64Value":"-5302428716536692736",)"
R"("uint32Value":42,"uint64Value":"530242871653669","floatValue":3.4e+38,)"
R"("doubleValue":-55.5,"stringValue":"foo bar baz","enumValue":"BAR",)"
R"("messageValue":{"value":2048},"repeatedBoolValue":[true],"repeatedInt32Value":[0,-42])"
R"(,"repeatedUint64Value":["1","2"],"repeatedDoubleValue":[1.5,-2],)"
R"("repeatedStringValue":["foo","bar ",""],"repeatedEnumValue":["BAR","FOO"],)"
R"("repeatedMessageValue":[{"value":40},{"value":96}]})"));
}
TEST(JsonUtilTest, ParseMap) {
TEST_P(JsonTest, ParseMap) {
TestMap message;
(*message.mutable_string_map())["hello"] = 1234;
auto printed = ToJson(message);
ASSERT_THAT(printed, IsOkAndHolds("{\"stringMap\":{\"hello\":1234}}"));
ASSERT_THAT(printed, IsOkAndHolds(R"({"stringMap":{"hello":1234}})"));
TestMap other;
ASSERT_OK(FromJson(*printed, &other));
EXPECT_EQ(other.DebugString(), message.DebugString());
auto other = ToProto<TestMap>(*printed);
ASSERT_OK(other);
EXPECT_EQ(other->DebugString(), message.DebugString());
}
TEST(JsonUtilTest, ParsePrimitiveMapIn) {
TEST_P(JsonTest, ParsePrimitiveMapIn) {
MapIn message;
JsonPrintOptions print_options;
print_options.always_print_primitive_fields = true;
auto printed = ToJson(message, print_options);
ASSERT_THAT(
ToJson(message, print_options),
IsOkAndHolds(
"{\"other\":\"\",\"things\":[],\"mapInput\":{},\"mapAny\":{}}"));
IsOkAndHolds(R"({"other":"","things":[],"mapInput":{},"mapAny":{}})"));
MapIn other;
ASSERT_OK(FromJson(*printed, &other));
EXPECT_EQ(other.DebugString(), message.DebugString());
auto other = ToProto<MapIn>(*printed);
ASSERT_OK(other);
EXPECT_EQ(other->DebugString(), message.DebugString());
}
TEST(JsonUtilTest, PrintPrimitiveOneof) {
TEST_P(JsonTest, PrintPrimitiveOneof) {
TestOneof message;
JsonPrintOptions options;
options.always_print_primitive_fields = true;
message.mutable_oneof_message_value();
EXPECT_THAT(ToJson(message, options),
IsOkAndHolds("{\"oneofMessageValue\":{\"value\":0}}"));
IsOkAndHolds(R"({"oneofMessageValue":{"value":0}})"));
message.set_oneof_int32_value(1);
EXPECT_THAT(ToJson(message, options),
IsOkAndHolds("{\"oneofInt32Value\":1}"));
IsOkAndHolds(R"({"oneofInt32Value":1})"));
}
TEST(JsonUtilTest, TestParseIgnoreUnknownFields) {
TestMessage m;
TEST_P(JsonTest, TestParseIgnoreUnknownFields) {
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_OK(FromJson("{\"unknownName\":0}", &m, options));
EXPECT_OK(ToProto<TestMessage>(R"({"unknownName":0})", options));
}
TEST(JsonUtilTest, TestParseErrors) {
TestMessage m;
TEST_P(JsonTest, TestParseErrors) {
// Parsing should fail if the field name can not be recognized.
EXPECT_THAT(FromJson(R"json({"unknownName": 0})json", &m),
EXPECT_THAT(ToProto<TestMessage>(R"({"unknownName": 0})"),
StatusIs(util::StatusCode::kInvalidArgument));
// Parsing should fail if the value is invalid.
EXPECT_THAT(FromJson(R"json("{"int32Value": 2147483648})json", &m),
EXPECT_THAT(ToProto<TestMessage>(R"("{"int32Value": 2147483648})"),
StatusIs(util::StatusCode::kInvalidArgument));
}
TEST(JsonUtilTest, TestDynamicMessage) {
TEST_P(JsonTest, TestDynamicMessage) {
// Create a new DescriptorPool with the same protos as the generated one.
DescriptorPoolDatabase database(*DescriptorPool::generated_pool());
DescriptorPool pool(&database);
@ -381,7 +475,7 @@ TEST(JsonUtilTest, TestDynamicMessage) {
std::unique_ptr<Message> message(
factory.GetPrototype(pool.FindMessageTypeByName("proto3.TestMessage"))
->New());
EXPECT_OK(FromJson(R"json(
ASSERT_OK(ToProto(*message, R"json(
{
"int32Value": 1024,
"repeatedInt32Value": [1, 2],
@ -393,12 +487,12 @@ TEST(JsonUtilTest, TestDynamicMessage) {
{"value": 96}
]
}
)json",
message.get()));
)json"));
// Convert to generated message for easy inspection.
TestMessage generated;
EXPECT_TRUE(generated.ParseFromString(message->SerializeAsString()));
EXPECT_EQ(generated.int32_value(), 1024);
ASSERT_EQ(generated.repeated_int32_value_size(), 2);
EXPECT_EQ(generated.repeated_int32_value(0), 1);
@ -415,8 +509,8 @@ TEST(JsonUtilTest, TestDynamicMessage) {
EXPECT_EQ(*message_json, *generated_json);
}
TEST(JsonUtilTest, TestParsingAny) {
StringPiece input = R"json(
TEST_P(JsonTest, TestParsingAny) {
auto m = ToProto<TestAny>(R"json(
{
"value": {
"@type": "type.googleapis.com/proto3.TestMessage",
@ -425,26 +519,24 @@ TEST(JsonUtilTest, TestParsingAny) {
"message_value": {"value": 1}
}
}
)json";
TestAny m;
ASSERT_OK(FromJson(input, &m));
)json");
ASSERT_OK(m);
TestMessage t;
EXPECT_TRUE(m.value().UnpackTo(&t));
ASSERT_TRUE(m->value().UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
EXPECT_THAT(
ToJson(m),
ToJson(*m),
IsOkAndHolds(
R"({"value":{"@type":"type.googleapis.com/proto3.TestMessage",)"
R"("int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}})"));
}
TEST(JsonUtilTest, TestParsingAnyMiddleAtType) {
StringPiece input = R"json(
TEST_P(JsonTest, TestParsingAnyMiddleAtType) {
auto m = ToProto<TestAny>(R"json(
{
"value": {
"int32_value": 5,
@ -453,20 +545,18 @@ TEST(JsonUtilTest, TestParsingAnyMiddleAtType) {
"message_value": {"value": 1}
}
}
)json";
TestAny m;
ASSERT_OK(FromJson(input, &m));
)json");
ASSERT_OK(m);
TestMessage t;
EXPECT_TRUE(m.value().UnpackTo(&t));
ASSERT_TRUE(m->value().UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
}
TEST(JsonUtilTest, TestParsingAnyEndAtType) {
StringPiece input = R"json(
TEST_P(JsonTest, TestParsingAnyEndAtType) {
auto m = ToProto<TestAny>(R"json(
{
"value": {
"int32_value": 5,
@ -475,20 +565,18 @@ TEST(JsonUtilTest, TestParsingAnyEndAtType) {
"@type": "type.googleapis.com/proto3.TestMessage"
}
}
)json";
TestAny m;
ASSERT_OK(FromJson(input, &m));
)json");
ASSERT_OK(m);
TestMessage t;
EXPECT_TRUE(m.value().UnpackTo(&t));
ASSERT_TRUE(m->value().UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
}
TEST(JsonUtilTest, TestParsingNestedAnys) {
StringPiece input = R"json(
TEST_P(JsonTest, TestParsingNestedAnys) {
auto m = ToProto<TestAny>(R"json(
{
"value": {
"value": {
@ -500,52 +588,47 @@ TEST(JsonUtilTest, TestParsingNestedAnys) {
"@type": "type.googleapis.com/google.protobuf.Any"
}
}
)json";
TestAny m;
ASSERT_OK(FromJson(input, &m));
)json");
ASSERT_OK(m);
google::protobuf::Any inner;
EXPECT_TRUE(m.value().UnpackTo(&inner));
ASSERT_TRUE(m->value().UnpackTo(&inner));
TestMessage t;
EXPECT_TRUE(inner.UnpackTo(&t));
ASSERT_TRUE(inner.UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
EXPECT_THAT(
ToJson(m),
ToJson(*m),
IsOkAndHolds(
R"({"value":{"@type":"type.googleapis.com/google.protobuf.Any",)"
R"("value":{"@type":"type.googleapis.com/proto3.TestMessage",)"
R"("int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}}})"));
}
TEST(JsonUtilTest, ParseWrappers) {
StringPiece input = R"json(
TEST_P(JsonTest, ParseWrappers) {
auto m = ToProto<TestWrapper>(R"json(
{
"boolValue": true,
"int32Value": 42,
"stringValue": "ieieo",
}
)json";
TestWrapper m;
auto proto_buffer = FromJson(input, &m);
ASSERT_OK(proto_buffer);
)json");
ASSERT_OK(m);
EXPECT_TRUE(m.bool_value().value());
EXPECT_EQ(m.int32_value().value(), 42);
EXPECT_EQ(m.string_value().value(), "ieieo");
EXPECT_TRUE(m->bool_value().value());
EXPECT_EQ(m->int32_value().value(), 42);
EXPECT_EQ(m->string_value().value(), "ieieo");
EXPECT_THAT(
ToJson(m),
ToJson(*m),
IsOkAndHolds(
R"({"boolValue":true,"int32Value":42,"stringValue":"ieieo"})"));
}
TEST(JsonUtilTest, TestParsingUnknownAnyFields) {
TEST_P(JsonTest, TestParsingUnknownAnyFields) {
StringPiece input = R"json(
{
"value": {
@ -556,321 +639,109 @@ TEST(JsonUtilTest, TestParsingUnknownAnyFields) {
}
)json";
TestAny m;
EXPECT_THAT(FromJson(input, &m),
EXPECT_THAT(ToProto<TestAny>(input),
StatusIs(util::StatusCode::kInvalidArgument));
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_OK(FromJson(input, &m, options));
auto m = ToProto<TestAny>(input, options);
ASSERT_OK(m);
TestMessage t;
EXPECT_TRUE(m.value().UnpackTo(&t));
ASSERT_TRUE(m->value().UnpackTo(&t));
EXPECT_EQ(t.string_value(), "expected_value");
}
TEST(JsonUtilTest, TestParsingUnknownEnumsProto2) {
StringPiece input = R"json({"a": "UNKNOWN_VALUE"})json";
protobuf_unittest::TestNumbers m;
JsonParseOptions options;
EXPECT_THAT(FromJson(input, &m, options),
TEST_P(JsonTest, TestParsingUnknownEnumsProto2) {
StringPiece input = R"json({"ayuLmao": "UNKNOWN_VALUE"})json";
EXPECT_THAT(ToProto<protobuf_unittest::TestNumbers>(input),
StatusIs(util::StatusCode::kInvalidArgument));
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_OK(FromJson(input, &m, options));
EXPECT_FALSE(m.has_a());
auto m = ToProto<protobuf_unittest::TestNumbers>(input, options);
ASSERT_OK(m);
EXPECT_FALSE(m->has_a());
}
TEST(JsonUtilTest, TestParsingUnknownEnumsProto3) {
TEST_P(JsonTest, TestParsingUnknownEnumsProto3) {
TestMessage m;
StringPiece input = R"json({"enum_value":"UNKNOWN_VALUE"})json";
m.set_enum_value(proto3::BAR);
EXPECT_THAT(FromJson(input, &m),
StatusIs(util::StatusCode::kInvalidArgument));
ASSERT_EQ(m.enum_value(), proto3::BAR); // Keep previous value
ASSERT_THAT(ToProto(m, input), StatusIs(util::StatusCode::kInvalidArgument));
EXPECT_EQ(m.enum_value(), proto3::BAR); // Keep previous value
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_OK(FromJson(input, &m, options));
ASSERT_OK(ToProto(m, input, options));
EXPECT_EQ(m.enum_value(), 0); // Unknown enum value must be decoded as 0
}
TEST(JsonUtilTest, TestParsingUnknownEnumsProto3FromInt) {
TEST_P(JsonTest, TestParsingUnknownEnumsProto3FromInt) {
TestMessage m;
StringPiece input = R"json({"enum_value":12345})json";
m.set_enum_value(proto3::BAR);
EXPECT_OK(FromJson(input, &m));
ASSERT_EQ(m.enum_value(), 12345);
ASSERT_OK(ToProto(m, input));
EXPECT_EQ(m.enum_value(), 12345);
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_OK(FromJson(input, &m, options));
ASSERT_OK(ToProto(m, input, options));
EXPECT_EQ(m.enum_value(), 12345);
}
// Trying to pass an object as an enum field value is always treated as an
// error
TEST(JsonUtilTest, TestParsingUnknownEnumsProto3FromObject) {
TestMessage m;
TEST_P(JsonTest, TestParsingUnknownEnumsProto3FromObject) {
StringPiece input = R"json({"enum_value": {}})json";
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_THAT(FromJson(input, &m, options),
EXPECT_THAT(ToProto<TestMessage>(input),
StatusIs(util::StatusCode::kInvalidArgument));
EXPECT_THAT(FromJson(input, &m),
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_THAT(ToProto<TestMessage>(input, options),
StatusIs(util::StatusCode::kInvalidArgument));
}
TEST(JsonUtilTest, TestParsingUnknownEnumsProto3FromArray) {
TestMessage m;
TEST_P(JsonTest, TestParsingUnknownEnumsProto3FromArray) {
StringPiece input = R"json({"enum_value": []})json";
EXPECT_THAT(FromJson(input, &m),
EXPECT_THAT(ToProto<TestMessage>(input),
StatusIs(util::StatusCode::kInvalidArgument));
JsonParseOptions options;
options.ignore_unknown_fields = true;
EXPECT_THAT(FromJson(input, &m, options),
EXPECT_THAT(ToProto<TestMessage>(input, options),
StatusIs(util::StatusCode::kInvalidArgument));
}
TEST(JsonUtilTest, TestParsingEnumCaseSensitive) {
TEST_P(JsonTest, TestParsingEnumCaseSensitive) {
TestMessage m;
StringPiece input = R"json({"enum_value": "bar"})json";
m.set_enum_value(proto3::FOO);
EXPECT_THAT(FromJson(input, &m),
EXPECT_THAT(ToProto(m, R"json({"enum_value": "bar"})json"),
StatusIs(util::StatusCode::kInvalidArgument));
// Default behavior is case-sensitive, so keep previous value.
ASSERT_EQ(m.enum_value(), proto3::FOO);
EXPECT_EQ(m.enum_value(), proto3::FOO);
}
TEST(JsonUtilTest, TestParsingEnumIgnoreCase) {
TEST_P(JsonTest, TestParsingEnumIgnoreCase) {
TestMessage m;
StringPiece input = R"json({"enum_value":"bar"})json";
m.set_enum_value(proto3::FOO);
JsonParseOptions options;
options.case_insensitive_enum_parsing = true;
EXPECT_OK(FromJson(input, &m, options));
ASSERT_EQ(m.enum_value(), proto3::BAR);
}
class TypeResolverTest : public testing::Test {
protected:
util::StatusOr<std::string> Proto2Json(StringPiece proto,
StringPiece type,
const JsonPrintOptions& options = {}) {
io::ArrayInputStream in(proto.data(), proto.size());
std::string result;
io::StringOutputStream out(&result);
RETURN_IF_ERROR(BinaryToJsonStream(
resolver_.get(), StrCat("type.googleapis.com/", type), &in,
&out));
return result;
}
util::StatusOr<std::string> Json2Proto(StringPiece json,
StringPiece type,
const JsonParseOptions& options = {}) {
io::ArrayInputStream in(json.data(), json.size());
std::string result;
io::StringOutputStream out(&result);
RETURN_IF_ERROR(JsonToBinaryStream(
resolver_.get(), StrCat("type.googleapis.com/", type), &in,
&out));
return result;
}
std::unique_ptr<TypeResolver> resolver_{NewTypeResolverForDescriptorPool(
"type.googleapis.com", DescriptorPool::generated_pool())};
};
TEST_F(TypeResolverTest, ParseFromResolver) {
StringPiece json = R"json(
{
"boolValue": true,
"int32Value": 1234567891,
"int64Value": -5302428716536692736,
"uint32Value": 42,
"uint64Value": 530242871653669,
"floatValue": 3.4e+38,
"doubleValue": -55.5,
"stringValue": "foo bar baz",
"enumValue": "BAR",
"messageValue": {
"value": 2048
},
"repeatedBoolValue": [true],
"repeatedInt32Value": [0, -42],
"repeatedUint64Value": [1, 2],
"repeatedDoubleValue": [1.5, -2.1],
"repeatedStringValue": ["foo", "bar ", ""],
"repeatedEnumValue": [1, "FOO"],
"repeatedMessageValue": [
{"value": 40},
{"value": 96}
]
}
)json";
auto proto_buffer = Json2Proto(json, "proto3.TestMessage");
ASSERT_OK(proto_buffer);
// Some random message but good enough to verify that the parsing wrapper
// functions are working properly.
TestMessage m;
ASSERT_TRUE(m.ParseFromString(*proto_buffer));
EXPECT_TRUE(m.bool_value());
EXPECT_EQ(m.int32_value(), 1234567891);
EXPECT_EQ(m.int64_value(), -5302428716536692736);
EXPECT_EQ(m.uint32_value(), 42);
EXPECT_EQ(m.uint64_value(), 530242871653669);
EXPECT_EQ(m.float_value(), 3.4e+38f);
EXPECT_EQ(m.double_value(), -55.5);
EXPECT_EQ(m.string_value(), "foo bar baz");
EXPECT_EQ(m.enum_value(), proto3::EnumType::BAR);
EXPECT_EQ(m.message_value().value(), 2048);
ASSERT_EQ(m.repeated_bool_value_size(), 1);
EXPECT_TRUE(m.repeated_bool_value(0));
ASSERT_EQ(m.repeated_int32_value_size(), 2);
EXPECT_EQ(m.repeated_int32_value(0), 0);
EXPECT_EQ(m.repeated_int32_value(1), -42);
ASSERT_EQ(m.repeated_uint64_value_size(), 2);
EXPECT_EQ(m.repeated_uint64_value(0), 1);
EXPECT_EQ(m.repeated_uint64_value(1), 2);
ASSERT_EQ(m.repeated_double_value_size(), 2);
EXPECT_EQ(m.repeated_double_value(0), 1.5);
EXPECT_EQ(m.repeated_double_value(1), -2.1);
ASSERT_EQ(m.repeated_string_value_size(), 3);
EXPECT_EQ(m.repeated_string_value(0), "foo");
EXPECT_EQ(m.repeated_string_value(1), "bar ");
EXPECT_EQ(m.repeated_string_value(2), "");
ASSERT_EQ(m.repeated_enum_value_size(), 2);
EXPECT_EQ(m.repeated_enum_value(0), proto3::EnumType::BAR);
EXPECT_EQ(m.repeated_enum_value(1), proto3::EnumType::FOO);
ASSERT_EQ(m.repeated_message_value_size(), 2);
EXPECT_EQ(m.repeated_message_value(0).value(), 40);
EXPECT_EQ(m.repeated_message_value(1).value(), 96);
StringPiece compacted_json =
R"({"boolValue":true,"int32Value":1234567891,"int64Value":"-5302428716536692736",)"
R"("uint32Value":42,"uint64Value":"530242871653669","floatValue":3.4e+38,)"
R"("doubleValue":-55.5,"stringValue":"foo bar baz","enumValue":"BAR",)"
R"("messageValue":{"value":2048},"repeatedBoolValue":[true],"repeatedInt32Value":[0,-42])"
R"(,"repeatedUint64Value":["1","2"],"repeatedDoubleValue":[1.5,-2.1],)"
R"("repeatedStringValue":["foo","bar ",""],"repeatedEnumValue":["BAR","FOO"],)"
R"("repeatedMessageValue":[{"value":40},{"value":96}]})";
EXPECT_THAT(Proto2Json(*proto_buffer, "proto3.TestMessage"),
IsOkAndHolds(compacted_json));
// Proto3 messages always used packed, so this will make sure to exercise
// packed decoding.
std::string proto_buffer2;
m.SerializeToString(&proto_buffer2);
EXPECT_THAT(Proto2Json(proto_buffer2, "proto3.TestMessage"),
IsOkAndHolds(compacted_json));
ASSERT_OK(ToProto(m, R"json({"enum_value":"bar"})json", options));
EXPECT_EQ(m.enum_value(), proto3::BAR);
}
TEST_F(TypeResolverTest, ParseAnyFromResolver) {
StringPiece input = R"json(
{
"value": {
"@type": "type.googleapis.com/proto3.TestMessage",
"int32_value": 5,
"string_value": "expected_value",
"message_value": {"value": 1}
}
}
)json";
auto proto_buffer = Json2Proto(input, "proto3.TestAny");
ASSERT_OK(proto_buffer);
TestAny m;
ASSERT_TRUE(m.ParseFromString(*proto_buffer));
TestMessage t;
EXPECT_TRUE(m.value().UnpackTo(&t));
EXPECT_EQ(t.int32_value(), 5);
EXPECT_EQ(t.string_value(), "expected_value");
EXPECT_EQ(t.message_value().value(), 1);
EXPECT_THAT(
Proto2Json(*proto_buffer, "proto3.TestAny"),
IsOkAndHolds(
R"({"value":{"@type":"type.googleapis.com/proto3.TestMessage",)"
R"("int32Value":5,"stringValue":"expected_value","messageValue":{"value":1}}})"));
}
TEST_F(TypeResolverTest, ParseWrappers) {
StringPiece input = R"json(
{
"boolValue": true,
"int32Value": 42,
"stringValue": "ieieo",
}
)json";
auto proto_buffer = Json2Proto(input, "proto3.TestWrapper");
ASSERT_OK(proto_buffer);
TestWrapper m;
ASSERT_TRUE(m.ParseFromString(*proto_buffer));
EXPECT_TRUE(m.bool_value().value());
EXPECT_EQ(m.int32_value().value(), 42);
EXPECT_EQ(m.string_value().value(), "ieieo");
EXPECT_THAT(
Proto2Json(*proto_buffer, "proto3.TestWrapper"),
IsOkAndHolds(
R"({"boolValue":true,"int32Value":42,"stringValue":"ieieo"})"));
}
TEST_F(TypeResolverTest, RejectDuplicateOneof) {
StringPiece input = R"json(
{
"oneofInt32Value": 42,
"oneofStringValue": "bad",
}
)json";
EXPECT_THAT(Json2Proto(input, "proto3.TestOneof"),
StatusIs(util::StatusCode::kInvalidArgument));
}
TEST_F(TypeResolverTest, TestWrongJsonInput) {
EXPECT_THAT(Json2Proto(R"json({"unknown_field": "some_value"})json",
"proto3.TestMessage"),
StatusIs(util::StatusCode::kInvalidArgument));
}
TEST(JsonUtilTest, DISABLED_HtmlEscape) {
TEST_P(JsonTest, DISABLED_HtmlEscape) {
TestMessage m;
m.set_string_value("</script>");
JsonPrintOptions options;
EXPECT_THAT(ToJson(m, options),
EXPECT_THAT(ToJson(m),
IsOkAndHolds("{\"stringValue\":\"\\u003c/script\\u003e\"}"));
}

Loading…
Cancel
Save