Sync from Piper @396393195

PROTOBUF_SYNC_PIPER

BUILD

@@ -133,6 +133,7 @@ cc_library(
         "src/google/protobuf/extension_set.cc",
         "src/google/protobuf/generated_enum_util.cc",
         "src/google/protobuf/generated_message_table_driven_lite.cc",
+        "src/google/protobuf/generated_message_tctable_lite.cc",
         "src/google/protobuf/generated_message_util.cc",
         "src/google/protobuf/implicit_weak_message.cc",
         "src/google/protobuf/inlined_string_field.cc",
@@ -193,6 +194,7 @@ cc_library(
         "src/google/protobuf/generated_message_bases.cc",
         "src/google/protobuf/generated_message_reflection.cc",
         "src/google/protobuf/generated_message_table_driven.cc",
+        "src/google/protobuf/generated_message_tctable_full.cc",
         "src/google/protobuf/io/gzip_stream.cc",
         "src/google/protobuf/io/printer.cc",
         "src/google/protobuf/io/tokenizer.cc",
@@ -222,7 +224,6 @@ cc_library(
         "src/google/protobuf/util/internal/protostream_objectsource.cc",
         "src/google/protobuf/util/internal/protostream_objectwriter.cc",
         "src/google/protobuf/util/internal/type_info.cc",
-        "src/google/protobuf/util/internal/type_info_test_helper.cc",
         "src/google/protobuf/util/internal/utility.cc",
         "src/google/protobuf/util/json_util.cc",
         "src/google/protobuf/util/message_differencer.cc",
@@ -621,6 +622,8 @@ cc_proto_library(
 COMMON_TEST_SRCS = [
     # AUTOGEN(common_test_srcs)
     "src/google/protobuf/arena_test_util.cc",
+    "src/google/protobuf/map_lite_test_util.cc",
+    "src/google/protobuf/test_util_lite.cc",
     "src/google/protobuf/map_test_util.inc",
     "src/google/protobuf/reflection_tester.cc",
     "src/google/protobuf/test_util.cc",

conformance/conformance_nodejs.js

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 // Protocol Buffers - Google's data interchange format
 // Copyright 2008 Google Inc.  All rights reserved.
 // https://developers.google.com/protocol-buffers/
@@ -28,40 +29,6 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-#!/usr/bin/env node
-
-/*
- * Protocol Buffers - Google's data interchange format
- * Copyright 2008 Google Inc.  All rights reserved.
- * https://developers.google.com/protocol-buffers/
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
 var conformance = require('conformance_pb');
 var test_messages_proto3 = require('google/protobuf/test_messages_proto3_pb');
 var test_messages_proto2 = require('google/protobuf/test_messages_proto2_pb');

java/core/src/test/java/com/google/protobuf/MapForProto2LiteTest.java

@@ -32,6 +32,7 @@ package com.google.protobuf;
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
+import static org.junit.Assert.fail;
 
 import map_lite_test.MapForProto2TestProto.BizarroTestMap;
 import map_lite_test.MapForProto2TestProto.TestMap;
@@ -43,6 +44,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.TreeMap;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -803,4 +805,35 @@ public final class MapForProto2LiteTest {
       // expected
     }
   }
+
+  @Test
+  public void testPutAllWithNullStringValue() throws Exception {
+    TestMap.Builder sourceBuilder = TestMap.newBuilder();
+
+    // order preserving map used here to help test rollback
+    Map<Integer, String> data = new TreeMap<>();
+    data.put(7, "foo");
+    data.put(8, "bar");
+    data.put(9, null);
+    try {
+      sourceBuilder.putAllInt32ToStringField(data);
+      fail("allowed null string value");
+    } catch (NullPointerException expected) {
+      // Verify rollback of previously added values.
+      // They all go in or none do.
+      assertThat(sourceBuilder.getInt32ToStringFieldMap()).isEmpty();
+    }
+  }
+
+  @Test
+  public void testPutNullStringValue() throws Exception {
+    TestMap.Builder sourceBuilder = TestMap.newBuilder();
+
+    try {
+      sourceBuilder.putInt32ToStringField(8, null);
+      fail("allowed null string value");
+    } catch (NullPointerException expected) {
+    }
+  }
+
 }

java/core/src/test/java/com/google/protobuf/MapForProto2Test.java

@@ -32,6 +32,8 @@ package com.google.protobuf;
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
 
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import map_test.MapForProto2TestProto.BizarroTestMap;
@@ -51,6 +53,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -484,13 +487,6 @@ public class MapForProto2Test {
   public void testPutChecksNullKeysAndValues() throws Exception {
     TestMap.Builder builder = TestMap.newBuilder();
 
-    try {
-      builder.putInt32ToStringField(1, null);
-      assertWithMessage("expected exception").fail();
-    } catch (NullPointerException e) {
-      // expected.
-    }
-
     try {
       builder.putInt32ToBytesField(1, null);
       assertWithMessage("expected exception").fail();
@@ -1218,4 +1214,35 @@ public class MapForProto2Test {
     assertThat(message.getInt32ToEnumFieldMap()).isEqualTo(message.getInt32ToEnumFieldMap());
     assertThat(message.getInt32ToMessageFieldMap()).isEqualTo(message.getInt32ToMessageFieldMap());
   }
+
+  @Test
+  public void testPutAllWithNullStringValue() throws Exception {
+    TestMap.Builder sourceBuilder = TestMap.newBuilder();
+
+    // order preserving map used here to help test rollback
+    Map<Integer, String> data = new TreeMap<>();
+    data.put(7, "foo");
+    data.put(8, "bar");
+    data.put(9, null);
+    try {
+      sourceBuilder.putAllInt32ToStringField(data);
+      fail("allowed null string value");
+    } catch (NullPointerException expected) {
+      // Verify rollback of previously added values.
+      // They all go in or none do.
+      assertThat(sourceBuilder.getInt32ToStringFieldMap()).isEmpty();
+    }
+  }
+
+  @Test
+  public void testPutNullStringValue() throws Exception {
+    TestMap.Builder sourceBuilder = TestMap.newBuilder();
+
+    try {
+      sourceBuilder.putInt32ToStringField(8, null);
+      fail("allowed null string value");
+    } catch (NullPointerException expected) {
+      assertNotNull(expected.getMessage());
+    }
+  }
 }

java/core/src/test/java/com/google/protobuf/MapTest.java

@@ -32,6 +32,8 @@ package com.google.protobuf;
 
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
 
 import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.EnumDescriptor;
@@ -50,6 +52,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -464,6 +467,37 @@ public class MapTest {
     assertMapValuesSet(destination.build());
   }
 
+  @Test
+  public void testPutAllWithNullStringValue() throws Exception {
+    TestMap.Builder sourceBuilder = TestMap.newBuilder();
+
+    // order preserving map used here to help test rollback
+    Map<Integer, String> data = new TreeMap<>();
+    data.put(7, "foo");
+    data.put(8, "bar");
+    data.put(9, null);
+    try {
+      sourceBuilder.putAllInt32ToStringField(data);
+      fail("allowed null string value");
+    } catch (NullPointerException expected) {
+      // Verify rollback of previously added values.
+      // They all go in or none do.
+      assertThat(sourceBuilder.getInt32ToStringFieldMap()).isEmpty();
+    }
+  }
+
+  @Test
+  public void testPutNullStringValue() throws Exception {
+    TestMap.Builder sourceBuilder = TestMap.newBuilder();
+
+    try {
+      sourceBuilder.putInt32ToStringField(8, null);
+      fail("allowed null string value");
+    } catch (NullPointerException expected) {
+      assertNotNull(expected.getMessage());
+    }
+  }
+
   @Test
   public void testPutAllForUnknownEnumValues() throws Exception {
     TestMap source =

src/google/protobuf/compiler/java/java_map_field.cc

@@ -111,11 +111,13 @@ void SetMessageVariables(const FieldDescriptor* descriptor, int messageBitIndex,
   (*variables)["key_default_value"] = DefaultValue(key, true, name_resolver);
   (*variables)["key_null_check"] =
       IsReferenceType(keyJavaType)
-          ? "if (key == null) { throw new java.lang.NullPointerException(); }"
+          ? "if (key == null) { throw new NullPointerException(\"map key\"); }"
           : "";
   (*variables)["value_null_check"] =
-      IsReferenceType(valueJavaType)
+      valueJavaType != JAVATYPE_ENUM && IsReferenceType(valueJavaType)
-          ? "if (value == null) { throw new java.lang.NullPointerException(); }"
+          ? "if (value == null) {\n"
+            "  throw new NullPointerException(\"map value\");\n"
+            "}\n"
           : "";
   if (valueJavaType == JAVATYPE_ENUM) {
     // We store enums as Integers internally.
@@ -435,6 +437,7 @@ void ImmutableMapFieldGenerator::GenerateBuilderMembers(
           "    $key_type$ key,\n"
           "    $value_type$ value) {\n"
           "  $key_null_check$\n"
+          "  $value_null_check$\n"
           "  internalGetMutable$capitalized_name$().getMutableMap()\n"
           "      .put(key, value);\n"
           "  return this;\n"

update_file_lists.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -u
 
 # This script copies source file lists from src/Makefile.am to cmake files.
 
@@ -50,12 +50,12 @@ MAKEFILE=src/Makefile.am
 
 # Extract file lists from src/Makefile.am
 GZHEADERS=$(get_variable_value $MAKEFILE GZHEADERS)
-HEADERS=$(get_variable_value $MAKEFILE nobase_include_HEADERS)
+LIBPROTOBUF_HEADERS=$(get_variable_value $MAKEFILE nobase_include_HEADERS | grep -v /compiler/)
-PUBLIC_HEADERS=$(sort_files $GZHEADERS $HEADERS)
+LIBPROTOBUF_HEADERS=$(sort_files $GZHEADERS $LIBPROTOBUF_HEADERS)
 LIBPROTOBUF_LITE_SOURCES=$(get_source_files $MAKEFILE libprotobuf_lite_la_SOURCES)
 LIBPROTOBUF_SOURCES=$(get_source_files $MAKEFILE libprotobuf_la_SOURCES)
 LIBPROTOC_SOURCES=$(get_source_files $MAKEFILE libprotoc_la_SOURCES)
-LIBPROTOC_HEADERS=$(get_header_files $MAKEFILE libprotoc_la_SOURCES)
+LIBPROTOC_HEADERS=$(get_variable_value $MAKEFILE nobase_include_HEADERS | grep /compiler/)
 LITE_PROTOS=$(get_proto_files $MAKEFILE protoc_lite_outputs)
 PROTOS=$(get_proto_files $MAKEFILE protoc_outputs)
 PROTOS_BLACKLISTED=$(get_proto_files_blacklisted $MAKEFILE protoc_outputs)
@@ -98,7 +98,11 @@ set_cmake_value() {
       print \$0;
       len = split(values, vlist, \" \");
       for (i = 1; i <= len; ++i) {
-        printf(\"  %s%s\\n\", prefix, vlist[i]);
+        printf(\"  \");
+        if (vlist[i] !~ /^\\\$/) {
+          printf(\"%s\", prefix);
+        }
+        printf(\"%s\\n\", vlist[i]);
       }
       next;
     }
@@ -121,7 +125,7 @@ set_cmake_value $CMAKE_DIR/libprotoc.cmake libprotoc_files $CMAKE_PREFIX $LIBPRO
 set_cmake_value $CMAKE_DIR/libprotoc.cmake libprotoc_headers $CMAKE_PREFIX $LIBPROTOC_HEADERS
 set_cmake_value $CMAKE_DIR/tests.cmake lite_test_protos "" $LITE_PROTOS
 set_cmake_value $CMAKE_DIR/tests.cmake tests_protos "" $PROTOS_BLACKLISTED
-set_cmake_value $CMAKE_DIR/tests.cmake common_test_files $CMAKE_PREFIX $COMMON_TEST_SOURCES
+set_cmake_value $CMAKE_DIR/tests.cmake common_test_files $CMAKE_PREFIX '${common_lite_test_files}' $COMMON_TEST_SOURCES
 set_cmake_value $CMAKE_DIR/tests.cmake common_lite_test_files $CMAKE_PREFIX $COMMON_LITE_TEST_SOURCES
 set_cmake_value $CMAKE_DIR/tests.cmake tests_files $CMAKE_PREFIX $TEST_SOURCES
 set_cmake_value $CMAKE_DIR/tests.cmake non_msvc_tests_files $CMAKE_PREFIX $NON_MSVC_TEST_SOURCES
@@ -130,14 +134,14 @@ set_cmake_value $CMAKE_DIR/tests.cmake lite_arena_test_files $CMAKE_PREFIX $LITE
 
 # Generate extract_includes.bat
 echo "mkdir include" > $EXTRACT_INCLUDES_BAT
-for INCLUDE in $PUBLIC_HEADERS $WKT_PROTOS; do
+for INCLUDE in $LIBPROTOBUF_HEADERS $LIBPROTOC_HEADERS $WKT_PROTOS; do
   INCLUDE_DIR=$(dirname "$INCLUDE")
   while [ ! "$INCLUDE_DIR" = "." ]; do
     echo "mkdir include\\${INCLUDE_DIR//\//\\}"
     INCLUDE_DIR=$(dirname "$INCLUDE_DIR")
   done
 done | sort | uniq >> $EXTRACT_INCLUDES_BAT
-for INCLUDE in $PUBLIC_HEADERS $WKT_PROTOS; do
+for INCLUDE in $(sort_files $LIBPROTOBUF_HEADERS $LIBPROTOC_HEADERS) $WKT_PROTOS; do
   WINPATH=${INCLUDE//\//\\}
   echo "copy \"\${PROTOBUF_SOURCE_WIN32_PATH}\\..\\src\\$WINPATH\" include\\$WINPATH" >> $EXTRACT_INCLUDES_BAT
 done
@@ -186,7 +190,7 @@ if [ -f "$BAZEL_BUILD" ]; then
   set_bazel_value $BAZEL_BUILD lite_test_protos "" $LITE_PROTOS
   set_bazel_value $BAZEL_BUILD well_known_protos "" $WKT_PROTOS
   set_bazel_value $BAZEL_BUILD test_protos "" $PROTOS
-  set_bazel_value $BAZEL_BUILD common_test_srcs $BAZEL_PREFIX $COMMON_TEST_SOURCES
+  set_bazel_value $BAZEL_BUILD common_test_srcs $BAZEL_PREFIX $COMMON_LITE_TEST_SOURCES $COMMON_TEST_SOURCES
   set_bazel_value $BAZEL_BUILD test_srcs $BAZEL_PREFIX $TEST_SOURCES
   set_bazel_value $BAZEL_BUILD non_msvc_test_srcs $BAZEL_PREFIX $NON_MSVC_TEST_SOURCES
   set_bazel_value $BAZEL_BUILD test_plugin_srcs $BAZEL_PREFIX $TEST_PLUGIN_SOURCES