Merge branch 'main' into minitable-api

.github/workflows/bazel_tests.yml

@@ -1,4 +1,4 @@
-name: Check ClangFormat
+name: Bazel Tests
 
 on:
   push:
@@ -10,11 +10,28 @@ on:
   workflow_dispatch:
 
 jobs:
-  check_clang_format:
-    runs-on: ubuntu-20.04
+
+  ubuntu:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        include:
+          - { CC: clang, os: ubuntu-20.04, flags: "" }
+          - { CC: clang, os: ubuntu-20.04, flags: "-c opt" }  # Some warnings only fire with -c opt
+          - { CC: gcc, os: ubuntu-20.04, flags: "-c opt" }
+          - { CC: clang, os: ubuntu-20.04, flags: "--//:fasttable_enabled=true -- -cmake:test_generated_files" }
+          - { CC: clang, os: ubuntu-20.04, flags: "--config=asan -c dbg -- -benchmarks:benchmark -python/..." }
+          - { CC: clang, os: ubuntu-20.04, flags: "--config=ubsan -c dbg -- -benchmarks:benchmark -python/... -upb/bindings/lua/...", install: "libunwind-dev" }
+          - { CC: clang, os: macos-11, flags: "" }
+
     steps:
       - uses: actions/checkout@v2
-      - name: Run ClangFormat
-        run: find . | grep -E '\.(c|h|cc)$' | grep -E -v '^./(third_party|cmake)' | xargs clang-format -i
-      - name: Check for differences
-        run: git diff --exit-code
+      - name: Setup Python venv
+        run: rm -rf /tmp/venv && python3 -m venv /tmp/venv
+      - name: Install dependencies
+        run: sudo apt install -y ${{ matrix.install }}
+        if: matrix.install != ''
+      - name: Run tests
+        run: cd ${{ github.workspace }} && PATH=/tmp/venv/bin:$PATH CC=${{ matrix.CC }} bazel test --test_output=errors ... ${{ matrix.flags }}

.github/workflows/clang_format.yml

@@ -0,0 +1,20 @@
+name: Check ClangFormat
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  check_clang_format:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run ClangFormat
+        run: find . | grep -E '\.(c|h|cc)$' | grep -E -v '^./(third_party|cmake)' | xargs clang-format -i
+      - name: Check for differences
+        run: git diff --exit-code

BUILD

@@ -37,7 +37,6 @@ load(
     "upb_proto_library_copts",
     "upb_proto_reflection_library",
 )
-load("@rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test")
 
 licenses(["notice"])
 
@@ -346,15 +345,6 @@ cc_test(
     ],
 )
 
-cc_fuzz_test(
-    name = "file_descriptor_parsenew_fuzzer",
-    srcs = ["upb/file_descriptor_parsenew_fuzzer.cc"],
-    deps = [
-        "//:descriptor_upb_proto",
-        "//:upb",
-    ],
-)
-
 upb_proto_library(
     name = "conformance_proto_upb",
     testonly = 1,

WORKSPACE

@@ -6,6 +6,9 @@ load("//bazel:workspace_defs.bzl", "system_python")
 
 upb_deps()
 
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+protobuf_deps()
+
 http_archive(
     name = "lua",
     build_file = "//bazel:lua.BUILD",

bazel/workspace_deps.bzl

@@ -1,3 +1,4 @@
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
@@ -11,14 +12,10 @@ def upb_deps():
     )
 
     maybe(
-        http_archive,
+        git_repository,
         name = "com_google_protobuf",
-        sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422",
-        strip_prefix = "protobuf-3.19.1",
-        urls = [
-            "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz",
-            "https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz",
-        ],
+        commit = "2f91da585e96a7efe43505f714f03c7716a94ecb",
+        remote = "https://github.com/protocolbuffers/protobuf.git",
         patches = [
             "//bazel:protobuf.patch",
         ],
@@ -45,12 +42,3 @@ def upb_deps():
         strip_prefix = "bazel-skylib-main",
         urls = ["https://github.com/bazelbuild/bazel-skylib/archive/main.tar.gz"],
     )
-
-    maybe(
-        http_archive,
-        name = "zlib",
-        build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
-        sha256 = "629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff",
-        strip_prefix = "zlib-1.2.11",
-        url = "https://github.com/madler/zlib/archive/v1.2.11.tar.gz",
-    )

cmake/google/protobuf/descriptor.upb.c

@@ -309,20 +309,21 @@ static const upb_MiniTable_Sub google_protobuf_FieldOptions_submsgs[3] = {
   {.subenum = &google_protobuf_FieldOptions_JSType_enuminit},
 };
 
-static const upb_MiniTable_Field google_protobuf_FieldOptions__fields[7] = {
+static const upb_MiniTable_Field google_protobuf_FieldOptions__fields[8] = {
   {1, UPB_SIZE(4, 4), 1, 1, 14, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
   {2, UPB_SIZE(12, 12), 2, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
   {3, UPB_SIZE(13, 13), 3, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
   {5, UPB_SIZE(14, 14), 4, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
   {6, UPB_SIZE(8, 8), 5, 2, 14, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
   {10, UPB_SIZE(15, 15), 6, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
-  {999, UPB_SIZE(16, 16), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {15, UPB_SIZE(16, 16), 7, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {999, UPB_SIZE(20, 24), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
 };
 
 const upb_MiniTable google_protobuf_FieldOptions_msginit = {
   &google_protobuf_FieldOptions_submsgs[0],
   &google_protobuf_FieldOptions__fields[0],
-  UPB_SIZE(24, 24), 7, upb_ExtMode_Extendable, 3, 255, 0,
+  UPB_SIZE(24, 32), 8, upb_ExtMode_Extendable, 3, 255, 0,
 };
 
 static const upb_MiniTable_Sub google_protobuf_OneofOptions_submsgs[1] = {

cmake/google/protobuf/descriptor.upb.h

@@ -1638,8 +1638,12 @@ UPB_INLINE bool google_protobuf_FieldOptions_has_weak(const google_protobuf_Fiel
 UPB_INLINE bool google_protobuf_FieldOptions_weak(const google_protobuf_FieldOptions* msg) {
   return *UPB_PTR_AT(msg, UPB_SIZE(15, 15), bool);
 }
-UPB_INLINE bool google_protobuf_FieldOptions_has_uninterpreted_option(const google_protobuf_FieldOptions *msg) { return _upb_has_submsg_nohasbit(msg, UPB_SIZE(16, 16)); }
-UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_FieldOptions_uninterpreted_option(const google_protobuf_FieldOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(16, 16), len); }
+UPB_INLINE bool google_protobuf_FieldOptions_has_unverified_lazy(const google_protobuf_FieldOptions *msg) { return _upb_hasbit(msg, 7); }
+UPB_INLINE bool google_protobuf_FieldOptions_unverified_lazy(const google_protobuf_FieldOptions* msg) {
+  return *UPB_PTR_AT(msg, UPB_SIZE(16, 16), bool);
+}
+UPB_INLINE bool google_protobuf_FieldOptions_has_uninterpreted_option(const google_protobuf_FieldOptions *msg) { return _upb_has_submsg_nohasbit(msg, UPB_SIZE(20, 24)); }
+UPB_INLINE const google_protobuf_UninterpretedOption* const* google_protobuf_FieldOptions_uninterpreted_option(const google_protobuf_FieldOptions *msg, size_t *len) { return (const google_protobuf_UninterpretedOption* const*)_upb_array_accessor(msg, UPB_SIZE(20, 24), len); }
 
 UPB_INLINE void google_protobuf_FieldOptions_set_ctype(google_protobuf_FieldOptions *msg, int32_t value) {
   _upb_sethas(msg, 1);
@@ -1665,16 +1669,20 @@ UPB_INLINE void google_protobuf_FieldOptions_set_weak(google_protobuf_FieldOptio
   _upb_sethas(msg, 6);
   *UPB_PTR_AT(msg, UPB_SIZE(15, 15), bool) = value;
 }
+UPB_INLINE void google_protobuf_FieldOptions_set_unverified_lazy(google_protobuf_FieldOptions *msg, bool value) {
+  _upb_sethas(msg, 7);
+  *UPB_PTR_AT(msg, UPB_SIZE(16, 16), bool) = value;
+}
 UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FieldOptions_mutable_uninterpreted_option(google_protobuf_FieldOptions *msg, size_t *len) {
-  return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(16, 16), len);
+  return (google_protobuf_UninterpretedOption**)_upb_array_mutable_accessor(msg, UPB_SIZE(20, 24), len);
 }
 UPB_INLINE google_protobuf_UninterpretedOption** google_protobuf_FieldOptions_resize_uninterpreted_option(google_protobuf_FieldOptions *msg, size_t len, upb_Arena *arena) {
-  return (google_protobuf_UninterpretedOption**)_upb_Array_Resize_accessor2(msg, UPB_SIZE(16, 16), len, UPB_SIZE(2, 3), arena);
+  return (google_protobuf_UninterpretedOption**)_upb_Array_Resize_accessor2(msg, UPB_SIZE(20, 24), len, UPB_SIZE(2, 3), arena);
 }
 UPB_INLINE struct google_protobuf_UninterpretedOption* google_protobuf_FieldOptions_add_uninterpreted_option(google_protobuf_FieldOptions *msg, upb_Arena *arena) {
   struct google_protobuf_UninterpretedOption* sub = (struct google_protobuf_UninterpretedOption*)_upb_Message_New(&google_protobuf_UninterpretedOption_msginit, arena);
   bool ok = _upb_Array_Append_accessor2(
-      msg, UPB_SIZE(16, 16), UPB_SIZE(2, 3), &sub, arena);
+      msg, UPB_SIZE(20, 24), UPB_SIZE(2, 3), &sub, arena);
   if (!ok) return NULL;
   return sub;
 }

cmake/make_cmakelists.py

@@ -217,6 +217,9 @@ class WorkspaceFileFunctions(object):
   def upb_deps(self):
     pass
 
+  def protobuf_deps(self):
+    pass
+
   def rules_fuzzing_dependencies(self):
     pass
 

python/pb_unit_tests/descriptor_test_wrapper.py

@@ -40,5 +40,7 @@ descriptor_test.MakeDescriptorTest.testJsonName.__unittest_expecting_failure__ =
 # Our error message is better.
 descriptor_test.NewDescriptorTest.testImmutableCppDescriptor.__unittest_expecting_failure__ = True
 
+descriptor_test.DescriptorTest.testGetDebugString.__unittest_expecting_failure__ = True
+
 if __name__ == '__main__':
   unittest.main(module=descriptor_test, verbosity=2)

python/pb_unit_tests/message_test_wrapper.py

@@ -46,8 +46,12 @@ message_test.MessageTest.testFloatPrinting_proto3.__unittest_expecting_failure__
 message_test.Proto3Test.testCopyFromBadType.__unittest_expecting_failure__ = True
 message_test.Proto3Test.testMergeFromBadType.__unittest_expecting_failure__ = True
 
+message_test.MessageTest.testPickleRepeatedScalarContainer_proto2.__unittest_expecting_failure__ = True
+message_test.MessageTest.testPickleRepeatedScalarContainer_proto3.__unittest_expecting_failure__ = True
 message_test.Proto2Test.testPythonicInit.__unittest_expecting_failure__ = True
 message_test.Proto2Test.test_documentation.__unittest_expecting_failure__ = True
+message_test.Proto3Test.testModifyMapEntryWhileIterating.__unittest_expecting_failure__ = True
+message_test.Proto3Test.testProto3Optional.__unittest_expecting_failure__ = True
 
 if __name__ == '__main__':
   unittest.main(module=message_test, verbosity=2)

upb/decode.c

@@ -1066,7 +1066,7 @@ upb_DecodeStatus upb_Decode(const char* buf, size_t size, void* msg,
 
   if (size <= 16) {
     memset(&state.patch, 0, 32);
-    memcpy(&state.patch, buf, size);
+    if (size) memcpy(&state.patch, buf, size);
     buf = state.patch;
     state.end = buf + size;
     state.limit = 0;

upb/encode.c

@@ -556,12 +556,14 @@ static void encode_message(upb_encstate* e, const upb_Message* msg,
     }
   }
 
-  const upb_MiniTable_Field* f = &m->fields[m->field_count];
-  const upb_MiniTable_Field* first = &m->fields[0];
-  while (f != first) {
-    f--;
-    if (encode_shouldencode(e, msg, m->subs, f)) {
-      encode_field(e, msg, m->subs, f);
+  if (m->field_count) {
+    const upb_MiniTable_Field* f = &m->fields[m->field_count];
+    const upb_MiniTable_Field* first = &m->fields[0];
+    while (f != first) {
+      f--;
+      if (encode_shouldencode(e, msg, m->subs, f)) {
+        encode_field(e, msg, m->subs, f);
+      }
     }
   }
 

upb/fuzz/BUILD

@@ -0,0 +1,11 @@
+
+load("@rules_fuzzing//fuzzing:cc_defs.bzl", "cc_fuzz_test")
+
+cc_fuzz_test(
+    name = "file_descriptor_parsenew_fuzzer",
+    srcs = ["file_descriptor_parsenew_fuzzer.cc"],
+    deps = [
+        "//:descriptor_upb_proto",
+        "//:upb",
+    ],
+)

upb/util/def_to_proto.c

@@ -453,14 +453,14 @@ static google_protobuf_FileDescriptorProto* filedef_toproto(
       google_protobuf_FileDescriptorProto_resize_public_dependency(proto, n,
                                                                    ctx->arena);
   const int32_t* public_dep_nums = _upb_FileDef_PublicDependencyIndexes(f);
-  memcpy(public_deps, public_dep_nums, n * sizeof(int32_t));
+  if (n) memcpy(public_deps, public_dep_nums, n * sizeof(int32_t));
 
   n = upb_FileDef_WeakDependencyCount(f);
   int32_t* weak_deps =
       google_protobuf_FileDescriptorProto_resize_weak_dependency(proto, n,
                                                                  ctx->arena);
   const int32_t* weak_dep_nums = _upb_FileDef_WeakDependencyIndexes(f);
-  memcpy(weak_deps, weak_dep_nums, n * sizeof(int32_t));
+  if (n) memcpy(weak_deps, weak_dep_nums, n * sizeof(int32_t));
 
   n = upb_FileDef_TopLevelMessageCount(f);
   google_protobuf_DescriptorProto** msgs =

upb/util/required_fields.c

@@ -216,8 +216,10 @@ static void upb_util_FindUnsetInMessage(upb_FindContext* ctx,
         // Append the contents of the stack to the out array, then
         // NULL-terminate.
         upb_FieldPathVector_Reserve(ctx, &ctx->out_fields, ctx->stack.size + 2);
-        memcpy(&ctx->out_fields.path[ctx->out_fields.size], ctx->stack.path,
-               ctx->stack.size * sizeof(*ctx->stack.path));
+        if (ctx->stack.size) {
+          memcpy(&ctx->out_fields.path[ctx->out_fields.size], ctx->stack.path,
+                 ctx->stack.size * sizeof(*ctx->stack.path));
+        }
         ctx->out_fields.size += ctx->stack.size;
         ctx->out_fields.path[ctx->out_fields.size++] =
             (upb_FieldPathEntry){.field = f};