Merge branch 'main' into master

.bazelci/presubmit.yml

@@ -1,7 +1,9 @@
 ---
 tasks:
   ubuntu:
-    platform: ubuntu1604
+    platform: ubuntu1804
+    shell_commands:
+    - "sudo apt -y update && sudo apt -y install libreadline-dev cmake"
     test_targets:
     - //...
   macos:

.bazelignore

@@ -0,0 +1 @@
+_build

.bazelrc

@@ -0,0 +1,57 @@
+# temporary fix for https://github.com/bazelbuild/bazel/issues/12905 on macOS
+build --features=-debug_prefix_map_pwd_is_dot
+
+build --extra_toolchains=@system_python//:python_toolchain
+
+# Use our custom-configured c++ toolchain.
+
+build:m32 --copt=-m32 --linkopt=-m32
+build:asan --copt=-fsanitize=address --linkopt=-fsanitize=address
+
+# For Valgrind, we have to disable checks of "possible" leaks because the Python
+# interpreter does the sorts of things that flag Valgrind "possible" leak checks.
+# Ideally we could enforce a stricter check for the non-Python tests, but I don't
+# know of an easy way to do that.
+#
+# We also have to disable pymalloc to avoid triggering Valgrind.
+build:valgrind --run_under='valgrind --leak-check=full --track-origins=yes --trace-children=yes --show-leak-kinds=all --error-exitcode=1 --num-callers=500 ' --action_env=PYTHONMALLOC=malloc
+
+build:ubsan --copt=-fsanitize=undefined --linkopt=-fsanitize=undefined --action_env=UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1
+# Workaround for the fact that Bazel links with $CC, not $CXX
+# https://github.com/bazelbuild/bazel/issues/11122#issuecomment-613746748
+build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr
+# Workaround for https://bugs.llvm.org/show_bug.cgi?id=16404
+build:ubsan --linkopt=--rtlib=compiler-rt --linkopt=-lunwind
+
+build:Werror --copt=-Werror
+build:Werror --per_file_copt=json/parser@-Wno-error
+build:Werror --per_file_copt=com_google_protobuf@-Wno-error
+
+# GCC's -fanalyzer, a deeper static analysis than normal warnings.
+build:analyzer --copt=-fanalyzer --copt=-Werror
+build:analyzer --per_file_copt=json/parser@-fno-analyzer
+build:analyzer --per_file_copt=com_google_protobuf@-fno-analyzer
+build:analyzer --per_file_copt=com_github_google_benchmark@-fno-analyzer
+
+# --config=asan-libfuzzer
+build:asan-libfuzzer --action_env=CC=clang
+build:asan-libfuzzer --action_env=CXX=clang++
+build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
+build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
+build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=asan
+
+# --config=msan-libfuzzer
+build:msan-libfuzzer --action_env=CC=clang
+build:msan-libfuzzer --action_env=CXX=clang++
+build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
+build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
+build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=msan
+
+# --config=ubsan-libfuzzer
+build:ubsan-libfuzzer --action_env=CC=clang
+build:ubsan-libfuzzer --action_env=CXX=clang++
+build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer
+build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer
+build:ubsan-libfuzzer --copt=-fsanitize=undefined
+build:ubsan-libfuzzer --linkopt=-fsanitize=undefined
+build:ubsan-libfuzzer --linkopt=-fsanitize-link-c++-runtime

.clang-format

@@ -0,0 +1,3 @@
+BasedOnStyle: Google
+DerivePointerAlignment: false
+PointerAlignment: Left

.github/workflows/bazel_tests.yml

@@ -0,0 +1,38 @@
+name: Bazel Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+
+  ubuntu:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false   # Don't cancel all jobs if one fails.
+      matrix:
+        include:
+          - { CC: clang, os: ubuntu-20.04, flags: "" }
+          - { CC: clang, os: ubuntu-20.04, flags: "-c opt" }  # Some warnings only fire with -c opt
+          - { CC: gcc, os: ubuntu-20.04, flags: "-c opt" }
+          - { CC: clang, os: ubuntu-20.04, flags: "--//:fasttable_enabled=true -- -cmake:test_generated_files" }
+          - { CC: clang, os: ubuntu-20.04, flags: "--config=asan -c dbg -- -benchmarks:benchmark -python/..." }
+          - { CC: clang, os: ubuntu-20.04, flags: "--config=ubsan -c dbg -- -benchmarks:benchmark -python/... -upb/bindings/lua/...", install: "libunwind-dev" }
+          - { CC: clang, os: ubuntu-20.04, flags: "--copt=-m32 --linkopt=-m32 -- -... benchmarks:benchmark ", install: "g++-multilib" }
+          - { CC: clang, os: macos-11, flags: "" }
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Python venv
+        run: rm -rf /tmp/venv && python3 -m venv /tmp/venv
+      - name: Install dependencies
+        run: sudo apt update && sudo apt install -y ${{ matrix.install }}
+        if: matrix.install != ''
+      - name: Run tests
+        run: cd ${{ github.workspace }} && PATH=/tmp/venv/bin:$PATH CC=${{ matrix.CC }} bazel test --test_output=errors ... ${{ matrix.flags }}

.github/workflows/cifuzz.yml

@@ -0,0 +1,29 @@
+name: CIFuzz
+on: [pull_request]
+jobs:
+ Fuzzing:
+   runs-on: ubuntu-latest
+   strategy:
+     fail-fast: false
+     matrix:
+       sanitizer: [address, undefined]
+   steps:
+   - name: Build Fuzzers (${{ matrix.sanitizer }})
+     uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+     with:
+       oss-fuzz-project-name: 'upb'
+       dry-run: false
+       sanitizer: ${{ matrix.sanitizer }}
+   - name: Run Fuzzers (${{ matrix.sanitizer }})
+     uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+     with:
+       oss-fuzz-project-name: 'upb'
+       fuzz-seconds: 600
+       dry-run: false
+       sanitizer: ${{ matrix.sanitizer }}
+   - name: Upload Crash
+     uses: actions/upload-artifact@v1
+     if: failure()
+     with:
+       name: ${{ matrix.sanitizer }}-artifacts
+       path: ./out/artifacts

.github/workflows/clang_format.yml

@@ -0,0 +1,20 @@
+name: Check ClangFormat
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  check_clang_format:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run ClangFormat
+        run: find . | grep -E '\.(c|h|cc)$' | grep -E -v '^./(third_party|cmake)' | xargs clang-format -i
+      - name: Check for differences
+        run: git diff --exit-code

.gitignore

@@ -1,4 +1,4 @@
-*.s??
+*.sw?
 obj/
 lib/
 bazel-*

CONTRIBUTING.md

@@ -1,7 +1,37 @@
-## <a name="cla"></a> Signing the CLA
 
-Please sign the [Google Contributor License Agreement
+# How to Contribute
-(CLA)](https://cla.developers.google.com/)
+
-before sending pull requests. For any code changes to be
+We'd love to accept your patches and contributions to this project. There are
-accepted, the CLA must be signed. It's a quick process, I
+just a few small guidelines you need to follow.
-promise!
+
+## Get in touch
+
+If your idea will take you more than, say, 30 minutes to
+implement, please get in touch first via the issue tracker
+to touch base about your plan.  That will give an
+opportunity for early feedback and help avoid wasting your
+time.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution;
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to <https://cla.developers.google.com/> to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code Reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+## Community Guidelines
+
+This project follows [Google's Open Source Community
+Guidelines](https://opensource.google/conduct/).

DESIGN.md

@@ -1,72 +1,201 @@
 
-μpb Design
+# upb Design
-----------
+
-
+upb aims to be a minimal C protobuf kernel.  It has a C API, but its primary
-μpb has the following design goals:
+goal is to be the core runtime for a higher-level API.
-
+
-- C89 compatible.
+## Design goals
-- small code size (both for the core library and generated messages).
+
-- fast performance (hundreds of MB/s).
+- Full protobuf conformance
-- idiomatic for C programs.
+- Small code size
-- easy to wrap in high-level languages (Python, Ruby, Lua, etc) with
+- Fast performance (without compromising code size)
-  good performance and all standard protobuf features.
+- Easy to wrap in language runtimes
-- hands-off about memory management, allowing for easy integration
+- Easy to adapt to different memory management schemes (refcounting, GC, etc)
-  with existing VMs and/or garbage collectors.
+
-- offers binary ABI compatibility between apps, generated messages, and
+## Design parameters
-  the core library (doesn't require re-generating messages or recompiling
+
-  your application when the core library changes).
+- C99
-- provides all features that users expect from a protobuf library
+- 32 or 64-bit CPU (assumes 4 or 8 byte pointers)
-  (generated messages in C, reflection, text format, etc.).
+- Uses pointer tagging, but avoids other implementation-defined behavior
-- layered, so the core is small and doesn't require descriptors.
+- Aims to never invoke undefined behavior (tests with ASAN, UBSAN, etc)
-- tidy about symbol references, so that any messages or features that
+- No global state, fully re-entrant
-  aren't used by a C program can have their code GC'd by the linker.
+
-- possible to use protobuf binary format without leaking message/field
+
-  names into the binary.
+## Overall Structure
-
+
-μpb accomplishes these goals by keeping a very small core that does not contain
+The upb library is divided into two main parts:
-descriptors.  We need some way of knowing what fields are in each message and
+
-where they live, but instead of descriptors, we keep a small/lightweight summary
+- A core message representation, which supports binary format parsing
-of the .proto file.  We call this a `upb_msglayout`.  It contains the bare
+  and serialization.
-minimum of what we need to know to parse and serialize protobuf binary format
+  - `upb/upb.h`: arena allocator (`upb_arena`)
-into our internal representation for messages, `upb_msg`.
+  - `upb/msg_internal.h`: core message representation and parse tables
-
+  - `upb/msg.h`: accessing metadata common to all messages, like unknown fields
-The core then contains functions to parse/serialize a message, given a `upb_msg*`
+  - `upb/decode.h`: binary format parsing
-and a `const upb_msglayout*`.
+  - `upb/encode.h`: binary format serialization
-
+  - `upb/table_internal.h`: hash table (used for maps)
-This approach is similar to [nanopb](https://github.com/nanopb/nanopb) which
+  - `upbc/protoc-gen-upbc.cc`: compiler that generates `.upb.h`/`.upb.c` APIs for
-also compiles message definitions to a compact, internal representation without
+    accessing messages without reflection.
-names.  However nanopb does not aim to be a fully-featured library, and has no
+- A reflection add-on library that supports JSON and text format.
-support for text format, JSON, or descriptors.  μpb is unique in that it has a
+  - `upb/def.h`: schema representation and loading from descriptors
-small core similar to nanopb (though not quite as small), but also offers a
+  - `upb/reflection.h`: reflective access to message data.
-full-featured protobuf library for applications that want reflection, text
+  - `upb/json_encode.h`: JSON encoding
-format, JSON format, etc.
+  - `upb/json_decode.h`: JSON decoding
-
+  - `upb/text_encode.h`: text format encoding
-Without descriptors, the core doesn't have access to field names, so it cannot
+  - `upbc/protoc-gen-upbdefs.cc`: compiler that generates `.upbdefs.h`/`.upbdefs.c`
-parse/serialize to protobuf text format or JSON.  Instead this functionality
+    APIs for loading reflection.
-lives in separate modules that depend on the module implementing descriptors.
+
-With the descriptor module we can parse/serialize binary descriptors and
+## Core Message Representation
-validate that they follow all the rules of protobuf schemas.
+
-
+The representation for each message consists of:
-To provide binary compatibility, we version the structs that generated messages
+- One pointer (`upb_msg_internaldata*`) for unknown fields and extensions. This
-use to create a `upb_msglayout*`.  The current initializers are
+  pointer is `NULL` when no unknown fields or extensions are present.
-`upb_msglayout_msginit_v1`, `upb_msglayout_fieldinit_v1`, etc.  Then
+- Hasbits for any optional/required fields.
-`upb_msglayout*` uses these as its internal representation.  If upb changes its
+- Case integers for each oneof.
-internal representation for a `upb_msglayout*`, it will also include code to
+- Data for each field.
-convert the old representation to the new representation.  This will use some
+
-more memory/CPU at runtime to convert between the two, but apps that statically
+For example, a layout for a message with two `optional int32` fields would end
-link μpb will never need to worry about this.
+up looking something like this:
-
+
-TODO
+```c
-----
+// For illustration only, upb does not actually generate structs.
-
+typedef struct {
-1. revise our generated code until it is in a state where we feel comfortable
+  upb_msg_internaldata* internal;  // Unknown fields and extensions.
-   committing to API/ABI stability for it.  In particular there is an open
+  uint32_t hasbits;                // We are only using two hasbits.
-   question of whether non-ABI-compatible field accesses should have a
+  int32_t field1;
-   fastpath different from the ABI-compatible field access.
+  int32_t field2;
-1. Add missing features (maps, extensions, unknown fields).
+} package_name_MessageName;
-1. Flesh out C++ wrappers.
+```
-1. *(lower-priority)*: revise all of the existing encoders/decoders and
+
-   handlers.  We probably will want to keep handlers, since they let us decouple
+Note in particular that messages do *not* have:
-   encoders/decoders from `upb_msg`, but we need to simplify all of that a LOT.
+- A pointer to reflection or a parse table (upb messages are not self-describing).
-   Likely we will want to make handlers only per-message instead of per-field,
+- A pointer to an arena (the arena must be expicitly passed into any function that
-   except for variable-length fields.
+  allocates).
+
+The upb compiler computes a layout for each message, and determines the offset for
+each field using normal alignment rules (each data member must be aligned to a
+multiple of its size).  This layout is then embedded into the generated `.upb.h`
+and `.upb.c` headers in two different forms.  First as inline accessors that expect
+the data at a given offset:
+
+```c
+// Example of a generated accessor, from foo.upb.h
+UPB_INLINE int32_t package_name_MessageName_field1(
+    const upb_test_MessageName *msg) {
+  return *UPB_PTR_AT(msg, UPB_SIZE(4, 4), int32_t);
+}
+```
+
+Secondly, the layout is emitted as a table which is used by the parser and serializer.
+We call these tables "mini-tables" to distinguish them from the larger and more
+optimized "fast tables" used in `upb/decode_fast.c` (an experimental parser that is
+2-3x the speed of the main parser, though the main parser is already quite fast).
+
+```c
+// Definition of mini-table structure, from upb/msg_internal.h
+typedef struct {
+  uint32_t number;
+  uint16_t offset;
+  int16_t presence;       /* If >0, hasbit_index.  If <0, ~oneof_index. */
+  uint16_t submsg_index;  /* undefined if descriptortype != MESSAGE or GROUP. */
+  uint8_t descriptortype;
+  int8_t mode;            /* upb_fieldmode, with flags from upb_labelflags */
+} upb_msglayout_field;
+
+typedef enum {
+  _UPB_MODE_MAP = 0,
+  _UPB_MODE_ARRAY = 1,
+  _UPB_MODE_SCALAR = 2,
+} upb_fieldmode;
+
+typedef struct {
+  const struct upb_msglayout *const* submsgs;
+  const upb_msglayout_field *fields;
+  uint16_t size;
+  uint16_t field_count;
+  bool extendable;
+  uint8_t dense_below;
+  uint8_t table_mask;
+} upb_msglayout;
+
+// Example of a generated mini-table, from foo.upb.c
+static const upb_msglayout_field upb_test_MessageName__fields[2] = {
+  {1, UPB_SIZE(4, 4), 1, 0, 5, _UPB_MODE_SCALAR},
+  {2, UPB_SIZE(8, 8), 2, 0, 5, _UPB_MODE_SCALAR},
+};
+
+const upb_msglayout upb_test_MessageName_msginit = {
+  NULL,
+  &upb_test_MessageName__fields[0],
+  UPB_SIZE(16, 16), 2, false, 2, 255,
+};
+```
+
+The upb compiler computes separate layouts for 32 and 64 bit modes, since the
+pointer size will be 4 or 8 bytes respectively.  The upb compiler embeds both
+sizes into the source code, using a `UPB_SIZE(size32, size64)` macro that can
+choose the appropriate size at build time based on the size of `UINTPTR_MAX`.
+
+Note that `.upb.c` files contain data tables only.  There is no "generated code"
+except for the inline accessors in the `.upb.h` files: the entire footprint
+of `.upb.c` files is in `.rodata`, none in `.text` or `.data`.
+
+## Memory Management Model
+
+All memory management in upb is built around arenas.  A message is never
+considered to "own" the strings or sub-messages contained within it.  Instead a
+message and all of its sub-messages/strings/etc. are all owned by an arena and
+are freed when the arena is freed.  An entire message tree will probably be
+owned by a single arena, but this is not required or enforced.  As far as upb is
+concerned, it is up to the client how to partition its arenas.  upb only requires
+that when you ask it to serialize a message, that all reachable messages are
+still alive.
+
+The arena supports both a user-supplied initial block and a custom allocation
+callback, so there is a lot of flexibility in memory allocation strategy.  The
+allocation callback can even be `NULL` for heap-free operation.  The main
+constraint of the arena is that all of the memory in each arena must be freed
+together.
+
+`upb_arena` supports a novel operation called "fuse".  When two arenas are fused
+together, their lifetimes are irreversibly joined, such that none of the arena
+blocks in either arena will be freed until *both* arenas are freed with
+`upb_arena_free()`.  This is useful when joining two messages from separate
+arenas (making one a sub-message of the other).  Fuse is an a very cheap
+operation, and an unlimited number of arenas can be fused together efficiently.
+
+## Reflection and Descriptors
+
+upb offers a fully-featured reflection library.  There are two main ways of
+using reflection:
+
+1. You can load descriptors from strings using `upb_symtab_addfile()`.
+  The upb runtime will dynamically create mini-tables like what the upb compiler
+  would have created if you had compiled this type into a `.upb.c` file.
+2. You can load descriptors using generated `.upbdefs.h` interfaces.
+  This will load reflection that references the corresponding `.upb.c`
+  mini-tables instead of building a new mini-table on the fly.  This lets
+  you reflect on generated types that are linked into your program.
+
+upb's design for descriptors is similar to protobuf C++ in many ways, with
+the following correspondences:
+
+| C++ Type | upb type |
+| ---------| ---------|
+| `google::protobuf::DescriptorPool` | `upb_symtab`
+| `google::protobuf::Descriptor` | `upb_msgdef`
+| `google::protobuf::FieldDescriptor` | `upb_fielddef`
+| `google::protobuf::OneofDescriptor` | `upb_oneofdef`
+| `google::protobuf::EnumDescriptor` | `upb_enumdef`
+| `google::protobuf::FileDescriptor` | `upb_filedef`
+| `google::protobuf::ServiceDescriptor` | `upb_servicedef`
+| `google::protobuf::MethodDescriptor` | `upb_methoddef`
+
+Like in C++ descriptors (defs) are created by loading a
+`google_protobuf_FileDescriptorProto` into a `upb_symtab`.  This creates and
+links all of the def objects corresponding to that `.proto` file, and inserts
+the names into a symbol table so they can be looked up by name.
+
+Once you have loaded some descriptors into a `upb_symtab`, you can create and
+manipulate messages using the interfaces defined in `upb/reflection.h`.  If your
+descriptors are linked to your generated layouts using option (2) above, you can
+safely access the same messages using both reflection and generated interfaces.

LICENSE

@@ -1,5 +1,5 @@
 
-Copyright (c) 2009-2011, Google Inc.
+Copyright (c) 2009-2021, Google LLC
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
@@ -10,14 +10,14 @@ modification, are permitted provided that the following conditions are met:
     * Redistributions in binary form must reproduce the above copyright
       notice, this list of conditions and the following disclaimer in the
       documentation and/or other materials provided with the distribution.
-    * Neither the name of Google Inc. nor the names of any other
+    * Neither the name of Google LLC nor the names of any other
       contributors may be used to endorse or promote products
       derived from this software without specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY GOOGLE INC. ``AS IS'' AND ANY EXPRESS OR IMPLIED
 WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-EVENT SHALL GOOGLE INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+EVENT SHALL GOOGLE LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER

README.md

@@ -1,117 +1,64 @@
 
-# μpb - a small protobuf implementation in C
+# μpb: small, fast C protos
-
-|Platform|Build Status|
-|--------|------------|
-|macOS|[![Build Status](https://storage.googleapis.com/upb-kokoro-results/status-badge/macos.png)](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod%3Aupb%2Fmacos%2Fcontinuous)|
-|ubuntu|[![Build Status](https://storage.googleapis.com/upb-kokoro-results/status-badge/ubuntu.png)](https://fusion.corp.google.com/projectanalysis/summary/KOKORO/prod%3Aupb%2Fubuntu%2Fcontinuous)|
-
-μpb (often written 'upb') is a small protobuf implementation written in C.
-
-upb generates a C API for creating, parsing, and serializing messages
-as declared in `.proto` files.  upb is heavily arena-based: all
-messages always live in an arena (note: the arena can live in stack or
-static memory if desired).  Here is a simple example:
-
-```c
-#include "conformance/conformance.upb.h"
-
-void foo(const char* data, size_t size) {
-  upb_arena *arena;
-
-  /* Generated message type. */
-  conformance_ConformanceRequest *request;
-  conformance_ConformanceResponse *response;
-
-  arena = upb_arena_new();
-  request = conformance_ConformanceRequest_parse(data, size, arena);
-  response = conformance_ConformanceResponse_new(arena);
-
-  switch (conformance_ConformanceRequest_payload_case(request)) {
-    case conformance_ConformanceRequest_payload_protobuf_payload: {
-      upb_strview payload = conformance_ConformanceRequest_protobuf_payload(request);
-      // ...
-      break;
-    }
-
-    case conformance_ConformanceRequest_payload_NOT_SET:
-      fprintf(stderr, "conformance_upb: Request didn't have payload.\n");
-      break;
-
-    default: {
-      static const char msg[] = "Unsupported input format.";
-      conformance_ConformanceResponse_set_skipped(
-          response, upb_strview_make(msg, sizeof(msg)));
-      break;
-    }
-  }
-
-  /* Frees all messages on the arena. */
-  upb_arena_free(arena);
-}
-```
 
-API and ABI are both subject to change!  Please do not distribute
+μpb (often written 'upb') is a small
-as a shared library for this reason (for now at least).
+[protobuf](https://github.com/protocolbuffers/protobuf) implementation written
+in C.
 
-## Using upb in your project
+upb is the core runtime for protobuf languages extensions in
+[Ruby](https://github.com/protocolbuffers/protobuf/tree/master/ruby),
+[PHP](https://github.com/protocolbuffers/protobuf/tree/master/php), and (soon)
+Python.
 
-Currently only Bazel is supported (CMake support is partial and incomplete
+While upb offers a C API, the C API & ABI **are not stable**. For this reason,
-but full CMake support is an eventual goal).
+upb is not generally offered as a C library for direct consumption, and there
+are no releases.
 
-To use upb in your Bazel project, first add upb to your `WORKSPACE` file,
+## Features
-either as a `git_repository()` or as a `new_local_repository()` with a
-Git Submodule.  (For an example, see `examples/bazel/ in this repo).
 
-```python
+upb has comparable speed to protobuf C++, but is an order of magnitude smaller
-# Add this to your WORKSPACE file.
+in code size.
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
-git_repository(
+Like the main protobuf implementation in C++, it supports:
-    name = "upb",
-    remote = "https://github.com/protocolbuffers/upb.git",
-    commit = "d16bf99ac4658793748cda3251226059892b3b7b",
-)
 
-load("@upb//bazel:workspace_deps.bzl", "upb_deps")
+- a generated API (in C)
+- reflection
+- binary & JSON wire formats
+- text format serialization
+- all standard features of protobufs (oneofs, maps, unknown fields, extensions,
+  etc.)
+- full conformance with the protobuf conformance tests
 
-upb_deps()
+upb also supports some features that C++ does not:
-```
 
-Then in your BUILD file you can add `upb_proto_library()` rules that
+- **optional reflection:** generated messages are agnostic to whether
-generate code for a corresponding `proto_library()` rule.  For
+  reflection will be linked in or not.
-example:
+- **no global state:** no pre-main registration or other global state.
-
+- **fast reflection-based parsing:** messages loaded at runtime parse
-```python
+  just as fast as compiled-in messages.
-# Add this to your BUILD file.
+
-load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library")
+However there are a few features it does not support:
-
-proto_library(
-    name = "foo_proto",
-    srcs = ["foo.proto"],
-)
-
-upb_proto_library(
-    name = "foo_upbproto",
-    deps = [":foo_proto"],
-)
-
-cc_binary(
-    name = "test_binary",
-    srcs = ["test_binary.c"],
-    deps = [":foo_upbproto"],
-)
-```
 
-Then in your `.c` file you can #include the generated header:
+- text format parsing
+- deep descriptor verification: upb's descriptor validation is not as exhaustive
+  as `protoc`.
 
-```c
+## Install
-#include "foo.upb.h"
 
-/* Insert code that uses generated types. */
+For Ruby, use [RubyGems](https://rubygems.org/gems/google-protobuf):
+
+```
+$ gem install google-protobuf
 ```
 
-Alternatively, you can build and install upb using [vcpkg](https://github.com/microsoft/vcpkg/) dependency manager:
+For PHP, use [PECL](https://pecl.php.net/package/protobuf):
+
+```
+$ sudo pecl install protobuf
+```
+
+Alternatively, you can build and install upb using
+[vcpkg](https://github.com/microsoft/vcpkg/) dependency manager:
 
     git clone https://github.com/Microsoft/vcpkg.git
     cd vcpkg
@@ -119,27 +66,13 @@ Alternatively, you can build and install upb using [vcpkg](https://github.com/mi
     ./vcpkg integrate install
     ./vcpkg install upb
 
-The upb port in vcpkg is kept up to date by microsoft team members and community contributors.
+The upb port in vcpkg is kept up to date by microsoft team members and community
-If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
+contributors.
-
-## Old "handlers" interfaces
-
-This library contains several semi-deprecated interfaces (see BUILD
-file for more info about which interfaces are deprecated).  These
-deprecated interfaces are still used in some significant projects,
-such as the Ruby and PHP C bindings for protobuf in the [main protobuf
-repo](https://github.com/protocolbuffers/protobuf).  The goal is to
-migrate the Ruby/PHP bindings to use the newer, simpler interfaces
-instead.  Please do not use the old interfaces in new code.
-
-## Lua bindings
 
-This repo has some Lua bindings for the core library.  These are
+If the version is out of date, please
-experimental and very incomplete.  These are currently included in
+[create an issue or pull request](https://github.com/Microsoft/vcpkg) on the
-order to validate that the C API is suitable for wrapping.  As the
+vcpkg repository.
-project matures these Lua bindings may become publicly available.
 
-## Contact
+## Contributing
 
-Author: Josh Haberman ([jhaberman@gmail.com](mailto:jhaberman@gmail.com),
+Please see [CONTRIBUTING.md](CONTRIBUTING.md).
-[haberman@google.com](mailto:haberman@google.com))

WORKSPACE

@@ -2,9 +2,13 @@ workspace(name = "upb")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("//bazel:workspace_deps.bzl", "upb_deps")
+load("//bazel:workspace_defs.bzl", "system_python")
 
 upb_deps()
 
+load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
+protobuf_deps()
+
 http_archive(
     name = "lua",
     build_file = "//bazel:lua.BUILD",
@@ -16,14 +20,6 @@ http_archive(
     ],
 )
 
-http_archive(
-    name = "ragel",
-    build_file = "//bazel:ragel.BUILD",
-    sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f",
-    strip_prefix = "ragel-6.10",
-    urls = ["http://www.colm.net/files/ragel/ragel-6.10.tar.gz"],
-)
-
 http_archive(
      name = "com_google_googletest",
      urls = ["https://github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip"],  # 2019-01-07
@@ -33,7 +29,36 @@ http_archive(
 
 http_archive(
     name = "com_github_google_benchmark",
-    urls = ["https://github.com/google/benchmark/archive/16703ff83c1ae6d53e5155df3bb3ab0bc96083be.zip"],
+    urls = ["https://github.com/google/benchmark/archive/0baacde3618ca617da95375e0af13ce1baadea47.zip"],
-    strip_prefix = "benchmark-16703ff83c1ae6d53e5155df3bb3ab0bc96083be",
+    strip_prefix = "benchmark-0baacde3618ca617da95375e0af13ce1baadea47",
-    sha256 = "59f918c8ccd4d74b6ac43484467b500f1d64b40cc1010daa055375b322a43ba3",
+    sha256 = "62e2f2e6d8a744d67e4bbc212fcfd06647080de4253c97ad5c6749e09faf2cb0",
+)
+
+http_archive(
+    name = "com_google_googleapis",
+    urls = ["https://github.com/googleapis/googleapis/archive/refs/heads/master.zip"],
+    build_file = "//benchmarks:BUILD.googleapis",
+    strip_prefix = "googleapis-master",
+    patch_cmds = ["find google -type f -name BUILD.bazel -delete"],
+)
+
+system_python(
+    name = "system_python"
 )
+
+register_toolchains("@system_python//:python_toolchain")
+
+http_archive(
+    name = "rules_fuzzing",
+    sha256 = "23bb074064c6f488d12044934ab1b0631e8e6898d5cf2f6bde087adb01111573",
+    strip_prefix = "rules_fuzzing-0.3.1",
+    urls = ["https://github.com/bazelbuild/rules_fuzzing/archive/v0.3.1.zip"],
+)
+
+load("@rules_fuzzing//fuzzing:repositories.bzl", "rules_fuzzing_dependencies")
+
+rules_fuzzing_dependencies()
+
+load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init")
+
+rules_fuzzing_init()

bazel/BUILD

@@ -0,0 +1,53 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+
+licenses(["notice"])
+
+py_binary(
+    name = "amalgamate",
+    srcs = ["amalgamate.py"],
+    visibility = ["//:__pkg__"],
+)
+
+# py_proto_library() is private rule, only intended for internal use by upb.
+# Hopefully py_proto_library() will eventually be availble in rules_proto or
+# another upstream package.
+bzl_library(
+    name = "py_proto_library_bzl",
+    srcs = ["py_proto_library.bzl"],
+)
+
+bzl_library(
+    name = "upb_proto_library_bzl",
+    srcs = ["upb_proto_library.bzl"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@bazel_skylib//lib:paths",
+        "@bazel_tools//tools/cpp:toolchain_utils.bzl",
+        "@rules_proto//proto:defs",
+    ],
+)

bazel/amalgamate.py

@@ -0,0 +1,126 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+import re
+import os
+
+INCLUDE_RE = re.compile('^#include "([^"]*)"$')
+
+def parse_include(line):
+  match = INCLUDE_RE.match(line)
+  return match.groups()[0] if match else None
+
+class Amalgamator:
+  def __init__(self, output_path, prefix):
+    self.include_paths = ["."]
+    self.included = set(["upb/port_def.inc", "upb/port_undef.inc"])
+    self.output_h = open(output_path + prefix + "upb.h", "w")
+    self.output_c = open(output_path + prefix + "upb.c", "w")
+
+    self.output_c.write("/* Amalgamated source file */\n")
+    self.output_c.write('#include "%supb.h"\n' % (prefix))
+    self.output_c.write(open("upb/port_def.inc").read())
+
+    self.output_h.write("/* Amalgamated source file */\n")
+    self.output_h.write(open("upb/port_def.inc").read())
+
+  def add_include_path(self, path):
+      self.include_paths.append(path)
+
+  def finish(self):
+    self._add_header("upb/port_undef.inc")
+    self.add_src("upb/port_undef.inc")
+
+  def _process_file(self, infile_name, outfile):
+    file = None
+    for path in self.include_paths:
+        try:
+            full_path = os.path.join(path, infile_name)
+            file = open(full_path)
+            break
+        except IOError:
+            pass
+    if not file:
+        raise RuntimeError("Couldn't open file " + infile_name)
+
+    lines = file.readlines()
+
+    has_copyright = lines[1].startswith(" * Copyright")
+    if has_copyright:
+      while not lines[0].startswith(" */"):
+        lines.pop(0)
+      lines.pop(0)
+
+    lines.insert(0, "\n/** " + infile_name + " " + ("*" * 60) +"/");
+
+    for line in lines:
+      if not self._process_include(line, outfile):
+        outfile.write(line)
+
+  def _process_include(self, line, outfile):
+    include = parse_include(line)
+    if not include:
+      return False
+    if not (include.startswith("upb") or include.startswith("google")):
+      return False
+    if include.endswith("hpp"):
+      # Skip, we don't support the amalgamation from C++.
+      return True
+    else:
+      # Include this upb header inline.
+      if include not in self.included:
+        self.included.add(include)
+        self._add_header(include)
+      return True
+
+  def _add_header(self, filename):
+    self._process_file(filename, self.output_h)
+
+  def add_src(self, filename):
+    self._process_file(filename, self.output_c)
+
+# ---- main ----
+
+output_path = sys.argv[1]
+prefix = sys.argv[2]
+amalgamator = Amalgamator(output_path, prefix)
+files = []
+
+for arg in sys.argv[3:]:
+  arg = arg.strip()
+  if arg.startswith("-I"):
+    amalgamator.add_include_path(arg[2:])
+  elif arg.endswith(".h") or arg.endswith(".inc"):
+    pass
+  else:
+    files.append(arg)
+
+for filename in files:
+    amalgamator.add_src(filename)
+
+amalgamator.finish()

bazel/build_defs.bzl

@@ -1,7 +1,62 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 """Internal rules for building upb."""
 
 load(":upb_proto_library.bzl", "GeneratedSrcsInfo")
 
+UPB_DEFAULT_CPPOPTS = select({
+    "//:windows": [],
+    "//conditions:default": [
+        # copybara:strip_for_google3_begin
+        "-Wextra",
+        # "-Wshorten-64-to-32",  # not in GCC (and my Kokoro images doesn't have Clang)
+        "-Werror",
+        "-Wno-long-long",
+        # copybara:strip_end
+    ],
+})
+
+UPB_DEFAULT_COPTS = select({
+    "//:windows": [],
+    "//:fasttable_enabled_setting": ["-std=gnu99", "-DUPB_ENABLE_FASTTABLE"],
+    "//conditions:default": [
+        # copybara:strip_for_google3_begin
+        "-std=c99",
+        "-pedantic",
+        "-Werror=pedantic",
+        "-Wall",
+        "-Wstrict-prototypes",
+        # GCC (at least) emits spurious warnings for this that cannot be fixed
+        # without introducing redundant initialization (with runtime cost):
+        #   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80635
+        #"-Wno-maybe-uninitialized",
+        # copybara:strip_end
+    ],
+})
+
 def _librule(name):
     return name + "_lib"
 
@@ -50,57 +105,14 @@ def _remove_suffix(str, suffix):
     return str[:-len(suffix)]
 
 def make_shell_script(name, contents, out):
-    contents = (runfiles_init + contents).replace("$", "$$")
+    contents = runfiles_init + contents  # copybara:strip_for_google3
+    contents = contents.replace("$", "$$")
     native.genrule(
         name = "gen_" + name,
         outs = [out],
         cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % contents,
     )
 
-def generated_file_staleness_test(name, outs, generated_pattern):
-    """Tests that checked-in file(s) match the contents of generated file(s).
-
-    The resulting test will verify that all output files exist and have the
-    correct contents.  If the test fails, it can be invoked with --fix to
-    bring the checked-in files up to date.
-
-    Args:
-      name: Name of the rule.
-      outs: the checked-in files that are copied from generated files.
-      generated_pattern: the pattern for transforming each "out" file into a
-        generated file.  For example, if generated_pattern="generated/%s" then
-        a file foo.txt will look for generated file generated/foo.txt.
-    """
-
-    script_name = name + ".py"
-    script_src = "//:tools/staleness_test.py"
-
-    # Filter out non-existing rules so Blaze doesn't error out before we even
-    # run the test.
-    existing_outs = native.glob(include = outs)
-
-    # The file list contains a few extra bits of information at the end.
-    # These get unpacked by the Config class in staleness_test_lib.py.
-    file_list = outs + [generated_pattern, native.package_name() or ".", name]
-
-    native.genrule(
-        name = name + "_makescript",
-        outs = [script_name],
-        srcs = [script_src],
-        testonly = 1,
-        cmd = "cat $(location " + script_src + ") > $@; " +
-              "sed -i.bak -e 's|INSERT_FILE_LIST_HERE|" + "\\\n  ".join(file_list) + "|' $@",
-    )
-
-    native.py_test(
-        name = name,
-        srcs = [script_name],
-        data = existing_outs + [generated_pattern % file for file in outs],
-        deps = [
-            "//:staleness_test_lib",
-        ],
-    )
-
 # upb_amalgamation() rule, with file_list aspect.
 
 SrcList = provider(
@@ -135,24 +147,24 @@ def _upb_amalgamation(ctx):
     ctx.actions.run(
         inputs = inputs,
         outputs = ctx.outputs.outs,
-        arguments = [ctx.bin_dir.path + "/"] + [f.path for f in srcs] + ["-I" + root for root in _get_real_roots(inputs)],
+        arguments = [ctx.bin_dir.path + "/", ctx.attr.prefix] + [f.path for f in srcs] + ["-I" + root for root in _get_real_roots(inputs)],
         progress_message = "Making amalgamation",
-        executable = ctx.executable.amalgamator,
+        executable = ctx.executable._amalgamator,
     )
     return []
 
 upb_amalgamation = rule(
     attrs = {
-        "amalgamator": attr.label(
+        "_amalgamator": attr.label(
             executable = True,
-            cfg = "host",
+            cfg = "exec",
+            default = "//bazel:amalgamate",
+        ),
+        "prefix": attr.string(
+            default = "",
         ),
         "libs": attr.label_list(aspects = [_file_list_aspect]),
         "outs": attr.output_list(),
     },
     implementation = _upb_amalgamation,
 )
-
-def licenses(*args):
-    # No-op (for Google-internal usage).
-    pass

bazel/lua.BUILD

@@ -1,10 +1,34 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 package(
     default_visibility = ["//visibility:public"],
 )
 
 cc_library(
     name = "liblua_headers",
-    defines = ["LUA_USE_LINUX"],
     hdrs = [
         "src/lauxlib.h",
         "src/lua.h",
@@ -12,6 +36,7 @@ cc_library(
         "src/luaconf.h",
         "src/lualib.h",
     ],
+    defines = ["LUA_USE_LINUX"],
     includes = ["src"],
 )
 
@@ -72,7 +97,6 @@ cc_library(
         "src/lzio.c",
         "src/lzio.h",
     ],
-    defines = ["LUA_USE_LINUX"],
     hdrs = [
         "src/lauxlib.h",
         "src/lua.h",
@@ -80,6 +104,7 @@ cc_library(
         "src/luaconf.h",
         "src/lualib.h",
     ],
+    defines = ["LUA_USE_LINUX"],
     includes = ["src"],
     linkopts = [
         "-lm",
@@ -92,11 +117,11 @@ cc_binary(
     srcs = [
         "src/lua.c",
     ],
-    deps = [
-        ":liblua",
-    ],
     linkopts = [
         "-lreadline",
         "-rdynamic",
     ],
+    deps = [
+        ":liblua",
+    ],
 )

bazel/protobuf.patch

@@ -0,0 +1,89 @@
+--- BUILD
++++ BUILD
+@@ -525,7 +525,7 @@ cc_binary(
+ filegroup(
+     name = "testdata",
+     srcs = glob(["src/google/protobuf/testdata/**/*"]),
+-    visibility = ["//:__subpackages__"],
++    visibility = ["//visibility:public"],
+ )
+ 
+ RELATIVE_LITE_TEST_PROTOS = [
+@@ -931,13 +931,10 @@ py_library(
+         [
+             "python/google/protobuf/**/*.py",
+         ],
+-        exclude = [
+-            "python/google/protobuf/internal/*_test.py",
+-            "python/google/protobuf/internal/test_util.py",
+-        ],
+     ),
+     imports = ["python"],
+     srcs_version = "PY2AND3",
++    visibility = ["//visibility:public"],
+ )
+ 
+ cc_binary(
+@@ -1038,13 +1035,6 @@ py_proto_library(
+     name = "protobuf_python",
+     srcs = COPIED_WELL_KNOWN_PROTOS,
+     include = "python",
+-    data = select({
+-        "//conditions:default": [],
+-        ":use_fast_cpp_protos": [
+-            ":python/google/protobuf/internal/_api_implementation.so",
+-            ":python/google/protobuf/pyext/_message.so",
+-        ],
+-    }),
+     default_runtime = "",
+     protoc = ":protoc",
+     py_libs = [
+@@ -1080,6 +1070,7 @@ py_proto_library(
+     protoc = ":protoc",
+     srcs_version = "PY2AND3",
+     deps = [":protobuf_python"],
++    visibility = ["//visibility:public"],
+ )
+ 
+ py_proto_library(
+@@ -1093,6 +1084,7 @@ py_proto_library(
+     protoc = ":protoc",
+     srcs_version = "PY2AND3",
+     deps = [":python_common_test_protos"],
++    visibility = ["//visibility:public"],
+ )
+ 
+ py_library(
+
+--- python/google/protobuf/internal/test_util.py
++++ python/google/protobuf/internal/test_util.py
+@@ -634,6 +634,13 @@ def GoldenFile(filename):
+     # Found it.  Load the golden file from the testdata directory.
+     return open(full_path, 'rb')
+ 
++  # Search for cross-repo path.
++  full_path = os.path.join('external/com_google_protobuf/src/google/protobuf/testdata',
++                           filename)
++  if os.path.exists(full_path):
++    # Found it.  Load the golden file from the testdata directory.
++    return open(full_path, 'rb')
++
+   raise RuntimeError(
+       'Could not find golden files.  This test must be run from within the '
+       'protobuf source package so that it can read test data files from the '
+
+--- python/google/protobuf/internal/testing_refleaks.py
++++ python/google/protobuf/internal/testing_refleaks.py
+@@ -67,6 +67,12 @@ class ReferenceLeakCheckerMixin(object):
+   NB_RUNS = 3
+ 
+   def run(self, result=None):
++    testMethod = getattr(self, self._testMethodName)
++    expecting_failure_method = getattr(testMethod, "__unittest_expecting_failure__", False)
++    expecting_failure_class = getattr(self, "__unittest_expecting_failure__", False)
++    if expecting_failure_class or expecting_failure_method:
++      return
++
+     # python_message.py registers all Message classes to some pickle global
+     # registry, which makes the classes immortal.
+     # We save a copy of this registry, and reset it before we could references.

bazel/py_extension.bzl

@@ -0,0 +1,39 @@
+
+load(
+    "//bazel:build_defs.bzl",
+    "UPB_DEFAULT_COPTS",
+)
+
+def py_extension(name, srcs, deps=[]):
+    version_script = name + "_version_script.lds"
+    symbol = "PyInit_" + name
+    native.genrule(
+        name = "gen_" + version_script,
+        outs = [version_script],
+        cmd = "echo 'message { global: " + symbol + "; local: *; };' > $@",
+    )
+
+    native.cc_binary(
+        name = name,
+        srcs = srcs,
+        copts = UPB_DEFAULT_COPTS + [
+            # The Python API requires patterns that are ISO C incompatible, like
+            # casts between function pointers and object pointers.
+            "-Wno-pedantic",
+        ],
+        # We use a linker script to hide all symbols except the entry point for
+        # the module.
+        linkopts = select({
+            "@platforms//os:linux": ["-Wl,--version-script,$(location :" + version_script + ")"],
+            "@platforms//os:macos": [
+                "-Wl,-exported_symbol",
+                "-Wl,_" + symbol,
+            ],
+        }),
+        linkshared = True,
+        linkstatic = True,
+        deps = deps + [
+            ":" + version_script,
+            "@system_python//:python_headers",
+        ],
+    )

bazel/py_proto_library.bzl

@@ -0,0 +1,134 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""An implementation of py_proto_library().
+
+We have to implement this ourselves because there is currently no reasonable
+py_proto_library() rule available for Bazel.
+
+Our py_proto_library() is similar to how a real py_proto_library() should work.
+But it hasn't been deeply tested or reviewed, and upb should not be in the
+business of vending py_proto_library(), so we keep it private to upb.
+"""
+
+load("@bazel_skylib//lib:paths.bzl", "paths")
+load("@rules_proto//proto:defs.bzl", "ProtoInfo")  # copybara:strip_for_google3
+
+# Generic support code #########################################################
+
+def _get_real_short_path(file):
+    # For some reason, files from other archives have short paths that look like:
+    #   ../com_google_protobuf/google/protobuf/descriptor.proto
+    short_path = file.short_path
+    if short_path.startswith("../"):
+        second_slash = short_path.index("/", 3)
+        short_path = short_path[second_slash + 1:]
+
+    # Sometimes it has another few prefixes like:
+    #   _virtual_imports/any_proto/google/protobuf/any.proto
+    #   benchmarks/_virtual_imports/100_msgs_proto/benchmarks/100_msgs.proto
+    # We want just google/protobuf/any.proto.
+    virtual_imports = "_virtual_imports/"
+    if virtual_imports in short_path:
+        short_path = short_path.split(virtual_imports)[1].split("/", 1)[1]
+    return short_path
+
+def _get_real_root(file):
+    real_short_path = _get_real_short_path(file)
+    return file.path[:-len(real_short_path) - 1]
+
+def _generate_output_file(ctx, src, extension):
+    real_short_path = _get_real_short_path(src)
+    real_short_path = paths.relativize(real_short_path, ctx.label.package)
+    output_filename = paths.replace_extension(real_short_path, extension)
+    ret = ctx.actions.declare_file(output_filename)
+    return ret
+
+# py_proto_library() ###########################################################
+
+def _py_proto_library_rule_impl(ctx):
+    # A real py_proto_library() should enforce this constraint.
+    # We don't bother for now, since it saves us some effort not to.
+    #
+    # if len(ctx.attr.deps) != 1:
+    #     fail("only one deps dependency allowed.")
+
+    files = []
+    for dep in ctx.attr.deps:
+        files += dep[PyInfo].transitive_sources.to_list()
+    return [
+        DefaultInfo(files = depset(direct = files)),
+    ]
+
+def _py_proto_library_aspect_impl(target, ctx):
+    proto_info = target[ProtoInfo]
+    proto_sources = proto_info.direct_sources
+    srcs = [_generate_output_file(ctx, name, "_pb2.py") for name in proto_sources]
+    transitive_sets = proto_info.transitive_descriptor_sets.to_list()
+    ctx.actions.run(
+        inputs = depset(
+            direct = [proto_info.direct_descriptor_set],
+            transitive = [proto_info.transitive_descriptor_sets],
+        ),
+        outputs = srcs,
+        executable = ctx.executable._protoc,
+        arguments = [
+                        "--python_out=" + _get_real_root(srcs[0]),
+                        "--descriptor_set_in=" + ctx.configuration.host_path_separator.join([f.path for f in transitive_sets]),
+                    ] +
+                    [_get_real_short_path(file) for file in proto_sources],
+        progress_message = "Generating Python protos for :" + ctx.label.name,
+    )
+    outs_depset = depset(srcs)
+    return [
+        PyInfo(transitive_sources = outs_depset),
+    ]
+
+_py_proto_library_aspect = aspect(
+    attrs = {
+        "_protoc": attr.label(
+            executable = True,
+            cfg = "exec",
+            default = "@com_google_protobuf//:protoc",
+        ),
+    },
+    implementation = _py_proto_library_aspect_impl,
+    provides = [
+        PyInfo,
+    ],
+    attr_aspects = ["deps"],
+)
+
+py_proto_library = rule(
+    output_to_genfiles = True,
+    implementation = _py_proto_library_rule_impl,
+    attrs = {
+        "deps": attr.label_list(
+            aspects = [_py_proto_library_aspect],
+            allow_rules = ["proto_library"],
+            providers = [ProtoInfo],
+        ),
+    },
+)

bazel/pyproto_test_wrapper.bzl

@@ -0,0 +1,34 @@
+
+# copybara:strip_for_google3_begin
+
+def pyproto_test_wrapper(name):
+    src = name + "_wrapper.py"
+    native.py_test(
+        name = name,
+        srcs = [src],
+        legacy_create_init = False,
+        main = src,
+        data = ["@com_google_protobuf//:testdata"],
+        deps = [
+            "//python:message_ext",
+            "@com_google_protobuf//:python_common_test_protos",
+            "@com_google_protobuf//:python_specific_test_protos",
+            "@com_google_protobuf//:python_srcs",
+        ],
+    )
+
+# copybara:replace_for_google3_begin
+#
+# def pyproto_test_wrapper(name):
+#     src = name + "_wrapper.py"
+#     native.py_test(
+#         name = name,
+#         srcs = [src],
+#         main = src,
+#         deps = [
+#             "//net/proto2/python/internal:" + name + "_for_deps",
+#             "//net/proto2/python/public:use_upb_protos",
+#         ],
+#     )
+#
+# copybara:replace_for_google3_end

bazel/ragel.BUILD

@@ -1,193 +0,0 @@
-
-package(
-    default_visibility = ["//visibility:public"],
-)
-
-cc_binary(
-    name = "ragelc",
-    srcs = [
-        "ragel/rubycodegen.cpp",
-        "ragel/goipgoto.h",
-        "ragel/cdtable.h",
-        "ragel/rubycodegen.h",
-        "ragel/gotable.h",
-        "ragel/gocodegen.cpp",
-        "ragel/rubyfflat.cpp",
-        "ragel/common.cpp",
-        "ragel/gofflat.cpp",
-        "ragel/cdtable.cpp",
-        "ragel/cdsplit.cpp",
-        "ragel/rlparse.cpp",
-        "ragel/csfgoto.cpp",
-        "ragel/javacodegen.cpp",
-        "ragel/gocodegen.h",
-        "ragel/mlgoto.cpp",
-        "ragel/fsmgraph.cpp",
-        "ragel/version.h",
-        "ragel/mlfflat.h",
-        "ragel/fsmgraph.h",
-        "ragel/fsmbase.cpp",
-        "ragel/fsmstate.cpp",
-        "ragel/gotablish.cpp",
-        "ragel/rubyflat.cpp",
-        "ragel/cdfgoto.h",
-        "ragel/cscodegen.h",
-        "ragel/mlflat.cpp",
-        "ragel/rubyflat.h",
-        "ragel/goftable.h",
-        "ragel/rbxgoto.cpp",
-        "ragel/csfflat.cpp",
-        "ragel/gofgoto.cpp",
-        "ragel/gofgoto.h",
-        "ragel/ragel.h",
-        "ragel/goftable.cpp",
-        "ragel/cdcodegen.cpp",
-        "ragel/rlparse.h",
-        "ragel/cdsplit.h",
-        "ragel/xmlcodegen.cpp",
-        "ragel/goipgoto.cpp",
-        "ragel/dotcodegen.h",
-        "ragel/gogoto.cpp",
-        "ragel/csflat.h",
-        "ragel/csfflat.h",
-        #"ragel/config.h.in",
-        "ragel/csipgoto.cpp",
-        "ragel/mltable.cpp",
-        "ragel/mlflat.h",
-        "ragel/csftable.cpp",
-        "ragel/cdgoto.h",
-        "ragel/goflat.cpp",
-        "ragel/rubyfflat.h",
-        "ragel/mlftable.h",
-        "ragel/rubyftable.h",
-        "ragel/fsmap.cpp",
-        "ragel/redfsm.cpp",
-        "ragel/goflat.h",
-        "ragel/parsetree.cpp",
-        "ragel/fsmmin.cpp",
-        "ragel/dotcodegen.cpp",
-        "ragel/redfsm.h",
-        "ragel/mlcodegen.cpp",
-        "ragel/cdfgoto.cpp",
-        "ragel/cssplit.cpp",
-        "ragel/cstable.cpp",
-        "ragel/javacodegen.h",
-        "ragel/parsedata.cpp",
-        "ragel/buffer.h",
-        "ragel/gogoto.h",
-        "ragel/csgoto.h",
-        "ragel/pcheck.h",
-        "ragel/rubyftable.cpp",
-        "ragel/csfgoto.h",
-        "ragel/common.h",
-        "ragel/cdftable.h",
-        "ragel/mlgoto.h",
-        "ragel/csgoto.cpp",
-        "ragel/cdflat.h",
-        "ragel/cdipgoto.h",
-        "ragel/cstable.h",
-        "ragel/gendata.h",
-        "ragel/cdfflat.cpp",
-        "ragel/gotable.cpp",
-        "ragel/cdcodegen.h",
-        "ragel/gendata.cpp",
-        "ragel/rubytable.h",
-        "ragel/csflat.cpp",
-        "ragel/inputdata.h",
-        "ragel/inputdata.cpp",
-        "ragel/rubytable.cpp",
-        "ragel/fsmattach.cpp",
-        "ragel/csipgoto.h",
-        "ragel/cscodegen.cpp",
-        "ragel/cdfflat.h",
-        "ragel/rbxgoto.h",
-        "ragel/xmlcodegen.h",
-        "ragel/gofflat.h",
-        "ragel/parsedata.h",
-        "ragel/mlfgoto.h",
-        "ragel/cdflat.cpp",
-        "ragel/config.h",
-        "ragel/rlscan.cpp",
-        "ragel/mlcodegen.h",
-        "ragel/mlfflat.cpp",
-        "ragel/mlftable.cpp",
-        "ragel/mltable.h",
-        "ragel/cdipgoto.cpp",
-        "ragel/cdftable.cpp",
-        "ragel/parsetree.h",
-        "ragel/rlscan.h",
-        "ragel/main.cpp",
-        "ragel/cssplit.h",
-        "ragel/mlfgoto.cpp",
-        "ragel/csftable.h",
-        "ragel/gotablish.h",
-        "ragel/cdgoto.cpp",
-        "aapl/avlmelkey.h",
-        "aapl/dlistmel.h",
-        "aapl/avliset.h",
-        "aapl/avlkeyless.h",
-        "aapl/sbstset.h",
-        "aapl/sbsttable.h",
-        "aapl/quicksort.h",
-        "aapl/avlitree.h",
-        "aapl/avlcommon.h",
-        "aapl/bstset.h",
-        "aapl/avlmel.h",
-        "aapl/insertsort.h",
-        "aapl/dlist.h",
-        "aapl/avlmap.h",
-        "aapl/mergesort.h",
-        "aapl/resize.h",
-        "aapl/bstcommon.h",
-        "aapl/bstmap.h",
-        "aapl/compare.h",
-        "aapl/svector.h",
-        "aapl/avlset.h",
-        "aapl/bsttable.h",
-        "aapl/avlikeyless.h",
-        "aapl/bubblesort.h",
-        "aapl/table.h",
-        "aapl/avlbasic.h",
-        "aapl/vector.h",
-        "aapl/avlimap.h",
-        "aapl/dlistval.h",
-        "aapl/dlcommon.h",
-        "aapl/avlibasic.h",
-        "aapl/sbstmap.h",
-        "aapl/avlimel.h",
-        "aapl/avlimelkey.h",
-        "aapl/avltree.h",
-    ],
-    includes = ["ragel", "aapl"],
-)
-
-config_h_contents = """
-#define PACKAGE "ragel"
-
-/* Define to the address where bug reports for this package should be sent. */
-#define PACKAGE_BUGREPORT ""
-
-/* Define to the full name of this package. */
-#define PACKAGE_NAME "ragel"
-
-/* Define to the full name and version of this package. */
-#define PACKAGE_STRING "ragel 6.10"
-
-/* Define to the one symbol short name of this package. */
-#define PACKAGE_TARNAME "ragel"
-
-/* Define to the home page for this package. */
-#define PACKAGE_URL ""
-
-/* Define to the version of this package. */
-#define PACKAGE_VERSION "6.10"
-
-/* Version number of package */
-#define VERSION "6.10"
-"""
-
-genrule(
-    name = "gen_config_h",
-    outs = ["ragel/config.h"],
-    cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % config_h_contents,
-)

bazel/upb_proto_library.bzl

@@ -1,3 +1,28 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 """Public rules for using upb protos:
   - upb_proto_library()
   - upb_proto_reflection_library()
@@ -5,10 +30,11 @@
 
 load("@bazel_skylib//lib:paths.bzl", "paths")
 load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
+load("@rules_proto//proto:defs.bzl", "ProtoInfo")  # copybara:strip_for_google3
 
 # Generic support code #########################################################
 
-_is_bazel = not hasattr(native, "genmpm")
+_is_bazel = True  # copybara:replace_for_google3 _is_bazel = False
 
 def _get_real_short_path(file):
     # For some reason, files from other archives have short paths that look like:
@@ -17,25 +43,20 @@ def _get_real_short_path(file):
     if short_path.startswith("../"):
         second_slash = short_path.index("/", 3)
         short_path = short_path[second_slash + 1:]
+
     # Sometimes it has another few prefixes like:
     #   _virtual_imports/any_proto/google/protobuf/any.proto
+    #   benchmarks/_virtual_imports/100_msgs_proto/benchmarks/100_msgs.proto
     # We want just google/protobuf/any.proto.
-    if short_path.startswith("_virtual_imports"):
+    virtual_imports = "_virtual_imports/"
-        short_path = short_path.split("/", 2)[-1]
+    if virtual_imports in short_path:
+        short_path = short_path.split(virtual_imports)[1].split("/", 1)[1]
     return short_path
 
 def _get_real_root(file):
     real_short_path = _get_real_short_path(file)
     return file.path[:-len(real_short_path) - 1]
 
-def _get_real_roots(files):
-    roots = {}
-    for file in files:
-        real_root = _get_real_root(file)
-        if real_root:
-            roots[real_root] = True
-    return roots.keys()
-
 def _generate_output_file(ctx, src, extension):
     real_short_path = _get_real_short_path(src)
     real_short_path = paths.relativize(real_short_path, ctx.label.package)
@@ -50,7 +71,7 @@ def _filter_none(elems):
             out.append(elem)
     return out
 
-def _cc_library_func(ctx, name, hdrs, srcs, dep_ccinfos):
+def _cc_library_func(ctx, name, hdrs, srcs, copts, dep_ccinfos):
     """Like cc_library(), but callable from rules.
 
     Args:
@@ -86,6 +107,7 @@ def _cc_library_func(ctx, name, hdrs, srcs, dep_ccinfos):
         name = name,
         srcs = srcs,
         public_hdrs = hdrs,
+        user_compile_flags = copts,
         compilation_contexts = compilation_contexts,
         **blaze_only_args
     )
@@ -104,6 +126,44 @@ def _cc_library_func(ctx, name, hdrs, srcs, dep_ccinfos):
         linking_context = linking_context,
     )
 
+# Build setting for whether fasttable code generation is enabled ###############
+
+_FastTableEnabled = provider(
+    fields = {
+        "enabled": "whether fasttable is enabled",
+    },
+)
+
+def fasttable_enabled_impl(ctx):
+    raw_setting = ctx.build_setting_value
+
+    if raw_setting:
+        # TODO(haberman): check that the target CPU supports fasttable.
+        pass
+
+    return _FastTableEnabled(enabled = raw_setting)
+
+upb_fasttable_enabled = rule(
+    implementation = fasttable_enabled_impl,
+    build_setting = config.bool(flag = True),
+)
+
+# Dummy rule to expose select() copts to aspects  ##############################
+
+_UpbProtoLibraryCopts = provider(
+    fields = {
+        "copts": "copts for upb_proto_library()",
+    },
+)
+
+def upb_proto_library_copts_impl(ctx):
+    return _UpbProtoLibraryCopts(copts = ctx.attr.copts)
+
+upb_proto_library_copts = rule(
+    implementation = upb_proto_library_copts_impl,
+    attrs = {"copts": attr.string_list(default = [])},
+)
+
 # upb_proto_library / upb_proto_reflection_library shared code #################
 
 GeneratedSrcsInfo = provider(
@@ -118,21 +178,29 @@ _UpbDefsWrappedCcInfo = provider(fields = ["cc_info"])
 _WrappedGeneratedSrcsInfo = provider(fields = ["srcs"])
 _WrappedDefsGeneratedSrcsInfo = provider(fields = ["srcs"])
 
-def _compile_upb_protos(ctx, proto_info, proto_sources, ext):
+def _compile_upb_protos(ctx, generator, proto_info, proto_sources):
+    if len(proto_sources) == 0:
+        return GeneratedSrcsInfo(srcs = [], hdrs = [])
+
+    ext = "." + generator
+    tool = getattr(ctx.executable, "_gen_" + generator)
     srcs = [_generate_output_file(ctx, name, ext + ".c") for name in proto_sources]
     hdrs = [_generate_output_file(ctx, name, ext + ".h") for name in proto_sources]
     transitive_sets = proto_info.transitive_descriptor_sets.to_list()
+    fasttable_enabled = (hasattr(ctx.attr, "_fasttable_enabled") and
+                         ctx.attr._fasttable_enabled[_FastTableEnabled].enabled)
+    codegen_params = "fasttable:" if fasttable_enabled else ""
     ctx.actions.run(
         inputs = depset(
             direct = [proto_info.direct_descriptor_set],
             transitive = [proto_info.transitive_descriptor_sets],
         ),
-        tools = [ctx.executable._upbc],
+        tools = [tool],
         outputs = srcs + hdrs,
         executable = ctx.executable._protoc,
         arguments = [
-                        "--upb_out=" + _get_real_root(srcs[0]),
+                        "--" + generator + "_out=" + codegen_params + _get_real_root(srcs[0]),
-                        "--plugin=protoc-gen-upb=" + ctx.executable._upbc.path,
+                        "--plugin=protoc-gen-" + generator + "=" + tool.path,
                         "--descriptor_set_in=" + ctx.configuration.host_path_separator.join([f.path for f in transitive_sets]),
                     ] +
                     [_get_real_short_path(file) for file in proto_sources],
@@ -161,10 +229,7 @@ def _upb_proto_rule_impl(ctx):
         fail("proto_library rule must generate _UpbWrappedCcInfo or " +
              "_UpbDefsWrappedCcInfo (aspect should have handled this).")
 
-    if type(cc_info.linking_context.libraries_to_link) == "list":
+    lib = cc_info.linking_context.linker_inputs.to_list()[0].libraries[0]
-        lib = cc_info.linking_context.libraries_to_link[0]
-    else:
-        lib = cc_info.linking_context.libraries_to_link.to_list()[0]
     files = _filter_none([
         lib.static_library,
         lib.pic_static_library,
@@ -176,39 +241,38 @@ def _upb_proto_rule_impl(ctx):
         cc_info,
     ]
 
-def _upb_proto_aspect_impl(target, ctx, cc_provider, file_provider):
+def _upb_proto_aspect_impl(target, ctx, generator, cc_provider, file_provider):
     proto_info = target[ProtoInfo]
-    files = _compile_upb_protos(ctx, proto_info, proto_info.direct_sources, ctx.attr._ext)
+    files = _compile_upb_protos(ctx, generator, proto_info, proto_info.direct_sources)
-    deps = ctx.rule.attr.deps + ctx.attr._upb
+    deps = ctx.rule.attr.deps + getattr(ctx.attr, "_" + generator)
-    if cc_provider == _UpbDefsWrappedCcInfo:
-        deps += ctx.attr._upb_reflection
     dep_ccinfos = [dep[CcInfo] for dep in deps if CcInfo in dep]
     dep_ccinfos += [dep[_UpbWrappedCcInfo].cc_info for dep in deps if _UpbWrappedCcInfo in dep]
     dep_ccinfos += [dep[_UpbDefsWrappedCcInfo].cc_info for dep in deps if _UpbDefsWrappedCcInfo in dep]
-    if cc_provider == _UpbDefsWrappedCcInfo:
+    if generator == "upbdefs":
         if _UpbWrappedCcInfo not in target:
-            fail("Target should have _UpbDefsWrappedCcInfo provider")
+            fail("Target should have _UpbWrappedCcInfo provider")
         dep_ccinfos += [target[_UpbWrappedCcInfo].cc_info]
     cc_info = _cc_library_func(
         ctx = ctx,
-        name = ctx.rule.attr.name + ctx.attr._ext,
+        name = ctx.rule.attr.name + "." + generator,
         hdrs = files.hdrs,
         srcs = files.srcs,
+        copts = ctx.attr._copts[_UpbProtoLibraryCopts].copts,
         dep_ccinfos = dep_ccinfos,
     )
     return [cc_provider(cc_info = cc_info), file_provider(srcs = files)]
 
 def _upb_proto_library_aspect_impl(target, ctx):
-    return _upb_proto_aspect_impl(target, ctx, _UpbWrappedCcInfo, _WrappedGeneratedSrcsInfo)
+    return _upb_proto_aspect_impl(target, ctx, "upb", _UpbWrappedCcInfo, _WrappedGeneratedSrcsInfo)
 
 def _upb_proto_reflection_library_aspect_impl(target, ctx):
-    return _upb_proto_aspect_impl(target, ctx, _UpbDefsWrappedCcInfo, _WrappedDefsGeneratedSrcsInfo)
+    return _upb_proto_aspect_impl(target, ctx, "upbdefs", _UpbDefsWrappedCcInfo, _WrappedDefsGeneratedSrcsInfo)
 
 def _maybe_add(d):
     if not _is_bazel:
         d["_grep_includes"] = attr.label(
             allow_single_file = True,
-            cfg = "host",
+            cfg = "exec",
             default = "//tools/cpp:grep-includes",
         )
     return d
@@ -217,14 +281,17 @@ def _maybe_add(d):
 
 _upb_proto_library_aspect = aspect(
     attrs = _maybe_add({
-        "_upbc": attr.label(
+        "_copts": attr.label(
+            default = "//:upb_proto_library_copts__for_generated_code_only_do_not_use",
+        ),
+        "_gen_upb": attr.label(
             executable = True,
-            cfg = "host",
+            cfg = "exec",
-            default = "//:protoc-gen-upb",
+            default = "//upbc:protoc-gen-upb",
         ),
         "_protoc": attr.label(
             executable = True,
-            cfg = "host",
+            cfg = "exec",
             default = "@com_google_protobuf//:protoc",
         ),
         "_cc_toolchain": attr.label(
@@ -232,9 +299,8 @@ _upb_proto_library_aspect = aspect(
         ),
         "_upb": attr.label_list(default = [
             "//:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
-            "//:upb",
         ]),
-        "_ext": attr.string(default = ".upb"),
+        "_fasttable_enabled": attr.label(default = "//:fasttable_enabled"),
     }),
     implementation = _upb_proto_library_aspect_impl,
     provides = [
@@ -244,6 +310,7 @@ _upb_proto_library_aspect = aspect(
     attr_aspects = ["deps"],
     fragments = ["cpp"],
     toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
+    incompatible_use_toolchain_transition = True,
 )
 
 upb_proto_library = rule(
@@ -262,34 +329,27 @@ upb_proto_library = rule(
 
 _upb_proto_reflection_library_aspect = aspect(
     attrs = _maybe_add({
-        "_upbc": attr.label(
+        "_copts": attr.label(
+            default = "//:upb_proto_library_copts__for_generated_code_only_do_not_use",
+        ),
+        "_gen_upbdefs": attr.label(
             executable = True,
-            cfg = "host",
+            cfg = "exec",
-            default = "//:protoc-gen-upb",
+            default = "//upbc:protoc-gen-upbdefs",
         ),
         "_protoc": attr.label(
             executable = True,
-            cfg = "host",
+            cfg = "exec",
             default = "@com_google_protobuf//:protoc",
         ),
         "_cc_toolchain": attr.label(
             default = "@bazel_tools//tools/cpp:current_cc_toolchain",
         ),
-        # For unknown reasons, this gets overwritten.
+        "_upbdefs": attr.label_list(
-        "_upb": attr.label_list(
-            default = [
-                "//:generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
-                "//:upb",
-                "//:reflection",
-            ],
-        ),
-        "_upb_reflection": attr.label_list(
             default = [
-                "//:upb",
+                "//:generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me",
-                "//:reflection",
             ],
         ),
-        "_ext": attr.string(default = ".upbdefs"),
     }),
     implementation = _upb_proto_reflection_library_aspect_impl,
     provides = [
@@ -303,6 +363,7 @@ _upb_proto_reflection_library_aspect = aspect(
     attr_aspects = ["deps"],
     fragments = ["cpp"],
     toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
+    incompatible_use_toolchain_transition = True,
 )
 
 upb_proto_reflection_library = rule(

bazel/workspace_defs.bzl

@@ -0,0 +1,87 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Repository rule for using Python 3.x headers from the system."""
+
+_build_file = """
+load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
+cc_library(
+   name = "python_headers",
+   hdrs = glob(["python/**/*.h"]),
+   includes = ["python"],
+   visibility = ["//visibility:public"],
+)
+
+py_runtime(
+    name = "py3_runtime",
+    interpreter_path = "%s",
+    python_version = "PY3",
+)
+
+py_runtime_pair(
+    name = "runtime_pair",
+    py3_runtime = ":py3_runtime",
+)
+
+toolchain(
+    name = "python_toolchain",
+    toolchain = ":runtime_pair",
+    toolchain_type = "@rules_python//python:toolchain_type",
+)
+"""
+
+def _get_config_var(repository_ctx, name):
+    py_program = "import sysconfig; print(sysconfig.get_config_var('%s'), end='')"
+    result = repository_ctx.execute(["python3", "-c", py_program % (name)])
+    if result.return_code != 0:
+        fail("No python3 executable available on the system")
+    return result.stdout
+
+def _python_headers_impl(repository_ctx):
+    path = _get_config_var(repository_ctx, "INCLUDEPY")
+    repository_ctx.symlink(path, "python")
+    python3 = repository_ctx.which("python3")
+    repository_ctx.file("BUILD.bazel", _build_file % python3)
+
+# The system_python() repository rule exposes Python headers from the system.
+#
+# In WORKSPACE:
+#   system_python(
+#       name = "system_python_repo",
+#   )
+#
+# This repository exposes a single rule that you can depend on from BUILD:
+#   cc_library(
+#     name = "foobar",
+#     srcs = ["foobar.cc"],
+#     deps = ["@system_python_repo//:python_headers"],
+#   )
+#
+# The headers should correspond to the version of python obtained by running
+# the `python3` command on the system.
+system_python = repository_rule(
+    implementation = _python_headers_impl,
+    local = True,
+)

bazel/workspace_deps.bzl

@@ -1,36 +1,44 @@
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
 def upb_deps():
     maybe(
-        git_repository,
+        http_archive,
         name = "com_google_absl",
-        commit = "070f6e47b33a2909d039e620c873204f78809492",
+        url = "https://github.com/abseil/abseil-cpp/archive/b9b925341f9e90f5e7aa0cf23f036c29c7e454eb.zip",
-        remote = "https://github.com/abseil/abseil-cpp.git",
+        strip_prefix = "abseil-cpp-b9b925341f9e90f5e7aa0cf23f036c29c7e454eb",
-        shallow_since = "1541627663 -0500",
+        sha256 = "bb2a0b57c92b6666e8acb00f4cbbfce6ddb87e83625fb851b0e78db581340617",
     )
 
     maybe(
         git_repository,
         name = "com_google_protobuf",
+        commit = "2f91da585e96a7efe43505f714f03c7716a94ecb",
         remote = "https://github.com/protocolbuffers/protobuf.git",
-        commit = "d41002663fd04325ead28439dfd5ce2822b0d6fb",
+        patches = [
+            "//bazel:protobuf.patch",
+        ],
+        patch_cmds = [
+            "rm python/google/protobuf/__init__.py",
+            "rm python/google/protobuf/pyext/__init__.py",
+            "rm python/google/protobuf/internal/__init__.py",
+        ],
     )
 
+    rules_python_version = "740825b7f74930c62f44af95c9a4c1bd428d2c53"  # Latest @ 2021-06-23
+
     maybe(
         http_archive,
-        name = "bazel_skylib",
+        name = "rules_python",
-        strip_prefix = "bazel-skylib-master",
+        strip_prefix = "rules_python-{}".format(rules_python_version),
-        urls = ["https://github.com/bazelbuild/bazel-skylib/archive/master.tar.gz"],
+        url = "https://github.com/bazelbuild/rules_python/archive/{}.zip".format(rules_python_version),
+        sha256 = "09a3c4791c61b62c2cbc5b2cbea4ccc32487b38c7a2cc8f87a794d7a659cc742",
     )
 
     maybe(
         http_archive,
-        name = "zlib",
+        name = "bazel_skylib",
-        build_file = "@com_google_protobuf//:third_party/zlib.BUILD",
+        strip_prefix = "bazel-skylib-main",
-        sha256 = "629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff",
+        urls = ["https://github.com/bazelbuild/bazel-skylib/archive/main.tar.gz"],
-        strip_prefix = "zlib-1.2.11",
-        url = "https://github.com/madler/zlib/archive/v1.2.11.tar.gz",
     )

benchmarks/BUILD

@@ -0,0 +1,242 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+load(
+    "//bazel:upb_proto_library.bzl",
+    "upb_proto_library",
+    "upb_proto_reflection_library",
+)
+load(
+    ":build_defs.bzl",
+    "tmpl_cc_binary",
+    "cc_optimizefor_proto_library",
+    "expand_suffixes",
+    "proto_library",
+)
+
+licenses(["notice"])
+
+proto_library(
+    name = "descriptor_proto",
+    srcs = ["descriptor.proto"],
+)
+
+upb_proto_library(
+    name = "benchmark_descriptor_upb_proto",
+    deps = [":descriptor_proto"],
+)
+
+upb_proto_reflection_library(
+    name = "benchmark_descriptor_upb_proto_reflection",
+    deps = [":descriptor_proto"],
+)
+
+upb_proto_reflection_library(
+    name = "ads_upb_proto_reflection",
+    deps = ["@com_google_googleapis//:ads_proto"],
+)
+
+cc_proto_library(
+    name = "benchmark_descriptor_cc_proto",
+    deps = [":descriptor_proto"],
+)
+
+proto_library(
+    name = "benchmark_descriptor_sv_proto",
+    srcs = ["descriptor_sv.proto"],
+)
+
+cc_proto_library(
+    name = "benchmark_descriptor_sv_cc_proto",
+    deps = [":benchmark_descriptor_sv_proto"],
+)
+
+cc_test(
+    name = "benchmark",
+    testonly = 1,
+    srcs = ["benchmark.cc"],
+    deps = [
+        ":ads_upb_proto_reflection",
+        ":benchmark_descriptor_cc_proto",
+        ":benchmark_descriptor_sv_cc_proto",
+        ":benchmark_descriptor_upb_proto",
+        ":benchmark_descriptor_upb_proto_reflection",
+        "//:descriptor_upb_proto",
+        "//:reflection",
+        "@com_github_google_benchmark//:benchmark_main",
+        "@com_google_absl//absl/container:flat_hash_set",
+    ],
+)
+
+# Size benchmarks.
+
+SIZE_BENCHMARKS = {
+    "empty": "Empty",
+    "descriptor": "FileDescriptorSet",
+    "100_msgs": "Message100",
+    "200_msgs": "Message200",
+    "100_fields": "Message",
+    "200_fields": "Message",
+}
+
+py_binary(
+    name = "gen_synthetic_protos",
+    srcs = ["gen_synthetic_protos.py"],
+    python_version = "PY3",
+)
+
+py_binary(
+    name = "gen_upb_binary_c",
+    srcs = ["gen_upb_binary_c.py"],
+    python_version = "PY3",
+)
+
+py_binary(
+    name = "gen_protobuf_binary_cc",
+    srcs = ["gen_protobuf_binary_cc.py"],
+    python_version = "PY3",
+)
+
+genrule(
+    name = "do_gen_synthetic_protos",
+    tools = [":gen_synthetic_protos"],
+    outs = [
+        "100_msgs.proto",
+        "200_msgs.proto",
+        "100_fields.proto",
+        "200_fields.proto",
+    ],
+    cmd = "$(execpath :gen_synthetic_protos) $(RULEDIR)",
+)
+
+proto_library(
+    name = "100_msgs_proto",
+    srcs = ["100_msgs.proto"],
+)
+
+proto_library(
+    name = "200_msgs_proto",
+    srcs = ["200_msgs.proto"],
+)
+
+proto_library(
+    name = "100_fields_proto",
+    srcs = ["100_fields.proto"],
+)
+
+proto_library(
+    name = "200_fields_proto",
+    srcs = ["200_fields.proto"],
+)
+
+proto_library(
+    name = "empty_proto",
+    srcs = ["empty.proto"],
+)
+
+[(
+upb_proto_library(
+    name = k + "_upb_proto",
+    deps = [":" + k + "_proto"],
+),
+cc_proto_library(
+    name = k + "_cc_proto",
+    deps = [":" + k + "_proto"],
+),
+tmpl_cc_binary(
+    name = k + "_upb_binary",
+    testonly = 1,
+    gen = ":gen_upb_binary_c",
+    args = [
+        package_name() + "/" + k + ".upb.h",
+        "upb_benchmark_" + v,
+    ],
+    deps = [
+        ":" + k + "_upb_proto",
+    ],
+),
+tmpl_cc_binary(
+    name = k + "_protobuf_binary",
+    testonly = 1,
+    gen = ":gen_protobuf_binary_cc",
+    args = [
+        package_name() + "/" + k + ".pb.h",
+        "upb_benchmark::" + v,
+    ],
+    deps = [
+        ":" + k + "_cc_proto",
+    ],
+),
+cc_optimizefor_proto_library(
+    srcs = [k + ".proto"],
+    outs = [k + "_lite.proto"],
+    name = k + "_cc_lite_proto",
+    optimize_for = "LITE_RUNTIME",
+),
+tmpl_cc_binary(
+    name = k + "_lite_protobuf_binary",
+    testonly = 1,
+    gen = ":gen_protobuf_binary_cc",
+    args = [
+        package_name() + "/" + k + "_lite.pb.h",
+        "upb_benchmark::" + v,
+    ],
+    deps = [
+        ":" + k + "_cc_lite_proto",
+    ],
+),
+cc_optimizefor_proto_library(
+    srcs = [k + ".proto"],
+    outs = [k + "_codesize.proto"],
+    name = k + "_cc_codesize_proto",
+    optimize_for = "CODE_SIZE",
+),
+tmpl_cc_binary(
+    name = k + "_codesize_protobuf_binary",
+    testonly = 1,
+    gen = ":gen_protobuf_binary_cc",
+    args = [
+        package_name() + "/" + k + "_codesize.pb.h",
+        "upb_benchmark::" + v,
+    ],
+    deps = [
+        ":" + k + "_cc_codesize_proto",
+    ],
+)
+) for k, v in SIZE_BENCHMARKS.items()]
+
+genrule(
+    testonly = 1,
+    name = "size_data",
+    srcs = expand_suffixes(
+        SIZE_BENCHMARKS.keys(),
+        suffixes = ["_upb_binary", "_protobuf_binary", "_lite_protobuf_binary", "_codesize_protobuf_binary"],
+    ),
+    outs = ["size_data.txt"],
+    # We want --format=GNU which counts rodata with data, not text.
+    cmd = "size $$($$OSTYPE == 'linux-gnu' ? '--format=GNU -d' : '') $(SRCS) > $@",
+    # "size" sometimes isn't available remotely.
+    local = 1,
+)

benchmarks/BUILD.googleapis

@@ -0,0 +1,54 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+load(
+    "@rules_proto//proto:defs.bzl",
+    "proto_library",
+)
+
+proto_library(
+    name = "ads_proto",
+    srcs = glob([
+        "google/ads/googleads/v7/**/*.proto",
+        "google/api/**/*.proto",
+        "google/rpc/**/*.proto",
+        "google/longrunning/**/*.proto",
+        "google/logging/**/*.proto",
+    ]),
+    #srcs = ["google/ads/googleads/v5/services/google_ads_service.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_google_protobuf//:any_proto",
+        "@com_google_protobuf//:api_proto",
+        "@com_google_protobuf//:descriptor_proto",
+        "@com_google_protobuf//:duration_proto",
+        "@com_google_protobuf//:empty_proto",
+        "@com_google_protobuf//:field_mask_proto",
+        "@com_google_protobuf//:struct_proto",
+        "@com_google_protobuf//:timestamp_proto",
+        "@com_google_protobuf//:type_proto",
+        "@com_google_protobuf//:wrappers_proto",
+    ],
+)

benchmarks/benchmark.cc

@@ -0,0 +1,333 @@
+// Copyright (c) 2009-2021, Google LLC
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above copyright
+//       notice, this list of conditions and the following disclaimer in the
+//       documentation and/or other materials provided with the distribution.
+//     * Neither the name of Google LLC nor the
+//       names of its contributors may be used to endorse or promote products
+//       derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <benchmark/benchmark.h>
+#include <string.h>
+
+#include "absl/container/flat_hash_set.h"
+#include "benchmarks/descriptor.pb.h"
+#include "benchmarks/descriptor.upb.h"
+#include "benchmarks/descriptor.upbdefs.h"
+#include "benchmarks/descriptor_sv.pb.h"
+#include "google/ads/googleads/v7/services/google_ads_service.upbdefs.h"
+#include "google/protobuf/descriptor.pb.h"
+#include "google/protobuf/dynamic_message.h"
+#include "upb/def.hpp"
+
+upb_StringView descriptor = benchmarks_descriptor_proto_upbdefinit.descriptor;
+namespace protobuf = ::google::protobuf;
+
+/* A buffer big enough to parse descriptor.proto without going to heap. */
+char buf[65535];
+
+void CollectFileDescriptors(
+    const _upb_DefPool_Init* file,
+    std::vector<upb_StringView>& serialized_files,
+    absl::flat_hash_set<const _upb_DefPool_Init*>& seen) {
+  if (!seen.insert(file).second) return;
+  for (_upb_DefPool_Init** deps = file->deps; *deps; deps++) {
+    CollectFileDescriptors(*deps, serialized_files, seen);
+  }
+  serialized_files.push_back(file->descriptor);
+}
+
+static void BM_ArenaOneAlloc(benchmark::State& state) {
+  for (auto _ : state) {
+    upb_Arena* arena = upb_Arena_New();
+    upb_Arena_Malloc(arena, 1);
+    upb_Arena_Free(arena);
+  }
+}
+BENCHMARK(BM_ArenaOneAlloc);
+
+static void BM_ArenaInitialBlockOneAlloc(benchmark::State& state) {
+  for (auto _ : state) {
+    upb_Arena* arena = upb_Arena_Init(buf, sizeof(buf), NULL);
+    upb_Arena_Malloc(arena, 1);
+    upb_Arena_Free(arena);
+  }
+}
+BENCHMARK(BM_ArenaInitialBlockOneAlloc);
+
+enum LoadDescriptorMode {
+  NoLayout,
+  WithLayout,
+};
+
+// This function is mostly copied from upb/def.c, but it is modified to avoid
+// passing in the pre-generated mini-tables, in order to force upb to compute
+// them dynamically.  Generally you would never want to do this, but we want to
+// simulate the cost we would pay if we were loading these types purely from
+// descriptors, with no mini-tales available.
+bool LoadDefInit_BuildLayout(upb_DefPool* s, const _upb_DefPool_Init* init,
+                             size_t* bytes) {
+  _upb_DefPool_Init** deps = init->deps;
+  google_protobuf_FileDescriptorProto* file;
+  upb_Arena* arena;
+  upb_Status status;
+
+  upb_Status_Clear(&status);
+
+  if (upb_DefPool_FindFileByName(s, init->filename)) {
+    return true;
+  }
+
+  arena = upb_Arena_New();
+
+  for (; *deps; deps++) {
+    if (!LoadDefInit_BuildLayout(s, *deps, bytes)) goto err;
+  }
+
+  file = google_protobuf_FileDescriptorProto_parse_ex(
+      init->descriptor.data, init->descriptor.size, NULL,
+      kUpb_DecodeOption_AliasString, arena);
+  *bytes += init->descriptor.size;
+
+  if (!file) {
+    upb_Status_SetErrorFormat(
+        &status,
+        "Failed to parse compiled-in descriptor for file '%s'. This should "
+        "never happen.",
+        init->filename);
+    goto err;
+  }
+
+  // KEY DIFFERENCE: Here we pass in only the descriptor, and not the
+  // pre-generated minitables.
+  if (!upb_DefPool_AddFile(s, file, &status)) {
+    goto err;
+  }
+
+  upb_Arena_Free(arena);
+  return true;
+
+err:
+  fprintf(stderr,
+          "Error loading compiled-in descriptor for file '%s' (this should "
+          "never happen): %s\n",
+          init->filename, upb_Status_ErrorMessage(&status));
+  exit(1);
+}
+
+template <LoadDescriptorMode Mode>
+static void BM_LoadAdsDescriptor_Upb(benchmark::State& state) {
+  size_t bytes_per_iter = 0;
+  for (auto _ : state) {
+    upb::SymbolTable symtab;
+    if (Mode == NoLayout) {
+      google_ads_googleads_v7_services_SearchGoogleAdsRequest_getmsgdef(
+          symtab.ptr());
+      bytes_per_iter = _upb_DefPool_BytesLoaded(symtab.ptr());
+    } else {
+      bytes_per_iter = 0;
+      LoadDefInit_BuildLayout(
+          symtab.ptr(),
+          &google_ads_googleads_v7_services_google_ads_service_proto_upbdefinit,
+          &bytes_per_iter);
+    }
+  }
+  state.SetBytesProcessed(state.iterations() * bytes_per_iter);
+}
+BENCHMARK_TEMPLATE(BM_LoadAdsDescriptor_Upb, NoLayout);
+BENCHMARK_TEMPLATE(BM_LoadAdsDescriptor_Upb, WithLayout);
+
+template <LoadDescriptorMode Mode>
+static void BM_LoadAdsDescriptor_Proto2(benchmark::State& state) {
+  extern _upb_DefPool_Init
+      google_ads_googleads_v7_services_google_ads_service_proto_upbdefinit;
+  std::vector<upb_StringView> serialized_files;
+  absl::flat_hash_set<const _upb_DefPool_Init*> seen_files;
+  CollectFileDescriptors(
+      &google_ads_googleads_v7_services_google_ads_service_proto_upbdefinit,
+      serialized_files, seen_files);
+  size_t bytes_per_iter = 0;
+  for (auto _ : state) {
+    bytes_per_iter = 0;
+    protobuf::Arena arena;
+    protobuf::DescriptorPool pool;
+    for (auto file : serialized_files) {
+      protobuf::StringPiece input(file.data, file.size);
+      auto proto =
+          protobuf::Arena::CreateMessage<protobuf::FileDescriptorProto>(&arena);
+      bool ok = proto->ParseFrom<protobuf::MessageLite::kMergePartial>(input) &&
+                pool.BuildFile(*proto) != nullptr;
+      if (!ok) {
+        printf("Failed to add file.\n");
+        exit(1);
+      }
+      bytes_per_iter += input.size();
+    }
+
+    if (Mode == WithLayout) {
+      protobuf::DynamicMessageFactory factory;
+      const protobuf::Descriptor* d = pool.FindMessageTypeByName(
+          "google.ads.googleads.v7.services.SearchGoogleAdsResponse");
+      if (!d) {
+        printf("Failed to find descriptor.\n");
+        exit(1);
+      }
+      factory.GetPrototype(d);
+    }
+  }
+  state.SetBytesProcessed(state.iterations() * bytes_per_iter);
+}
+BENCHMARK_TEMPLATE(BM_LoadAdsDescriptor_Proto2, NoLayout);
+BENCHMARK_TEMPLATE(BM_LoadAdsDescriptor_Proto2, WithLayout);
+
+enum CopyStrings {
+  Copy,
+  Alias,
+};
+
+enum ArenaMode {
+  NoArena,
+  UseArena,
+  InitBlock,
+};
+
+template <ArenaMode AMode, CopyStrings Copy>
+static void BM_Parse_Upb_FileDesc(benchmark::State& state) {
+  for (auto _ : state) {
+    upb_Arena* arena;
+    if (AMode == InitBlock) {
+      arena = upb_Arena_Init(buf, sizeof(buf), NULL);
+    } else {
+      arena = upb_Arena_New();
+    }
+    upb_benchmark_FileDescriptorProto* set =
+        upb_benchmark_FileDescriptorProto_parse_ex(
+            descriptor.data, descriptor.size, NULL,
+            Copy == Alias ? kUpb_DecodeOption_AliasString : 0, arena);
+    if (!set) {
+      printf("Failed to parse.\n");
+      exit(1);
+    }
+    upb_Arena_Free(arena);
+  }
+  state.SetBytesProcessed(state.iterations() * descriptor.size);
+}
+BENCHMARK_TEMPLATE(BM_Parse_Upb_FileDesc, UseArena, Copy);
+BENCHMARK_TEMPLATE(BM_Parse_Upb_FileDesc, UseArena, Alias);
+BENCHMARK_TEMPLATE(BM_Parse_Upb_FileDesc, InitBlock, Copy);
+BENCHMARK_TEMPLATE(BM_Parse_Upb_FileDesc, InitBlock, Alias);
+
+template <ArenaMode AMode, class P>
+struct Proto2Factory;
+
+template <class P>
+struct Proto2Factory<NoArena, P> {
+ public:
+  P* GetProto() { return &proto_; }
+
+ private:
+  P proto_;
+};
+
+template <class P>
+struct Proto2Factory<UseArena, P> {
+ public:
+  P* GetProto() { return protobuf::Arena::CreateMessage<P>(&arena_); }
+
+ private:
+  protobuf::Arena arena_;
+};
+
+template <class P>
+struct Proto2Factory<InitBlock, P> {
+ public:
+  Proto2Factory() : arena_(GetOptions()) {}
+  P* GetProto() { return protobuf::Arena::CreateMessage<P>(&arena_); }
+
+ private:
+  protobuf::ArenaOptions GetOptions() {
+    protobuf::ArenaOptions opts;
+    opts.initial_block = buf;
+    opts.initial_block_size = sizeof(buf);
+    return opts;
+  }
+
+  protobuf::Arena arena_;
+};
+
+using FileDesc = ::upb_benchmark::FileDescriptorProto;
+using FileDescSV = ::upb_benchmark::sv::FileDescriptorProto;
+
+template <class P, ArenaMode AMode, CopyStrings kCopy>
+void BM_Parse_Proto2(benchmark::State& state) {
+  constexpr protobuf::MessageLite::ParseFlags kParseFlags =
+      kCopy == Copy
+          ? protobuf::MessageLite::ParseFlags::kMergePartial
+          : protobuf::MessageLite::ParseFlags::kMergePartialWithAliasing;
+  for (auto _ : state) {
+    Proto2Factory<AMode, P> proto_factory;
+    auto proto = proto_factory.GetProto();
+    protobuf::StringPiece input(descriptor.data, descriptor.size);
+    bool ok = proto->template ParseFrom<kParseFlags>(input);
+    if (!ok) {
+      printf("Failed to parse.\n");
+      exit(1);
+    }
+  }
+  state.SetBytesProcessed(state.iterations() * descriptor.size);
+}
+BENCHMARK_TEMPLATE(BM_Parse_Proto2, FileDesc, NoArena, Copy);
+BENCHMARK_TEMPLATE(BM_Parse_Proto2, FileDesc, UseArena, Copy);
+BENCHMARK_TEMPLATE(BM_Parse_Proto2, FileDesc, InitBlock, Copy);
+BENCHMARK_TEMPLATE(BM_Parse_Proto2, FileDescSV, InitBlock, Alias);
+
+static void BM_SerializeDescriptor_Proto2(benchmark::State& state) {
+  upb_benchmark::FileDescriptorProto proto;
+  proto.ParseFromArray(descriptor.data, descriptor.size);
+  for (auto _ : state) {
+    proto.SerializePartialToArray(buf, sizeof(buf));
+  }
+  state.SetBytesProcessed(state.iterations() * descriptor.size);
+}
+BENCHMARK(BM_SerializeDescriptor_Proto2);
+
+static void BM_SerializeDescriptor_Upb(benchmark::State& state) {
+  int64_t total = 0;
+  upb_Arena* arena = upb_Arena_New();
+  upb_benchmark_FileDescriptorProto* set =
+      upb_benchmark_FileDescriptorProto_parse(descriptor.data, descriptor.size,
+                                              arena);
+  if (!set) {
+    printf("Failed to parse.\n");
+    exit(1);
+  }
+  for (auto _ : state) {
+    upb_Arena* enc_arena = upb_Arena_Init(buf, sizeof(buf), NULL);
+    size_t size;
+    char* data =
+        upb_benchmark_FileDescriptorProto_serialize(set, enc_arena, &size);
+    if (!data) {
+      printf("Failed to serialize.\n");
+      exit(1);
+    }
+    total += size;
+  }
+  state.SetBytesProcessed(total);
+}
+BENCHMARK(BM_SerializeDescriptor_Upb);

benchmarks/build_defs.bzl

@@ -0,0 +1,87 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# copybara:insert_for_google3_begin
+# load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", _cc_proto_library="cc_proto_library")
+# copybara:insert_end
+
+# copybara:strip_for_google3_begin
+_cc_proto_library = native.cc_proto_library
+# copybara:strip_end
+
+def proto_library(**kwargs):
+    native.proto_library(
+        # copybara:insert_for_google3_begin
+        # cc_api_version = 2,
+        # copybara:insert_end
+        **kwargs,
+    )
+
+def tmpl_cc_binary(name, gen, args, replacements = [], **kwargs):
+    srcs = [name + ".cc"]
+    native.genrule(
+        name = name + "_gen_srcs",
+        tools = [gen],
+        outs = srcs,
+        cmd = "$(location " + gen + ") " + " ".join(args) + " > $@",
+    )
+
+    native.cc_binary(
+        # copybara:insert_for_google3_begin
+        # malloc="//base:system_malloc",
+        # features = ["-static_linking_mode"],
+        # copybara:insert_end
+        name = name,
+        srcs = srcs,
+        **kwargs,
+    )
+
+def cc_optimizefor_proto_library(name, srcs, outs, optimize_for):
+    if len(srcs) != 1:
+        fail("Currently srcs must have exactly 1 element")
+
+    native.genrule(
+        name = name + "_gen_proto",
+        srcs = srcs,
+        outs = outs,
+        cmd = "cp $< $@ && chmod a+w $@ && echo 'option optimize_for = " + optimize_for + ";' >> $@",
+    )
+
+    proto_library(
+        name = name + "_proto",
+        srcs = outs,
+    )
+
+    _cc_proto_library(
+        name = name,
+        deps = [":" + name + "_proto"],
+    )
+
+def expand_suffixes(vals, suffixes):
+    ret = []
+    for val in vals:
+        for suffix in suffixes:
+            ret.append(val + suffix)
+    return ret

benchmarks/compare.py

@@ -0,0 +1,117 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Benchmarks the current working directory against a given baseline.
+
+This script benchmarks both size and speed. Sample output:
+"""
+
+import contextlib
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+@contextlib.contextmanager
+def GitWorktree(commit):
+  tmpdir = tempfile.mkdtemp()
+  subprocess.run(['git', 'worktree', 'add', '-q', '-d', tmpdir, commit], check=True)
+  cwd = os.getcwd()
+  os.chdir(tmpdir)
+  try:
+    yield tmpdir
+  finally:
+    os.chdir(cwd)
+    subprocess.run(['git', 'worktree', 'remove', tmpdir], check=True)
+
+def Run(cmd):
+  subprocess.check_call(cmd, shell=True)
+
+def Benchmark(outbase, bench_cpu=True, runs=12, fasttable=False):
+  tmpfile = "/tmp/bench-output.json"
+  Run("rm -rf {}".format(tmpfile))
+  #Run("CC=clang bazel test ...")
+  if fasttable:
+    extra_args = " --//:fasttable_enabled=true"
+  else:
+    extra_args = ""
+
+  if bench_cpu:
+    Run("CC=clang bazel build -c opt --copt=-march=native benchmarks:benchmark" + extra_args)
+    Run("./bazel-bin/benchmarks/benchmark --benchmark_out_format=json --benchmark_out={} --benchmark_repetitions={} --benchmark_min_time=0.05 --benchmark_enable_random_interleaving=true".format(tmpfile, runs))
+    with open(tmpfile) as f:
+      bench_json = json.load(f)
+
+    # Translate into the format expected by benchstat.
+    txt_filename = outbase + ".txt"
+    with open(txt_filename, "w") as f:
+      for run in bench_json["benchmarks"]:
+        if run["run_type"] == "aggregate":
+          continue
+        name = run["name"]
+        name = name.replace(" ", "")
+        name = re.sub(r'^BM_', 'Benchmark', name)
+        values = (name, run["iterations"], run["cpu_time"])
+        print("{} {} {} ns/op".format(*values), file=f)
+    Run("sort {} -o {} ".format(txt_filename, txt_filename))
+
+  Run("CC=clang bazel build -c opt --copt=-g --copt=-march=native tests:conformance_upb" + extra_args)
+  Run("cp -f bazel-bin/tests/conformance_upb {}.bin".format(outbase))
+
+
+baseline = "main"
+bench_cpu = True
+fasttable = False
+
+if len(sys.argv) > 1:
+  baseline = sys.argv[1]
+
+  # Quickly verify that the baseline exists.
+  with GitWorktree(baseline):
+    pass
+
+# Benchmark our current directory first, since it's more likely to be broken.
+Benchmark("/tmp/new", bench_cpu, fasttable=fasttable)
+
+# Benchmark the baseline.
+with GitWorktree(baseline):
+  Benchmark("/tmp/old", bench_cpu, fasttable=fasttable)
+
+print()
+print()
+
+if bench_cpu:
+  Run("~/go/bin/benchstat /tmp/old.txt /tmp/new.txt")
+
+print()
+print()
+
+Run("objcopy --strip-debug /tmp/old.bin /tmp/old.bin.stripped")
+Run("objcopy --strip-debug /tmp/new.bin /tmp/new.bin.stripped")
+Run("~/code/bloaty/bloaty /tmp/new.bin.stripped -- /tmp/old.bin.stripped --debug-file=/tmp/old.bin --debug-file=/tmp/new.bin -d compileunits,symbols")

benchmarks/descriptor.proto

@@ -0,0 +1,905 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// The messages in this file describe the definitions found in .proto files.
+// A valid .proto file can be translated directly to a FileDescriptorProto
+// without any other information (e.g. without reading its imports).
+
+
+syntax = "proto2";
+
+package upb_benchmark;
+
+option go_package = "google.golang.org/protobuf/types/descriptorpb";
+option java_package = "com.google.protobuf";
+option java_outer_classname = "DescriptorProtos";
+option csharp_namespace = "Google.Protobuf.Reflection";
+option objc_class_prefix = "GPB";
+option cc_enable_arenas = true;
+
+// The protocol compiler can output a FileDescriptorSet containing the .proto
+// files it parses.
+message FileDescriptorSet {
+  repeated FileDescriptorProto file = 1;
+}
+
+// Describes a complete .proto file.
+message FileDescriptorProto {
+  optional string name = 1;     // file name, relative to root of source tree
+  optional string package = 2;  // e.g. "foo", "foo.bar", etc.
+
+  // Names of files imported by this file.
+  repeated string dependency = 3;
+  // Indexes of the public imported files in the dependency list above.
+  repeated int32 public_dependency = 10;
+  // Indexes of the weak imported files in the dependency list.
+  // For Google-internal migration only. Do not use.
+  repeated int32 weak_dependency = 11;
+
+  // All top-level definitions in this file.
+  repeated DescriptorProto message_type = 4;
+  repeated EnumDescriptorProto enum_type = 5;
+  repeated ServiceDescriptorProto service = 6;
+  repeated FieldDescriptorProto extension = 7;
+
+  optional FileOptions options = 8;
+
+  // This field contains optional information about the original source code.
+  // You may safely remove this entire field without harming runtime
+  // functionality of the descriptors -- the information is needed only by
+  // development tools.
+  optional SourceCodeInfo source_code_info = 9;
+
+  // The syntax of the proto file.
+  // The supported values are "proto2" and "proto3".
+  optional string syntax = 12;
+}
+
+// Describes a message type.
+message DescriptorProto {
+  optional string name = 1;
+
+  repeated FieldDescriptorProto field = 2;
+  repeated FieldDescriptorProto extension = 6;
+
+  repeated DescriptorProto nested_type = 3;
+  repeated EnumDescriptorProto enum_type = 4;
+
+  message ExtensionRange {
+    optional int32 start = 1;  // Inclusive.
+    optional int32 end = 2;    // Exclusive.
+
+    optional ExtensionRangeOptions options = 3;
+  }
+  repeated ExtensionRange extension_range = 5;
+
+  repeated OneofDescriptorProto oneof_decl = 8;
+
+  optional MessageOptions options = 7;
+
+  // Range of reserved tag numbers. Reserved tag numbers may not be used by
+  // fields or extension ranges in the same message. Reserved ranges may
+  // not overlap.
+  message ReservedRange {
+    optional int32 start = 1;  // Inclusive.
+    optional int32 end = 2;    // Exclusive.
+  }
+  repeated ReservedRange reserved_range = 9;
+  // Reserved field names, which may not be used by fields in the same message.
+  // A given name may only be reserved once.
+  repeated string reserved_name = 10;
+}
+
+message ExtensionRangeOptions {
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+// Describes a field within a message.
+message FieldDescriptorProto {
+  enum Type {
+    // 0 is reserved for errors.
+    // Order is weird for historical reasons.
+    TYPE_DOUBLE = 1;
+    TYPE_FLOAT = 2;
+    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT64 if
+    // negative values are likely.
+    TYPE_INT64 = 3;
+    TYPE_UINT64 = 4;
+    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT32 if
+    // negative values are likely.
+    TYPE_INT32 = 5;
+    TYPE_FIXED64 = 6;
+    TYPE_FIXED32 = 7;
+    TYPE_BOOL = 8;
+    TYPE_STRING = 9;
+    // Tag-delimited aggregate.
+    // Group type is deprecated and not supported in proto3. However, Proto3
+    // implementations should still be able to parse the group wire format and
+    // treat group fields as unknown fields.
+    TYPE_GROUP = 10;
+    TYPE_MESSAGE = 11;  // Length-delimited aggregate.
+
+    // New in version 2.
+    TYPE_BYTES = 12;
+    TYPE_UINT32 = 13;
+    TYPE_ENUM = 14;
+    TYPE_SFIXED32 = 15;
+    TYPE_SFIXED64 = 16;
+    TYPE_SINT32 = 17;  // Uses ZigZag encoding.
+    TYPE_SINT64 = 18;  // Uses ZigZag encoding.
+  }
+
+  enum Label {
+    // 0 is reserved for errors
+    LABEL_OPTIONAL = 1;
+    LABEL_REQUIRED = 2;
+    LABEL_REPEATED = 3;
+  }
+
+  optional string name = 1;
+  optional int32 number = 3;
+  optional Label label = 4;
+
+  // If type_name is set, this need not be set.  If both this and type_name
+  // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP.
+  optional Type type = 5;
+
+  // For message and enum types, this is the name of the type.  If the name
+  // starts with a '.', it is fully-qualified.  Otherwise, C++-like scoping
+  // rules are used to find the type (i.e. first the nested types within this
+  // message are searched, then within the parent, on up to the root
+  // namespace).
+  optional string type_name = 6;
+
+  // For extensions, this is the name of the type being extended.  It is
+  // resolved in the same manner as type_name.
+  optional string extendee = 2;
+
+  // For numeric types, contains the original text representation of the value.
+  // For booleans, "true" or "false".
+  // For strings, contains the default text contents (not escaped in any way).
+  // For bytes, contains the C escaped value.  All bytes >= 128 are escaped.
+  // TODO(kenton):  Base-64 encode?
+  optional string default_value = 7;
+
+  // If set, gives the index of a oneof in the containing type's oneof_decl
+  // list.  This field is a member of that oneof.
+  optional int32 oneof_index = 9;
+
+  // JSON name of this field. The value is set by protocol compiler. If the
+  // user has set a "json_name" option on this field, that option's value
+  // will be used. Otherwise, it's deduced from the field's name by converting
+  // it to camelCase.
+  optional string json_name = 10;
+
+  optional FieldOptions options = 8;
+
+  // If true, this is a proto3 "optional". When a proto3 field is optional, it
+  // tracks presence regardless of field type.
+  //
+  // When proto3_optional is true, this field must be belong to a oneof to
+  // signal to old proto3 clients that presence is tracked for this field. This
+  // oneof is known as a "synthetic" oneof, and this field must be its sole
+  // member (each proto3 optional field gets its own synthetic oneof). Synthetic
+  // oneofs exist in the descriptor only, and do not generate any API. Synthetic
+  // oneofs must be ordered after all "real" oneofs.
+  //
+  // For message fields, proto3_optional doesn't create any semantic change,
+  // since non-repeated message fields always track presence. However it still
+  // indicates the semantic detail of whether the user wrote "optional" or not.
+  // This can be useful for round-tripping the .proto file. For consistency we
+  // give message fields a synthetic oneof also, even though it is not required
+  // to track presence. This is especially important because the parser can't
+  // tell if a field is a message or an enum, so it must always create a
+  // synthetic oneof.
+  //
+  // Proto2 optional fields do not set this flag, because they already indicate
+  // optional with `LABEL_OPTIONAL`.
+  optional bool proto3_optional = 17;
+}
+
+// Describes a oneof.
+message OneofDescriptorProto {
+  optional string name = 1;
+  optional OneofOptions options = 2;
+}
+
+// Describes an enum type.
+message EnumDescriptorProto {
+  optional string name = 1;
+
+  repeated EnumValueDescriptorProto value = 2;
+
+  optional EnumOptions options = 3;
+
+  // Range of reserved numeric values. Reserved values may not be used by
+  // entries in the same enum. Reserved ranges may not overlap.
+  //
+  // Note that this is distinct from DescriptorProto.ReservedRange in that it
+  // is inclusive such that it can appropriately represent the entire int32
+  // domain.
+  message EnumReservedRange {
+    optional int32 start = 1;  // Inclusive.
+    optional int32 end = 2;    // Inclusive.
+  }
+
+  // Range of reserved numeric values. Reserved numeric values may not be used
+  // by enum values in the same enum declaration. Reserved ranges may not
+  // overlap.
+  repeated EnumReservedRange reserved_range = 4;
+
+  // Reserved enum value names, which may not be reused. A given name may only
+  // be reserved once.
+  repeated string reserved_name = 5;
+}
+
+// Describes a value within an enum.
+message EnumValueDescriptorProto {
+  optional string name = 1;
+  optional int32 number = 2;
+
+  optional EnumValueOptions options = 3;
+}
+
+// Describes a service.
+message ServiceDescriptorProto {
+  optional string name = 1;
+  repeated MethodDescriptorProto method = 2;
+
+  optional ServiceOptions options = 3;
+}
+
+// Describes a method of a service.
+message MethodDescriptorProto {
+  optional string name = 1;
+
+  // Input and output type names.  These are resolved in the same way as
+  // FieldDescriptorProto.type_name, but must refer to a message type.
+  optional string input_type = 2;
+  optional string output_type = 3;
+
+  optional MethodOptions options = 4;
+
+  // Identifies if client streams multiple client messages
+  optional bool client_streaming = 5 [default = false];
+  // Identifies if server streams multiple server messages
+  optional bool server_streaming = 6 [default = false];
+}
+
+
+// ===================================================================
+// Options
+
+// Each of the definitions above may have "options" attached.  These are
+// just annotations which may cause code to be generated slightly differently
+// or may contain hints for code that manipulates protocol messages.
+//
+// Clients may define custom options as extensions of the *Options messages.
+// These extensions may not yet be known at parsing time, so the parser cannot
+// store the values in them.  Instead it stores them in a field in the *Options
+// message called uninterpreted_option. This field must have the same name
+// across all *Options messages. We then use this field to populate the
+// extensions when we build a descriptor, at which point all protos have been
+// parsed and so all extensions are known.
+//
+// Extension numbers for custom options may be chosen as follows:
+// * For options which will only be used within a single application or
+//   organization, or for experimental options, use field numbers 50000
+//   through 99999.  It is up to you to ensure that you do not use the
+//   same number for multiple options.
+// * For options which will be published and used publicly by multiple
+//   independent entities, e-mail protobuf-global-extension-registry@google.com
+//   to reserve extension numbers. Simply provide your project name (e.g.
+//   Objective-C plugin) and your project website (if available) -- there's no
+//   need to explain how you intend to use them. Usually you only need one
+//   extension number. You can declare multiple options with only one extension
+//   number by putting them in a sub-message. See the Custom Options section of
+//   the docs for examples:
+//   https://developers.google.com/protocol-buffers/docs/proto#options
+//   If this turns out to be popular, a web service will be set up
+//   to automatically assign option numbers.
+
+message FileOptions {
+
+  // Sets the Java package where classes generated from this .proto will be
+  // placed.  By default, the proto package is used, but this is often
+  // inappropriate because proto packages do not normally start with backwards
+  // domain names.
+  optional string java_package = 1;
+
+
+  // If set, all the classes from the .proto file are wrapped in a single
+  // outer class with the given name.  This applies to both Proto1
+  // (equivalent to the old "--one_java_file" option) and Proto2 (where
+  // a .proto always translates to a single class, but you may want to
+  // explicitly choose the class name).
+  optional string java_outer_classname = 8;
+
+  // If set true, then the Java code generator will generate a separate .java
+  // file for each top-level message, enum, and service defined in the .proto
+  // file.  Thus, these types will *not* be nested inside the outer class
+  // named by java_outer_classname.  However, the outer class will still be
+  // generated to contain the file's getDescriptor() method as well as any
+  // top-level extensions defined in the file.
+  optional bool java_multiple_files = 10 [default = false];
+
+  // This option does nothing.
+  optional bool java_generate_equals_and_hash = 20 [deprecated=true];
+
+  // If set true, then the Java2 code generator will generate code that
+  // throws an exception whenever an attempt is made to assign a non-UTF-8
+  // byte sequence to a string field.
+  // Message reflection will do the same.
+  // However, an extension field still accepts non-UTF-8 byte sequences.
+  // This option has no effect on when used with the lite runtime.
+  optional bool java_string_check_utf8 = 27 [default = false];
+
+
+  // Generated classes can be optimized for speed or code size.
+  enum OptimizeMode {
+    SPEED = 1;         // Generate complete code for parsing, serialization,
+                       // etc.
+    CODE_SIZE = 2;     // Use ReflectionOps to implement these methods.
+    LITE_RUNTIME = 3;  // Generate code using MessageLite and the lite runtime.
+  }
+  optional OptimizeMode optimize_for = 9 [default = SPEED];
+
+  // Sets the Go package where structs generated from this .proto will be
+  // placed. If omitted, the Go package will be derived from the following:
+  //   - The basename of the package import path, if provided.
+  //   - Otherwise, the package statement in the .proto file, if present.
+  //   - Otherwise, the basename of the .proto file, without extension.
+  optional string go_package = 11;
+
+
+
+
+  // Should generic services be generated in each language?  "Generic" services
+  // are not specific to any particular RPC system.  They are generated by the
+  // main code generators in each language (without additional plugins).
+  // Generic services were the only kind of service generation supported by
+  // early versions of google.protobuf.
+  //
+  // Generic services are now considered deprecated in favor of using plugins
+  // that generate code specific to your particular RPC system.  Therefore,
+  // these default to false.  Old code which depends on generic services should
+  // explicitly set them to true.
+  optional bool cc_generic_services = 16 [default = false];
+  optional bool java_generic_services = 17 [default = false];
+  optional bool py_generic_services = 18 [default = false];
+  optional bool php_generic_services = 42 [default = false];
+
+  // Is this file deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for everything in the file, or it will be completely ignored; in the very
+  // least, this is a formalization for deprecating files.
+  optional bool deprecated = 23 [default = false];
+
+  // Enables the use of arenas for the proto messages in this file. This applies
+  // only to generated classes for C++.
+  optional bool cc_enable_arenas = 31 [default = true];
+
+
+  // Sets the objective c class prefix which is prepended to all objective c
+  // generated classes from this .proto. There is no default.
+  optional string objc_class_prefix = 36;
+
+  // Namespace for generated classes; defaults to the package.
+  optional string csharp_namespace = 37;
+
+  // By default Swift generators will take the proto package and CamelCase it
+  // replacing '.' with underscore and use that to prefix the types/symbols
+  // defined. When this options is provided, they will use this value instead
+  // to prefix the types/symbols defined.
+  optional string swift_prefix = 39;
+
+  // Sets the php class prefix which is prepended to all php generated classes
+  // from this .proto. Default is empty.
+  optional string php_class_prefix = 40;
+
+  // Use this option to change the namespace of php generated classes. Default
+  // is empty. When this option is empty, the package name will be used for
+  // determining the namespace.
+  optional string php_namespace = 41;
+
+  // Use this option to change the namespace of php generated metadata classes.
+  // Default is empty. When this option is empty, the proto file name will be
+  // used for determining the namespace.
+  optional string php_metadata_namespace = 44;
+
+  // Use this option to change the package of ruby generated classes. Default
+  // is empty. When this option is not set, the package name will be used for
+  // determining the ruby package.
+  optional string ruby_package = 45;
+
+
+  // The parser stores options it doesn't recognize here.
+  // See the documentation for the "Options" section above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message.
+  // See the documentation for the "Options" section above.
+  extensions 1000 to max;
+
+  reserved 38;
+}
+
+message MessageOptions {
+  // Set true to use the old proto1 MessageSet wire format for extensions.
+  // This is provided for backwards-compatibility with the MessageSet wire
+  // format.  You should not use this for any other reason:  It's less
+  // efficient, has fewer features, and is more complicated.
+  //
+  // The message must be defined exactly as follows:
+  //   message Foo {
+  //     option message_set_wire_format = true;
+  //     extensions 4 to max;
+  //   }
+  // Note that the message cannot have any defined fields; MessageSets only
+  // have extensions.
+  //
+  // All extensions of your type must be singular messages; e.g. they cannot
+  // be int32s, enums, or repeated messages.
+  //
+  // Because this is an option, the above two restrictions are not enforced by
+  // the protocol compiler.
+  optional bool message_set_wire_format = 1 [default = false];
+
+  // Disables the generation of the standard "descriptor()" accessor, which can
+  // conflict with a field of the same name.  This is meant to make migration
+  // from proto1 easier; new code should avoid fields named "descriptor".
+  optional bool no_standard_descriptor_accessor = 2 [default = false];
+
+  // Is this message deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the message, or it will be completely ignored; in the very least,
+  // this is a formalization for deprecating messages.
+  optional bool deprecated = 3 [default = false];
+
+  // Whether the message is an automatically generated map entry type for the
+  // maps field.
+  //
+  // For maps fields:
+  //     map<KeyType, ValueType> map_field = 1;
+  // The parsed descriptor looks like:
+  //     message MapFieldEntry {
+  //         option map_entry = true;
+  //         optional KeyType key = 1;
+  //         optional ValueType value = 2;
+  //     }
+  //     repeated MapFieldEntry map_field = 1;
+  //
+  // Implementations may choose not to generate the map_entry=true message, but
+  // use a native map in the target language to hold the keys and values.
+  // The reflection APIs in such implementations still need to work as
+  // if the field is a repeated message field.
+  //
+  // NOTE: Do not set the option in .proto files. Always use the maps syntax
+  // instead. The option should only be implicitly set by the proto compiler
+  // parser.
+  optional bool map_entry = 7;
+
+  reserved 8;  // javalite_serializable
+  reserved 9;  // javanano_as_lite
+
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message FieldOptions {
+  // The ctype option instructs the C++ code generator to use a different
+  // representation of the field than it normally would.  See the specific
+  // options below.  This option is not yet implemented in the open source
+  // release -- sorry, we'll try to include it in a future version!
+  optional CType ctype = 1 [default = STRING];
+  enum CType {
+    // Default mode.
+    STRING = 0;
+
+    CORD = 1;
+
+    STRING_PIECE = 2;
+  }
+  // The packed option can be enabled for repeated primitive fields to enable
+  // a more efficient representation on the wire. Rather than repeatedly
+  // writing the tag and type for each element, the entire array is encoded as
+  // a single length-delimited blob. In proto3, only explicit setting it to
+  // false will avoid using packed encoding.
+  optional bool packed = 2;
+
+  // The jstype option determines the JavaScript type used for values of the
+  // field.  The option is permitted only for 64 bit integral and fixed types
+  // (int64, uint64, sint64, fixed64, sfixed64).  A field with jstype JS_STRING
+  // is represented as JavaScript string, which avoids loss of precision that
+  // can happen when a large value is converted to a floating point JavaScript.
+  // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to
+  // use the JavaScript "number" type.  The behavior of the default option
+  // JS_NORMAL is implementation dependent.
+  //
+  // This option is an enum to permit additional types to be added, e.g.
+  // goog.math.Integer.
+  optional JSType jstype = 6 [default = JS_NORMAL];
+  enum JSType {
+    // Use the default type.
+    JS_NORMAL = 0;
+
+    // Use JavaScript strings.
+    JS_STRING = 1;
+
+    // Use JavaScript numbers.
+    JS_NUMBER = 2;
+  }
+
+  // Should this field be parsed lazily?  Lazy applies only to message-type
+  // fields.  It means that when the outer message is initially parsed, the
+  // inner message's contents will not be parsed but instead stored in encoded
+  // form.  The inner message will actually be parsed when it is first accessed.
+  //
+  // This is only a hint.  Implementations are free to choose whether to use
+  // eager or lazy parsing regardless of the value of this option.  However,
+  // setting this option true suggests that the protocol author believes that
+  // using lazy parsing on this field is worth the additional bookkeeping
+  // overhead typically needed to implement it.
+  //
+  // This option does not affect the public interface of any generated code;
+  // all method signatures remain the same.  Furthermore, thread-safety of the
+  // interface is not affected by this option; const methods remain safe to
+  // call from multiple threads concurrently, while non-const methods continue
+  // to require exclusive access.
+  //
+  //
+  // Note that implementations may choose not to check required fields within
+  // a lazy sub-message.  That is, calling IsInitialized() on the outer message
+  // may return true even if the inner message has missing required fields.
+  // This is necessary because otherwise the inner message would have to be
+  // parsed in order to perform the check, defeating the purpose of lazy
+  // parsing.  An implementation which chooses not to check required fields
+  // must be consistent about it.  That is, for any particular sub-message, the
+  // implementation must either *always* check its required fields, or *never*
+  // check its required fields, regardless of whether or not the message has
+  // been parsed.
+  optional bool lazy = 5 [default = false];
+
+  // Is this field deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for accessors, or it will be completely ignored; in the very least, this
+  // is a formalization for deprecating fields.
+  optional bool deprecated = 3 [default = false];
+
+  // For Google-internal migration only. Do not use.
+  optional bool weak = 10 [default = false];
+
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+
+  reserved 4;  // removed jtype
+}
+
+message OneofOptions {
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message EnumOptions {
+
+  // Set this option to true to allow mapping different tag names to the same
+  // value.
+  optional bool allow_alias = 2;
+
+  // Is this enum deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the enum, or it will be completely ignored; in the very least, this
+  // is a formalization for deprecating enums.
+  optional bool deprecated = 3 [default = false];
+
+  reserved 5;  // javanano_as_lite
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message EnumValueOptions {
+  // Is this enum value deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the enum value, or it will be completely ignored; in the very least,
+  // this is a formalization for deprecating enum values.
+  optional bool deprecated = 1 [default = false];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message ServiceOptions {
+
+  // Note:  Field numbers 1 through 32 are reserved for Google's internal RPC
+  //   framework.  We apologize for hoarding these numbers to ourselves, but
+  //   we were already using them long before we decided to release Protocol
+  //   Buffers.
+
+  // Is this service deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the service, or it will be completely ignored; in the very least,
+  // this is a formalization for deprecating services.
+  optional bool deprecated = 33 [default = false];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message MethodOptions {
+
+  // Note:  Field numbers 1 through 32 are reserved for Google's internal RPC
+  //   framework.  We apologize for hoarding these numbers to ourselves, but
+  //   we were already using them long before we decided to release Protocol
+  //   Buffers.
+
+  // Is this method deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the method, or it will be completely ignored; in the very least,
+  // this is a formalization for deprecating methods.
+  optional bool deprecated = 33 [default = false];
+
+  // Is this method side-effect-free (or safe in HTTP parlance), or idempotent,
+  // or neither? HTTP based RPC implementation may choose GET verb for safe
+  // methods, and PUT verb for idempotent methods instead of the default POST.
+  enum IdempotencyLevel {
+    IDEMPOTENCY_UNKNOWN = 0;
+    NO_SIDE_EFFECTS = 1;  // implies idempotent
+    IDEMPOTENT = 2;       // idempotent, but may have side effects
+  }
+  optional IdempotencyLevel idempotency_level = 34
+      [default = IDEMPOTENCY_UNKNOWN];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+
+// A message representing a option the parser does not recognize. This only
+// appears in options protos created by the compiler::Parser class.
+// DescriptorPool resolves these when building Descriptor objects. Therefore,
+// options protos in descriptor objects (e.g. returned by Descriptor::options(),
+// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions
+// in them.
+message UninterpretedOption {
+  // The name of the uninterpreted option.  Each string represents a segment in
+  // a dot-separated name.  is_extension is true iff a segment represents an
+  // extension (denoted with parentheses in options specs in .proto files).
+  // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents
+  // "foo.(bar.baz).qux".
+  message NamePart {
+    optional string name_part = 1;
+    optional bool is_extension = 2;
+  }
+  repeated NamePart name = 2;
+
+  // The value of the uninterpreted option, in whatever type the tokenizer
+  // identified it as during parsing. Exactly one of these should be set.
+  optional string identifier_value = 3;
+  optional uint64 positive_int_value = 4;
+  optional int64 negative_int_value = 5;
+  optional double double_value = 6;
+  optional bytes string_value = 7;
+  optional string aggregate_value = 8;
+}
+
+// ===================================================================
+// Optional source code info
+
+// Encapsulates information about the original source file from which a
+// FileDescriptorProto was generated.
+message SourceCodeInfo {
+  // A Location identifies a piece of source code in a .proto file which
+  // corresponds to a particular definition.  This information is intended
+  // to be useful to IDEs, code indexers, documentation generators, and similar
+  // tools.
+  //
+  // For example, say we have a file like:
+  //   message Foo {
+  //     optional string foo = 1;
+  //   }
+  // Let's look at just the field definition:
+  //   optional string foo = 1;
+  //   ^       ^^     ^^  ^  ^^^
+  //   a       bc     de  f  ghi
+  // We have the following locations:
+  //   span   path               represents
+  //   [a,i)  [ 4, 0, 2, 0 ]     The whole field definition.
+  //   [a,b)  [ 4, 0, 2, 0, 4 ]  The label (optional).
+  //   [c,d)  [ 4, 0, 2, 0, 5 ]  The type (string).
+  //   [e,f)  [ 4, 0, 2, 0, 1 ]  The name (foo).
+  //   [g,h)  [ 4, 0, 2, 0, 3 ]  The number (1).
+  //
+  // Notes:
+  // - A location may refer to a repeated field itself (i.e. not to any
+  //   particular index within it).  This is used whenever a set of elements are
+  //   logically enclosed in a single code segment.  For example, an entire
+  //   extend block (possibly containing multiple extension definitions) will
+  //   have an outer location whose path refers to the "extensions" repeated
+  //   field without an index.
+  // - Multiple locations may have the same path.  This happens when a single
+  //   logical declaration is spread out across multiple places.  The most
+  //   obvious example is the "extend" block again -- there may be multiple
+  //   extend blocks in the same scope, each of which will have the same path.
+  // - A location's span is not always a subset of its parent's span.  For
+  //   example, the "extendee" of an extension declaration appears at the
+  //   beginning of the "extend" block and is shared by all extensions within
+  //   the block.
+  // - Just because a location's span is a subset of some other location's span
+  //   does not mean that it is a descendant.  For example, a "group" defines
+  //   both a type and a field in a single declaration.  Thus, the locations
+  //   corresponding to the type and field and their components will overlap.
+  // - Code which tries to interpret locations should probably be designed to
+  //   ignore those that it doesn't understand, as more types of locations could
+  //   be recorded in the future.
+  repeated Location location = 1;
+  message Location {
+    // Identifies which part of the FileDescriptorProto was defined at this
+    // location.
+    //
+    // Each element is a field number or an index.  They form a path from
+    // the root FileDescriptorProto to the place where the definition.  For
+    // example, this path:
+    //   [ 4, 3, 2, 7, 1 ]
+    // refers to:
+    //   file.message_type(3)  // 4, 3
+    //       .field(7)         // 2, 7
+    //       .name()           // 1
+    // This is because FileDescriptorProto.message_type has field number 4:
+    //   repeated DescriptorProto message_type = 4;
+    // and DescriptorProto.field has field number 2:
+    //   repeated FieldDescriptorProto field = 2;
+    // and FieldDescriptorProto.name has field number 1:
+    //   optional string name = 1;
+    //
+    // Thus, the above path gives the location of a field name.  If we removed
+    // the last element:
+    //   [ 4, 3, 2, 7 ]
+    // this path refers to the whole field declaration (from the beginning
+    // of the label to the terminating semicolon).
+    repeated int32 path = 1 [packed = true];
+
+    // Always has exactly three or four elements: start line, start column,
+    // end line (optional, otherwise assumed same as start line), end column.
+    // These are packed into a single field for efficiency.  Note that line
+    // and column numbers are zero-based -- typically you will want to add
+    // 1 to each before displaying to a user.
+    repeated int32 span = 2 [packed = true];
+
+    // If this SourceCodeInfo represents a complete declaration, these are any
+    // comments appearing before and after the declaration which appear to be
+    // attached to the declaration.
+    //
+    // A series of line comments appearing on consecutive lines, with no other
+    // tokens appearing on those lines, will be treated as a single comment.
+    //
+    // leading_detached_comments will keep paragraphs of comments that appear
+    // before (but not connected to) the current element. Each paragraph,
+    // separated by empty lines, will be one comment element in the repeated
+    // field.
+    //
+    // Only the comment content is provided; comment markers (e.g. //) are
+    // stripped out.  For block comments, leading whitespace and an asterisk
+    // will be stripped from the beginning of each line other than the first.
+    // Newlines are included in the output.
+    //
+    // Examples:
+    //
+    //   optional int32 foo = 1;  // Comment attached to foo.
+    //   // Comment attached to bar.
+    //   optional int32 bar = 2;
+    //
+    //   optional string baz = 3;
+    //   // Comment attached to baz.
+    //   // Another line attached to baz.
+    //
+    //   // Comment attached to qux.
+    //   //
+    //   // Another line attached to qux.
+    //   optional double qux = 4;
+    //
+    //   // Detached comment for corge. This is not leading or trailing comments
+    //   // to qux or corge because there are blank lines separating it from
+    //   // both.
+    //
+    //   // Detached comment for corge paragraph 2.
+    //
+    //   optional string corge = 5;
+    //   /* Block comment attached
+    //    * to corge.  Leading asterisks
+    //    * will be removed. */
+    //   /* Block comment attached to
+    //    * grault. */
+    //   optional int32 grault = 6;
+    //
+    //   // ignored detached comments.
+    optional string leading_comments = 3;
+    optional string trailing_comments = 4;
+    repeated string leading_detached_comments = 6;
+  }
+}
+
+// Describes the relationship between generated code and its original source
+// file. A GeneratedCodeInfo message is associated with only one generated
+// source file, but may contain references to different source .proto files.
+message GeneratedCodeInfo {
+  // An Annotation connects some span of text in generated code to an element
+  // of its generating .proto file.
+  repeated Annotation annotation = 1;
+  message Annotation {
+    // Identifies the element in the original source .proto file. This field
+    // is formatted the same as SourceCodeInfo.Location.path.
+    repeated int32 path = 1 [packed = true];
+
+    // Identifies the filesystem path to the original source .proto.
+    optional string source_file = 2;
+
+    // Identifies the starting offset in bytes in the generated code
+    // that relates to the identified object.
+    optional int32 begin = 3;
+
+    // Identifies the ending offset in bytes in the generated code that
+    // relates to the identified offset. The end offset should be one past
+    // the last relevant byte (so the length of the text = end - begin).
+    optional int32 end = 4;
+  }
+}

benchmarks/descriptor_sv.proto

@@ -0,0 +1,890 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+//
+// The messages in this file describe the definitions found in .proto files.
+// A valid .proto file can be translated directly to a FileDescriptorProto
+// without any other information (e.g. without reading its imports).
+
+syntax = "proto2";
+
+package upb_benchmark.sv;
+
+option go_package = "google.golang.org/protobuf/types/descriptorpb";
+option java_package = "com.google.protobuf";
+option java_outer_classname = "DescriptorProtos";
+option csharp_namespace = "Google.Protobuf.Reflection";
+option objc_class_prefix = "GPB";
+option cc_enable_arenas = true;
+
+// The protocol compiler can output a FileDescriptorSet containing the .proto
+// files it parses.
+message FileDescriptorSet {
+  repeated FileDescriptorProto file = 1;
+}
+
+// Describes a complete .proto file.
+message FileDescriptorProto {
+  optional string name = 1
+      [ctype = STRING_PIECE];  // file name, relative to root of source tree
+  optional string package = 2
+      [ctype = STRING_PIECE];  // e.g. "foo", "foo.bar", etc.
+
+  // Names of files imported by this file.
+  repeated string dependency = 3 [ctype = STRING_PIECE];
+  // Indexes of the public imported files in the dependency list above.
+  repeated int32 public_dependency = 10;
+  // Indexes of the weak imported files in the dependency list.
+  // For Google-internal migration only. Do not use.
+  repeated int32 weak_dependency = 11;
+
+  // All top-level definitions in this file.
+  repeated DescriptorProto message_type = 4;
+  repeated EnumDescriptorProto enum_type = 5;
+  repeated ServiceDescriptorProto service = 6;
+  repeated FieldDescriptorProto extension = 7;
+
+  optional FileOptions options = 8;
+
+  // This field contains optional information about the original source code.
+  // You may safely remove this entire field without harming runtime
+  // functionality of the descriptors -- the information is needed only by
+  // development tools.
+  optional SourceCodeInfo source_code_info = 9;
+
+  // The syntax of the proto file.
+  // The supported values are "proto2" and "proto3".
+  optional string syntax = 12 [ctype = STRING_PIECE];
+}
+
+// Describes a message type.
+message DescriptorProto {
+  optional string name = 1 [ctype = STRING_PIECE];
+
+  repeated FieldDescriptorProto field = 2;
+  repeated FieldDescriptorProto extension = 6;
+
+  repeated DescriptorProto nested_type = 3;
+  repeated EnumDescriptorProto enum_type = 4;
+
+  message ExtensionRange {
+    optional int32 start = 1;  // Inclusive.
+    optional int32 end = 2;    // Exclusive.
+
+    optional ExtensionRangeOptions options = 3;
+  }
+  repeated ExtensionRange extension_range = 5;
+
+  repeated OneofDescriptorProto oneof_decl = 8;
+
+  optional MessageOptions options = 7;
+
+  // Range of reserved tag numbers. Reserved tag numbers may not be used by
+  // fields or extension ranges in the same message. Reserved ranges may
+  // not overlap.
+  message ReservedRange {
+    optional int32 start = 1;  // Inclusive.
+    optional int32 end = 2;    // Exclusive.
+  }
+  repeated ReservedRange reserved_range = 9;
+  // Reserved field names, which may not be used by fields in the same message.
+  // A given name may only be reserved once.
+  repeated string reserved_name = 10 [ctype = STRING_PIECE];
+}
+
+message ExtensionRangeOptions {
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+// Describes a field within a message.
+message FieldDescriptorProto {
+  enum Type {
+    // 0 is reserved for errors.
+    // Order is weird for historical reasons.
+    TYPE_DOUBLE = 1;
+    TYPE_FLOAT = 2;
+    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT64 if
+    // negative values are likely.
+    TYPE_INT64 = 3;
+    TYPE_UINT64 = 4;
+    // Not ZigZag encoded.  Negative numbers take 10 bytes.  Use TYPE_SINT32 if
+    // negative values are likely.
+    TYPE_INT32 = 5;
+    TYPE_FIXED64 = 6;
+    TYPE_FIXED32 = 7;
+    TYPE_BOOL = 8;
+    TYPE_STRING = 9;
+    // Tag-delimited aggregate.
+    // Group type is deprecated and not supported in proto3. However, Proto3
+    // implementations should still be able to parse the group wire format and
+    // treat group fields as unknown fields.
+    TYPE_GROUP = 10;
+    TYPE_MESSAGE = 11;  // Length-delimited aggregate.
+
+    // New in version 2.
+    TYPE_BYTES = 12;
+    TYPE_UINT32 = 13;
+    TYPE_ENUM = 14;
+    TYPE_SFIXED32 = 15;
+    TYPE_SFIXED64 = 16;
+    TYPE_SINT32 = 17;  // Uses ZigZag encoding.
+    TYPE_SINT64 = 18;  // Uses ZigZag encoding.
+  }
+
+  enum Label {
+    // 0 is reserved for errors
+    LABEL_OPTIONAL = 1;
+    LABEL_REQUIRED = 2;
+    LABEL_REPEATED = 3;
+  }
+
+  optional string name = 1 [ctype = STRING_PIECE];
+  optional int32 number = 3;
+  optional Label label = 4;
+
+  // If type_name is set, this need not be set.  If both this and type_name
+  // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP.
+  optional Type type = 5;
+
+  // For message and enum types, this is the name of the type.  If the name
+  // starts with a '.', it is fully-qualified.  Otherwise, C++-like scoping
+  // rules are used to find the type (i.e. first the nested types within this
+  // message are searched, then within the parent, on up to the root
+  // namespace).
+  optional string type_name = 6 [ctype = STRING_PIECE];
+
+  // For extensions, this is the name of the type being extended.  It is
+  // resolved in the same manner as type_name.
+  optional string extendee = 2 [ctype = STRING_PIECE];
+
+  // For numeric types, contains the original text representation of the value.
+  // For booleans, "true" or "false".
+  // For strings, contains the default text contents (not escaped in any way).
+  // For bytes, contains the C escaped value.  All bytes >= 128 are escaped.
+  // TODO(kenton):  Base-64 encode?
+  optional string default_value = 7 [ctype = STRING_PIECE];
+
+  // If set, gives the index of a oneof in the containing type's oneof_decl
+  // list.  This field is a member of that oneof.
+  optional int32 oneof_index = 9;
+
+  // JSON name of this field. The value is set by protocol compiler. If the
+  // user has set a "json_name" option on this field, that option's value
+  // will be used. Otherwise, it's deduced from the field's name by converting
+  // it to camelCase.
+  optional string json_name = 10 [ctype = STRING_PIECE];
+
+  optional FieldOptions options = 8;
+
+  // If true, this is a proto3 "optional". When a proto3 field is optional, it
+  // tracks presence regardless of field type.
+  //
+  // When proto3_optional is true, this field must be belong to a oneof to
+  // signal to old proto3 clients that presence is tracked for this field. This
+  // oneof is known as a "synthetic" oneof, and this field must be its sole
+  // member (each proto3 optional field gets its own synthetic oneof). Synthetic
+  // oneofs exist in the descriptor only, and do not generate any API. Synthetic
+  // oneofs must be ordered after all "real" oneofs.
+  //
+  // For message fields, proto3_optional doesn't create any semantic change,
+  // since non-repeated message fields always track presence. However it still
+  // indicates the semantic detail of whether the user wrote "optional" or not.
+  // This can be useful for round-tripping the .proto file. For consistency we
+  // give message fields a synthetic oneof also, even though it is not required
+  // to track presence. This is especially important because the parser can't
+  // tell if a field is a message or an enum, so it must always create a
+  // synthetic oneof.
+  //
+  // Proto2 optional fields do not set this flag, because they already indicate
+  // optional with `LABEL_OPTIONAL`.
+  optional bool proto3_optional = 17;
+}
+
+// Describes a oneof.
+message OneofDescriptorProto {
+  optional string name = 1 [ctype = STRING_PIECE];
+  optional OneofOptions options = 2;
+}
+
+// Describes an enum type.
+message EnumDescriptorProto {
+  optional string name = 1 [ctype = STRING_PIECE];
+
+  repeated EnumValueDescriptorProto value = 2;
+
+  optional EnumOptions options = 3;
+
+  // Range of reserved numeric values. Reserved values may not be used by
+  // entries in the same enum. Reserved ranges may not overlap.
+  //
+  // Note that this is distinct from DescriptorProto.ReservedRange in that it
+  // is inclusive such that it can appropriately represent the entire int32
+  // domain.
+  message EnumReservedRange {
+    optional int32 start = 1;  // Inclusive.
+    optional int32 end = 2;    // Inclusive.
+  }
+
+  // Range of reserved numeric values. Reserved numeric values may not be used
+  // by enum values in the same enum declaration. Reserved ranges may not
+  // overlap.
+  repeated EnumReservedRange reserved_range = 4;
+
+  // Reserved enum value names, which may not be reused. A given name may only
+  // be reserved once.
+  repeated string reserved_name = 5 [ctype = STRING_PIECE];
+}
+
+// Describes a value within an enum.
+message EnumValueDescriptorProto {
+  optional string name = 1 [ctype = STRING_PIECE];
+  optional int32 number = 2;
+
+  optional EnumValueOptions options = 3;
+}
+
+// Describes a service.
+message ServiceDescriptorProto {
+  optional string name = 1 [ctype = STRING_PIECE];
+  repeated MethodDescriptorProto method = 2;
+
+  optional ServiceOptions options = 3;
+}
+
+// Describes a method of a service.
+message MethodDescriptorProto {
+  optional string name = 1 [ctype = STRING_PIECE];
+
+  // Input and output type names.  These are resolved in the same way as
+  // FieldDescriptorProto.type_name, but must refer to a message type.
+  optional string input_type = 2 [ctype = STRING_PIECE];
+  optional string output_type = 3 [ctype = STRING_PIECE];
+
+  optional MethodOptions options = 4;
+
+  // Identifies if client streams multiple client messages
+  optional bool client_streaming = 5 [default = false];
+  // Identifies if server streams multiple server messages
+  optional bool server_streaming = 6 [default = false];
+}
+
+// ===================================================================
+// Options
+
+// Each of the definitions above may have "options" attached.  These are
+// just annotations which may cause code to be generated slightly differently
+// or may contain hints for code that manipulates protocol messages.
+//
+// Clients may define custom options as extensions of the *Options messages.
+// These extensions may not yet be known at parsing time, so the parser cannot
+// store the values in them.  Instead it stores them in a field in the *Options
+// message called uninterpreted_option. This field must have the same name
+// across all *Options messages. We then use this field to populate the
+// extensions when we build a descriptor, at which point all protos have been
+// parsed and so all extensions are known.
+//
+// Extension numbers for custom options may be chosen as follows:
+// * For options which will only be used within a single application or
+//   organization, or for experimental options, use field numbers 50000
+//   through 99999.  It is up to you to ensure that you do not use the
+//   same number for multiple options.
+// * For options which will be published and used publicly by multiple
+//   independent entities, e-mail protobuf-global-extension-registry@google.com
+//   to reserve extension numbers. Simply provide your project name (e.g.
+//   Objective-C plugin) and your project website (if available) -- there's no
+//   need to explain how you intend to use them. Usually you only need one
+//   extension number. You can declare multiple options with only one extension
+//   number by putting them in a sub-message. See the Custom Options section of
+//   the docs for examples:
+//   https://developers.google.com/protocol-buffers/docs/proto#options
+//   If this turns out to be popular, a web service will be set up
+//   to automatically assign option numbers.
+
+message FileOptions {
+  // Sets the Java package where classes generated from this .proto will be
+  // placed.  By default, the proto package is used, but this is often
+  // inappropriate because proto packages do not normally start with backwards
+  // domain names.
+  optional string java_package = 1 [ctype = STRING_PIECE];
+
+  // If set, all the classes from the .proto file are wrapped in a single
+  // outer class with the given name.  This applies to both Proto1
+  // (equivalent to the old "--one_java_file" option) and Proto2 (where
+  // a .proto always translates to a single class, but you may want to
+  // explicitly choose the class name).
+  optional string java_outer_classname = 8 [ctype = STRING_PIECE];
+
+  // If set true, then the Java code generator will generate a separate .java
+  // file for each top-level message, enum, and service defined in the .proto
+  // file.  Thus, these types will *not* be nested inside the outer class
+  // named by java_outer_classname.  However, the outer class will still be
+  // generated to contain the file's getDescriptor() method as well as any
+  // top-level extensions defined in the file.
+  optional bool java_multiple_files = 10 [default = false];
+
+  // This option does nothing.
+  optional bool java_generate_equals_and_hash = 20 [deprecated = true];
+
+  // If set true, then the Java2 code generator will generate code that
+  // throws an exception whenever an attempt is made to assign a non-UTF-8
+  // byte sequence to a string field.
+  // Message reflection will do the same.
+  // However, an extension field still accepts non-UTF-8 byte sequences.
+  // This option has no effect on when used with the lite runtime.
+  optional bool java_string_check_utf8 = 27 [default = false];
+
+  // Generated classes can be optimized for speed or code size.
+  enum OptimizeMode {
+    SPEED = 1;         // Generate complete code for parsing, serialization,
+                       // etc.
+    CODE_SIZE = 2;     // Use ReflectionOps to implement these methods.
+    LITE_RUNTIME = 3;  // Generate code using MessageLite and the lite runtime.
+  }
+  optional OptimizeMode optimize_for = 9 [default = SPEED];
+
+  // Sets the Go package where structs generated from this .proto will be
+  // placed. If omitted, the Go package will be derived from the following:
+  //   - The basename of the package import path, if provided.
+  //   - Otherwise, the package statement in the .proto file, if present.
+  //   - Otherwise, the basename of the .proto file, without extension.
+  optional string go_package = 11 [ctype = STRING_PIECE];
+
+  // Should generic services be generated in each language?  "Generic" services
+  // are not specific to any particular RPC system.  They are generated by the
+  // main code generators in each language (without additional plugins).
+  // Generic services were the only kind of service generation supported by
+  // early versions of google.protobuf.
+  //
+  // Generic services are now considered deprecated in favor of using plugins
+  // that generate code specific to your particular RPC system.  Therefore,
+  // these default to false.  Old code which depends on generic services should
+  // explicitly set them to true.
+  optional bool cc_generic_services = 16 [default = false];
+  optional bool java_generic_services = 17 [default = false];
+  optional bool py_generic_services = 18 [default = false];
+  optional bool php_generic_services = 42 [default = false];
+
+  // Is this file deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for everything in the file, or it will be completely ignored; in the very
+  // least, this is a formalization for deprecating files.
+  optional bool deprecated = 23 [default = false];
+
+  // Enables the use of arenas for the proto messages in this file. This applies
+  // only to generated classes for C++.
+  optional bool cc_enable_arenas = 31 [default = true];
+
+  // Sets the objective c class prefix which is prepended to all objective c
+  // generated classes from this .proto. There is no default.
+  optional string objc_class_prefix = 36 [ctype = STRING_PIECE];
+
+  // Namespace for generated classes; defaults to the package.
+  optional string csharp_namespace = 37 [ctype = STRING_PIECE];
+
+  // By default Swift generators will take the proto package and CamelCase it
+  // replacing '.' with underscore and use that to prefix the types/symbols
+  // defined. When this options is provided, they will use this value instead
+  // to prefix the types/symbols defined.
+  optional string swift_prefix = 39 [ctype = STRING_PIECE];
+
+  // Sets the php class prefix which is prepended to all php generated classes
+  // from this .proto. Default is empty.
+  optional string php_class_prefix = 40 [ctype = STRING_PIECE];
+
+  // Use this option to change the namespace of php generated classes. Default
+  // is empty. When this option is empty, the package name will be used for
+  // determining the namespace.
+  optional string php_namespace = 41 [ctype = STRING_PIECE];
+
+  // Use this option to change the namespace of php generated metadata classes.
+  // Default is empty. When this option is empty, the proto file name will be
+  // used for determining the namespace.
+  optional string php_metadata_namespace = 44 [ctype = STRING_PIECE];
+
+  // Use this option to change the package of ruby generated classes. Default
+  // is empty. When this option is not set, the package name will be used for
+  // determining the ruby package.
+  optional string ruby_package = 45 [ctype = STRING_PIECE];
+
+  // The parser stores options it doesn't recognize here.
+  // See the documentation for the "Options" section above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message.
+  // See the documentation for the "Options" section above.
+  extensions 1000 to max;
+
+  reserved 38;
+}
+
+message MessageOptions {
+  // Set true to use the old proto1 MessageSet wire format for extensions.
+  // This is provided for backwards-compatibility with the MessageSet wire
+  // format.  You should not use this for any other reason:  It's less
+  // efficient, has fewer features, and is more complicated.
+  //
+  // The message must be defined exactly as follows:
+  //   message Foo {
+  //     option message_set_wire_format = true;
+  //     extensions 4 to max;
+  //   }
+  // Note that the message cannot have any defined fields; MessageSets only
+  // have extensions.
+  //
+  // All extensions of your type must be singular messages; e.g. they cannot
+  // be int32s, enums, or repeated messages.
+  //
+  // Because this is an option, the above two restrictions are not enforced by
+  // the protocol compiler.
+  optional bool message_set_wire_format = 1 [default = false];
+
+  // Disables the generation of the standard "descriptor()" accessor, which can
+  // conflict with a field of the same name.  This is meant to make migration
+  // from proto1 easier; new code should avoid fields named "descriptor".
+  optional bool no_standard_descriptor_accessor = 2 [default = false];
+
+  // Is this message deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the message, or it will be completely ignored; in the very least,
+  // this is a formalization for deprecating messages.
+  optional bool deprecated = 3 [default = false];
+
+  // Whether the message is an automatically generated map entry type for the
+  // maps field.
+  //
+  // For maps fields:
+  //     map<KeyType, ValueType> map_field = 1;
+  // The parsed descriptor looks like:
+  //     message MapFieldEntry {
+  //         option map_entry = true;
+  //         optional KeyType key = 1;
+  //         optional ValueType value = 2;
+  //     }
+  //     repeated MapFieldEntry map_field = 1;
+  //
+  // Implementations may choose not to generate the map_entry=true message, but
+  // use a native map in the target language to hold the keys and values.
+  // The reflection APIs in such implementations still need to work as
+  // if the field is a repeated message field.
+  //
+  // NOTE: Do not set the option in .proto files. Always use the maps syntax
+  // instead. The option should only be implicitly set by the proto compiler
+  // parser.
+  optional bool map_entry = 7;
+
+  reserved 8;  // javalite_serializable
+  reserved 9;  // javanano_as_lite
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message FieldOptions {
+  // The ctype option instructs the C++ code generator to use a different
+  // representation of the field than it normally would.  See the specific
+  // options below.  This option is not yet implemented in the open source
+  // release -- sorry, we'll try to include it in a future version!
+  optional CType ctype = 1 [default = STRING];
+  enum CType {
+    // Default mode.
+    STRING = 0;
+
+    CORD = 1;
+
+    STRING_PIECE = 2;
+  }
+  // The packed option can be enabled for repeated primitive fields to enable
+  // a more efficient representation on the wire. Rather than repeatedly
+  // writing the tag and type for each element, the entire array is encoded as
+  // a single length-delimited blob. In proto3, only explicit setting it to
+  // false will avoid using packed encoding.
+  optional bool packed = 2;
+
+  // The jstype option determines the JavaScript type used for values of the
+  // field.  The option is permitted only for 64 bit integral and fixed types
+  // (int64, uint64, sint64, fixed64, sfixed64).  A field with jstype JS_STRING
+  // is represented as JavaScript string, which avoids loss of precision that
+  // can happen when a large value is converted to a floating point JavaScript.
+  // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to
+  // use the JavaScript "number" type.  The behavior of the default option
+  // JS_NORMAL is implementation dependent.
+  //
+  // This option is an enum to permit additional types to be added, e.g.
+  // goog.math.Integer.
+  optional JSType jstype = 6 [default = JS_NORMAL];
+  enum JSType {
+    // Use the default type.
+    JS_NORMAL = 0;
+
+    // Use JavaScript strings.
+    JS_STRING = 1;
+
+    // Use JavaScript numbers.
+    JS_NUMBER = 2;
+  }
+
+  // Should this field be parsed lazily?  Lazy applies only to message-type
+  // fields.  It means that when the outer message is initially parsed, the
+  // inner message's contents will not be parsed but instead stored in encoded
+  // form.  The inner message will actually be parsed when it is first accessed.
+  //
+  // This is only a hint.  Implementations are free to choose whether to use
+  // eager or lazy parsing regardless of the value of this option.  However,
+  // setting this option true suggests that the protocol author believes that
+  // using lazy parsing on this field is worth the additional bookkeeping
+  // overhead typically needed to implement it.
+  //
+  // This option does not affect the public interface of any generated code;
+  // all method signatures remain the same.  Furthermore, thread-safety of the
+  // interface is not affected by this option; const methods remain safe to
+  // call from multiple threads concurrently, while non-const methods continue
+  // to require exclusive access.
+  //
+  //
+  // Note that implementations may choose not to check required fields within
+  // a lazy sub-message.  That is, calling IsInitialized() on the outer message
+  // may return true even if the inner message has missing required fields.
+  // This is necessary because otherwise the inner message would have to be
+  // parsed in order to perform the check, defeating the purpose of lazy
+  // parsing.  An implementation which chooses not to check required fields
+  // must be consistent about it.  That is, for any particular sub-message, the
+  // implementation must either *always* check its required fields, or *never*
+  // check its required fields, regardless of whether or not the message has
+  // been parsed.
+  optional bool lazy = 5 [default = false];
+
+  // Is this field deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for accessors, or it will be completely ignored; in the very least, this
+  // is a formalization for deprecating fields.
+  optional bool deprecated = 3 [default = false];
+
+  // For Google-internal migration only. Do not use.
+  optional bool weak = 10 [default = false];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+
+  reserved 4;  // removed jtype
+}
+
+message OneofOptions {
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message EnumOptions {
+  // Set this option to true to allow mapping different tag names to the same
+  // value.
+  optional bool allow_alias = 2;
+
+  // Is this enum deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the enum, or it will be completely ignored; in the very least, this
+  // is a formalization for deprecating enums.
+  optional bool deprecated = 3 [default = false];
+
+  reserved 5;  // javanano_as_lite
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message EnumValueOptions {
+  // Is this enum value deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the enum value, or it will be completely ignored; in the very least,
+  // this is a formalization for deprecating enum values.
+  optional bool deprecated = 1 [default = false];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message ServiceOptions {
+  // Note:  Field numbers 1 through 32 are reserved for Google's internal RPC
+  //   framework.  We apologize for hoarding these numbers to ourselves, but
+  //   we were already using them long before we decided to release Protocol
+  //   Buffers.
+
+  // Is this service deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the service, or it will be completely ignored; in the very least,
+  // this is a formalization for deprecating services.
+  optional bool deprecated = 33 [default = false];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+message MethodOptions {
+  // Note:  Field numbers 1 through 32 are reserved for Google's internal RPC
+  //   framework.  We apologize for hoarding these numbers to ourselves, but
+  //   we were already using them long before we decided to release Protocol
+  //   Buffers.
+
+  // Is this method deprecated?
+  // Depending on the target platform, this can emit Deprecated annotations
+  // for the method, or it will be completely ignored; in the very least,
+  // this is a formalization for deprecating methods.
+  optional bool deprecated = 33 [default = false];
+
+  // Is this method side-effect-free (or safe in HTTP parlance), or idempotent,
+  // or neither? HTTP based RPC implementation may choose GET verb for safe
+  // methods, and PUT verb for idempotent methods instead of the default POST.
+  enum IdempotencyLevel {
+    IDEMPOTENCY_UNKNOWN = 0;
+    NO_SIDE_EFFECTS = 1;  // implies idempotent
+    IDEMPOTENT = 2;       // idempotent, but may have side effects
+  }
+  optional IdempotencyLevel idempotency_level = 34
+      [default = IDEMPOTENCY_UNKNOWN];
+
+  // The parser stores options it doesn't recognize here. See above.
+  repeated UninterpretedOption uninterpreted_option = 999;
+
+  // Clients can define custom options in extensions of this message. See above.
+  extensions 1000 to max;
+}
+
+// A message representing a option the parser does not recognize. This only
+// appears in options protos created by the compiler::Parser class.
+// DescriptorPool resolves these when building Descriptor objects. Therefore,
+// options protos in descriptor objects (e.g. returned by Descriptor::options(),
+// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions
+// in them.
+message UninterpretedOption {
+  // The name of the uninterpreted option.  Each string represents a segment in
+  // a dot-separated name.  is_extension is true iff a segment represents an
+  // extension (denoted with parentheses in options specs in .proto files).
+  // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents
+  // "foo.(bar.baz).qux".
+  message NamePart {
+    optional string name_part = 1 [ctype = STRING_PIECE];
+    optional bool is_extension = 2;
+  }
+  repeated NamePart name = 2;
+
+  // The value of the uninterpreted option, in whatever type the tokenizer
+  // identified it as during parsing. Exactly one of these should be set.
+  optional string identifier_value = 3 [ctype = STRING_PIECE];
+  optional uint64 positive_int_value = 4;
+  optional int64 negative_int_value = 5;
+  optional double double_value = 6;
+  optional bytes string_value = 7;
+  optional string aggregate_value = 8 [ctype = STRING_PIECE];
+}
+
+// ===================================================================
+// Optional source code info
+
+// Encapsulates information about the original source file from which a
+// FileDescriptorProto was generated.
+message SourceCodeInfo {
+  // A Location identifies a piece of source code in a .proto file which
+  // corresponds to a particular definition.  This information is intended
+  // to be useful to IDEs, code indexers, documentation generators, and similar
+  // tools.
+  //
+  // For example, say we have a file like:
+  //   message Foo {
+  //     optional string foo = 1 [ctype = STRING_PIECE];
+  //   }
+  // Let's look at just the field definition:
+  //   optional string foo = 1 [ctype = STRING_PIECE];
+  //   ^       ^^     ^^  ^  ^^^
+  //   a       bc     de  f  ghi
+  // We have the following locations:
+  //   span   path               represents
+  //   [a,i)  [ 4, 0, 2, 0 ]     The whole field definition.
+  //   [a,b)  [ 4, 0, 2, 0, 4 ]  The label (optional).
+  //   [c,d)  [ 4, 0, 2, 0, 5 ]  The type (string).
+  //   [e,f)  [ 4, 0, 2, 0, 1 ]  The name (foo).
+  //   [g,h)  [ 4, 0, 2, 0, 3 ]  The number (1).
+  //
+  // Notes:
+  // - A location may refer to a repeated field itself (i.e. not to any
+  //   particular index within it).  This is used whenever a set of elements are
+  //   logically enclosed in a single code segment.  For example, an entire
+  //   extend block (possibly containing multiple extension definitions) will
+  //   have an outer location whose path refers to the "extensions" repeated
+  //   field without an index.
+  // - Multiple locations may have the same path.  This happens when a single
+  //   logical declaration is spread out across multiple places.  The most
+  //   obvious example is the "extend" block again -- there may be multiple
+  //   extend blocks in the same scope, each of which will have the same path.
+  // - A location's span is not always a subset of its parent's span.  For
+  //   example, the "extendee" of an extension declaration appears at the
+  //   beginning of the "extend" block and is shared by all extensions within
+  //   the block.
+  // - Just because a location's span is a subset of some other location's span
+  //   does not mean that it is a descendant.  For example, a "group" defines
+  //   both a type and a field in a single declaration.  Thus, the locations
+  //   corresponding to the type and field and their components will overlap.
+  // - Code which tries to interpret locations should probably be designed to
+  //   ignore those that it doesn't understand, as more types of locations could
+  //   be recorded in the future.
+  repeated Location location = 1;
+  message Location {
+    // Identifies which part of the FileDescriptorProto was defined at this
+    // location.
+    //
+    // Each element is a field number or an index.  They form a path from
+    // the root FileDescriptorProto to the place where the definition.  For
+    // example, this path:
+    //   [ 4, 3, 2, 7, 1 ]
+    // refers to:
+    //   file.message_type(3)  // 4, 3
+    //       .field(7)         // 2, 7
+    //       .name()           // 1
+    // This is because FileDescriptorProto.message_type has field number 4:
+    //   repeated DescriptorProto message_type = 4;
+    // and DescriptorProto.field has field number 2:
+    //   repeated FieldDescriptorProto field = 2;
+    // and FieldDescriptorProto.name has field number 1:
+    //   optional string name = 1 [ctype = STRING_PIECE];
+    //
+    // Thus, the above path gives the location of a field name.  If we removed
+    // the last element:
+    //   [ 4, 3, 2, 7 ]
+    // this path refers to the whole field declaration (from the beginning
+    // of the label to the terminating semicolon).
+    repeated int32 path = 1 [packed = true];
+
+    // Always has exactly three or four elements: start line, start column,
+    // end line (optional, otherwise assumed same as start line), end column.
+    // These are packed into a single field for efficiency.  Note that line
+    // and column numbers are zero-based -- typically you will want to add
+    // 1 to each before displaying to a user.
+    repeated int32 span = 2 [packed = true];
+
+    // If this SourceCodeInfo represents a complete declaration, these are any
+    // comments appearing before and after the declaration which appear to be
+    // attached to the declaration.
+    //
+    // A series of line comments appearing on consecutive lines, with no other
+    // tokens appearing on those lines, will be treated as a single comment.
+    //
+    // leading_detached_comments will keep paragraphs of comments that appear
+    // before (but not connected to) the current element. Each paragraph,
+    // separated by empty lines, will be one comment element in the repeated
+    // field.
+    //
+    // Only the comment content is provided; comment markers (e.g. //) are
+    // stripped out.  For block comments, leading whitespace and an asterisk
+    // will be stripped from the beginning of each line other than the first.
+    // Newlines are included in the output.
+    //
+    // Examples:
+    //
+    //   optional int32 foo = 1;  // Comment attached to foo.
+    //   // Comment attached to bar.
+    //   optional int32 bar = 2;
+    //
+    //   optional string baz = 3 [ctype = STRING_PIECE];
+    //   // Comment attached to baz.
+    //   // Another line attached to baz.
+    //
+    //   // Comment attached to qux.
+    //   //
+    //   // Another line attached to qux.
+    //   optional double qux = 4;
+    //
+    //   // Detached comment for corge. This is not leading or trailing comments
+    //   // to qux or corge because there are blank lines separating it from
+    //   // both.
+    //
+    //   // Detached comment for corge paragraph 2.
+    //
+    //   optional string corge = 5 [ctype = STRING_PIECE];
+    //   /* Block comment attached
+    //    * to corge.  Leading asterisks
+    //    * will be removed. */
+    //   /* Block comment attached to
+    //    * grault. */
+    //   optional int32 grault = 6;
+    //
+    //   // ignored detached comments.
+    optional string leading_comments = 3 [ctype = STRING_PIECE];
+    optional string trailing_comments = 4 [ctype = STRING_PIECE];
+    repeated string leading_detached_comments = 6 [ctype = STRING_PIECE];
+  }
+}
+
+// Describes the relationship between generated code and its original source
+// file. A GeneratedCodeInfo message is associated with only one generated
+// source file, but may contain references to different source .proto files.
+message GeneratedCodeInfo {
+  // An Annotation connects some span of text in generated code to an element
+  // of its generating .proto file.
+  repeated Annotation annotation = 1;
+  message Annotation {
+    // Identifies the element in the original source .proto file. This field
+    // is formatted the same as SourceCodeInfo.Location.path.
+    repeated int32 path = 1 [packed = true];
+
+    // Identifies the filesystem path to the original source .proto.
+    optional string source_file = 2 [ctype = STRING_PIECE];
+
+    // Identifies the starting offset in bytes in the generated code
+    // that relates to the identified object.
+    optional int32 begin = 3;
+
+    // Identifies the ending offset in bytes in the generated code that
+    // relates to the identified offset. The end offset should be one past
+    // the last relevant byte (so the length of the text = end - begin).
+    optional int32 end = 4;
+  }
+}

benchmarks/empty.proto

@@ -0,0 +1,6 @@
+
+syntax = "proto3";
+
+package upb_benchmark;
+
+message Empty {}

benchmarks/gen_protobuf_binary_cc.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+import re
+
+include = sys.argv[1]
+msg_basename = sys.argv[2]
+count = 1
+
+m = re.search(r'(.*\D)(\d+)$', sys.argv[2])
+if m:
+  msg_basename = m.group(1)
+  count = int(m.group(2))
+
+print('''
+#include "{include}"
+
+char buf[1];
+
+int main() {{
+'''.format(include=include))
+
+def RefMessage(name):
+  print('''
+  {{
+    {name} proto;
+    proto.ParseFromArray(buf, 0);
+    proto.SerializePartialToArray(&buf[0], 0);
+  }}
+  '''.format(name=name))
+
+RefMessage(msg_basename)
+
+for i in range(2, count + 1):
+  RefMessage(msg_basename + str(i))
+
+print('''
+  return 0;
+}''')

benchmarks/gen_synthetic_protos.py

@@ -0,0 +1,118 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+import random
+
+base = sys.argv[1]
+
+field_freqs = [
+    (('bool', 'optional'), 8.321),
+    (('bool', 'repeated'), 0.033),
+    (('bytes', 'optional'), 0.809),
+    (('bytes', 'repeated'), 0.065),
+    (('double', 'optional'), 2.845),
+    (('double', 'repeated'), 0.143),
+    (('fixed32', 'optional'), 0.084),
+    (('fixed32', 'repeated'), 0.012),
+    (('fixed64', 'optional'), 0.204),
+    (('fixed64', 'repeated'), 0.027),
+    (('float', 'optional'), 2.355),
+    (('float', 'repeated'), 0.132),
+    (('int32', 'optional'), 6.717),
+    (('int32', 'repeated'), 0.366),
+    (('int64', 'optional'), 9.678),
+    (('int64', 'repeated'), 0.425),
+    (('sfixed32', 'optional'), 0.018),
+    (('sfixed32', 'repeated'), 0.005),
+    (('sfixed64', 'optional'), 0.022),
+    (('sfixed64', 'repeated'), 0.005),
+    (('sint32', 'optional'), 0.026),
+    (('sint32', 'repeated'), 0.009),
+    (('sint64', 'optional'), 0.018),
+    (('sint64', 'repeated'), 0.006),
+    (('string', 'optional'), 25.461),
+    (('string', 'repeated'), 2.606),
+    (('Enum', 'optional'), 6.16),
+    (('Enum', 'repeated'), 0.576),
+    (('Message', 'optional'), 22.472),
+    (('Message', 'repeated'), 7.766),
+    (('uint32', 'optional'), 1.289),
+    (('uint32', 'repeated'), 0.051),
+    (('uint64', 'optional'), 1.044),
+    (('uint64', 'repeated'), 0.079),
+]
+
+population = [item[0] for item in field_freqs]
+weights = [item[1] for item in field_freqs]
+
+def choices(k):
+  if sys.version_info >= (3, 6):
+    return random.choices(population=population, weights=weights, k=k)
+  else:
+    print("WARNING: old Python version, field types are not properly weighted!")
+    return [random.choice(population) for _ in range(k)]
+
+with open(base + "/100_msgs.proto", "w") as f:
+  f.write('syntax = "proto3";\n')
+  f.write('package upb_benchmark;\n')
+  f.write('message Message {}\n')
+  for i in range(2, 101):
+    f.write('message Message{i} {{}}\n'.format(i=i))
+
+with open(base + "/200_msgs.proto", "w") as f:
+  f.write('syntax = "proto3";\n')
+  f.write('package upb_benchmark;\n')
+  f.write('message Message {}\n')
+  for i in range(2, 501):
+    f.write('message Message{i} {{}}\n'.format(i=i))
+
+with open(base + "/100_fields.proto", "w") as f:
+  f.write('syntax = "proto2";\n')
+  f.write('package upb_benchmark;\n')
+  f.write('enum Enum { ZERO = 0; }\n')
+  f.write('message Message {\n')
+  i = 1
+  random.seed(a=0, version=2)
+  for field in choices(100):
+    field_type, label = field
+    f.write('  {label} {field_type} field{i} = {i};\n'.format(i=i, label=label, field_type=field_type))
+    i += 1
+  f.write('}\n')
+
+with open(base + "/200_fields.proto", "w") as f:
+  f.write('syntax = "proto2";\n')
+  f.write('package upb_benchmark;\n')
+  f.write('enum Enum { ZERO = 0; }\n')
+  f.write('message Message {\n')
+  i = 1
+  random.seed(a=0, version=2)
+  for field in choices(200):
+    field_type, label = field
+    f.write('  {label} {field_type} field{i} = {i};\n'.format(i=i, label=label,field_type=field_type))
+    i += 1
+  f.write('}\n')

benchmarks/gen_upb_binary_c.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+import re
+
+include = sys.argv[1]
+msg_basename = sys.argv[2]
+count = 1
+
+m = re.search(r'(.*\D)(\d+)$', sys.argv[2])
+if m:
+  msg_basename = m.group(1)
+  count = int(m.group(2))
+
+print('''
+#include "{include}"
+
+char buf[1];
+
+int main() {{
+  upb_Arena *arena = upb_Arena_New();
+  size_t size;
+'''.format(include=include))
+
+def RefMessage(name):
+  print('''
+  {{
+    {name} *proto = {name}_parse(buf, 1, arena);
+    {name}_serialize(proto, arena, &size);
+  }}
+  '''.format(name=name))
+
+RefMessage(msg_basename)
+
+for i in range(2, count + 1):
+  RefMessage(msg_basename + str(i))
+
+print('''
+  return 0;
+}''')

cmake/BUILD

@@ -0,0 +1,107 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+load(
+    ":build_defs.bzl",
+    "generated_file_staleness_test",
+)
+load(
+    "//bazel:build_defs.bzl",
+    "make_shell_script",
+)
+
+licenses(["notice"])
+
+exports_files(["staleness_test.py"])
+
+py_library(
+    name = "staleness_test_lib",
+    testonly = 1,
+    srcs = ["staleness_test_lib.py"],
+)
+
+py_binary(
+    name = "make_cmakelists",
+    srcs = ["make_cmakelists.py"],
+)
+
+genrule(
+    name = "gen_cmakelists",
+    srcs = [
+        "//:BUILD",
+        "//:WORKSPACE",
+        "//:cmake_files",
+        ":cmake_files",
+    ],
+    outs = ["generated-in/CMakeLists.txt"],
+    cmd = "$(location :make_cmakelists) $@",
+    tools = [":make_cmakelists"],
+)
+
+genrule(
+    name = "copy_protos",
+    srcs = ["//:descriptor_upb_proto"],
+    outs = [
+        "generated-in/google/protobuf/descriptor.upb.c",
+        "generated-in/google/protobuf/descriptor.upb.h",
+    ],
+    cmd = "cp $(SRCS) $(@D)/generated-in/google/protobuf",
+)
+
+generated_file_staleness_test(
+    name = "test_generated_files",
+    outs = [
+        "CMakeLists.txt",
+        "google/protobuf/descriptor.upb.c",
+        "google/protobuf/descriptor.upb.h",
+    ],
+    generated_pattern = "generated-in/%s",
+)
+
+# Test the CMake build #########################################################
+
+filegroup(
+    name = "cmake_files",
+    srcs = glob([
+        "**/*",
+    ]),
+)
+
+make_shell_script(
+    name = "gen_run_cmake_build",
+    out = "run_cmake_build.sh",
+    contents = "find . && mkdir build && cd build && cmake ../cmake && make -j8 && make test",
+)
+
+sh_test(
+    name = "cmake_build",
+    srcs = ["run_cmake_build.sh"],
+    data = [
+        ":cmake_files",
+        "//:cmake_files",
+        "//third_party/utf8_range:cmake_files",
+    ],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)

cmake/CMakeLists.txt

@@ -12,6 +12,7 @@ cmake_minimum_required (VERSION 3.0)
 cmake_policy(SET CMP0048 NEW)
 
 project(upb)
+set(CMAKE_C_STANDARD 99)
 
 
 # Prevent CMake from setting -rdynamic on Linux (!!).
@@ -48,8 +49,8 @@ if(UPB_ENABLE_UBSAN)
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address")
 endif()
 
-include_directories(.)
+include_directories(..)
-include_directories(generated_for_cmake)
+include_directories(../cmake)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 if(APPLE)
@@ -60,105 +61,80 @@ endif()
 
 enable_testing()
 
-add_library(port
+add_library(port INTERFACE)
-  upb/port.c)
 add_library(upb
-  upb/decode.c
+  ../upb/decode.c
-  upb/encode.c
+  ../upb/decode_internal.h
-  upb/msg.c
+  ../upb/encode.c
-  upb/msg.h
+  ../upb/msg.c
-  upb/table.c
+  ../upb/msg_internal.h
-  upb/table.int.h
+  ../upb/table.c
-  upb/upb.c
+  ../upb/table_internal.h
-  upb/decode.h
+  ../upb/upb.c
-  upb/encode.h
+  ../upb/upb_internal.h
-  upb/upb.h)
+  ../upb/decode.h
+  ../upb/encode.h
+  ../upb/msg.h
+  ../upb/upb.h
+  ../upb/upb.hpp)
 target_link_libraries(upb
-  port)
+  fastdecode
+  port
+  /third_party/utf8_range)
+add_library(fastdecode
+  ../upb/decode.h
+  ../upb/decode_fast.c
+  ../upb/decode_fast.h
+  ../upb/decode_internal.h
+  ../upb/msg.h
+  ../upb/msg_internal.h
+  ../upb/upb_internal.h)
+target_link_libraries(fastdecode
+  port
+  table
+  /third_party/utf8_range)
 add_library(generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE)
 target_link_libraries(generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE
   table
   upb)
+add_library(generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE)
+target_link_libraries(generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE
+  descriptor_upb_proto
+  reflection
+  table)
 add_library(reflection
-  upb/def.c
+  ../upb/def.c
-  upb/msg.h
+  ../upb/msg.h
-  upb/reflection.c
+  ../upb/reflection.c
-  upb/def.h
+  ../upb/def.h
-  upb/reflection.h)
+  ../upb/def.hpp
+  ../upb/reflection.h
+  ../upb/reflection.hpp)
 target_link_libraries(reflection
-  descriptor_upbproto
+  descriptor_upb_proto
   port
   table
   upb)
 add_library(textformat
-  upb/text_encode.c
+  ../upb/text_encode.c
-  upb/text_encode.h)
+  ../upb/upb_internal.h
+  ../upb/text_encode.h)
 target_link_libraries(textformat
   port
-  reflection)
+  reflection
+  table)
 add_library(json
-  upb/json_decode.c
+  ../upb/json_decode.c
-  upb/json_encode.c
+  ../upb/json_encode.c
-  upb/json_decode.h
+  ../upb/upb_internal.h
-  upb/json_encode.h)
+  ../upb/json_decode.h
+  ../upb/json_encode.h)
 target_link_libraries(json
   port
   reflection
   upb)
 add_library(table INTERFACE)
 target_link_libraries(table INTERFACE
-  port
+  port)
-  upb)
-add_library(handlers
-  upb/handlers.c
-  upb/handlers-inl.h
-  upb/sink.c
-  upb/handlers.h
-  upb/sink.h)
-target_link_libraries(handlers
-  port
-  reflection
-  table
-  upb)
-add_library(upb_pb
-  upb/pb/compile_decoder.c
-  upb/pb/decoder.c
-  upb/pb/decoder.int.h
-  upb/pb/encoder.c
-  upb/pb/textprinter.c
-  upb/pb/varint.c
-  upb/pb/varint.int.h
-  upb/pb/decoder.h
-  upb/pb/encoder.h
-  upb/pb/textprinter.h)
-target_link_libraries(upb_pb
-  descriptor_upbproto
-  handlers
-  port
-  reflection
-  table
-  upb)
-add_library(upb_json
-  generated_for_cmake/upb/json/parser.c
-  upb/json/printer.c
-  upb/json/parser.h
-  upb/json/printer.h)
-target_link_libraries(upb_json
-  upb
-  upb_pb)
-add_library(upb_cc_bindings INTERFACE)
-target_link_libraries(upb_cc_bindings INTERFACE
-  descriptor_upbproto
-  handlers
-  port
-  upb)
-add_library(upb_test
-  tests/testmain.cc
-  tests/test_util.h
-  tests/upb_test.h)
-target_link_libraries(upb_test
-  handlers
-  port
-  upb)
 
 

cmake/README.md

@@ -0,0 +1,23 @@
+
+# upb CMake build (EXPERIMENTAL)
+
+upb's CMake support is experimental. The core library builds successfully
+under CMake, and this is verified by the Bazel tests in this directory.
+However there is no support for building the upb compiler or for generating
+.upb.c/upb.h files. This means upb's CMake support is incomplete at best,
+unless your application is intended to be purely reflective.
+
+If you find this CMake setup useful in its current state, please consider
+filing an issue so we know. If you have suggestions for how it could be
+more useful (and particularly if you can contribute some code for it)
+please feel free to file an issue for that too. Do keep in mind that upb
+does not currently provide any ABI stability, so we want to avoid providing
+a shared library.
+
+The CMakeLists.txt is generated from the Bazel BUILD files using the Python
+scripts in this directory. We want to avoid having two separate sources of
+truth that both need to be updated when a file is added or removed.
+
+This directory also contains some generated files that would be created
+on the fly during a Bazel build. These are automaticaly kept in sync by
+the Bazel test `//cmake:test_generated_files`.

cmake/build_defs.bzl

@@ -0,0 +1,69 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+def generated_file_staleness_test(name, outs, generated_pattern):
+    """Tests that checked-in file(s) match the contents of generated file(s).
+
+    The resulting test will verify that all output files exist and have the
+    correct contents.  If the test fails, it can be invoked with --fix to
+    bring the checked-in files up to date.
+
+    Args:
+      name: Name of the rule.
+      outs: the checked-in files that are copied from generated files.
+      generated_pattern: the pattern for transforming each "out" file into a
+        generated file.  For example, if generated_pattern="generated/%s" then
+        a file foo.txt will look for generated file generated/foo.txt.
+    """
+
+    script_name = name + ".py"
+    script_src = ":staleness_test.py"
+
+    # Filter out non-existing rules so Blaze doesn't error out before we even
+    # run the test.
+    existing_outs = native.glob(include = outs)
+
+    # The file list contains a few extra bits of information at the end.
+    # These get unpacked by the Config class in staleness_test_lib.py.
+    file_list = outs + [generated_pattern, native.package_name() or ".", name]
+
+    native.genrule(
+        name = name + "_makescript",
+        outs = [script_name],
+        srcs = [script_src],
+        testonly = 1,
+        cmd = "cat $(location " + script_src + ") > $@; " +
+              "sed -i.bak -e 's|INSERT_FILE_LIST_HERE|" + "\\\n  ".join(file_list) + "|' $@",
+    )
+
+    native.py_test(
+        name = name,
+        srcs = [script_name],
+        data = existing_outs + [generated_pattern % file for file in outs],
+        python_version = "PY3",
+        deps = [
+            ":staleness_test_lib",
+        ],
+    )

cmake/google/protobuf/descriptor.upb.c

@@ -0,0 +1,577 @@
+/* This file was generated by upbc (the upb compiler) from the input
+ * file:
+ *
+ *     google/protobuf/descriptor.proto
+ *
+ * Do not edit -- your changes will be discarded when the file is
+ * regenerated. */
+
+#include <stddef.h>
+#include "upb/msg_internal.h"
+#include "google/protobuf/descriptor.upb.h"
+
+#include "upb/port_def.inc"
+
+static const upb_MiniTable_Sub google_protobuf_FileDescriptorSet_submsgs[1] = {
+  {.submsg = &google_protobuf_FileDescriptorProto_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_FileDescriptorSet__fields[1] = {
+  {1, UPB_SIZE(0, 0), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_FileDescriptorSet_msginit = {
+  &google_protobuf_FileDescriptorSet_submsgs[0],
+  &google_protobuf_FileDescriptorSet__fields[0],
+  UPB_SIZE(8, 8), 1, kUpb_ExtMode_NonExtendable, 1, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_FileDescriptorProto_submsgs[6] = {
+  {.submsg = &google_protobuf_DescriptorProto_msginit},
+  {.submsg = &google_protobuf_EnumDescriptorProto_msginit},
+  {.submsg = &google_protobuf_FieldDescriptorProto_msginit},
+  {.submsg = &google_protobuf_FileOptions_msginit},
+  {.submsg = &google_protobuf_ServiceDescriptorProto_msginit},
+  {.submsg = &google_protobuf_SourceCodeInfo_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_FileDescriptorProto__fields[12] = {
+  {1, UPB_SIZE(4, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(12, 24), 2, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(36, 72), 0, 0, 12, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {4, UPB_SIZE(40, 80), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {5, UPB_SIZE(44, 88), 0, 1, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {6, UPB_SIZE(48, 96), 0, 4, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {7, UPB_SIZE(52, 104), 0, 2, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {8, UPB_SIZE(28, 56), 3, 3, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {9, UPB_SIZE(32, 64), 4, 5, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {10, UPB_SIZE(56, 112), 0, 0, 5, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {11, UPB_SIZE(60, 120), 0, 0, 5, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {12, UPB_SIZE(20, 40), 5, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_FileDescriptorProto_msginit = {
+  &google_protobuf_FileDescriptorProto_submsgs[0],
+  &google_protobuf_FileDescriptorProto__fields[0],
+  UPB_SIZE(64, 128), 12, kUpb_ExtMode_NonExtendable, 12, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_DescriptorProto_submsgs[7] = {
+  {.submsg = &google_protobuf_DescriptorProto_msginit},
+  {.submsg = &google_protobuf_DescriptorProto_ExtensionRange_msginit},
+  {.submsg = &google_protobuf_DescriptorProto_ReservedRange_msginit},
+  {.submsg = &google_protobuf_EnumDescriptorProto_msginit},
+  {.submsg = &google_protobuf_FieldDescriptorProto_msginit},
+  {.submsg = &google_protobuf_MessageOptions_msginit},
+  {.submsg = &google_protobuf_OneofDescriptorProto_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_DescriptorProto__fields[10] = {
+  {1, UPB_SIZE(4, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(16, 32), 0, 4, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(20, 40), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {4, UPB_SIZE(24, 48), 0, 3, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {5, UPB_SIZE(28, 56), 0, 1, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {6, UPB_SIZE(32, 64), 0, 4, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {7, UPB_SIZE(12, 24), 2, 5, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {8, UPB_SIZE(36, 72), 0, 6, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {9, UPB_SIZE(40, 80), 0, 2, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {10, UPB_SIZE(44, 88), 0, 0, 12, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_DescriptorProto_msginit = {
+  &google_protobuf_DescriptorProto_submsgs[0],
+  &google_protobuf_DescriptorProto__fields[0],
+  UPB_SIZE(48, 96), 10, kUpb_ExtMode_NonExtendable, 10, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_DescriptorProto_ExtensionRange_submsgs[1] = {
+  {.submsg = &google_protobuf_ExtensionRangeOptions_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_DescriptorProto_ExtensionRange__fields[3] = {
+  {1, UPB_SIZE(4, 4), 1, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(8, 8), 2, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(12, 16), 3, 0, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_DescriptorProto_ExtensionRange_msginit = {
+  &google_protobuf_DescriptorProto_ExtensionRange_submsgs[0],
+  &google_protobuf_DescriptorProto_ExtensionRange__fields[0],
+  UPB_SIZE(16, 24), 3, kUpb_ExtMode_NonExtendable, 3, 255, 0,
+};
+
+static const upb_MiniTable_Field google_protobuf_DescriptorProto_ReservedRange__fields[2] = {
+  {1, UPB_SIZE(4, 4), 1, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(8, 8), 2, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_DescriptorProto_ReservedRange_msginit = {
+  NULL,
+  &google_protobuf_DescriptorProto_ReservedRange__fields[0],
+  UPB_SIZE(16, 16), 2, kUpb_ExtMode_NonExtendable, 2, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_ExtensionRangeOptions_submsgs[1] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_ExtensionRangeOptions__fields[1] = {
+  {999, UPB_SIZE(0, 0), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_ExtensionRangeOptions_msginit = {
+  &google_protobuf_ExtensionRangeOptions_submsgs[0],
+  &google_protobuf_ExtensionRangeOptions__fields[0],
+  UPB_SIZE(8, 8), 1, kUpb_ExtMode_Extendable, 0, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_FieldDescriptorProto_submsgs[3] = {
+  {.submsg = &google_protobuf_FieldOptions_msginit},
+  {.subenum = &google_protobuf_FieldDescriptorProto_Label_enuminit},
+  {.subenum = &google_protobuf_FieldDescriptorProto_Type_enuminit},
+};
+
+static const upb_MiniTable_Field google_protobuf_FieldDescriptorProto__fields[11] = {
+  {1, UPB_SIZE(24, 24), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(32, 40), 2, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(12, 12), 3, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {4, UPB_SIZE(4, 4), 4, 1, 14, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {5, UPB_SIZE(8, 8), 5, 2, 14, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {6, UPB_SIZE(40, 56), 6, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {7, UPB_SIZE(48, 72), 7, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {8, UPB_SIZE(64, 104), 8, 0, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {9, UPB_SIZE(16, 16), 9, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {10, UPB_SIZE(56, 88), 10, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {17, UPB_SIZE(20, 20), 11, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_FieldDescriptorProto_msginit = {
+  &google_protobuf_FieldDescriptorProto_submsgs[0],
+  &google_protobuf_FieldDescriptorProto__fields[0],
+  UPB_SIZE(72, 112), 11, kUpb_ExtMode_NonExtendable, 10, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_OneofDescriptorProto_submsgs[1] = {
+  {.submsg = &google_protobuf_OneofOptions_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_OneofDescriptorProto__fields[2] = {
+  {1, UPB_SIZE(4, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(12, 24), 2, 0, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_OneofDescriptorProto_msginit = {
+  &google_protobuf_OneofDescriptorProto_submsgs[0],
+  &google_protobuf_OneofDescriptorProto__fields[0],
+  UPB_SIZE(16, 32), 2, kUpb_ExtMode_NonExtendable, 2, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_EnumDescriptorProto_submsgs[3] = {
+  {.submsg = &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit},
+  {.submsg = &google_protobuf_EnumOptions_msginit},
+  {.submsg = &google_protobuf_EnumValueDescriptorProto_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_EnumDescriptorProto__fields[5] = {
+  {1, UPB_SIZE(4, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(16, 32), 0, 2, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(12, 24), 2, 1, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {4, UPB_SIZE(20, 40), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {5, UPB_SIZE(24, 48), 0, 0, 12, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_EnumDescriptorProto_msginit = {
+  &google_protobuf_EnumDescriptorProto_submsgs[0],
+  &google_protobuf_EnumDescriptorProto__fields[0],
+  UPB_SIZE(32, 64), 5, kUpb_ExtMode_NonExtendable, 5, 255, 0,
+};
+
+static const upb_MiniTable_Field google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[2] = {
+  {1, UPB_SIZE(4, 4), 1, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(8, 8), 2, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit = {
+  NULL,
+  &google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[0],
+  UPB_SIZE(16, 16), 2, kUpb_ExtMode_NonExtendable, 2, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_EnumValueDescriptorProto_submsgs[1] = {
+  {.submsg = &google_protobuf_EnumValueOptions_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_EnumValueDescriptorProto__fields[3] = {
+  {1, UPB_SIZE(8, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(4, 4), 2, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(16, 24), 3, 0, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_EnumValueDescriptorProto_msginit = {
+  &google_protobuf_EnumValueDescriptorProto_submsgs[0],
+  &google_protobuf_EnumValueDescriptorProto__fields[0],
+  UPB_SIZE(24, 32), 3, kUpb_ExtMode_NonExtendable, 3, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_ServiceDescriptorProto_submsgs[2] = {
+  {.submsg = &google_protobuf_MethodDescriptorProto_msginit},
+  {.submsg = &google_protobuf_ServiceOptions_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_ServiceDescriptorProto__fields[3] = {
+  {1, UPB_SIZE(4, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(16, 32), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(12, 24), 2, 1, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_ServiceDescriptorProto_msginit = {
+  &google_protobuf_ServiceDescriptorProto_submsgs[0],
+  &google_protobuf_ServiceDescriptorProto__fields[0],
+  UPB_SIZE(24, 48), 3, kUpb_ExtMode_NonExtendable, 3, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_MethodDescriptorProto_submsgs[1] = {
+  {.submsg = &google_protobuf_MethodOptions_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_MethodDescriptorProto__fields[6] = {
+  {1, UPB_SIZE(4, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(12, 24), 2, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(20, 40), 3, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {4, UPB_SIZE(28, 56), 4, 0, 11, kUpb_FieldMode_Scalar | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {5, UPB_SIZE(1, 1), 5, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {6, UPB_SIZE(2, 2), 6, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_MethodDescriptorProto_msginit = {
+  &google_protobuf_MethodDescriptorProto_submsgs[0],
+  &google_protobuf_MethodDescriptorProto__fields[0],
+  UPB_SIZE(32, 64), 6, kUpb_ExtMode_NonExtendable, 6, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_FileOptions_submsgs[2] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+  {.subenum = &google_protobuf_FileOptions_OptimizeMode_enuminit},
+};
+
+static const upb_MiniTable_Field google_protobuf_FileOptions__fields[21] = {
+  {1, UPB_SIZE(20, 24), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {8, UPB_SIZE(28, 40), 2, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {9, UPB_SIZE(4, 4), 3, 1, 14, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {10, UPB_SIZE(8, 8), 4, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {11, UPB_SIZE(36, 56), 5, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {16, UPB_SIZE(9, 9), 6, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {17, UPB_SIZE(10, 10), 7, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {18, UPB_SIZE(11, 11), 8, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {20, UPB_SIZE(12, 12), 9, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {23, UPB_SIZE(13, 13), 10, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {27, UPB_SIZE(14, 14), 11, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {31, UPB_SIZE(15, 15), 12, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {36, UPB_SIZE(44, 72), 13, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {37, UPB_SIZE(52, 88), 14, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {39, UPB_SIZE(60, 104), 15, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {40, UPB_SIZE(68, 120), 16, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {41, UPB_SIZE(76, 136), 17, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {42, UPB_SIZE(16, 16), 18, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {44, UPB_SIZE(84, 152), 19, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {45, UPB_SIZE(92, 168), 20, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {999, UPB_SIZE(100, 184), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_FileOptions_msginit = {
+  &google_protobuf_FileOptions_submsgs[0],
+  &google_protobuf_FileOptions__fields[0],
+  UPB_SIZE(104, 192), 21, kUpb_ExtMode_Extendable, 1, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_MessageOptions_submsgs[1] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_MessageOptions__fields[5] = {
+  {1, UPB_SIZE(1, 1), 1, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(2, 2), 2, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(3, 3), 3, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {7, UPB_SIZE(4, 4), 4, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {999, UPB_SIZE(8, 8), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_MessageOptions_msginit = {
+  &google_protobuf_MessageOptions_submsgs[0],
+  &google_protobuf_MessageOptions__fields[0],
+  UPB_SIZE(16, 16), 5, kUpb_ExtMode_Extendable, 3, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_FieldOptions_submsgs[3] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+  {.subenum = &google_protobuf_FieldOptions_CType_enuminit},
+  {.subenum = &google_protobuf_FieldOptions_JSType_enuminit},
+};
+
+static const upb_MiniTable_Field google_protobuf_FieldOptions__fields[8] = {
+  {1, UPB_SIZE(4, 4), 1, 1, 14, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(12, 12), 2, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(13, 13), 3, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {5, UPB_SIZE(14, 14), 4, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {6, UPB_SIZE(8, 8), 5, 2, 14, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {10, UPB_SIZE(15, 15), 6, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {15, UPB_SIZE(16, 16), 7, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {999, UPB_SIZE(20, 24), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_FieldOptions_msginit = {
+  &google_protobuf_FieldOptions_submsgs[0],
+  &google_protobuf_FieldOptions__fields[0],
+  UPB_SIZE(24, 32), 8, kUpb_ExtMode_Extendable, 3, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_OneofOptions_submsgs[1] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_OneofOptions__fields[1] = {
+  {999, UPB_SIZE(0, 0), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_OneofOptions_msginit = {
+  &google_protobuf_OneofOptions_submsgs[0],
+  &google_protobuf_OneofOptions__fields[0],
+  UPB_SIZE(8, 8), 1, kUpb_ExtMode_Extendable, 0, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_EnumOptions_submsgs[1] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_EnumOptions__fields[3] = {
+  {2, UPB_SIZE(1, 1), 1, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(2, 2), 2, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_EnumOptions_msginit = {
+  &google_protobuf_EnumOptions_submsgs[0],
+  &google_protobuf_EnumOptions__fields[0],
+  UPB_SIZE(8, 16), 3, kUpb_ExtMode_Extendable, 0, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_EnumValueOptions_submsgs[1] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_EnumValueOptions__fields[2] = {
+  {1, UPB_SIZE(1, 1), 1, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_EnumValueOptions_msginit = {
+  &google_protobuf_EnumValueOptions_submsgs[0],
+  &google_protobuf_EnumValueOptions__fields[0],
+  UPB_SIZE(8, 16), 2, kUpb_ExtMode_Extendable, 1, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_ServiceOptions_submsgs[1] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_ServiceOptions__fields[2] = {
+  {33, UPB_SIZE(1, 1), 1, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {999, UPB_SIZE(4, 8), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_ServiceOptions_msginit = {
+  &google_protobuf_ServiceOptions_submsgs[0],
+  &google_protobuf_ServiceOptions__fields[0],
+  UPB_SIZE(8, 16), 2, kUpb_ExtMode_Extendable, 0, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_MethodOptions_submsgs[2] = {
+  {.submsg = &google_protobuf_UninterpretedOption_msginit},
+  {.subenum = &google_protobuf_MethodOptions_IdempotencyLevel_enuminit},
+};
+
+static const upb_MiniTable_Field google_protobuf_MethodOptions__fields[3] = {
+  {33, UPB_SIZE(8, 8), 1, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+  {34, UPB_SIZE(4, 4), 2, 1, 14, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {999, UPB_SIZE(12, 16), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_MethodOptions_msginit = {
+  &google_protobuf_MethodOptions_submsgs[0],
+  &google_protobuf_MethodOptions__fields[0],
+  UPB_SIZE(16, 24), 3, kUpb_ExtMode_Extendable, 0, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_UninterpretedOption_submsgs[1] = {
+  {.submsg = &google_protobuf_UninterpretedOption_NamePart_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_UninterpretedOption__fields[7] = {
+  {2, UPB_SIZE(56, 80), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(32, 32), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {4, UPB_SIZE(8, 8), 2, 0, 4, kUpb_FieldMode_Scalar | (kUpb_FieldRep_8Byte << kUpb_FieldRep_Shift)},
+  {5, UPB_SIZE(16, 16), 3, 0, 3, kUpb_FieldMode_Scalar | (kUpb_FieldRep_8Byte << kUpb_FieldRep_Shift)},
+  {6, UPB_SIZE(24, 24), 4, 0, 1, kUpb_FieldMode_Scalar | (kUpb_FieldRep_8Byte << kUpb_FieldRep_Shift)},
+  {7, UPB_SIZE(40, 48), 5, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {8, UPB_SIZE(48, 64), 6, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_UninterpretedOption_msginit = {
+  &google_protobuf_UninterpretedOption_submsgs[0],
+  &google_protobuf_UninterpretedOption__fields[0],
+  UPB_SIZE(64, 96), 7, kUpb_ExtMode_NonExtendable, 0, 255, 0,
+};
+
+static const upb_MiniTable_Field google_protobuf_UninterpretedOption_NamePart__fields[2] = {
+  {1, UPB_SIZE(4, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(1, 1), 2, 0, 8, kUpb_FieldMode_Scalar | (kUpb_FieldRep_1Byte << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_UninterpretedOption_NamePart_msginit = {
+  NULL,
+  &google_protobuf_UninterpretedOption_NamePart__fields[0],
+  UPB_SIZE(16, 32), 2, kUpb_ExtMode_NonExtendable, 2, 255, 2,
+};
+
+static const upb_MiniTable_Sub google_protobuf_SourceCodeInfo_submsgs[1] = {
+  {.submsg = &google_protobuf_SourceCodeInfo_Location_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_SourceCodeInfo__fields[1] = {
+  {1, UPB_SIZE(0, 0), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_SourceCodeInfo_msginit = {
+  &google_protobuf_SourceCodeInfo_submsgs[0],
+  &google_protobuf_SourceCodeInfo__fields[0],
+  UPB_SIZE(8, 8), 1, kUpb_ExtMode_NonExtendable, 1, 255, 0,
+};
+
+static const upb_MiniTable_Field google_protobuf_SourceCodeInfo_Location__fields[5] = {
+  {1, UPB_SIZE(20, 40), 0, 0, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(24, 48), 0, 0, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(4, 8), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {4, UPB_SIZE(12, 24), 2, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {6, UPB_SIZE(28, 56), 0, 0, 12, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_SourceCodeInfo_Location_msginit = {
+  NULL,
+  &google_protobuf_SourceCodeInfo_Location__fields[0],
+  UPB_SIZE(32, 64), 5, kUpb_ExtMode_NonExtendable, 4, 255, 0,
+};
+
+static const upb_MiniTable_Sub google_protobuf_GeneratedCodeInfo_submsgs[1] = {
+  {.submsg = &google_protobuf_GeneratedCodeInfo_Annotation_msginit},
+};
+
+static const upb_MiniTable_Field google_protobuf_GeneratedCodeInfo__fields[1] = {
+  {1, UPB_SIZE(0, 0), 0, 0, 11, kUpb_FieldMode_Array | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_GeneratedCodeInfo_msginit = {
+  &google_protobuf_GeneratedCodeInfo_submsgs[0],
+  &google_protobuf_GeneratedCodeInfo__fields[0],
+  UPB_SIZE(8, 8), 1, kUpb_ExtMode_NonExtendable, 1, 255, 0,
+};
+
+static const upb_MiniTable_Field google_protobuf_GeneratedCodeInfo_Annotation__fields[4] = {
+  {1, UPB_SIZE(20, 32), 0, 0, 5, kUpb_FieldMode_Array | kUpb_LabelFlags_IsPacked | (kUpb_FieldRep_Pointer << kUpb_FieldRep_Shift)},
+  {2, UPB_SIZE(12, 16), 1, 0, 12, kUpb_FieldMode_Scalar | (kUpb_FieldRep_StringView << kUpb_FieldRep_Shift)},
+  {3, UPB_SIZE(4, 4), 2, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+  {4, UPB_SIZE(8, 8), 3, 0, 5, kUpb_FieldMode_Scalar | (kUpb_FieldRep_4Byte << kUpb_FieldRep_Shift)},
+};
+
+const upb_MiniTable google_protobuf_GeneratedCodeInfo_Annotation_msginit = {
+  NULL,
+  &google_protobuf_GeneratedCodeInfo_Annotation__fields[0],
+  UPB_SIZE(24, 48), 4, kUpb_ExtMode_NonExtendable, 4, 255, 0,
+};
+
+static const upb_MiniTable *messages_layout[27] = {
+  &google_protobuf_FileDescriptorSet_msginit,
+  &google_protobuf_FileDescriptorProto_msginit,
+  &google_protobuf_DescriptorProto_msginit,
+  &google_protobuf_DescriptorProto_ExtensionRange_msginit,
+  &google_protobuf_DescriptorProto_ReservedRange_msginit,
+  &google_protobuf_ExtensionRangeOptions_msginit,
+  &google_protobuf_FieldDescriptorProto_msginit,
+  &google_protobuf_OneofDescriptorProto_msginit,
+  &google_protobuf_EnumDescriptorProto_msginit,
+  &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit,
+  &google_protobuf_EnumValueDescriptorProto_msginit,
+  &google_protobuf_ServiceDescriptorProto_msginit,
+  &google_protobuf_MethodDescriptorProto_msginit,
+  &google_protobuf_FileOptions_msginit,
+  &google_protobuf_MessageOptions_msginit,
+  &google_protobuf_FieldOptions_msginit,
+  &google_protobuf_OneofOptions_msginit,
+  &google_protobuf_EnumOptions_msginit,
+  &google_protobuf_EnumValueOptions_msginit,
+  &google_protobuf_ServiceOptions_msginit,
+  &google_protobuf_MethodOptions_msginit,
+  &google_protobuf_UninterpretedOption_msginit,
+  &google_protobuf_UninterpretedOption_NamePart_msginit,
+  &google_protobuf_SourceCodeInfo_msginit,
+  &google_protobuf_SourceCodeInfo_Location_msginit,
+  &google_protobuf_GeneratedCodeInfo_msginit,
+  &google_protobuf_GeneratedCodeInfo_Annotation_msginit,
+};
+
+const upb_MiniTable_Enum google_protobuf_FieldDescriptorProto_Type_enuminit = {
+  NULL,
+  0x7fffeULL,
+  0,
+};
+
+const upb_MiniTable_Enum google_protobuf_FieldDescriptorProto_Label_enuminit = {
+  NULL,
+  0xeULL,
+  0,
+};
+
+const upb_MiniTable_Enum google_protobuf_FileOptions_OptimizeMode_enuminit = {
+  NULL,
+  0xeULL,
+  0,
+};
+
+const upb_MiniTable_Enum google_protobuf_FieldOptions_CType_enuminit = {
+  NULL,
+  0x7ULL,
+  0,
+};
+
+const upb_MiniTable_Enum google_protobuf_FieldOptions_JSType_enuminit = {
+  NULL,
+  0x7ULL,
+  0,
+};
+
+const upb_MiniTable_Enum google_protobuf_MethodOptions_IdempotencyLevel_enuminit = {
+  NULL,
+  0x7ULL,
+  0,
+};
+
+static const upb_MiniTable_Enum *enums_layout[6] = {
+  &google_protobuf_FieldDescriptorProto_Type_enuminit,
+  &google_protobuf_FieldDescriptorProto_Label_enuminit,
+  &google_protobuf_FileOptions_OptimizeMode_enuminit,
+  &google_protobuf_FieldOptions_CType_enuminit,
+  &google_protobuf_FieldOptions_JSType_enuminit,
+  &google_protobuf_MethodOptions_IdempotencyLevel_enuminit,
+};
+
+const upb_MiniTable_File google_protobuf_descriptor_proto_upb_file_layout = {
+  messages_layout,
+  enums_layout,
+  NULL,
+  27,
+  6,
+  0,
+};
+
+#include "upb/port_undef.inc"
+

cmake/make_cmakelists.py

@@ -1,4 +1,29 @@
-#!/usr/bin/env python
+#!/usr/bin/python
+#
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 """TODO(haberman): DO NOT SUBMIT without one-line documentation for make_cmakelists.
 
@@ -36,7 +61,9 @@ class BuildFileFunctions(object):
     pass
 
   def cc_library(self, **kwargs):
-    if kwargs["name"] == "amalgamation" or kwargs["name"] == "upbc_generator":
+    if kwargs["name"].endswith("amalgamation"):
+      return
+    if kwargs["name"] == "upbc_generator":
       return
     if kwargs["name"] == "lupb":
       return
@@ -44,9 +71,9 @@ class BuildFileFunctions(object):
     found_files = []
     for file in files:
         if os.path.isfile(file):
-            found_files.append(file)
+            found_files.append("../" + file)
-        elif os.path.isfile("generated_for_cmake/" + file):
+        elif os.path.isfile("cmake/" + file):
-            found_files.append("generated_for_cmake/" + file)
+            found_files.append("../cmake/" + file)
         else:
             print("Warning: no such file: " + file)
 
@@ -94,6 +121,9 @@ class BuildFileFunctions(object):
     # self._add_deps(kwargs)
     pass
 
+  def cc_fuzz_test(self, **kwargs):
+    pass
+
   def py_library(self, **kwargs):
     pass
 
@@ -115,6 +145,9 @@ class BuildFileFunctions(object):
   def proto_library(self, **kwargs):
     pass
 
+  def cc_proto_library(self, **kwargs):
+    pass
+
   def generated_file_staleness_test(self, **kwargs):
     pass
 
@@ -124,6 +157,9 @@ class BuildFileFunctions(object):
   def upb_proto_library(self, **kwargs):
     pass
 
+  def upb_proto_library_copts(self, **kwargs):
+    pass
+
   def upb_proto_reflection_library(self, **kwargs):
     pass
 
@@ -136,6 +172,9 @@ class BuildFileFunctions(object):
   def config_setting(self, **kwargs):
     pass
 
+  def upb_fasttable_enabled(self, **kwargs):
+    pass
+
   def select(self, arg_dict):
     return []
 
@@ -161,6 +200,7 @@ class WorkspaceFileFunctions(object):
 
   def workspace(self, **kwargs):
     self.converter.prelude += "project(%s)\n" % (kwargs["name"])
+    self.converter.prelude += "set(CMAKE_C_STANDARD 99)\n"
 
   def http_archive(self, **kwargs):
     pass
@@ -168,12 +208,30 @@ class WorkspaceFileFunctions(object):
   def git_repository(self, **kwargs):
     pass
 
+  def new_git_repository(self, **kwargs):
+    pass
+
   def bazel_version_repository(self, **kwargs):
     pass
 
   def upb_deps(self):
     pass
 
+  def protobuf_deps(self):
+    pass
+
+  def rules_fuzzing_dependencies(self):
+    pass
+
+  def rules_fuzzing_init(self):
+    pass
+
+  def system_python(self, **kwargs):
+    pass
+
+  def register_toolchains(self, toolchain):
+    pass
+
 
 class Converter(object):
   def __init__(self):
@@ -237,8 +295,8 @@ class Converter(object):
       set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address")
     endif()
 
-    include_directories(.)
+    include_directories(..)
-    include_directories(generated_for_cmake)
+    include_directories(../cmake)
     include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
     if(APPLE)
@@ -258,6 +316,8 @@ converter = Converter()
 
 def GetDict(obj):
   ret = {}
+  ret["UPB_DEFAULT_COPTS"] = []  # HACK
+  ret["UPB_DEFAULT_CPPOPTS"] = []  # HACK
   for k in dir(obj):
     if not k.startswith("_"):
       ret[k] = getattr(obj, k);

cmake/staleness_test.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""The py_test() script for generated_file_staleness_test() rules.
+
+Note that this file is preprocessed!  The INSERT_<...> text below is replaced
+with the actual list of files before we actually run the script.
+"""
+
+from __future__ import absolute_import
+
+from cmake import staleness_test_lib
+import unittest
+import sys
+
+file_list = """
+  INSERT_FILE_LIST_HERE
+""".split()
+
+config = staleness_test_lib.Config(file_list)
+
+
+class TestFilesMatch(unittest.TestCase):
+
+  def testFilesMatch(self):
+    errors = staleness_test_lib.CheckFilesMatch(config)
+    self.assertFalse(errors, errors)
+
+
+if len(sys.argv) > 1 and sys.argv[1] == "--fix":
+  staleness_test_lib.FixFiles(config)
+else:
+  unittest.main()

cmake/staleness_test_lib.py

@@ -1,3 +1,30 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 """Shared code for validating generated_file_staleness_test() rules.
 
 This code is used by test scripts generated from
@@ -7,6 +34,7 @@ generated_file_staleness_test() rules.
 from __future__ import absolute_import
 from __future__ import print_function
 
+import sys
 import os
 from shutil import copyfile
 
@@ -47,13 +75,13 @@ def _GetFilePairs(config):
 
   ret = []
 
-  has_bazel_genfiles = os.path.exists("bazel-genfiles")
+  has_bazel_genfiles = os.path.exists("bazel-bin")
 
   for filename in config.file_list:
     target = os.path.join(config.package_name, filename)
     generated = os.path.join(config.package_name, config.pattern % filename)
     if has_bazel_genfiles:
-      generated = os.path.join("bazel-genfiles", generated)
+      generated = os.path.join("bazel-bin", generated)
 
     # Generated files should always exist.  Blaze should guarantee this before
     # we are run.
@@ -61,6 +89,7 @@ def _GetFilePairs(config):
       print("Generated file '%s' does not exist." % generated)
       print("Please run this command to generate it:")
       print("  bazel build %s:%s" % (config.package_name, config.target_name))
+      sys.exit(1)
     ret.append(_FilePair(target, generated))
 
   return ret
@@ -87,9 +116,8 @@ def _GetMissingAndStaleFiles(file_pairs):
       missing_files.append(pair)
       continue
 
-    generated = open(pair.generated).read()
+    with open(pair.generated) as g, open(pair.target) as t:
-    target = open(pair.target).read()
+      if g.read() != t.read():
-    if generated != target:
         stale_files.append(pair)
 
   return missing_files, stale_files

doc/render.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+import subprocess
+import sys
+import shutil
+import os
+
+if len(sys.argv) < 2:
+    print("Must pass a filename argument")
+    sys.exit(1)
+
+in_filename = sys.argv[1]
+out_filename = in_filename.replace(".in.md", ".md")
+out_dir = in_filename.replace(".in.md", "")
+
+if in_filename == out_filename:
+    print("File must end in .in.md")
+    sys.exit(1)
+
+if os.path.isdir(out_dir):
+    shutil.rmtree(out_dir)
+
+os.mkdir(out_dir)
+file_num = 1
+
+with open(out_filename, "wb") as out_file, open(in_filename, "rb") as in_file:
+    for line in in_file:
+        if line.startswith(b"```dot"):
+            dot_lines = []
+            while True:
+                dot_line = next(in_file)
+                if dot_line == b"```\n":
+                    break
+                dot_lines.append(dot_line)
+            dot_input = b"".join(dot_lines)
+            svg_filename = out_dir + "/" + str(file_num) + ".svg"
+            svg = subprocess.check_output(['dot', '-Tsvg', '-o', svg_filename], input=dot_input)
+            out_file.write(b"<div align=center>\n")
+            out_file.write(b"<img src='%s'/>\n" % (svg_filename.encode('utf-8')))
+            out_file.write(b"</div>\n")
+            file_num += 1
+        else:
+            out_file.write(line)

doc/vs-cpp-protos.md

@@ -0,0 +1,254 @@
+
+# upb vs. C++ Protobuf Design
+
+[upb](https://github.com/protocolbuffers/upb) is a small C protobuf library.
+While some of the design follows in the footsteps of the C++ Protobuf Library,
+upb departs from C++'s design in several key ways.  This document compares
+and contrasts the two libraries on several design points.
+
+## Design Goals
+
+Before we begin, it is worth calling out that upb and C++ have different design
+goals, and this motivates some of the differences we will see.
+
+C++ protobuf is a user-level library: it is designed to be used directly by C++
+applications.  These applications will expect a full-featured C++ API surface
+that uses C++ idioms.  The C++ library is also willing to add features to
+increase server performance, even if these features would add size or complexity
+to the library.  Because C++ protobuf is a user-level library, API stability is
+of utmost importance: breaking API changes are rare and carefully managed when
+they do occur.  The focus on C++ also means that ABI compatibility with C is not
+a priority.
+
+upb, on the other hand, is designed primarily to be wrapped by other languages.
+It is a C protobuf kernel that forms the basis on which a user-level protobuf
+library can be built.  This means we prefer to keep the API surface as small and
+orthogonal as possible.  While upb supports all protobuf features required for
+full conformance, upb prioritizes simplicity and small code size, and avoids
+adding features like lazy fields that can accelerate some use cases but at great
+cost in terms of complexity.  As upb is not aimed directly at users, there is
+much more freedom to make API-breaking changes when necessary, which helps the
+core to stay small and simple.  We want to be compatible with all FFI
+interfaces, so C ABI compatibility is a must.
+
+Despite these differences, C++ protos and upb offer [roughly the same core set
+of features](https://github.com/protocolbuffers/upb#features).
+
+## Arenas
+
+upb and C++ protos both offer arena allocation, but there are some key
+differences.
+
+### C++
+
+As a matter of history, when C++ protos were open-sourced in 2008, they did not
+support arenas.  Originally there was only unique ownership, whereby each
+message uniquely owns all child messages and will free them when the parent is
+freed.
+
+Arena allocation was added as a feature in 2014 as a way of dramatically
+reducing allocation and (especially) deallocation costs.  But the library was
+not at liberty to remove the unique ownership model, because it would break far
+too many users.  As a result, C++ has supported a **hybrid allocation model**
+ever since, allowing users to allocate messages either directly from the
+stack/heap or from an arena.  The library attempts to ensure that there are
+no dangling pointers by performing automatic copies in some cases (for example
+`a->set_allocated_b(b)`, where `a` and `b` are on different arenas).
+
+C++'s arena object itself `google::protobuf::Arena` is **thread-safe** by
+design, which allows users to allocate from multiple threads simultaneously
+without external synchronization.  The user can supply an initial block of
+memory to the arena, and can choose some parameters to control the arena block
+size.  The user can also supply block alloc/dealloc functions, but the alloc
+function is expected to always return some memory.  The C++ library in general
+does not attempt to handle out of memory conditions.
+
+### upb
+
+upb uses **arena allocation exclusively**. All messages must be allocated from
+an arena, and can only be freed by freeing the arena.  It is entirely the user's
+responsibility to ensure that there are no dangling pointers: when a user sets a
+message field, this will always trivially overwrite the pointer and will never
+perform an implicit copy.
+
+upb's `upb::Arena` is **thread-compatible**, which means it cannot be used
+concurrently without synchronization.  The arena can be seeded with an initial
+block of memory, but it does not explicitly support any parameters for choosing
+block size.  It support a custom alloc/dealloc function, and this function is
+allowed to return `NULL` if no dynamic memory is available.  This allows upb
+arenas to have a max/fixed size, and makes it possible in theory to write code
+that is tolerant to out-of-memory errors.
+
+upb's arena also supports a novel operation known as **fuse**, which joins two
+arenas together into a single lifetime.  Though both arenas must still be freed
+separately, none of the memory will actually be freed until *both* arenas have
+been freed.  This is useful for avoiding dangling pointers when reparenting a
+message with one that may be on a different arena.
+
+### Comparison
+
+**hybrid allocation vs. arena-only**:
+* The C++ hybrid allocation model introduces a great deal of complexity and
+  unpredictability into the library.  upb benefits from having a much simpler
+  and more predictable design.
+* Some of the complexity in C++'s hybrid model arises from the fact that arenas
+  were added after the fact.  Designing for a hybrid model from the outset
+  would likely yield a simpler result.
+* Unique ownership does support some usage patterns that arenas cannot directly
+  accommodate.  For example, you can reparent a message and the child will precisely
+  follow the lifetime of its new parent.  An arena would require you to either
+  perform a deep copy or extend the lifetime.
+
+**thread-compatible vs. thread-safe arena**
+* A thread-safe arena (as in C++) is safer and easier to use.  A thread-compatible
+  arena requires that the user prove that the arena cannot be used concurrently.
+* [Thread Sanitizer](https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual)
+  is far more accessible than it was in 2014 (when C++ introduced a thread-safe
+  arena).  We now have more tools at our disposal to ensure that we do not trigger
+  data races in a thread-compatible arena like upb.
+* Thread-compatible arenas are more performant.
+* Thread-compatible arenas have a far simpler implementation.  The C++ thread-safe
+  arena relies on thread-local variables, which introduce complications on some
+  platforms.  It also requires far more subtle reasoning for correctness and
+  performance.
+
+**fuse vs. no fuse**
+* The `upb_Arena_Fuse()` operation is a key part of how upb supports reparenting
+  of messages when the parent may be on a different arena.  Without this, upb has
+  no way of supporting `foo.bar = bar` in dynamic languages without performing a
+  deep copy.
+* A downside of `upb_Arena_Fuse()` is that passing an arena to a function can allow
+  that function to extend the lifetime of the arena in potentially
+  unpredictable ways.  This can be prevented if necessary, as fuse can fail, eg. if
+  one arena has an initial block.  But this adds some complexity by requiring callers
+  to handle the case where fuse fails.
+
+## Code Generation vs. Tables
+
+The C++ protobuf library has always been built around code generation, while upb
+generates only tables.  In other words, `foo.pb.cc` files contain functions,
+whereas `foo.upb.c` files emit only data structures.
+
+### C++
+
+C++ generated code emits a large number of functions into `foo.pb.cc` files.
+An incomplete list:
+
+* `FooMsg::FooMsg()` (constructor): initializes all fields to their default value.
+* `FooMsg::~FooMsg()` (destructor): frees any present child messages.
+* `FooMsg::Clear()`: clears all fields back to their default/empty value.
+* `FooMsg::_InternalParse()`: generated code for parsing a message.
+* `FooMsg::_InternalSerialize()`: generated code for serializing a message.
+* `FooMsg::ByteSizeLong()`: calculates serialized size, as a first pass before serializing.
+* `FooMsg::MergeFrom()`: copies/appends present fields from another message.
+* `FooMsg::IsInitialized()`: checks whether required fields are set.
+
+This code lives in the `.text` section and contains function calls to the generated
+classes for child messages.
+
+### upb
+
+upb does not generate any code into `foo.upb.c` files, only data structures.  upb uses a
+compact data table known as a *mini table* to represent the schema and all fields.
+
+upb uses mini tables to perform all of the operations that would traditionally be done
+with generated code.  Revisiting the list from the previous section:
+
+* `FooMsg::FooMsg()` (constructor): upb instead initializes all messages with `memset(msg, 0, size)`.
+   Non-zero defaults are injected in the accessors.
+* `FooMsg::~FooMsg()` (destructor): upb messages are freed by freeing the arena.
+* `FooMsg::Clear()`: can be performed with `memset(msg, 0, size)`.
+* `FooMsg::_InternalParse()`: upb's parser uses mini tables as data, instead of generating code.
+* `FooMsg::_InternalSerialize()`: upb's serializer also uses mini-tables instead of generated code.
+* `FooMsg::ByteSizeLong()`: upb performs serialization in reverse so that an initial pass is not required.
+* `FooMsg::MergeFrom()`: upb supports this via serialize+parse from the other message.
+* `FooMsg::IsInitialized()`: upb's encoder and decoder have special flags to check for required fields.
+  A util library `upb/util/required_fields.h` handles the corner cases.
+
+### Comparison
+
+If we compare compiled code size, upb is far smaller.  Here is a comparison of the code
+size of a trivial binary that does nothing but a parse and serialize of `descriptor.proto`.
+This means we are seeing both the overhead of the core library itself as well as the
+generated code (or table) for `descriptor.proto`.  (For extra clarity we should break this
+down by generated code vs core library in the future).
+
+
+| Library         | `.text` | `.data` | `.bss` |
+|------------     |---------|---------|--------|
+| upb             |  26Ki   | 0.6Ki   | 0.01Ki |
+| C++ (lite)      | 187Ki   | 2.8Ki   | 1.25Ki |
+| C++ (code size) | 904Ki   | 6.1Ki   | 1.88Ki |
+| C++ (full)      | 983Ki   | 6.1Ki   | 1.88Ki |
+
+"C++ (code size)" refers to protos compiled with `optimize_for = CODE_SIZE`, a mode
+in which generated code contains reflection only, in an attempt to make the
+generated code size smaller (however it requires the full runtime instead
+of the lite runtime).
+
+## Bifurcated vs. Optional Reflection
+
+upb and C++ protos both offer reflection without making it mandatory.  However
+the models for enabling/disabling reflection are very different.
+
+### C++
+
+C++ messages offer full reflection by default.  Messages in C++ generally
+derive from `Message`, and the base class provides a member function
+`Reflection* Message::GetReflection()` which returns the reflection object.
+
+It follows that any message deriving from `Message` will always have reflection
+linked into the binary, whether or not the reflection object is ever used.
+Because `GetReflection()` is a function on the base class, it is not possible
+to statically determine if a given message's reflection is used:
+
+```c++
+Reflection* GetReflection(const Message& message) {
+    // Can refer to any message in the whole binary.
+    return message.GetReflection();
+}
+```
+
+The C++ library does provide a way of omitting reflection: `MessageLite`.  We can
+cause a message to be lite in two different ways:
+
+* `optimize_for = LITE_RUNTIME` in a `.proto` file will cause all messages in that
+  file to be lite.
+* `lite` as a codegen param: this will force all messages to lite, even if the
+  `.proto` file does not have `optimize_for = LITE_RUNTIME`.
+
+A lite message will derive from `MessageLite` instead of `Message`.  Since
+`MessageLite` has no `GetReflection()` function, this means no reflection is
+available, so we can avoid taking the code size hit.
+
+### upb
+
+upb does not have the `Message` vs. `MessageLite` bifurcation.  There is only one
+kind of message type `upb_Message`, which means there is no need to configure in
+a `.proto` file which messages will need reflection and which will not.
+Every message has the *option* to link in reflection from a separate `foo.upbdefs.o`
+file, without needing to change the message itself in any way.
+
+upb does not provide the equivalent of `Message::GetReflection()`: there is no
+facility for retrieving the reflection of a message whose type is not known statically.
+It would be possible to layer such a facility on top of the upb core, though this
+would probably require some kind of code generation.
+
+### Comparison
+
+* Most messages in C++ will not bother to declare themselves as "lite".  This means
+  that many C++ messages will link in reflection even when it is never used, bloating
+  binaries unnecessarily.
+* `optimize_for = LITE_RUNTIME` is difficult to use in practice, because it prevents 
+  any non-lite protos from `import`ing that file.
+* Forcing all protos to lite via a codegen parameter (for example, when building for
+  mobile) is more practical than `optimize_for = LITE_RUNTIME`.  But this will break
+  the compile for any code that tries to upcast to `Message`, or tries to use a
+  non-lite method.
+* The one major advantage of the C++ model is that it can support `msg.DebugString()`
+  on a type-erased proto.  For upb you have to explicitly pass the `upb_MessageDef*`
+  separately if you want to perform an operation like printing a proto to text format.
+
+## Explicit Registration vs. Globals
+
+TODO

doc/wrapping-upb.in.md

@@ -0,0 +1,372 @@
+
+<!---
+This document contains embedded graphviz diagrams inside ```dot blocks.
+
+To convert it to rendered form using render.py:
+  $ ./render.py wrapping-upb.in.md
+
+You can also live-preview this document with all diagrams using Markdown Preview Enhanced
+in Visual Studio Code:
+  https://marketplace.visualstudio.com/items?itemName=shd101wyy.markdown-preview-enhanced
+--->
+
+# Wrapping upb in other languages
+
+upb is a C kernel that is designed to be wrapped in other languages.  This is a
+guide for creating a new protobuf implementation based on upb.
+
+## What you will need
+
+There are certain things that the language runtime must provide in order to be
+wrapped by upb.
+
+1. **Finalizers, Destructors, or Cleaners**: This is one unavoidable
+   requirement: the language *must* provide finalizers or destructors of some sort.
+   There must be a way of calling a C function when the language GCs or otherwise
+   destroys an object.  We don't care much whether it is a finalizer, a destructor,
+   or a cleaner, as long as it gets called eventually when the object is destroyed.
+   Without finalizers, we would have no way of cleaning up upb data and everything
+   would leak.
+2. **HashMap with weak values**: This is not an absolute requirement, but in
+   languages with automatic memory management, we generally end up wanting a
+   hash map with weak values to act as a `upb_msg* -> wrapper` object cache.
+   We want the values to be weak (not the keys).
+
+## Reflection vs. Direct Access
+
+Each language wrapping upb gets to decide whether it will access messages
+through *reflection* or through *direct access*.  This decision has some deep
+implications that will affect the design, features, and performance of your
+library.
+
+### Reflection
+
+The simplest option is to load full reflection data into the upb library at
+runtime.  You can load reflection data using serialized descriptors, which are a
+stable and widely supported format across all protobuf tooling.
+
+```c
+  // A upb_symtab is a dynamic container that we can load reflection data into.
+  upb_symtab* symtab = upb_symtab_new();
+
+  // We load reflection data via a serialized descriptor.  The code generator
+  // for your language should embed serialized descriptors into your generated
+  // files. For each generated file loaded by your library, you can add the
+  // serialized descriptor to the symtab as shown.
+  upb_arena *tmp = upb_arena_new();
+  google_protobuf_FileDescriptorProto* file =
+      google_protobuf_FileDescriptorProto_parse(desc_data, desc_size, tmp);
+  if (!file || !upb_symtab_addfile(symtab, file, NULL)) {
+    // Handle error.
+  }
+  upb_arena_free(tmp);
+
+  // At application exit, we free the symtab.
+  upb_symtab_free(symtab);
+```
+
+The `upb_symtab` will give you full access to all data from the `.proto` file,
+including convenient APIs like looking up a field by name. It will allow you to
+use JSON and text format.  The APIs for accessing a message through reflection
+are simple and well-supported.  These APIs cleanly encapsulate upb's internal
+implementation details.  
+
+```c
+  upb_symtab* symtab = BuildSymtab();
+
+  // Look up a message type in the symtab.
+  const upb_msgdef* m = upb_symtab_lookupmsg(symtab, "FooMessage");
+
+  // Construct a new message of this type, via reflection.
+  upb_arena *arena = upb_arena_new();
+  upb_msg *msg = upb_msg_new(m, arena);
+
+  // Set a message field using reflection.
+  const upb_fielddef* f = upb_msgdef_ntof("bar_field");
+  upb_msgval val = {.int32_val = 123};
+  upb_msg_set(m, f, val, arena);
+
+  // Free the message and symtab.
+  upb_arena_free(arena);
+  upb_symtab_free(symtab);
+```
+
+Using reflection is a natural choice in heavily reflective, dynamic runtimes
+like Python, Ruby, PHP, or Lua.  These languages generally perform method
+dispatch through a dictionary/hash table anyway, so we are not adding any extra
+overhead by using upb's hash table to lookup fields by name at field access
+time.
+
+### Direct Access
+
+Using reflection has some downsides.  Reflection data is relatively large, both
+in your binary (at rest) and in RAM (at runtime).  It contains names of
+everything, and these names will be exposed in your binary.  Reflection APIs for
+accessing a message will have more overhead than you might want, especially if
+crossing the FFI boundary for your language runtime imposes significant
+overhead.
+
+We can reduce these overheads by using *direct access*.  upb's parser and
+serializer do not actually require full reflection data, they use a more compact
+data structure known as **mini tables**.  Mini tables will take up less space
+than reflection, both in the binary and in RAM, and they will not leak field
+names.  Mini tables will let us parse and serialize binary wire format data
+without reflection.
+
+```c
+  // TODO: demonstrate upb API for loading mini table data at runtime.
+  // This API does not exist yet.
+```
+
+To access messages themselves without the reflection API, we will be using
+different, lower-level APIs that will require you to supply precise data such as
+the offset of a given field.  This is information that will come from the upb
+compiler framework, and the correctness (and even memory safety!) of the program
+will rely on you passing these values through from the upb compiler libraries to
+the upb runtime correctly.
+
+```c
+  // TODO: demonstrate using low-level APIs for direct field access.
+  // These APIs do not exist yet.
+```
+
+It can even be possible in certain circumstances to bypass the upb API completely
+and access raw field data directly at a given offset, using unsafe APIs like
+`sun.misc.unsafe`.  This can theoretically allow for field access that is no
+more expensive than referencing a struct/class field.
+
+```java
+import sun.misc.Unsafe;
+
+class FooProto {
+  private final long addr;
+  private final Arena arena;
+
+  // Accessor that a Java library built on upb could conceivably generate.
+  long getFoo() {
+    // The offset 1234 came from the upb compiler library, and was injected by the
+    // Java+upb code generator.
+    return Unsafe.getLong(self.addr + 1234);
+  }
+}
+```
+
+It is always possible to load reflection data as desired, even if your library
+is designed primarily around direct access.  Users who want to use JSON, text
+format, or reflection could potentially load reflection data from separate
+generated modules, for cases where they do not mind the size overhead or the
+leaking of field names. You do not give up any of these possibilities by using
+direct access.
+
+However, using direct access does have some noticeable downsides.  It requires
+tighter coupling with upb's implementation details, as the mini table format is
+upb-specific and requires building your code generator against upb's compiler
+libraries.  Any direct access of memory is especially tightly coupled, and would
+need to be changed if upb's in-memory format ever changes.  It also is more
+prone to hard-to-debug memory errors if you make any mistakes.
+
+## Memory Management
+
+One of the core design challenges when wrapping upb is memory management.  Every
+language runtime will have some memory management system, whether it is
+garbage collection, reference counting, manual memory management, or some hybrid
+of these.  upb is written in C and uses arenas for memory management, but upb is
+designed to integrate with a wide variety of memory management schemes, and it
+provides a number of tools for making this integration as smooth as possible.
+
+### Arenas
+
+upb defines data structures in C to represent messages, arrays (repeated
+fields), and maps.  A protobuf message is a hierarchical tree of these objects.
+For example, a relatively simple protobuf tree might look something like this:
+
+```dot {align="center"}
+digraph G {
+  rankdir=LR; 
+  newrank=true;
+  node [style="rounded,filled" shape=box colorscheme=accent8 fillcolor=1, ordering=out]
+  upb_msg -> upb_msg2;
+  upb_msg -> upb_array;
+  upb_msg [label="upb Message" fillcolor=1]
+  upb_msg2 [label="upb Message"];
+  upb_array [label="upb Array"]
+}
+```
+
+All upb objects are allocated from an arena.  An arena lets you allocate objects
+individually, but you cannot free individual objects; you can only free the arena
+as a whole.  When the arena is freed, all of the individual objects allocated
+from that arena are freed together.
+
+```dot {align="center"}
+digraph G {
+  rankdir=LR;
+  newrank=true;
+  subgraph cluster_0 {
+    label = "upb Arena"
+    graph[style="rounded,filled" fillcolor=gray]
+    node [style="rounded,filled" shape=box colorscheme=accent8 fillcolor=1, ordering=out]
+    upb_msg -> upb_array;
+    upb_msg -> upb_msg2;
+    upb_msg [label="upb Message" fillcolor=1]
+    upb_msg2 [label="upb Message"];
+    upb_array [label="upb Array"];
+  }
+}
+```
+
+In simple cases, the entire tree of objects will all live in a single arena.
+This has the nice property that there cannot be any dangling pointers between
+objects, since all objects are freed at the same time.
+
+However upb allows you to create links between any two objects, whether or
+not they are in the same arena.  The library does not know or care what arenas
+the objects are in when you create links between them.
+
+```dot {align="center"}
+digraph G {
+  rankdir=LR;
+  newrank=true;
+  subgraph cluster_0 {
+    label = "upb Arena 1"
+    graph[style="rounded,filled" fillcolor=gray]
+    node [style="rounded,filled" shape=box colorscheme=accent8 fillcolor=1, ordering=out]
+    upb_msg -> upb_array;
+    upb_msg -> upb_msg2;
+    upb_msg [label="upb Message 1" fillcolor=1]
+    upb_msg2 [label="upb Message 2"];
+    upb_array [label="upb Array"];
+  }
+  subgraph cluster_1 {
+    label = "upb Arena 2"
+    graph[style="rounded,filled" fillcolor=gray]
+    node [style="rounded,filled" shape=box colorscheme=accent8 fillcolor=1]
+    upb_msg3;
+  }
+  upb_msg2 -> upb_msg3;
+  upb_msg3 [label="upb Message 3"];
+}
+```
+
+When objects are on separate arenas, it is the user's responsibility to ensure
+that there are no dangling pointers.  In the example above, this means Arena 2
+must outlive Message 1 and Message 2.
+
+### Integrating GC with upb
+
+In languages with automatic memory management, the goal is to handle all of the
+arenas behind the scenes, so that the user does not have to manage them manually
+or even know that they exist.
+
+We can achieve this goal if we set up the object graph in a particular way.  The
+general strategy is to create wrapper objects around all of the C objects,
+including the arena.  Our key goal is to make sure the arena wrapper is not
+GC'd until all of the C objects in that arena have become unreachable.
+
+For this example, we will assume we are wrapping upb in Python:
+
+```dot {align="center"}
+digraph G { 
+  rankdir=LR;
+  newrank=true;
+  compound=true;
+  
+  subgraph cluster_1 {
+    label = "upb Arena"
+    graph[style="rounded,filled" fillcolor=gray]
+    node [style="rounded,filled" shape=box colorscheme=accent8 fillcolor=1, ordering=out]
+    upb_msg -> upb_array [style=dashed];
+    upb_msg -> upb_msg2 [style=dashed];
+    upb_msg [label="upb Message" fillcolor=1]
+    upb_msg2 [label="upb Message"];
+    upb_array [label="upb Array"]
+    dummy [style=invis]
+  }
+  subgraph cluster_python {
+    node [style="rounded,filled" shape=box colorscheme=accent8 fillcolor=2]
+    peripheries=0
+    py_upb_msg [label="Python Message"];
+    py_upb_msg2 [label="Python Message"];
+    py_upb_arena [label="Python Arena"];
+  }
+  py_upb_msg -> upb_msg [style=dashed];
+  py_upb_msg2->upb_msg2 [style=dashed];
+  py_upb_msg2 -> py_upb_arena [color=springgreen4];
+  py_upb_msg -> py_upb_arena [color=springgreen4];
+  py_upb_arena -> dummy [lhead=cluster_1, color=red];
+  {
+     rank=same;
+     upb_msg;
+     py_upb_msg;
+  }
+  {
+     rank=same;
+     upb_array;
+     upb_msg2;
+     py_upb_msg2;
+  }
+  {  rank=same;
+     dummy;
+     py_upb_arena;
+  }
+  dummy->upb_array [style=invis];
+  dummy->upb_msg2 [style=invis];
+
+  subgraph cluster_01 { 
+    node [shape=plaintext]
+    peripheries=0
+    key [label=<<table border="0" cellpadding="2" cellspacing="0" cellborder="0">
+      <tr><td align="right" port="i1">raw ptr</td></tr>
+      <tr><td align="right" port="i2">unique ptr</td></tr>
+      <tr><td align="right" port="i3">shared (GC) ptr</td></tr>
+      </table>>]
+    key2 [label=<<table border="0" cellpadding="2" cellspacing="0" cellborder="0">
+      <tr><td port="i1">&nbsp;</td></tr>
+      <tr><td port="i2">&nbsp;</td></tr>
+      <tr><td port="i3">&nbsp;</td></tr>
+      </table>>]
+    key:i1:e -> key2:i1:w [style=dashed]
+    key:i2:e -> key2:i2:w [color=red]
+    key:i3:e -> key2:i3:w [color=springgreen4]
+  }
+    key2:i1:w -> upb_msg [style=invis];
+  {
+    rank=same;
+    key;
+    upb_msg;
+  }
+}
+```
+
+In this example we have three different kinds of pointers:
+
+* **raw ptr**: This is a pointer that carries no ownership.
+* **unique ptr**: This is a pointer has *unique ownership* of the target.  The owner
+  will free the target in its destructor (or finalizer, or cleaner).  There can
+  only be a single unique pointer to a given object.
+* **shared (GC) ptr**: This is a pointer that has *shared ownership* of the
+  target.  Many objects can point to the target, and the target will be deleted
+  only when all such references are gone.  In a runtime with automatic memory
+  management (GC), this is a reference that participates in GC.  In Python such
+  references use reference counting, but in other VMs they may use mark and
+  sweep or some other form of GC instead.
+
+The Python Message wrappers have only raw pointers to the underlying message,
+but they contain a shared pointer to the arena that will ensure that the raw
+pointer remains valid.  Only when all message wrapper objects are destroyed
+will the Python Arena become unreachable, and the upb arena ultimately freed.
+
+### Links between arenas with "Fuse"
+
+The design given above works well for objects that live in a single arena. But
+what if a user wants to create a link between two objects in different arenas?
+
+TODO
+
+## UTF-8 vs. UTF-16
+
+TODO
+
+## Object Cache
+
+TODO

doc/wrapping-upb.md

@@ -0,0 +1,260 @@
+
+<!---
+This document contains embedded graphviz diagrams inside ```dot blocks.
+
+To convert it to rendered form using render.py:
+  $ ./render.py wrapping-upb.in.md
+
+You can also live-preview this document with all diagrams using Markdown Preview Enhanced
+in Visual Studio Code:
+  https://marketplace.visualstudio.com/items?itemName=shd101wyy.markdown-preview-enhanced
+--->
+
+# Wrapping upb in other languages
+
+upb is a C kernel that is designed to be wrapped in other languages.  This is a
+guide for creating a new protobuf implementation based on upb.
+
+## What you will need
+
+There are certain things that the language runtime must provide in order to be
+wrapped by upb.
+
+1. **Finalizers, Destructors, or Cleaners**: This is one unavoidable
+   requirement: the language *must* provide finalizers or destructors of some sort.
+   There must be a way of calling a C function when the language GCs or otherwise
+   destroys an object.  We don't care much whether it is a finalizer, a destructor,
+   or a cleaner, as long as it gets called eventually when the object is destroyed.
+   Without finalizers, we would have no way of cleaning up upb data and everything
+   would leak.
+2. **HashMap with weak values**: This is not an absolute requirement, but in
+   languages with automatic memory management, we generally end up wanting a
+   hash map with weak values to act as a `upb_msg* -> wrapper` object cache.
+   We want the values to be weak (not the keys).
+
+## Reflection vs. Direct Access
+
+Each language wrapping upb gets to decide whether it will access messages
+through *reflection* or through *direct access*.  This decision has some deep
+implications that will affect the design, features, and performance of your
+library.
+
+### Reflection
+
+The simplest option is to load full reflection data into the upb library at
+runtime.  You can load reflection data using serialized descriptors, which are a
+stable and widely supported format across all protobuf tooling.
+
+```c
+  // A upb_symtab is a dynamic container that we can load reflection data into.
+  upb_symtab* symtab = upb_symtab_new();
+
+  // We load reflection data via a serialized descriptor.  The code generator
+  // for your language should embed serialized descriptors into your generated
+  // files. For each generated file loaded by your library, you can add the
+  // serialized descriptor to the symtab as shown.
+  upb_arena *tmp = upb_arena_new();
+  google_protobuf_FileDescriptorProto* file =
+      google_protobuf_FileDescriptorProto_parse(desc_data, desc_size, tmp);
+  if (!file || !upb_symtab_addfile(symtab, file, NULL)) {
+    // Handle error.
+  }
+  upb_arena_free(tmp);
+
+  // At application exit, we free the symtab.
+  upb_symtab_free(symtab);
+```
+
+The `upb_symtab` will give you full access to all data from the `.proto` file,
+including convenient APIs like looking up a field by name. It will allow you to
+use JSON and text format.  The APIs for accessing a message through reflection
+are simple and well-supported.  These APIs cleanly encapsulate upb's internal
+implementation details.  
+
+```c
+  upb_symtab* symtab = BuildSymtab();
+
+  // Look up a message type in the symtab.
+  const upb_msgdef* m = upb_symtab_lookupmsg(symtab, "FooMessage");
+
+  // Construct a new message of this type, via reflection.
+  upb_arena *arena = upb_arena_new();
+  upb_msg *msg = upb_msg_new(m, arena);
+
+  // Set a message field using reflection.
+  const upb_fielddef* f = upb_msgdef_ntof("bar_field");
+  upb_msgval val = {.int32_val = 123};
+  upb_msg_set(m, f, val, arena);
+
+  // Free the message and symtab.
+  upb_arena_free(arena);
+  upb_symtab_free(symtab);
+```
+
+Using reflection is a natural choice in heavily reflective, dynamic runtimes
+like Python, Ruby, PHP, or Lua.  These languages generally perform method
+dispatch through a dictionary/hash table anyway, so we are not adding any extra
+overhead by using upb's hash table to lookup fields by name at field access
+time.
+
+### Direct Access
+
+Using reflection has some downsides.  Reflection data is relatively large, both
+in your binary (at rest) and in RAM (at runtime).  It contains names of
+everything, and these names will be exposed in your binary.  Reflection APIs for
+accessing a message will have more overhead than you might want, especially if
+crossing the FFI boundary for your language runtime imposes significant
+overhead.
+
+We can reduce these overheads by using *direct access*.  upb's parser and
+serializer do not actually require full reflection data, they use a more compact
+data structure known as **mini tables**.  Mini tables will take up less space
+than reflection, both in the binary and in RAM, and they will not leak field
+names.  Mini tables will let us parse and serialize binary wire format data
+without reflection.
+
+```c
+  // TODO: demonstrate upb API for loading mini table data at runtime.
+  // This API does not exist yet.
+```
+
+To access messages themselves without the reflection API, we will be using
+different, lower-level APIs that will require you to supply precise data such as
+the offset of a given field.  This is information that will come from the upb
+compiler framework, and the correctness (and even memory safety!) of the program
+will rely on you passing these values through from the upb compiler libraries to
+the upb runtime correctly.
+
+```c
+  // TODO: demonstrate using low-level APIs for direct field access.
+  // These APIs do not exist yet.
+```
+
+It can even be possible in certain circumstances to bypass the upb API completely
+and access raw field data directly at a given offset, using unsafe APIs like
+`sun.misc.unsafe`.  This can theoretically allow for field access that is no
+more expensive than referencing a struct/class field.
+
+```java
+import sun.misc.Unsafe;
+
+class FooProto {
+  private final long addr;
+  private final Arena arena;
+
+  // Accessor that a Java library built on upb could conceivably generate.
+  long getFoo() {
+    // The offset 1234 came from the upb compiler library, and was injected by the
+    // Java+upb code generator.
+    return Unsafe.getLong(self.addr + 1234);
+  }
+}
+```
+
+It is always possible to load reflection data as desired, even if your library
+is designed primarily around direct access.  Users who want to use JSON, text
+format, or reflection could potentially load reflection data from separate
+generated modules, for cases where they do not mind the size overhead or the
+leaking of field names. You do not give up any of these possibilities by using
+direct access.
+
+However, using direct access does have some noticeable downsides.  It requires
+tighter coupling with upb's implementation details, as the mini table format is
+upb-specific and requires building your code generator against upb's compiler
+libraries.  Any direct access of memory is especially tightly coupled, and would
+need to be changed if upb's in-memory format ever changes.  It also is more
+prone to hard-to-debug memory errors if you make any mistakes.
+
+## Memory Management
+
+One of the core design challenges when wrapping upb is memory management.  Every
+language runtime will have some memory management system, whether it is
+garbage collection, reference counting, manual memory management, or some hybrid
+of these.  upb is written in C and uses arenas for memory management, but upb is
+designed to integrate with a wide variety of memory management schemes, and it
+provides a number of tools for making this integration as smooth as possible.
+
+### Arenas
+
+upb defines data structures in C to represent messages, arrays (repeated
+fields), and maps.  A protobuf message is a hierarchical tree of these objects.
+For example, a relatively simple protobuf tree might look something like this:
+
+<div align=center>
+<img src='wrapping-upb/1.svg'/>
+</div>
+
+All upb objects are allocated from an arena.  An arena lets you allocate objects
+individually, but you cannot free individual objects; you can only free the arena
+as a whole.  When the arena is freed, all of the individual objects allocated
+from that arena are freed together.
+
+<div align=center>
+<img src='wrapping-upb/2.svg'/>
+</div>
+
+In simple cases, the entire tree of objects will all live in a single arena.
+This has the nice property that there cannot be any dangling pointers between
+objects, since all objects are freed at the same time.
+
+However upb allows you to create links between any two objects, whether or
+not they are in the same arena.  The library does not know or care what arenas
+the objects are in when you create links between them.
+
+<div align=center>
+<img src='wrapping-upb/3.svg'/>
+</div>
+
+When objects are on separate arenas, it is the user's responsibility to ensure
+that there are no dangling pointers.  In the example above, this means Arena 2
+must outlive Message 1 and Message 2.
+
+### Integrating GC with upb
+
+In languages with automatic memory management, the goal is to handle all of the
+arenas behind the scenes, so that the user does not have to manage them manually
+or even know that they exist.
+
+We can achieve this goal if we set up the object graph in a particular way.  The
+general strategy is to create wrapper objects around all of the C objects,
+including the arena.  Our key goal is to make sure the arena wrapper is not
+GC'd until all of the C objects in that arena have become unreachable.
+
+For this example, we will assume we are wrapping upb in Python:
+
+<div align=center>
+<img src='wrapping-upb/4.svg'/>
+</div>
+
+In this example we have three different kinds of pointers:
+
+* **raw ptr**: This is a pointer that carries no ownership.
+* **unique ptr**: This is a pointer has *unique ownership* of the target.  The owner
+  will free the target in its destructor (or finalizer, or cleaner).  There can
+  only be a single unique pointer to a given object.
+* **shared (GC) ptr**: This is a pointer that has *shared ownership* of the
+  target.  Many objects can point to the target, and the target will be deleted
+  only when all such references are gone.  In a runtime with automatic memory
+  management (GC), this is a reference that participates in GC.  In Python such
+  references use reference counting, but in other VMs they may use mark and
+  sweep or some other form of GC instead.
+
+The Python Message wrappers have only raw pointers to the underlying message,
+but they contain a shared pointer to the arena that will ensure that the raw
+pointer remains valid.  Only when all message wrapper objects are destroyed
+will the Python Arena become unreachable, and the upb arena ultimately freed.
+
+### Links between arenas with "Fuse"
+
+The design given above works well for objects that live in a single arena. But
+what if a user wants to create a link between two objects in different arenas?
+
+TODO
+
+## UTF-8 vs. UTF-16
+
+TODO
+
+## Object Cache
+
+TODO

doc/wrapping-upb/1.svg

@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.43.0 (0)
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="222pt" height="98pt"
+ viewBox="0.00 0.00 222.00 98.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 94)">
+<title>G</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-94 218,-94 218,4 -4,4"/>
+<!-- upb_msg -->
+<g id="node1" class="node">
+<title>upb_msg</title>
+<path fill="#7fc97f" stroke="black" d="M77,-63C77,-63 12,-63 12,-63 6,-63 0,-57 0,-51 0,-51 0,-39 0,-39 0,-33 6,-27 12,-27 12,-27 77,-27 77,-27 83,-27 89,-33 89,-39 89,-39 89,-51 89,-51 89,-57 83,-63 77,-63"/>
+<text text-anchor="middle" x="44.5" y="-41.3" font-family="Times,serif" font-size="14.00">upb Message</text>
+</g>
+<!-- upb_msg2 -->
+<g id="node2" class="node">
+<title>upb_msg2</title>
+<path fill="#7fc97f" stroke="black" d="M202,-90C202,-90 137,-90 137,-90 131,-90 125,-84 125,-78 125,-78 125,-66 125,-66 125,-60 131,-54 137,-54 137,-54 202,-54 202,-54 208,-54 214,-60 214,-66 214,-66 214,-78 214,-78 214,-84 208,-90 202,-90"/>
+<text text-anchor="middle" x="169.5" y="-68.3" font-family="Times,serif" font-size="14.00">upb Message</text>
+</g>
+<!-- upb_msg&#45;&gt;upb_msg2 -->
+<g id="edge1" class="edge">
+<title>upb_msg&#45;&gt;upb_msg2</title>
+<path fill="none" stroke="black" d="M89.21,-54.6C97.55,-56.43 106.36,-58.36 114.97,-60.25"/>
+<polygon fill="black" stroke="black" points="114.41,-63.71 124.93,-62.44 115.91,-56.87 114.41,-63.71"/>
+</g>
+<!-- upb_array -->
+<g id="node3" class="node">
+<title>upb_array</title>
+<path fill="#7fc97f" stroke="black" d="M193,-36C193,-36 146,-36 146,-36 140,-36 134,-30 134,-24 134,-24 134,-12 134,-12 134,-6 140,0 146,0 146,0 193,0 193,0 199,0 205,-6 205,-12 205,-12 205,-24 205,-24 205,-30 199,-36 193,-36"/>
+<text text-anchor="middle" x="169.5" y="-14.3" font-family="Times,serif" font-size="14.00">upb Array</text>
+</g>
+<!-- upb_msg&#45;&gt;upb_array -->
+<g id="edge2" class="edge">
+<title>upb_msg&#45;&gt;upb_array</title>
+<path fill="none" stroke="black" d="M89.21,-35.4C100.42,-32.94 112.49,-30.3 123.75,-27.82"/>
+<polygon fill="black" stroke="black" points="124.8,-31.18 133.81,-25.61 123.3,-24.34 124.8,-31.18"/>
+</g>
+</g>
+</svg>

doc/wrapping-upb/2.svg

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.43.0 (0)
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="254pt" height="153pt"
+ viewBox="0.00 0.00 254.00 153.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 149)">
+<title>G</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-149 250,-149 250,4 -4,4"/>
+<g id="clust1" class="cluster">
+<title>cluster_0</title>
+<path fill="gray" stroke="black" d="M20,-8C20,-8 226,-8 226,-8 232,-8 238,-14 238,-20 238,-20 238,-125 238,-125 238,-131 232,-137 226,-137 226,-137 20,-137 20,-137 14,-137 8,-131 8,-125 8,-125 8,-20 8,-20 8,-14 14,-8 20,-8"/>
+<text text-anchor="middle" x="123" y="-121.8" font-family="Times,serif" font-size="14.00">upb Arena</text>
+</g>
+<!-- upb_msg -->
+<g id="node1" class="node">
+<title>upb_msg</title>
+<path fill="#7fc97f" stroke="black" d="M93,-79C93,-79 28,-79 28,-79 22,-79 16,-73 16,-67 16,-67 16,-55 16,-55 16,-49 22,-43 28,-43 28,-43 93,-43 93,-43 99,-43 105,-49 105,-55 105,-55 105,-67 105,-67 105,-73 99,-79 93,-79"/>
+<text text-anchor="middle" x="60.5" y="-57.3" font-family="Times,serif" font-size="14.00">upb Message</text>
+</g>
+<!-- upb_array -->
+<g id="node2" class="node">
+<title>upb_array</title>
+<path fill="#7fc97f" stroke="black" d="M209,-52C209,-52 162,-52 162,-52 156,-52 150,-46 150,-40 150,-40 150,-28 150,-28 150,-22 156,-16 162,-16 162,-16 209,-16 209,-16 215,-16 221,-22 221,-28 221,-28 221,-40 221,-40 221,-46 215,-52 209,-52"/>
+<text text-anchor="middle" x="185.5" y="-30.3" font-family="Times,serif" font-size="14.00">upb Array</text>
+</g>
+<!-- upb_msg&#45;&gt;upb_array -->
+<g id="edge1" class="edge">
+<title>upb_msg&#45;&gt;upb_array</title>
+<path fill="none" stroke="black" d="M105.21,-51.4C116.42,-48.94 128.49,-46.3 139.75,-43.82"/>
+<polygon fill="black" stroke="black" points="140.8,-47.18 149.81,-41.61 139.3,-40.34 140.8,-47.18"/>
+</g>
+<!-- upb_msg2 -->
+<g id="node3" class="node">
+<title>upb_msg2</title>
+<path fill="#7fc97f" stroke="black" d="M218,-106C218,-106 153,-106 153,-106 147,-106 141,-100 141,-94 141,-94 141,-82 141,-82 141,-76 147,-70 153,-70 153,-70 218,-70 218,-70 224,-70 230,-76 230,-82 230,-82 230,-94 230,-94 230,-100 224,-106 218,-106"/>
+<text text-anchor="middle" x="185.5" y="-84.3" font-family="Times,serif" font-size="14.00">upb Message</text>
+</g>
+<!-- upb_msg&#45;&gt;upb_msg2 -->
+<g id="edge2" class="edge">
+<title>upb_msg&#45;&gt;upb_msg2</title>
+<path fill="none" stroke="black" d="M105.21,-70.6C113.55,-72.43 122.36,-74.36 130.97,-76.25"/>
+<polygon fill="black" stroke="black" points="130.41,-79.71 140.93,-78.44 131.91,-72.87 130.41,-79.71"/>
+</g>
+</g>
+</svg>

doc/wrapping-upb/3.svg

@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.43.0 (0)
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="409pt" height="153pt"
+ viewBox="0.00 0.00 409.00 153.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 149)">
+<title>G</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-149 405,-149 405,4 -4,4"/>
+<g id="clust1" class="cluster">
+<title>cluster_0</title>
+<path fill="gray" stroke="black" d="M20,-8C20,-8 246,-8 246,-8 252,-8 258,-14 258,-20 258,-20 258,-125 258,-125 258,-131 252,-137 246,-137 246,-137 20,-137 20,-137 14,-137 8,-131 8,-125 8,-125 8,-20 8,-20 8,-14 14,-8 20,-8"/>
+<text text-anchor="middle" x="133" y="-121.8" font-family="Times,serif" font-size="14.00">upb Arena 1</text>
+</g>
+<g id="clust2" class="cluster">
+<title>cluster_1</title>
+<path fill="gray" stroke="black" d="M290,-62C290,-62 381,-62 381,-62 387,-62 393,-68 393,-74 393,-74 393,-125 393,-125 393,-131 387,-137 381,-137 381,-137 290,-137 290,-137 284,-137 278,-131 278,-125 278,-125 278,-74 278,-74 278,-68 284,-62 290,-62"/>
+<text text-anchor="middle" x="335.5" y="-121.8" font-family="Times,serif" font-size="14.00">upb Arena 2</text>
+</g>
+<!-- upb_msg -->
+<g id="node1" class="node">
+<title>upb_msg</title>
+<path fill="#7fc97f" stroke="black" d="M103,-79C103,-79 28,-79 28,-79 22,-79 16,-73 16,-67 16,-67 16,-55 16,-55 16,-49 22,-43 28,-43 28,-43 103,-43 103,-43 109,-43 115,-49 115,-55 115,-55 115,-67 115,-67 115,-73 109,-79 103,-79"/>
+<text text-anchor="middle" x="65.5" y="-57.3" font-family="Times,serif" font-size="14.00">upb Message 1</text>
+</g>
+<!-- upb_array -->
+<g id="node2" class="node">
+<title>upb_array</title>
+<path fill="#7fc97f" stroke="black" d="M224,-52C224,-52 177,-52 177,-52 171,-52 165,-46 165,-40 165,-40 165,-28 165,-28 165,-22 171,-16 177,-16 177,-16 224,-16 224,-16 230,-16 236,-22 236,-28 236,-28 236,-40 236,-40 236,-46 230,-52 224,-52"/>
+<text text-anchor="middle" x="200.5" y="-30.3" font-family="Times,serif" font-size="14.00">upb Array</text>
+</g>
+<!-- upb_msg&#45;&gt;upb_array -->
+<g id="edge1" class="edge">
+<title>upb_msg&#45;&gt;upb_array</title>
+<path fill="none" stroke="black" d="M115.27,-51.1C128.26,-48.46 142.22,-45.63 154.96,-43.04"/>
+<polygon fill="black" stroke="black" points="155.78,-46.45 164.88,-41.03 154.39,-39.59 155.78,-46.45"/>
+</g>
+<!-- upb_msg2 -->
+<g id="node3" class="node">
+<title>upb_msg2</title>
+<path fill="#7fc97f" stroke="black" d="M238,-106C238,-106 163,-106 163,-106 157,-106 151,-100 151,-94 151,-94 151,-82 151,-82 151,-76 157,-70 163,-70 163,-70 238,-70 238,-70 244,-70 250,-76 250,-82 250,-82 250,-94 250,-94 250,-100 244,-106 238,-106"/>
+<text text-anchor="middle" x="200.5" y="-84.3" font-family="Times,serif" font-size="14.00">upb Message 2</text>
+</g>
+<!-- upb_msg&#45;&gt;upb_msg2 -->
+<g id="edge2" class="edge">
+<title>upb_msg&#45;&gt;upb_msg2</title>
+<path fill="none" stroke="black" d="M115.27,-70.9C123.58,-72.59 132.29,-74.36 140.83,-76.09"/>
+<polygon fill="black" stroke="black" points="140.24,-79.54 150.74,-78.1 141.63,-72.68 140.24,-79.54"/>
+</g>
+<!-- upb_msg3 -->
+<g id="node4" class="node">
+<title>upb_msg3</title>
+<path fill="#7fc97f" stroke="black" d="M373,-106C373,-106 298,-106 298,-106 292,-106 286,-100 286,-94 286,-94 286,-82 286,-82 286,-76 292,-70 298,-70 298,-70 373,-70 373,-70 379,-70 385,-76 385,-82 385,-82 385,-94 385,-94 385,-100 379,-106 373,-106"/>
+<text text-anchor="middle" x="335.5" y="-84.3" font-family="Times,serif" font-size="14.00">upb Message 3</text>
+</g>
+<!-- upb_msg2&#45;&gt;upb_msg3 -->
+<g id="edge3" class="edge">
+<title>upb_msg2&#45;&gt;upb_msg3</title>
+<path fill="none" stroke="black" d="M250.27,-88C258.49,-88 267.1,-88 275.55,-88"/>
+<polygon fill="black" stroke="black" points="275.74,-91.5 285.74,-88 275.74,-84.5 275.74,-91.5"/>
+</g>
+</g>
+</svg>

doc/wrapping-upb/4.svg

@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.43.0 (0)
+ -->
+<!-- Title: G Pages: 1 -->
+<svg width="417pt" height="303pt"
+ viewBox="0.00 0.00 417.00 303.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 299)">
+<title>G</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-299 413,-299 413,4 -4,4"/>
+<g id="clust1" class="cluster">
+<title>cluster_1</title>
+<path fill="gray" stroke="black" d="M29,-98C29,-98 380,-98 380,-98 386,-98 392,-104 392,-110 392,-110 392,-215 392,-215 392,-221 386,-227 380,-227 380,-227 29,-227 29,-227 23,-227 17,-221 17,-215 17,-215 17,-110 17,-110 17,-104 23,-98 29,-98"/>
+<text text-anchor="middle" x="204.5" y="-211.8" font-family="Times,serif" font-size="14.00">upb Arena</text>
+</g>
+<g id="clust2" class="cluster">
+<title>cluster_python</title>
+</g>
+<g id="clust6" class="cluster">
+<title>cluster_01</title>
+</g>
+<!-- upb_msg -->
+<g id="node1" class="node">
+<title>upb_msg</title>
+<path fill="#7fc97f" stroke="black" d="M102,-169C102,-169 37,-169 37,-169 31,-169 25,-163 25,-157 25,-157 25,-145 25,-145 25,-139 31,-133 37,-133 37,-133 102,-133 102,-133 108,-133 114,-139 114,-145 114,-145 114,-157 114,-157 114,-163 108,-169 102,-169"/>
+<text text-anchor="middle" x="69.5" y="-147.3" font-family="Times,serif" font-size="14.00">upb Message</text>
+</g>
+<!-- upb_array -->
+<g id="node2" class="node">
+<title>upb_array</title>
+<path fill="#7fc97f" stroke="black" d="M363,-142C363,-142 316,-142 316,-142 310,-142 304,-136 304,-130 304,-130 304,-118 304,-118 304,-112 310,-106 316,-106 316,-106 363,-106 363,-106 369,-106 375,-112 375,-118 375,-118 375,-130 375,-130 375,-136 369,-142 363,-142"/>
+<text text-anchor="middle" x="339.5" y="-120.3" font-family="Times,serif" font-size="14.00">upb Array</text>
+</g>
+<!-- upb_msg&#45;&gt;upb_array -->
+<g id="edge1" class="edge">
+<title>upb_msg&#45;&gt;upb_array</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M114.33,-139.43C128.49,-136.14 144.32,-132.93 159,-131 204.59,-125.01 257.21,-123.6 293.64,-123.48"/>
+<polygon fill="black" stroke="black" points="293.93,-126.98 303.93,-123.48 293.93,-119.98 293.93,-126.98"/>
+</g>
+<!-- upb_msg2 -->
+<g id="node3" class="node">
+<title>upb_msg2</title>
+<path fill="#7fc97f" stroke="black" d="M372,-196C372,-196 307,-196 307,-196 301,-196 295,-190 295,-184 295,-184 295,-172 295,-172 295,-166 301,-160 307,-160 307,-160 372,-160 372,-160 378,-160 384,-166 384,-172 384,-172 384,-184 384,-184 384,-190 378,-196 372,-196"/>
+<text text-anchor="middle" x="339.5" y="-174.3" font-family="Times,serif" font-size="14.00">upb Message</text>
+</g>
+<!-- upb_msg&#45;&gt;upb_msg2 -->
+<g id="edge2" class="edge">
+<title>upb_msg&#45;&gt;upb_msg2</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M108.38,-169.06C123.72,-175.44 141.85,-181.82 159,-185 200.72,-192.74 248.8,-190.27 284.7,-186.27"/>
+<polygon fill="black" stroke="black" points="285.44,-189.7 294.95,-185.04 284.61,-182.75 285.44,-189.7"/>
+</g>
+<!-- dummy -->
+<!-- dummy&#45;&gt;upb_array -->
+<!-- dummy&#45;&gt;upb_msg2 -->
+<!-- py_upb_msg -->
+<g id="node5" class="node">
+<title>py_upb_msg</title>
+<path fill="#beaed4" stroke="black" d="M111,-279C111,-279 28,-279 28,-279 22,-279 16,-273 16,-267 16,-267 16,-255 16,-255 16,-249 22,-243 28,-243 28,-243 111,-243 111,-243 117,-243 123,-249 123,-255 123,-255 123,-267 123,-267 123,-273 117,-279 111,-279"/>
+<text text-anchor="middle" x="69.5" y="-257.3" font-family="Times,serif" font-size="14.00">Python Message</text>
+</g>
+<!-- py_upb_msg&#45;&gt;upb_msg -->
+<g id="edge3" class="edge">
+<title>py_upb_msg&#45;&gt;upb_msg</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M69.5,-242.68C69.5,-221.49 69.5,-200.3 69.5,-179.11"/>
+<polygon fill="black" stroke="black" points="73,-179.05 69.5,-169.05 66,-179.05 73,-179.05"/>
+</g>
+<!-- py_upb_arena -->
+<g id="node7" class="node">
+<title>py_upb_arena</title>
+<path fill="#beaed4" stroke="black" d="M238,-279C238,-279 171,-279 171,-279 165,-279 159,-273 159,-267 159,-267 159,-255 159,-255 159,-249 165,-243 171,-243 171,-243 238,-243 238,-243 244,-243 250,-249 250,-255 250,-255 250,-267 250,-267 250,-273 244,-279 238,-279"/>
+<text text-anchor="middle" x="204.5" y="-257.3" font-family="Times,serif" font-size="14.00">Python Arena</text>
+</g>
+<!-- py_upb_msg&#45;&gt;py_upb_arena -->
+<g id="edge6" class="edge">
+<title>py_upb_msg&#45;&gt;py_upb_arena</title>
+<path fill="none" stroke="#008b45" d="M123.07,-261C131.43,-261 140.1,-261 148.51,-261"/>
+<polygon fill="#008b45" stroke="#008b45" points="148.62,-264.5 158.62,-261 148.62,-257.5 148.62,-264.5"/>
+</g>
+<!-- py_upb_msg2 -->
+<g id="node6" class="node">
+<title>py_upb_msg2</title>
+<path fill="#beaed4" stroke="black" d="M381,-279C381,-279 298,-279 298,-279 292,-279 286,-273 286,-267 286,-267 286,-255 286,-255 286,-249 292,-243 298,-243 298,-243 381,-243 381,-243 387,-243 393,-249 393,-255 393,-255 393,-267 393,-267 393,-273 387,-279 381,-279"/>
+<text text-anchor="middle" x="339.5" y="-257.3" font-family="Times,serif" font-size="14.00">Python Message</text>
+</g>
+<!-- py_upb_msg2&#45;&gt;upb_msg2 -->
+<g id="edge4" class="edge">
+<title>py_upb_msg2&#45;&gt;upb_msg2</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M339.5,-242.76C339.5,-230.63 339.5,-218.49 339.5,-206.35"/>
+<polygon fill="black" stroke="black" points="343,-206.16 339.5,-196.16 336,-206.16 343,-206.16"/>
+</g>
+<!-- py_upb_msg2&#45;&gt;py_upb_arena -->
+<g id="edge5" class="edge">
+<title>py_upb_msg2&#45;&gt;py_upb_arena</title>
+<path fill="none" stroke="#008b45" d="M285.91,-261C277.49,-261 268.77,-261 260.31,-261"/>
+<polygon fill="#008b45" stroke="#008b45" points="260.16,-257.5 250.16,-261 260.16,-264.5 260.16,-257.5"/>
+</g>
+<!-- py_upb_arena&#45;&gt;dummy -->
+<g id="edge7" class="edge">
+<title>py_upb_arena&#45;&gt;dummy</title>
+<path fill="none" stroke="red" d="M204.5,-242.76C204.5,-240.95 204.5,-239.15 204.5,-237.34"/>
+<polygon fill="red" stroke="red" points="208,-237 204.5,-227 201,-237 208,-237"/>
+</g>
+<!-- key -->
+<g id="node8" class="node">
+<title>key</title>
+<text text-anchor="start" x="72.5" y="-63.8" font-family="Times,serif" font-size="14.00">raw ptr</text>
+<text text-anchor="start" x="56.5" y="-44.8" font-family="Times,serif" font-size="14.00">unique ptr</text>
+<text text-anchor="start" x="26.5" y="-25.8" font-family="Times,serif" font-size="14.00">shared (GC) ptr</text>
+</g>
+<!-- key2 -->
+<g id="node9" class="node">
+<title>key2</title>
+<text text-anchor="start" x="202.5" y="-63.8" font-family="Times,serif" font-size="14.00"> </text>
+<text text-anchor="start" x="202.5" y="-44.8" font-family="Times,serif" font-size="14.00"> </text>
+<text text-anchor="start" x="202.5" y="-25.8" font-family="Times,serif" font-size="14.00"> </text>
+</g>
+<!-- key&#45;&gt;key2 -->
+<g id="edge10" class="edge">
+<title>key:e&#45;&gt;key2:w</title>
+<path fill="none" stroke="black" stroke-dasharray="5,2" d="M115.5,-68C149.19,-68 160.08,-68 189.31,-68"/>
+<polygon fill="black" stroke="black" points="189.5,-71.5 199.5,-68 189.5,-64.5 189.5,-71.5"/>
+</g>
+<!-- key&#45;&gt;key2 -->
+<g id="edge11" class="edge">
+<title>key:e&#45;&gt;key2:w</title>
+<path fill="none" stroke="red" d="M115.5,-48C149.19,-48 160.08,-48 189.31,-48"/>
+<polygon fill="red" stroke="red" points="189.5,-51.5 199.5,-48 189.5,-44.5 189.5,-51.5"/>
+</g>
+<!-- key&#45;&gt;key2 -->
+<g id="edge12" class="edge">
+<title>key:e&#45;&gt;key2:w</title>
+<path fill="none" stroke="#008b45" d="M115.5,-29C149.19,-29 160.08,-29 189.31,-29"/>
+<polygon fill="#008b45" stroke="#008b45" points="189.5,-32.5 199.5,-29 189.5,-25.5 189.5,-32.5"/>
+</g>
+<!-- key2&#45;&gt;upb_msg -->
+</g>
+</svg>

examples/bazel/BUILD

@@ -1,18 +0,0 @@
-
-load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library")
-
-proto_library(
-    name = "foo_proto",
-    srcs = ["foo.proto"],
-)
-
-upb_proto_library(
-    name = "foo_upbproto",
-    deps = [":foo_proto"],
-)
-
-cc_binary(
-    name = "test_binary",
-    srcs = ["test_binary.c"],
-    deps = [":foo_upbproto"],
-)

examples/bazel/WORKSPACE

@@ -1,14 +0,0 @@
-
-workspace(name = "upb_example")
-
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-
-git_repository(
-    name = "upb",
-    remote = "https://github.com/protocolbuffers/upb.git",
-    commit = "d16bf99ac4658793748cda3251226059892b3b7b",
-)
-
-load("@upb//bazel:workspace_deps.bzl", "upb_deps")
-
-upb_deps()

examples/bazel/foo.proto

@@ -1,7 +0,0 @@
-
-syntax = "proto2";
-
-message Foo {
-  optional int64 time = 1;
-  optional string greeting = 2;
-}

examples/bazel/test_binary.c

@@ -1,17 +0,0 @@
-
-#include <time.h>
-
-#include "foo.upb.h"
-
-int main() {
-  upb_arena *arena = upb_arena_new();
-  Foo* foo = Foo_new(arena);
-  const char greeting[] = "Hello, World!\n";
-
-  Foo_set_time(foo, time(NULL));
-  /* Warning: the proto will not copy this, the string data must outlive
-   * the proto. */
-  Foo_set_greeting(foo, upb_strview_makez(greeting));
-
-  upb_arena_free(arena);
-}

generated_for_cmake/google/protobuf/descriptor.upb.c

@@ -1,485 +0,0 @@
-/* This file was generated by upbc (the upb compiler) from the input
- * file:
- *
- *     google/protobuf/descriptor.proto
- *
- * Do not edit -- your changes will be discarded when the file is
- * regenerated. */
-
-#include <stddef.h>
-#include "upb/msg.h"
-#include "google/protobuf/descriptor.upb.h"
-
-#include "upb/port_def.inc"
-
-static const upb_msglayout *const google_protobuf_FileDescriptorSet_submsgs[1] = {
-  &google_protobuf_FileDescriptorProto_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_FileDescriptorSet__fields[1] = {
-  {1, UPB_SIZE(0, 0), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_FileDescriptorSet_msginit = {
-  &google_protobuf_FileDescriptorSet_submsgs[0],
-  &google_protobuf_FileDescriptorSet__fields[0],
-  UPB_SIZE(4, 8), 1, false,
-};
-
-static const upb_msglayout *const google_protobuf_FileDescriptorProto_submsgs[6] = {
-  &google_protobuf_DescriptorProto_msginit,
-  &google_protobuf_EnumDescriptorProto_msginit,
-  &google_protobuf_FieldDescriptorProto_msginit,
-  &google_protobuf_FileOptions_msginit,
-  &google_protobuf_ServiceDescriptorProto_msginit,
-  &google_protobuf_SourceCodeInfo_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_FileDescriptorProto__fields[12] = {
-  {1, UPB_SIZE(4, 8), 1, 0, 9, 1},
-  {2, UPB_SIZE(12, 24), 2, 0, 9, 1},
-  {3, UPB_SIZE(36, 72), 0, 0, 9, 3},
-  {4, UPB_SIZE(40, 80), 0, 0, 11, 3},
-  {5, UPB_SIZE(44, 88), 0, 1, 11, 3},
-  {6, UPB_SIZE(48, 96), 0, 4, 11, 3},
-  {7, UPB_SIZE(52, 104), 0, 2, 11, 3},
-  {8, UPB_SIZE(28, 56), 4, 3, 11, 1},
-  {9, UPB_SIZE(32, 64), 5, 5, 11, 1},
-  {10, UPB_SIZE(56, 112), 0, 0, 5, 3},
-  {11, UPB_SIZE(60, 120), 0, 0, 5, 3},
-  {12, UPB_SIZE(20, 40), 3, 0, 9, 1},
-};
-
-const upb_msglayout google_protobuf_FileDescriptorProto_msginit = {
-  &google_protobuf_FileDescriptorProto_submsgs[0],
-  &google_protobuf_FileDescriptorProto__fields[0],
-  UPB_SIZE(64, 128), 12, false,
-};
-
-static const upb_msglayout *const google_protobuf_DescriptorProto_submsgs[8] = {
-  &google_protobuf_DescriptorProto_msginit,
-  &google_protobuf_DescriptorProto_ExtensionRange_msginit,
-  &google_protobuf_DescriptorProto_ReservedRange_msginit,
-  &google_protobuf_EnumDescriptorProto_msginit,
-  &google_protobuf_FieldDescriptorProto_msginit,
-  &google_protobuf_MessageOptions_msginit,
-  &google_protobuf_OneofDescriptorProto_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_DescriptorProto__fields[10] = {
-  {1, UPB_SIZE(4, 8), 1, 0, 9, 1},
-  {2, UPB_SIZE(16, 32), 0, 4, 11, 3},
-  {3, UPB_SIZE(20, 40), 0, 0, 11, 3},
-  {4, UPB_SIZE(24, 48), 0, 3, 11, 3},
-  {5, UPB_SIZE(28, 56), 0, 1, 11, 3},
-  {6, UPB_SIZE(32, 64), 0, 4, 11, 3},
-  {7, UPB_SIZE(12, 24), 2, 5, 11, 1},
-  {8, UPB_SIZE(36, 72), 0, 6, 11, 3},
-  {9, UPB_SIZE(40, 80), 0, 2, 11, 3},
-  {10, UPB_SIZE(44, 88), 0, 0, 9, 3},
-};
-
-const upb_msglayout google_protobuf_DescriptorProto_msginit = {
-  &google_protobuf_DescriptorProto_submsgs[0],
-  &google_protobuf_DescriptorProto__fields[0],
-  UPB_SIZE(48, 96), 10, false,
-};
-
-static const upb_msglayout *const google_protobuf_DescriptorProto_ExtensionRange_submsgs[1] = {
-  &google_protobuf_ExtensionRangeOptions_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_DescriptorProto_ExtensionRange__fields[3] = {
-  {1, UPB_SIZE(4, 4), 1, 0, 5, 1},
-  {2, UPB_SIZE(8, 8), 2, 0, 5, 1},
-  {3, UPB_SIZE(12, 16), 3, 0, 11, 1},
-};
-
-const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit = {
-  &google_protobuf_DescriptorProto_ExtensionRange_submsgs[0],
-  &google_protobuf_DescriptorProto_ExtensionRange__fields[0],
-  UPB_SIZE(16, 24), 3, false,
-};
-
-static const upb_msglayout_field google_protobuf_DescriptorProto_ReservedRange__fields[2] = {
-  {1, UPB_SIZE(4, 4), 1, 0, 5, 1},
-  {2, UPB_SIZE(8, 8), 2, 0, 5, 1},
-};
-
-const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit = {
-  NULL,
-  &google_protobuf_DescriptorProto_ReservedRange__fields[0],
-  UPB_SIZE(12, 12), 2, false,
-};
-
-static const upb_msglayout *const google_protobuf_ExtensionRangeOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_ExtensionRangeOptions__fields[1] = {
-  {999, UPB_SIZE(0, 0), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit = {
-  &google_protobuf_ExtensionRangeOptions_submsgs[0],
-  &google_protobuf_ExtensionRangeOptions__fields[0],
-  UPB_SIZE(4, 8), 1, false,
-};
-
-static const upb_msglayout *const google_protobuf_FieldDescriptorProto_submsgs[1] = {
-  &google_protobuf_FieldOptions_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_FieldDescriptorProto__fields[10] = {
-  {1, UPB_SIZE(32, 32), 5, 0, 9, 1},
-  {2, UPB_SIZE(40, 48), 6, 0, 9, 1},
-  {3, UPB_SIZE(24, 24), 3, 0, 5, 1},
-  {4, UPB_SIZE(8, 8), 1, 0, 14, 1},
-  {5, UPB_SIZE(16, 16), 2, 0, 14, 1},
-  {6, UPB_SIZE(48, 64), 7, 0, 9, 1},
-  {7, UPB_SIZE(56, 80), 8, 0, 9, 1},
-  {8, UPB_SIZE(72, 112), 10, 0, 11, 1},
-  {9, UPB_SIZE(28, 28), 4, 0, 5, 1},
-  {10, UPB_SIZE(64, 96), 9, 0, 9, 1},
-};
-
-const upb_msglayout google_protobuf_FieldDescriptorProto_msginit = {
-  &google_protobuf_FieldDescriptorProto_submsgs[0],
-  &google_protobuf_FieldDescriptorProto__fields[0],
-  UPB_SIZE(80, 128), 10, false,
-};
-
-static const upb_msglayout *const google_protobuf_OneofDescriptorProto_submsgs[1] = {
-  &google_protobuf_OneofOptions_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_OneofDescriptorProto__fields[2] = {
-  {1, UPB_SIZE(4, 8), 1, 0, 9, 1},
-  {2, UPB_SIZE(12, 24), 2, 0, 11, 1},
-};
-
-const upb_msglayout google_protobuf_OneofDescriptorProto_msginit = {
-  &google_protobuf_OneofDescriptorProto_submsgs[0],
-  &google_protobuf_OneofDescriptorProto__fields[0],
-  UPB_SIZE(16, 32), 2, false,
-};
-
-static const upb_msglayout *const google_protobuf_EnumDescriptorProto_submsgs[3] = {
-  &google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit,
-  &google_protobuf_EnumOptions_msginit,
-  &google_protobuf_EnumValueDescriptorProto_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_EnumDescriptorProto__fields[5] = {
-  {1, UPB_SIZE(4, 8), 1, 0, 9, 1},
-  {2, UPB_SIZE(16, 32), 0, 2, 11, 3},
-  {3, UPB_SIZE(12, 24), 2, 1, 11, 1},
-  {4, UPB_SIZE(20, 40), 0, 0, 11, 3},
-  {5, UPB_SIZE(24, 48), 0, 0, 9, 3},
-};
-
-const upb_msglayout google_protobuf_EnumDescriptorProto_msginit = {
-  &google_protobuf_EnumDescriptorProto_submsgs[0],
-  &google_protobuf_EnumDescriptorProto__fields[0],
-  UPB_SIZE(32, 64), 5, false,
-};
-
-static const upb_msglayout_field google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[2] = {
-  {1, UPB_SIZE(4, 4), 1, 0, 5, 1},
-  {2, UPB_SIZE(8, 8), 2, 0, 5, 1},
-};
-
-const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit = {
-  NULL,
-  &google_protobuf_EnumDescriptorProto_EnumReservedRange__fields[0],
-  UPB_SIZE(12, 12), 2, false,
-};
-
-static const upb_msglayout *const google_protobuf_EnumValueDescriptorProto_submsgs[1] = {
-  &google_protobuf_EnumValueOptions_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_EnumValueDescriptorProto__fields[3] = {
-  {1, UPB_SIZE(8, 8), 2, 0, 9, 1},
-  {2, UPB_SIZE(4, 4), 1, 0, 5, 1},
-  {3, UPB_SIZE(16, 24), 3, 0, 11, 1},
-};
-
-const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit = {
-  &google_protobuf_EnumValueDescriptorProto_submsgs[0],
-  &google_protobuf_EnumValueDescriptorProto__fields[0],
-  UPB_SIZE(24, 32), 3, false,
-};
-
-static const upb_msglayout *const google_protobuf_ServiceDescriptorProto_submsgs[2] = {
-  &google_protobuf_MethodDescriptorProto_msginit,
-  &google_protobuf_ServiceOptions_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_ServiceDescriptorProto__fields[3] = {
-  {1, UPB_SIZE(4, 8), 1, 0, 9, 1},
-  {2, UPB_SIZE(16, 32), 0, 0, 11, 3},
-  {3, UPB_SIZE(12, 24), 2, 1, 11, 1},
-};
-
-const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit = {
-  &google_protobuf_ServiceDescriptorProto_submsgs[0],
-  &google_protobuf_ServiceDescriptorProto__fields[0],
-  UPB_SIZE(24, 48), 3, false,
-};
-
-static const upb_msglayout *const google_protobuf_MethodDescriptorProto_submsgs[1] = {
-  &google_protobuf_MethodOptions_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_MethodDescriptorProto__fields[6] = {
-  {1, UPB_SIZE(4, 8), 3, 0, 9, 1},
-  {2, UPB_SIZE(12, 24), 4, 0, 9, 1},
-  {3, UPB_SIZE(20, 40), 5, 0, 9, 1},
-  {4, UPB_SIZE(28, 56), 6, 0, 11, 1},
-  {5, UPB_SIZE(1, 1), 1, 0, 8, 1},
-  {6, UPB_SIZE(2, 2), 2, 0, 8, 1},
-};
-
-const upb_msglayout google_protobuf_MethodDescriptorProto_msginit = {
-  &google_protobuf_MethodDescriptorProto_submsgs[0],
-  &google_protobuf_MethodDescriptorProto__fields[0],
-  UPB_SIZE(32, 64), 6, false,
-};
-
-static const upb_msglayout *const google_protobuf_FileOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_FileOptions__fields[21] = {
-  {1, UPB_SIZE(28, 32), 11, 0, 9, 1},
-  {8, UPB_SIZE(36, 48), 12, 0, 9, 1},
-  {9, UPB_SIZE(8, 8), 1, 0, 14, 1},
-  {10, UPB_SIZE(16, 16), 2, 0, 8, 1},
-  {11, UPB_SIZE(44, 64), 13, 0, 9, 1},
-  {16, UPB_SIZE(17, 17), 3, 0, 8, 1},
-  {17, UPB_SIZE(18, 18), 4, 0, 8, 1},
-  {18, UPB_SIZE(19, 19), 5, 0, 8, 1},
-  {20, UPB_SIZE(20, 20), 6, 0, 8, 1},
-  {23, UPB_SIZE(21, 21), 7, 0, 8, 1},
-  {27, UPB_SIZE(22, 22), 8, 0, 8, 1},
-  {31, UPB_SIZE(23, 23), 9, 0, 8, 1},
-  {36, UPB_SIZE(52, 80), 14, 0, 9, 1},
-  {37, UPB_SIZE(60, 96), 15, 0, 9, 1},
-  {39, UPB_SIZE(68, 112), 16, 0, 9, 1},
-  {40, UPB_SIZE(76, 128), 17, 0, 9, 1},
-  {41, UPB_SIZE(84, 144), 18, 0, 9, 1},
-  {42, UPB_SIZE(24, 24), 10, 0, 8, 1},
-  {44, UPB_SIZE(92, 160), 19, 0, 9, 1},
-  {45, UPB_SIZE(100, 176), 20, 0, 9, 1},
-  {999, UPB_SIZE(108, 192), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_FileOptions_msginit = {
-  &google_protobuf_FileOptions_submsgs[0],
-  &google_protobuf_FileOptions__fields[0],
-  UPB_SIZE(112, 208), 21, false,
-};
-
-static const upb_msglayout *const google_protobuf_MessageOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_MessageOptions__fields[5] = {
-  {1, UPB_SIZE(1, 1), 1, 0, 8, 1},
-  {2, UPB_SIZE(2, 2), 2, 0, 8, 1},
-  {3, UPB_SIZE(3, 3), 3, 0, 8, 1},
-  {7, UPB_SIZE(4, 4), 4, 0, 8, 1},
-  {999, UPB_SIZE(8, 8), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_MessageOptions_msginit = {
-  &google_protobuf_MessageOptions_submsgs[0],
-  &google_protobuf_MessageOptions__fields[0],
-  UPB_SIZE(12, 16), 5, false,
-};
-
-static const upb_msglayout *const google_protobuf_FieldOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_FieldOptions__fields[7] = {
-  {1, UPB_SIZE(8, 8), 1, 0, 14, 1},
-  {2, UPB_SIZE(24, 24), 3, 0, 8, 1},
-  {3, UPB_SIZE(25, 25), 4, 0, 8, 1},
-  {5, UPB_SIZE(26, 26), 5, 0, 8, 1},
-  {6, UPB_SIZE(16, 16), 2, 0, 14, 1},
-  {10, UPB_SIZE(27, 27), 6, 0, 8, 1},
-  {999, UPB_SIZE(28, 32), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_FieldOptions_msginit = {
-  &google_protobuf_FieldOptions_submsgs[0],
-  &google_protobuf_FieldOptions__fields[0],
-  UPB_SIZE(32, 40), 7, false,
-};
-
-static const upb_msglayout *const google_protobuf_OneofOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_OneofOptions__fields[1] = {
-  {999, UPB_SIZE(0, 0), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_OneofOptions_msginit = {
-  &google_protobuf_OneofOptions_submsgs[0],
-  &google_protobuf_OneofOptions__fields[0],
-  UPB_SIZE(4, 8), 1, false,
-};
-
-static const upb_msglayout *const google_protobuf_EnumOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_EnumOptions__fields[3] = {
-  {2, UPB_SIZE(1, 1), 1, 0, 8, 1},
-  {3, UPB_SIZE(2, 2), 2, 0, 8, 1},
-  {999, UPB_SIZE(4, 8), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_EnumOptions_msginit = {
-  &google_protobuf_EnumOptions_submsgs[0],
-  &google_protobuf_EnumOptions__fields[0],
-  UPB_SIZE(8, 16), 3, false,
-};
-
-static const upb_msglayout *const google_protobuf_EnumValueOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_EnumValueOptions__fields[2] = {
-  {1, UPB_SIZE(1, 1), 1, 0, 8, 1},
-  {999, UPB_SIZE(4, 8), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_EnumValueOptions_msginit = {
-  &google_protobuf_EnumValueOptions_submsgs[0],
-  &google_protobuf_EnumValueOptions__fields[0],
-  UPB_SIZE(8, 16), 2, false,
-};
-
-static const upb_msglayout *const google_protobuf_ServiceOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_ServiceOptions__fields[2] = {
-  {33, UPB_SIZE(1, 1), 1, 0, 8, 1},
-  {999, UPB_SIZE(4, 8), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_ServiceOptions_msginit = {
-  &google_protobuf_ServiceOptions_submsgs[0],
-  &google_protobuf_ServiceOptions__fields[0],
-  UPB_SIZE(8, 16), 2, false,
-};
-
-static const upb_msglayout *const google_protobuf_MethodOptions_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_MethodOptions__fields[3] = {
-  {33, UPB_SIZE(16, 16), 2, 0, 8, 1},
-  {34, UPB_SIZE(8, 8), 1, 0, 14, 1},
-  {999, UPB_SIZE(20, 24), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_MethodOptions_msginit = {
-  &google_protobuf_MethodOptions_submsgs[0],
-  &google_protobuf_MethodOptions__fields[0],
-  UPB_SIZE(24, 32), 3, false,
-};
-
-static const upb_msglayout *const google_protobuf_UninterpretedOption_submsgs[1] = {
-  &google_protobuf_UninterpretedOption_NamePart_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_UninterpretedOption__fields[7] = {
-  {2, UPB_SIZE(56, 80), 0, 0, 11, 3},
-  {3, UPB_SIZE(32, 32), 4, 0, 9, 1},
-  {4, UPB_SIZE(8, 8), 1, 0, 4, 1},
-  {5, UPB_SIZE(16, 16), 2, 0, 3, 1},
-  {6, UPB_SIZE(24, 24), 3, 0, 1, 1},
-  {7, UPB_SIZE(40, 48), 5, 0, 12, 1},
-  {8, UPB_SIZE(48, 64), 6, 0, 9, 1},
-};
-
-const upb_msglayout google_protobuf_UninterpretedOption_msginit = {
-  &google_protobuf_UninterpretedOption_submsgs[0],
-  &google_protobuf_UninterpretedOption__fields[0],
-  UPB_SIZE(64, 96), 7, false,
-};
-
-static const upb_msglayout_field google_protobuf_UninterpretedOption_NamePart__fields[2] = {
-  {1, UPB_SIZE(4, 8), 2, 0, 9, 2},
-  {2, UPB_SIZE(1, 1), 1, 0, 8, 2},
-};
-
-const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit = {
-  NULL,
-  &google_protobuf_UninterpretedOption_NamePart__fields[0],
-  UPB_SIZE(16, 32), 2, false,
-};
-
-static const upb_msglayout *const google_protobuf_SourceCodeInfo_submsgs[1] = {
-  &google_protobuf_SourceCodeInfo_Location_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_SourceCodeInfo__fields[1] = {
-  {1, UPB_SIZE(0, 0), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_SourceCodeInfo_msginit = {
-  &google_protobuf_SourceCodeInfo_submsgs[0],
-  &google_protobuf_SourceCodeInfo__fields[0],
-  UPB_SIZE(4, 8), 1, false,
-};
-
-static const upb_msglayout_field google_protobuf_SourceCodeInfo_Location__fields[5] = {
-  {1, UPB_SIZE(20, 40), 0, 0, 5, 3},
-  {2, UPB_SIZE(24, 48), 0, 0, 5, 3},
-  {3, UPB_SIZE(4, 8), 1, 0, 9, 1},
-  {4, UPB_SIZE(12, 24), 2, 0, 9, 1},
-  {6, UPB_SIZE(28, 56), 0, 0, 9, 3},
-};
-
-const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit = {
-  NULL,
-  &google_protobuf_SourceCodeInfo_Location__fields[0],
-  UPB_SIZE(32, 64), 5, false,
-};
-
-static const upb_msglayout *const google_protobuf_GeneratedCodeInfo_submsgs[1] = {
-  &google_protobuf_GeneratedCodeInfo_Annotation_msginit,
-};
-
-static const upb_msglayout_field google_protobuf_GeneratedCodeInfo__fields[1] = {
-  {1, UPB_SIZE(0, 0), 0, 0, 11, 3},
-};
-
-const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit = {
-  &google_protobuf_GeneratedCodeInfo_submsgs[0],
-  &google_protobuf_GeneratedCodeInfo__fields[0],
-  UPB_SIZE(4, 8), 1, false,
-};
-
-static const upb_msglayout_field google_protobuf_GeneratedCodeInfo_Annotation__fields[4] = {
-  {1, UPB_SIZE(20, 32), 0, 0, 5, 3},
-  {2, UPB_SIZE(12, 16), 3, 0, 9, 1},
-  {3, UPB_SIZE(4, 4), 1, 0, 5, 1},
-  {4, UPB_SIZE(8, 8), 2, 0, 5, 1},
-};
-
-const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit = {
-  NULL,
-  &google_protobuf_GeneratedCodeInfo_Annotation__fields[0],
-  UPB_SIZE(24, 48), 4, false,
-};
-
-#include "upb/port_undef.inc"
-

kokoro/ubuntu/build.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-# Install the latest version of Bazel.
-use_bazel.sh latest
-
-# Verify/query CMake
-echo PATH=$PATH
-ls -l `which cmake`
-cmake --version
-
-# Log the bazel path and version.
-which bazel
-bazel version
-
-cd $(dirname $0)/../..
-bazel test --test_output=errors :all

kokoro/ubuntu/continuous.cfg

@@ -1,2 +0,0 @@
-build_file: "upb/kokoro/ubuntu/build.sh"
-timeout_mins: 15

kokoro/ubuntu/presubmit.cfg

@@ -1,2 +0,0 @@
-build_file: "upb/kokoro/ubuntu/build.sh"
-timeout_mins: 15

python/BUILD

@@ -0,0 +1,163 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+load("//bazel:py_proto_library.bzl", "py_proto_library")
+load("//bazel:py_extension.bzl", "py_extension")  # copybara:strip_for_google3
+load("@rules_python//python:packaging.bzl", "py_wheel")
+
+licenses(["notice"])
+
+py_extension(
+    name = "_message",
+    srcs = [
+        "convert.c",
+        "convert.h",
+        "descriptor.c",
+        "descriptor.h",
+        "descriptor_containers.c",
+        "descriptor_containers.h",
+        "descriptor_pool.c",
+        "descriptor_pool.h",
+        "extension_dict.c",
+        "extension_dict.h",
+        "map.c",
+        "map.h",
+        "message.c",
+        "message.h",
+        "protobuf.c",
+        "protobuf.h",
+        "python.h",
+        "repeated.c",
+        "repeated.h",
+    ],
+    deps = [
+        "//:descriptor_upb_proto_reflection",
+        "//:reflection",
+        "//:table",
+        "//:textformat",
+        "//:upb",
+        "//upb/util:compare",
+        "//upb/util:def_to_proto",
+        "//upb/util:required_fields",
+    ],
+)
+
+py_extension(
+    name = "_api_implementation",
+    srcs = ["api_implementation.c"],
+)
+
+# copybara:strip_for_google3_begin
+
+py_test(
+    name = "minimal_test",
+    srcs = [
+        "minimal_test.py",
+    ],
+    imports = ["."],
+    legacy_create_init = False,
+    deps = [
+        "//python:message_ext",
+        "@com_google_protobuf//:python_common_test_protos",
+        "@com_google_protobuf//:python_specific_test_protos",
+        "@com_google_protobuf//:python_srcs",
+    ],
+)
+
+# Copy the extensions into the location recognized by Python.
+# .abi3.so indicates use of the limited API, and cross-version ABI compatibility.
+EXT_SUFFIX = ".abi3.so"
+
+genrule(
+    name = "copy_message",
+    srcs = [":_message"],
+    outs = ["google/protobuf/pyext/_message" + EXT_SUFFIX],
+    cmd = "cp $< $@",
+)
+
+genrule(
+    name = "copy_api_implementation",
+    srcs = [":_api_implementation"],
+    outs = ["google/protobuf/internal/_api_implementation" + EXT_SUFFIX],
+    cmd = "cp $< $@",
+    visibility = ["//python:__subpackages__"],
+)
+
+filegroup(
+    name = "extension_files",
+    srcs = [
+        "google/protobuf/pyext/_message" + EXT_SUFFIX,
+        "google/protobuf/internal/_api_implementation" + EXT_SUFFIX,
+    ],
+)
+
+py_library(
+    name = "message_ext",
+    data = [":extension_files"],
+    imports = ["."],
+    visibility = ["//python:__subpackages__"],
+)
+
+
+py_proto_library(
+    name = "well_known_proto_pb2",
+    deps = [
+        "@com_google_protobuf//:any_proto",
+        "@com_google_protobuf//:api_proto",
+        "@com_google_protobuf//:compiler_plugin_proto",
+        "@com_google_protobuf//:descriptor_proto",
+        "@com_google_protobuf//:duration_proto",
+        "@com_google_protobuf//:empty_proto",
+        "@com_google_protobuf//:field_mask_proto",
+        "@com_google_protobuf//:source_context_proto",
+        "@com_google_protobuf//:struct_proto",
+        "@com_google_protobuf//:timestamp_proto",
+        "@com_google_protobuf//:type_proto",
+        "@com_google_protobuf//:wrappers_proto",
+    ],
+)
+
+py_wheel(
+    name = "binary_wheel",
+    abi = "abi3",
+    distribution = "protobuf",
+    # TODO(https://github.com/protocolbuffers/upb/issues/502): we need to make
+    # this a select() that is calculated from the platform we are actually
+    # building on.
+    platform = "manylinux2014_x86_64",
+    python_tag = "cp36",
+    strip_path_prefixes = ["python/"],
+    version = "4.20.0",
+    deps = [
+        ":extension_files",
+        ":well_known_proto_pb2",
+        # TODO(https://github.com/protocolbuffers/upb/issues/503): currently
+        # this includes the unit tests.  We should filter these out so we are
+        # only distributing true source files.
+        "@com_google_protobuf//:python_srcs",
+    ],
+)
+
+# copybara:strip_end

python/api_implementation.c

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <Python.h>
+
+static struct PyModuleDef module_def = {
+    PyModuleDef_HEAD_INIT,
+    "google.protobuf.internal._api_implementation",
+    "Protobuf Module",
+    -1,
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+    NULL};
+
+PyMODINIT_FUNC PyInit__api_implementation(void) {
+  PyObject* module = PyModule_Create(&module_def);
+
+  if (PyModule_AddIntConstant(module, "api_version", 2)) {
+    Py_DECREF(module);
+    return NULL;
+  }
+
+  return module;
+}

python/convert.c

@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "python/convert.h"
+
+#include "python/message.h"
+#include "python/protobuf.h"
+#include "upb/reflection.h"
+#include "upb/util/compare.h"
+
+PyObject* PyUpb_UpbToPy(upb_MessageValue val, const upb_FieldDef* f,
+                        PyObject* arena) {
+  switch (upb_FieldDef_CType(f)) {
+    case kUpb_CType_Enum:
+    case kUpb_CType_Int32:
+      return PyLong_FromLong(val.int32_val);
+    case kUpb_CType_Int64:
+      return PyLong_FromLongLong(val.int64_val);
+    case kUpb_CType_UInt32:
+      return PyLong_FromSize_t(val.uint32_val);
+    case kUpb_CType_UInt64:
+      return PyLong_FromUnsignedLongLong(val.uint64_val);
+    case kUpb_CType_Float:
+      return PyFloat_FromDouble(val.float_val);
+    case kUpb_CType_Double:
+      return PyFloat_FromDouble(val.double_val);
+    case kUpb_CType_Bool:
+      return PyBool_FromLong(val.bool_val);
+    case kUpb_CType_Bytes:
+      return PyBytes_FromStringAndSize(val.str_val.data, val.str_val.size);
+    case kUpb_CType_String: {
+      PyObject* ret =
+          PyUnicode_DecodeUTF8(val.str_val.data, val.str_val.size, NULL);
+      // If the string can't be decoded in UTF-8, just return a bytes object
+      // that contains the raw bytes. This can't happen if the value was
+      // assigned using the members of the Python message object, but can happen
+      // if the values were parsed from the wire (binary).
+      if (ret == NULL) {
+        PyErr_Clear();
+        ret = PyBytes_FromStringAndSize(val.str_val.data, val.str_val.size);
+      }
+      return ret;
+    }
+    case kUpb_CType_Message:
+      return PyUpb_CMessage_Get((upb_Message*)val.msg_val,
+                                upb_FieldDef_MessageSubDef(f), arena);
+    default:
+      PyErr_Format(PyExc_SystemError,
+                   "Getting a value from a field of unknown type %d",
+                   upb_FieldDef_CType(f));
+      return NULL;
+  }
+}
+
+static bool PyUpb_GetInt64(PyObject* obj, int64_t* val) {
+  // We require that the value is either an integer or has an __index__
+  // conversion.
+  obj = PyNumber_Index(obj);
+  if (!obj) return false;
+  // If the value is already a Python long, PyLong_AsLongLong() retrieves it.
+  // Otherwise is converts to integer using __int__.
+  *val = PyLong_AsLongLong(obj);
+  bool ok = true;
+  if (PyErr_Occurred()) {
+    assert(PyErr_ExceptionMatches(PyExc_OverflowError));
+    PyErr_Clear();
+    PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
+    ok = false;
+  }
+  Py_DECREF(obj);
+  return ok;
+}
+
+static bool PyUpb_GetUint64(PyObject* obj, uint64_t* val) {
+  // We require that the value is either an integer or has an __index__
+  // conversion.
+  obj = PyNumber_Index(obj);
+  if (!obj) return false;
+  *val = PyLong_AsUnsignedLongLong(obj);
+  bool ok = true;
+  if (PyErr_Occurred()) {
+    assert(PyErr_ExceptionMatches(PyExc_OverflowError));
+    PyErr_Clear();
+    PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
+    ok = false;
+  }
+  Py_DECREF(obj);
+  return ok;
+}
+
+static bool PyUpb_GetInt32(PyObject* obj, int32_t* val) {
+  int64_t i64;
+  if (!PyUpb_GetInt64(obj, &i64)) return false;
+  if (i64 < INT32_MIN || i64 > INT32_MAX) {
+    PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
+    return false;
+  }
+  *val = i64;
+  return true;
+}
+
+static bool PyUpb_GetUint32(PyObject* obj, uint32_t* val) {
+  uint64_t u64;
+  if (!PyUpb_GetUint64(obj, &u64)) return false;
+  if (u64 > UINT32_MAX) {
+    PyErr_Format(PyExc_ValueError, "Value out of range: %S", obj);
+    return false;
+  }
+  *val = u64;
+  return true;
+}
+
+// If `arena` is specified, copies the string data into the given arena.
+// Otherwise aliases the given data.
+static upb_MessageValue PyUpb_MaybeCopyString(const char* ptr, size_t size,
+                                              upb_Arena* arena) {
+  upb_MessageValue ret;
+  ret.str_val.size = size;
+  if (arena) {
+    char* buf = upb_Arena_Malloc(arena, size);
+    memcpy(buf, ptr, size);
+    ret.str_val.data = buf;
+  } else {
+    ret.str_val.data = ptr;
+  }
+  return ret;
+}
+
+static bool PyUpb_PyToUpbEnum(PyObject* obj, const upb_EnumDef* e,
+                              upb_MessageValue* val) {
+  if (PyUnicode_Check(obj)) {
+    Py_ssize_t size;
+    const char* name = PyUnicode_AsUTF8AndSize(obj, &size);
+    const upb_EnumValueDef* ev =
+        upb_EnumDef_FindValueByNameWithSize(e, name, size);
+    if (!ev) {
+      PyErr_Format(PyExc_ValueError, "unknown enum label \"%s\"", name);
+      return false;
+    }
+    val->int32_val = upb_EnumValueDef_Number(ev);
+    return true;
+  } else {
+    int32_t i32;
+    if (!PyUpb_GetInt32(obj, &i32)) return false;
+    if (upb_FileDef_Syntax(upb_EnumDef_File(e)) == kUpb_Syntax_Proto2 &&
+        !upb_EnumDef_CheckNumber(e, i32)) {
+      PyErr_Format(PyExc_ValueError, "invalid enumerator %d", (int)i32);
+      return false;
+    }
+    val->int32_val = i32;
+    return true;
+  }
+}
+
+bool PyUpb_PyToUpb(PyObject* obj, const upb_FieldDef* f, upb_MessageValue* val,
+                   upb_Arena* arena) {
+  switch (upb_FieldDef_CType(f)) {
+    case kUpb_CType_Enum:
+      return PyUpb_PyToUpbEnum(obj, upb_FieldDef_EnumSubDef(f), val);
+    case kUpb_CType_Int32:
+      return PyUpb_GetInt32(obj, &val->int32_val);
+    case kUpb_CType_Int64:
+      return PyUpb_GetInt64(obj, &val->int64_val);
+    case kUpb_CType_UInt32:
+      return PyUpb_GetUint32(obj, &val->uint32_val);
+    case kUpb_CType_UInt64:
+      return PyUpb_GetUint64(obj, &val->uint64_val);
+    case kUpb_CType_Float:
+      val->float_val = PyFloat_AsDouble(obj);
+      return !PyErr_Occurred();
+    case kUpb_CType_Double:
+      val->double_val = PyFloat_AsDouble(obj);
+      return !PyErr_Occurred();
+    case kUpb_CType_Bool:
+      val->bool_val = PyLong_AsLong(obj);
+      return !PyErr_Occurred();
+    case kUpb_CType_Bytes: {
+      char* ptr;
+      Py_ssize_t size;
+      if (PyBytes_AsStringAndSize(obj, &ptr, &size) < 0) return false;
+      *val = PyUpb_MaybeCopyString(ptr, size, arena);
+      return true;
+    }
+    case kUpb_CType_String: {
+      Py_ssize_t size;
+      const char* ptr;
+      PyObject* unicode = NULL;
+      if (PyBytes_Check(obj)) {
+        unicode = obj = PyUnicode_FromEncodedObject(obj, "utf-8", NULL);
+        if (!obj) return false;
+      }
+      ptr = PyUnicode_AsUTF8AndSize(obj, &size);
+      if (PyErr_Occurred()) {
+        Py_XDECREF(unicode);
+        return false;
+      }
+      *val = PyUpb_MaybeCopyString(ptr, size, arena);
+      Py_XDECREF(unicode);
+      return true;
+    }
+    case kUpb_CType_Message:
+      PyErr_Format(PyExc_ValueError, "Message objects may not be assigned",
+                   upb_FieldDef_CType(f));
+      return false;
+    default:
+      PyErr_Format(PyExc_SystemError,
+                   "Getting a value from a field of unknown type %d",
+                   upb_FieldDef_CType(f));
+      return false;
+  }
+}
+
+bool PyUpb_Message_IsEqual(const upb_Message* msg1, const upb_Message* msg2,
+                           const upb_MessageDef* m);
+
+// -----------------------------------------------------------------------------
+// Equal
+// -----------------------------------------------------------------------------
+
+bool PyUpb_ValueEq(upb_MessageValue val1, upb_MessageValue val2,
+                   const upb_FieldDef* f) {
+  switch (upb_FieldDef_CType(f)) {
+    case kUpb_CType_Bool:
+      return val1.bool_val == val2.bool_val;
+    case kUpb_CType_Int32:
+    case kUpb_CType_UInt32:
+    case kUpb_CType_Enum:
+      return val1.int32_val == val2.int32_val;
+    case kUpb_CType_Int64:
+    case kUpb_CType_UInt64:
+      return val1.int64_val == val2.int64_val;
+    case kUpb_CType_Float:
+      return val1.float_val == val2.float_val;
+    case kUpb_CType_Double:
+      return val1.double_val == val2.double_val;
+    case kUpb_CType_String:
+    case kUpb_CType_Bytes:
+      return val1.str_val.size == val2.str_val.size &&
+             memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) ==
+                 0;
+    case kUpb_CType_Message:
+      return PyUpb_Message_IsEqual(val1.msg_val, val2.msg_val,
+                                   upb_FieldDef_MessageSubDef(f));
+    default:
+      return false;
+  }
+}
+
+bool PyUpb_Map_IsEqual(const upb_Map* map1, const upb_Map* map2,
+                       const upb_FieldDef* f) {
+  assert(upb_FieldDef_IsMap(f));
+  if (map1 == map2) return true;
+
+  size_t size1 = map1 ? upb_Map_Size(map1) : 0;
+  size_t size2 = map2 ? upb_Map_Size(map2) : 0;
+  if (size1 != size2) return false;
+  if (size1 == 0) return true;
+
+  const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+  const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1);
+  size_t iter = kUpb_Map_Begin;
+
+  while (upb_MapIterator_Next(map1, &iter)) {
+    upb_MessageValue key = upb_MapIterator_Key(map1, iter);
+    upb_MessageValue val1 = upb_MapIterator_Value(map1, iter);
+    upb_MessageValue val2;
+    if (!upb_Map_Get(map2, key, &val2)) return false;
+    if (!PyUpb_ValueEq(val1, val2, val_f)) return false;
+  }
+
+  return true;
+}
+
+static bool PyUpb_ArrayElem_IsEqual(const upb_Array* arr1,
+                                    const upb_Array* arr2, size_t i,
+                                    const upb_FieldDef* f) {
+  assert(i < upb_Array_Size(arr1));
+  assert(i < upb_Array_Size(arr2));
+  upb_MessageValue val1 = upb_Array_Get(arr1, i);
+  upb_MessageValue val2 = upb_Array_Get(arr2, i);
+  return PyUpb_ValueEq(val1, val2, f);
+}
+
+bool PyUpb_Array_IsEqual(const upb_Array* arr1, const upb_Array* arr2,
+                         const upb_FieldDef* f) {
+  assert(upb_FieldDef_IsRepeated(f) && !upb_FieldDef_IsMap(f));
+  if (arr1 == arr2) return true;
+
+  size_t n1 = arr1 ? upb_Array_Size(arr1) : 0;
+  size_t n2 = arr2 ? upb_Array_Size(arr2) : 0;
+  if (n1 != n2) return false;
+
+  // Half the length rounded down.  Important: the empty list rounds to 0.
+  size_t half = n1 / 2;
+
+  // Search from the ends-in.  We expect differences to more quickly manifest
+  // at the ends than in the middle.  If the length is odd we will miss the
+  // middle element.
+  for (size_t i = 0; i < half; i++) {
+    if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, i, f)) return false;
+    if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, n1 - 1 - i, f)) return false;
+  }
+
+  // For an odd-lengthed list, pick up the middle element.
+  if (n1 & 1) {
+    if (!PyUpb_ArrayElem_IsEqual(arr1, arr2, half, f)) return false;
+  }
+
+  return true;
+}
+
+bool PyUpb_Message_IsEqual(const upb_Message* msg1, const upb_Message* msg2,
+                           const upb_MessageDef* m) {
+  if (msg1 == msg2) return true;
+  if (upb_Message_ExtensionCount(msg1) != upb_Message_ExtensionCount(msg2))
+    return false;
+
+  // Compare messages field-by-field.  This is slightly tricky, because while
+  // we can iterate over normal fields in a predictable order, the extension
+  // order is unpredictable and may be different between msg1 and msg2.
+  // So we use the following strategy:
+  //   1. Iterate over all msg1 fields (including extensions).
+  //   2. For non-extension fields, we find the corresponding field by simply
+  //      using upb_Message_Next(msg2).  If the two messages have the same set
+  //      of fields, this will yield the same field.
+  //   3. For extension fields, we have to actually search for the corresponding
+  //      field, which we do with upb_Message_Get(msg2, ext_f1).
+  //   4. Once iteration over msg1 is complete, we call upb_Message_Next(msg2)
+  //   one
+  //      final time to verify that we have visited all of msg2's regular fields
+  //      (we pass NULL for ext_dict so that iteration will *not* return
+  //      extensions).
+  //
+  // We don't need to visit all of msg2's extensions, because we verified up
+  // front that both messages have the same number of extensions.
+  const upb_DefPool* symtab = upb_FileDef_Pool(upb_MessageDef_File(m));
+  const upb_FieldDef *f1, *f2;
+  upb_MessageValue val1, val2;
+  size_t iter1 = kUpb_Message_Begin;
+  size_t iter2 = kUpb_Message_Begin;
+  while (upb_Message_Next(msg1, m, symtab, &f1, &val1, &iter1)) {
+    if (upb_FieldDef_IsExtension(f1)) {
+      val2 = upb_Message_Get(msg2, f1);
+    } else {
+      if (!upb_Message_Next(msg2, m, NULL, &f2, &val2, &iter2) || f1 != f2) {
+        return false;
+      }
+    }
+
+    if (upb_FieldDef_IsMap(f1)) {
+      if (!PyUpb_Map_IsEqual(val1.map_val, val2.map_val, f1)) return false;
+    } else if (upb_FieldDef_IsRepeated(f1)) {
+      if (!PyUpb_Array_IsEqual(val1.array_val, val2.array_val, f1)) {
+        return false;
+      }
+    } else {
+      if (!PyUpb_ValueEq(val1, val2, f1)) return false;
+    }
+  }
+
+  if (upb_Message_Next(msg2, m, NULL, &f2, &val2, &iter2)) return false;
+
+  size_t usize1, usize2;
+  const char* uf1 = upb_Message_GetUnknown(msg1, &usize1);
+  const char* uf2 = upb_Message_GetUnknown(msg2, &usize2);
+  // 100 is arbitrary, we're trying to prevent stack overflow but it's not
+  // obvious how deep we should allow here.
+  return upb_Message_UnknownFieldsAreEqual(uf1, usize1, uf2, usize2, 100) ==
+         kUpb_UnknownCompareResult_Equal;
+}

python/convert.h

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYUPB_CONVERT_H__
+#define PYUPB_CONVERT_H__
+
+#include "protobuf.h"
+#include "upb/def.h"
+#include "upb/reflection.h"
+
+// Converts `val` to a Python object according to the type information in `f`.
+// Any newly-created Python objects that reference non-primitive data from `val`
+// will take a reference on `arena`; the caller must ensure that `val` belongs
+// to `arena`. If the conversion cannot be performed, returns NULL and sets a
+// Python error.
+PyObject* PyUpb_UpbToPy(upb_MessageValue val, const upb_FieldDef* f,
+                        PyObject* arena);
+
+// Converts `obj` to a upb_MessageValue `*val` according to the type information
+// in `f`. If `arena` is provided, any string data will be copied into `arena`,
+// otherwise the returned value will alias the Python-owned data (this can be
+// useful for an ephemeral upb_MessageValue).  If the conversion cannot be
+// performed, returns false.
+bool PyUpb_PyToUpb(PyObject* obj, const upb_FieldDef* f, upb_MessageValue* val,
+                   upb_Arena* arena);
+
+// Returns true if the given values (of type `f`) are equal.
+bool PyUpb_ValueEq(upb_MessageValue val1, upb_MessageValue val2,
+                   const upb_FieldDef* f);
+
+// Returns true if the given messages (of type `m`) are equal.
+bool PyUpb_Message_IsEqual(const upb_Message* msg1, const upb_Message* msg2,
+                           const upb_MessageDef* m);
+
+// Returns true if the two arrays (with element type `f`) are equal.
+bool PyUpb_Array_IsEqual(const upb_Array* arr1, const upb_Array* arr2,
+                         const upb_FieldDef* f);
+
+#endif  // PYUPB_CONVERT_H__

python/descriptor.h

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYUPB_DESCRIPTOR_H__
+#define PYUPB_DESCRIPTOR_H__
+
+#include <stdbool.h>
+
+#include "python/python.h"
+#include "upb/def.h"
+
+typedef enum {
+  kPyUpb_Descriptor = 0,
+  kPyUpb_EnumDescriptor = 1,
+  kPyUpb_EnumValueDescriptor = 2,
+  kPyUpb_FieldDescriptor = 3,
+  kPyUpb_FileDescriptor = 4,
+  kPyUpb_MethodDescriptor = 5,
+  kPyUpb_OneofDescriptor = 6,
+  kPyUpb_ServiceDescriptor = 7,
+  kPyUpb_Descriptor_Count = 8,
+} PyUpb_DescriptorType;
+
+// Given a descriptor object |desc|, returns a Python message class object for
+// the msgdef |m|, which must be from the same pool.
+PyObject* PyUpb_Descriptor_GetClass(const upb_MessageDef* m);
+
+// Returns a Python wrapper object for the given def. This will return an
+// existing object if one already exists, otherwise a new object will be
+// created.  The caller always owns a ref on the returned object.
+PyObject* PyUpb_Descriptor_Get(const upb_MessageDef* msgdef);
+PyObject* PyUpb_EnumDescriptor_Get(const upb_EnumDef* enumdef);
+PyObject* PyUpb_FieldDescriptor_Get(const upb_FieldDef* field);
+PyObject* PyUpb_FileDescriptor_Get(const upb_FileDef* file);
+PyObject* PyUpb_OneofDescriptor_Get(const upb_OneofDef* oneof);
+PyObject* PyUpb_EnumValueDescriptor_Get(const upb_EnumValueDef* enumval);
+PyObject* PyUpb_Descriptor_GetOrCreateWrapper(const upb_MessageDef* msg);
+PyObject* PyUpb_ServiceDescriptor_Get(const upb_ServiceDef* s);
+PyObject* PyUpb_MethodDescriptor_Get(const upb_MethodDef* s);
+
+// Returns the underlying |def| for a given wrapper object. The caller must
+// have already verified that the given Python object is of the expected type.
+const upb_FileDef* PyUpb_FileDescriptor_GetDef(PyObject* file);
+const upb_FieldDef* PyUpb_FieldDescriptor_GetDef(PyObject* file);
+const upb_MessageDef* PyUpb_Descriptor_GetDef(PyObject* _self);
+const void* PyUpb_AnyDescriptor_GetDef(PyObject* _self);
+
+// Returns the underlying |def| for a given wrapper object. The caller must
+// have already verified that the given Python object is of the expected type.
+const upb_FileDef* PyUpb_FileDescriptor_GetDef(PyObject* file);
+const void* PyUpb_AnyDescriptor_GetDef(PyObject* _self);
+
+// Module-level init.
+bool PyUpb_InitDescriptor(PyObject* m);
+
+#endif  // PYUPB_DESCRIPTOR_H__

python/descriptor_containers.c

@@ -0,0 +1,704 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "descriptor_containers.h"
+
+#include "descriptor.h"
+#include "protobuf.h"
+#include "upb/def.h"
+
+// -----------------------------------------------------------------------------
+// DescriptorIterator
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  const PyUpb_GenericSequence_Funcs* funcs;
+  const void* parent;    // upb_MessageDef*, upb_DefPool*, etc.
+  PyObject* parent_obj;  // Python object that keeps parent alive, we own a ref.
+  int index;             // Current iterator index.
+} PyUpb_DescriptorIterator;
+
+PyUpb_DescriptorIterator* PyUpb_DescriptorIterator_Self(PyObject* obj) {
+  assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->descriptor_iterator_type);
+  return (PyUpb_DescriptorIterator*)obj;
+}
+
+static void PyUpb_DescriptorIterator_Dealloc(PyObject* _self) {
+  PyUpb_DescriptorIterator* self = PyUpb_DescriptorIterator_Self(_self);
+  Py_DECREF(self->parent_obj);
+  PyUpb_Dealloc(self);
+}
+
+PyObject* PyUpb_DescriptorIterator_New(const PyUpb_GenericSequence_Funcs* funcs,
+                                       const void* parent,
+                                       PyObject* parent_obj) {
+  PyUpb_ModuleState* s = PyUpb_ModuleState_Get();
+  PyUpb_DescriptorIterator* iter =
+      (void*)PyType_GenericAlloc(s->descriptor_iterator_type, 0);
+  iter->funcs = funcs;
+  iter->parent = parent;
+  iter->parent_obj = parent_obj;
+  iter->index = 0;
+  Py_INCREF(iter->parent_obj);
+  return &iter->ob_base;
+}
+
+PyObject* PyUpb_DescriptorIterator_IterNext(PyObject* _self) {
+  PyUpb_DescriptorIterator* self = PyUpb_DescriptorIterator_Self(_self);
+  int size = self->funcs->get_elem_count(self->parent);
+  if (self->index >= size) return NULL;
+  const void* elem = self->funcs->index(self->parent, self->index);
+  self->index++;
+  return self->funcs->get_elem_wrapper(elem);
+}
+
+static PyType_Slot PyUpb_DescriptorIterator_Slots[] = {
+    {Py_tp_dealloc, PyUpb_DescriptorIterator_Dealloc},
+    {Py_tp_iter, PyObject_SelfIter},
+    {Py_tp_iternext, PyUpb_DescriptorIterator_IterNext},
+    {0, NULL}};
+
+static PyType_Spec PyUpb_DescriptorIterator_Spec = {
+    PYUPB_MODULE_NAME ".DescriptorIterator",  // tp_name
+    sizeof(PyUpb_DescriptorIterator),         // tp_basicsize
+    0,                                        // tp_itemsize
+    Py_TPFLAGS_DEFAULT,                       // tp_flags
+    PyUpb_DescriptorIterator_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// GenericSequence
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  const PyUpb_GenericSequence_Funcs* funcs;
+  const void* parent;    // upb_MessageDef*, upb_DefPool*, etc.
+  PyObject* parent_obj;  // Python object that keeps parent alive, we own a ref.
+} PyUpb_GenericSequence;
+
+PyUpb_GenericSequence* PyUpb_GenericSequence_Self(PyObject* obj) {
+  assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->generic_sequence_type);
+  return (PyUpb_GenericSequence*)obj;
+}
+
+static void PyUpb_GenericSequence_Dealloc(PyObject* _self) {
+  PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self);
+  Py_CLEAR(self->parent_obj);
+  PyUpb_Dealloc(self);
+}
+
+PyObject* PyUpb_GenericSequence_New(const PyUpb_GenericSequence_Funcs* funcs,
+                                    const void* parent, PyObject* parent_obj) {
+  PyUpb_ModuleState* s = PyUpb_ModuleState_Get();
+  PyUpb_GenericSequence* seq =
+      (PyUpb_GenericSequence*)PyType_GenericAlloc(s->generic_sequence_type, 0);
+  seq->funcs = funcs;
+  seq->parent = parent;
+  seq->parent_obj = parent_obj;
+  Py_INCREF(parent_obj);
+  return &seq->ob_base;
+}
+
+static Py_ssize_t PyUpb_GenericSequence_Length(PyObject* _self) {
+  PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self);
+  return self->funcs->get_elem_count(self->parent);
+}
+
+static PyObject* PyUpb_GenericSequence_GetItem(PyObject* _self,
+                                               Py_ssize_t index) {
+  PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self);
+  Py_ssize_t size = self->funcs->get_elem_count(self->parent);
+  if (index < 0 || index >= size) {
+    PyErr_Format(PyExc_IndexError, "list index (%zd) out of range", index);
+    return NULL;
+  }
+  const void* elem = self->funcs->index(self->parent, index);
+  return self->funcs->get_elem_wrapper(elem);
+}
+
+// A sequence container can only be equal to another sequence container, or (for
+// backward compatibility) to a list containing the same items.
+// Returns 1 if equal, 0 if unequal, -1 on error.
+static int PyUpb_GenericSequence_IsEqual(PyUpb_GenericSequence* self,
+                                         PyObject* other) {
+  // Check the identity of C++ pointers.
+  if (PyObject_TypeCheck(other, Py_TYPE(self))) {
+    PyUpb_GenericSequence* other_seq = (void*)other;
+    return self->parent == other_seq->parent && self->funcs == other_seq->funcs;
+  }
+
+  if (!PyList_Check(other)) return 0;
+
+  // return list(self) == other
+  // We can clamp `i` to int because GenericSequence uses int for size (this
+  // is useful when we do int iteration below).
+  int n = PyUpb_GenericSequence_Length((PyObject*)self);
+  if ((Py_ssize_t)n != PyList_Size(other)) {
+    return false;
+  }
+
+  PyObject* item1;
+  for (int i = 0; i < n; i++) {
+    item1 = PyUpb_GenericSequence_GetItem((PyObject*)self, i);
+    PyObject* item2 = PyList_GetItem(other, i);
+    if (!item1 || !item2) goto error;
+    int cmp = PyObject_RichCompareBool(item1, item2, Py_EQ);
+    Py_DECREF(item1);
+    if (cmp != 1) return cmp;
+  }
+  // All items were found and equal
+  return 1;
+
+error:
+  Py_XDECREF(item1);
+  return -1;
+}
+
+static PyObject* PyUpb_GenericSequence_RichCompare(PyObject* _self,
+                                                   PyObject* other, int opid) {
+  PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self);
+  if (opid != Py_EQ && opid != Py_NE) {
+    Py_RETURN_NOTIMPLEMENTED;
+  }
+  bool ret = PyUpb_GenericSequence_IsEqual(self, other);
+  if (opid == Py_NE) ret = !ret;
+  return PyBool_FromLong(ret);
+}
+
+// Linear search.  Could optimize this in some cases (defs that have index),
+// but not all (FileDescriptor.dependencies).
+static int PyUpb_GenericSequence_Find(PyObject* _self, PyObject* item) {
+  PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self);
+  const void* item_ptr = PyUpb_AnyDescriptor_GetDef(item);
+  int count = self->funcs->get_elem_count(self->parent);
+  for (int i = 0; i < count; i++) {
+    if (self->funcs->index(self->parent, i) == item_ptr) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+static PyObject* PyUpb_GenericSequence_Index(PyObject* self, PyObject* item) {
+  int position = PyUpb_GenericSequence_Find(self, item);
+  if (position < 0) {
+    PyErr_SetNone(PyExc_ValueError);
+    return NULL;
+  } else {
+    return PyLong_FromLong(position);
+  }
+}
+
+static PyObject* PyUpb_GenericSequence_Count(PyObject* _self, PyObject* item) {
+  PyUpb_GenericSequence* self = PyUpb_GenericSequence_Self(_self);
+  const void* item_ptr = PyUpb_AnyDescriptor_GetDef(item);
+  int n = self->funcs->get_elem_count(self->parent);
+  int count = 0;
+  for (int i = 0; i < n; i++) {
+    if (self->funcs->index(self->parent, i) == item_ptr) {
+      count++;
+    }
+  }
+  return PyLong_FromLong(count);
+}
+
+static PyObject* PyUpb_GenericSequence_Append(PyObject* self, PyObject* args) {
+  PyErr_Format(PyExc_TypeError, "'%R' is not a mutable sequence", self);
+  return NULL;
+}
+
+static PyMethodDef PyUpb_GenericSequence_Methods[] = {
+    {"index", PyUpb_GenericSequence_Index, METH_O},
+    {"count", PyUpb_GenericSequence_Count, METH_O},
+    {"append", PyUpb_GenericSequence_Append, METH_O},
+    // This was implemented for Python/C++ but so far has not been required.
+    //{ "__reversed__", (PyCFunction)Reversed, METH_NOARGS, },
+    {NULL}};
+
+static PyType_Slot PyUpb_GenericSequence_Slots[] = {
+    {Py_tp_dealloc, &PyUpb_GenericSequence_Dealloc},
+    {Py_tp_methods, &PyUpb_GenericSequence_Methods},
+    {Py_sq_length, PyUpb_GenericSequence_Length},
+    {Py_sq_item, PyUpb_GenericSequence_GetItem},
+    {Py_tp_richcompare, &PyUpb_GenericSequence_RichCompare},
+    // These were implemented for Python/C++ but so far have not been required.
+    // {Py_tp_repr, &PyUpb_GenericSequence_Repr},
+    // {Py_sq_contains, PyUpb_GenericSequence_Contains},
+    // {Py_mp_subscript, PyUpb_GenericSequence_Subscript},
+    // {Py_mp_ass_subscript, PyUpb_GenericSequence_AssignSubscript},
+    {0, NULL},
+};
+
+static PyType_Spec PyUpb_GenericSequence_Spec = {
+    PYUPB_MODULE_NAME "._GenericSequence",  // tp_name
+    sizeof(PyUpb_GenericSequence),          // tp_basicsize
+    0,                                      // tp_itemsize
+    Py_TPFLAGS_DEFAULT,                     // tp_flags
+    PyUpb_GenericSequence_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// ByNameMap
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  const PyUpb_ByNameMap_Funcs* funcs;
+  const void* parent;    // upb_MessageDef*, upb_DefPool*, etc.
+  PyObject* parent_obj;  // Python object that keeps parent alive, we own a ref.
+} PyUpb_ByNameMap;
+
+PyUpb_ByNameMap* PyUpb_ByNameMap_Self(PyObject* obj) {
+  assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_name_map_type);
+  return (PyUpb_ByNameMap*)obj;
+}
+
+static void PyUpb_ByNameMap_Dealloc(PyObject* _self) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  Py_DECREF(self->parent_obj);
+  PyUpb_Dealloc(self);
+}
+
+PyObject* PyUpb_ByNameMap_New(const PyUpb_ByNameMap_Funcs* funcs,
+                              const void* parent, PyObject* parent_obj) {
+  PyUpb_ModuleState* s = PyUpb_ModuleState_Get();
+  PyUpb_ByNameMap* map = (void*)PyType_GenericAlloc(s->by_name_map_type, 0);
+  map->funcs = funcs;
+  map->parent = parent;
+  map->parent_obj = parent_obj;
+  Py_INCREF(parent_obj);
+  return &map->ob_base;
+}
+
+static Py_ssize_t PyUpb_ByNameMap_Length(PyObject* _self) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  return self->funcs->base.get_elem_count(self->parent);
+}
+
+static PyObject* PyUpb_ByNameMap_Subscript(PyObject* _self, PyObject* key) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  const char* name = PyUpb_GetStrData(key);
+  const void* elem = name ? self->funcs->lookup(self->parent, name) : NULL;
+
+  if (elem) {
+    return self->funcs->base.get_elem_wrapper(elem);
+  } else {
+    PyErr_SetObject(PyExc_KeyError, key);
+    return NULL;
+  }
+}
+
+static int PyUpb_ByNameMap_AssignSubscript(PyObject* self, PyObject* key,
+                                           PyObject* value) {
+  PyErr_Format(PyExc_TypeError, PYUPB_MODULE_NAME
+               ".ByNameMap' object does not support item assignment");
+  return -1;
+}
+
+static int PyUpb_ByNameMap_Contains(PyObject* _self, PyObject* key) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  const char* name = PyUpb_GetStrData(key);
+  const void* elem = name ? self->funcs->lookup(self->parent, name) : NULL;
+  return elem ? 1 : 0;
+}
+
+static PyObject* PyUpb_ByNameMap_Get(PyObject* _self, PyObject* args) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  PyObject* key;
+  PyObject* default_value = Py_None;
+  if (!PyArg_UnpackTuple(args, "get", 1, 2, &key, &default_value)) {
+    return NULL;
+  }
+
+  const char* name = PyUpb_GetStrData(key);
+  const void* elem = name ? self->funcs->lookup(self->parent, name) : NULL;
+
+  if (elem) {
+    return self->funcs->base.get_elem_wrapper(elem);
+  } else {
+    Py_INCREF(default_value);
+    return default_value;
+  }
+}
+
+static PyObject* PyUpb_ByNameMap_GetIter(PyObject* _self) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  return PyUpb_DescriptorIterator_New(&self->funcs->base, self->parent,
+                                      self->parent_obj);
+}
+
+static PyObject* PyUpb_ByNameMap_Keys(PyObject* _self, PyObject* args) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  int n = self->funcs->base.get_elem_count(self->parent);
+  PyObject* ret = PyList_New(n);
+  if (!ret) return NULL;
+  for (int i = 0; i < n; i++) {
+    const void* elem = self->funcs->base.index(self->parent, i);
+    PyObject* key = PyUnicode_FromString(self->funcs->get_elem_name(elem));
+    if (!key) goto error;
+    PyList_SetItem(ret, i, key);
+  }
+  return ret;
+
+error:
+  Py_XDECREF(ret);
+  return NULL;
+}
+
+static PyObject* PyUpb_ByNameMap_Values(PyObject* _self, PyObject* args) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  int n = self->funcs->base.get_elem_count(self->parent);
+  PyObject* ret = PyList_New(n);
+  if (!ret) return NULL;
+  for (int i = 0; i < n; i++) {
+    const void* elem = self->funcs->base.index(self->parent, i);
+    PyObject* py_elem = self->funcs->base.get_elem_wrapper(elem);
+    if (!py_elem) goto error;
+    PyList_SetItem(ret, i, py_elem);
+  }
+  return ret;
+
+error:
+  Py_XDECREF(ret);
+  return NULL;
+}
+
+static PyObject* PyUpb_ByNameMap_Items(PyObject* _self, PyObject* args) {
+  PyUpb_ByNameMap* self = (PyUpb_ByNameMap*)_self;
+  int n = self->funcs->base.get_elem_count(self->parent);
+  PyObject* ret = PyList_New(n);
+  PyObject* item;
+  PyObject* py_elem;
+  if (!ret) return NULL;
+  for (int i = 0; i < n; i++) {
+    const void* elem = self->funcs->base.index(self->parent, i);
+    item = PyTuple_New(2);
+    py_elem = self->funcs->base.get_elem_wrapper(elem);
+    if (!item || !py_elem) goto error;
+    PyTuple_SetItem(item, 0,
+                    PyUnicode_FromString(self->funcs->get_elem_name(elem)));
+    PyTuple_SetItem(item, 1, py_elem);
+    PyList_SetItem(ret, i, item);
+  }
+  return ret;
+
+error:
+  Py_XDECREF(py_elem);
+  Py_XDECREF(item);
+  Py_XDECREF(ret);
+  return NULL;
+}
+
+// A mapping container can only be equal to another mapping container, or (for
+// backward compatibility) to a dict containing the same items.
+// Returns 1 if equal, 0 if unequal, -1 on error.
+static int PyUpb_ByNameMap_IsEqual(PyUpb_ByNameMap* self, PyObject* other) {
+  // Check the identity of C++ pointers.
+  if (PyObject_TypeCheck(other, Py_TYPE(self))) {
+    PyUpb_ByNameMap* other_map = (void*)other;
+    return self->parent == other_map->parent && self->funcs == other_map->funcs;
+  }
+
+  if (!PyDict_Check(other)) return 0;
+
+  PyObject* self_dict = PyDict_New();
+  PyDict_Merge(self_dict, (PyObject*)self, 0);
+  int eq = PyObject_RichCompareBool(self_dict, other, Py_EQ);
+  Py_DECREF(self_dict);
+  return eq;
+}
+
+static PyObject* PyUpb_ByNameMap_RichCompare(PyObject* _self, PyObject* other,
+                                             int opid) {
+  PyUpb_ByNameMap* self = PyUpb_ByNameMap_Self(_self);
+  if (opid != Py_EQ && opid != Py_NE) {
+    Py_RETURN_NOTIMPLEMENTED;
+  }
+  bool ret = PyUpb_ByNameMap_IsEqual(self, other);
+  if (opid == Py_NE) ret = !ret;
+  return PyBool_FromLong(ret);
+}
+
+static PyMethodDef PyUpb_ByNameMap_Methods[] = {
+    {"get", (PyCFunction)&PyUpb_ByNameMap_Get, METH_VARARGS},
+    {"keys", PyUpb_ByNameMap_Keys, METH_NOARGS},
+    {"values", PyUpb_ByNameMap_Values, METH_NOARGS},
+    {"items", PyUpb_ByNameMap_Items, METH_NOARGS},
+    {NULL}};
+
+static PyType_Slot PyUpb_ByNameMap_Slots[] = {
+    {Py_mp_ass_subscript, PyUpb_ByNameMap_AssignSubscript},
+    {Py_mp_length, PyUpb_ByNameMap_Length},
+    {Py_mp_subscript, PyUpb_ByNameMap_Subscript},
+    {Py_sq_contains, &PyUpb_ByNameMap_Contains},
+    {Py_tp_dealloc, &PyUpb_ByNameMap_Dealloc},
+    {Py_tp_iter, PyUpb_ByNameMap_GetIter},
+    {Py_tp_methods, &PyUpb_ByNameMap_Methods},
+    {Py_tp_richcompare, &PyUpb_ByNameMap_RichCompare},
+    {0, NULL},
+};
+
+static PyType_Spec PyUpb_ByNameMap_Spec = {
+    PYUPB_MODULE_NAME "._ByNameMap",  // tp_name
+    sizeof(PyUpb_ByNameMap),          // tp_basicsize
+    0,                                // tp_itemsize
+    Py_TPFLAGS_DEFAULT,               // tp_flags
+    PyUpb_ByNameMap_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// ByNumberMap
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  const PyUpb_ByNumberMap_Funcs* funcs;
+  const void* parent;    // upb_MessageDef*, upb_DefPool*, etc.
+  PyObject* parent_obj;  // Python object that keeps parent alive, we own a ref.
+} PyUpb_ByNumberMap;
+
+PyUpb_ByNumberMap* PyUpb_ByNumberMap_Self(PyObject* obj) {
+  assert(Py_TYPE(obj) == PyUpb_ModuleState_Get()->by_number_map_type);
+  return (PyUpb_ByNumberMap*)obj;
+}
+
+PyObject* PyUpb_ByNumberMap_New(const PyUpb_ByNumberMap_Funcs* funcs,
+                                const void* parent, PyObject* parent_obj) {
+  PyUpb_ModuleState* s = PyUpb_ModuleState_Get();
+  PyUpb_ByNumberMap* map = (void*)PyType_GenericAlloc(s->by_number_map_type, 0);
+  map->funcs = funcs;
+  map->parent = parent;
+  map->parent_obj = parent_obj;
+  Py_INCREF(parent_obj);
+  return &map->ob_base;
+}
+
+static void PyUpb_ByNumberMap_Dealloc(PyObject* _self) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  Py_DECREF(self->parent_obj);
+  PyUpb_Dealloc(self);
+}
+
+static Py_ssize_t PyUpb_ByNumberMap_Length(PyObject* _self) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  return self->funcs->base.get_elem_count(self->parent);
+}
+
+static const void* PyUpb_ByNumberMap_LookupHelper(PyUpb_ByNumberMap* self,
+                                                  PyObject* key) {
+  long num = PyLong_AsLong(key);
+  if (num == -1 && PyErr_Occurred()) {
+    PyErr_Clear();
+    return NULL;
+  } else {
+    return self->funcs->lookup(self->parent, num);
+  }
+}
+
+static PyObject* PyUpb_ByNumberMap_Subscript(PyObject* _self, PyObject* key) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  const void* elem = PyUpb_ByNumberMap_LookupHelper(self, key);
+  if (elem) {
+    return self->funcs->base.get_elem_wrapper(elem);
+  } else {
+    PyErr_SetObject(PyExc_KeyError, key);
+    return NULL;
+  }
+}
+
+static int PyUpb_ByNumberMap_AssignSubscript(PyObject* self, PyObject* key,
+                                             PyObject* value) {
+  PyErr_Format(PyExc_TypeError, PYUPB_MODULE_NAME
+               ".ByNumberMap' object does not support item assignment");
+  return -1;
+}
+
+static PyObject* PyUpb_ByNumberMap_Get(PyObject* _self, PyObject* args) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  PyObject* key;
+  PyObject* default_value = Py_None;
+  if (!PyArg_UnpackTuple(args, "get", 1, 2, &key, &default_value)) {
+    return NULL;
+  }
+
+  const void* elem = PyUpb_ByNumberMap_LookupHelper(self, key);
+  if (elem) {
+    return self->funcs->base.get_elem_wrapper(elem);
+  } else {
+    return PyUpb_NewRef(default_value);
+  }
+}
+
+static PyObject* PyUpb_ByNumberMap_GetIter(PyObject* _self) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  return PyUpb_DescriptorIterator_New(&self->funcs->base, self->parent,
+                                      self->parent_obj);
+}
+
+static PyObject* PyUpb_ByNumberMap_Keys(PyObject* _self, PyObject* args) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  int n = self->funcs->base.get_elem_count(self->parent);
+  PyObject* ret = PyList_New(n);
+  if (!ret) return NULL;
+  for (int i = 0; i < n; i++) {
+    const void* elem = self->funcs->base.index(self->parent, i);
+    PyObject* key = PyLong_FromLong(self->funcs->get_elem_num(elem));
+    if (!key) goto error;
+    PyList_SetItem(ret, i, key);
+  }
+  return ret;
+
+error:
+  Py_XDECREF(ret);
+  return NULL;
+}
+
+static PyObject* PyUpb_ByNumberMap_Values(PyObject* _self, PyObject* args) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  int n = self->funcs->base.get_elem_count(self->parent);
+  PyObject* ret = PyList_New(n);
+  if (!ret) return NULL;
+  for (int i = 0; i < n; i++) {
+    const void* elem = self->funcs->base.index(self->parent, i);
+    PyObject* py_elem = self->funcs->base.get_elem_wrapper(elem);
+    if (!py_elem) goto error;
+    PyList_SetItem(ret, i, py_elem);
+  }
+  return ret;
+
+error:
+  Py_XDECREF(ret);
+  return NULL;
+}
+
+static PyObject* PyUpb_ByNumberMap_Items(PyObject* _self, PyObject* args) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  int n = self->funcs->base.get_elem_count(self->parent);
+  PyObject* ret = PyList_New(n);
+  PyObject* item;
+  PyObject* py_elem;
+  if (!ret) return NULL;
+  for (int i = 0; i < n; i++) {
+    const void* elem = self->funcs->base.index(self->parent, i);
+    int number = self->funcs->get_elem_num(elem);
+    item = PyTuple_New(2);
+    py_elem = self->funcs->base.get_elem_wrapper(elem);
+    if (!item || !py_elem) goto error;
+    PyTuple_SetItem(item, 0, PyLong_FromLong(number));
+    PyTuple_SetItem(item, 1, py_elem);
+    PyList_SetItem(ret, i, item);
+  }
+  return ret;
+
+error:
+  Py_XDECREF(py_elem);
+  Py_XDECREF(item);
+  Py_XDECREF(ret);
+  return NULL;
+}
+
+static int PyUpb_ByNumberMap_Contains(PyObject* _self, PyObject* key) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  const void* elem = PyUpb_ByNumberMap_LookupHelper(self, key);
+  return elem ? 1 : 0;
+}
+
+// A mapping container can only be equal to another mapping container, or (for
+// backward compatibility) to a dict containing the same items.
+// Returns 1 if equal, 0 if unequal, -1 on error.
+static int PyUpb_ByNumberMap_IsEqual(PyUpb_ByNumberMap* self, PyObject* other) {
+  // Check the identity of C++ pointers.
+  if (PyObject_TypeCheck(other, Py_TYPE(self))) {
+    PyUpb_ByNumberMap* other_map = (void*)other;
+    return self->parent == other_map->parent && self->funcs == other_map->funcs;
+  }
+
+  if (!PyDict_Check(other)) return 0;
+
+  PyObject* self_dict = PyDict_New();
+  PyDict_Merge(self_dict, (PyObject*)self, 0);
+  int eq = PyObject_RichCompareBool(self_dict, other, Py_EQ);
+  Py_DECREF(self_dict);
+  return eq;
+}
+
+static PyObject* PyUpb_ByNumberMap_RichCompare(PyObject* _self, PyObject* other,
+                                               int opid) {
+  PyUpb_ByNumberMap* self = PyUpb_ByNumberMap_Self(_self);
+  if (opid != Py_EQ && opid != Py_NE) {
+    Py_RETURN_NOTIMPLEMENTED;
+  }
+  bool ret = PyUpb_ByNumberMap_IsEqual(self, other);
+  if (opid == Py_NE) ret = !ret;
+  return PyBool_FromLong(ret);
+}
+
+static PyMethodDef PyUpb_ByNumberMap_Methods[] = {
+    {"get", (PyCFunction)&PyUpb_ByNumberMap_Get, METH_VARARGS},
+    {"keys", PyUpb_ByNumberMap_Keys, METH_NOARGS},
+    {"values", PyUpb_ByNumberMap_Values, METH_NOARGS},
+    {"items", PyUpb_ByNumberMap_Items, METH_NOARGS},
+    {NULL}};
+
+static PyType_Slot PyUpb_ByNumberMap_Slots[] = {
+    {Py_mp_ass_subscript, PyUpb_ByNumberMap_AssignSubscript},
+    {Py_mp_length, PyUpb_ByNumberMap_Length},
+    {Py_mp_subscript, PyUpb_ByNumberMap_Subscript},
+    {Py_sq_contains, &PyUpb_ByNumberMap_Contains},
+    {Py_tp_dealloc, &PyUpb_ByNumberMap_Dealloc},
+    {Py_tp_iter, PyUpb_ByNumberMap_GetIter},
+    {Py_tp_methods, &PyUpb_ByNumberMap_Methods},
+    {Py_tp_richcompare, &PyUpb_ByNumberMap_RichCompare},
+    {0, NULL},
+};
+
+static PyType_Spec PyUpb_ByNumberMap_Spec = {
+    PYUPB_MODULE_NAME "._ByNumberMap",  // tp_name
+    sizeof(PyUpb_ByNumberMap),          // tp_basicsize
+    0,                                  // tp_itemsize
+    Py_TPFLAGS_DEFAULT,                 // tp_flags
+    PyUpb_ByNumberMap_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// Top Level
+// -----------------------------------------------------------------------------
+
+bool PyUpb_InitDescriptorContainers(PyObject* m) {
+  PyUpb_ModuleState* s = PyUpb_ModuleState_GetFromModule(m);
+
+  s->by_name_map_type = PyUpb_AddClass(m, &PyUpb_ByNameMap_Spec);
+  s->by_number_map_type = PyUpb_AddClass(m, &PyUpb_ByNumberMap_Spec);
+  s->descriptor_iterator_type =
+      PyUpb_AddClass(m, &PyUpb_DescriptorIterator_Spec);
+  s->generic_sequence_type = PyUpb_AddClass(m, &PyUpb_GenericSequence_Spec);
+
+  return s->by_name_map_type && s->by_number_map_type &&
+         s->descriptor_iterator_type && s->generic_sequence_type;
+}

python/descriptor_containers.h

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYUPB_DESCRIPTOR_CONTAINERS_H__
+#define PYUPB_DESCRIPTOR_CONTAINERS_H__
+
+// This file defines immutable Python containiner types whose data comes from
+// an underlying descriptor (def).
+//
+// Because there are many instances of these types that vend different kinds of
+// data (fields, oneofs, enums, etc) these types accept a "vtable" of function
+// pointers. This saves us from having to define numerous distinct Python types
+// for each kind of data we want to vend.
+//
+// The underlying upb APIs follow a consistent pattern that allows us to use
+// those functions directly inside these vtables, greatly reducing the amount of
+// "adaptor" code we need to write.
+
+#include <stdbool.h>
+
+#include "protobuf.h"
+#include "upb/def.h"
+
+// -----------------------------------------------------------------------------
+// PyUpb_GenericSequence
+// -----------------------------------------------------------------------------
+
+// A Python object that vends a sequence of descriptors.
+
+typedef struct {
+  // Returns the number of elements in the map.
+  int (*get_elem_count)(const void* parent);
+  // Returns an element by index.
+  const void* (*index)(const void* parent, int idx);
+  // Returns a Python object wrapping this element, caller owns a ref.
+  PyObject* (*get_elem_wrapper)(const void* elem);
+} PyUpb_GenericSequence_Funcs;
+
+// Returns a new GenericSequence.  The vtable `funcs` must outlive this object
+// (generally it should be static).  The GenericSequence will take a ref on
+// `parent_obj`, which must be sufficient to keep `parent` alive.  The object
+// `parent` will be passed as an argument to the functions in `funcs`.
+PyObject* PyUpb_GenericSequence_New(const PyUpb_GenericSequence_Funcs* funcs,
+                                    const void* parent, PyObject* parent_obj);
+
+// -----------------------------------------------------------------------------
+// PyUpb_ByNameMap
+// -----------------------------------------------------------------------------
+
+// A Python object that vends a name->descriptor map.
+
+typedef struct {
+  PyUpb_GenericSequence_Funcs base;
+  // Looks up by name and returns either a pointer to the element or NULL.
+  const void* (*lookup)(const void* parent, const char* key);
+  // Returns the name associated with this element.
+  const char* (*get_elem_name)(const void* elem);
+} PyUpb_ByNameMap_Funcs;
+
+// Returns a new ByNameMap.  The vtable `funcs` must outlive this object
+// (generally it should be static).  The ByNameMap will take a ref on
+// `parent_obj`, which must be sufficient to keep `parent` alive.  The object
+// `parent` will be passed as an argument to the functions in `funcs`.
+PyObject* PyUpb_ByNameMap_New(const PyUpb_ByNameMap_Funcs* funcs,
+                              const void* parent, PyObject* parent_obj);
+
+// -----------------------------------------------------------------------------
+// PyUpb_ByNumberMap
+// -----------------------------------------------------------------------------
+
+// A Python object that vends a number->descriptor map.
+
+typedef struct {
+  PyUpb_GenericSequence_Funcs base;
+  // Looks up by name and returns either a pointer to the element or NULL.
+  const void* (*lookup)(const void* parent, int num);
+  // Returns the name associated with this element.
+  int (*get_elem_num)(const void* elem);
+} PyUpb_ByNumberMap_Funcs;
+
+// Returns a new ByNumberMap.  The vtable `funcs` must outlive this object
+// (generally it should be static).  The ByNumberMap will take a ref on
+// `parent_obj`, which must be sufficient to keep `parent` alive.  The object
+// `parent` will be passed as an argument to the functions in `funcs`.
+PyObject* PyUpb_ByNumberMap_New(const PyUpb_ByNumberMap_Funcs* funcs,
+                                const void* parent, PyObject* parent_obj);
+
+bool PyUpb_InitDescriptorContainers(PyObject* m);
+
+#endif  // PYUPB_DESCRIPTOR_CONTAINERS_H__

python/descriptor_pool.c

@@ -0,0 +1,643 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "python/descriptor_pool.h"
+
+#include "google/protobuf/descriptor.upbdefs.h"
+#include "python/convert.h"
+#include "python/descriptor.h"
+#include "python/message.h"
+#include "python/protobuf.h"
+#include "upb/def.h"
+#include "upb/util/def_to_proto.h"
+
+// -----------------------------------------------------------------------------
+// DescriptorPool
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  upb_DefPool* symtab;
+  PyObject* db;  // The DescriptorDatabase underlying this pool.  May be NULL.
+} PyUpb_DescriptorPool;
+
+PyObject* PyUpb_DescriptorPool_GetDefaultPool() {
+  PyUpb_ModuleState* s = PyUpb_ModuleState_Get();
+  return s->default_pool;
+}
+
+const upb_MessageDef* PyUpb_DescriptorPool_GetFileProtoDef(void) {
+  PyUpb_ModuleState* s = PyUpb_ModuleState_Get();
+  if (!s->c_descriptor_symtab) {
+    s->c_descriptor_symtab = upb_DefPool_New();
+  }
+  return google_protobuf_FileDescriptorProto_getmsgdef(s->c_descriptor_symtab);
+}
+
+static PyObject* PyUpb_DescriptorPool_DoCreateWithCache(
+    PyTypeObject* type, PyObject* db, PyUpb_WeakMap* obj_cache) {
+  PyUpb_DescriptorPool* pool = (void*)PyType_GenericAlloc(type, 0);
+  pool->symtab = upb_DefPool_New();
+  pool->db = db;
+  Py_XINCREF(pool->db);
+  PyUpb_WeakMap_Add(obj_cache, pool->symtab, &pool->ob_base);
+  return &pool->ob_base;
+}
+
+static PyObject* PyUpb_DescriptorPool_DoCreate(PyTypeObject* type,
+                                               PyObject* db) {
+  return PyUpb_DescriptorPool_DoCreateWithCache(type, db,
+                                                PyUpb_ObjCache_Instance());
+}
+
+upb_DefPool* PyUpb_DescriptorPool_GetSymtab(PyObject* pool) {
+  return ((PyUpb_DescriptorPool*)pool)->symtab;
+}
+
+static int PyUpb_DescriptorPool_Traverse(PyUpb_DescriptorPool* self,
+                                         visitproc visit, void* arg) {
+  Py_VISIT(self->db);
+  return 0;
+}
+
+static int PyUpb_DescriptorPool_Clear(PyUpb_DescriptorPool* self) {
+  Py_CLEAR(self->db);
+  return 0;
+}
+
+PyObject* PyUpb_DescriptorPool_Get(const upb_DefPool* symtab) {
+  PyObject* pool = PyUpb_ObjCache_Get(symtab);
+  assert(pool);
+  return pool;
+}
+
+static void PyUpb_DescriptorPool_Dealloc(PyUpb_DescriptorPool* self) {
+  PyUpb_DescriptorPool_Clear(self);
+  upb_DefPool_Free(self->symtab);
+  PyUpb_ObjCache_Delete(self->symtab);
+  PyUpb_Dealloc(self);
+}
+
+/*
+ * DescriptorPool.__new__()
+ *
+ * Implements:
+ *   DescriptorPool(descriptor_db=None)
+ */
+static PyObject* PyUpb_DescriptorPool_New(PyTypeObject* type, PyObject* args,
+                                          PyObject* kwargs) {
+  char* kwlist[] = {"descriptor_db", 0};
+  PyObject* db = NULL;
+
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, &db)) {
+    return NULL;
+  }
+
+  if (db == Py_None) db = NULL;
+  return PyUpb_DescriptorPool_DoCreate(type, db);
+}
+
+static PyObject* PyUpb_DescriptorPool_DoAdd(PyObject* _self,
+                                            PyObject* file_desc);
+
+static bool PyUpb_DescriptorPool_TryLoadFileProto(PyUpb_DescriptorPool* self,
+                                                  PyObject* proto) {
+  if (proto == NULL) {
+    if (PyErr_ExceptionMatches(PyExc_KeyError)) {
+      // Expected error: item was simply not found.
+      PyErr_Clear();
+      return true;  // We didn't accomplish our goal, but we didn't error out.
+    }
+    return false;
+  }
+  if (proto == Py_None) return true;
+  PyObject* ret = PyUpb_DescriptorPool_DoAdd((PyObject*)self, proto);
+  bool ok = ret != NULL;
+  Py_XDECREF(ret);
+  return ok;
+}
+
+static bool PyUpb_DescriptorPool_TryLoadSymbol(PyUpb_DescriptorPool* self,
+                                               PyObject* sym) {
+  if (!self->db) return false;
+  PyObject* file_proto =
+      PyObject_CallMethod(self->db, "FindFileContainingSymbol", "O", sym);
+  bool ret = PyUpb_DescriptorPool_TryLoadFileProto(self, file_proto);
+  Py_XDECREF(file_proto);
+  return ret;
+}
+
+static bool PyUpb_DescriptorPool_TryLoadFilename(PyUpb_DescriptorPool* self,
+                                                 PyObject* filename) {
+  if (!self->db) return false;
+  PyObject* file_proto =
+      PyObject_CallMethod(self->db, "FindFileByName", "O", filename);
+  bool ret = PyUpb_DescriptorPool_TryLoadFileProto(self, file_proto);
+  Py_XDECREF(file_proto);
+  return ret;
+}
+
+bool PyUpb_DescriptorPool_CheckNoDatabase(PyObject* _self) { return true; }
+
+static bool PyUpb_DescriptorPool_LoadDependentFiles(
+    PyUpb_DescriptorPool* self, google_protobuf_FileDescriptorProto* proto) {
+  size_t n;
+  const upb_StringView* deps =
+      google_protobuf_FileDescriptorProto_dependency(proto, &n);
+  for (size_t i = 0; i < n; i++) {
+    const upb_FileDef* dep = upb_DefPool_FindFileByNameWithSize(
+        self->symtab, deps[i].data, deps[i].size);
+    if (!dep) {
+      PyObject* filename =
+          PyUnicode_FromStringAndSize(deps[i].data, deps[i].size);
+      if (!filename) return false;
+      bool ok = PyUpb_DescriptorPool_TryLoadFilename(self, filename);
+      Py_DECREF(filename);
+      if (!ok) return false;
+    }
+  }
+  return true;
+}
+
+static PyObject* PyUpb_DescriptorPool_DoAddSerializedFile(
+    PyObject* _self, PyObject* serialized_pb) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+  upb_Arena* arena = upb_Arena_New();
+  if (!arena) PYUPB_RETURN_OOM;
+  PyObject* result = NULL;
+
+  char* buf;
+  Py_ssize_t size;
+  if (PyBytes_AsStringAndSize(serialized_pb, &buf, &size) < 0) {
+    goto done;
+  }
+
+  google_protobuf_FileDescriptorProto* proto =
+      google_protobuf_FileDescriptorProto_parse(buf, size, arena);
+  if (!proto) {
+    PyErr_SetString(PyExc_TypeError, "Couldn't parse file content!");
+    goto done;
+  }
+
+  upb_StringView name = google_protobuf_FileDescriptorProto_name(proto);
+  const upb_FileDef* file =
+      upb_DefPool_FindFileByNameWithSize(self->symtab, name.data, name.size);
+
+  if (file) {
+    // If the existing file is equal to the new file, then silently ignore the
+    // duplicate add.
+    google_protobuf_FileDescriptorProto* existing =
+        upb_FileDef_ToProto(file, arena);
+    if (!existing) {
+      PyErr_SetNone(PyExc_MemoryError);
+      goto done;
+    }
+    const upb_MessageDef* m = PyUpb_DescriptorPool_GetFileProtoDef();
+    if (PyUpb_Message_IsEqual(proto, existing, m)) {
+      Py_INCREF(Py_None);
+      result = Py_None;
+      goto done;
+    }
+  }
+
+  if (self->db) {
+    if (!PyUpb_DescriptorPool_LoadDependentFiles(self, proto)) goto done;
+  }
+
+  upb_Status status;
+  upb_Status_Clear(&status);
+
+  const upb_FileDef* filedef =
+      upb_DefPool_AddFile(self->symtab, proto, &status);
+  if (!filedef) {
+    PyErr_Format(PyExc_TypeError,
+                 "Couldn't build proto file into descriptor pool: %s",
+                 upb_Status_ErrorMessage(&status));
+    goto done;
+  }
+
+  result = PyUpb_FileDescriptor_Get(filedef);
+
+done:
+  upb_Arena_Free(arena);
+  return result;
+}
+
+static PyObject* PyUpb_DescriptorPool_DoAdd(PyObject* _self,
+                                            PyObject* file_desc) {
+  if (!PyUpb_CMessage_Verify(file_desc)) return NULL;
+  const upb_MessageDef* m = PyUpb_CMessage_GetMsgdef(file_desc);
+  const char* file_proto_name =
+      PYUPB_DESCRIPTOR_PROTO_PACKAGE ".FileDescriptorProto";
+  if (strcmp(upb_MessageDef_FullName(m), file_proto_name) != 0) {
+    return PyErr_Format(PyExc_TypeError, "Can only add FileDescriptorProto");
+  }
+  PyObject* subargs = PyTuple_New(0);
+  if (!subargs) return NULL;
+  PyObject* serialized =
+      PyUpb_CMessage_SerializeToString(file_desc, subargs, NULL);
+  Py_DECREF(subargs);
+  if (!serialized) return NULL;
+  PyObject* ret = PyUpb_DescriptorPool_DoAddSerializedFile(_self, serialized);
+  Py_DECREF(serialized);
+  return ret;
+}
+
+/*
+ * PyUpb_DescriptorPool_AddSerializedFile()
+ *
+ * Implements:
+ *   DescriptorPool.AddSerializedFile(self, serialized_file_descriptor)
+ *
+ * Adds the given serialized FileDescriptorProto to the pool.
+ */
+static PyObject* PyUpb_DescriptorPool_AddSerializedFile(
+    PyObject* _self, PyObject* serialized_pb) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+  if (self->db) {
+    PyErr_SetString(
+        PyExc_ValueError,
+        "Cannot call AddSerializedFile on a DescriptorPool that uses a "
+        "DescriptorDatabase. Add your file to the underlying database.");
+    return false;
+  }
+  return PyUpb_DescriptorPool_DoAddSerializedFile(_self, serialized_pb);
+}
+
+static PyObject* PyUpb_DescriptorPool_Add(PyObject* _self,
+                                          PyObject* file_desc) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+  if (self->db) {
+    PyErr_SetString(
+        PyExc_ValueError,
+        "Cannot call Add on a DescriptorPool that uses a DescriptorDatabase. "
+        "Add your file to the underlying database.");
+    return false;
+  }
+  return PyUpb_DescriptorPool_DoAdd(_self, file_desc);
+}
+
+/*
+ * PyUpb_DescriptorPool_FindFileByName()
+ *
+ * Implements:
+ *   DescriptorPool.FindFileByName(self, name)
+ */
+static PyObject* PyUpb_DescriptorPool_FindFileByName(PyObject* _self,
+                                                     PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+
+  const upb_FileDef* file = upb_DefPool_FindFileByName(self->symtab, name);
+  if (file == NULL && self->db) {
+    if (!PyUpb_DescriptorPool_TryLoadFilename(self, arg)) return NULL;
+    file = upb_DefPool_FindFileByName(self->symtab, name);
+  }
+  if (file == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Couldn't find file %.200s", name);
+  }
+
+  return PyUpb_FileDescriptor_Get(file);
+}
+
+/*
+ * PyUpb_DescriptorPool_FindExtensionByName()
+ *
+ * Implements:
+ *   DescriptorPool.FindExtensionByName(self, name)
+ */
+static PyObject* PyUpb_DescriptorPool_FindExtensionByName(PyObject* _self,
+                                                          PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+
+  const upb_FieldDef* field =
+      upb_DefPool_FindExtensionByName(self->symtab, name);
+  if (field == NULL && self->db) {
+    if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL;
+    field = upb_DefPool_FindExtensionByName(self->symtab, name);
+  }
+  if (field == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Couldn't find extension %.200s", name);
+  }
+
+  return PyUpb_FieldDescriptor_Get(field);
+}
+
+/*
+ * PyUpb_DescriptorPool_FindMessageTypeByName()
+ *
+ * Implements:
+ *   DescriptorPool.FindMessageTypeByName(self, name)
+ */
+static PyObject* PyUpb_DescriptorPool_FindMessageTypeByName(PyObject* _self,
+                                                            PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+
+  const upb_MessageDef* m = upb_DefPool_FindMessageByName(self->symtab, name);
+  if (m == NULL && self->db) {
+    if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL;
+    m = upb_DefPool_FindMessageByName(self->symtab, name);
+  }
+  if (m == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name);
+  }
+
+  return PyUpb_Descriptor_Get(m);
+}
+
+// Splits a dotted symbol like foo.bar.baz on the last dot.  Returns the portion
+// after the last dot (baz) and updates `*parent_size` to the length of the
+// parent (foo.bar).  Returns NULL if no dots were present.
+static const char* PyUpb_DescriptorPool_SplitSymbolName(const char* sym,
+                                                        size_t* parent_size) {
+  const char* last_dot = strrchr(sym, '.');
+  if (!last_dot) return NULL;
+  *parent_size = last_dot - sym;
+  return last_dot + 1;
+}
+
+/*
+ * PyUpb_DescriptorPool_FindFieldByName()
+ *
+ * Implements:
+ *   DescriptorPool.FindFieldByName(self, name)
+ */
+static PyObject* PyUpb_DescriptorPool_FindFieldByName(PyObject* _self,
+                                                      PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+
+  size_t parent_size;
+  const char* child = PyUpb_DescriptorPool_SplitSymbolName(name, &parent_size);
+  const upb_FieldDef* f = NULL;
+  if (child) {
+    const upb_MessageDef* parent =
+        upb_DefPool_FindMessageByNameWithSize(self->symtab, name, parent_size);
+    if (parent == NULL && self->db) {
+      if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL;
+      parent = upb_DefPool_FindMessageByNameWithSize(self->symtab, name,
+                                                     parent_size);
+    }
+    if (parent) {
+      f = upb_MessageDef_FindFieldByName(parent, child);
+    }
+  }
+
+  if (!f) {
+    return PyErr_Format(PyExc_KeyError, "Couldn't find message %.200s", name);
+  }
+
+  return PyUpb_FieldDescriptor_Get(f);
+}
+
+/*
+ * PyUpb_DescriptorPool_FindEnumTypeByName()
+ *
+ * Implements:
+ *   DescriptorPool.FindEnumTypeByName(self, name)
+ */
+static PyObject* PyUpb_DescriptorPool_FindEnumTypeByName(PyObject* _self,
+                                                         PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+
+  const upb_EnumDef* e = upb_DefPool_FindEnumByName(self->symtab, name);
+  if (e == NULL && self->db) {
+    if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL;
+    e = upb_DefPool_FindEnumByName(self->symtab, name);
+  }
+  if (e == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Couldn't find enum %.200s", name);
+  }
+
+  return PyUpb_EnumDescriptor_Get(e);
+}
+
+/*
+ * PyUpb_DescriptorPool_FindOneofByName()
+ *
+ * Implements:
+ *   DescriptorPool.FindOneofByName(self, name)
+ */
+static PyObject* PyUpb_DescriptorPool_FindOneofByName(PyObject* _self,
+                                                      PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+
+  size_t parent_size;
+  const char* child = PyUpb_DescriptorPool_SplitSymbolName(name, &parent_size);
+
+  if (child) {
+    const upb_MessageDef* parent =
+        upb_DefPool_FindMessageByNameWithSize(self->symtab, name, parent_size);
+    if (parent == NULL && self->db) {
+      if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL;
+      parent = upb_DefPool_FindMessageByNameWithSize(self->symtab, name,
+                                                     parent_size);
+    }
+    if (parent) {
+      const upb_OneofDef* o = upb_MessageDef_FindOneofByName(parent, child);
+      return PyUpb_OneofDescriptor_Get(o);
+    }
+  }
+
+  return PyErr_Format(PyExc_KeyError, "Couldn't find oneof %.200s", name);
+}
+
+static PyObject* PyUpb_DescriptorPool_FindServiceByName(PyObject* _self,
+                                                        PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+
+  const upb_ServiceDef* s = upb_DefPool_FindServiceByName(self->symtab, name);
+  if (s == NULL && self->db) {
+    if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL;
+    s = upb_DefPool_FindServiceByName(self->symtab, name);
+  }
+  if (s == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Couldn't find service %.200s", name);
+  }
+
+  return PyUpb_ServiceDescriptor_Get(s);
+}
+
+static PyObject* PyUpb_DescriptorPool_FindMethodByName(PyObject* _self,
+                                                       PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+  size_t parent_size;
+  const char* child = PyUpb_DescriptorPool_SplitSymbolName(name, &parent_size);
+
+  if (!child) goto err;
+  const upb_ServiceDef* parent =
+      upb_DefPool_FindServiceByNameWithSize(self->symtab, name, parent_size);
+  if (parent == NULL && self->db) {
+    if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL;
+    parent =
+        upb_DefPool_FindServiceByNameWithSize(self->symtab, name, parent_size);
+  }
+  if (!parent) goto err;
+  const upb_MethodDef* m = upb_ServiceDef_FindMethodByName(parent, child);
+  if (!m) goto err;
+  return PyUpb_MethodDescriptor_Get(m);
+
+err:
+  return PyErr_Format(PyExc_KeyError, "Couldn't find method %.200s", name);
+}
+
+static PyObject* PyUpb_DescriptorPool_FindFileContainingSymbol(PyObject* _self,
+                                                               PyObject* arg) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+
+  const char* name = PyUpb_VerifyStrData(arg);
+  if (!name) return NULL;
+
+  const upb_FileDef* f =
+      upb_DefPool_FindFileContainingSymbol(self->symtab, name);
+  if (f == NULL && self->db) {
+    if (!PyUpb_DescriptorPool_TryLoadSymbol(self, arg)) return NULL;
+    f = upb_DefPool_FindFileContainingSymbol(self->symtab, name);
+  }
+  if (f == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Couldn't find symbol %.200s", name);
+  }
+
+  return PyUpb_FileDescriptor_Get(f);
+}
+
+static PyObject* PyUpb_DescriptorPool_FindExtensionByNumber(PyObject* _self,
+                                                            PyObject* args) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+  PyObject* message_descriptor;
+  int number;
+  if (!PyArg_ParseTuple(args, "Oi", &message_descriptor, &number)) {
+    return NULL;
+  }
+
+  const upb_FieldDef* f = upb_DefPool_FindExtensionByNumber(
+      self->symtab, PyUpb_Descriptor_GetDef(message_descriptor), number);
+  if (f == NULL) {
+    return PyErr_Format(PyExc_KeyError, "Couldn't find Extension %d", number);
+  }
+
+  return PyUpb_FieldDescriptor_Get(f);
+}
+
+static PyObject* PyUpb_DescriptorPool_FindAllExtensions(PyObject* _self,
+                                                        PyObject* msg_desc) {
+  PyUpb_DescriptorPool* self = (PyUpb_DescriptorPool*)_self;
+  const upb_MessageDef* m = PyUpb_Descriptor_GetDef(msg_desc);
+  size_t n;
+  const upb_FieldDef** ext = upb_DefPool_GetAllExtensions(self->symtab, m, &n);
+  PyObject* ret = PyList_New(n);
+  for (size_t i = 0; i < n; i++) {
+    PyObject* field = PyUpb_FieldDescriptor_Get(ext[i]);
+    if (!field) return NULL;
+    PyList_SetItem(ret, i, field);
+  }
+  return ret;
+}
+
+static PyMethodDef PyUpb_DescriptorPool_Methods[] = {
+    {"Add", PyUpb_DescriptorPool_Add, METH_O,
+     "Adds the FileDescriptorProto and its types to this pool."},
+    {"AddSerializedFile", PyUpb_DescriptorPool_AddSerializedFile, METH_O,
+     "Adds a serialized FileDescriptorProto to this pool."},
+    {"FindFileByName", PyUpb_DescriptorPool_FindFileByName, METH_O,
+     "Searches for a file descriptor by its .proto name."},
+    {"FindMessageTypeByName", PyUpb_DescriptorPool_FindMessageTypeByName,
+     METH_O, "Searches for a message descriptor by full name."},
+    {"FindFieldByName", PyUpb_DescriptorPool_FindFieldByName, METH_O,
+     "Searches for a field descriptor by full name."},
+    {"FindExtensionByName", PyUpb_DescriptorPool_FindExtensionByName, METH_O,
+     "Searches for extension descriptor by full name."},
+    {"FindEnumTypeByName", PyUpb_DescriptorPool_FindEnumTypeByName, METH_O,
+     "Searches for enum type descriptor by full name."},
+    {"FindOneofByName", PyUpb_DescriptorPool_FindOneofByName, METH_O,
+     "Searches for oneof descriptor by full name."},
+    {"FindServiceByName", PyUpb_DescriptorPool_FindServiceByName, METH_O,
+     "Searches for service descriptor by full name."},
+    {"FindMethodByName", PyUpb_DescriptorPool_FindMethodByName, METH_O,
+     "Searches for method descriptor by full name."},
+    {"FindFileContainingSymbol", PyUpb_DescriptorPool_FindFileContainingSymbol,
+     METH_O, "Gets the FileDescriptor containing the specified symbol."},
+    {"FindExtensionByNumber", PyUpb_DescriptorPool_FindExtensionByNumber,
+     METH_VARARGS, "Gets the extension descriptor for the given number."},
+    {"FindAllExtensions", PyUpb_DescriptorPool_FindAllExtensions, METH_O,
+     "Gets all known extensions of the given message descriptor."},
+    {NULL}};
+
+static PyType_Slot PyUpb_DescriptorPool_Slots[] = {
+    {Py_tp_clear, PyUpb_DescriptorPool_Clear},
+    {Py_tp_dealloc, PyUpb_DescriptorPool_Dealloc},
+    {Py_tp_methods, PyUpb_DescriptorPool_Methods},
+    {Py_tp_new, PyUpb_DescriptorPool_New},
+    {Py_tp_traverse, PyUpb_DescriptorPool_Traverse},
+    {0, NULL}};
+
+static PyType_Spec PyUpb_DescriptorPool_Spec = {
+    PYUPB_MODULE_NAME ".DescriptorPool",
+    sizeof(PyUpb_DescriptorPool),
+    0,  // tp_itemsize
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,
+    PyUpb_DescriptorPool_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// Top Level
+// -----------------------------------------------------------------------------
+
+bool PyUpb_InitDescriptorPool(PyObject* m) {
+  PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m);
+  PyTypeObject* descriptor_pool_type =
+      PyUpb_AddClass(m, &PyUpb_DescriptorPool_Spec);
+
+  if (!descriptor_pool_type) return false;
+
+  state->default_pool = PyUpb_DescriptorPool_DoCreateWithCache(
+      descriptor_pool_type, NULL, state->obj_cache);
+  return state->default_pool &&
+         PyModule_AddObject(m, "default_pool", state->default_pool) == 0;
+}

python/descriptor_pool.h

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYUPB_DESCRIPTOR_POOL_H__
+#define PYUPB_DESCRIPTOR_POOL_H__
+
+#include <stdbool.h>
+
+#include "protobuf.h"
+
+// Returns a Python wrapper object for the given symtab. The symtab must have
+// been created from a Python DescriptorPool originally.
+PyObject* PyUpb_DescriptorPool_Get(const upb_DefPool* symtab);
+
+// Given a Python DescriptorPool, returns the underlying symtab.
+upb_DefPool* PyUpb_DescriptorPool_GetSymtab(PyObject* pool);
+
+// Returns the default DescriptorPool (a global singleton).
+PyObject* PyUpb_DescriptorPool_GetDefaultPool(void);
+
+// Module-level init.
+bool PyUpb_InitDescriptorPool(PyObject* m);
+
+#endif  // PYUPB_DESCRIPTOR_POOL_H__

python/extension_dict.c

@@ -0,0 +1,247 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "python/extension_dict.h"
+
+#include "python/message.h"
+#include "python/protobuf.h"
+
+// -----------------------------------------------------------------------------
+// ExtensionDict
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  PyObject* msg;  // Owning ref to our parent pessage.
+} PyUpb_ExtensionDict;
+
+PyObject* PyUpb_ExtensionDict_New(PyObject* msg) {
+  PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
+  PyUpb_ExtensionDict* ext_dict =
+      (void*)PyType_GenericAlloc(state->extension_dict_type, 0);
+  ext_dict->msg = msg;
+  Py_INCREF(ext_dict->msg);
+  return &ext_dict->ob_base;
+}
+
+static PyObject* PyUpb_ExtensionDict_FindExtensionByName(PyObject* _self,
+                                                         PyObject* key) {
+  PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
+  const char* name = PyUpb_GetStrData(key);
+  const upb_MessageDef* m = PyUpb_CMessage_GetMsgdef(self->msg);
+  const upb_FileDef* file = upb_MessageDef_File(m);
+  const upb_DefPool* symtab = upb_FileDef_Pool(file);
+  const upb_FieldDef* ext = upb_DefPool_FindExtensionByName(symtab, name);
+  if (ext) {
+    return PyUpb_FieldDescriptor_Get(ext);
+  } else {
+    Py_RETURN_NONE;
+  }
+}
+
+static PyObject* PyUpb_ExtensionDict_FindExtensionByNumber(PyObject* _self,
+                                                           PyObject* arg) {
+  PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
+  const upb_MessageDef* m = PyUpb_CMessage_GetMsgdef(self->msg);
+  const upb_MiniTable* l = upb_MessageDef_MiniTable(m);
+  const upb_FileDef* file = upb_MessageDef_File(m);
+  const upb_DefPool* symtab = upb_FileDef_Pool(file);
+  const upb_ExtensionRegistry* reg = upb_DefPool_ExtensionRegistry(symtab);
+  int64_t number = PyLong_AsLong(arg);
+  const upb_MiniTable_Extension* ext =
+      (upb_MiniTable_Extension*)_upb_extreg_get(reg, l, number);
+  if (ext) {
+    const upb_FieldDef* f = _upb_DefPool_FindExtensionByMiniTable(symtab, ext);
+    return PyUpb_FieldDescriptor_Get(f);
+  } else {
+    Py_RETURN_NONE;
+  }
+}
+
+static void PyUpb_ExtensionDict_Dealloc(PyUpb_ExtensionDict* self) {
+  PyUpb_CMessage_ClearExtensionDict(self->msg);
+  Py_DECREF(self->msg);
+  PyUpb_Dealloc(self);
+}
+
+static PyObject* PyUpb_ExtensionDict_RichCompare(PyObject* _self,
+                                                 PyObject* _other, int opid) {
+  // Only equality comparisons are implemented.
+  if (opid != Py_EQ && opid != Py_NE) {
+    Py_INCREF(Py_NotImplemented);
+    return Py_NotImplemented;
+  }
+  PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
+  bool equals = false;
+  if (PyObject_TypeCheck(_other, Py_TYPE(_self))) {
+    PyUpb_ExtensionDict* other = (PyUpb_ExtensionDict*)_other;
+    equals = self->msg == other->msg;
+  }
+  bool ret = opid == Py_EQ ? equals : !equals;
+  return PyBool_FromLong(ret);
+}
+
+static int PyUpb_ExtensionDict_Contains(PyObject* _self, PyObject* key) {
+  PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
+  const upb_FieldDef* f = PyUpb_CMessage_GetExtensionDef(self->msg, key);
+  if (!f) return -1;
+  upb_Message* msg = PyUpb_CMessage_GetIfReified(self->msg);
+  if (!msg) return 0;
+  if (upb_FieldDef_IsRepeated(f)) {
+    upb_MessageValue val = upb_Message_Get(msg, f);
+    return upb_Array_Size(val.array_val) > 0;
+  } else {
+    return upb_Message_Has(msg, f);
+  }
+}
+
+static Py_ssize_t PyUpb_ExtensionDict_Length(PyObject* _self) {
+  PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
+  upb_Message* msg = PyUpb_CMessage_GetIfReified(self->msg);
+  return msg ? upb_Message_ExtensionCount(msg) : 0;
+}
+
+static PyObject* PyUpb_ExtensionDict_Subscript(PyObject* _self, PyObject* key) {
+  PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
+  const upb_FieldDef* f = PyUpb_CMessage_GetExtensionDef(self->msg, key);
+  if (!f) return NULL;
+  return PyUpb_CMessage_GetFieldValue(self->msg, f);
+}
+
+static int PyUpb_ExtensionDict_AssignSubscript(PyObject* _self, PyObject* key,
+                                               PyObject* val) {
+  PyUpb_ExtensionDict* self = (PyUpb_ExtensionDict*)_self;
+  const upb_FieldDef* f = PyUpb_CMessage_GetExtensionDef(self->msg, key);
+  if (!f) return -1;
+  if (val) {
+    return PyUpb_CMessage_SetFieldValue(self->msg, f, val, PyExc_TypeError);
+  } else {
+    PyUpb_CMessage_DoClearField(self->msg, f);
+    return 0;
+  }
+}
+
+static PyObject* PyUpb_ExtensionIterator_New(PyObject* _ext_dict);
+
+static PyMethodDef PyUpb_ExtensionDict_Methods[] = {
+    {"_FindExtensionByName", PyUpb_ExtensionDict_FindExtensionByName, METH_O,
+     "Finds an extension by name."},
+    {"_FindExtensionByNumber", PyUpb_ExtensionDict_FindExtensionByNumber,
+     METH_O, "Finds an extension by number."},
+    {NULL, NULL},
+};
+
+static PyType_Slot PyUpb_ExtensionDict_Slots[] = {
+    {Py_tp_dealloc, PyUpb_ExtensionDict_Dealloc},
+    {Py_tp_methods, PyUpb_ExtensionDict_Methods},
+    //{Py_tp_getset, PyUpb_ExtensionDict_Getters},
+    //{Py_tp_hash, PyObject_HashNotImplemented},
+    {Py_tp_richcompare, PyUpb_ExtensionDict_RichCompare},
+    {Py_tp_iter, PyUpb_ExtensionIterator_New},
+    {Py_sq_contains, PyUpb_ExtensionDict_Contains},
+    {Py_sq_length, PyUpb_ExtensionDict_Length},
+    {Py_mp_length, PyUpb_ExtensionDict_Length},
+    {Py_mp_subscript, PyUpb_ExtensionDict_Subscript},
+    {Py_mp_ass_subscript, PyUpb_ExtensionDict_AssignSubscript},
+    {0, NULL}};
+
+static PyType_Spec PyUpb_ExtensionDict_Spec = {
+    PYUPB_MODULE_NAME ".ExtensionDict",  // tp_name
+    sizeof(PyUpb_ExtensionDict),         // tp_basicsize
+    0,                                   // tp_itemsize
+    Py_TPFLAGS_DEFAULT,                  // tp_flags
+    PyUpb_ExtensionDict_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// ExtensionIterator
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  PyObject* msg;
+  size_t iter;
+} PyUpb_ExtensionIterator;
+
+static PyObject* PyUpb_ExtensionIterator_New(PyObject* _ext_dict) {
+  PyUpb_ExtensionDict* ext_dict = (PyUpb_ExtensionDict*)_ext_dict;
+  PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
+  PyUpb_ExtensionIterator* iter =
+      (void*)PyType_GenericAlloc(state->extension_iterator_type, 0);
+  if (!iter) return NULL;
+  iter->msg = ext_dict->msg;
+  iter->iter = kUpb_Message_Begin;
+  Py_INCREF(iter->msg);
+  return &iter->ob_base;
+}
+
+static void PyUpb_ExtensionIterator_Dealloc(void* _self) {
+  PyUpb_ExtensionIterator* self = (PyUpb_ExtensionIterator*)_self;
+  Py_DECREF(self->msg);
+  PyUpb_Dealloc(_self);
+}
+
+PyObject* PyUpb_ExtensionIterator_IterNext(PyObject* _self) {
+  PyUpb_ExtensionIterator* self = (PyUpb_ExtensionIterator*)_self;
+  upb_Message* msg = PyUpb_CMessage_GetIfReified(self->msg);
+  if (!msg) return NULL;
+  const upb_MessageDef* m = PyUpb_CMessage_GetMsgdef(self->msg);
+  const upb_DefPool* symtab = upb_FileDef_Pool(upb_MessageDef_File(m));
+  while (true) {
+    const upb_FieldDef* f;
+    upb_MessageValue val;
+    if (!upb_Message_Next(msg, m, symtab, &f, &val, &self->iter)) return NULL;
+    if (upb_FieldDef_IsExtension(f)) return PyUpb_FieldDescriptor_Get(f);
+  }
+}
+
+static PyType_Slot PyUpb_ExtensionIterator_Slots[] = {
+    {Py_tp_dealloc, PyUpb_ExtensionIterator_Dealloc},
+    {Py_tp_iter, PyObject_SelfIter},
+    {Py_tp_iternext, PyUpb_ExtensionIterator_IterNext},
+    {0, NULL}};
+
+static PyType_Spec PyUpb_ExtensionIterator_Spec = {
+    PYUPB_MODULE_NAME ".ExtensionIterator",  // tp_name
+    sizeof(PyUpb_ExtensionIterator),         // tp_basicsize
+    0,                                       // tp_itemsize
+    Py_TPFLAGS_DEFAULT,                      // tp_flags
+    PyUpb_ExtensionIterator_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// Top Level
+// -----------------------------------------------------------------------------
+
+bool PyUpb_InitExtensionDict(PyObject* m) {
+  PyUpb_ModuleState* s = PyUpb_ModuleState_GetFromModule(m);
+
+  s->extension_dict_type = PyUpb_AddClass(m, &PyUpb_ExtensionDict_Spec);
+  s->extension_iterator_type = PyUpb_AddClass(m, &PyUpb_ExtensionIterator_Spec);
+
+  return s->extension_dict_type && s->extension_iterator_type;
+}

python/extension_dict.h

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYUPB_EXTENSION_DICT_H__
+#define PYUPB_EXTENSION_DICT_H__
+
+#include <stdbool.h>
+
+#include "python/python.h"
+
+PyObject* PyUpb_ExtensionDict_New(PyObject* msg);
+
+bool PyUpb_InitExtensionDict(PyObject* m);
+
+#endif  // PYUPB_EXTENSION_DICT_H__

python/map.c

@@ -0,0 +1,506 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "python/map.h"
+
+#include "python/convert.h"
+#include "python/message.h"
+#include "python/protobuf.h"
+
+// -----------------------------------------------------------------------------
+// MapContainer
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  PyObject* arena;
+  // The field descriptor (upb_FieldDef*).
+  // The low bit indicates whether the container is reified (see ptr below).
+  //   - low bit set: repeated field is a stub (empty map, no underlying data).
+  //   - low bit clear: repeated field is reified (points to upb_Array).
+  uintptr_t field;
+  union {
+    PyObject* parent;  // stub: owning pointer to parent message.
+    upb_Map* map;      // reified: the data for this array.
+  } ptr;
+  int version;
+} PyUpb_MapContainer;
+
+static PyObject* PyUpb_MapIterator_New(PyUpb_MapContainer* map);
+
+static bool PyUpb_MapContainer_IsStub(PyUpb_MapContainer* self) {
+  return self->field & 1;
+}
+
+// If the map is reified, returns it.  Otherwise, returns NULL.
+// If NULL is returned, the object is empty and has no underlying data.
+static upb_Map* PyUpb_MapContainer_GetIfReified(PyUpb_MapContainer* self) {
+  return PyUpb_MapContainer_IsStub(self) ? NULL : self->ptr.map;
+}
+
+static const upb_FieldDef* PyUpb_MapContainer_GetField(
+    PyUpb_MapContainer* self) {
+  return (const upb_FieldDef*)(self->field & ~(uintptr_t)1);
+}
+
+static void PyUpb_MapContainer_Dealloc(void* _self) {
+  PyUpb_MapContainer* self = _self;
+  Py_DECREF(self->arena);
+  if (PyUpb_MapContainer_IsStub(self)) {
+    PyUpb_CMessage_CacheDelete(self->ptr.parent,
+                               PyUpb_MapContainer_GetField(self));
+    Py_DECREF(self->ptr.parent);
+  } else {
+    PyUpb_ObjCache_Delete(self->ptr.map);
+  }
+  PyUpb_Dealloc(_self);
+}
+
+PyTypeObject* PyUpb_MapContainer_GetClass(const upb_FieldDef* f) {
+  assert(upb_FieldDef_IsMap(f));
+  PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
+  return upb_FieldDef_IsSubMessage(f) ? state->message_map_container_type
+                                      : state->scalar_map_container_type;
+}
+
+PyObject* PyUpb_MapContainer_NewStub(PyObject* parent, const upb_FieldDef* f,
+                                     PyObject* arena) {
+  // We only create stubs when the parent is reified, by convention.  However
+  // this is not an invariant: the parent could become reified at any time.
+  assert(PyUpb_CMessage_GetIfReified(parent) == NULL);
+  PyTypeObject* cls = PyUpb_MapContainer_GetClass(f);
+  PyUpb_MapContainer* map = (void*)PyType_GenericAlloc(cls, 0);
+  map->arena = arena;
+  map->field = (uintptr_t)f | 1;
+  map->ptr.parent = parent;
+  map->version = 0;
+  Py_INCREF(arena);
+  Py_INCREF(parent);
+  return &map->ob_base;
+}
+
+void PyUpb_MapContainer_Reify(PyObject* _self, upb_Map* map) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  if (!map) {
+    const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+    upb_Arena* arena = PyUpb_Arena_Get(self->arena);
+    const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+    const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0);
+    const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1);
+    map = upb_Map_New(arena, upb_FieldDef_CType(key_f),
+                      upb_FieldDef_CType(val_f));
+  }
+  PyUpb_ObjCache_Add(map, &self->ob_base);
+  Py_DECREF(self->ptr.parent);
+  self->ptr.map = map;  // Overwrites self->ptr.parent.
+  self->field &= ~(uintptr_t)1;
+  assert(!PyUpb_MapContainer_IsStub(self));
+}
+
+void PyUpb_MapContainer_Invalidate(PyObject* obj) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)obj;
+  self->version++;
+}
+
+upb_Map* PyUpb_MapContainer_EnsureReified(PyObject* _self) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  self->version++;
+  upb_Map* map = PyUpb_MapContainer_GetIfReified(self);
+  if (map) return map;  // Already writable.
+
+  const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+  upb_Arena* arena = PyUpb_Arena_Get(self->arena);
+  const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+  const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0);
+  const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1);
+  map =
+      upb_Map_New(arena, upb_FieldDef_CType(key_f), upb_FieldDef_CType(val_f));
+  upb_MessageValue msgval = {.map_val = map};
+  PyUpb_CMessage_SetConcreteSubobj(self->ptr.parent, f, msgval);
+  PyUpb_MapContainer_Reify((PyObject*)self, map);
+  return map;
+}
+
+int PyUpb_MapContainer_AssignSubscript(PyObject* _self, PyObject* key,
+                                       PyObject* val) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_Map* map = PyUpb_MapContainer_EnsureReified(_self);
+  const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+  const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+  const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0);
+  const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1);
+  upb_Arena* arena = PyUpb_Arena_Get(self->arena);
+  upb_MessageValue u_key, u_val;
+  if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return -1;
+
+  if (val) {
+    if (!PyUpb_PyToUpb(val, val_f, &u_val, arena)) return -1;
+    upb_Map_Set(map, u_key, u_val, arena);
+  } else {
+    if (!upb_Map_Delete(map, u_key)) {
+      PyErr_Format(PyExc_KeyError, "Key not present in map");
+      return -1;
+    }
+  }
+  return 0;
+}
+
+PyObject* PyUpb_MapContainer_Subscript(PyObject* _self, PyObject* key) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_Map* map = PyUpb_MapContainer_GetIfReified(self);
+  const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+  const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+  const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0);
+  const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1);
+  upb_Arena* arena = PyUpb_Arena_Get(self->arena);
+  upb_MessageValue u_key, u_val;
+  if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return NULL;
+  if (!map || !upb_Map_Get(map, u_key, &u_val)) {
+    map = PyUpb_MapContainer_EnsureReified(_self);
+    upb_Arena* arena = PyUpb_Arena_Get(self->arena);
+    if (upb_FieldDef_IsSubMessage(val_f)) {
+      u_val.msg_val = upb_Message_New(upb_FieldDef_MessageSubDef(val_f), arena);
+    } else {
+      memset(&u_val, 0, sizeof(u_val));
+    }
+    upb_Map_Set(map, u_key, u_val, arena);
+  }
+  return PyUpb_UpbToPy(u_val, val_f, self->arena);
+}
+
+PyObject* PyUpb_MapContainer_Contains(PyObject* _self, PyObject* key) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_Map* map = PyUpb_MapContainer_GetIfReified(self);
+  if (!map) Py_RETURN_FALSE;
+  const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+  const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+  const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0);
+  upb_MessageValue u_key;
+  if (!PyUpb_PyToUpb(key, key_f, &u_key, NULL)) return NULL;
+  if (upb_Map_Get(map, u_key, NULL)) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
+PyObject* PyUpb_MapContainer_Clear(PyObject* _self, PyObject* key) {
+  upb_Map* map = PyUpb_MapContainer_EnsureReified(_self);
+  upb_Map_Clear(map);
+  Py_RETURN_NONE;
+}
+
+static PyObject* PyUpb_MapContainer_Get(PyObject* _self, PyObject* args,
+                                        PyObject* kwargs) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  static const char* kwlist[] = {"key", "default", NULL};
+  PyObject* key;
+  PyObject* default_value = NULL;
+  upb_Map* map = PyUpb_MapContainer_GetIfReified(self);
+  if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", (char**)kwlist, &key,
+                                   &default_value)) {
+    return NULL;
+  }
+
+  const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+  const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+  const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0);
+  const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1);
+  upb_Arena* arena = PyUpb_Arena_Get(self->arena);
+  upb_MessageValue u_key, u_val;
+  if (!PyUpb_PyToUpb(key, key_f, &u_key, arena)) return NULL;
+  if (map && upb_Map_Get(map, u_key, &u_val)) {
+    return PyUpb_UpbToPy(u_val, val_f, self->arena);
+  }
+  if (default_value) {
+    Py_INCREF(default_value);
+    return default_value;
+  }
+  Py_RETURN_NONE;
+}
+
+static PyObject* PyUpb_MapContainer_GetEntryClass(PyObject* _self,
+                                                  PyObject* arg) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+  const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+  return PyUpb_Descriptor_GetClass(entry_m);
+}
+
+Py_ssize_t PyUpb_MapContainer_Length(PyObject* _self) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_Map* map = PyUpb_MapContainer_GetIfReified(self);
+  return map ? upb_Map_Size(map) : 0;
+}
+
+PyUpb_MapContainer* PyUpb_MapContainer_Check(PyObject* _self) {
+  PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
+  if (!PyObject_TypeCheck(_self, state->message_map_container_type) &&
+      !PyObject_TypeCheck(_self, state->scalar_map_container_type)) {
+    PyErr_Format(PyExc_TypeError, "Expected protobuf map, but got %R", _self);
+    return NULL;
+  }
+  return (PyUpb_MapContainer*)_self;
+}
+
+int PyUpb_CMessage_InitMapAttributes(PyObject* map, PyObject* value,
+                                     const upb_FieldDef* f);
+
+static PyObject* PyUpb_MapContainer_MergeFrom(PyObject* _self, PyObject* _arg) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+
+  if (PyDict_Check(_arg)) {
+    return PyErr_Format(PyExc_AttributeError, "Merging of dict is not allowed");
+  }
+
+  if (PyUpb_CMessage_InitMapAttributes(_self, _arg, f) < 0) {
+    return NULL;
+  }
+
+  Py_RETURN_NONE;
+}
+
+static PyObject* PyUpb_MapContainer_Repr(PyObject* _self) {
+  PyUpb_MapContainer* self = (PyUpb_MapContainer*)_self;
+  upb_Map* map = PyUpb_MapContainer_GetIfReified(self);
+  PyObject* dict = PyDict_New();
+  if (map) {
+    const upb_FieldDef* f = PyUpb_MapContainer_GetField(self);
+    const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+    const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0);
+    const upb_FieldDef* val_f = upb_MessageDef_Field(entry_m, 1);
+    size_t iter = kUpb_Map_Begin;
+    while (upb_MapIterator_Next(map, &iter)) {
+      PyObject* key =
+          PyUpb_UpbToPy(upb_MapIterator_Key(map, iter), key_f, self->arena);
+      PyObject* val =
+          PyUpb_UpbToPy(upb_MapIterator_Value(map, iter), val_f, self->arena);
+      if (!key || !val) {
+        Py_XDECREF(key);
+        Py_XDECREF(val);
+        Py_DECREF(dict);
+        return NULL;
+      }
+      PyDict_SetItem(dict, key, val);
+      Py_DECREF(key);
+      Py_DECREF(val);
+    }
+  }
+  PyObject* repr = PyObject_Repr(dict);
+  Py_DECREF(dict);
+  return repr;
+}
+
+PyObject* PyUpb_MapContainer_GetOrCreateWrapper(upb_Map* map,
+                                                const upb_FieldDef* f,
+                                                PyObject* arena) {
+  PyUpb_MapContainer* ret = (void*)PyUpb_ObjCache_Get(map);
+  if (ret) return &ret->ob_base;
+
+  PyTypeObject* cls = PyUpb_MapContainer_GetClass(f);
+  ret = (void*)PyType_GenericAlloc(cls, 0);
+  ret->arena = arena;
+  ret->field = (uintptr_t)f;
+  ret->ptr.map = map;
+  ret->version = 0;
+  Py_INCREF(arena);
+  PyUpb_ObjCache_Add(map, &ret->ob_base);
+  return &ret->ob_base;
+}
+
+// -----------------------------------------------------------------------------
+// ScalarMapContainer
+// -----------------------------------------------------------------------------
+
+static PyMethodDef PyUpb_ScalarMapContainer_Methods[] = {
+    {"__contains__", PyUpb_MapContainer_Contains, METH_O,
+     "Tests whether a key is a member of the map."},
+    {"clear", PyUpb_MapContainer_Clear, METH_NOARGS,
+     "Removes all elements from the map."},
+    {"get", (PyCFunction)PyUpb_MapContainer_Get, METH_VARARGS | METH_KEYWORDS,
+     "Gets the value for the given key if present, or otherwise a default"},
+    {"GetEntryClass", PyUpb_MapContainer_GetEntryClass, METH_NOARGS,
+     "Return the class used to build Entries of (key, value) pairs."},
+    {"MergeFrom", PyUpb_MapContainer_MergeFrom, METH_O,
+     "Merges a map into the current map."},
+    /*
+   { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+     "Makes a deep copy of the class." },
+   { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+     "Outputs picklable representation of the repeated field." },
+   */
+    {NULL, NULL},
+};
+
+static PyType_Slot PyUpb_ScalarMapContainer_Slots[] = {
+    {Py_tp_dealloc, PyUpb_MapContainer_Dealloc},
+    {Py_mp_length, PyUpb_MapContainer_Length},
+    {Py_mp_subscript, PyUpb_MapContainer_Subscript},
+    {Py_mp_ass_subscript, PyUpb_MapContainer_AssignSubscript},
+    {Py_tp_methods, PyUpb_ScalarMapContainer_Methods},
+    {Py_tp_iter, PyUpb_MapIterator_New},
+    {Py_tp_repr, PyUpb_MapContainer_Repr},
+    {Py_tp_hash, PyObject_HashNotImplemented},
+    {0, NULL},
+};
+
+static PyType_Spec PyUpb_ScalarMapContainer_Spec = {
+    PYUPB_MODULE_NAME ".ScalarMapContainer",
+    sizeof(PyUpb_MapContainer),
+    0,
+    Py_TPFLAGS_DEFAULT,
+    PyUpb_ScalarMapContainer_Slots,
+};
+
+// -----------------------------------------------------------------------------
+// MessageMapContainer
+// -----------------------------------------------------------------------------
+
+static PyMethodDef PyUpb_MessageMapContainer_Methods[] = {
+    {"__contains__", PyUpb_MapContainer_Contains, METH_O,
+     "Tests whether the map contains this element."},
+    {"clear", PyUpb_MapContainer_Clear, METH_NOARGS,
+     "Removes all elements from the map."},
+    {"get", (PyCFunction)PyUpb_MapContainer_Get, METH_VARARGS | METH_KEYWORDS,
+     "Gets the value for the given key if present, or otherwise a default"},
+    {"get_or_create", PyUpb_MapContainer_Subscript, METH_O,
+     "Alias for getitem, useful to make explicit that the map is mutated."},
+    {"GetEntryClass", PyUpb_MapContainer_GetEntryClass, METH_NOARGS,
+     "Return the class used to build Entries of (key, value) pairs."},
+    {"MergeFrom", PyUpb_MapContainer_MergeFrom, METH_O,
+     "Merges a map into the current map."},
+    /*
+   { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS,
+     "Makes a deep copy of the class." },
+   { "__reduce__", (PyCFunction)Reduce, METH_NOARGS,
+     "Outputs picklable representation of the repeated field." },
+   */
+    {NULL, NULL},
+};
+
+static PyType_Slot PyUpb_MessageMapContainer_Slots[] = {
+    {Py_tp_dealloc, PyUpb_MapContainer_Dealloc},
+    {Py_mp_length, PyUpb_MapContainer_Length},
+    {Py_mp_subscript, PyUpb_MapContainer_Subscript},
+    {Py_mp_ass_subscript, PyUpb_MapContainer_AssignSubscript},
+    {Py_tp_methods, PyUpb_MessageMapContainer_Methods},
+    {Py_tp_iter, PyUpb_MapIterator_New},
+    {Py_tp_repr, PyUpb_MapContainer_Repr},
+    {Py_tp_hash, PyObject_HashNotImplemented},
+    {0, NULL}};
+
+static PyType_Spec PyUpb_MessageMapContainer_Spec = {
+    PYUPB_MODULE_NAME ".MessageMapContainer", sizeof(PyUpb_MapContainer), 0,
+    Py_TPFLAGS_DEFAULT, PyUpb_MessageMapContainer_Slots};
+
+// -----------------------------------------------------------------------------
+// MapIterator
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  PyObject_HEAD;
+  PyUpb_MapContainer* map;  // We own a reference.
+  size_t iter;
+  int version;
+} PyUpb_MapIterator;
+
+static PyObject* PyUpb_MapIterator_New(PyUpb_MapContainer* map) {
+  PyUpb_ModuleState* state = PyUpb_ModuleState_Get();
+  PyUpb_MapIterator* iter =
+      (void*)PyType_GenericAlloc(state->map_iterator_type, 0);
+  iter->map = map;
+  iter->iter = kUpb_Map_Begin;
+  iter->version = map->version;
+  Py_INCREF(map);
+  return &iter->ob_base;
+}
+
+static void PyUpb_MapIterator_Dealloc(void* _self) {
+  PyUpb_MapIterator* self = (PyUpb_MapIterator*)_self;
+  Py_DECREF(&self->map->ob_base);
+  PyUpb_Dealloc(_self);
+}
+
+PyObject* PyUpb_MapIterator_IterNext(PyObject* _self) {
+  PyUpb_MapIterator* self = (PyUpb_MapIterator*)_self;
+  if (self->version != self->map->version) {
+    return PyErr_Format(PyExc_RuntimeError, "Map modified during iteration.");
+  }
+  upb_Map* map = PyUpb_MapContainer_GetIfReified(self->map);
+  if (!map) return NULL;
+  if (!upb_MapIterator_Next(map, &self->iter)) return NULL;
+  upb_MessageValue key = upb_MapIterator_Key(map, self->iter);
+  const upb_FieldDef* f = PyUpb_MapContainer_GetField(self->map);
+  const upb_MessageDef* entry_m = upb_FieldDef_MessageSubDef(f);
+  const upb_FieldDef* key_f = upb_MessageDef_Field(entry_m, 0);
+  return PyUpb_UpbToPy(key, key_f, self->map->arena);
+}
+
+static PyType_Slot PyUpb_MapIterator_Slots[] = {
+    {Py_tp_dealloc, PyUpb_MapIterator_Dealloc},
+    {Py_tp_iter, PyObject_SelfIter},
+    {Py_tp_iternext, PyUpb_MapIterator_IterNext},
+    {0, NULL}};
+
+static PyType_Spec PyUpb_MapIterator_Spec = {
+    PYUPB_MODULE_NAME ".MapIterator", sizeof(PyUpb_MapIterator), 0,
+    Py_TPFLAGS_DEFAULT, PyUpb_MapIterator_Slots};
+
+// -----------------------------------------------------------------------------
+// Top Level
+// -----------------------------------------------------------------------------
+
+static PyObject* GetMutableMappingBase(void) {
+  PyObject* collections = NULL;
+  PyObject* mapping = NULL;
+  PyObject* bases = NULL;
+  if ((collections = PyImport_ImportModule("collections.abc")) &&
+      (mapping = PyObject_GetAttrString(collections, "MutableMapping"))) {
+    bases = Py_BuildValue("(O)", mapping);
+  }
+  Py_XDECREF(collections);
+  Py_XDECREF(mapping);
+  return bases;
+}
+
+bool PyUpb_Map_Init(PyObject* m) {
+  PyUpb_ModuleState* state = PyUpb_ModuleState_GetFromModule(m);
+  PyObject* bases = GetMutableMappingBase();
+  if (!bases) return false;
+
+  state->message_map_container_type =
+      PyUpb_AddClassWithBases(m, &PyUpb_MessageMapContainer_Spec, bases);
+  state->scalar_map_container_type =
+      PyUpb_AddClassWithBases(m, &PyUpb_ScalarMapContainer_Spec, bases);
+  state->map_iterator_type = PyUpb_AddClass(m, &PyUpb_MapIterator_Spec);
+
+  Py_DECREF(bases);
+
+  return state->message_map_container_type &&
+         state->scalar_map_container_type && state->map_iterator_type;
+}

python/map.h

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYUPB_MAP_H__
+#define PYUPB_MAP_H__
+
+#include <stdbool.h>
+
+#include "python/python.h"
+#include "upb/def.h"
+
+// Creates a new repeated field stub for field `f` of message object `parent`.
+// Precondition: `parent` must be a stub.
+PyObject* PyUpb_MapContainer_NewStub(PyObject* parent, const upb_FieldDef* f,
+                                     PyObject* arena);
+
+// Returns a map object wrapping `map`, of field type `f`, which must be on
+// `arena`.  If an existing wrapper object exists, it will be returned,
+// otherwise a new object will be created.  The caller always owns a ref on the
+// returned value.
+PyObject* PyUpb_MapContainer_GetOrCreateWrapper(upb_Map* map,
+                                                const upb_FieldDef* f,
+                                                PyObject* arena);
+
+// Reifies a map stub to point to the concrete data in `map`.
+// If `map` is NULL, an appropriate empty map will be constructed.
+void PyUpb_MapContainer_Reify(PyObject* self, upb_Map* map);
+
+// Reifies this map object if it is not already reified.
+upb_Map* PyUpb_MapContainer_EnsureReified(PyObject* self);
+
+// Assigns `self[key] = val` for the map `self`.
+int PyUpb_MapContainer_AssignSubscript(PyObject* self, PyObject* key,
+                                       PyObject* val);
+
+// Invalidates any existing iterators for the map `obj`.
+void PyUpb_MapContainer_Invalidate(PyObject* obj);
+
+// Module-level init.
+bool PyUpb_Map_Init(PyObject* m);
+
+#endif  // PYUPB_MAP_H__

python/message.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2009-2021, Google LLC
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Google LLC nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef PYPB_MESSAGE_H__
+#define PYPB_MESSAGE_H__
+
+#include <stdbool.h>
+
+#include "python/protobuf.h"
+#include "upb/reflection.h"
+
+// Removes the wrapper object for this field from the unset subobject cache.
+void PyUpb_CMessage_CacheDelete(PyObject* _self, const upb_FieldDef* f);
+
+// Sets the field value for `f` to `subobj`, evicting the wrapper object from
+// the "unset subobject" cache now that real data exists for it.  The caller
+// must also update the wrapper associated with `f` to point to `subobj` also.
+void PyUpb_CMessage_SetConcreteSubobj(PyObject* _self, const upb_FieldDef* f,
+                                      upb_MessageValue subobj);
+
+// Gets a Python wrapper object for message `u_msg` of type `m`, returning a
+// cached wrapper if one was previously created.  If a new object is created,
+// it will reference `arena`, which must own `u_msg`.
+PyObject* PyUpb_CMessage_Get(upb_Message* u_msg, const upb_MessageDef* m,
+                             PyObject* arena);
+
+// Verifies that a Python object is a message.  Sets a TypeError exception and
+// returns false on failure.
+bool PyUpb_CMessage_Verify(PyObject* self);
+
+// Gets the upb_Message* for this message object if the message is reified.
+// Otherwise returns NULL.
+upb_Message* PyUpb_CMessage_GetIfReified(PyObject* _self);
+
+// Returns the `upb_MessageDef` for a given CMessage.
+const upb_MessageDef* PyUpb_CMessage_GetMsgdef(PyObject* self);
+
+// Functions that match the corresponding methods on the message object.
+PyObject* PyUpb_CMessage_MergeFrom(PyObject* self, PyObject* arg);
+PyObject* PyUpb_CMessage_MergeFromString(PyObject* self, PyObject* arg);
+PyObject* PyUpb_CMessage_SerializeToString(PyObject* self, PyObject* args,
+                                           PyObject* kwargs);
+
+// Sets fields of the message according to the attribuges in `kwargs`.
+int PyUpb_CMessage_InitAttributes(PyObject* _self, PyObject* args,
+                                  PyObject* kwargs);
+
+// Checks that `key` is a field descriptor for an extension type, and that the
+// extendee is this message.  Otherwise returns NULL and sets a KeyError.
+const upb_FieldDef* PyUpb_CMessage_GetExtensionDef(PyObject* _self,
+                                                   PyObject* key);
+
+// Clears the given field in this message.
+void PyUpb_CMessage_DoClearField(PyObject* _self, const upb_FieldDef* f);
+
+// Clears the ExtensionDict from the message.  The message must have an
+// ExtensionDict set.
+void PyUpb_CMessage_ClearExtensionDict(PyObject* _self);
+
+// Implements the equivalent of getattr(msg, field), once `field` has
+// already been resolved to a `upb_FieldDef*`.
+PyObject* PyUpb_CMessage_GetFieldValue(PyObject* _self,
+                                       const upb_FieldDef* field);
+
+// Implements the equivalent of setattr(msg, field, value), once `field` has
+// already been resolved to a `upb_FieldDef*`.
+int PyUpb_CMessage_SetFieldValue(PyObject* _self, const upb_FieldDef* field,
+                                 PyObject* value, PyObject* exc);
+
+// Returns the version associated with this message.  The version will be
+// incremented when the message changes.
+int PyUpb_CMessage_GetVersion(PyObject* _self);
+
+// Module-level init.
+bool PyUpb_InitMessage(PyObject* m);
+
+#endif  // PYPB_MESSAGE_H__

python/minimal_test.py

@@ -0,0 +1,183 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+"""A bare-bones unit test that doesn't load any generated code."""
+
+
+import unittest
+from google.protobuf.pyext import _message
+from google.protobuf.internal import api_implementation
+from google.protobuf import unittest_pb2
+from google.protobuf import map_unittest_pb2
+from google.protobuf import descriptor_pool
+from google.protobuf import text_format
+from google.protobuf import message_factory
+from google.protobuf import message
+from google.protobuf.internal import factory_test1_pb2
+from google.protobuf.internal import factory_test2_pb2
+from google.protobuf.internal import more_extensions_pb2
+from google.protobuf import descriptor_pb2
+
+class TestMessageExtension(unittest.TestCase):
+
+    def test_descriptor_pool(self):
+        serialized_desc = b'\n\ntest.proto\"\x0e\n\x02M1*\x08\x08\x01\x10\x80\x80\x80\x80\x02:\x15\n\x08test_ext\x12\x03.M1\x18\x01 \x01(\x05'
+        pool = _message.DescriptorPool()
+        file_desc = pool.AddSerializedFile(serialized_desc)
+        self.assertEqual("test.proto", file_desc.name)
+        ext_desc = pool.FindExtensionByName("test_ext")
+        self.assertEqual(1, ext_desc.number)
+
+        # Test object cache: repeatedly retrieving the same descriptor
+        # should result in the same object
+        self.assertIs(ext_desc, pool.FindExtensionByName("test_ext"))
+
+
+    def test_lib_is_upb(self):
+        # Ensure we are not pulling in a different protobuf library on the
+        # system.
+        print(_message._IS_UPB)
+        self.assertTrue(_message._IS_UPB)
+        self.assertEqual(api_implementation.Type(), "cpp")
+
+    def test_repeated_field_slice_delete(self):
+        def test_slice(start, end, step):
+            vals = list(range(20))
+            message = unittest_pb2.TestAllTypes(repeated_int32=vals)
+            del vals[start:end:step]
+            del message.repeated_int32[start:end:step]
+            self.assertEqual(vals, list(message.repeated_int32))
+        test_slice(3, 11, 1)
+        test_slice(3, 11, 2)
+        test_slice(3, 11, 3)
+        test_slice(11, 3, -1)
+        test_slice(11, 3, -2)
+        test_slice(11, 3, -3)
+        test_slice(10, 25, 4)
+    
+    def testExtensionsErrors(self):
+        msg = unittest_pb2.TestAllTypes()
+        self.assertRaises(AttributeError, getattr, msg, 'Extensions')
+    
+    def testClearStubMapField(self):
+        msg = map_unittest_pb2.TestMapSubmessage()
+        int32_map = msg.test_map.map_int32_int32
+        msg.test_map.ClearField("map_int32_int32")
+        int32_map[123] = 456
+        self.assertEqual(0, msg.test_map.ByteSize())
+
+    def testClearReifiedMapField(self):
+        msg = map_unittest_pb2.TestMap()
+        int32_map = msg.map_int32_int32
+        int32_map[123] = 456
+        msg.ClearField("map_int32_int32")
+        int32_map[111] = 222
+        self.assertEqual(0, msg.ByteSize())
+
+    def testClearStubRepeatedField(self):
+        msg = unittest_pb2.NestedTestAllTypes()
+        int32_array = msg.payload.repeated_int32
+        msg.payload.ClearField("repeated_int32")
+        int32_array.append(123)
+        self.assertEqual(0, msg.payload.ByteSize())
+
+    def testClearReifiedRepeatdField(self):
+        msg = unittest_pb2.TestAllTypes()
+        int32_array = msg.repeated_int32
+        int32_array.append(123)
+        self.assertNotEqual(0, msg.ByteSize())
+        msg.ClearField("repeated_int32")
+        int32_array.append(123)
+        self.assertEqual(0, msg.ByteSize())
+
+    def testFloatPrinting(self):
+        message = unittest_pb2.TestAllTypes()
+        message.optional_float = -0.0
+        self.assertEqual(str(message), 'optional_float: -0\n')
+
+class OversizeProtosTest(unittest.TestCase):
+  def setUp(self):
+    msg = unittest_pb2.NestedTestAllTypes()
+    m = msg
+    for i in range(101):
+      m = m.child
+    m.Clear()
+    self.p_serialized = msg.SerializeToString()
+    
+  def testAssertOversizeProto(self):
+    from google.protobuf.pyext._message import SetAllowOversizeProtos
+    SetAllowOversizeProtos(False)
+    q = unittest_pb2.NestedTestAllTypes()
+    with self.assertRaises(message.DecodeError):
+      q.ParseFromString(self.p_serialized)
+      print(q)
+  
+  def testSucceedOversizeProto(self):
+    from google.protobuf.pyext._message import SetAllowOversizeProtos
+    SetAllowOversizeProtos(True)
+    q = unittest_pb2.NestedTestAllTypes()
+    q.ParseFromString(self.p_serialized)
+
+  def testExtensionIter(self):
+    extendee_proto = more_extensions_pb2.ExtendedMessage()
+
+    extension_int32 = more_extensions_pb2.optional_int_extension
+    extendee_proto.Extensions[extension_int32] = 23
+
+    extension_repeated = more_extensions_pb2.repeated_int_extension
+    extendee_proto.Extensions[extension_repeated].append(11)
+
+    extension_msg = more_extensions_pb2.optional_message_extension
+    extendee_proto.Extensions[extension_msg].foreign_message_int = 56
+
+    # Set some normal fields.
+    extendee_proto.optional_int32 = 1
+    extendee_proto.repeated_string.append('hi')
+
+    expected = {
+        extension_int32: True,
+        extension_msg: True,
+        extension_repeated: True
+    }
+    count = 0
+    for item in extendee_proto.Extensions:
+      del expected[item]
+      self.assertIn(item, extendee_proto.Extensions)
+      count += 1
+    self.assertEqual(count, 3)
+    self.assertEqual(len(expected), 0)
+  
+  def testIsInitializedStub(self):
+    proto = unittest_pb2.TestRequiredForeign()
+    self.assertTrue(proto.IsInitialized())
+    self.assertFalse(proto.optional_message.IsInitialized())
+    errors = []
+    self.assertFalse(proto.optional_message.IsInitialized(errors))
+    self.assertEqual(['a', 'b', 'c'], errors)
+    self.assertRaises(message.EncodeError, proto.optional_message.SerializeToString)
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)

python/pb_unit_tests/BUILD

@@ -0,0 +1,62 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+load("//bazel:pyproto_test_wrapper.bzl", "pyproto_test_wrapper")
+
+licenses(["notice"])
+
+pyproto_test_wrapper(name = "descriptor_database_test")
+
+pyproto_test_wrapper(name = "descriptor_pool_test")
+
+pyproto_test_wrapper(name = "descriptor_test")
+
+pyproto_test_wrapper(name = "generator_test")  # copybara:strip_for_google3
+
+pyproto_test_wrapper(name = "json_format_test")
+
+pyproto_test_wrapper(name = "keywords_test")
+
+pyproto_test_wrapper(name = "message_factory_test")
+
+pyproto_test_wrapper(name = "message_test")
+
+pyproto_test_wrapper(name = "proto_builder_test")
+
+pyproto_test_wrapper(name = "reflection_test")
+
+pyproto_test_wrapper(name = "service_reflection_test")
+
+pyproto_test_wrapper(name = "symbol_database_test")
+
+pyproto_test_wrapper(name = "text_encoding_test")
+
+pyproto_test_wrapper(name = "text_format_test")
+
+pyproto_test_wrapper(name = "unknown_fields_test")
+
+pyproto_test_wrapper(name = "well_known_types_test")
+
+pyproto_test_wrapper(name = "wire_format_test")

python/pb_unit_tests/README.md

@@ -0,0 +1,11 @@
+
+# Protobuf Unit Tests
+
+This directory contains wrappers around the Python unit tests defined in
+the protobuf repo.  Python+upb is intended to be a drop-in replacement for
+protobuf Python, so we should be able to pass the same set of unit tests.
+
+Our wrappers contain exclusion lists for tests we know we are not currently
+passing.  Ideally these exclusion lists will become empty once Python+upb is
+fully implemented.  However there may be a few edge cases that we decide
+are not worth matching with perfect parity.

python/pb_unit_tests/descriptor_database_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import descriptor_database_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=descriptor_database_test, verbosity=2)

python/pb_unit_tests/descriptor_pool_test_wrapper.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import descriptor_pool_test
+import unittest
+import copy
+
+# copybara:insert_for_google3_begin
+# from google3.testing.pybase import googletest
+# copybara:insert_end
+
+# This is testing that certain methods unconditionally throw TypeError.
+# In the new extension we simply don't define them at all.
+descriptor_pool_test.AddDescriptorTest.testAddTypeError.__unittest_expecting_failure__ = True
+
+descriptor_pool_test.SecondaryDescriptorFromDescriptorDB.testErrorCollector.__unittest_expecting_failure__ = True
+
+if __name__ == '__main__':
+  # copybara:strip_for_google3_begin
+  unittest.main(module=descriptor_pool_test, verbosity=2)
+  # copybara:replace_for_google3_begin
+  # googletest.main()
+  # copybara:replace_for_google3_end

python/pb_unit_tests/descriptor_test_wrapper.py

@@ -0,0 +1,46 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import descriptor_test
+import unittest
+
+# Our behavior here matches pure-Python, which does not allow
+# foo.enum_values_by_name.get([]).  We reject it because we return a true
+# dict (like pure Python), which does not allow hashing by a list.
+descriptor_test.GeneratedDescriptorTest.testDescriptor.__unittest_expecting_failure__ = True
+
+# These fail because they attempt to add fields with conflicting JSON names.
+# We don't want to support this going forward.
+descriptor_test.MakeDescriptorTest.testCamelcaseName.__unittest_expecting_failure__ = True
+descriptor_test.MakeDescriptorTest.testJsonName.__unittest_expecting_failure__ = True
+
+# We pass this test, but the error message is slightly different.
+# Our error message is better.
+descriptor_test.NewDescriptorTest.testImmutableCppDescriptor.__unittest_expecting_failure__ = True
+
+descriptor_test.DescriptorTest.testGetDebugString.__unittest_expecting_failure__ = True
+
+if __name__ == '__main__':
+  unittest.main(module=descriptor_test, verbosity=2)

python/pb_unit_tests/generator_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import generator_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=generator_test, verbosity=2)

python/pb_unit_tests/json_format_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import json_format_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=json_format_test, verbosity=2)

python/pb_unit_tests/keywords_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import keywords_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=keywords_test, verbosity=2)

python/pb_unit_tests/message_factory_test_wrapper.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import message_factory_test
+import unittest
+
+message_factory_test.MessageFactoryTest.testCreatePrototypeOverride.__unittest_expecting_failure__ = True
+message_factory_test.MessageFactoryTest.testDuplicateExtensionNumber.__unittest_expecting_failure__ = True
+message_factory_test.MessageFactoryTest.testGetMessages.__unittest_expecting_failure__ = True
+message_factory_test.MessageFactoryTest.testGetPrototype.__unittest_expecting_failure__ = True
+
+if __name__ == '__main__':
+  unittest.main(module=message_factory_test, verbosity=2)

python/pb_unit_tests/message_test_wrapper.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import message_test
+import unittest
+
+# We don't want to support extending repeated fields with nothing; this behavior
+# is marked for deprecation in the existing library.
+message_test.MessageTest.testExtendFloatWithNothing_proto2.__unittest_expecting_failure__ = True
+message_test.MessageTest.testExtendFloatWithNothing_proto3.__unittest_expecting_failure__ = True
+message_test.MessageTest.testExtendInt32WithNothing_proto2.__unittest_expecting_failure__ = True
+message_test.MessageTest.testExtendInt32WithNothing_proto3.__unittest_expecting_failure__ = True
+message_test.MessageTest.testExtendStringWithNothing_proto2.__unittest_expecting_failure__ = True
+message_test.MessageTest.testExtendStringWithNothing_proto3.__unittest_expecting_failure__ = True
+
+# Python/C++ customizes the C++ TextFormat to always print trailing ".0" for
+# floats. upb doesn't do this, it matches C++ TextFormat.
+message_test.MessageTest.testFloatPrinting_proto2.__unittest_expecting_failure__ = True
+message_test.MessageTest.testFloatPrinting_proto3.__unittest_expecting_failure__ = True
+
+# For these tests we are throwing the correct error, only the text of the error
+# message is a mismatch.  For technical reasons around the limited API, matching
+# the existing error message exactly is not feasible.
+message_test.Proto3Test.testCopyFromBadType.__unittest_expecting_failure__ = True
+message_test.Proto3Test.testMergeFromBadType.__unittest_expecting_failure__ = True
+
+message_test.MessageTest.testPickleRepeatedScalarContainer_proto2.__unittest_expecting_failure__ = True
+message_test.MessageTest.testPickleRepeatedScalarContainer_proto3.__unittest_expecting_failure__ = True
+message_test.Proto2Test.testPythonicInit.__unittest_expecting_failure__ = True
+message_test.Proto2Test.test_documentation.__unittest_expecting_failure__ = True
+message_test.Proto3Test.testModifyMapEntryWhileIterating.__unittest_expecting_failure__ = True
+message_test.Proto3Test.testProto3Optional.__unittest_expecting_failure__ = True
+
+if __name__ == '__main__':
+  unittest.main(module=message_test, verbosity=2)

python/pb_unit_tests/proto_builder_test_wrapper.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import proto_builder_test
+import unittest
+
+proto_builder_test.ProtoBuilderTest.testMakeLargeProtoClass.__unittest_expecting_failure__ = True
+
+if __name__ == '__main__':
+  unittest.main(module=proto_builder_test, verbosity=2)

python/pb_unit_tests/reflection_test_wrapper.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import reflection_test
+import unittest
+
+# These tests depend on a specific iteration order for extensions, which is not
+# reasonable to guarantee.
+reflection_test.Proto2ReflectionTest.testExtensionIter.__unittest_expecting_failure__ = True
+
+# These tests depend on a specific serialization order for extensions, which is
+# not reasonable to guarantee.
+reflection_test.SerializationTest.testCanonicalSerializationOrder.__unittest_expecting_failure__ = True
+reflection_test.SerializationTest.testCanonicalSerializationOrderSameAsCpp.__unittest_expecting_failure__ = True
+
+# This test relies on the internal implementation using Python descriptors.
+# This is an implementation detail that users should not depend on.
+reflection_test.SerializationTest.testFieldDataDescriptor.__unittest_expecting_failure__ = True
+
+reflection_test.ReflectionTest.testDeepCopy_proto2.__unittest_expecting_failure__ = True
+reflection_test.ReflectionTest.testDeepCopy_proto3.__unittest_expecting_failure__ = True
+reflection_test.SerializationTest.testFieldProperties.__unittest_expecting_failure__ = True
+
+if __name__ == '__main__':
+  unittest.main(module=reflection_test, verbosity=2)

python/pb_unit_tests/service_reflection_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import service_reflection_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=service_reflection_test, verbosity=2)

python/pb_unit_tests/symbol_database_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import symbol_database_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=symbol_database_test, verbosity=2)

python/pb_unit_tests/text_encoding_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import text_encoding_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=text_encoding_test, verbosity=2)

python/pb_unit_tests/text_format_test_wrapper.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import text_format_test
+import unittest
+
+# These rely on the UnknownFields accessor, which we are trying to deprecate.
+text_format_test.OnlyWorksWithProto2RightNowTests.testPrintUnknownFields.__unittest_expecting_failure__ = True
+
+# copybara:strip_for_google3_begin
+from google.protobuf.internal import _parameterized  # copybara:strip_for_google3
+sep = _parameterized._SEPARATOR
+getattr(text_format_test.TextFormatMessageToStringTests, "testPrintUnknownFieldsEmbeddedMessageInBytes" + sep + "0").__unittest_expecting_failure__ = True
+getattr(text_format_test.TextFormatMessageToStringTests, "testPrintUnknownFieldsEmbeddedMessageInBytes" + sep + "1").__unittest_expecting_failure__ = True
+# copybara:strip_end
+
+if __name__ == '__main__':
+  unittest.main(module=text_format_test, verbosity=2)

python/pb_unit_tests/unknown_fields_test_wrapper.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import unknown_fields_test
+import unittest
+
+unknown_fields_test.UnknownEnumValuesTest.testCheckUnknownFieldValueForEnum.__unittest_expecting_failure__ = True
+unknown_fields_test.UnknownFieldsAccessorsTest.testCheckUnknownFieldValue.__unittest_expecting_failure__ = True
+unknown_fields_test.UnknownFieldsAccessorsTest.testClear.__unittest_expecting_failure__ = True
+unknown_fields_test.UnknownFieldsAccessorsTest.testMergeFrom.__unittest_expecting_failure__ = True
+unknown_fields_test.UnknownFieldsAccessorsTest.testSubUnknownFields.__unittest_expecting_failure__ = True
+unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownExtensions.__unittest_expecting_failure__ = True
+unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownField.__unittest_expecting_failure__ = True
+unknown_fields_test.UnknownFieldsAccessorsTest.testUnknownFieldsNoMemoryLeak.__unittest_expecting_failure__ = True
+unknown_fields_test.UnknownFieldsTest.testSerializeMessageSetWireFormatUnknownExtension.__unittest_expecting_failure__ = True
+
+if __name__ == '__main__':
+  unittest.main(module=unknown_fields_test, verbosity=2)

python/pb_unit_tests/well_known_types_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import well_known_types_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=well_known_types_test, verbosity=2)

python/pb_unit_tests/wire_format_test_wrapper.py

@@ -0,0 +1,30 @@
+# Copyright (c) 2009-2021, Google LLC
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Google LLC nor the
+#       names of its contributors may be used to endorse or promote products
+#       derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from google.protobuf.internal import wire_format_test
+import unittest
+
+if __name__ == '__main__':
+  unittest.main(module=wire_format_test, verbosity=2)