diff --git a/bazel/pyproto_test_wrapper.bzl b/bazel/pyproto_test_wrapper.bzl new file mode 100644 index 0000000000..fb909a1db8 --- /dev/null +++ b/bazel/pyproto_test_wrapper.bzl @@ -0,0 +1,34 @@ + +# copybara:strip_for_google3_begin + +def pyproto_test_wrapper(name): + src = name + "_wrapper.py" + native.py_test( + name = name, + srcs = [src], + legacy_create_init = False, + main = src, + data = ["@com_google_protobuf//:testdata"], + deps = [ + "//python:message_ext", + "@com_google_protobuf//:python_common_test_protos", + "@com_google_protobuf//:python_specific_test_protos", + "@com_google_protobuf//:python_srcs", + ], + ) + +# copybara:replace_for_google3_begin +# +# def pyproto_test_wrapper(name): +# src = name + "_wrapper.py" +# native.py_test( +# name = name, +# srcs = [src], +# main = src, +# deps = [ +# "//net/proto2/python/internal:" + name + "_for_deps", +# "//net/proto2/python/public:use_upb_protos", +# ], +# ) +# +# copybara:replace_for_google3_end diff --git a/python/descriptor.c b/python/descriptor.c index 71712b2de8..543c9d7a8c 100644 --- a/python/descriptor.c +++ b/python/descriptor.c @@ -107,7 +107,7 @@ static PyObject* PyUpb_DescriptorBase_GetOptions(PyUpb_DescriptorBase* self, if (!self->options) { // Load descriptors protos if they are not loaded already. We have to do // this lazily, otherwise, it would lead to circular imports. - PyObject* mod = PyImport_ImportModule("google.protobuf.descriptor_pb2"); + PyObject* mod = PyImport_ImportModule(PYUPB_DESCRIPTOR_MODULE); Py_DECREF(mod); // Find the correct options message. @@ -323,7 +323,7 @@ static PyObject* PyUpb_Descriptor_GetOptions(PyObject* _self, PyObject* args) { return PyUpb_DescriptorBase_GetOptions( self, upb_MessageDef_Options(self->def), &google_protobuf_MessageOptions_msginit, - "google.protobuf.MessageOptions"); + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".MessageOptions"); } static PyObject* PyUpb_Descriptor_CopyToProto(PyObject* _self, @@ -331,7 +331,7 @@ static PyObject* PyUpb_Descriptor_CopyToProto(PyObject* _self, return PyUpb_DescriptorBase_CopyToProto( _self, (PyUpb_ToProto_Func*)&upb_MessageDef_ToProto, &google_protobuf_DescriptorProto_msginit, - "google.protobuf.DescriptorProto", py_proto); + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".DescriptorProto", py_proto); } static PyObject* PyUpb_Descriptor_EnumValueName(PyObject* _self, @@ -733,7 +733,8 @@ static PyObject* PyUpb_EnumDescriptor_GetOptions(PyObject* _self, PyUpb_DescriptorBase* self = (void*)_self; return PyUpb_DescriptorBase_GetOptions(self, upb_EnumDef_Options(self->def), &google_protobuf_EnumOptions_msginit, - "google.protobuf.EnumOptions"); + PYUPB_DESCRIPTOR_PROTO_PACKAGE + ".EnumOptions"); } static PyObject* PyUpb_EnumDescriptor_CopyToProto(PyObject* _self, @@ -741,7 +742,7 @@ static PyObject* PyUpb_EnumDescriptor_CopyToProto(PyObject* _self, return PyUpb_DescriptorBase_CopyToProto( _self, (PyUpb_ToProto_Func*)&upb_EnumDef_ToProto, &google_protobuf_EnumDescriptorProto_msginit, - "google.protobuf.EnumDescriptorProto", py_proto); + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".EnumDescriptorProto", py_proto); } static PyGetSetDef PyUpb_EnumDescriptor_Getters[] = { @@ -822,7 +823,7 @@ static PyObject* PyUpb_EnumValueDescriptor_GetOptions(PyObject* _self, return PyUpb_DescriptorBase_GetOptions( self, upb_EnumValueDef_Options(self->def), &google_protobuf_EnumValueOptions_msginit, - "google.protobuf.EnumValueOptions"); + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".EnumValueOptions"); } static PyGetSetDef PyUpb_EnumValueDescriptor_Getters[] = { @@ -1015,7 +1016,8 @@ static PyObject* PyUpb_FieldDescriptor_GetOptions(PyObject* _self, PyUpb_DescriptorBase* self = (void*)_self; return PyUpb_DescriptorBase_GetOptions(self, upb_FieldDef_Options(self->def), &google_protobuf_FieldOptions_msginit, - "google.protobuf.FieldOptions"); + PYUPB_DESCRIPTOR_PROTO_PACKAGE + ".FieldOptions"); } static PyGetSetDef PyUpb_FieldDescriptor_Getters[] = { @@ -1254,7 +1256,8 @@ static PyObject* PyUpb_FileDescriptor_GetOptions(PyObject* _self, PyUpb_DescriptorBase* self = (void*)_self; return PyUpb_DescriptorBase_GetOptions(self, upb_FileDef_Options(self->def), &google_protobuf_FileOptions_msginit, - "google.protobuf.FileOptions"); + PYUPB_DESCRIPTOR_PROTO_PACKAGE + ".FileOptions"); } static PyObject* PyUpb_FileDescriptor_CopyToProto(PyObject* _self, @@ -1262,7 +1265,7 @@ static PyObject* PyUpb_FileDescriptor_CopyToProto(PyObject* _self, return PyUpb_DescriptorBase_CopyToProto( _self, (PyUpb_ToProto_Func*)&upb_FileDef_ToProto, &google_protobuf_FileDescriptorProto_msginit, - "google.protobuf.FileDescriptorProto", py_proto); + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".FileDescriptorProto", py_proto); } static PyGetSetDef PyUpb_FileDescriptor_Getters[] = { @@ -1367,7 +1370,8 @@ static PyObject* PyUpb_MethodDescriptor_GetOptions(PyObject* _self, PyUpb_DescriptorBase* self = (void*)_self; return PyUpb_DescriptorBase_GetOptions(self, upb_MethodDef_Options(self->def), &google_protobuf_MethodOptions_msginit, - "google.protobuf.MethodOptions"); + PYUPB_DESCRIPTOR_PROTO_PACKAGE + ".MethodOptions"); } static PyObject* PyUpb_MethodDescriptor_CopyToProto(PyObject* _self, @@ -1375,7 +1379,7 @@ static PyObject* PyUpb_MethodDescriptor_CopyToProto(PyObject* _self, return PyUpb_DescriptorBase_CopyToProto( _self, (PyUpb_ToProto_Func*)&upb_MethodDef_ToProto, &google_protobuf_MethodDescriptorProto_msginit, - "google.protobuf.MethodDescriptorProto", py_proto); + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".MethodDescriptorProto", py_proto); } static PyGetSetDef PyUpb_MethodDescriptor_Getters[] = { @@ -1471,7 +1475,8 @@ static PyObject* PyUpb_OneofDescriptor_GetOptions(PyObject* _self, PyUpb_DescriptorBase* self = (void*)_self; return PyUpb_DescriptorBase_GetOptions(self, upb_OneofDef_Options(self->def), &google_protobuf_OneofOptions_msginit, - "google.protobuf.OneofOptions"); + PYUPB_DESCRIPTOR_PROTO_PACKAGE + ".OneofOptions"); } static PyGetSetDef PyUpb_OneofDescriptor_Getters[] = { @@ -1572,7 +1577,7 @@ static PyObject* PyUpb_ServiceDescriptor_GetOptions(PyObject* _self, return PyUpb_DescriptorBase_GetOptions( self, upb_ServiceDef_Options(self->def), &google_protobuf_ServiceOptions_msginit, - "google.protobuf.ServiceOptions"); + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".ServiceOptions"); } static PyObject* PyUpb_ServiceDescriptor_CopyToProto(PyObject* _self, @@ -1580,7 +1585,7 @@ static PyObject* PyUpb_ServiceDescriptor_CopyToProto(PyObject* _self, return PyUpb_DescriptorBase_CopyToProto( _self, (PyUpb_ToProto_Func*)&upb_ServiceDef_ToProto, &google_protobuf_ServiceDescriptorProto_msginit, - "google.protobuf.ServiceDescriptorProto", py_proto); + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".ServiceDescriptorProto", py_proto); } static PyObject* PyUpb_ServiceDescriptor_FindMethodByName(PyObject* _self, diff --git a/python/descriptor_pool.c b/python/descriptor_pool.c index 06011d702e..5a61b4e6c7 100644 --- a/python/descriptor_pool.c +++ b/python/descriptor_pool.c @@ -251,7 +251,8 @@ static PyObject* PyUpb_DescriptorPool_DoAdd(PyObject* _self, PyObject* file_desc) { if (!PyUpb_CMessage_Verify(file_desc)) return NULL; const upb_MessageDef* m = PyUpb_CMessage_GetMsgdef(file_desc); - const char* file_proto_name = "google.protobuf.FileDescriptorProto"; + const char* file_proto_name = + PYUPB_DESCRIPTOR_PROTO_PACKAGE ".FileDescriptorProto"; if (strcmp(upb_MessageDef_FullName(m), file_proto_name) != 0) { return PyErr_Format(PyExc_TypeError, "Can only add FileDescriptorProto"); } diff --git a/python/message.c b/python/message.c index 095f56d720..384c5b31d1 100644 --- a/python/message.c +++ b/python/message.c @@ -1833,7 +1833,8 @@ bool PyUpb_InitMessage(PyObject* m) { state->listfields_item_key = PyObject_GetAttrString( (PyObject*)state->cmessage_type, "_ListFieldsItemKey"); - PyObject* mod = PyImport_ImportModule("google.protobuf.message"); + PyObject* mod = + PyImport_ImportModule(PYUPB_PROTOBUF_PUBLIC_PACKAGE ".message"); if (mod == NULL) return false; state->encode_error_class = PyObject_GetAttrString(mod, "EncodeError"); @@ -1841,8 +1842,8 @@ bool PyUpb_InitMessage(PyObject* m) { state->message_class = PyObject_GetAttrString(mod, "Message"); Py_DECREF(mod); - PyObject* enum_type_wrapper = - PyImport_ImportModule("google.protobuf.internal.enum_type_wrapper"); + PyObject* enum_type_wrapper = PyImport_ImportModule( + PYUPB_PROTOBUF_INTERNAL_PACKAGE ".enum_type_wrapper"); if (enum_type_wrapper == NULL) return false; state->enum_type_wrapper_class = diff --git a/python/pb_unit_tests/BUILD b/python/pb_unit_tests/BUILD index bcea96ecf1..ecb1d7b900 100644 --- a/python/pb_unit_tests/BUILD +++ b/python/pb_unit_tests/BUILD @@ -23,231 +23,40 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +load("//bazel:pyproto_test_wrapper.bzl", "pyproto_test_wrapper") + licenses(["notice"]) -py_test( - name = "descriptor_database_test", - srcs = ["descriptor_database_test_wrapper.py"], - legacy_create_init = False, - main = "descriptor_database_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "descriptor_pool_test", - srcs = ["descriptor_pool_test_wrapper.py"], - legacy_create_init = False, - main = "descriptor_pool_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "descriptor_test", - srcs = ["descriptor_test_wrapper.py"], - legacy_create_init = False, - main = "descriptor_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "generator_test", - srcs = ["generator_test_wrapper.py"], - legacy_create_init = False, - main = "generator_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "json_format_test", - srcs = ["json_format_test_wrapper.py"], - legacy_create_init = False, - main = "json_format_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "keywords_test", - srcs = ["keywords_test_wrapper.py"], - legacy_create_init = False, - main = "keywords_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "message_factory_test", - srcs = ["message_factory_test_wrapper.py"], - legacy_create_init = False, - main = "message_factory_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "message_test", - srcs = ["message_test_wrapper.py"], - data = [ - "@com_google_protobuf//:testdata", - ], - legacy_create_init = False, - main = "message_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "proto_builder_test", - srcs = ["proto_builder_test_wrapper.py"], - legacy_create_init = False, - main = "proto_builder_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "reflection_test", - srcs = ["reflection_test_wrapper.py"], - legacy_create_init = False, - main = "reflection_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "service_reflection_test", - srcs = ["service_reflection_test_wrapper.py"], - legacy_create_init = False, - main = "service_reflection_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "symbol_database_test", - srcs = ["symbol_database_test_wrapper.py"], - legacy_create_init = False, - main = "symbol_database_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "text_encoding_test", - srcs = ["text_encoding_test_wrapper.py"], - legacy_create_init = False, - main = "text_encoding_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "text_format_test", - srcs = ["text_format_test_wrapper.py"], - data = [ - "@com_google_protobuf//:testdata", - ], - legacy_create_init = False, - main = "text_format_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "unknown_fields_test", - srcs = ["unknown_fields_test_wrapper.py"], - legacy_create_init = False, - main = "unknown_fields_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "well_known_types_test", - srcs = ["well_known_types_test_wrapper.py"], - legacy_create_init = False, - main = "well_known_types_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) - -py_test( - name = "wire_format_test", - srcs = ["wire_format_test_wrapper.py"], - legacy_create_init = False, - main = "wire_format_test_wrapper.py", - deps = [ - "//python:message_ext", - "@com_google_protobuf//:python_common_test_protos", - "@com_google_protobuf//:python_specific_test_protos", - "@com_google_protobuf//:python_srcs", - ], -) +pyproto_test_wrapper(name = "descriptor_database_test") + +pyproto_test_wrapper(name = "descriptor_pool_test") + +pyproto_test_wrapper(name = "descriptor_test") + +pyproto_test_wrapper(name = "generator_test") # copybara:strip_for_google3 + +pyproto_test_wrapper(name = "json_format_test") + +pyproto_test_wrapper(name = "keywords_test") + +pyproto_test_wrapper(name = "message_factory_test") + +pyproto_test_wrapper(name = "message_test") + +pyproto_test_wrapper(name = "proto_builder_test") + +pyproto_test_wrapper(name = "reflection_test") + +pyproto_test_wrapper(name = "service_reflection_test") + +pyproto_test_wrapper(name = "symbol_database_test") + +pyproto_test_wrapper(name = "text_encoding_test") + +pyproto_test_wrapper(name = "text_format_test") + +pyproto_test_wrapper(name = "unknown_fields_test") + +pyproto_test_wrapper(name = "well_known_types_test") + +pyproto_test_wrapper(name = "wire_format_test") diff --git a/python/protobuf.c b/python/protobuf.c index 35600ae788..ae6b6d5c29 100644 --- a/python/protobuf.c +++ b/python/protobuf.c @@ -94,8 +94,8 @@ PyUpb_ModuleState* PyUpb_ModuleState_Get(void) { PyObject* PyUpb_GetWktBases(PyUpb_ModuleState* state) { if (!state->wkt_bases) { - PyObject* wkt_module = - PyImport_ImportModule("google.protobuf.internal.well_known_types"); + PyObject* wkt_module = PyImport_ImportModule(PYUPB_PROTOBUF_INTERNAL_PACKAGE + ".well_known_types"); if (wkt_module == NULL) { return false; diff --git a/python/protobuf.h b/python/protobuf.h index fdad168b86..d09ab0e8db 100644 --- a/python/protobuf.h +++ b/python/protobuf.h @@ -34,7 +34,19 @@ #include "python/python.h" #include "upb/table_internal.h" +// copybara:strip_for_google3_begin +#define PYUPB_PROTOBUF_PUBLIC_PACKAGE "google.protobuf" +#define PYUPB_PROTOBUF_INTERNAL_PACKAGE "google.protobuf.internal" +#define PYUPB_DESCRIPTOR_PROTO_PACKAGE "google.protobuf" +#define PYUPB_DESCRIPTOR_MODULE "google.protobuf.descriptor_pb2" #define PYUPB_MODULE_NAME "google.protobuf.pyext._message" +// copybara:replace_for_google3_begin +// #define PYUPB_PROTOBUF_PUBLIC_PACKAGE "google3.net.proto2.python.public" +// #define PYUPB_PROTOBUF_INTERNAL_PACKAGE "google3.net.proto2.python.internal" +// #define PYUPB_DESCRIPTOR_PROTO_PACKAGE "proto2" +// #define PYUPB_DESCRIPTOR_MODULE "google3.net.proto2.proto.descriptor_pb2" +// #define PYUPB_MODULE_NAME "google3.third_party.upb.python._message" +// copybara:replace_for_google3_end #define PYUPB_RETURN_OOM return PyErr_SetNone(PyExc_MemoryError), NULL diff --git a/upb/def.c b/upb/def.c index 6b70263991..8ae873db4a 100644 --- a/upb/def.c +++ b/upb/def.c @@ -188,6 +188,7 @@ struct upb_MethodDef { const char* full_name; const upb_MessageDef* input_type; const upb_MessageDef* output_type; + int index; bool client_streaming; bool server_streaming; }; @@ -980,6 +981,8 @@ const char* upb_MethodDef_FullName(const upb_MethodDef* m) { return m->full_name; } +int upb_MethodDef_Index(const upb_MethodDef* m) { return m->index; } + const char* upb_MethodDef_Name(const upb_MethodDef* m) { return shortdefname(m->full_name); } @@ -2382,6 +2385,7 @@ static void create_service( m->service = s; m->full_name = makefullname(ctx, s->full_name, name); + m->index = i; m->client_streaming = google_protobuf_MethodDescriptorProto_client_streaming(method_proto); m->server_streaming = @@ -2810,15 +2814,10 @@ static void resolve_msgdef(symtab_addctx* ctx, upb_MessageDef* m) { resolve_fielddef(ctx, m->full_name, (upb_FieldDef*)&m->fields[i]); } - for (int i = 0; i < m->nested_ext_count; i++) { - resolve_fielddef(ctx, m->full_name, (upb_FieldDef*)&m->nested_exts[i]); - } - - if (!ctx->layout) make_layout(ctx, m); - m->in_message_set = false; - if (m->nested_ext_count == 1) { - const upb_FieldDef* ext = &m->nested_exts[0]; + for (int i = 0; i < m->nested_ext_count; i++) { + upb_FieldDef* ext = (upb_FieldDef*)&m->nested_exts[i]; + resolve_fielddef(ctx, m->full_name, ext); if (ext->type_ == kUpb_FieldType_Message && ext->label_ == kUpb_Label_Optional && ext->sub.msgdef == m && google_protobuf_MessageOptions_message_set_wire_format( @@ -2827,6 +2826,8 @@ static void resolve_msgdef(symtab_addctx* ctx, upb_MessageDef* m) { } } + if (!ctx->layout) make_layout(ctx, m); + for (int i = 0; i < m->nested_msg_count; i++) { resolve_msgdef(ctx, (upb_MessageDef*)&m->nested_msgs[i]); } diff --git a/upb/def.h b/upb/def.h index 2d22896e89..1c7adb4b15 100644 --- a/upb/def.h +++ b/upb/def.h @@ -317,6 +317,7 @@ const google_protobuf_MethodOptions* upb_MethodDef_Options( const upb_MethodDef* m); bool upb_MethodDef_HasOptions(const upb_MethodDef* m); const char* upb_MethodDef_FullName(const upb_MethodDef* m); +int upb_MethodDef_Index(const upb_MethodDef* m); const char* upb_MethodDef_Name(const upb_MethodDef* m); const upb_ServiceDef* upb_MethodDef_Service(const upb_MethodDef* m); const upb_MessageDef* upb_MethodDef_InputType(const upb_MethodDef* m);