From 39362b35b236242729cc5a0cb6fb0f7233fecd5e Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Wed, 14 Oct 2015 17:12:11 -0700 Subject: [PATCH 1/7] Extract protoc action into .bzl for cc. This is needed to support python bazel build. --- BUILD | 49 ++++++++++++----------- protobuf.bzl | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 25 deletions(-) create mode 100644 protobuf.bzl diff --git a/BUILD b/BUILD index 1aca02cfd3..a060f06e25 100644 --- a/BUILD +++ b/BUILD @@ -18,6 +18,8 @@ COPTS = [ # Bazel should provide portable link_opts for pthread. LINK_OPTS = ["-lpthread"] +load("protobuf", "cc_proto_library") + cc_library( name = "protobuf_lite", srcs = [ @@ -140,6 +142,14 @@ WELL_KNOWN_PROTOS = [ "google/protobuf/wrappers.proto", ] +cc_proto_library( + name = "cc_wkt_protos", + srcs = ["src/" + s for s in WELL_KNOWN_PROTOS], + internal_bootstrap_hack = 1, + prefix = "src", + deps = ":protobuf", +) + ################################################################################ # Protocol Buffers Compiler ################################################################################ @@ -258,23 +268,22 @@ cc_binary( ################################################################################ genrule( name = "generate_java_descriptor_proto", - tools = [":protoc"], - srcs = [ "src/google/protobuf/descriptor.proto", ], - outs = [ "com/google/protobuf/DescriptorProtos.java" ], + srcs = ["src/google/protobuf/descriptor.proto"], + outs = ["com/google/protobuf/DescriptorProtos.java"], cmd = "$(location :protoc) --java_out=$(@D)/../../.. $<", + tools = [":protoc"], ) java_library( name = "java_proto", - visibility = ["//visibility:public"], srcs = glob([ - "java/src/main/java/com/google/protobuf/*.java" + "java/src/main/java/com/google/protobuf/*.java", ]) + [ - ":generate_java_descriptor_proto", - ] + ":generate_java_descriptor_proto", + ], + visibility = ["//visibility:public"], ) - ################################################################################ # Tests ################################################################################ @@ -328,22 +337,11 @@ TEST_PROTOS = [ "google/protobuf/util/json_format_proto3.proto", ] -PROTOS = LITE_TEST_PROTOS + TEST_PROTOS - -INPUTS = PROTOS + WELL_KNOWN_PROTOS - -OUTPUTS = ["src/" + x[:-5] + "pb.h" for x in PROTOS] + \ - ["src/" + x[:-5] + "pb.cc" for x in PROTOS] - -genrule( - name = "gen_test_protos", - srcs = ["src/" + x for x in INPUTS], - outs = OUTPUTS, - cmd = - "$(location :protoc) --cpp_out=$(@D)/src" + - "".join([" -I" + x + "=$(location src/" + x + ")" for x in INPUTS]) + - "".join([" $(location src/" + x + ")" for x in PROTOS]), - tools = [":protoc"], +cc_proto_library( + name = "cc_test_protos", + srcs = ["src/" + s for s in (LITE_TEST_PROTOS + TEST_PROTOS)], + prefix = "src", + proto_deps = [":cc_wkt_protos"], ) COMMON_TEST_SRCS = [ @@ -372,7 +370,7 @@ cc_binary( cc_test( name = "protobuf_test", - srcs = OUTPUTS + COMMON_TEST_SRCS + [ + srcs = COMMON_TEST_SRCS + [ # AUTOGEN(test_srcs) "src/google/protobuf/any_test.cc", "src/google/protobuf/arena_unittest.cc", @@ -449,6 +447,7 @@ cc_test( deps = [ ":protobuf", ":protoc_lib", + ":cc_test_protos", "//external:gtest_main", ], ) diff --git a/protobuf.bzl b/protobuf.bzl new file mode 100644 index 0000000000..63ef95fb00 --- /dev/null +++ b/protobuf.bzl @@ -0,0 +1,107 @@ +# -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED + + +def _gen_dir(ctx): + if not ctx.attr.prefix: + return ctx.label.package + if not ctx.label.package: + return ctx.attr.prefix + return ctx.label.package + '/' + ctx.attr.prefix + +def CcOuts(srcs): + return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \ + [s[:-len(".proto")] + ".pb.cc" for s in srcs] + +def PyOuts(srcs): + return [s[:-len(".proto")] + "_pb2.py" for s in srcs] + +def _proto_srcs_impl(ctx): + """General implementation for calculating proto srcs""" + srcs = ctx.files.srcs + deps = [] + deps += ctx.files.srcs + gen_dir = _gen_dir(ctx) + import_flags = ["-I" + gen_dir] + for dep in ctx.attr.deps: + import_flags += dep.proto.import_flags + deps += dep.proto.deps + + args = [] + if ctx.attr.gen_cc: + args += ["--cpp_out=" + ctx.var["GENDIR"] + "/" + gen_dir] + if ctx.attr.gen_py: + args += ["--python_out=" + ctx.var["GENDIR"] + "/" + gen_dir] + + if args: + ctx.action( + inputs= srcs + deps, + outputs=ctx.outputs.outs, + arguments= args + import_flags + [s.path for s in srcs], + executable=ctx.executable.protoc + ) + + return struct( + proto=struct( + srcs = srcs, + import_flags = import_flags, + deps = deps, + ), + ) + +_proto_srcs = rule( + implementation = _proto_srcs_impl, + output_to_genfiles = True, + attrs = { + "srcs": attr.label_list(allow_files=True), + "deps": attr.label_list(providers=["proto"]), + "prefix": attr.string(), + "protoc": attr.label(executable=True, single_file=True, mandatory=True), + "gen_cc": attr.bool(), + "gen_py": attr.bool(), + "outs": attr.output_list() + } +) + +def cc_proto_library( + name, + srcs=[], + protoc=":protoc", + internal_bootstrap_hack=False, + prefix="", + proto_deps=[], + deps=[], + **kargs): + + if internal_bootstrap_hack: + # For pre-checked-in generated files, we add the internal_bootstrap_hack + # which will skip the codegen action. + _proto_srcs( + name = name + "_genproto", + srcs = srcs, + deps = [s + "_genproto" for s in proto_deps], + prefix = prefix, + protoc = protoc, + ) + # An empty cc_library to make rule dependency consistent. + native.cc_library( + name = name, + **kargs) + return + + outs = CcOuts(srcs) + _proto_srcs( + name = name + "_genproto", + srcs = srcs, + deps = [s + "_genproto" for s in proto_deps], + prefix = prefix, + protoc = protoc, + gen_cc = 1, + outs = outs, + ) + + native.cc_library( + name = name, + srcs = outs, + deps = deps + proto_deps, + includes = [prefix], + **kargs) From ee8131a62f2c2b3cf12c6899214dd48ee0907902 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Wed, 14 Oct 2015 17:20:05 -0700 Subject: [PATCH 2/7] buildifier the file --- protobuf.bzl | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/protobuf.bzl b/protobuf.bzl index 63ef95fb00..a6ef20e10d 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -1,6 +1,5 @@ # -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED - def _gen_dir(ctx): if not ctx.attr.prefix: return ctx.label.package @@ -49,17 +48,21 @@ def _proto_srcs_impl(ctx): ) _proto_srcs = rule( - implementation = _proto_srcs_impl, - output_to_genfiles = True, attrs = { - "srcs": attr.label_list(allow_files=True), - "deps": attr.label_list(providers=["proto"]), - "prefix": attr.string(), - "protoc": attr.label(executable=True, single_file=True, mandatory=True), - "gen_cc": attr.bool(), - "gen_py": attr.bool(), - "outs": attr.output_list() - } + "srcs": attr.label_list(allow_files = True), + "deps": attr.label_list(providers = ["proto"]), + "prefix": attr.string(), + "protoc": attr.label( + executable = True, + single_file = True, + mandatory = True, + ), + "gen_cc": attr.bool(), + "gen_py": attr.bool(), + "outs": attr.output_list(), + }, + output_to_genfiles = True, + implementation = _proto_srcs_impl, ) def cc_proto_library( From 125a91be0847a0981e4db5e44990400a56157a8f Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Wed, 14 Oct 2015 17:37:39 -0700 Subject: [PATCH 3/7] Format code. --- protobuf.bzl | 68 ++++++++++++++++++++++++++-------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/protobuf.bzl b/protobuf.bzl index a6ef20e10d..45dfc6c26e 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -9,10 +9,10 @@ def _gen_dir(ctx): def CcOuts(srcs): return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \ - [s[:-len(".proto")] + ".pb.cc" for s in srcs] + [s[:-len(".proto")] + ".pb.cc" for s in srcs] def PyOuts(srcs): - return [s[:-len(".proto")] + "_pb2.py" for s in srcs] + return [s[:-len(".proto")] + "_pb2.py" for s in srcs] def _proto_srcs_impl(ctx): """General implementation for calculating proto srcs""" @@ -33,19 +33,19 @@ def _proto_srcs_impl(ctx): if args: ctx.action( - inputs= srcs + deps, + inputs=srcs + deps, outputs=ctx.outputs.outs, - arguments= args + import_flags + [s.path for s in srcs], + arguments=args + import_flags + [s.path for s in srcs], executable=ctx.executable.protoc ) return struct( proto=struct( - srcs = srcs, - import_flags = import_flags, - deps = deps, - ), - ) + srcs=srcs, + import_flags=import_flags, + deps=deps, + ), + ) _proto_srcs = rule( attrs = { @@ -66,45 +66,45 @@ _proto_srcs = rule( ) def cc_proto_library( - name, - srcs=[], - protoc=":protoc", - internal_bootstrap_hack=False, - prefix="", - proto_deps=[], - deps=[], - **kargs): + name, + srcs=[], + protoc=":protoc", + internal_bootstrap_hack=False, + prefix="", + proto_deps=[], + deps=[], + **kargs): if internal_bootstrap_hack: # For pre-checked-in generated files, we add the internal_bootstrap_hack # which will skip the codegen action. _proto_srcs( - name = name + "_genproto", - srcs = srcs, - deps = [s + "_genproto" for s in proto_deps], - prefix = prefix, - protoc = protoc, + name=name + "_genproto", + srcs=srcs, + deps=[s + "_genproto" for s in proto_deps], + prefix=prefix, + protoc=protoc, ) # An empty cc_library to make rule dependency consistent. native.cc_library( - name = name, + name=name, **kargs) return outs = CcOuts(srcs) _proto_srcs( - name = name + "_genproto", - srcs = srcs, - deps = [s + "_genproto" for s in proto_deps], - prefix = prefix, - protoc = protoc, - gen_cc = 1, - outs = outs, + name=name + "_genproto", + srcs=srcs, + deps=[s + "_genproto" for s in proto_deps], + prefix=prefix, + protoc=protoc, + gen_cc=1, + outs=outs, ) native.cc_library( - name = name, - srcs = outs, - deps = deps + proto_deps, - includes = [prefix], + name=name, + srcs=outs, + deps=deps + proto_deps, + includes=[prefix], **kargs) From 25d75b56655068f6150e8d8d816afc2f7417e0c5 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Wed, 14 Oct 2015 17:41:14 -0700 Subject: [PATCH 4/7] make cc_out and py_out internal. --- protobuf.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/protobuf.bzl b/protobuf.bzl index 45dfc6c26e..ecd58b8752 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -7,11 +7,11 @@ def _gen_dir(ctx): return ctx.attr.prefix return ctx.label.package + '/' + ctx.attr.prefix -def CcOuts(srcs): +def _cc_outs(srcs): return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \ [s[:-len(".proto")] + ".pb.cc" for s in srcs] -def PyOuts(srcs): +def _py_outs(srcs): return [s[:-len(".proto")] + "_pb2.py" for s in srcs] def _proto_srcs_impl(ctx): @@ -91,7 +91,7 @@ def cc_proto_library( **kargs) return - outs = CcOuts(srcs) + outs = _cc_outs(srcs) _proto_srcs( name=name + "_genproto", srcs=srcs, From 9c7d9c0925ebed800e4e41b298de9d97fd429679 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Thu, 15 Oct 2015 10:51:32 -0700 Subject: [PATCH 5/7] style/naming fix --- protobuf.bzl | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/protobuf.bzl b/protobuf.bzl index ecd58b8752..a745998c8b 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -14,8 +14,8 @@ def _cc_outs(srcs): def _py_outs(srcs): return [s[:-len(".proto")] + "_pb2.py" for s in srcs] -def _proto_srcs_impl(ctx): - """General implementation for calculating proto srcs""" +def _proto_gen_impl(ctx): + """General implementation for generating protos""" srcs = ctx.files.srcs deps = [] deps += ctx.files.srcs @@ -36,7 +36,7 @@ def _proto_srcs_impl(ctx): inputs=srcs + deps, outputs=ctx.outputs.outs, arguments=args + import_flags + [s.path for s in srcs], - executable=ctx.executable.protoc + executable=ctx.executable.protoc, ) return struct( @@ -47,7 +47,7 @@ def _proto_srcs_impl(ctx): ), ) -_proto_srcs = rule( +_proto_gen = rule( attrs = { "srcs": attr.label_list(allow_files = True), "deps": attr.label_list(providers = ["proto"]), @@ -62,7 +62,7 @@ _proto_srcs = rule( "outs": attr.output_list(), }, output_to_genfiles = True, - implementation = _proto_srcs_impl, + implementation = _proto_gen_impl, ) def cc_proto_library( @@ -78,7 +78,7 @@ def cc_proto_library( if internal_bootstrap_hack: # For pre-checked-in generated files, we add the internal_bootstrap_hack # which will skip the codegen action. - _proto_srcs( + _proto_gen( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in proto_deps], @@ -88,11 +88,12 @@ def cc_proto_library( # An empty cc_library to make rule dependency consistent. native.cc_library( name=name, - **kargs) + **kargs, + ) return outs = _cc_outs(srcs) - _proto_srcs( + _proto_gen( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in proto_deps], @@ -107,4 +108,5 @@ def cc_proto_library( srcs=outs, deps=deps + proto_deps, includes=[prefix], - **kargs) + **kargs, + ) From d8701b5741a668c5f2b4781b12f5d5517e38b9e7 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Fri, 16 Oct 2015 11:44:21 -0700 Subject: [PATCH 6/7] Rename deps/proto_deps to cc_libs/deps --- BUILD | 4 ++-- protobuf.bzl | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/BUILD b/BUILD index a060f06e25..bce030a761 100644 --- a/BUILD +++ b/BUILD @@ -147,7 +147,7 @@ cc_proto_library( srcs = ["src/" + s for s in WELL_KNOWN_PROTOS], internal_bootstrap_hack = 1, prefix = "src", - deps = ":protobuf", + cc_libs = [":protobuf"], ) ################################################################################ @@ -341,7 +341,7 @@ cc_proto_library( name = "cc_test_protos", srcs = ["src/" + s for s in (LITE_TEST_PROTOS + TEST_PROTOS)], prefix = "src", - proto_deps = [":cc_wkt_protos"], + deps = [":cc_wkt_protos"], ) COMMON_TEST_SRCS = [ diff --git a/protobuf.bzl b/protobuf.bzl index a745998c8b..c6685d3017 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -71,8 +71,8 @@ def cc_proto_library( protoc=":protoc", internal_bootstrap_hack=False, prefix="", - proto_deps=[], deps=[], + cc_libs=[], **kargs): if internal_bootstrap_hack: @@ -81,22 +81,21 @@ def cc_proto_library( _proto_gen( name=name + "_genproto", srcs=srcs, - deps=[s + "_genproto" for s in proto_deps], + deps=[s + "_genproto" for s in deps], prefix=prefix, protoc=protoc, ) # An empty cc_library to make rule dependency consistent. native.cc_library( name=name, - **kargs, - ) + **kargs) return outs = _cc_outs(srcs) _proto_gen( name=name + "_genproto", srcs=srcs, - deps=[s + "_genproto" for s in proto_deps], + deps=[s + "_genproto" for s in deps], prefix=prefix, protoc=protoc, gen_cc=1, @@ -106,7 +105,6 @@ def cc_proto_library( native.cc_library( name=name, srcs=outs, - deps=deps + proto_deps, + deps=cc_libs + deps, includes=[prefix], - **kargs, - ) + **kargs) From 3101e7337cdade287bdba97dd7c08ee5d4c14587 Mon Sep 17 00:00:00 2001 From: Jisi Liu Date: Fri, 16 Oct 2015 12:46:26 -0700 Subject: [PATCH 7/7] Change prefix to include, and add docs --- BUILD | 4 ++-- protobuf.bzl | 37 +++++++++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/BUILD b/BUILD index bce030a761..69ea6cf3e1 100644 --- a/BUILD +++ b/BUILD @@ -146,7 +146,7 @@ cc_proto_library( name = "cc_wkt_protos", srcs = ["src/" + s for s in WELL_KNOWN_PROTOS], internal_bootstrap_hack = 1, - prefix = "src", + include = "src", cc_libs = [":protobuf"], ) @@ -340,7 +340,7 @@ TEST_PROTOS = [ cc_proto_library( name = "cc_test_protos", srcs = ["src/" + s for s in (LITE_TEST_PROTOS + TEST_PROTOS)], - prefix = "src", + include = "src", deps = [":cc_wkt_protos"], ) diff --git a/protobuf.bzl b/protobuf.bzl index c6685d3017..79dabd0246 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -1,11 +1,11 @@ # -*- mode: python; -*- PYTHON-PREPROCESSING-REQUIRED def _gen_dir(ctx): - if not ctx.attr.prefix: + if not ctx.attr.include: return ctx.label.package if not ctx.label.package: - return ctx.attr.prefix - return ctx.label.package + '/' + ctx.attr.prefix + return ctx.attr.include + return ctx.label.package + '/' + ctx.attr.include def _cc_outs(srcs): return [s[:-len(".proto")] + ".pb.h" for s in srcs] + \ @@ -51,7 +51,7 @@ _proto_gen = rule( attrs = { "srcs": attr.label_list(allow_files = True), "deps": attr.label_list(providers = ["proto"]), - "prefix": attr.string(), + "include": attr.string(), "protoc": attr.label( executable = True, single_file = True, @@ -68,12 +68,29 @@ _proto_gen = rule( def cc_proto_library( name, srcs=[], - protoc=":protoc", - internal_bootstrap_hack=False, - prefix="", deps=[], cc_libs=[], + include="", + protoc=":protoc", + internal_bootstrap_hack=False, **kargs): + """Bazel rule to create a C++ protobuf library from proto source files + + Args: + name: the name of the cc_proto_library. + srcs: the .proto files of the cc_proto_library. + deps: a list of dependency labels; must be cc_proto_library. + cc_libs: a list of other cc_library targets depended by the generated + cc_library. + include: a string indicating the include path of the .proto files. + protoc: the label of the protocol compiler to generate the sources. + internal_bootstrap_hack: a flag indicate the cc_proto_library is used only + for bootstraping. When it is set to True, no files will be generated. + The rule will simply be a provider for .proto files, so that other + cc_proto_library can depend on it. + **kargs: other keyword arguments that are passed to cc_library. + + """ if internal_bootstrap_hack: # For pre-checked-in generated files, we add the internal_bootstrap_hack @@ -82,7 +99,7 @@ def cc_proto_library( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - prefix=prefix, + include=include, protoc=protoc, ) # An empty cc_library to make rule dependency consistent. @@ -96,7 +113,7 @@ def cc_proto_library( name=name + "_genproto", srcs=srcs, deps=[s + "_genproto" for s in deps], - prefix=prefix, + include=include, protoc=protoc, gen_cc=1, outs=outs, @@ -106,5 +123,5 @@ def cc_proto_library( name=name, srcs=outs, deps=cc_libs + deps, - includes=[prefix], + includes=[include], **kargs)