From 5b27b4f300c9c8fdb1f178151ef29a22a674d184 Mon Sep 17 00:00:00 2001 From: deannagarcia <69992229+deannagarcia@users.noreply.github.com> Date: Thu, 19 Jan 2023 15:10:20 -0800 Subject: [PATCH] Add ruby release targets (#11468) Add bazel targets to create ruby release artifacts. Should be run with: ``` bazel run ruby:release bazel run ruby:jruby_release ``` Closes #11468 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/11468 from deannagarcia:rubyTargets b7b7eb68657c3aa632813cc4f8896fec5db6d411 PiperOrigin-RevId: 503277136 --- .github/workflows/ruby_install.yml | 58 ++++++++++++++++++ WORKSPACE | 8 +++ protobuf_deps.bzl | 4 +- protobuf_version.bzl | 1 + ruby/BUILD.bazel | 39 ++++++++++++ ruby/build_jruby_release.sh | 63 +++++++++++++++++++ ruby/build_release.sh | 88 +++++++++++++++++++++++++++ ruby/ext/google/protobuf_c/extconf.rb | 4 +- ruby/lib/google/BUILD.bazel | 2 + 9 files changed, 263 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ruby_install.yml mode change 100644 => 100755 ruby/BUILD.bazel create mode 100755 ruby/build_jruby_release.sh create mode 100755 ruby/build_release.sh diff --git a/.github/workflows/ruby_install.yml b/.github/workflows/ruby_install.yml new file mode 100644 index 0000000000..c9f38c79c7 --- /dev/null +++ b/.github/workflows/ruby_install.yml @@ -0,0 +1,58 @@ +name: Ruby Install Tests + +on: + push: + branches: + - main + - '[0-9]+.x' + pull_request: + branches: + - main + - '[0-9]+.x' + workflow_dispatch: + +jobs: + test_ruby_gems: + name: Test ruby gems + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + ruby: [2.6, 2.7, 3.0, 3.1, 3.2, jruby-9.2, jruby-9.3] + + steps: + - uses: actions/checkout@v2 + - name: Install bazel + run: | + sudo apt-get install -qy wget + mkdir $HOME/bin + wget -O $HOME/bin/bazel https://github.com/bazelbuild/bazel/releases/download/6.0.0/bazel-6.0.0-linux-x86_64 + chmod a+x $HOME/bin/bazel + - name: Install git + run: | + sudo apt-get install -qy --no-install-recommends git + - name: Install ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Build cruby gem + run: $HOME/bin/bazel run ruby:release + if: ${{ !contains(matrix.ruby, 'jruby') }} + - name: Install cruby gem + run: gem install bazel-bin/ruby/release.runfiles/com_google_protobuf/tmp/google-protobuf-* + if: ${{ !contains(matrix.ruby, 'jruby') }} + - name: Build jruby gem + run: $HOME/bin/bazel run ruby:jruby_release + if: ${{ contains(matrix.ruby, 'jruby') }} + - name: Install jruby gem + run: gem install bazel-bin/ruby/jruby_release.runfiles/com_google_protobuf/tmp/google-protobuf-* + if: ${{ contains(matrix.ruby, 'jruby') }} + - name: Test installation + run: | + bazel run //:protoc -- --proto_path=$GITHUB_WORKSPACE/src --proto_path=$GITHUB_WORKSPACE/ruby/tests --proto_path=$GITHUB_WORKSPACE/ruby --ruby_out=$GITHUB_WORKSPACE/ruby tests/test_import_proto2.proto + bazel run //:protoc -- --proto_path=$GITHUB_WORKSPACE/src --proto_path=$GITHUB_WORKSPACE/ruby/tests --proto_path=$GITHUB_WORKSPACE/ruby --ruby_out=$GITHUB_WORKSPACE/ruby tests/basic_test.proto + ruby ruby/tests/basic.rb diff --git a/WORKSPACE b/WORKSPACE index 36d4ee5e1d..3ed1bee114 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -35,6 +35,14 @@ http_archive( load("//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") protobuf_deps() +load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") + +rules_jvm_external_deps() + +load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") + +rules_jvm_external_setup() + load("@rules_jvm_external//:defs.bzl", "maven_install") maven_install( diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 1269a2dcf4..4a9e722a7f 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -110,8 +110,8 @@ def protobuf_deps(): _github_archive( name = "rules_ruby", repo = "https://github.com/protocolbuffers/rules_ruby", - commit = "59732544ce3a4bc4e8e4d4e8c8f318c931c17eae", - sha256 = "4fc45adf1056c824afde9a52b743b915eeada3633539eb1dbd641a66dc9f4c4f", + commit = "5cf6ff74161d7f985b9bf86bb3c5fb16cef6337b", + sha256 = "c88dd69eb50fcfd7fbc5d7db79adc6631ef0e1d80b3c94efe33ac5ee3ccc37f7", ) if not native.existing_rule("rules_jvm_external"): diff --git a/protobuf_version.bzl b/protobuf_version.bzl index 4ac031cb3b..ec39a4e123 100644 --- a/protobuf_version.bzl +++ b/protobuf_version.bzl @@ -2,3 +2,4 @@ PROTOC_VERSION = "21.12" PROTOBUF_JAVA_VERSION = "3.21.12" PROTOBUF_PYTHON_VERSION = "4.21.12" PROTOBUF_PHP_VERSION = "3.21.12" +PROTOBUF_RUBY_VERSION = "3.21.12" diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel old mode 100644 new mode 100755 index 6095c1d328..b469bc8fc4 --- a/ruby/BUILD.bazel +++ b/ruby/BUILD.bazel @@ -6,6 +6,7 @@ load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_ruby//ruby:defs.bzl", "ruby_library") load("//:protobuf.bzl", "internal_ruby_proto_library") load("//conformance:defs.bzl", "conformance_test") +load("//:protobuf_version.bzl", "PROTOBUF_RUBY_VERSION") ################################################################################ # Ruby Runtime @@ -19,6 +20,44 @@ ruby_library( ], ) +sh_binary( + name = "jruby_release", + data = [ + "//ruby/lib/google:copy_jar", + "//ruby/lib/google:dist_files", + "//:well_known_ruby_protos", + "google-protobuf.gemspec", + ], + srcs = [ + "build_jruby_release.sh", + ], + deps = ["@bazel_tools//tools/bash/runfiles"], + target_compatible_with = select({ + "@rules_ruby//ruby/runtime:config_jruby": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + +sh_binary( + name = "release", + data = [ + "@utf8_range//:utf8_range_srcs", + "@utf8_range//:LICENSE", + "//:well_known_ruby_protos", + "//ruby/ext/google/protobuf_c:dist_files", + "//ruby/lib/google:dist_files", + "google-protobuf.gemspec", + ], + srcs = [ + "build_release.sh", + ], + deps = ["@bazel_tools//tools/bash/runfiles"], + target_compatible_with = select({ + "@rules_ruby//ruby/runtime:config_ruby": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + ################################################################################ # Tests ################################################################################ diff --git a/ruby/build_jruby_release.sh b/ruby/build_jruby_release.sh new file mode 100755 index 0000000000..9ff66d680a --- /dev/null +++ b/ruby/build_jruby_release.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# This file should be executed with jruby + +set -ex + +# --- begin runfiles.bash initialization --- +# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash). +set -euo pipefail +if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + if [[ -f "$0.runfiles_manifest" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest" + elif [[ -f "$0.runfiles/MANIFEST" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST" + elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + export RUNFILES_DIR="$0.runfiles" + fi +fi +if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" +elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \ + "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)" +else +echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash" +exit 1 +fi +# --- end runfiles.bash initialization --- + +# Make a temporary directory and move to it to do all packaging work +mkdir -p tmp +cd tmp + +# Move all generated files to lib/google/protobuf +mkdir -p lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/any_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/api_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/descriptor_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/duration_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/empty_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/field_mask_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/source_context_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/struct_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/timestamp_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/type_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/wrappers_pb.rb)" lib/google/protobuf + +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf_java.jar)" lib/google + +# Move all source files to the correct location +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf.rb)" lib/google +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf/descriptor_dsl.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf/message_exts.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf/repeated_field.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf/well_known_types.rb)" lib/google/protobuf + +# Move gemspec file to current directory +cp "$(rlocation com_google_protobuf/ruby/google.protobuf.gemspec)" . + +# Make all files global readable/writable/executable +chmod -R 777 ./ + +# Build gem +gem build google-protobuf.gemspec diff --git a/ruby/build_release.sh b/ruby/build_release.sh new file mode 100755 index 0000000000..c9b68d1514 --- /dev/null +++ b/ruby/build_release.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# This file should be executed with ruby + +set -ex + +# --- begin runfiles.bash initialization --- +# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash). +set -euo pipefail +if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + if [[ -f "$0.runfiles_manifest" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest" + elif [[ -f "$0.runfiles/MANIFEST" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST" + elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + export RUNFILES_DIR="$0.runfiles" + fi +fi +if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" +elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \ + "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)" +else +echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash" +exit 1 +fi +# --- end runfiles.bash initialization --- + +# rvm use ruby-3.0 + +# Make a temporary directory and move to it to do all packaging work +mkdir -p tmp +cd tmp + +# Move all generated files to lib/google/protobuf +mkdir -p lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/any_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/api_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/descriptor_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/duration_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/empty_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/field_mask_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/source_context_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/struct_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/timestamp_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/type_pb.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/src/google/protobuf/wrappers_pb.rb)" lib/google/protobuf + +# Move all utf-8 files to ext/google/protobuf_c/third_party/utf8_range +UTF8_DIR=ext/google/protobuf_c/third_party/utf8_range +mkdir -p $UTF8_DIR +cp "$(rlocation utf8_range/LICENSE)" $UTF8_DIR/LICENSE +cp "$(rlocation utf8_range/naive.c)" $UTF8_DIR +cp "$(rlocation utf8_range/range2-neon.c)" $UTF8_DIR +cp "$(rlocation utf8_range/range2-sse.c)" $UTF8_DIR +cp "$(rlocation utf8_range/utf8_range.h)" $UTF8_DIR + +# Move all source files to the correct location +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/convert.c)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/convert.h)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/defs.c)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/defs.h)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/extconf.rb)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/map.c)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/map.h)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/message.c)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/message.h)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/protobuf.c)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/protobuf.h)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/repeated_field.c)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/repeated_field.h)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/ruby-upb.c)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/ruby-upb.h)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/ext/google/protobuf_c/wrap_memcpy.c)" ext/google/protobuf_c +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf.rb)" lib/google +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf/descriptor_dsl.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf/message_exts.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf/repeated_field.rb)" lib/google/protobuf +cp "$(rlocation com_google_protobuf/ruby/lib/google/protobuf/well_known_types.rb)" lib/google/protobuf + +# Move gemspec file to current directory +cp "$(rlocation com_google_protobuf/ruby/google.protobuf.gemspec)" . + +# Make all files global readable/writable/executable +chmod -R 777 ./ + +# Build gem +gem build google-protobuf.gemspec diff --git a/ruby/ext/google/protobuf_c/extconf.rb b/ruby/ext/google/protobuf_c/extconf.rb index 38bb8c970a..04c6f78fd3 100755 --- a/ruby/ext/google/protobuf_c/extconf.rb +++ b/ruby/ext/google/protobuf_c/extconf.rb @@ -7,7 +7,7 @@ ext_name = "google/protobuf_c" dir_config(ext_name) if RUBY_PLATFORM =~ /darwin/ || RUBY_PLATFORM =~ /linux/ || RUBY_PLATFORM =~ /freebsd/ - $CFLAGS += " -std=gnu99 -O3 -DNDEBUG -fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement" + $CFLAGS += " -std=gnu99 -O3 -DNDEBUG -fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement -I$(srcdir)/third_party/utf8_range" else $CFLAGS += " -std=gnu99 -O3 -DNDEBUG" end @@ -25,4 +25,4 @@ $srcs = ["protobuf.c", "convert.c", "defs.c", "message.c", "repeated_field.c", "map.c", "ruby-upb.c", "wrap_memcpy.c", "naive.c", "range2-neon.c", "range2-sse.c"] -create_makefile(ext_name, Dir.pwd+"/../../../../ext/google/protobuf_c") +create_makefile(ext_name) diff --git a/ruby/lib/google/BUILD.bazel b/ruby/lib/google/BUILD.bazel index 6c90959b9d..4cfa5cbcfe 100644 --- a/ruby/lib/google/BUILD.bazel +++ b/ruby/lib/google/BUILD.bazel @@ -27,6 +27,7 @@ java_binary( runtime_deps = [ "//ruby/src/main/java:protobuf_java" ], + deploy_env = ["@rules_ruby//ruby/runtime:jruby_binary"], create_executable = False, ) @@ -37,6 +38,7 @@ genrule( outs = ["protobuf_java.jar"], cmd = "cp $< $@", tags = ["manual"], + visibility = ["//ruby:__subpackages__"], ) ruby_library(