From 90d19ac385999a7d89716bd3f141825b8137b193 Mon Sep 17 00:00:00 2001 From: apolcyn Date: Fri, 6 Sep 2024 15:02:52 -0700 Subject: [PATCH] [ruby] add a sanity check to load grpc in a child process before most ruby tests (#37649) To try to get more info about these dlopen flakes Closes #37649 COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37649 from apolcyn:add_check bb160cf32f06ec6ba520ac81e5a58bad8fad41ff PiperOrigin-RevId: 671902460 --- src/ruby/end2end/bad_usage_fork_test.rb | 1 + ...data_doesnt_kill_background_thread_test.rb | 1 + .../end2end/call_credentials_timeout_test.rb | 1 + .../errors_load_before_grpc_lib_test.rb | 2 + src/ruby/end2end/fork_test.rb | 1 + .../end2end/load_grpc_with_gc_stress_test.rb | 2 + .../logger_load_before_grpc_lib_test.rb | 2 + .../end2end/prefork_postfork_loop_test.rb | 1 + .../prefork_without_using_grpc_test.rb | 1 + src/ruby/end2end/sanity_check_dlopen.rb | 74 +++++++++++++++++++ src/ruby/end2end/secure_fork_test.rb | 1 + src/ruby/end2end/simple_fork_test.rb | 1 + .../status_codes_load_before_grpc_lib_test.rb | 2 + tools/run_tests/helper_scripts/build_ruby.sh | 12 +++ 14 files changed, 102 insertions(+) create mode 100755 src/ruby/end2end/sanity_check_dlopen.rb diff --git a/src/ruby/end2end/bad_usage_fork_test.rb b/src/ruby/end2end/bad_usage_fork_test.rb index f6ba65d34c6..c796b8510a7 100755 --- a/src/ruby/end2end/bad_usage_fork_test.rb +++ b/src/ruby/end2end/bad_usage_fork_test.rb @@ -24,6 +24,7 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' require 'grpc' require 'end2end_common' diff --git a/src/ruby/end2end/call_credentials_returning_bad_metadata_doesnt_kill_background_thread_test.rb b/src/ruby/end2end/call_credentials_returning_bad_metadata_doesnt_kill_background_thread_test.rb index eda85c6da09..d6da32a1b86 100755 --- a/src/ruby/end2end/call_credentials_returning_bad_metadata_doesnt_kill_background_thread_test.rb +++ b/src/ruby/end2end/call_credentials_returning_bad_metadata_doesnt_kill_background_thread_test.rb @@ -21,6 +21,7 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' require 'grpc' require 'end2end_common' diff --git a/src/ruby/end2end/call_credentials_timeout_test.rb b/src/ruby/end2end/call_credentials_timeout_test.rb index 3d442d854c1..acc81765718 100755 --- a/src/ruby/end2end/call_credentials_timeout_test.rb +++ b/src/ruby/end2end/call_credentials_timeout_test.rb @@ -21,6 +21,7 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' require 'grpc' require 'end2end_common' diff --git a/src/ruby/end2end/errors_load_before_grpc_lib_test.rb b/src/ruby/end2end/errors_load_before_grpc_lib_test.rb index 56f7714fc7c..7c753678720 100755 --- a/src/ruby/end2end/errors_load_before_grpc_lib_test.rb +++ b/src/ruby/end2end/errors_load_before_grpc_lib_test.rb @@ -18,6 +18,8 @@ this_dir = File.expand_path(File.dirname(__FILE__)) grpc_lib_dir = File.join(File.dirname(this_dir), 'lib') $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) +require_relative './sanity_check_dlopen' + def check_to_status(error) my_status = error.to_status fail('GRPC BadStatus#to_status not expected to return nil') if my_status.nil? diff --git a/src/ruby/end2end/fork_test.rb b/src/ruby/end2end/fork_test.rb index 6c92c0893c6..38a8843c985 100755 --- a/src/ruby/end2end/fork_test.rb +++ b/src/ruby/end2end/fork_test.rb @@ -24,6 +24,7 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' require 'grpc' require 'end2end_common' diff --git a/src/ruby/end2end/load_grpc_with_gc_stress_test.rb b/src/ruby/end2end/load_grpc_with_gc_stress_test.rb index 9b0ca765f86..4ed3e1af3f6 100755 --- a/src/ruby/end2end/load_grpc_with_gc_stress_test.rb +++ b/src/ruby/end2end/load_grpc_with_gc_stress_test.rb @@ -21,6 +21,8 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' + GC.stress = 0x04 require 'grpc' diff --git a/src/ruby/end2end/logger_load_before_grpc_lib_test.rb b/src/ruby/end2end/logger_load_before_grpc_lib_test.rb index 76c37875040..8e58667bff3 100755 --- a/src/ruby/end2end/logger_load_before_grpc_lib_test.rb +++ b/src/ruby/end2end/logger_load_before_grpc_lib_test.rb @@ -18,6 +18,8 @@ this_dir = File.expand_path(File.dirname(__FILE__)) grpc_lib_dir = File.join(File.dirname(this_dir), 'lib') $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) +require_relative './sanity_check_dlopen' + def main fail('GRPC constant loaded before expected') if Object.const_defined?(:GRPC) require 'grpc/logconfig' diff --git a/src/ruby/end2end/prefork_postfork_loop_test.rb b/src/ruby/end2end/prefork_postfork_loop_test.rb index a09f85bcc77..0613b374b06 100755 --- a/src/ruby/end2end/prefork_postfork_loop_test.rb +++ b/src/ruby/end2end/prefork_postfork_loop_test.rb @@ -24,6 +24,7 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' require 'grpc' require 'end2end_common' diff --git a/src/ruby/end2end/prefork_without_using_grpc_test.rb b/src/ruby/end2end/prefork_without_using_grpc_test.rb index f6791b27ad6..ec6736a9031 100755 --- a/src/ruby/end2end/prefork_without_using_grpc_test.rb +++ b/src/ruby/end2end/prefork_without_using_grpc_test.rb @@ -24,6 +24,7 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' require 'grpc' require 'end2end_common' diff --git a/src/ruby/end2end/sanity_check_dlopen.rb b/src/ruby/end2end/sanity_check_dlopen.rb new file mode 100755 index 00000000000..e536100bad9 --- /dev/null +++ b/src/ruby/end2end/sanity_check_dlopen.rb @@ -0,0 +1,74 @@ +#!/usr/bin/env ruby +# +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Some tests are flaking by failing to dlopen grpc. Perform some sanity checks. +this_dir = File.expand_path(File.dirname(__FILE__)) +grpc_bin_dir = File.join(File.join(File.dirname(this_dir), 'lib'), 'grpc') +grpc_c_sha256_path = File.join(grpc_bin_dir, 'grpc_c_sha256') +grpc_c_so_path = if RUBY_PLATFORM =~ /darwin/ + File.join(grpc_bin_dir, 'grpc_c.bundle') + else + File.join(grpc_bin_dir, 'grpc_c.so') + end + +require 'digest' + +# first try to detect corruption b/t the build and now +actual_sha256 = Digest::SHA256.file(grpc_c_so_path).hexdigest +expected_sha256 = File.read(grpc_c_sha256_path).chomp +raise "detected corruption in #{grpc_c_so_path}: sha256: |#{actual_sha256}| != expected sha256: |#{expected_sha256}|" if actual_sha256 != expected_sha256 +STDERR.puts "verified sha256 of #{grpc_c_so_path}" + +def try_command(command) + STDERR.puts "==== run |#{command}| BEGIN ====" + output = `#{command} || true` + STDERR.puts output + STDERR.puts "==== run |#{command}| DONE ====" +end + +try_command('vm_stat') +try_command('free') +try_command('ulimit -v') + +# sanity check that we can load grpc in a child process, log things like available +# memory on the off chance we might be low. +pid = fork do + STDERR.puts "==== sanity check child process BEGIN ====" + def dump(file_path) + STDERR.puts "==== dump file: #{file_path} BEGIN ====" + if File.exist?(file_path) + File.open(file_path, 'r') do |file| + file.each_line do |line| + puts line + end + end + else + STDERR.puts "file: #{file_path} does not exist" + end + STDERR.puts "==== dump file: #{file_path} DONE ====" + end + dump("/proc/#{Process.pid}/limits") + dump("/proc/#{Process.pid}/status") + STDERR.puts "==== sanity check require grpc in child process BEGIN =====" + require 'grpc' + STDERR.puts "==== sanity check require grpc in child process DONE =====" + dump("/proc/#{Process.pid}/limits") + dump("/proc/#{Process.pid}/status") + STDERR.puts "==== sanity check child process DONE ====" +end +_, status = Process.wait2(pid) +fail "sanity check require grpc in child process FAILED exit code #{status.exitstatus}" unless status.success? +STDERR.puts "==== sanity check require grpc in child process SUCCESS =====" diff --git a/src/ruby/end2end/secure_fork_test.rb b/src/ruby/end2end/secure_fork_test.rb index d6af2fb2024..a37087913de 100755 --- a/src/ruby/end2end/secure_fork_test.rb +++ b/src/ruby/end2end/secure_fork_test.rb @@ -24,6 +24,7 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' require 'grpc' require 'end2end_common' diff --git a/src/ruby/end2end/simple_fork_test.rb b/src/ruby/end2end/simple_fork_test.rb index ca5e867457c..999dd378567 100755 --- a/src/ruby/end2end/simple_fork_test.rb +++ b/src/ruby/end2end/simple_fork_test.rb @@ -24,6 +24,7 @@ $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) $LOAD_PATH.unshift(protos_lib_dir) unless $LOAD_PATH.include?(protos_lib_dir) $LOAD_PATH.unshift(this_dir) unless $LOAD_PATH.include?(this_dir) +require 'sanity_check_dlopen' require 'grpc' require 'end2end_common' diff --git a/src/ruby/end2end/status_codes_load_before_grpc_lib_test.rb b/src/ruby/end2end/status_codes_load_before_grpc_lib_test.rb index 010c94cb91b..4eca92073d6 100755 --- a/src/ruby/end2end/status_codes_load_before_grpc_lib_test.rb +++ b/src/ruby/end2end/status_codes_load_before_grpc_lib_test.rb @@ -18,6 +18,8 @@ this_dir = File.expand_path(File.dirname(__FILE__)) grpc_lib_dir = File.join(File.dirname(this_dir), 'lib') $LOAD_PATH.unshift(grpc_lib_dir) unless $LOAD_PATH.include?(grpc_lib_dir) +require_relative './sanity_check_dlopen' + def main fail('GRPC constant loaded before expected') if Object.const_defined?(:GRPC) require 'grpc/core/status_codes' diff --git a/tools/run_tests/helper_scripts/build_ruby.sh b/tools/run_tests/helper_scripts/build_ruby.sh index cbf7a3cb191..2d1c8b51925 100755 --- a/tools/run_tests/helper_scripts/build_ruby.sh +++ b/tools/run_tests/helper_scripts/build_ruby.sh @@ -36,6 +36,18 @@ if [ "$SYSTEM" == "Darwin" ]; then fi bundle exec rake compile +# Log stuff and save a hash of the binary verify later at test runtime, in order +# to detect corruption. +if [ "$SYSTEM" == "Darwin" ]; then + ls -l src/ruby/lib/grpc/grpc_c.bundle + file src/ruby/lib/grpc/grpc_c.bundle + shasum -a 256 src/ruby/lib/grpc/grpc_c.bundle | awk '{print $1}' > src/ruby/lib/grpc/grpc_c_sha256 +else + ls -l src/ruby/lib/grpc/grpc_c.so + file src/ruby/lib/grpc/grpc_c.so + sha256sum src/ruby/lib/grpc/grpc_c.so | awk '{print $1}' > src/ruby/lib/grpc/grpc_c_sha256 +fi + # build grpc_ruby_plugin mkdir -p cmake/build pushd cmake/build