Merge branch 'master' into 3.0.0-GA

We need to pick up some of the last-minute Ruby fixes.

Makefile.am

@@ -705,6 +705,7 @@ ruby_EXTRA_DIST=                                                             \
   ruby/tests/repeated_field_test.rb                                          \
   ruby/tests/stress.rb                                                       \
   ruby/tests/generated_code.proto                                            \
+  ruby/tests/test_import.proto                                               \
   ruby/tests/generated_code_test.rb                                          \
   ruby/travis-test.sh
 

conformance/conformance_ruby.rb

@@ -30,7 +30,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-require 'conformance'
+require 'conformance_pb'
 
 $test_count = 0
 $verbose = false

protoc-artifacts/README.md

@@ -102,6 +102,25 @@ When you have done deployment for all platforms, go to
 https://oss.sonatype.org/#stagingRepositories, verify that the staging
 repository has all the binaries, close and release this repository.
 
+## Upload zip packages to github release page.
+After uploading protoc artifacts to Maven Central repository, run the
+build-zip.sh script to bulid zip packages for these protoc binaries
+and upload these zip packages to the download section of the github
+release. For example:
+```
+$ ./build-zip.sh 3.0.0-beta-4
+```
+The above command will create 5 zip files:
+```
+dist/protoc-3.0.0-beta-4-win32.zip
+dist/protoc-3.0.0-beta-4-osx-x86_32.zip
+dist/protoc-3.0.0-beta-4-osx-x86_64.zip
+dist/protoc-3.0.0-beta-4-linux-x86_32.zip
+dist/protoc-3.0.0-beta-4-linux-x86_64.zip
+```
+Before running the script, make sure the artifacts are accessible from:
+http://repo1.maven.org/maven2/com/google/protobuf/protoc/
+
 ### Tips for deploying on Linux
 We build on Centos 6.6 to provide a good compatibility for not very new
 systems. We have provided a ``Dockerfile`` under this directory to build the

protoc-artifacts/build-zip.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+if [ $# -eq 0 ]; then
+  cat <<EOF
+Usage: $0 <VERSION_NUMBER>
+
+Example:
+  $ $0 3.0.0-beta-4
+
+This script will download pre-built protoc binaries from maven repository
+and package them with well-known type .proto files to create .zip packages
+suitable to be included in the github release page. Each invocation will
+create 5 zip packages:
+  dist/protoc-<VERSION_NUMBER>-win32.zip
+  dist/protoc-<VERSION_NUMBER>-osx-x86_32.zip
+  dist/protoc-<VERSION_NUMBER>-osx-x86_64.zip
+  dist/protoc-<VERSION_NUMBER>-linux-x86_32.zip
+  dist/protoc-<VERSION_NUMBER>-linux-x86_64.zip
+EOF
+  exit 1
+fi
+
+VERSION_NUMBER=$1
+
+# <zip file name> <binary file name> pairs.
+declare -a FILE_NAMES=( \
+  win32.zip windows-x86_32.exe \
+  osx-x86_32.zip osx-x86_32.exe \
+  osx-x86_64.zip osx-x86_64.exe \
+  linux-x86_32.zip linux-x86_32.exe \
+  linux-x86_64.zip linux-x86_64.exe \
+)
+
+# List of all well-known types to be included.
+declare -a WELL_KNOWN_TYPES=(           \
+  google/protobuf/descriptor.proto      \
+  google/protobuf/any.proto             \
+  google/protobuf/api.proto             \
+  google/protobuf/duration.proto        \
+  google/protobuf/empty.proto           \
+  google/protobuf/field_mask.proto      \
+  google/protobuf/source_context.proto  \
+  google/protobuf/struct.proto          \
+  google/protobuf/timestamp.proto       \
+  google/protobuf/type.proto            \
+  google/protobuf/wrappers.proto        \
+  google/protobuf/compiler/plugin.proto \
+)
+
+set -e
+
+# A temporary working directory to put all files.
+DIR=$(mktemp -d)
+
+# Copy over well-known types.
+mkdir -p ${DIR}/include/google/protobuf/compiler
+for PROTO in ${WELL_KNOWN_TYPES[@]}; do
+  cp -f ../src/${PROTO} ${DIR}/include/${PROTO}
+done
+
+# Create a readme file.
+cat <<EOF > ${DIR}/readme.txt
+Protocol Buffers - Google's data interchange format
+Copyright 2008 Google Inc.
+https://developers.google.com/protocol-buffers/
+
+This package contains a precompiled binary version of the protocol buffer
+compiler (protoc). This binary is intended for users who want to use Protocol
+Buffers in languages other than C++ but do not want to compile protoc
+themselves. To install, simply place this binary somewhere in your PATH.
+
+Please refer to our official github site for more installation instructions:
+  https://github.com/google/protobuf
+EOF
+
+mkdir -p dist
+mkdir -p ${DIR}/bin
+# Create a zip file for each binary.
+for((i=0;i<${#FILE_NAMES[@]};i+=2));do
+  ZIP_NAME=${FILE_NAMES[$i]}
+  BINARY_NAME=${FILE_NAMES[$(($i+1))]}
+  BINARY_URL=http://repo1.maven.org/maven2/com/google/protobuf/protoc/${VERSION_NUMBER}/protoc-${VERSION_NUMBER}-${BINARY_NAME}
+  if ! wget ${BINARY_URL} -O ${DIR}/bin/protoc &> /dev/null; then
+    echo "[ERROR] Failed to download ${BINARY_URL}" >&2
+    echo "[ERROR] Skipped protoc-${VERSION_NAME}-${ZIP_NAME}" >&2
+    continue
+  fi
+  TARGET_ZIP_FILE=`pwd`/dist/protoc-${VERSION_NUMBER}-${ZIP_NAME}
+  pushd $DIR &> /dev/null
+  chmod +x bin/protoc
+  zip -r ${TARGET_ZIP_FILE} include bin readme.txt &> /dev/null
+  popd &> /dev/null
+  echo "[INFO] Successfully created ${TARGET_ZIP_FILE}"
+done

ruby/Rakefile

@@ -31,7 +31,7 @@ genproto_output = []
 unless ENV['IN_DOCKER'] == 'true'
   well_known_protos.each do |proto_file|
     input_file = "../src/" + proto_file
-    output_file = "lib/" + proto_file.sub(/\.proto$/, ".rb")
+    output_file = "lib/" + proto_file.sub(/\.proto$/, "_pb.rb")
     genproto_output << output_file
     file output_file => input_file do |file_task|
       sh "../src/protoc -I../src --ruby_out=lib #{input_file}"
@@ -80,10 +80,15 @@ end
 
 # Proto for tests.
 genproto_output << "tests/generated_code.rb"
+genproto_output << "tests/test_import.rb"
 file "tests/generated_code.rb" => "tests/generated_code.proto" do |file_task|
   sh "../src/protoc --ruby_out=. tests/generated_code.proto"
 end
 
+file "tests/test_import.rb" => "tests/test_import.proto" do |file_task|
+  sh "../src/protoc --ruby_out=. tests/test_import.proto"
+end
+
 task :genproto => genproto_output
 
 task :clean do

ruby/tests/generated_code.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package A.B.C;
+package a.b.c;
 
 message TestMessage {
   int32 optional_int32 = 1;

ruby/tests/generated_code_test.rb

@@ -3,7 +3,8 @@
 # generated_code.rb is in the same directory as this test.
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
 
-require 'generated_code'
+require 'generated_code_pb'
+require 'test_import_pb'
 require 'test/unit'
 
 class GeneratedCodeTest < Test::Unit::TestCase
@@ -13,5 +14,6 @@ class GeneratedCodeTest < Test::Unit::TestCase
     # successfully creates message definitions and classes, not to test every
     # aspect of the extension (basic.rb is for that).
     m = A::B::C::TestMessage.new()
+    m2 = FooBar::TestImportedMessage.new()
   end
 end

ruby/tests/test_import.proto

@@ -0,0 +1,5 @@
+syntax = "proto3";
+
+package foo_bar;
+
+message TestImportedMessage {}

src/google/protobuf/compiler/ruby/ruby_generator.cc

@@ -48,7 +48,7 @@ namespace ruby {
 
 // Forward decls.
 std::string IntToString(int32 value);
-std::string StripDotProto(const std::string& proto_file);
+std::string GetRequireName(const std::string& proto_file);
 std::string LabelForField(google::protobuf::FieldDescriptor* field);
 std::string TypeName(google::protobuf::FieldDescriptor* field);
 void GenerateMessage(const google::protobuf::Descriptor* message,
@@ -70,13 +70,13 @@ std::string IntToString(int32 value) {
   return os.str();
 }
 
-std::string StripDotProto(const std::string& proto_file) {
+std::string GetRequireName(const std::string& proto_file) {
   int lastindex = proto_file.find_last_of(".");
-  return proto_file.substr(0, lastindex);
+  return proto_file.substr(0, lastindex) + "_pb";
 }
 
 std::string GetOutputFilename(const std::string& proto_file) {
-  return StripDotProto(proto_file) + ".rb";
+  return GetRequireName(proto_file) + ".rb";
 }
 
 std::string LabelForField(const google::protobuf::FieldDescriptor* field) {
@@ -237,15 +237,52 @@ void GenerateEnum(const google::protobuf::EnumDescriptor* en,
     "end\n");
 }
 
-// Module names, class names, and enum value names need to be Ruby constants,
-// which must start with a capital letter.
+// Locale-agnostic utility functions.
+bool IsLower(char ch) { return ch >= 'a' && ch <= 'z'; }
+
+bool IsUpper(char ch) { return ch >= 'A' && ch <= 'Z'; }
+
+bool IsAlpha(char ch) { return IsLower(ch) || IsUpper(ch); }
+
+char ToUpper(char ch) { return IsLower(ch) ? (ch - 'a' + 'A') : ch; }
+
+
+// Package names in protobuf are snake_case by convention, but Ruby module
+// names must be PascalCased.
+//
+//   foo_bar_baz -> FooBarBaz
+std::string PackageToModule(const std::string& name) {
+  bool next_upper = true;
+  std::string result;
+  result.reserve(name.size());
+
+  for (int i = 0; i < name.size(); i++) {
+    if (name[i] == '_') {
+      next_upper = true;
+    } else {
+      if (next_upper) {
+        result.push_back(ToUpper(name[i]));
+      } else {
+        result.push_back(name[i]);
+      }
+      next_upper = false;
+    }
+  }
+
+  return result;
+}
+
+// Class and enum names in protobuf should be PascalCased by convention, but
+// since there is nothing enforcing this we need to ensure that they are valid
+// Ruby constants.  That mainly means making sure that the first character is
+// an upper-case letter.
 std::string RubifyConstant(const std::string& name) {
   std::string ret = name;
   if (!ret.empty()) {
-    if (ret[0] >= 'a' && ret[0] <= 'z') {
+    if (IsLower(ret[0])) {
       // If it starts with a lowercase letter, capitalize it.
-      ret[0] = ret[0] - 'a' + 'A';
-    } else if (ret[0] < 'A' || ret[0] > 'Z') {
+      ret[0] = ToUpper(ret[0]);
+    } else if (!IsAlpha(ret[0])) {
       // Otherwise (e.g. if it begins with an underscore), we need to come up
       // with some prefix that starts with a capital letter. We could be smarter
       // here, e.g. try to strip leading underscores, but this may cause other
@@ -254,6 +291,7 @@ std::string RubifyConstant(const std::string& name) {
       ret = "PB_" + ret;
     }
   }
+
   return ret;
 }
 
@@ -314,7 +352,7 @@ int GeneratePackageModules(
       component = package_name.substr(0, dot_index);
       package_name = package_name.substr(dot_index + 1);
     }
-    component = RubifyConstant(component);
+    component = PackageToModule(component);
     printer->Print(
       "module $name$\n",
       "name", component);
@@ -391,7 +429,7 @@ bool MaybeEmitDependency(const FileDescriptor* import,
     return true;
   } else {
     printer->Print(
-      "require '$name$'\n", "name", StripDotProto(import->name()));
+      "require '$name$'\n", "name", GetRequireName(import->name()));
     return true;
   }
 }

src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc

@@ -91,12 +91,12 @@ TEST(RubyGeneratorTest, GeneratorTest) {
   // Load the generated output and compare to the expected result.
   string output;
   GOOGLE_CHECK_OK(File::GetContents(
-      TestTempDir() + "/ruby_generated_code.rb",
+      TestTempDir() + "/ruby_generated_code_pb.rb",
       &output,
       true));
   string expected_output;
   GOOGLE_CHECK_OK(File::GetContents(
-      ruby_tests + "/ruby_generated_code.rb",
+      ruby_tests + "/ruby_generated_code_pb.rb",
       &expected_output,
       true));
   EXPECT_EQ(expected_output, output);