Merge pull request #8850 from haberman/rubybuilder
Move DSL implementation from C to pure Rubypull/8854/head
Joshua Haberman
3 years ago
committed by
GitHub
commit
88e6a36562
9 changed files with 521 additions and 1419 deletions
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,439 @@ |
|||||||
|
#!/usr/bin/ruby |
||||||
|
# |
||||||
|
# Code that implements the DSL for defining proto messages. |
||||||
|
|
||||||
|
require 'google/protobuf/descriptor_pb' |
||||||
|
|
||||||
|
module Google |
||||||
|
module Protobuf |
||||||
|
module Internal |
||||||
|
class Builder |
||||||
|
def initialize(pool) |
||||||
|
@pool = pool |
||||||
|
@default_file = nil # Constructed lazily |
||||||
|
end |
||||||
|
|
||||||
|
def add_file(name, options={}, &block) |
||||||
|
builder = FileBuilder.new(@pool, name, options) |
||||||
|
builder.instance_eval(&block) |
||||||
|
internal_add_file(builder) |
||||||
|
end |
||||||
|
|
||||||
|
def add_message(name, &block) |
||||||
|
internal_default_file.add_message(name, &block) |
||||||
|
end |
||||||
|
|
||||||
|
def add_enum(name, &block) |
||||||
|
internal_default_file.add_enum(name, &block) |
||||||
|
end |
||||||
|
|
||||||
|
# ---- Internal methods, not part of the DSL ---- |
||||||
|
|
||||||
|
def build |
||||||
|
if @default_file |
||||||
|
internal_add_file(@default_file) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
private def internal_add_file(file_builder) |
||||||
|
proto = file_builder.build |
||||||
|
serialized = Google::Protobuf::FileDescriptorProto.encode(proto) |
||||||
|
@pool.add_serialized_file(serialized) |
||||||
|
end |
||||||
|
|
||||||
|
private def internal_default_file |
||||||
|
@default_file ||= FileBuilder.new(@pool, "ruby_default_file.proto") |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
class FileBuilder |
||||||
|
def initialize(pool, name, options={}) |
||||||
|
@pool = pool |
||||||
|
@file_proto = Google::Protobuf::FileDescriptorProto.new( |
||||||
|
name: name, |
||||||
|
syntax: options.fetch(:syntax, "proto3") |
||||||
|
) |
||||||
|
end |
||||||
|
|
||||||
|
def add_message(name, &block) |
||||||
|
builder = MessageBuilder.new(name, self, @file_proto) |
||||||
|
builder.instance_eval(&block) |
||||||
|
builder.internal_add_synthetic_oneofs |
||||||
|
end |
||||||
|
|
||||||
|
def add_enum(name, &block) |
||||||
|
EnumBuilder.new(name, @file_proto).instance_eval(&block) |
||||||
|
end |
||||||
|
|
||||||
|
# ---- Internal methods, not part of the DSL ---- |
||||||
|
|
||||||
|
# These methods fix up the file descriptor to account for differences |
||||||
|
# between the DSL and FileDescriptorProto. |
||||||
|
|
||||||
|
# The DSL can omit a package name; here we infer what the package is if |
||||||
|
# was not specified. |
||||||
|
def infer_package(names) |
||||||
|
# Package is longest common prefix ending in '.', if any. |
||||||
|
if not names.empty? |
||||||
|
min, max = names.minmax |
||||||
|
last_common_dot = nil |
||||||
|
min.size.times { |i| |
||||||
|
if min[i] != max[i] then break end |
||||||
|
if min[i] == "." then last_common_dot = i end |
||||||
|
} |
||||||
|
if last_common_dot |
||||||
|
return min.slice(0, last_common_dot) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
nil |
||||||
|
end |
||||||
|
|
||||||
|
def rewrite_enum_default(field) |
||||||
|
if field.type != :TYPE_ENUM or !field.has_default_value? or !field.has_type_name? |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
value = field.default_value |
||||||
|
type_name = field.type_name |
||||||
|
|
||||||
|
if value.empty? or value[0].ord < "0".ord or value[0].ord > "9".ord |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
if type_name.empty? || type_name[0] != "." |
||||||
|
return |
||||||
|
end |
||||||
|
|
||||||
|
type_name = type_name[1..-1] |
||||||
|
as_int = Integer(value) rescue return |
||||||
|
|
||||||
|
enum_desc = @pool.lookup(type_name) |
||||||
|
if enum_desc.is_a?(Google::Protobuf::EnumDescriptor) |
||||||
|
# Enum was defined in a previous file. |
||||||
|
name = enum_desc.lookup_value(as_int) |
||||||
|
if name |
||||||
|
# Update the default value in the proto. |
||||||
|
field.default_value = name |
||||||
|
end |
||||||
|
else |
||||||
|
# See if enum was defined in this file. |
||||||
|
@file_proto.enum_type.each { |enum_proto| |
||||||
|
if enum_proto.name == type_name |
||||||
|
enum_proto.value.each { |enum_value_proto| |
||||||
|
if enum_value_proto.number == as_int |
||||||
|
# Update the default value in the proto. |
||||||
|
field.default_value = enum_value_proto.name |
||||||
|
return |
||||||
|
end |
||||||
|
} |
||||||
|
# We found the right enum, but no value matched. |
||||||
|
return |
||||||
|
end |
||||||
|
} |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
# Historically we allowed enum defaults to be specified as a number. |
||||||
|
# In retrospect this was a mistake as descriptors require defaults to |
||||||
|
# be specified as a label. This can make a difference if multiple |
||||||
|
# labels have the same number. |
||||||
|
# |
||||||
|
# Here we do a pass over all enum defaults and rewrite numeric defaults |
||||||
|
# by looking up their labels. This is complicated by the fact that the |
||||||
|
# enum definition can live in either the symtab or the file_proto. |
||||||
|
# |
||||||
|
# We take advantage of the fact that this is called *before* enums or |
||||||
|
# messages are nested in other messages, so we only have to iterate |
||||||
|
# one level deep. |
||||||
|
def rewrite_enum_defaults |
||||||
|
@file_proto.message_type.each { |msg| |
||||||
|
msg.field.each { |field| |
||||||
|
rewrite_enum_default(field) |
||||||
|
} |
||||||
|
} |
||||||
|
end |
||||||
|
|
||||||
|
# We have to do some relatively complicated logic here for backward |
||||||
|
# compatibility. |
||||||
|
# |
||||||
|
# In descriptor.proto, messages are nested inside other messages if that is |
||||||
|
# what the original .proto file looks like. For example, suppose we have this |
||||||
|
# foo.proto: |
||||||
|
# |
||||||
|
# package foo; |
||||||
|
# message Bar { |
||||||
|
# message Baz {} |
||||||
|
# } |
||||||
|
# |
||||||
|
# The descriptor for this must look like this: |
||||||
|
# |
||||||
|
# file { |
||||||
|
# name: "test.proto" |
||||||
|
# package: "foo" |
||||||
|
# message_type { |
||||||
|
# name: "Bar" |
||||||
|
# nested_type { |
||||||
|
# name: "Baz" |
||||||
|
# } |
||||||
|
# } |
||||||
|
# } |
||||||
|
# |
||||||
|
# However, the Ruby generated code has always generated messages in a flat, |
||||||
|
# non-nested way: |
||||||
|
# |
||||||
|
# Google::Protobuf::DescriptorPool.generated_pool.build do |
||||||
|
# add_message "foo.Bar" do |
||||||
|
# end |
||||||
|
# add_message "foo.Bar.Baz" do |
||||||
|
# end |
||||||
|
# end |
||||||
|
# |
||||||
|
# Here we need to do a translation where we turn this generated code into the |
||||||
|
# above descriptor. We need to infer that "foo" is the package name, and not |
||||||
|
# a message itself. */ |
||||||
|
|
||||||
|
def split_parent_name(msg_or_enum) |
||||||
|
name = msg_or_enum.name |
||||||
|
idx = name.rindex(?.) |
||||||
|
if idx |
||||||
|
return name[0...idx], name[idx+1..-1] |
||||||
|
else |
||||||
|
return nil, name |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
def get_parent_msg(msgs_by_name, name, parent_name) |
||||||
|
parent_msg = msgs_by_name[parent_name] |
||||||
|
if parent_msg.nil? |
||||||
|
raise "To define name #{name}, there must be a message named #{parent_name} to enclose it" |
||||||
|
end |
||||||
|
return parent_msg |
||||||
|
end |
||||||
|
|
||||||
|
def fix_nesting |
||||||
|
# Calculate and update package. |
||||||
|
msgs_by_name = @file_proto.message_type.map { |msg| [msg.name, msg] }.to_h |
||||||
|
enum_names = @file_proto.enum_type.map { |enum_proto| enum_proto.name } |
||||||
|
|
||||||
|
package = infer_package(msgs_by_name.keys + enum_names) |
||||||
|
if package |
||||||
|
@file_proto.package = package |
||||||
|
end |
||||||
|
|
||||||
|
# Update nesting based on package. |
||||||
|
final_msgs = Google::Protobuf::RepeatedField.new(:message, Google::Protobuf::DescriptorProto) |
||||||
|
final_enums = Google::Protobuf::RepeatedField.new(:message, Google::Protobuf::EnumDescriptorProto) |
||||||
|
|
||||||
|
# Note: We don't iterate over msgs_by_name.values because we want to |
||||||
|
# preserve order as listed in the DSL. |
||||||
|
@file_proto.message_type.each { |msg| |
||||||
|
parent_name, msg.name = split_parent_name(msg) |
||||||
|
if parent_name == package |
||||||
|
final_msgs << msg |
||||||
|
else |
||||||
|
get_parent_msg(msgs_by_name, msg.name, parent_name).nested_type << msg |
||||||
|
end |
||||||
|
} |
||||||
|
|
||||||
|
@file_proto.enum_type.each { |enum| |
||||||
|
parent_name, enum.name = split_parent_name(enum) |
||||||
|
if parent_name == package |
||||||
|
final_enums << enum |
||||||
|
else |
||||||
|
get_parent_msg(msgs_by_name, enum.name, parent_name).enum_type << enum |
||||||
|
end |
||||||
|
} |
||||||
|
|
||||||
|
@file_proto.message_type = final_msgs |
||||||
|
@file_proto.enum_type = final_enums |
||||||
|
end |
||||||
|
|
||||||
|
def internal_file_proto |
||||||
|
@file_proto |
||||||
|
end |
||||||
|
|
||||||
|
def build |
||||||
|
rewrite_enum_defaults |
||||||
|
fix_nesting |
||||||
|
return @file_proto |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
class MessageBuilder |
||||||
|
def initialize(name, file_builder, file_proto) |
||||||
|
@file_builder = file_builder |
||||||
|
@msg_proto = Google::Protobuf::DescriptorProto.new( |
||||||
|
:name => name |
||||||
|
) |
||||||
|
file_proto.message_type << @msg_proto |
||||||
|
end |
||||||
|
|
||||||
|
def optional(name, type, number, type_class=nil, options=nil) |
||||||
|
internal_add_field(:LABEL_OPTIONAL, name, type, number, type_class, options) |
||||||
|
end |
||||||
|
|
||||||
|
def proto3_optional(name, type, number, type_class=nil, options=nil) |
||||||
|
internal_add_field(:LABEL_OPTIONAL, name, type, number, type_class, options, |
||||||
|
proto3_optional: true) |
||||||
|
end |
||||||
|
|
||||||
|
def required(name, type, number, type_class=nil, options=nil) |
||||||
|
internal_add_field(:LABEL_REQUIRED, name, type, number, type_class, options) |
||||||
|
end |
||||||
|
|
||||||
|
def repeated(name, type, number, type_class = nil) |
||||||
|
internal_add_field(:LABEL_REPEATED, name, type, number, type_class, nil) |
||||||
|
end |
||||||
|
|
||||||
|
def oneof(name, &block) |
||||||
|
OneofBuilder.new(name, self).instance_eval(&block) |
||||||
|
end |
||||||
|
|
||||||
|
# Defines a new map field on this message type with the given key and |
||||||
|
# value types, tag number, and type class (for message and enum value |
||||||
|
# types). The key type must be :int32/:uint32/:int64/:uint64, :bool, or |
||||||
|
# :string. The value type type must be a Ruby symbol (as accepted by |
||||||
|
# FieldDescriptor#type=) and the type_class must be a string, if |
||||||
|
# present (as accepted by FieldDescriptor#submsg_name=). |
||||||
|
def map(name, key_type, value_type, number, value_type_class = nil) |
||||||
|
if key_type == :float or key_type == :double or key_type == :enum or |
||||||
|
key_type == :message |
||||||
|
raise ArgError, "Not an acceptable key type: " + key_type |
||||||
|
end |
||||||
|
entry_name = "#{@msg_proto.name}_MapEntry_#{name}" |
||||||
|
|
||||||
|
@file_builder.add_message entry_name do |
||||||
|
optional :key, key_type, 1 |
||||||
|
optional :value, value_type, 2, value_type_class |
||||||
|
end |
||||||
|
options = @file_builder.internal_file_proto.message_type.last.options ||= MessageOptions.new |
||||||
|
options.map_entry = true |
||||||
|
repeated name, :message, number, entry_name |
||||||
|
end |
||||||
|
|
||||||
|
# ---- Internal methods, not part of the DSL ---- |
||||||
|
|
||||||
|
def internal_add_synthetic_oneofs |
||||||
|
# We have to build a set of all names, to ensure that synthetic oneofs |
||||||
|
# are not creating conflicts |
||||||
|
names = {} |
||||||
|
@msg_proto.field.each { |field| names[field.name] = true } |
||||||
|
@msg_proto.oneof_decl.each { |oneof| names[oneof.name] = true } |
||||||
|
|
||||||
|
@msg_proto.field.each { |field| |
||||||
|
if field.proto3_optional |
||||||
|
# Prepend '_' until we are no longer conflicting. |
||||||
|
oneof_name = field.name |
||||||
|
while names[oneof_name] |
||||||
|
oneof_name = "_" + oneof_name |
||||||
|
end |
||||||
|
names[oneof_name] = true |
||||||
|
field.oneof_index = @msg_proto.oneof_decl.size |
||||||
|
@msg_proto.oneof_decl << Google::Protobuf::OneofDescriptorProto.new( |
||||||
|
name: oneof_name |
||||||
|
) |
||||||
|
end |
||||||
|
} |
||||||
|
end |
||||||
|
|
||||||
|
def internal_add_field(label, name, type, number, type_class, options, |
||||||
|
oneof_index: nil, proto3_optional: false) |
||||||
|
# Allow passing either: |
||||||
|
# - (name, type, number, options) or |
||||||
|
# - (name, type, number, type_class, options) |
||||||
|
if options.nil? and type_class.instance_of?(Hash) |
||||||
|
options = type_class; |
||||||
|
type_class = nil; |
||||||
|
end |
||||||
|
|
||||||
|
field_proto = Google::Protobuf::FieldDescriptorProto.new( |
||||||
|
:label => label, |
||||||
|
:name => name, |
||||||
|
:type => ("TYPE_" + type.to_s.upcase).to_sym, |
||||||
|
:number => number |
||||||
|
) |
||||||
|
|
||||||
|
if type_class |
||||||
|
# Make it an absolute type name by prepending a dot. |
||||||
|
field_proto.type_name = "." + type_class |
||||||
|
end |
||||||
|
|
||||||
|
if oneof_index |
||||||
|
field_proto.oneof_index = oneof_index |
||||||
|
end |
||||||
|
|
||||||
|
if proto3_optional |
||||||
|
field_proto.proto3_optional = true |
||||||
|
end |
||||||
|
|
||||||
|
if options |
||||||
|
if options.key?(:default) |
||||||
|
default = options[:default] |
||||||
|
if !default.instance_of?(String) |
||||||
|
# Call #to_s since all defaults are strings in the descriptor. |
||||||
|
default = default.to_s |
||||||
|
end |
||||||
|
# XXX: we should be C-escaping bytes defaults. |
||||||
|
field_proto.default_value = default.dup.force_encoding("UTF-8") |
||||||
|
end |
||||||
|
if options.key?(:json_name) |
||||||
|
field_proto.json_name = options[:json_name] |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
@msg_proto.field << field_proto |
||||||
|
end |
||||||
|
|
||||||
|
def internal_msg_proto |
||||||
|
@msg_proto |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
class OneofBuilder |
||||||
|
def initialize(name, msg_builder) |
||||||
|
@msg_builder = msg_builder |
||||||
|
oneof_proto = Google::Protobuf::OneofDescriptorProto.new( |
||||||
|
:name => name |
||||||
|
) |
||||||
|
msg_proto = msg_builder.internal_msg_proto |
||||||
|
@oneof_index = msg_proto.oneof_decl.size |
||||||
|
msg_proto.oneof_decl << oneof_proto |
||||||
|
end |
||||||
|
|
||||||
|
def optional(name, type, number, type_class=nil, options=nil) |
||||||
|
@msg_builder.internal_add_field( |
||||||
|
:LABEL_OPTIONAL, name, type, number, type_class, options, |
||||||
|
oneof_index: @oneof_index) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
class EnumBuilder |
||||||
|
def initialize(name, file_proto) |
||||||
|
@enum_proto = Google::Protobuf::EnumDescriptorProto.new( |
||||||
|
:name => name |
||||||
|
) |
||||||
|
file_proto.enum_type << @enum_proto |
||||||
|
end |
||||||
|
|
||||||
|
def value(name, number) |
||||||
|
enum_value_proto = Google::Protobuf::EnumValueDescriptorProto.new( |
||||||
|
name: name, |
||||||
|
number: number |
||||||
|
) |
||||||
|
@enum_proto.value << enum_value_proto |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
end |
||||||
|
|
||||||
|
# Re-open the class (the rest of the class is implemented in C) |
||||||
|
class DescriptorPool |
||||||
|
def build(&block) |
||||||
|
builder = Internal::Builder.new(self) |
||||||
|
builder.instance_eval(&block) |
||||||
|
builder.build |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
Loading…
Reference in new issue