protobuf/tools/make_c_api.lua

--[[

  Code to generate a C API in:
    foo.proto -> foo.upb.h
                 foo.upb.c

  This code is evolving very quickly and so there are lots of little things
  that aren't perfect right now.  As it settles a little more, the code
  quality should improve.

--]]

local upb = require "upb"
local dump_cinit = require "dump_cinit"
local export = {}

local typemap = {
  [upb.TYPE_BOOL]     = "bool",
  [upb.TYPE_FLOAT]    = "float",
  [upb.TYPE_INT32]    = "int32_t",
  [upb.TYPE_UINT32]   = "uint32_t",
  [upb.TYPE_DOUBLE]   = "double",
  [upb.TYPE_INT64]    = "int64_t",
  [upb.TYPE_UINT64]   = "uint64_t",
  [upb.TYPE_STRING]   = "upb_stringview",
  [upb.TYPE_BYTES]    = "upb_stringview",
}

function strip_proto(filename)
  return string.gsub(filename, '%.proto$','')
end

local function join(...)
  return table.concat({...}, ".")
end

local function to_cident(...)
  return string.gsub(join(...), "[%./]", "_")
end

local function to_preproc(...)
  return string.upper(to_cident(...))
end

-- Strips away last path element, ie:
--   foo.Bar.Baz -> foo.Bar
local function remove_name(name)
  local package_end = 0
  for i=1,string.len(name) do
    if string.byte(name, i) == string.byte(".", 1) then
      package_end = i - 1
    end
  end
  return string.sub(name, 1, package_end)
end

local function enum_value_symbol(enumdef, name)
  return to_cident(remove_name(enumdef:full_name())) .. "_" .. name
end

local function dump_enum_vals(enumdef, append)
  local enum_vals = {}

  for k, v in enumdef:values() do
    enum_vals[#enum_vals + 1] = {k, v}
  end

  table.sort(enum_vals, function(a, b) return a[2] < b[2] end)

  -- protobuf convention is that enum values are scoped at the level of the
  -- enum itself, to follow C++.  Ie, if you have the enum:
  -- message Foo {
  --   enum E {
  --     VAL1 = 1;
  --     VAL2 = 2;
  --   }
  -- }
  --
  -- The name of VAL1 is Foo.VAL1, not Foo.E.VAL1.
  --
  -- This seems a bit sketchy, but people often name their enum values
  -- accordingly, ie:
  --
  -- enum Foo {
  --   FOO_VAL1 = 1;
  --   FOO_VAL2 = 2;
  -- }
  --
  -- So if we don't respect this also, we end up with constants that look like:
  --
  --   GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_TYPE_TYPE_DOUBLE = 1
  --
  -- (notice the duplicated "TYPE").
  local cident = to_cident(remove_name(enumdef:full_name()))
  for i, pair in ipairs(enum_vals) do
    k, v = pair[1], pair[2]
    append('  %s = %d', enum_value_symbol(enumdef, k), v)
    if i == #enum_vals then
      append('\n')
    else
      append(',\n')
    end
  end
end

local function field_default(field)
  if field:type() == upb.TYPE_MESSAGE then
    return "NULL"
  elseif field:type() == upb.TYPE_STRING or
         field:type() == upb.TYPE_BYTES then
    local default = field:default() or ""
    return string.format('upb_stringview_make("%s", strlen("%s"))', field:default(), field:default())
  elseif field:type() == upb.TYPE_ENUM then
    return enum_value_symbol(field:subdef(), field:default())
  else
    return field:default();
  end
end

local function ctype(field, const)
  if const then
    const = "const "
  else
    const = ""
  end

  if field:label() == upb.LABEL_REPEATED then
    return const .. "upb_array*"
  elseif field:type() == upb.TYPE_MESSAGE then
    if field:containing_type():file() == field:subdef():file() then
      return const .. to_cident(field:subdef():full_name()) .. "*"
    else
      return const .. "struct " .. to_cident(field:subdef():full_name()) .. "*"
    end
  elseif field:type() == upb.TYPE_ENUM then
    return to_cident(field:subdef():full_name())
  else
    return typemap[field:type()] or "void*"
  end
end

local function emit_file_warning(filedef, append)
  append('/* This file was generated by upbc (the upb compiler) from the input\n')
  append(' * file:\n')
  append(' *\n')
  append(' *     %s\n', filedef:name())
  append(' *\n')
  append(' * Do not edit -- your changes will be discarded when the file is\n')
  append(' * regenerated. */\n\n')
end

local function field_layout_rank(field)
  -- Order:
  --   1, 2, 3. primitive fields (8, 4, 1 byte)
  --   4. string fields
  --   5. submessage fields
  --   6. repeated fields
  --
  -- This has the following nice properties:
  --
  --  1. padding alignment is (nearly) minimized.
  --  2. fields that might have defaults (1-4) are segregated
  --     from fields that are always zero-initialized (5-7).
  --
  -- We skip oneof fields, because they are emitted in a separate pass.
  local rank
  if field:containing_oneof() then
    rank = 100  -- These go last (actually we skip them).
  elseif field:label() == upb.LABEL_REPEATED then
    rank = 6
  elseif field:type() == upb.TYPE_MESSAGE then
    rank = 5
  elseif field:type() == upb.TYPE_STRING or field:type() == upb.TYPE_BYTES then
    rank = 4
  elseif field:type() == upb.TYPE_BOOL then
    rank = 3
  elseif field:type() == upb.TYPE_FLOAT or
         field:type() == upb.TYPE_INT32 or
         field:type() == upb.TYPE_UINT32 then
    rank = 2
  else
    rank = 1
  end

  -- Break ties with field number.
  return (rank * 2^29) + field:number()
end

local function has_hasbit(field)
  if field:containing_type():file():syntax() == upb.SYNTAX_PROTO2 then
    return field:label() ~= upb.LABEL_REPEATED and not field:containing_oneof()
  else
    return false
  end
end

local function write_h_file(filedef, append)
  emit_file_warning(filedef, append)
  local basename_preproc = to_preproc(filedef:name())
  append('#ifndef %s_UPB_H_\n', basename_preproc)
  append('#define %s_UPB_H_\n\n', basename_preproc)

  append('#include "upb/msg.h"\n\n')

  append('UPB_BEGIN_EXTERN_C\n\n')

  for msg in filedef:defs(upb.DEF_MSG) do
    -- TODO(haberman): forward declare C++ type names so we can use
    -- UPB_DECLARE_TYPE().
    local msgname = to_cident(msg:full_name())
    append('struct %s;\n', msgname)
    append('typedef struct %s %s;\n', msgname, msgname)
  end

  append("/* Enums */\n\n")
  for _, def in ipairs(sorted_defs(filedef:defs(upb.DEF_ENUM))) do
    local cident = to_cident(def:full_name())
    append('typedef enum {\n')
    dump_enum_vals(def, append)
    append('} %s;\n\n', cident)
  end

  for msg in filedef:defs(upb.DEF_MSG) do
    local msgname = to_cident(msg:full_name())
    append('/* %s */\n', msgname)
    append('extern const upb_msglayout_msginit_v1 %s_msginit;\n', msgname)
    append('%s *%s_new(upb_env *env);\n', msgname, msgname)
    append('%s *%s_parsenew(upb_stringview buf, upb_env *env);\n',
           msgname, msgname)
    append('char *%s_serialize(%s *msg, upb_env *env, size_t *len);\n',
           msgname, msgname)
    append('void %s_free(%s *msg, upb_env *env);\n', msgname, msgname)
    append('\n')

    append('/* getters. */\n')
    local setters, get_setters = dump_cinit.str_appender()
    for field in msg:fields() do
      local fieldname = to_cident(field:name())
      if field:type() == upb.TYPE_MESSAGE and
          field:subdef():file() ~= filedef then
        -- Forward declaration for message type declared in another file.
        append('struct %s;\n', to_cident(field:subdef():full_name()))
      end
      append('%s %s_%s(const %s *msg);\n',
             ctype(field, true), msgname, fieldname, msgname)
      setters('void %s_set_%s(%s *msg, %s value);\n',
              msgname, fieldname, msgname, ctype(field))
    end

    for oneof in msg:oneofs() do
      local fullname = to_cident(oneof:containing_type():full_name() .. "." .. oneof:name())
      append('typedef enum {\n')
      for field in oneof:fields() do
        append('  %s = %d,\n', fullname .. "_" .. field:name(), field:number())
      end
      append('  %s_NOT_SET = 0,\n', fullname)
      append('} %s_oneofcases;\n', fullname)
      append('%s_oneofcases %s_case(const %s *msg);\n', fullname, fullname, msgname)
    end

    append('\n')
    append('/* setters. */\n')
    append(get_setters())

    append('\n')
    append('\n')
  end

  append('UPB_END_EXTERN_C')

  append('\n')
  append('\n')

  append('#endif  /* %s_UPB_H_ */\n', basename_preproc)
end

local function write_c_file(filedef, hfilename, append)
  emit_file_warning(filedef, append)

  append('#include <stddef.h>\n')
  append('#include "upb/decode.h"\n')
  append('#include "upb/encode.h"\n')
  append('#include "upb/msg.h"\n')
  append('#include "upb/upb.h"\n')
  append('#include "%s"\n\n', hfilename)

  for dep in filedef:dependencies() do
    local outbase = strip_proto(dep:name())
    append('#include "%s.upb.h"\n', outbase)
  end

  append('\n')

  for msg in filedef:defs(upb.DEF_MSG) do
    local msgname = to_cident(msg:full_name())

    local fields_array_ref = "NULL"
    local submsgs_array_ref = "NULL"
    local oneofs_array_ref = "NULL"
    local field_count = 0
    local submsg_count = 0
    local submsg_set = {}
    local submsg_indexes = {}
    local hasbit_count = 0
    local hasbit_indexes = {}
    local oneof_count = 0
    local oneof_indexes = {}

    -- Create a layout order for oneofs.
    local oneofs_layout_order = {}
    for oneof in msg:oneofs() do
      table.insert(oneofs_layout_order, oneof)
    end
    table.sort(oneofs_layout_order, function(a, b)
      return a:name() < b:name()
    end)

    for _, oneof in ipairs(oneofs_layout_order) do
      oneof_indexes[oneof] = oneof_count
      oneof_count = oneof_count + 1
    end

    -- Create a layout order for fields.  We use this order for the struct and
    -- for offsets, but our list of fields we keep in field number order.
    local fields_layout_order = {}
    for field in msg:fields() do
      table.insert(fields_layout_order, field)
    end
    table.sort(fields_layout_order, function(a, b)
      return field_layout_rank(a) < field_layout_rank(b)
    end)

    -- Another sorted array in field number order.
    local fields_number_order = {}
    for field in msg:fields() do
      table.insert(fields_number_order, field)
    end
    table.sort(fields_number_order, function(a, b)
      return a:number() < b:number()
    end)

    append('struct %s {\n', msgname)

    -- Non-oneof fields.
    for _, field in ipairs(fields_layout_order) do
      field_count = field_count + 1

      if field:type() == upb.TYPE_MESSAGE then
        submsg_count = submsg_count + 1
        submsg_set[field:subdef()] = true
      end

      if field:containing_oneof() then
        -- Handled below.
      else
        if has_hasbit(field) then
          hasbit_indexes[field] = hasbit_count
          hasbit_count = hasbit_count + 1
        end

        append('  %s %s;\n', ctype(field), field:name())
      end
    end

    -- Oneof fields.
    for oneof in msg:oneofs() do
      local fullname = to_cident(oneof:containing_type():full_name() .. "." .. oneof:name())
      append('  union {\n')
      for field in oneof:fields() do
        append('    %s %s;\n', ctype(field), field:name())
      end
      append('  } %s;\n', oneof:name())
      append('  %s_oneofcases %s_case;\n', fullname, oneof:name())
    end

    append('};\n\n')

    if oneof_count > 0 then
      local oneofs_array_name = msgname .. "_oneofs"
      oneofs_array_ref = "&" .. oneofs_array_name .. "[0]"
      append('static const upb_msglayout_oneofinit_v1 %s[%s] = {\n',
             oneofs_array_name, oneof_count)
      for _, oneof in ipairs(oneofs_layout_order) do
        append('  {offsetof(%s, %s), offsetof(%s, %s_case)},\n',
               msgname, oneof:name(), msgname, oneof:name())
      end
      append('};\n\n')
    end

    if submsg_count > 0 then
      -- TODO(haberman): could save a little bit of space by only generating a
      -- "submsgs" array for every strongly-connected component.
      local submsgs_array_name = msgname .. "_submsgs"
      submsgs_array_ref = "&" .. submsgs_array_name .. "[0]"
      append('static const upb_msglayout_msginit_v1 *const %s[%s] = {\n',
             submsgs_array_name, submsg_count)

      -- Create a deterministically-sorted array of submessage entries.
      local submsg_array = {}
      for k, v in pairs(submsg_set) do
        table.insert(submsg_array, k)
      end
      table.sort(submsg_array, function(a, b)
        return a:full_name() < b:full_name()
      end)

      for i, submsg in ipairs(submsg_array) do
        append('  &%s_msginit,\n', to_cident(submsg:full_name()))
        submsg_indexes[submsg] = i - 1
      end

      append('};\n\n')
    end

    if field_count > 0 then
      local fields_array_name = msgname .. "__fields"
      fields_array_ref = "&" .. fields_array_name .. "[0]"
      append('static const upb_msglayout_fieldinit_v1 %s[%s] = {\n',
             fields_array_name, field_count)
      for _, field in ipairs(fields_number_order) do
        local submsg_index = "UPB_NO_SUBMSG"
        local oneof_index = "UPB_NOT_IN_ONEOF"
        if field:type() == upb.TYPE_MESSAGE then
          submsg_index = submsg_indexes[field:subdef()]
        end
        if field:containing_oneof() then
          oneof_index = oneof_indexes[field:containing_oneof()]
        end
        append('  {%s, offsetof(%s, %s), %s, %s, %s, %s, %s},\n',
               field:number(),
               msgname,
               (field:containing_oneof() and field:containing_oneof():name()) or field:name(),
               hasbit_indexes[field] or "UPB_NO_HASBIT",
               oneof_index,
               submsg_index,
               field:descriptor_type(),
               field:label())
      end
      append('};\n\n')
    end

    append('const upb_msglayout_msginit_v1 %s_msginit = {\n', msgname)
    append('  %s,\n', submsgs_array_ref)
    append('  %s,\n', fields_array_ref)
    append('  %s,\n', oneofs_array_ref)
    append('  NULL, /* TODO. default_msg */\n')
    append('  UPB_ALIGNED_SIZEOF(%s), %s, %s, %s, %s\n',
           msgname, field_count,
           0, -- TODO: oneof_count
           'false', -- TODO: extendable
          msg:file():syntax() == upb.SYNTAX_PROTO2
          )
    append('};\n\n')

    append('%s *%s_new(upb_env *env) {\n', msgname, msgname)
    append('  %s *msg = upb_env_malloc(env, sizeof(*msg));\n',
           msgname)
    append('  memset(msg, 0, sizeof(*msg));  /* TODO: defaults */\n')
    append('  return msg;\n')
    append('}\n')

    append('%s *%s_parsenew(upb_stringview buf, upb_env *env) {\n',
           msgname, msgname)
    append('  %s *msg = %s_new(env);\n', msgname, msgname)
    append('  if (upb_decode(buf, msg, &%s_msginit, env)) {\n', msgname)
    append('    return msg;\n')
    append('  } else {\n')
    append('    return NULL;\n')
    append('  }\n')
    append('}\n')

    append('char *%s_serialize(%s *msg, upb_env *env, size_t *size) {\n',
           msgname, msgname)
    append('  return upb_encode(msg, &%s_msginit, env, size);\n', msgname)
    append('}\n')

    for field in msg:fields() do
      append('%s %s_%s(const %s *msg) {\n',
             ctype(field, true), msgname, field:name(), msgname);
      if field:containing_oneof() then
        local oneof = field:containing_oneof()
        append('  return msg->%s_case == %s ? msg->%s.%s : %s;\n',
               oneof:name(), field:number(), oneof:name(), field:name(),
               field_default(field))
      else
        append('  return msg->%s;\n', field:name())
      end
      append('}\n')
      append('void %s_set_%s(%s *msg, %s value) {\n',
             msgname, field:name(), msgname, ctype(field));
      if field:containing_oneof() then
        local oneof = field:containing_oneof()
        append('  msg->%s.%s = value;\n', oneof:name(), field:name())
        append('  msg->%s_case = %s;\n', oneof:name(), field:number())
      else
        append('  msg->%s = value;\n', field:name())
      end
      append('}\n')
    end

    for oneof in msg:oneofs() do
      local fullname = to_cident(oneof:containing_type():full_name() .. "." .. oneof:name())
      append('%s_oneofcases %s_case(const %s *msg) {\n', fullname, fullname, msgname)
      append('  return msg->%s_case;\n', oneof:name())
      append('}\n')
    end
  end
end

function export.write_gencode(filedef, hfilename, append_h, append_c)
  write_h_file(filedef, append_h)
  write_c_file(filedef, hfilename, append_c)
end

return export