Merge pull request #3400 from murgatroid99/core_gyp

Add gyp file for compiling C core

templates/grpc.gyp.template

@@ -0,0 +1,125 @@
+%YAML 1.2
+--- |
+  # GRPC gyp file
+  # This currently builds C code.
+  # This file has been automatically generated from a template file.
+  # Please look at the templates directory instead.
+  # This file can be regenerated from the template by running
+  # tools/buildgen/generate_projects.sh
+
+  # Copyright 2015, Google Inc.
+  # All rights reserved.
+  #
+  # Redistribution and use in source and binary forms, with or without
+  # modification, are permitted provided that the following conditions are
+  # met:
+  #
+  #     * Redistributions of source code must retain the above copyright
+  # notice, this list of conditions and the following disclaimer.
+  #     * Redistributions in binary form must reproduce the above
+  # copyright notice, this list of conditions and the following disclaimer
+  # in the documentation and/or other materials provided with the
+  # distribution.
+  #     * Neither the name of Google Inc. nor the names of its
+  # contributors may be used to endorse or promote products derived from
+  # this software without specific prior written permission.
+  #
+  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+  # Some of this file is built with the help of
+  # https://n8.io/converting-a-c-library-to-gyp/
+  {
+    # TODO: Finish windows support
+    'target_defaults': {
+      'default_configuration': 'Debug',
+      'configurations': {
+        'Debug': {
+          'defines': [ 'DEBUG', '_DEBUG' ],
+          'msvs_settings': {
+            'VCCLCompilerTool': {
+              'RuntimeLibrary': 1, # static debug
+            },
+          },
+        },
+        'Release': {
+          'defines': [ 'NDEBUG' ],
+          'msvs_settings': {
+            'VCCLCompilerTool': {
+              'RuntimeLibrary': 0, # static release
+            },
+          },
+        }
+      },
+      'msvs_settings': {
+        'VCLinkerTool': {
+          'GenerateDebugInformation': 'true',
+        },
+      },
+      # TODO: Add fallback for Windows, and if pkg-config is not available
+      'defines': [
+        'TSI_OPENSSL_ALPN_SUPPORT=<!(pkg-config --atleast-version=1.0.2 openssl >/dev/null 2>&1 && echo 1 || echo 0)'
+      ],
+      'include_dirs': [
+        '.',
+        'include'
+      ],
+      # TODO: Check for libraries with pkg-config
+      'libraries': [
+        '-lcrypto',
+        '-lssl',
+        '-ldl',
+        '-lpthread',
+        '-lz'
+      ]
+    },
+    'targets': [
+      % for lib in libs:
+      # TODO: Add C++ targets
+      % if lib.language == 'c':
+      {
+        'target_name': '${lib.name}',
+        'product_prefix': 'lib',
+        'type': 'static_library',
+        'dependencies': [
+          % for dep in getattr(lib, 'deps', []):
+          '${dep}',
+          % endfor
+        ],
+        'sources': [
+          % for source in lib.src:
+          '${source}',
+          % endfor
+        ],
+      },
+      % endif
+      % endfor
+      % for tgt in targets:
+      % if tgt.language == 'c':
+      {
+        'target_name': '${tgt.name}',
+        'type': 'executable',
+        'dependencies': [
+          % for dep in getattr(tgt, 'deps', []):
+          '${dep}',
+          % endfor
+        ],
+        'sources': [
+          % for source in tgt.src:
+          '${source}',
+          % endfor
+        ]
+      },
+      % endif
+      % endfor
+    ]
+  }

tools/jenkins/grpc_jenkins_slave/Dockerfile

@@ -43,6 +43,7 @@ RUN apt-get update && apt-get install -y \
   gcc \
   gcc-multilib \
   git \
+  gyp \
   libc6 \
   libc6-dbg \
   libc6-dev \

tools/jenkins/grpc_jenkins_slave_32bits/Dockerfile

@@ -43,6 +43,7 @@ RUN apt-get update && apt-get install -y \
   gcc \
   gcc-multilib \
   git \
+  gyp \
   libc6 \
   libc6-dbg \
   libc6-dev \

tools/run_tests/run_tests.py

@@ -120,6 +120,18 @@ class ValgrindConfig(object):
                           hash_targets=None)
 
 
+def get_c_tests(travis, test_lang) :
+  out = []
+  platforms_str = 'ci_platforms' if travis else 'platforms'
+  with open('tools/run_tests/tests.json') as f:
+    js = json.load(f)
+    return [tgt
+            for tgt in js
+            if tgt['language'] == test_lang and
+                platform_string() in tgt[platforms_str] and
+                not (travis and tgt['flaky'])]
+
+
 class CLanguage(object):
 
   def __init__(self, make_target, test_lang):
@@ -129,16 +141,9 @@ class CLanguage(object):
 
   def test_specs(self, config, travis):
     out = []
-    with open('tools/run_tests/tests.json') as f:
+    binaries = get_c_tests(travis, self.test_lang)
-      js = json.load(f)
-      platforms_str = 'ci_platforms' if travis else 'platforms'
-      binaries = [tgt
-                  for tgt in js
-                  if tgt['language'] == self.test_lang and
-                      config.build_config not in tgt['exclude_configs'] and
-                      platform_string() in tgt[platforms_str]]
     for target in binaries:
-      if travis and target['flaky']:
+      if config.build_config in target['exclude_configs']:
         continue
       if self.platform == 'windows':
         binary = 'vsprojects/%s/%s.exe' % (
@@ -157,9 +162,15 @@ class CLanguage(object):
       return ['buildtests_%s' % self.make_target]
     return ['buildtests_%s' % self.make_target, 'tools_%s' % self.make_target]
 
+  def pre_build_steps(self):
+    return []
+
   def build_steps(self):
     return []
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return True
 
@@ -167,18 +178,60 @@ class CLanguage(object):
     return self.make_target
 
 
+def gyp_test_paths(travis, config=None):
+  binaries = get_c_tests(travis, 'c')
+  out = []
+  for target in binaries:
+    if config is not None and config.build_config in target['exclude_configs']:
+        continue
+    binary = 'out/Debug/%s' % target['name']
+    out.append(binary)
+  return sorted(out)
+
+
+class GYPCLanguage(object):
+
+  def test_specs(self, config, travis):
+    return [config.job_spec([binary], [binary])
+            for binary in gyp_test_paths(travis, config)]
+
+  def pre_build_steps(self):
+    return [['gyp', '--depth=.', '--suffix=-gyp', 'grpc.gyp']]
+
+  def make_targets(self):
+    return gyp_test_paths(False)
+
+  def build_steps(self):
+    return []
+
+  def makefile_name(self):
+    return 'Makefile-gyp'
+
+  def supports_multi_config(self):
+    return False
+
+  def __str__(self):
+    return 'gyp'
+
+
 class NodeLanguage(object):
 
   def test_specs(self, config, travis):
     return [config.job_spec(['tools/run_tests/run_node.sh'], None,
                             environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
+  def pre_build_steps(self):
+    return []
+
   def make_targets(self):
     return ['static_c', 'shared_c']
 
   def build_steps(self):
     return [['tools/run_tests/build_node.sh']]
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return False
 
@@ -192,12 +245,18 @@ class PhpLanguage(object):
     return [config.job_spec(['src/php/bin/run_tests.sh'], None,
                             environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
+  def pre_build_steps(self):
+    return []
+
   def make_targets(self):
     return ['static_c', 'shared_c']
 
   def build_steps(self):
     return [['tools/run_tests/build_php.sh']]
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return False
 
@@ -221,6 +280,9 @@ class PythonLanguage(object):
         shortname='py.test',
     )]
 
+  def pre_build_steps(self):
+    return []
+
   def make_targets(self):
     return ['static_c', 'grpc_python_plugin', 'shared_c']
 
@@ -238,6 +300,9 @@ class PythonLanguage(object):
                        do_newline=True)
     return commands
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return False
 
@@ -251,12 +316,18 @@ class RubyLanguage(object):
     return [config.job_spec(['tools/run_tests/run_ruby.sh'], None,
                             environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
+  def pre_build_steps(self):
+    return []
+
   def make_targets(self):
     return ['static_c']
 
   def build_steps(self):
     return [['tools/run_tests/build_ruby.sh']]
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return False
 
@@ -282,6 +353,9 @@ class CSharpLanguage(object):
             environ=_FORCE_ENVIRON_FOR_WRAPPERS)
             for assembly in assemblies]
 
+  def pre_build_steps(self):
+    return []
+
   def make_targets(self):
     # For Windows, this target doesn't really build anything,
     # everything is build by buildall script later.
@@ -296,6 +370,9 @@ class CSharpLanguage(object):
     else:
       return [['tools/run_tests/build_csharp.sh']]
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return False
 
@@ -309,12 +386,18 @@ class ObjCLanguage(object):
     return [config.job_spec(['src/objective-c/tests/run_tests.sh'], None,
                             environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
 
+  def pre_build_steps(self):
+    return []
+
   def make_targets(self):
     return ['grpc_objective_c_plugin', 'interop_server']
 
   def build_steps(self):
     return [['src/objective-c/tests/build_tests.sh']]
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return False
 
@@ -328,12 +411,18 @@ class Sanity(object):
     return [config.job_spec('tools/run_tests/run_sanity.sh', None),
             config.job_spec('tools/run_tests/check_sources_and_headers.py', None)]
 
+  def pre_build_steps(self):
+    return []
+
   def make_targets(self):
     return ['run_dep_checks']
 
   def build_steps(self):
     return []
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return False
 
@@ -346,12 +435,18 @@ class Build(object):
   def test_specs(self, config, travis):
     return []
 
+  def pre_build_steps(self):
+    return []
+
   def make_targets(self):
     return ['static']
 
   def build_steps(self):
     return []
 
+  def makefile_name(self):
+    return 'Makefile'
+
   def supports_multi_config(self):
     return True
 
@@ -382,6 +477,7 @@ _DEFAULT = ['opt']
 _LANGUAGES = {
     'c++': CLanguage('cxx', 'c++'),
     'c': CLanguage('c', 'c'),
+    'gyp': GYPCLanguage(),
     'node': NodeLanguage(),
     'php': PhpLanguage(),
     'python': PythonLanguage(),
@@ -514,33 +610,41 @@ if len(build_configs) > 1:
       sys.exit(1)
 
 if platform.system() == 'Windows':
-  def make_jobspec(cfg, targets):
+  def make_jobspec(cfg, targets, makefile='Makefile'):
     extra_args = []
     # better do parallel compilation
     extra_args.extend(["/m"])
     # disable PDB generation: it's broken, and we don't need it during CI
     extra_args.extend(["/p:GenerateDebugInformation=false", "/p:DebugInformationFormat=None"])
     return [
-      jobset.JobSpec(['vsprojects\\build.bat', 
+      jobset.JobSpec(['vsprojects\\build.bat',
-                      'vsprojects\\%s.sln' % target, 
+                      'vsprojects\\%s.sln' % target,
                       '/p:Configuration=%s' % _WINDOWS_CONFIG[cfg]] +
                       extra_args,
                       shell=True, timeout_seconds=90*60)
       for target in targets]
 else:
-  def make_jobspec(cfg, targets):
+  def make_jobspec(cfg, targets, makefile='Makefile'):
     return [jobset.JobSpec([os.getenv('MAKE', 'make'),
+                            '-f', makefile,
                             '-j', '%d' % (multiprocessing.cpu_count() + 1),
                             'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' %
                                 args.slowdown,
                             'CONFIG=%s' % cfg] + targets,
                            timeout_seconds=30*60)]
+make_targets = {}
+for l in languages:
+  makefile = l.makefile_name()
+  make_targets[makefile] = make_targets.get(makefile, set()).union(
+      set(l.make_targets()))
 
-make_targets = list(set(itertools.chain.from_iterable(
+build_steps = list(set(
-                                         l.make_targets() for l in languages)))
+                   jobset.JobSpec(cmdline, environ={'CONFIG': cfg})
-build_steps = []
+                   for cfg in build_configs
+                   for l in languages
+                   for cmdline in l.pre_build_steps()))
 if make_targets:
-  make_commands = itertools.chain.from_iterable(make_jobspec(cfg, make_targets) for cfg in build_configs)
+  make_commands = itertools.chain.from_iterable(make_jobspec(cfg, list(targets), makefile) for cfg in build_configs for (makefile, targets) in make_targets.iteritems())
   build_steps.extend(set(make_commands))
 build_steps.extend(set(
                    jobset.JobSpec(cmdline, environ={'CONFIG': cfg})