From b542c821d1c84d9844e897527d41f55aa60fbe42 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:46:15 -0700 Subject: [PATCH 01/47] new .props that has c++ dependencies --- vsprojects/cpptest.props | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 vsprojects/cpptest.props diff --git a/vsprojects/cpptest.props b/vsprojects/cpptest.props new file mode 100644 index 00000000000..c5ffd54f990 --- /dev/null +++ b/vsprojects/cpptest.props @@ -0,0 +1,18 @@ + + + + + + + + $(ProjectDir)\..\..;$(ProjectDir)\..\..\include;$(ProjectDir)\..\..\third_party\protobuf\src;$(ProjectDir)\..\..\third_party\gtest\include;%(AdditionalIncludeDirectories) + _SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;_UNICODE;UNICODE;%(PreprocessorDefinitions) + EnableAllWarnings + + + grpc++_test_util.lib;grpc_test_util.lib;gpr_test_util.lib;gtestd.lib;gflags.lib;shlwapi.lib;gpr.lib;grpc.lib;grpc++.lib;%(AdditionalDependencies) + $(ProjectDir)\..\..\third_party\gtest\msvc\gtest\Debug;$(ProjectDir)\..\..\third_party\gflags\lib\Debug;$(ProjectDir)\..\Debug;$(ProjectDir)\..\packages\grpc.dependencies.openssl.1.0.2.2\build\native\lib\$(PlatformToolset)\$(Platform)\$(Configuration)\static;%(AdditionalLibraryDirectories) + + + + From b19d4ebba01b5a90b69230bcfdd7426bc791aacf Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:47:54 -0700 Subject: [PATCH 02/47] changed vsproject generation to make c++ tests; put a hint in the .sln about lib vs. target; expand .proto includes --- templates/vsprojects/vcxproj_defs.include | 27 ++++++++++++++++--- tools/buildgen/plugins/generate_vsprojects.py | 2 +- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/templates/vsprojects/vcxproj_defs.include b/templates/vsprojects/vcxproj_defs.include index 55ec37393fa..39c73860629 100644 --- a/templates/vsprojects/vcxproj_defs.include +++ b/templates/vsprojects/vcxproj_defs.include @@ -12,19 +12,21 @@ if t.name == name: target = t if not configuration_type and target: + print target.name if target.build == 'test' or target.build == 'tool': configuration_type = 'Application' if not configuration_type: configuration_type = 'StaticLibrary' if not project_guid: project_guid = project.vs_project_guid + if target.build == 'test' and target.language == 'c++': + props.extend(['cpptest']) if configuration_type == 'Application': + print target.build if target.build == 'protoc': props.extend(['protoc']) else: props.extend(['winsock', 'protobuf', 'zlib', 'openssl']) - if target.language == 'c++': - props.extend(['protobuf']) props.extend(['global']) %>\ @@ -113,6 +115,13 @@ ${gen_package_props(packages)}\ ${name} + % if "zlib" in packages: + static + Debug + % endif + % if "openssl" in packages: + Debug + % endif ${name} @@ -200,8 +209,20 @@ ${gen_package_props(packages)}\ % if project.get('src',[]): % for src_name in project.src: + % if src_name.endswith(".proto"): +<% src_name_parts = src_name.split(".") %>\ + + + + + + + + + % else: + % endif % endfor % elif configuration_type != 'StaticLibrary': @@ -230,4 +251,4 @@ ${gen_package_targets(packages)}\ ${gen_package_ensure(packages)}\ -\ \ No newline at end of file +\ diff --git a/tools/buildgen/plugins/generate_vsprojects.py b/tools/buildgen/plugins/generate_vsprojects.py index 150e72e0b26..413056fe89f 100755 --- a/tools/buildgen/plugins/generate_vsprojects.py +++ b/tools/buildgen/plugins/generate_vsprojects.py @@ -70,7 +70,7 @@ def mako_plugin(dictionary): if project.get('vs_project_guid', None)] projects = [project for project in projects - if project['language'] != 'c++' or project['build'] == 'all' or project['build'] == 'protoc'] + if project['language'] != 'c++' or project['build'] == 'all' or project['build'] == 'protoc' or (project['language'] == 'c++' and (project['build'] == 'test' or project['build'] == 'private'))] project_dict = dict([(p['name'], p) for p in projects]) From 6e081e0453b8194bb389ebd811bdc253551045d2 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:48:35 -0700 Subject: [PATCH 03/47] changed .sln to include hint about lib vs. target --- templates/vsprojects/sln_defs.include | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/templates/vsprojects/sln_defs.include b/templates/vsprojects/sln_defs.include index ee05d0fbde2..4ba0fbff075 100644 --- a/templates/vsprojects/sln_defs.include +++ b/templates/vsprojects/sln_defs.include @@ -15,6 +15,13 @@ cpp_proj_type = "{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}" %>\ % for project in solution_projects: Project("${cpp_proj_type}") = "${project.name}", "${project.name}\${project.name}.vcxproj", "${project.vs_project_guid}" + ProjectSection(myProperties) = preProject + % if project.is_library: + lib = "True" + % else: + lib = "False" + % endif + EndProjectSection % if project.get('deps', None): ProjectSection(ProjectDependencies) = postProject % for dep in project.get('deps', []): @@ -47,4 +54,4 @@ Global HideSolutionNode = FALSE EndGlobalSection EndGlobal -\ \ No newline at end of file +\ From 1ee691d92291cfb4553f19d17329963b11feee85 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:49:59 -0700 Subject: [PATCH 04/47] changed generate_debug_projects.sh to use the .sln hint when making vcxproj templates --- .../vsprojects/generate_debug_projects.sh | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/templates/vsprojects/generate_debug_projects.sh b/templates/vsprojects/generate_debug_projects.sh index 1bbe40b6c8b..f103ebe01a9 100755 --- a/templates/vsprojects/generate_debug_projects.sh +++ b/templates/vsprojects/generate_debug_projects.sh @@ -37,26 +37,40 @@ cd `dirname $0`/../.. -./tools/buildgen/generate_projects.sh +git add . #because we're using "git diff" to find changes to grpc.sln and then make files based on those changes, this prevents this file or other files from possibly being found in the diff -git diff | -grep \\+Project | -cut -d\" -f 4 | -sort -u | -grep _test$ | -while read p ; do - mkdir -p templates/vsprojects/$p - echo '<%namespace file="../vcxproj_defs.include" import="gen_project"/>${gen_project("'$p'", targets)}' > templates/vsprojects/$p/$p.vcxproj.template -done +./tools/buildgen/generate_projects-old.sh + +line_number=0 git diff | -grep \\+Project | -cut -d\" -f 4 | -sort -u | -grep -v _test$ | +grep -A2 \\+Project | #find "Project" immediately after a backslash (escaped), plus 2 additional lines to capture the "libs = ", plus 1 line of "--". matches will come from the generated grpc.sln while read p ; do - mkdir -p templates/vsprojects/$p - echo '<%namespace file="../vcxproj_defs.include" import="gen_project"/>${gen_project("'$p'", libs)}' > templates/vsprojects/$p/$p.vcxproj.template + line_number=$((line_number + 1)) + if [ "$line_number" -gt "4" ]; then + line_number=1; + fi + echo $line_number + echo $p + if [ "$line_number" -eq "1" ]; then + project_name=$(echo "$p" | cut -d\" -f 4) #sed: extract line N only; cut with delimiter: ". select only field 4 + fi + if [ "$line_number" -eq "3" ]; then + lib_setting=$(echo "$p" | cut -d\" -f 2) # + echo "project_name" + echo $project_name + echo "lib_setting" + echo $lib_setting + mkdir -p templates/vsprojects/$project_name + if [ "$lib_setting" = "True" ]; then + echo "lib: true" + echo '<%namespace file="../vcxproj_defs.include" import="gen_project"/>${gen_project("'$project_name'", libs)}' > templates/vsprojects/$project_name/$project_name.vcxproj.template + else + echo "lib: not true" + echo '<%namespace file="../vcxproj_defs.include" import="gen_project"/>${gen_project("'$project_name'", targets)}' > templates/vsprojects/$project_name/$project_name.vcxproj.template + fi + fi + # sleep .5 #for testing done -./tools/buildgen/generate_projects.sh +./tools/buildgen/generate_projects-old.sh From c27b25f78982f06a789c0fe19e9e2dfc1537e064 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:50:37 -0700 Subject: [PATCH 05/47] new includes / one less warning --- vsprojects/global.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vsprojects/global.props b/vsprojects/global.props index 14acf4668b3..6858bd7b9d4 100644 --- a/vsprojects/global.props +++ b/vsprojects/global.props @@ -5,10 +5,10 @@ - $(ProjectDir)\..\..;$(ProjectDir)\..\..\include;$(ProjectDir)\..\..\third_party\protobuf\src;$(AdditionalIncludeDirectories) - _CRT_SECURE_NO_WARNINGS;_UNICODE;UNICODE;%(PreprocessorDefinitions) + $(ProjectDir)\..\..;$(ProjectDir)\..\..\include;$(ProjectDir)\..\..\third_party\protobuf\src;$(ProjectDir)\..\packages\grpc.dependencies.openssl.1.0.2.2\build\native\include;$(ProjectDir)\..\..\third_party\gflags\include;%(AdditionalIncludeDirectories) + _SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;_UNICODE;UNICODE;%(PreprocessorDefinitions) EnableAllWarnings - \ No newline at end of file + From 845019c1d1afcb628312913cab87f769e3bf7c39 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:51:03 -0700 Subject: [PATCH 06/47] helper script to build all .proto files in /test/ in-place --- templates/vsprojects/build_test_protos.sh | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 templates/vsprojects/build_test_protos.sh diff --git a/templates/vsprojects/build_test_protos.sh b/templates/vsprojects/build_test_protos.sh new file mode 100644 index 00000000000..54edd3699af --- /dev/null +++ b/templates/vsprojects/build_test_protos.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +GRPC_CPP_PLUGIN_PATH=`which grpc_cpp_plugin` + +cd `dirname $0`/../.. + +find ./test -type f -name "*.proto" | +while read p ; do + echo "processing $p" + DIR=$(dirname "$p") + protoc $p --grpc_out=./ --plugin=protoc-gen-grpc=$GRPC_CPP_PLUGIN_PATH + protoc $p --cpp_out=./ +done From 878b63e1eba6a5ccc99ffde67c216f081e9cda0f Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:51:31 -0700 Subject: [PATCH 07/47] old not parallel version of generate_projects.sh, for debuggin --- tools/buildgen/generate_projects-old.sh | 76 +++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tools/buildgen/generate_projects-old.sh diff --git a/tools/buildgen/generate_projects-old.sh b/tools/buildgen/generate_projects-old.sh new file mode 100644 index 00000000000..53998677465 --- /dev/null +++ b/tools/buildgen/generate_projects-old.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +set -e + +if [ "x$TEST" = "x" ] ; then + TEST=false +fi + + +cd `dirname $0`/../.. +mako_renderer=tools/buildgen/mako_renderer.py + +if [ "x$TEST" != "x" ] ; then + tools/buildgen/build-cleaner.py build.json +fi + +. tools/buildgen/generate_build_additions.sh + +global_plugins=`find ./tools/buildgen/plugins -name '*.py' | + sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` + +for dir in . ; do + local_plugins=`find $dir/templates -name '*.py' | + sort | grep -v __init__ | awk ' { printf "-p %s ", $0 } '` + + plugins="$global_plugins $local_plugins" + + find -L $dir/templates -type f -and -name *.template | while read file ; do + out=${dir}/${file#$dir/templates/} # strip templates dir prefix + out=${out%.*} # strip template extension + echo "generating file: $out" + json_files="build.json $gen_build_files" + data=`for i in $json_files ; do echo $i ; done | awk ' { printf "-d %s ", $0 } '` + if [ "x$TEST" = "xtrue" ] ; then + actual_out=$out + out=`mktemp /tmp/gentXXXXXX` + fi + mkdir -p `dirname $out` # make sure dest directory exist + $mako_renderer $plugins $data -o $out $file + if [ "x$TEST" = "xtrue" ] ; then + diff -q $out $actual_out + rm $out + fi + done +done + +rm $gen_build_files From d6fafad06761b46ada8c6b64a7fae2032ef34377 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:51:52 -0700 Subject: [PATCH 08/47] documentation of new vcxproj/.sln debug buildgen --- vsprojects/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/vsprojects/README.md b/vsprojects/README.md index dade9e500e2..06e87b3aed2 100644 --- a/vsprojects/README.md +++ b/vsprojects/README.md @@ -19,8 +19,38 @@ After that, you can build the solution using one of these options: 1. open `grpc.sln` with Visual Studio and hit "Build". 2. build from commandline using `msbuild grpc.sln /p:Configuration=Debug` +#C/C++ Test Dependencies + * gtest isn't available as a git repo like the other dependencies. download it and add it to `/third_party/gtest/` (the folder will end up with `/build-aux/`, `/cmake/`, `/codegear/`, etc. folders in it). + * if using vs2013: open/import the gtest solution in `/msvc/`, and save over the first solution (you will have to change it from read-only). change all projects to use `/MDd` (Property Pages - C/C++ - Code Generation - Runtime Library) and build. This is a "multithreaded debug" setting and it needs to match grpc. + * build all + * open protobuf solution in `/third_party/protobuf/vsprojects` + * if using vs2013: on import the gtest stuff will probably fail, I think the paths are interpreted wrong. it's ok. + * tests and test_plugin will fail when built. also ok + * build all + * gflags is automatically imported as a git submodule but it needs to have CMake run on it to be ready for a specific platform + * download [CMake](http://www.cmake.org/) windows installer; install + * open visual studio developer command prompt (not sure if dev command prompt is necessary) + * run `cmake ` + * this will build a `.sln` and fill up the `/third_party/gflags/include/gflags/` directory with headers + * build all + * install [NuGet](http://www.nuget.org) + * nuget should automatically bring in built versions of zlib and openssl when building grpc.sln (the versions in `/third_party/` are not used). If it doesn't work use `tools->nuget...->manage...`. The packages are put in `/vsprojects/packages/` + +#C/C++ Test Build Steps + * set up dependencies (above) + * add `"debug": true,` to the top of build.json. This is the base file for all build tracking, see [templates](https://github.com/grpc/grpc/tree/master/templates) for more information + * `"debug": true,` gets picked up by `/tools/buildgen/plugins/generate_vsprojects.py`. It tells the script to add visual studio GUIDs to all projects. Otherwise only the projects that already have GUIDs in build.json will be built + * A basic git version of grpc only has templates for non-test items. run `/templates/vsprojects/generate_debug_projects.sh` to make debug templates/projects. This runs a regular visual studio buildgen process, which creates the `.sln` file with all of the new debug projects, then uses git diff to find the new project names from the `.sln` that need templates added. It builds the new templates based on the diff, then re-runs the visual studio buildgen, which builds the vs projects for each of the new debug targets + * copy over the `/vsprojects/` folder to your windows build setup (assuming this was built on linux in order to have easy access to python/mako and shell scripts) + * run `/templates/vsprojects/build_test_protos.sh` + * this builds all `.proto` files in `/test/` in-place. there might be a better place to put them that mirrors what happens in the linux build process (todo) + * each `.proto` file gets built into a `.grpc.pb.cc`, .`grpc.pb.h`, `.pb.cc`, and `.pb.h`. These are included in each test project in lieu of the `.proto` includes specified in `build.json`. This substitution is done by `/templates/vsprojects/vcxproj_defs.include` + * copy over the `/test/` folder in order to get the new files (assuming this was built on linux in order to have an easy protobuf+grpc plugin installation) + #Testing +This is incomplete (only runs some tests for now), todo. The above .sln-based buildgen makes more tets but isn't tied in to automatic test running yet. + Use `run_tests.py`, that also supports Windows (with a bit limited experience). ``` > REM Run from repository root. From 2c960065e20bba00df6f590eb090d70a705e8cf2 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:52:08 -0700 Subject: [PATCH 09/47] nmake: continue on error --- vsprojects/make.bat | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vsprojects/make.bat b/vsprojects/make.bat index 04737aeefc4..08c97bcd21f 100644 --- a/vsprojects/make.bat +++ b/vsprojects/make.bat @@ -5,6 +5,7 @@ setlocal @rem Set VS variables (uses Visual Studio 2013) @call "%VS120COMNTOOLS%\..\..\vc\vcvarsall.bat" x86 -nmake /f Grpc.mak %* +@rem /K: continue on error +nmake /K /f Grpc.mak %* exit /b %ERRORLEVEL% -endlocal \ No newline at end of file +endlocal From c58e50137327760a5b046a357cec9b1462844690 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Sun, 19 Jul 2015 10:52:32 -0700 Subject: [PATCH 10/47] changes to grpc.mak.template to support c++ test building --- templates/vsprojects/Grpc.mak.template | 73 ++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/templates/vsprojects/Grpc.mak.template b/templates/vsprojects/Grpc.mak.template index 9d85376648b..9d2f676d380 100644 --- a/templates/vsprojects/Grpc.mak.template +++ b/templates/vsprojects/Grpc.mak.template @@ -33,16 +33,18 @@ <%def name="to_windows_path(path)">${path.replace('/','\\')}\ <% build_from_project_file = set(['gpr', - 'grpc', - 'grpc_unsecure', 'gpr_test_util', + 'grpc', 'grpc_test_util', 'grpc_test_util_unsecure', + 'grpc_unsecure', + 'grpc++', + 'grpc++_unsecure' ]) buildable_targets = [ target for target in targets + libs if target.build in ['all', 'test', 'private', 'tool', 'benchmark'] and target.language in ['c', 'c++'] and - all([src.endswith('.c') for src in target.src]) and + all([(src.endswith('.c') or src.endswith('.cc') or src.endswith('.proto')) for src in target.src]) and 'windows' in target.get('platforms', ['windows']) ] c_test_targets = [ target for target in buildable_targets if target.build == 'test' and not target.language == 'c++' ] cxx_test_targets = [ target for target in buildable_targets if target.build == 'test' and target.language == 'c++' ] @@ -60,8 +62,19 @@ REPO_ROOT=.. OPENSSL_INCLUDES = .\packages\${get_openssl()}\build\native\include ZLIB_INCLUDES = .\packages\${get_zlib()}\build\native\include INCLUDES=/I$(REPO_ROOT) /I$(REPO_ROOT)\include /I$(OPENSSL_INCLUDES) /I$(ZLIB_INCLUDES) -DEFINES=/D WIN32 /D _LIB /D _USE_32BIT_TIME_T /D _UNICODE /D UNICODE /D _CRT_SECURE_NO_WARNINGS + +GFLAGS_INCLUDES = .\..\third_party\gflags\include +GTEST_INCLUDES = .\..\third_party\gtest\include +PROTOBUF_INCLUDES = .\..\third_party\protobuf\src +CXX_INCLUDES=/I$(GFLAGS_INCLUDES) /I$(GTEST_INCLUDES) /I$(PROTOBUF_INCLUDES) + +#_SCL_SECURE_NO_WARNINGS supresses a ton of "potentially unsafe use of std lib" warnings +DEFINES=/D WIN32 /D _LIB /D _USE_32BIT_TIME_T /D _UNICODE /D UNICODE /D _CRT_SECURE_NO_WARNINGS /D _SCL_SECURE_NO_WARNINGS + +#important options: /TC vs. /TP: compile as C vs. compile as C++ CFLAGS=/c $(INCLUDES) /Z7 /W3 /WX- /sdl $(DEFINES) /EHsc /RTC1 /MDd /GS /fp:precise /Zc:wchar_t /Zc:forScope /Gd /TC /analyze- +CXXFLAGS=/c $(INCLUDES) $(CXX_INCLUDES) /Z7 /W3 /WX- /sdl $(DEFINES) /EHsc /RTC1 /MDd /GS /fp:precise /Zc:wchar_t /Zc:forScope /Gd /TP /analyze- + LFLAGS=/DEBUG /INCREMENTAL /SUBSYSTEM:CONSOLE /TLBID:1 /DYNAMICBASE /NXCOMPAT /MACHINE:X86 OPENSSL_LIBS=.\packages\${get_openssl()}\build\native\lib\v120\Win32\Debug\static\ssleay32.lib .\packages\${get_openssl()}\build\native\lib\v120\Win32\Debug\static\libeay32.lib @@ -70,6 +83,12 @@ GENERAL_LIBS=advapi32.lib comdlg32.lib gdi32.lib kernel32.lib odbc32.lib odbccp3 ZLIB_LIBS=.\packages\${get_zlib()}\build\native\lib\v120\Win32\Debug\static\cdecl\zlib.lib LIBS=$(OPENSSL_LIBS) $(ZLIB_LIBS) $(GENERAL_LIBS) $(WINSOCK_LIBS) +#shlwapi.lib provides PathMatchSpec() for gflags in windows +GFLAGS_LIBS=.\..\third_party\gflags\lib\Debug\gflags.lib shlwapi.lib +GTEST_LIBS=.\..\third_party\gtest\msvc\gtest\Debug\gtestd.lib +PROTOBUF_LIBS=.\..\third_party\protobuf\vsprojects\Debug\libprotobuf.lib +CXX_LIBS=$(GFLAGS_LIBS) $(GTEST_LIBS) $(PROTOBUF_LIBS) + all: buildtests tools: @@ -99,16 +118,33 @@ buildtests_c: \ ${target.name}.exe \ % endfor - echo All tests built. + echo All C tests built. buildtests_cxx: \ % for target in cxx_test_targets: ${target.name}.exe \ % endfor - echo All tests built. + echo All C++ tests built. % for target in buildable_targets: + +## replace all .proto includes with .pb.cc / .grpc.pb.cc +%if target.src: +%for source in target.src: +%if source.endswith(".proto"): +<% + src_name_parts = source.split(".") + target.src.append(src_name_parts[0] + ".pb.cc") + target.src.append(src_name_parts[0] + ".grpc.pb.cc") +%>\ +%endif +%endfor +%endif +## remove all .proto includes +<% + target.src = [item for item in target.src if not re.search('([^/]+)\.proto$', item)] +%>\ %if target.name in build_from_project_file: build_${target.name}: msbuild grpc.sln /t:${target.name} /p:Configuration=Debug /p:Linkage-grpc_dependencies_zlib=static @@ -116,11 +152,22 @@ build_${target.name}: %if target.build == 'private': Debug\${target.name}.lib: \ %else: -${target.name}.exe: build_libs \ +${target.name}.exe: \ +%for dep in target.get('deps', []): +%if dep in build_from_project_file: +build_${dep} \ +%else: +Debug\${dep}.lib \ +%endif +%endfor %endif $(OUT_DIR) echo Building ${target.name} +%if target.language == 'c++': + $(CC) $(CXXFLAGS) /Fo:$(OUT_DIR)\ \ +%else: $(CC) $(CFLAGS) /Fo:$(OUT_DIR)\ \ +%endif %for source in target.src: $(REPO_ROOT)\${to_windows_path(source)} \ %endfor @@ -135,13 +182,19 @@ $(REPO_ROOT)\${to_windows_path('vsprojects/dummy.c')} \ %for dep in target.get('deps', []): Debug\${dep}.lib \ %endfor +%if target.language == 'c++': +$(CXX_LIBS) \ +%endif $(LIBS) \ %endif -%for source in target.src: -$(OUT_DIR)\${re.search('([^/]+)\.c$', source).group(1)}.obj \ -%endfor %if not target.src: $(OUT_DIR)\dummy.obj \ +%else: +%for source in target.src: +%if re.search('([^/]+)\.c{1,2}$', source): +$(OUT_DIR)\${re.search('([^/]+)\.c{1,2}$', source).group(1)}.obj \ +%endif +%endfor %endif %if target.build != 'private': From 882ef3ff9ee0f5f5a82e26cfe2cfd2ba1536a339 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Mon, 20 Jul 2015 14:14:41 -0700 Subject: [PATCH 11/47] mark not building windows tests as platforms = posix in build.json --- build.json | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/build.json b/build.json index af8bd54460e..84c292cd7f3 100644 --- a/build.json +++ b/build.json @@ -1890,6 +1890,9 @@ "grpc", "gpr_test_util", "gpr" + ], + "platforms": [ + "posix" ] }, { @@ -1907,6 +1910,9 @@ "grpc", "gpr_test_util", "gpr" + ], + "platforms": [ + "posix" ] }, { @@ -1965,6 +1971,9 @@ "grpc", "gpr_test_util", "gpr" + ], + "platforms": [ + "posix" ] }, { @@ -2188,6 +2197,9 @@ "gpr_test_util", "gpr", "grpc++_test_config" + ], + "platforms": [ + "posix" ] }, { @@ -2206,6 +2218,9 @@ "gpr_test_util", "gpr", "grpc++_test_config" + ], + "platforms": [ + "posix" ] }, { @@ -2220,6 +2235,9 @@ "grpc", "gpr_test_util", "gpr" + ], + "platforms": [ + "posix" ] }, { @@ -2324,6 +2342,9 @@ "grpc", "gpr_test_util", "gpr" + ], + "platforms": [ + "posix" ] }, { @@ -2342,6 +2363,9 @@ "gpr_test_util", "gpr", "grpc++_test_config" + ], + "platforms": [ + "posix" ] }, { @@ -2360,6 +2384,9 @@ "gpr_test_util", "gpr", "grpc++_test_config" + ], + "platforms": [ + "posix" ] }, { @@ -2411,6 +2438,9 @@ "grpc", "gpr_test_util", "gpr" + ], + "platforms": [ + "posix" ] }, { @@ -2460,6 +2490,9 @@ "grpc", "gpr_test_util", "gpr" + ], + "platforms": [ + "posix" ] }, { @@ -2477,6 +2510,9 @@ "grpc", "gpr_test_util", "gpr" + ], + "platforms": [ + "posix" ] }, { From 639ae0ff5f1deacf185ed38e8be86e4ca5fb7e43 Mon Sep 17 00:00:00 2001 From: Michael Larson Date: Tue, 21 Jul 2015 12:05:42 -0700 Subject: [PATCH 12/47] documentation for windows testing using grpc.mak --- vsprojects/README.md | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/vsprojects/README.md b/vsprojects/README.md index 06e87b3aed2..e6cbf833590 100644 --- a/vsprojects/README.md +++ b/vsprojects/README.md @@ -36,32 +36,45 @@ After that, you can build the solution using one of these options: * install [NuGet](http://www.nuget.org) * nuget should automatically bring in built versions of zlib and openssl when building grpc.sln (the versions in `/third_party/` are not used). If it doesn't work use `tools->nuget...->manage...`. The packages are put in `/vsprojects/packages/` -#C/C++ Test Build Steps +#C/C++ Test Solution/Project Build Steps + * A basic git version of grpc only has templates for non-test items. This checklist adds test items to grpc.sln and makes individual vs projects for them * set up dependencies (above) * add `"debug": true,` to the top of build.json. This is the base file for all build tracking, see [templates](https://github.com/grpc/grpc/tree/master/templates) for more information * `"debug": true,` gets picked up by `/tools/buildgen/plugins/generate_vsprojects.py`. It tells the script to add visual studio GUIDs to all projects. Otherwise only the projects that already have GUIDs in build.json will be built - * A basic git version of grpc only has templates for non-test items. run `/templates/vsprojects/generate_debug_projects.sh` to make debug templates/projects. This runs a regular visual studio buildgen process, which creates the `.sln` file with all of the new debug projects, then uses git diff to find the new project names from the `.sln` that need templates added. It builds the new templates based on the diff, then re-runs the visual studio buildgen, which builds the vs projects for each of the new debug targets + * run `/templates/vsprojects/generate_debug_projects.sh` to make debug templates/projects. This runs a regular visual studio buildgen process, which creates the `.sln` file with all of the new debug projects, then uses git diff to find the new project names from the `.sln` that need templates added. It builds the new templates based on the diff, then re-runs the visual studio buildgen, which builds the vs projects for each of the new debug targets * copy over the `/vsprojects/` folder to your windows build setup (assuming this was built on linux in order to have easy access to python/mako and shell scripts) * run `/templates/vsprojects/build_test_protos.sh` * this builds all `.proto` files in `/test/` in-place. there might be a better place to put them that mirrors what happens in the linux build process (todo) * each `.proto` file gets built into a `.grpc.pb.cc`, .`grpc.pb.h`, `.pb.cc`, and `.pb.h`. These are included in each test project in lieu of the `.proto` includes specified in `build.json`. This substitution is done by `/templates/vsprojects/vcxproj_defs.include` * copy over the `/test/` folder in order to get the new files (assuming this was built on linux in order to have an easy protobuf+grpc plugin installation) -#Testing +#Making and running tests with `/tools/run_tests/run_tests.py` or `/vsprojects/make.bat` +`run_tests.py` and `make.bat` both rely on `/vsprojects/grpc.mak`, an NMAKE script that includes C/C++ tests in addition to the base grpc projects. It builds the base projects by calling grpc.sln, but most things are built with a command line similar to a makefile workflow. -This is incomplete (only runs some tests for now), todo. The above .sln-based buildgen makes more tets but isn't tied in to automatic test running yet. + arguments for `/vsprojects/make.bat`: -Use `run_tests.py`, that also supports Windows (with a bit limited experience). -``` -> REM Run from repository root. -> python tools\run_tests\run_tests.py -l c -``` + * no options or `all` or `buildtests`: builds all tests + * `buildtests_c`: just c tests + * `buildtests_cxx`: just c++ tests + * names of individual tests: just those tests (example: `make.bat gpr_string_test`) -Also, you can `make.bat` directly to build and run gRPC tests. -``` -> REM Run from this directory. -> make.bat alarm_test -``` +using `run_tests.py` on windows: + + * when `run_tests.py` detects that it's running on windows it calls `make.bat` to build the tests and expects to find tests in `/vsprojects/test_bins/` + +`run_tests.py` options: + + * `run_tests.py --help` + * `run_tests.py -l c`: run c language tests + * `run_tests.py -l c++`: run c++ language tests + * note: `run_tests.py` doesn't normally show build steps, so if a build fails it is best to fall back to `make.bat` + * if `make.bat` fails, it might be easier to open up the `.sln` file in the visual studio gui (see above for how to build the test projects) and build the offending test from its project file. The `.mak` and project file templates are slightly different, so it's possible that a project will build one way and not another. Please report this if it happens. + +It can be helpful to disable the firewall when running tests so that 400 connection warnings don't pop up. + +Individual tests can be run by directly running the executable in `/vsprojects/run_tests/` (this is `/bins/opt/` on linux). Many C tests have no output; they either pass or fail internally and communicate this with their exit code (`0=pass`, `nonzero=fail`) + +`run_tests.py` will fail if it can't build something, so not-building tests are disabled with a "platforms = posix" note in build.json. The buildgen tools will not add a test to a windows build unless it is marked "windows" or has no platforms identified. As tests are ported they will get this mark removed. # Building protoc plugins For generating service stub code, gRPC relies on plugins for `protoc` (the protocol buffer compiler). The solution `grpc_protoc_plugins.sln` allows you to build From 4a4f1496c1b2fe6dc8336ee5f00c09568165a42a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 21 Jul 2015 16:32:29 -0700 Subject: [PATCH 13/47] Rename grpc_channel_create to grpc_insecure_channel_create --- include/grpc/grpc.h | 4 ++-- src/core/surface/channel_create.c | 4 ++-- src/csharp/ext/grpc_csharp_ext.c | 2 +- src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m | 3 ++- src/php/ext/grpc/channel.c | 4 ++-- src/python/src/grpc/_adapter/_c/types/channel.c | 2 +- src/ruby/ext/grpc/rb_channel.c | 2 +- test/core/end2end/dualstack_socket_test.c | 2 +- test/core/end2end/fixtures/chttp2_fullstack.c | 2 +- test/core/end2end/fixtures/chttp2_fullstack_compression.c | 3 ++- test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c | 2 +- test/core/end2end/fixtures/chttp2_fullstack_with_poll.c | 2 +- test/core/end2end/no_server_test.c | 2 +- test/core/fling/client.c | 2 +- 14 files changed, 19 insertions(+), 17 deletions(-) diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 504f0cce039..7b2fedad635 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -438,8 +438,8 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops, clients will want to simply pass NULL. See grpc_channel_args definition for more on this. The data in 'args' need only live through the invocation of this function. */ -grpc_channel *grpc_channel_create(const char *target, - const grpc_channel_args *args); +grpc_channel *grpc_insecure_channel_create(const char *target, + const grpc_channel_args *args); /** Create a lame client: this client fails every operation attempted on it. */ grpc_channel *grpc_lame_client_channel_create(void); diff --git a/src/core/surface/channel_create.c b/src/core/surface/channel_create.c index 91c7b35550a..bfe10a3b8c3 100644 --- a/src/core/surface/channel_create.c +++ b/src/core/surface/channel_create.c @@ -151,8 +151,8 @@ static const grpc_subchannel_factory_vtable subchannel_factory_vtable = { Asynchronously: - resolve target - connect to it (trying alternatives as presented) - perform handshakes */ -grpc_channel *grpc_channel_create(const char *target, - const grpc_channel_args *args) { +grpc_channel *grpc_insecure_channel_create(const char *target, + const grpc_channel_args *args) { grpc_channel *channel = NULL; #define MAX_FILTERS 3 const grpc_channel_filter *filters[MAX_FILTERS]; diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c index 7dd1959a5f7..eed286c7d6e 100644 --- a/src/csharp/ext/grpc_csharp_ext.c +++ b/src/csharp/ext/grpc_csharp_ext.c @@ -315,7 +315,7 @@ grpcsharp_completion_queue_pluck(grpc_completion_queue *cq, void *tag) { GPR_EXPORT grpc_channel *GPR_CALLTYPE grpcsharp_channel_create(const char *target, const grpc_channel_args *args) { - return grpc_channel_create(target, args); + return grpc_insecure_channel_create(target, args); } GPR_EXPORT void GPR_CALLTYPE grpcsharp_channel_destroy(grpc_channel *channel) { diff --git a/src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m b/src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m index d27f7ca565b..d522ddaae68 100644 --- a/src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m @@ -38,7 +38,8 @@ @implementation GRPCUnsecuredChannel - (instancetype)initWithHost:(NSString *)host { - return (self = [super initWithChannel:grpc_channel_create(host.UTF8String, NULL)]); + return (self = [super initWithChannel:grpc_insecure_channel_create( + host.UTF8String, NULL)]); } @end diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c index b8262db162b..501e42a6435 100644 --- a/src/php/ext/grpc/channel.c +++ b/src/php/ext/grpc/channel.c @@ -152,7 +152,7 @@ PHP_METHOD(Channel, __construct) { override = target; override_len = target_length; if (args_array == NULL) { - channel->wrapped = grpc_channel_create(target, NULL); + channel->wrapped = grpc_insecure_channel_create(target, NULL); } else { array_hash = Z_ARRVAL_P(args_array); if (zend_hash_find(array_hash, "credentials", sizeof("credentials"), @@ -182,7 +182,7 @@ PHP_METHOD(Channel, __construct) { } php_grpc_read_args_array(args_array, &args); if (creds == NULL) { - channel->wrapped = grpc_channel_create(target, &args); + channel->wrapped = grpc_insecure_channel_create(target, &args); } else { gpr_log(GPR_DEBUG, "Initialized secure channel"); channel->wrapped = diff --git a/src/python/src/grpc/_adapter/_c/types/channel.c b/src/python/src/grpc/_adapter/_c/types/channel.c index c235597466c..feb256cf000 100644 --- a/src/python/src/grpc/_adapter/_c/types/channel.c +++ b/src/python/src/grpc/_adapter/_c/types/channel.c @@ -104,7 +104,7 @@ Channel *pygrpc_Channel_new( if (creds) { self->c_chan = grpc_secure_channel_create(creds->c_creds, target, &c_args); } else { - self->c_chan = grpc_channel_create(target, &c_args); + self->c_chan = grpc_insecure_channel_create(target, &c_args); } pygrpc_discard_channel_args(c_args); return self; diff --git a/src/ruby/ext/grpc/rb_channel.c b/src/ruby/ext/grpc/rb_channel.c index 9bf1a9f945c..0cb6fa2f800 100644 --- a/src/ruby/ext/grpc/rb_channel.c +++ b/src/ruby/ext/grpc/rb_channel.c @@ -146,7 +146,7 @@ static VALUE grpc_rb_channel_init(int argc, VALUE *argv, VALUE self) { target_chars = StringValueCStr(target); grpc_rb_hash_convert_to_channel_args(channel_args, &args); if (credentials == Qnil) { - ch = grpc_channel_create(target_chars, &args); + ch = grpc_insecure_channel_create(target_chars, &args); } else { creds = grpc_rb_get_wrapped_credentials(credentials); ch = grpc_secure_channel_create(creds, target_chars, &args); diff --git a/test/core/end2end/dualstack_socket_test.c b/test/core/end2end/dualstack_socket_test.c index 05ad42ca1f3..1fac2f9da1a 100644 --- a/test/core/end2end/dualstack_socket_test.c +++ b/test/core/end2end/dualstack_socket_test.c @@ -106,7 +106,7 @@ void test_connect(const char *server_host, const char *client_host, int port, /* Create client. */ gpr_join_host_port(&client_hostport, client_host, port); - client = grpc_channel_create(client_hostport, NULL); + client = grpc_insecure_channel_create(client_hostport, NULL); gpr_log(GPR_INFO, "Testing with server=%s client=%s (expecting %s)", server_hostport, client_hostport, expect_ok ? "success" : "failure"); diff --git a/test/core/end2end/fixtures/chttp2_fullstack.c b/test/core/end2end/fixtures/chttp2_fullstack.c index 8a1530e63b0..6647b949ba2 100644 --- a/test/core/end2end/fixtures/chttp2_fullstack.c +++ b/test/core/end2end/fixtures/chttp2_fullstack.c @@ -72,7 +72,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_fullstack( void chttp2_init_client_fullstack(grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { fullstack_fixture_data *ffd = f->fixture_data; - f->client = grpc_channel_create(ffd->localaddr, client_args); + f->client = grpc_insecure_channel_create(ffd->localaddr, client_args); GPR_ASSERT(f->client); } diff --git a/test/core/end2end/fixtures/chttp2_fullstack_compression.c b/test/core/end2end/fixtures/chttp2_fullstack_compression.c index 0a9a3122968..f3d1fa22dce 100644 --- a/test/core/end2end/fixtures/chttp2_fullstack_compression.c +++ b/test/core/end2end/fixtures/chttp2_fullstack_compression.c @@ -82,7 +82,8 @@ void chttp2_init_client_fullstack_compression(grpc_end2end_test_fixture *f, } ffd->client_args_compression = grpc_channel_args_set_compression_algorithm( client_args, GRPC_COMPRESS_GZIP); - f->client = grpc_channel_create(ffd->localaddr, ffd->client_args_compression); + f->client = grpc_insecure_channel_create(ffd->localaddr, + ffd->client_args_compression); } void chttp2_init_server_fullstack_compression(grpc_end2end_test_fixture *f, diff --git a/test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c b/test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c index 351e1c5459b..89ad7b8c2d8 100644 --- a/test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c +++ b/test/core/end2end/fixtures/chttp2_fullstack_uds_posix.c @@ -78,7 +78,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_fullstack( void chttp2_init_client_fullstack(grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { fullstack_fixture_data *ffd = f->fixture_data; - f->client = grpc_channel_create(ffd->localaddr, client_args); + f->client = grpc_insecure_channel_create(ffd->localaddr, client_args); } void chttp2_init_server_fullstack(grpc_end2end_test_fixture *f, diff --git a/test/core/end2end/fixtures/chttp2_fullstack_with_poll.c b/test/core/end2end/fixtures/chttp2_fullstack_with_poll.c index 69860d04d5e..93786d0943a 100644 --- a/test/core/end2end/fixtures/chttp2_fullstack_with_poll.c +++ b/test/core/end2end/fixtures/chttp2_fullstack_with_poll.c @@ -72,7 +72,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_fullstack( void chttp2_init_client_fullstack(grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { fullstack_fixture_data *ffd = f->fixture_data; - f->client = grpc_channel_create(ffd->localaddr, client_args); + f->client = grpc_insecure_channel_create(ffd->localaddr, client_args); } void chttp2_init_server_fullstack(grpc_end2end_test_fixture *f, diff --git a/test/core/end2end/no_server_test.c b/test/core/end2end/no_server_test.c index 79797f93751..6f1133c009a 100644 --- a/test/core/end2end/no_server_test.c +++ b/test/core/end2end/no_server_test.c @@ -61,7 +61,7 @@ int main(int argc, char **argv) { cqv = cq_verifier_create(cq); /* create a call, channel to a non existant server */ - chan = grpc_channel_create("nonexistant:54321", NULL); + chan = grpc_insecure_channel_create("nonexistant:54321", NULL); call = grpc_channel_create_call(chan, cq, "/Foo", "nonexistant", deadline); op = ops; diff --git a/test/core/fling/client.c b/test/core/fling/client.c index 2b196543792..5647a16101e 100644 --- a/test/core/fling/client.c +++ b/test/core/fling/client.c @@ -183,7 +183,7 @@ int main(int argc, char **argv) { return 1; } - channel = grpc_channel_create(target, NULL); + channel = grpc_insecure_channel_create(target, NULL); cq = grpc_completion_queue_create(); the_buffer = grpc_raw_byte_buffer_create(&slice, payload_size); histogram = gpr_histogram_create(0.01, 60e9); From 33c0d9d83db8e47cb7090d20b1fcfa288964e4c0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 21 Jul 2015 18:11:44 -0700 Subject: [PATCH 14/47] C++ is also a language that can be insecure --- src/cpp/client/insecure_credentials.cc | 2 +- src/node/ext/channel.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cpp/client/insecure_credentials.cc b/src/cpp/client/insecure_credentials.cc index 5ad87845675..e802fa8034e 100644 --- a/src/cpp/client/insecure_credentials.cc +++ b/src/cpp/client/insecure_credentials.cc @@ -49,7 +49,7 @@ class InsecureCredentialsImpl GRPC_FINAL : public Credentials { grpc_channel_args channel_args; args.SetChannelArgs(&channel_args); return std::shared_ptr(new Channel( - target, grpc_channel_create(target.c_str(), &channel_args))); + target, grpc_insecure_channel_create(target.c_str(), &channel_args))); } // InsecureCredentials should not be applied to a call. diff --git a/src/node/ext/channel.cc b/src/node/ext/channel.cc index d37bf763ddc..28fa2550988 100644 --- a/src/node/ext/channel.cc +++ b/src/node/ext/channel.cc @@ -103,7 +103,7 @@ NAN_METHOD(Channel::New) { NanUtf8String *host = new NanUtf8String(args[0]); NanUtf8String *host_override = NULL; if (args[1]->IsUndefined()) { - wrapped_channel = grpc_channel_create(**host, NULL); + wrapped_channel = grpc_insecure_channel_create(**host, NULL); } else if (args[1]->IsObject()) { grpc_credentials *creds = NULL; Handle args_hash(args[1]->ToObject()->Clone()); @@ -148,7 +148,7 @@ NAN_METHOD(Channel::New) { } } if (creds == NULL) { - wrapped_channel = grpc_channel_create(**host, &channel_args); + wrapped_channel = grpc_insecure_channel_create(**host, &channel_args); } else { wrapped_channel = grpc_secure_channel_create(creds, **host, &channel_args); From 5029b30d1ce8509a4b2f7c17e88bddb8a5206f56 Mon Sep 17 00:00:00 2001 From: Julien Boeuf Date: Tue, 21 Jul 2015 23:02:16 -0700 Subject: [PATCH 15/47] Adding option to force client auth on the server SSL creds. --- include/grpc++/server_credentials.h | 1 + include/grpc/grpc_security.h | 9 ++++++--- src/core/security/credentials.c | 8 +++++--- src/core/security/security_connector.c | 7 ++++--- src/core/security/security_connector.h | 1 + src/core/tsi/ssl_transport_security.c | 8 +++++--- src/core/tsi/ssl_transport_security.h | 16 ++++++++++------ src/cpp/server/secure_server_credentials.cc | 3 ++- src/csharp/ext/grpc_csharp_ext.c | 3 ++- src/node/ext/server_credentials.cc | 4 +++- src/php/ext/grpc/server_credentials.c | 6 ++++-- .../grpc/_adapter/_c/types/server_credentials.c | 4 +++- src/ruby/ext/grpc/rb_server_credentials.c | 5 +++-- .../fixtures/chttp2_simple_ssl_fullstack.c | 2 +- .../chttp2_simple_ssl_fullstack_with_poll.c | 2 +- .../chttp2_simple_ssl_with_oauth2_fullstack.c | 2 +- test/core/fling/server.c | 2 +- 17 files changed, 53 insertions(+), 30 deletions(-) diff --git a/include/grpc++/server_credentials.h b/include/grpc++/server_credentials.h index 83ae9fd1eb0..94041c7757c 100644 --- a/include/grpc++/server_credentials.h +++ b/include/grpc++/server_credentials.h @@ -64,6 +64,7 @@ struct SslServerCredentialsOptions { }; grpc::string pem_root_certs; std::vector pem_key_cert_pairs; + bool force_client_auth; }; // Builds SSL ServerCredentials given SSL specific options diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 37d66c04ae5..7c5bd6433f7 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -87,7 +87,7 @@ typedef struct { directory). - pem_key_cert_pair is a pointer on the object containing client's private key and certificate chain. This parameter can be NULL if the client does - not have such a key/cert pair. */ + not have such a key/cert pair. */ grpc_credentials *grpc_ssl_credentials_create( const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair); @@ -177,10 +177,13 @@ void grpc_server_credentials_release(grpc_server_credentials *creds); - pem_key_cert_pairs is an array private key / certificate chains of the server. This parameter cannot be NULL. - num_key_cert_pairs indicates the number of items in the private_key_files - and cert_chain_files parameters. It should be at least 1. */ + and cert_chain_files parameters. It should be at least 1. + - force_client_auth, if set to non-zero will force the client to authenticate + with an SSL cert. Note that this option is ignored if pem_root_certs is + NULL. */ grpc_server_credentials *grpc_ssl_server_credentials_create( const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs, - size_t num_key_cert_pairs); + size_t num_key_cert_pairs, int force_client_auth); /* Creates a fake server transport security credentials object for testing. */ grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( diff --git a/src/core/security/credentials.c b/src/core/security/credentials.c index fb59fa4b0e5..a4d998a4290 100644 --- a/src/core/security/credentials.c +++ b/src/core/security/credentials.c @@ -259,8 +259,10 @@ static void ssl_build_config(const char *pem_root_certs, static void ssl_build_server_config( const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs, - size_t num_key_cert_pairs, grpc_ssl_server_config *config) { + size_t num_key_cert_pairs, int force_client_auth, + grpc_ssl_server_config *config) { size_t i; + config->force_client_auth = force_client_auth; if (pem_root_certs != NULL) { ssl_copy_key_material(pem_root_certs, &config->pem_root_certs, &config->pem_root_certs_size); @@ -302,14 +304,14 @@ grpc_credentials *grpc_ssl_credentials_create( grpc_server_credentials *grpc_ssl_server_credentials_create( const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs, - size_t num_key_cert_pairs) { + size_t num_key_cert_pairs, int force_client_auth) { grpc_ssl_server_credentials *c = gpr_malloc(sizeof(grpc_ssl_server_credentials)); memset(c, 0, sizeof(grpc_ssl_server_credentials)); c->base.type = GRPC_CREDENTIALS_TYPE_SSL; c->base.vtable = &ssl_server_vtable; ssl_build_server_config(pem_root_certs, pem_key_cert_pairs, - num_key_cert_pairs, &c->config); + num_key_cert_pairs, force_client_auth, &c->config); return &c->base; } diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c index f6e423eb279..726b4c1e123 100644 --- a/src/core/security/security_connector.c +++ b/src/core/security/security_connector.c @@ -653,9 +653,10 @@ grpc_security_status grpc_ssl_server_security_connector_create( config->pem_private_keys_sizes, (const unsigned char **)config->pem_cert_chains, config->pem_cert_chains_sizes, config->num_key_cert_pairs, - config->pem_root_certs, config->pem_root_certs_size, ssl_cipher_suites(), - alpn_protocol_strings, alpn_protocol_string_lengths, - (uint16_t)num_alpn_protocols, &c->handshaker_factory); + config->pem_root_certs, config->pem_root_certs_size, + config->force_client_auth, ssl_cipher_suites(), alpn_protocol_strings, + alpn_protocol_string_lengths, (uint16_t)num_alpn_protocols, + &c->handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); diff --git a/src/core/security/security_connector.h b/src/core/security/security_connector.h index a4c723f0260..2c9aa1c5a4e 100644 --- a/src/core/security/security_connector.h +++ b/src/core/security/security_connector.h @@ -201,6 +201,7 @@ typedef struct { size_t num_key_cert_pairs; unsigned char *pem_root_certs; size_t pem_root_certs_size; + int force_client_auth; } grpc_ssl_server_config; /* Creates an SSL server_security_connector. diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 6156a39d093..609fc06ed5d 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -1293,8 +1293,8 @@ tsi_result tsi_create_ssl_server_handshaker_factory( const size_t* pem_private_keys_sizes, const unsigned char** pem_cert_chains, const size_t* pem_cert_chains_sizes, size_t key_cert_pair_count, const unsigned char* pem_client_root_certs, - size_t pem_client_root_certs_size, const char* cipher_list, - const unsigned char** alpn_protocols, + size_t pem_client_root_certs_size, int force_client_auth, + const char* cipher_list, const unsigned char** alpn_protocols, const unsigned char* alpn_protocols_lengths, uint16_t num_alpn_protocols, tsi_ssl_handshaker_factory** factory) { tsi_ssl_server_handshaker_factory* impl = NULL; @@ -1349,6 +1349,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory( if (result != TSI_OK) break; if (pem_client_root_certs != NULL) { + int flags = SSL_VERIFY_PEER; STACK_OF(X509_NAME)* root_names = NULL; result = ssl_ctx_load_verification_certs( impl->ssl_contexts[i], pem_client_root_certs, @@ -1358,7 +1359,8 @@ tsi_result tsi_create_ssl_server_handshaker_factory( break; } SSL_CTX_set_client_CA_list(impl->ssl_contexts[i], root_names); - SSL_CTX_set_verify(impl->ssl_contexts[i], SSL_VERIFY_PEER, NULL); + if (force_client_auth) flags |= SSL_VERIFY_FAIL_IF_NO_PEER_CERT; + SSL_CTX_set_verify(impl->ssl_contexts[i], flags, NULL); /* TODO(jboeuf): Add revocation verification. */ } diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index b2aa2f393e5..4bf6c81b75d 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -107,10 +107,14 @@ tsi_result tsi_create_ssl_client_handshaker_factory( - key_cert_pair_count indicates the number of items in the private_key_files and cert_chain_files parameters. - pem_client_roots is the buffer containing the PEM encoding of the client - root certificates. This parameter may be NULL in which case the server - will not ask the client to authenticate itself with a certificate (server- - only authentication mode). - - pem_client_roots_size is the size of the associated buffer. + root certificates. This parameter may be NULL in which case the server will + not authenticate the client. If not NULL, the force_client_auth parameter + specifies if the server will accept only authenticated clients or both + authenticated and non-authenticated clients. + - pem_client_root_certs_size is the size of the associated buffer. + - force_client_auth, if set to non-zero will force the client to authenticate + with an SSL cert. Note that this option is ignored if pem_client_root_certs + is NULL or pem_client_roots_certs_size is 0 - cipher_suites contains an optional list of the ciphers that the server supports. The format of this string is described in: https://www.openssl.org/docs/apps/ciphers.html. @@ -131,8 +135,8 @@ tsi_result tsi_create_ssl_server_handshaker_factory( const size_t* pem_private_keys_sizes, const unsigned char** pem_cert_chains, const size_t* pem_cert_chains_sizes, size_t key_cert_pair_count, const unsigned char* pem_client_root_certs, - size_t pem_client_root_certs_size, const char* cipher_suites, - const unsigned char** alpn_protocols, + size_t pem_client_root_certs_size, int force_client_auth, + const char* cipher_suites, const unsigned char** alpn_protocols, const unsigned char* alpn_protocols_lengths, uint16_t num_alpn_protocols, tsi_ssl_handshaker_factory** factory); diff --git a/src/cpp/server/secure_server_credentials.cc b/src/cpp/server/secure_server_credentials.cc index 3e262dd74f7..32c45e22806 100644 --- a/src/cpp/server/secure_server_credentials.cc +++ b/src/cpp/server/secure_server_credentials.cc @@ -51,7 +51,8 @@ std::shared_ptr SslServerCredentials( } grpc_server_credentials* c_creds = grpc_ssl_server_credentials_create( options.pem_root_certs.empty() ? nullptr : options.pem_root_certs.c_str(), - &pem_key_cert_pairs[0], pem_key_cert_pairs.size()); + &pem_key_cert_pairs[0], pem_key_cert_pairs.size(), + options.force_client_auth); return std::shared_ptr( new SecureServerCredentials(c_creds)); } diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c index 7dd1959a5f7..3c0035d4538 100644 --- a/src/csharp/ext/grpc_csharp_ext.c +++ b/src/csharp/ext/grpc_csharp_ext.c @@ -718,8 +718,9 @@ grpcsharp_ssl_server_credentials_create( key_cert_pairs[i].private_key = key_cert_pair_private_key_array[i]; } } + /* TODO: Add a force_client_auth parameter and pass it here. */ creds = grpc_ssl_server_credentials_create(pem_root_certs, key_cert_pairs, - num_key_cert_pairs); + num_key_cert_pairs, 0); gpr_free(key_cert_pairs); return creds; } diff --git a/src/node/ext/server_credentials.cc b/src/node/ext/server_credentials.cc index d2b63cdc4ec..709105c72fc 100644 --- a/src/node/ext/server_credentials.cc +++ b/src/node/ext/server_credentials.cc @@ -140,8 +140,10 @@ NAN_METHOD(ServerCredentials::CreateSsl) { return NanThrowTypeError("createSsl's third argument must be a Buffer"); } key_cert_pair.cert_chain = ::node::Buffer::Data(args[2]); + // TODO Add a force_client_auth parameter and pass it as the last parameter + // here. NanReturnValue(WrapStruct( - grpc_ssl_server_credentials_create(root_certs, &key_cert_pair, 1))); + grpc_ssl_server_credentials_create(root_certs, &key_cert_pair, 1, 0))); } NAN_METHOD(ServerCredentials::CreateFake) { diff --git a/src/php/ext/grpc/server_credentials.c b/src/php/ext/grpc/server_credentials.c index c4c1fabb1a5..ec1c3c4f52c 100644 --- a/src/php/ext/grpc/server_credentials.c +++ b/src/php/ext/grpc/server_credentials.c @@ -115,8 +115,10 @@ PHP_METHOD(ServerCredentials, createSsl) { "createSsl expects 3 strings", 1 TSRMLS_CC); return; } - grpc_server_credentials *creds = - grpc_ssl_server_credentials_create(pem_root_certs, &pem_key_cert_pair, 1); + /* TODO: add a force_client_auth field in ServerCredentials and pass it as + * the last parameter. */ + grpc_server_credentials *creds = grpc_ssl_server_credentials_create( + pem_root_certs, &pem_key_cert_pair, 1, 0); zval *creds_object = grpc_php_wrap_server_credentials(creds); RETURN_DESTROY_ZVAL(creds_object); } diff --git a/src/python/src/grpc/_adapter/_c/types/server_credentials.c b/src/python/src/grpc/_adapter/_c/types/server_credentials.c index 2e02c8fe816..2277b5b907f 100644 --- a/src/python/src/grpc/_adapter/_c/types/server_credentials.c +++ b/src/python/src/grpc/_adapter/_c/types/server_credentials.c @@ -131,8 +131,10 @@ ServerCredentials *pygrpc_ServerCredentials_ssl( } self = (ServerCredentials *)type->tp_alloc(type, 0); + /* TODO: Add a force_client_auth parameter in the python object and pass it + here as the last arg. */ self->c_creds = grpc_ssl_server_credentials_create( - root_certs, key_cert_pairs, num_key_cert_pairs); + root_certs, key_cert_pairs, num_key_cert_pairs, 0); gpr_free(key_cert_pairs); return self; } diff --git a/src/ruby/ext/grpc/rb_server_credentials.c b/src/ruby/ext/grpc/rb_server_credentials.c index 5f409358907..62c211d7691 100644 --- a/src/ruby/ext/grpc/rb_server_credentials.c +++ b/src/ruby/ext/grpc/rb_server_credentials.c @@ -176,11 +176,12 @@ static VALUE grpc_rb_server_credentials_init(VALUE self, VALUE pem_root_certs, } key_cert_pair.private_key = RSTRING_PTR(pem_private_key); key_cert_pair.cert_chain = RSTRING_PTR(pem_cert_chain); + /* TODO Add a force_client_auth parameter and pass it here. */ if (pem_root_certs == Qnil) { - creds = grpc_ssl_server_credentials_create(NULL, &key_cert_pair, 1); + creds = grpc_ssl_server_credentials_create(NULL, &key_cert_pair, 1, 0); } else { creds = grpc_ssl_server_credentials_create(RSTRING_PTR(pem_root_certs), - &key_cert_pair, 1); + &key_cert_pair, 1, 0); } if (creds == NULL) { rb_raise(rb_eRuntimeError, "could not create a credentials, not sure why"); diff --git a/test/core/end2end/fixtures/chttp2_simple_ssl_fullstack.c b/test/core/end2end/fixtures/chttp2_simple_ssl_fullstack.c index 73a36116fb1..6d5669d05a8 100644 --- a/test/core/end2end/fixtures/chttp2_simple_ssl_fullstack.c +++ b/test/core/end2end/fixtures/chttp2_simple_ssl_fullstack.c @@ -115,7 +115,7 @@ static void chttp2_init_server_simple_ssl_secure_fullstack( grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, test_server1_cert}; grpc_server_credentials *ssl_creds = - grpc_ssl_server_credentials_create(NULL, &pem_cert_key_pair, 1); + grpc_ssl_server_credentials_create(NULL, &pem_cert_key_pair, 1, 0); chttp2_init_server_secure_fullstack(f, server_args, ssl_creds); } diff --git a/test/core/end2end/fixtures/chttp2_simple_ssl_fullstack_with_poll.c b/test/core/end2end/fixtures/chttp2_simple_ssl_fullstack_with_poll.c index b1ac3e535f1..d0cc3dd74a2 100644 --- a/test/core/end2end/fixtures/chttp2_simple_ssl_fullstack_with_poll.c +++ b/test/core/end2end/fixtures/chttp2_simple_ssl_fullstack_with_poll.c @@ -115,7 +115,7 @@ static void chttp2_init_server_simple_ssl_secure_fullstack( grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, test_server1_cert}; grpc_server_credentials *ssl_creds = - grpc_ssl_server_credentials_create(NULL, &pem_cert_key_pair, 1); + grpc_ssl_server_credentials_create(NULL, &pem_cert_key_pair, 1, 0); chttp2_init_server_secure_fullstack(f, server_args, ssl_creds); } diff --git a/test/core/end2end/fixtures/chttp2_simple_ssl_with_oauth2_fullstack.c b/test/core/end2end/fixtures/chttp2_simple_ssl_with_oauth2_fullstack.c index de418bf7ee0..f74ed9365f7 100644 --- a/test/core/end2end/fixtures/chttp2_simple_ssl_with_oauth2_fullstack.c +++ b/test/core/end2end/fixtures/chttp2_simple_ssl_with_oauth2_fullstack.c @@ -120,7 +120,7 @@ static void chttp2_init_server_simple_ssl_secure_fullstack( grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {test_server1_key, test_server1_cert}; grpc_server_credentials *ssl_creds = - grpc_ssl_server_credentials_create(NULL, &pem_key_cert_pair, 1); + grpc_ssl_server_credentials_create(NULL, &pem_key_cert_pair, 1, 0); chttp2_init_server_secure_fullstack(f, server_args, ssl_creds); } diff --git a/test/core/fling/server.c b/test/core/fling/server.c index 082bbd368a7..8f349044d97 100644 --- a/test/core/fling/server.c +++ b/test/core/fling/server.c @@ -210,7 +210,7 @@ int main(int argc, char **argv) { grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {test_server1_key, test_server1_cert}; grpc_server_credentials *ssl_creds = - grpc_ssl_server_credentials_create(NULL, &pem_key_cert_pair, 1); + grpc_ssl_server_credentials_create(NULL, &pem_key_cert_pair, 1, 0); server = grpc_server_create(NULL); GPR_ASSERT(grpc_server_add_secure_http2_port(server, addr, ssl_creds)); grpc_server_credentials_release(ssl_creds); From 6999c096d8fb683cbb78dd27be477d22d64c291f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 22 Jul 2015 08:14:12 -0700 Subject: [PATCH 16/47] Canonicalize metadata keys in core --- src/core/channel/compress_filter.c | 10 ++--- src/core/channel/http_server_filter.c | 6 +-- src/core/security/client_auth_filter.c | 10 ++--- src/core/surface/call.c | 6 +-- src/core/surface/channel.c | 22 +++++----- src/core/surface/server.c | 8 ++-- src/core/transport/chttp2/stream_encoder.c | 4 +- src/core/transport/chttp2_transport.c | 2 +- src/core/transport/metadata.c | 40 ++++++++++++++++--- src/core/transport/metadata.h | 5 ++- ...quest_response_with_metadata_and_payload.c | 4 +- test/core/transport/metadata_test.c | 16 ++++---- 12 files changed, 82 insertions(+), 51 deletions(-) diff --git a/src/core/channel/compress_filter.c b/src/core/channel/compress_filter.c index 14cb3da62da..e264440d71b 100644 --- a/src/core/channel/compress_filter.c +++ b/src/core/channel/compress_filter.c @@ -282,19 +282,19 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master, grpc_channel_args_get_compression_algorithm(args); channeld->mdstr_request_compression_algorithm_key = - grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY); + grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY, 0); channeld->mdstr_outgoing_compression_algorithm_key = - grpc_mdstr_from_string(mdctx, "grpc-encoding"); + grpc_mdstr_from_string(mdctx, "grpc-encoding", 0); for (algo_idx = 0; algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) { - char *algorith_name; - GPR_ASSERT(grpc_compression_algorithm_name(algo_idx, &algorith_name) != 0); + char *algorithm_name; + GPR_ASSERT(grpc_compression_algorithm_name(algo_idx, &algorithm_name) != 0); channeld->mdelem_compression_algorithms[algo_idx] = grpc_mdelem_from_metadata_strings( mdctx, grpc_mdstr_ref(channeld->mdstr_outgoing_compression_algorithm_key), - grpc_mdstr_from_string(mdctx, algorith_name)); + grpc_mdstr_from_string(mdctx, algorithm_name, 0)); } GPR_ASSERT(!is_last); diff --git a/src/core/channel/http_server_filter.c b/src/core/channel/http_server_filter.c index a6cbb5a7f4a..98f7b8a637a 100644 --- a/src/core/channel/http_server_filter.c +++ b/src/core/channel/http_server_filter.c @@ -250,9 +250,9 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master, channeld->http_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "http"); channeld->https_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "https"); channeld->grpc_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "grpc"); - channeld->path_key = grpc_mdstr_from_string(mdctx, ":path"); - channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority"); - channeld->host_key = grpc_mdstr_from_string(mdctx, "host"); + channeld->path_key = grpc_mdstr_from_string(mdctx, ":path", 0); + channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority", 0); + channeld->host_key = grpc_mdstr_from_string(mdctx, "host", 0); channeld->content_type = grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc"); diff --git a/src/core/security/client_auth_filter.c b/src/core/security/client_auth_filter.c index 9e49a807f12..a0e4d86b6fe 100644 --- a/src/core/security/client_auth_filter.c +++ b/src/core/security/client_auth_filter.c @@ -80,7 +80,7 @@ static void bubble_up_error(grpc_call_element *elem, const char *error_msg) { channel_data *chand = elem->channel_data; grpc_transport_stream_op_add_cancellation( &calld->op, GRPC_STATUS_UNAUTHENTICATED, - grpc_mdstr_from_string(chand->md_ctx, error_msg)); + grpc_mdstr_from_string(chand->md_ctx, error_msg, 0)); grpc_call_next_op(elem, &calld->op); } @@ -316,10 +316,10 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master, (grpc_channel_security_connector *)GRPC_SECURITY_CONNECTOR_REF( sc, "client_auth_filter"); chand->md_ctx = metadata_context; - chand->authority_string = grpc_mdstr_from_string(chand->md_ctx, ":authority"); - chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path"); - chand->error_msg_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-message"); - chand->status_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-status"); + chand->authority_string = grpc_mdstr_from_string(chand->md_ctx, ":authority", 0); + chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path", 0); + chand->error_msg_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-message", 0); + chand->status_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-status", 0); } /* Destructor for channel data */ diff --git a/src/core/surface/call.c b/src/core/surface/call.c index 6e643b591cf..278f568f212 100644 --- a/src/core/surface/call.c +++ b/src/core/surface/call.c @@ -932,7 +932,7 @@ static int prepare_application_metadata(grpc_call *call, size_t count, GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data)); l->md = grpc_mdelem_from_string_and_buffer(call->metadata_context, md->key, (const gpr_uint8 *)md->value, - md->value_length); + md->value_length, 1); if (!grpc_mdstr_is_legal_header(l->md->key)) { gpr_log(GPR_ERROR, "attempt to send invalid metadata key"); return 0; @@ -1203,7 +1203,7 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, static grpc_call_error cancel_with_status(grpc_call *c, grpc_status_code status, const char *description) { grpc_mdstr *details = - description ? grpc_mdstr_from_string(c->metadata_context, description) + description ? grpc_mdstr_from_string(c->metadata_context, description, 0) : NULL; GPR_ASSERT(status != GRPC_STATUS_OK); @@ -1491,7 +1491,7 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops, op->data.send_status_from_server.status_details != NULL ? grpc_mdstr_from_string( call->metadata_context, - op->data.send_status_from_server.status_details) + op->data.send_status_from_server.status_details, 0) : NULL; req = &reqs[out++]; req->op = GRPC_IOREQ_SEND_CLOSE; diff --git a/src/core/surface/channel.c b/src/core/surface/channel.c index a6438ff512a..9035b6bc6e2 100644 --- a/src/core/surface/channel.c +++ b/src/core/surface/channel.c @@ -97,19 +97,19 @@ grpc_channel *grpc_channel_create_from_filters( /* decremented by grpc_channel_destroy */ gpr_ref_init(&channel->refs, 1); channel->metadata_context = mdctx; - channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status"); + channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status", 0); channel->grpc_compression_algorithm_string = - grpc_mdstr_from_string(mdctx, "grpc-encoding"); - channel->grpc_message_string = grpc_mdstr_from_string(mdctx, "grpc-message"); + grpc_mdstr_from_string(mdctx, "grpc-encoding", 0); + channel->grpc_message_string = grpc_mdstr_from_string(mdctx, "grpc-message", 0); for (i = 0; i < NUM_CACHED_STATUS_ELEMS; i++) { char buf[GPR_LTOA_MIN_BUFSIZE]; gpr_ltoa(i, buf); channel->grpc_status_elem[i] = grpc_mdelem_from_metadata_strings( mdctx, GRPC_MDSTR_REF(channel->grpc_status_string), - grpc_mdstr_from_string(mdctx, buf)); + grpc_mdstr_from_string(mdctx, buf, 0)); } - channel->path_string = grpc_mdstr_from_string(mdctx, ":path"); - channel->authority_string = grpc_mdstr_from_string(mdctx, ":authority"); + channel->path_string = grpc_mdstr_from_string(mdctx, ":path", 0); + channel->authority_string = grpc_mdstr_from_string(mdctx, ":authority", 0); gpr_mu_init(&channel->registered_call_mu); channel->registered_calls = NULL; @@ -159,10 +159,10 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel, channel, cq, grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->path_string), - grpc_mdstr_from_string(channel->metadata_context, method)), + grpc_mdstr_from_string(channel->metadata_context, method, 0)), grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->authority_string), - grpc_mdstr_from_string(channel->metadata_context, host)), + grpc_mdstr_from_string(channel->metadata_context, host, 0)), deadline); } @@ -171,10 +171,10 @@ void *grpc_channel_register_call(grpc_channel *channel, const char *method, registered_call *rc = gpr_malloc(sizeof(registered_call)); rc->path = grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->path_string), - grpc_mdstr_from_string(channel->metadata_context, method)); + grpc_mdstr_from_string(channel->metadata_context, method, 0)); rc->authority = grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->authority_string), - grpc_mdstr_from_string(channel->metadata_context, host)); + grpc_mdstr_from_string(channel->metadata_context, host, 0)); gpr_mu_lock(&channel->registered_call_mu); rc->next = channel->registered_calls; channel->registered_calls = rc; @@ -275,7 +275,7 @@ grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel, int i) { gpr_ltoa(i, tmp); return grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->grpc_status_string), - grpc_mdstr_from_string(channel->metadata_context, tmp)); + grpc_mdstr_from_string(channel->metadata_context, tmp, 0)); } } diff --git a/src/core/surface/server.c b/src/core/surface/server.c index f2d6b11bc79..fdd772deefd 100644 --- a/src/core/surface/server.c +++ b/src/core/surface/server.c @@ -677,8 +677,8 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master, GPR_ASSERT(!is_last); chand->server = NULL; chand->channel = NULL; - chand->path_key = grpc_mdstr_from_string(metadata_context, ":path"); - chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority"); + chand->path_key = grpc_mdstr_from_string(metadata_context, ":path", 0); + chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority", 0); chand->next = chand->prev = chand; chand->registered_methods = NULL; chand->connectivity_state = GRPC_CHANNEL_IDLE; @@ -899,8 +899,8 @@ void grpc_server_setup_transport(grpc_server *s, grpc_transport *transport, chand->registered_methods = gpr_malloc(alloc); memset(chand->registered_methods, 0, alloc); for (rm = s->registered_methods; rm; rm = rm->next) { - host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host) : NULL; - method = grpc_mdstr_from_string(mdctx, rm->method); + host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host, 0) : NULL; + method = grpc_mdstr_from_string(mdctx, rm->method, 0); hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash); for (probes = 0; chand->registered_methods[(hash + probes) % slots] .server_registered_method != NULL; diff --git a/src/core/transport/chttp2/stream_encoder.c b/src/core/transport/chttp2/stream_encoder.c index d30059abf8c..3dfea3ca4ba 100644 --- a/src/core/transport/chttp2/stream_encoder.c +++ b/src/core/transport/chttp2/stream_encoder.c @@ -441,7 +441,7 @@ static void deadline_enc(grpc_chttp2_hpack_compressor *c, gpr_timespec deadline, gpr_time_sub(deadline, gpr_now(GPR_CLOCK_REALTIME)), timeout_str); mdelem = grpc_mdelem_from_metadata_strings( c->mdctx, GRPC_MDSTR_REF(c->timeout_key_str), - grpc_mdstr_from_string(c->mdctx, timeout_str)); + grpc_mdstr_from_string(c->mdctx, timeout_str, 0)); mdelem = hpack_enc(c, mdelem, st); if (mdelem) GRPC_MDELEM_UNREF(mdelem); } @@ -456,7 +456,7 @@ void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor *c, grpc_mdctx *ctx) { memset(c, 0, sizeof(*c)); c->mdctx = ctx; - c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout"); + c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout", 0); } void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor *c) { diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index c923d5e42fb..634bedd6253 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -228,7 +228,7 @@ static void init_transport(grpc_chttp2_transport *t, t->global.pings.next = t->global.pings.prev = &t->global.pings; t->parsing.is_client = is_client; t->parsing.str_grpc_timeout = - grpc_mdstr_from_string(t->metadata_context, "grpc-timeout"); + grpc_mdstr_from_string(t->metadata_context, "grpc-timeout", 0); t->parsing.deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; t->writing.is_client = is_client; diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c index e95b7a21f9a..c5012baa248 100644 --- a/src/core/transport/metadata.c +++ b/src/core/transport/metadata.c @@ -309,7 +309,36 @@ static void slice_unref(void *p) { unlock(ctx); } -grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str) { +grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str, int canonicalize_key) { + if (canonicalize_key) { + size_t len; + size_t i; + int canonical = 1; + + for (i = 0; str[i]; i++) { + if (str[i] >= 'A' && str[i] <= 'Z') { + canonical = 0; + } + } + len = i; + + if (canonical) { + return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, len); + } else { + char *copy = gpr_malloc(len + 1); + grpc_mdstr *ret; + for (i = 0; i < len; i++) { + if (str[i] >= 'A' && str[i] <= 'Z') { + copy[i] = str[i] - 'A' + 'a'; + } else { + copy[i] = str[i]; + } + } + ret = grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)copy, len); + gpr_free(copy); + return ret; + } + } return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, strlen(str)); } @@ -491,8 +520,8 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx, grpc_mdelem *grpc_mdelem_from_strings(grpc_mdctx *ctx, const char *key, const char *value) { return grpc_mdelem_from_metadata_strings(ctx, - grpc_mdstr_from_string(ctx, key), - grpc_mdstr_from_string(ctx, value)); + grpc_mdstr_from_string(ctx, key, 0), + grpc_mdstr_from_string(ctx, value, 0)); } grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key, @@ -504,9 +533,10 @@ grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key, grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx, const char *key, const gpr_uint8 *value, - size_t value_length) { + size_t value_length, + int canonicalize_key) { return grpc_mdelem_from_metadata_strings( - ctx, grpc_mdstr_from_string(ctx, key), + ctx, grpc_mdstr_from_string(ctx, key, canonicalize_key), grpc_mdstr_from_buffer(ctx, value, value_length)); } diff --git a/src/core/transport/metadata.h b/src/core/transport/metadata.h index 99b15322c39..15ef9bb555e 100644 --- a/src/core/transport/metadata.h +++ b/src/core/transport/metadata.h @@ -95,7 +95,7 @@ size_t grpc_mdctx_get_mdtab_free_test_only(grpc_mdctx *mdctx); /* Constructors for grpc_mdstr instances; take a variety of data types that clients may have handy */ -grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str); +grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str, int perform_key_canonicalization); /* Unrefs the slice. */ grpc_mdstr *grpc_mdstr_from_slice(grpc_mdctx *ctx, gpr_slice slice); grpc_mdstr *grpc_mdstr_from_buffer(grpc_mdctx *ctx, const gpr_uint8 *str, @@ -117,7 +117,8 @@ grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key, grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx, const char *key, const gpr_uint8 *value, - size_t value_length); + size_t value_length, + int canonicalize_key); /* Mutator and accessor for grpc_mdelem user data. The destructor function is used as a type tag and is checked during user_data fetch. */ diff --git a/test/core/end2end/tests/request_response_with_metadata_and_payload.c b/test/core/end2end/tests/request_response_with_metadata_and_payload.c index ef6dfe9561f..9821d7852f3 100644 --- a/test/core/end2end/tests/request_response_with_metadata_and_payload.c +++ b/test/core/end2end/tests/request_response_with_metadata_and_payload.c @@ -111,8 +111,8 @@ static void test_request_response_with_metadata_and_payload( gpr_timespec deadline = five_seconds_time(); grpc_metadata meta_c[2] = {{"key1", "val1", 4, {{NULL, NULL, NULL}}}, {"key2", "val2", 4, {{NULL, NULL, NULL}}}}; - grpc_metadata meta_s[2] = {{"key3", "val3", 4, {{NULL, NULL, NULL}}}, - {"key4", "val4", 4, {{NULL, NULL, NULL}}}}; + grpc_metadata meta_s[2] = {{"KeY3", "val3", 4, {{NULL, NULL, NULL}}}, + {"KeY4", "val4", 4, {{NULL, NULL, NULL}}}}; grpc_end2end_test_fixture f = begin_test( config, "test_request_response_with_metadata_and_payload", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); diff --git a/test/core/transport/metadata_test.c b/test/core/transport/metadata_test.c index a932e04f330..4a4d4bcb8a1 100644 --- a/test/core/transport/metadata_test.c +++ b/test/core/transport/metadata_test.c @@ -63,9 +63,9 @@ static void test_create_string(void) { LOG_TEST("test_create_string"); ctx = grpc_mdctx_create(); - s1 = grpc_mdstr_from_string(ctx, "hello"); - s2 = grpc_mdstr_from_string(ctx, "hello"); - s3 = grpc_mdstr_from_string(ctx, "very much not hello"); + s1 = grpc_mdstr_from_string(ctx, "hello", 0); + s2 = grpc_mdstr_from_string(ctx, "hello", 0); + s3 = grpc_mdstr_from_string(ctx, "very much not hello", 0); GPR_ASSERT(s1 == s2); GPR_ASSERT(s3 != s1); GPR_ASSERT(gpr_slice_str_cmp(s1->slice, "hello") == 0); @@ -190,7 +190,7 @@ static void test_things_stick_around(void) { for (i = 0; i < nstrs; i++) { gpr_asprintf(&buffer, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx%dx", i); - strs[i] = grpc_mdstr_from_string(ctx, buffer); + strs[i] = grpc_mdstr_from_string(ctx, buffer, 0); shuf[i] = i; gpr_free(buffer); } @@ -212,7 +212,7 @@ static void test_things_stick_around(void) { GRPC_MDSTR_UNREF(strs[shuf[i]]); for (j = i + 1; j < nstrs; j++) { gpr_asprintf(&buffer, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx%dx", shuf[j]); - test = grpc_mdstr_from_string(ctx, buffer); + test = grpc_mdstr_from_string(ctx, buffer, 0); GPR_ASSERT(test == strs[shuf[j]]); GRPC_MDSTR_UNREF(test); gpr_free(buffer); @@ -235,13 +235,13 @@ static void test_slices_work(void) { ctx = grpc_mdctx_create(); str = grpc_mdstr_from_string( - ctx, "123456789012345678901234567890123456789012345678901234567890"); + ctx, "123456789012345678901234567890123456789012345678901234567890", 0); slice = gpr_slice_ref(str->slice); GRPC_MDSTR_UNREF(str); gpr_slice_unref(slice); str = grpc_mdstr_from_string( - ctx, "123456789012345678901234567890123456789012345678901234567890"); + ctx, "123456789012345678901234567890123456789012345678901234567890", 0); slice = gpr_slice_ref(str->slice); gpr_slice_unref(slice); GRPC_MDSTR_UNREF(str); @@ -258,7 +258,7 @@ static void test_base64_and_huffman_works(void) { LOG_TEST("test_base64_and_huffman_works"); ctx = grpc_mdctx_create(); - str = grpc_mdstr_from_string(ctx, "abcdefg"); + str = grpc_mdstr_from_string(ctx, "abcdefg", 0); slice1 = grpc_mdstr_as_base64_encoded_and_huffman_compressed(str); slice2 = grpc_chttp2_base64_encode_and_huffman_compress(str->slice); GPR_ASSERT(0 == gpr_slice_cmp(slice1, slice2)); From 6c22619d7b656eef1dc3e4d3617f58d9d2678953 Mon Sep 17 00:00:00 2001 From: vjpai Date: Wed, 22 Jul 2015 14:34:58 -0700 Subject: [PATCH 17/47] Debugging support to check for double-push and valid pop --- src/core/support/stack_lockfree.c | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/core/support/stack_lockfree.c b/src/core/support/stack_lockfree.c index 28443303797..f24e272207f 100644 --- a/src/core/support/stack_lockfree.c +++ b/src/core/support/stack_lockfree.c @@ -72,6 +72,11 @@ typedef union lockfree_node { struct gpr_stack_lockfree { lockfree_node *entries; lockfree_node head; /* An atomic entry describing curr head */ + +#ifndef NDEBUG + /* Bitmap of pushed entries to check for double-push or pop */ + gpr_atm pushed[(INVALID_ENTRY_INDEX+1)/(8*sizeof(gpr_atm))]; +#endif }; gpr_stack_lockfree *gpr_stack_lockfree_create(int entries) { @@ -86,6 +91,9 @@ gpr_stack_lockfree *gpr_stack_lockfree_create(int entries) { /* Clear out all entries */ memset(stack->entries, 0, entries * sizeof(stack->entries[0])); memset(&stack->head, 0, sizeof(stack->head)); +#ifndef NDEBUG + memset(&stack->pushed, 0, sizeof(stack->pushed)); +#endif /* Point the head at reserved dummy entry */ stack->head.contents.index = INVALID_ENTRY_INDEX; @@ -106,6 +114,19 @@ int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) { /* Also post-increment the aba_ctr */ newhead.contents.aba_ctr = stack->entries[entry].contents.aba_ctr++; +#ifndef NDEBUG + /* Check for double push */ + { + int pushed_index = entry / (8*sizeof(gpr_atm)); + int pushed_bit = entry % (8*sizeof(gpr_atm)); + gpr_atm old_val; + + old_val = gpr_atm_no_barrier_fetch_add(&stack->pushed[pushed_index], + (gpr_atm)(1UL << pushed_bit)); + GPR_ASSERT((old_val & (1UL<head.atm)); @@ -119,6 +140,7 @@ int gpr_stack_lockfree_push(gpr_stack_lockfree *stack, int entry) { int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack) { lockfree_node head; lockfree_node newhead; + do { head.atm = gpr_atm_acq_load(&(stack->head.atm)); if (head.contents.index == INVALID_ENTRY_INDEX) { @@ -128,5 +150,18 @@ int gpr_stack_lockfree_pop(gpr_stack_lockfree *stack) { gpr_atm_no_barrier_load(&(stack->entries[head.contents.index].atm)); } while (!gpr_atm_no_barrier_cas(&(stack->head.atm), head.atm, newhead.atm)); +#ifndef NDEBUG + /* Check for valid pop */ + { + int pushed_index = head.contents.index / (8*sizeof(gpr_atm)); + int pushed_bit = head.contents.index % (8*sizeof(gpr_atm)); + gpr_atm old_val; + + old_val = gpr_atm_no_barrier_fetch_add(&stack->pushed[pushed_index], + -(gpr_atm)(1UL << pushed_bit)); + GPR_ASSERT((old_val & (1UL< Date: Wed, 22 Jul 2015 17:19:53 -0700 Subject: [PATCH 18/47] ObjC formatting fix --- src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m b/src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m index d522ddaae68..8518f78c5b2 100644 --- a/src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m +++ b/src/objective-c/GRPCClient/private/GRPCUnsecuredChannel.m @@ -38,8 +38,7 @@ @implementation GRPCUnsecuredChannel - (instancetype)initWithHost:(NSString *)host { - return (self = [super initWithChannel:grpc_insecure_channel_create( - host.UTF8String, NULL)]); + return (self = [super initWithChannel:grpc_insecure_channel_create(host.UTF8String, NULL)]); } @end From 6c2d56b7efd8a00287bcd1ef5fbbdaf3fee0d567 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 22 Jul 2015 17:22:33 -0700 Subject: [PATCH 19/47] Addressing review comments --- src/core/transport/metadata.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c index c5012baa248..967fd4898c5 100644 --- a/src/core/transport/metadata.c +++ b/src/core/transport/metadata.c @@ -318,6 +318,7 @@ grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str, int canonic for (i = 0; str[i]; i++) { if (str[i] >= 'A' && str[i] <= 'Z') { canonical = 0; + /* Keep going in loop just to get string length */ } } len = i; @@ -325,7 +326,7 @@ grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str, int canonic if (canonical) { return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, len); } else { - char *copy = gpr_malloc(len + 1); + char *copy = gpr_malloc(len); grpc_mdstr *ret; for (i = 0; i < len; i++) { if (str[i] >= 'A' && str[i] <= 'Z') { From f7e7d089fe3bec0ca2cea87183c15ca1f735caaf Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 23 Jul 2015 10:52:23 -0700 Subject: [PATCH 20/47] Integration fix --- src/core/channel/http_client_filter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/channel/http_client_filter.c b/src/core/channel/http_client_filter.c index 6ae84880707..91125cb1495 100644 --- a/src/core/channel/http_client_filter.c +++ b/src/core/channel/http_client_filter.c @@ -233,7 +233,7 @@ static grpc_mdstr *user_agent_from_args(grpc_mdctx *mdctx, tmp = gpr_strvec_flatten(&v, NULL); gpr_strvec_destroy(&v); - result = grpc_mdstr_from_string(mdctx, tmp); + result = grpc_mdstr_from_string(mdctx, tmp, 0); gpr_free(tmp); return result; @@ -260,7 +260,7 @@ static void init_channel_elem(grpc_channel_element *elem, grpc_channel *master, grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc"); channeld->status = grpc_mdelem_from_strings(mdctx, ":status", "200"); channeld->user_agent = grpc_mdelem_from_metadata_strings( - mdctx, grpc_mdstr_from_string(mdctx, "user-agent"), + mdctx, grpc_mdstr_from_string(mdctx, "user-agent", 0), user_agent_from_args(mdctx, args)); } From ac91eddb606fce140a4f2ee3e99fe81f2efa59bc Mon Sep 17 00:00:00 2001 From: Marcin Wyszynski Date: Thu, 23 Jul 2015 19:59:46 +0200 Subject: [PATCH 21/47] Avoid implicit conversion on array_length --- src/ruby/ext/grpc/rb_call.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c index bfb9f6ff01b..7470698e7af 100644 --- a/src/ruby/ext/grpc/rb_call.c +++ b/src/ruby/ext/grpc/rb_call.c @@ -235,7 +235,7 @@ static VALUE grpc_rb_call_set_metadata(VALUE self, VALUE metadata) { */ static int grpc_rb_md_ary_fill_hash_cb(VALUE key, VALUE val, VALUE md_ary_obj) { grpc_metadata_array *md_ary = NULL; - int array_length; + long array_length; int i; /* Construct a metadata object from key and value and add it */ From ea0c18b3f2f93c4f693646a22810a8fad00a7fb3 Mon Sep 17 00:00:00 2001 From: Marcin Wyszynski Date: Thu, 23 Jul 2015 20:00:32 +0200 Subject: [PATCH 22/47] Make time_t to int conversion explicit on tv_nsec --- src/ruby/ext/grpc/rb_grpc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c index 829f8255979..b0b37a87050 100644 --- a/src/ruby/ext/grpc/rb_grpc.c +++ b/src/ruby/ext/grpc/rb_grpc.c @@ -139,7 +139,7 @@ gpr_timespec grpc_rb_time_timeval(VALUE time, int interval) { rb_raise(rb_eRangeError, "%f out of Time range", RFLOAT_VALUE(time)); } - t.tv_nsec = (time_t)(d * 1e9 + 0.5); + t.tv_nsec = (int)(time_t)(d * 1e9 + 0.5); } break; From 1a2ac33f1fcc2b9f38647d2d7912de7d8c277afa Mon Sep 17 00:00:00 2001 From: Marcin Wyszynski Date: Thu, 23 Jul 2015 20:12:33 +0200 Subject: [PATCH 23/47] Avoid stupid double conversion --- src/ruby/ext/grpc/rb_grpc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c index b0b37a87050..1bb402026e5 100644 --- a/src/ruby/ext/grpc/rb_grpc.c +++ b/src/ruby/ext/grpc/rb_grpc.c @@ -139,7 +139,7 @@ gpr_timespec grpc_rb_time_timeval(VALUE time, int interval) { rb_raise(rb_eRangeError, "%f out of Time range", RFLOAT_VALUE(time)); } - t.tv_nsec = (int)(time_t)(d * 1e9 + 0.5); + t.tv_nsec = (int)(d * 1e9 + 0.5); } break; From e2f2e9a31a39caf56a216db0c53c74a0a52606de Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 23 Jul 2015 11:35:10 -0700 Subject: [PATCH 24/47] Integration fix --- src/python/src/grpc/_adapter/_low_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/src/grpc/_adapter/_low_test.py b/src/python/src/grpc/_adapter/_low_test.py index a49cd007bfa..9a8edfad0cb 100644 --- a/src/python/src/grpc/_adapter/_low_test.py +++ b/src/python/src/grpc/_adapter/_low_test.py @@ -97,7 +97,7 @@ class InsecureServerInsecureClient(unittest.TestCase): CLIENT_METADATA_BIN_VALUE = b'\0'*1000 SERVER_INITIAL_METADATA_KEY = 'init_me_me_me' SERVER_INITIAL_METADATA_VALUE = 'whodawha?' - SERVER_TRAILING_METADATA_KEY = 'California_is_in_a_drought' + SERVER_TRAILING_METADATA_KEY = 'california_is_in_a_drought' SERVER_TRAILING_METADATA_VALUE = 'zomg it is' SERVER_STATUS_CODE = _types.StatusCode.OK SERVER_STATUS_DETAILS = 'our work is never over' From 4243ca878e2de795eccb7516bc45865755bc0e5b Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Thu, 23 Jul 2015 23:47:56 +0200 Subject: [PATCH 25/47] Supporting FreeBSD for Jenkins. --- tools/jenkins/run_jenkins.sh | 11 ++++++++++- tools/run_tests/run_tests.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 56f9e82ca58..f4fdbc44218 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Copyright 2015, Google Inc. # All rights reserved. # @@ -31,6 +31,8 @@ # This script is invoked by Jenkins and triggers a test run based on # env variable settings. # +# Bootstrap into bash +[ -z $1 ] && exec bash $0 bootstrapped # Setting up rvm environment BEFORE we set -ex. [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh # To prevent cygwin bash complaining about empty lines ending with \r @@ -103,11 +105,18 @@ then /cygdrive/c/nuget/nuget.exe restore src/csharp/Grpc.sln python tools/run_tests/run_tests.py -t -l $language -x report.xml || true + elif [ "$platform" == "macos" ] then echo "building $language on MacOS" ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml || true + +elif [ "$platform" == "freebsd" ] +then + echo "building $language on FreeBSD" + + MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml || true else echo "Unknown platform $platform" exit 1 diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 1f44fc34fa8..682cb96785b 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -458,7 +458,7 @@ if platform.system() == 'Windows': cwd='vsprojects', shell=True) else: def make_jobspec(cfg, targets): - return jobset.JobSpec(['make', + return jobset.JobSpec([os.getenv('MAKE', 'make'), '-j', '%d' % (multiprocessing.cpu_count() + 1), 'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % args.slowdown, From a5fea60e8d8c7bdf0cc82ac04a5c086b900d7531 Mon Sep 17 00:00:00 2001 From: Marcin Wyszynski Date: Fri, 24 Jul 2015 10:24:32 +0200 Subject: [PATCH 26/47] array_length and it's counter types should match --- src/ruby/ext/grpc/rb_call.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c index 7470698e7af..a7607a83a36 100644 --- a/src/ruby/ext/grpc/rb_call.c +++ b/src/ruby/ext/grpc/rb_call.c @@ -236,7 +236,7 @@ static VALUE grpc_rb_call_set_metadata(VALUE self, VALUE metadata) { static int grpc_rb_md_ary_fill_hash_cb(VALUE key, VALUE val, VALUE md_ary_obj) { grpc_metadata_array *md_ary = NULL; long array_length; - int i; + long i; /* Construct a metadata object from key and value and add it */ TypedData_Get_Struct(md_ary_obj, grpc_metadata_array, From 46f2860b5d1dcf1903c925cd8f6c99c8ab566e31 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 24 Jul 2015 14:23:44 -0700 Subject: [PATCH 27/47] Added documentation command and settings to Node package --- src/node/jsdoc_conf.json | 22 ++++++++++++++++++++++ src/node/package.json | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/node/jsdoc_conf.json diff --git a/src/node/jsdoc_conf.json b/src/node/jsdoc_conf.json new file mode 100644 index 00000000000..876a8e19c6e --- /dev/null +++ b/src/node/jsdoc_conf.json @@ -0,0 +1,22 @@ +{ + "tags": { + "allowUnknownTags": true + }, + "source": { + "include": [ "index.js", "src" ], + "includePattern": ".+\\.js(doc)?$", + "excludePattern": "(^|\\/|\\\\)_" + }, + "opts": { + "package": "package.json", + "readme": "README.md" + }, + "plugins": [], + "templates": { + "cleverLinks": false, + "monospaceLinks": false, + "default": { + "outputSourceFiles": true + } + } +} diff --git a/src/node/package.json b/src/node/package.json index 1caf1587923..756d41b0af2 100644 --- a/src/node/package.json +++ b/src/node/package.json @@ -21,7 +21,8 @@ }, "scripts": { "lint": "node ./node_modules/jshint/bin/jshint src test examples interop index.js", - "test": "node ./node_modules/mocha/bin/mocha && npm run-script lint" + "test": "node ./node_modules/mocha/bin/mocha && npm run-script lint", + "gen_docs": "./node_modules/.bin/jsdoc -c jsdoc_conf.json" }, "dependencies": { "bindings": "^1.2.0", @@ -32,6 +33,7 @@ "devDependencies": { "async": "^0.9.0", "google-auth-library": "^0.9.2", + "jsdoc": "^3.3.2", "jshint": "^2.5.0", "minimist": "^1.1.0", "mocha": "~1.21.0", From cd7a4054de7b77a425b544fa8b3f424b9fede142 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sat, 25 Jul 2015 01:47:25 -0700 Subject: [PATCH 28/47] make work with out-of-band oauth2 token more pleasant --- src/csharp/Grpc.Auth/Grpc.Auth.csproj | 2 +- ...ceptorFactory.cs => OAuth2Interceptors.cs} | 29 ++++++++++++++++--- .../Grpc.IntegrationTesting/InteropClient.cs | 13 ++++----- 3 files changed, 32 insertions(+), 12 deletions(-) rename src/csharp/Grpc.Auth/{OAuth2InterceptorFactory.cs => OAuth2Interceptors.cs} (79%) diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj index afb8204c07e..5615ffd620b 100644 --- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj +++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj @@ -79,7 +79,7 @@ - + diff --git a/src/csharp/Grpc.Auth/OAuth2InterceptorFactory.cs b/src/csharp/Grpc.Auth/OAuth2Interceptors.cs similarity index 79% rename from src/csharp/Grpc.Auth/OAuth2InterceptorFactory.cs rename to src/csharp/Grpc.Auth/OAuth2Interceptors.cs index 420c4cb5371..c785ca5a16b 100644 --- a/src/csharp/Grpc.Auth/OAuth2InterceptorFactory.cs +++ b/src/csharp/Grpc.Auth/OAuth2Interceptors.cs @@ -47,17 +47,31 @@ using Grpc.Core.Utils; namespace Grpc.Auth { - public static class OAuth2InterceptorFactory + public static class OAuth2Interceptors { /// - /// Creates OAuth2 interceptor. + /// Creates OAuth2 interceptor that will obtain access token from GoogleCredentials. /// - public static MetadataInterceptorDelegate Create(GoogleCredential googleCredential) + public static MetadataInterceptorDelegate FromCredential(GoogleCredential googleCredential) { var interceptor = new OAuth2Interceptor(googleCredential.InternalCredential, SystemClock.Default); return new MetadataInterceptorDelegate(interceptor.InterceptHeaders); } + /// + /// Creates OAuth2 interceptor that will use given OAuth2 token. + /// + /// + /// + public static MetadataInterceptorDelegate FromAccessToken(string oauth2Token) + { + Preconditions.CheckNotNull(oauth2Token); + return new MetadataInterceptorDelegate((metadata) => + { + metadata.Add(OAuth2Interceptor.CreateBearerTokenHeader(oauth2Token)); + }); + } + /// /// Injects OAuth2 authorization header into initial metadata (= request headers). /// @@ -97,8 +111,15 @@ namespace Grpc.Auth public void InterceptHeaders(Metadata metadata) { var accessToken = GetAccessToken(CancellationToken.None); - metadata.Add(new Metadata.Entry(AuthorizationHeader, Schema + " " + accessToken)); + metadata.Add(CreateBearerTokenHeader(accessToken)); + } + + public static Metadata.Entry CreateBearerTokenHeader(string accessToken) + { + return new Metadata.Entry(AuthorizationHeader, Schema + " " + accessToken); } } + + } } diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs index ce255f94237..0612067f1a4 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs @@ -127,7 +127,7 @@ namespace Grpc.IntegrationTesting { credential = credential.CreateScoped(new[] { AuthScope }); } - client.HeaderInterceptor = OAuth2InterceptorFactory.Create(credential); + client.HeaderInterceptor = OAuth2Interceptors.FromCredential(credential); } RunTestCase(options.testCase, client); @@ -368,11 +368,7 @@ namespace Grpc.IntegrationTesting Assert.IsTrue(credential.RequestAccessTokenAsync(CancellationToken.None).Result); string oauth2Token = credential.Token.AccessToken; - // Intercept calls with an OAuth2 token obtained out-of-band. - client.HeaderInterceptor = new MetadataInterceptorDelegate((metadata) => - { - metadata.Add(new Metadata.Entry("Authorization", "Bearer " + oauth2Token)); - }); + client.HeaderInterceptor = OAuth2Interceptors.FromAccessToken(oauth2Token); var request = SimpleRequest.CreateBuilder() .SetFillUsername(true) @@ -393,13 +389,16 @@ namespace Grpc.IntegrationTesting var credential = GoogleCredential.GetApplicationDefault().CreateScoped(new[] { AuthScope }); Assert.IsTrue(credential.RequestAccessTokenAsync(CancellationToken.None).Result); string oauth2Token = credential.Token.AccessToken; + var headerInterceptor = OAuth2Interceptors.FromAccessToken(oauth2Token); var request = SimpleRequest.CreateBuilder() .SetFillUsername(true) .SetFillOauthScope(true) .Build(); - var response = client.UnaryCall(request, headers: new Metadata { new Metadata.Entry("Authorization", "Bearer " + oauth2Token) }); + var headers = new Metadata(); + headerInterceptor(headers); + var response = client.UnaryCall(request, headers: headers); Assert.AreEqual(AuthScopeResponse, response.OauthScope); Assert.AreEqual(ServiceAccountUser, response.Username); From 925e4a634978ec0e8cd43fe2352a2c6240953f82 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Sun, 26 Jul 2015 16:51:23 -0700 Subject: [PATCH 29/47] Enrich census initialization and feature code --- include/grpc/census.h | 32 ++++++++++++++++++-------------- src/core/census/grpc_context.c | 2 +- src/core/census/initialize.c | 19 ++++++++++++------- src/core/surface/init.c | 7 +++++-- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 379783905aa..4b5fd1b2116 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -44,26 +44,30 @@ extern "C" { #endif -/* Identify census functionality that can be enabled via census_initialize(). */ -enum census_functions { - CENSUS_NONE = 0, /* Do not enable census. */ - CENSUS_TRACING = 1, /* Enable census tracing. */ - CENSUS_STATS = 2, /* Enable Census stats collection. */ - CENSUS_CPU = 4, /* Enable Census CPU usage collection. */ - CENSUS_ALL = CENSUS_TRACING | CENSUS_STATS | CENSUS_CPU +/* Identify census features that can be enabled via census_initialize(). */ +enum census_features { + CENSUS_FEATURE_NONE = 0, /* Do not enable census. */ + CENSUS_FEATURE_TRACING = 1, /* Enable census tracing. */ + CENSUS_FEATURE_STATS = 2, /* Enable Census stats collection. */ + CENSUS_FEATURE_CPU = 4, /* Enable Census CPU usage collection. */ + CENSUS_FEATURE_ALL = + CENSUS_FEATURE_TRACING | CENSUS_FEATURE_STATS | CENSUS_FEATURE_CPU }; -/* Shutdown and startup census subsystem. The 'functions' argument should be - * the OR (|) of census_functions values. If census fails to initialize, then +/** Shutdown and startup census subsystem. The 'functions' argument should be + * the OR (|) of census_features values. If census fails to initialize, then * census_initialize() will return a non-zero value. It is an error to call * census_initialize() more than once (without an intervening * census_shutdown()). */ -int census_initialize(int functions); -void census_shutdown(); +int census_initialize(int features); +void census_shutdown(void); -/* If any census feature has been initialized, this funtion will return a - * non-zero value. */ -int census_available(); +/** Return the features supported by the current census implementation (not all + * features will be available on all platforms). */ +int census_supported(void); + +/** Return the census features currently enabled. */ +int census_enabled(void); /* Internally, Census relies on a context, which should be propagated across * RPC's. From the RPC subsystems viewpoint, this is an opaque data structure. diff --git a/src/core/census/grpc_context.c b/src/core/census/grpc_context.c index 0ed63469b64..d4243cb246f 100644 --- a/src/core/census/grpc_context.c +++ b/src/core/census/grpc_context.c @@ -39,7 +39,7 @@ static void grpc_census_context_destroy(void *context) { } void grpc_census_call_set_context(grpc_call *call, census_context *context) { - if (!census_available()) { + if (census_enabled() == CENSUS_FEATURE_NONE) { return; } if (context == NULL) { diff --git a/src/core/census/initialize.c b/src/core/census/initialize.c index 80165206417..8d60f790eb7 100644 --- a/src/core/census/initialize.c +++ b/src/core/census/initialize.c @@ -33,20 +33,25 @@ #include -static int census_fns_enabled = CENSUS_NONE; +static int features_enabled = CENSUS_FEATURE_NONE; -int census_initialize(int functions) { - if (census_fns_enabled != CENSUS_NONE) { +int census_initialize(int features) { + if (features_enabled != CENSUS_FEATURE_NONE) { return 1; } - if (functions != CENSUS_NONE) { + if (features != CENSUS_FEATURE_NONE) { return 1; } else { - census_fns_enabled = functions; + features_enabled = features; return 0; } } -void census_shutdown() { census_fns_enabled = CENSUS_NONE; } +void census_shutdown(void) { features_enabled = CENSUS_FEATURE_NONE; } -int census_available() { return (census_fns_enabled != CENSUS_NONE); } +int census_supported(void) { + /* TODO(aveitch): improve this as we implement features... */ + return CENSUS_FEATURE_NONE; +} + +int census_enabled(void) { return features_enabled; } diff --git a/src/core/surface/init.c b/src/core/surface/init.c index 5cba4793174..99fbde50c48 100644 --- a/src/core/surface/init.c +++ b/src/core/surface/init.c @@ -78,8 +78,11 @@ void grpc_init(void) { grpc_security_pre_init(); grpc_iomgr_init(); grpc_tracer_init("GRPC_TRACE"); - if (census_initialize(CENSUS_NONE)) { - gpr_log(GPR_ERROR, "Could not initialize census."); + /* Only initialize census if noone else has. */ + if (census_enabled() == CENSUS_FEATURE_NONE) { + if (census_initialize(census_supported())) { /* enable all features. */ + gpr_log(GPR_ERROR, "Could not initialize census."); + } } grpc_timers_global_init(); } From db98e085704308bf03b2c68a9183336a3b37b422 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 27 Jul 2015 10:19:45 -0700 Subject: [PATCH 30/47] Exposed channel target and call peer in PHP --- src/php/ext/grpc/call.c | 14 +++++++++++++- src/php/ext/grpc/channel.c | 14 +++++++++++++- src/php/lib/Grpc/AbstractCall.php | 7 +++++++ src/php/lib/Grpc/BaseStub.php | 7 +++++++ .../generated_code/AbstractGeneratedCodeTest.php | 7 ++++++- src/php/tests/interop/interop_client.php | 9 +++++---- src/php/tests/unit_tests/CallTest.php | 4 ++++ src/php/tests/unit_tests/EndToEndTest.php | 8 ++++++-- 8 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/php/ext/grpc/call.c b/src/php/ext/grpc/call.c index b67bae75682..1f76c7359d4 100644 --- a/src/php/ext/grpc/call.c +++ b/src/php/ext/grpc/call.c @@ -472,6 +472,16 @@ cleanup: RETURN_DESTROY_ZVAL(result); } +/** + * Get the endpoint this call/stream is connected to + * @return string The URI of the endpoint + */ +PHP_METHOD(Call, getPeer) { + wrapped_grpc_call *call = + (wrapped_grpc_call *)zend_object_store_get_object(getThis() TSRMLS_CC); + RETURN_STRING(grpc_call_get_peer(call->wrapped), 1); +} + /** * Cancel the call. This will cause the call to end with STATUS_CANCELLED if it * has not already ended with another status. @@ -485,7 +495,9 @@ PHP_METHOD(Call, cancel) { static zend_function_entry call_methods[] = { PHP_ME(Call, __construct, NULL, ZEND_ACC_PUBLIC | ZEND_ACC_CTOR) PHP_ME(Call, startBatch, NULL, ZEND_ACC_PUBLIC) - PHP_ME(Call, cancel, NULL, ZEND_ACC_PUBLIC) PHP_FE_END}; + PHP_ME(Call, getPeer, NULL, ZEND_ACC_PUBLIC) + PHP_ME(Call, cancel, NULL, ZEND_ACC_PUBLIC) + PHP_FE_END}; void grpc_init_call(TSRMLS_D) { zend_class_entry ce; diff --git a/src/php/ext/grpc/channel.c b/src/php/ext/grpc/channel.c index b8262db162b..fe69fa5e29a 100644 --- a/src/php/ext/grpc/channel.c +++ b/src/php/ext/grpc/channel.c @@ -194,6 +194,16 @@ PHP_METHOD(Channel, __construct) { memcpy(channel->target, override, override_len); } +/** + * Get the endpoint this call/stream is connected to + * @return string The URI of the endpoint + */ +PHP_METHOD(Channel, getTarget) { + wrapped_grpc_channel *channel = + (wrapped_grpc_channel *)zend_object_store_get_object(getThis() TSRMLS_CC); + RETURN_STRING(grpc_channel_get_target(channel->wrapped), 1); +} + /** * Close the channel */ @@ -208,7 +218,9 @@ PHP_METHOD(Channel, close) { static zend_function_entry channel_methods[] = { PHP_ME(Channel, __construct, NULL, ZEND_ACC_PUBLIC | ZEND_ACC_CTOR) - PHP_ME(Channel, close, NULL, ZEND_ACC_PUBLIC) PHP_FE_END}; + PHP_ME(Channel, getTarget, NULL, ZEND_ACC_PUBLIC) + PHP_ME(Channel, close, NULL, ZEND_ACC_PUBLIC) + PHP_FE_END}; void grpc_init_channel(TSRMLS_D) { zend_class_entry ce; diff --git a/src/php/lib/Grpc/AbstractCall.php b/src/php/lib/Grpc/AbstractCall.php index 5b28417a0df..35057224f8c 100644 --- a/src/php/lib/Grpc/AbstractCall.php +++ b/src/php/lib/Grpc/AbstractCall.php @@ -67,6 +67,13 @@ abstract class AbstractCall { return $this->metadata; } + /** + * @return string The URI of the endpoint. + */ + public function getPeer() { + return $this->call->getPeer(); + } + /** * Cancels the call */ diff --git a/src/php/lib/Grpc/BaseStub.php b/src/php/lib/Grpc/BaseStub.php index 8c438e4bf92..a0c677908c7 100755 --- a/src/php/lib/Grpc/BaseStub.php +++ b/src/php/lib/Grpc/BaseStub.php @@ -67,6 +67,13 @@ class BaseStub { $this->channel = new Channel($hostname, $opts); } + /** + * @return string The URI of the endpoint. + */ + public function getTarget() { + return $this->channel->getTarget(); + } + /** * Close the communication channel associated with this stub */ diff --git a/src/php/tests/generated_code/AbstractGeneratedCodeTest.php b/src/php/tests/generated_code/AbstractGeneratedCodeTest.php index 6102aaf0a8b..8b7e67f57c2 100644 --- a/src/php/tests/generated_code/AbstractGeneratedCodeTest.php +++ b/src/php/tests/generated_code/AbstractGeneratedCodeTest.php @@ -43,7 +43,9 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase { $div_arg = new math\DivArgs(); $div_arg->setDividend(7); $div_arg->setDivisor(4); - list($response, $status) = self::$client->Div($div_arg)->wait(); + $call = self::$client->Div($div_arg); + $this->assertTrue(is_string($call->getPeer())); + list($response, $status) = $call->wait(); $this->assertSame(1, $response->getQuotient()); $this->assertSame(3, $response->getRemainder()); $this->assertSame(\Grpc\STATUS_OK, $status->code); @@ -53,6 +55,7 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase { $fib_arg = new math\FibArgs(); $fib_arg->setLimit(7); $call = self::$client->Fib($fib_arg); + $this->assertTrue(is_string($call->getPeer())); $result_array = iterator_to_array($call->responses()); $extract_num = function($num){ return $num->getNum(); @@ -72,6 +75,7 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase { } }; $call = self::$client->Sum($num_iter()); + $this->assertTrue(is_string($call->getPeer())); list($response, $status) = $call->wait(); $this->assertSame(21, $response->getNum()); $this->assertSame(\Grpc\STATUS_OK, $status->code); @@ -79,6 +83,7 @@ abstract class AbstractGeneratedCodeTest extends PHPUnit_Framework_TestCase { public function testBidiStreaming() { $call = self::$client->DivMany(); + $this->assertTrue(is_string($call->getPeer())); for ($i = 0; $i < 7; $i++) { $div_arg = new math\DivArgs(); $div_arg->setDividend(2 * $i + 1); diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php index 20415775574..4bcf7df7f72 100755 --- a/src/php/tests/interop/interop_client.php +++ b/src/php/tests/interop/interop_client.php @@ -332,10 +332,11 @@ if (in_array($args['test_case'], array( $opts['update_metadata'] = $auth->getUpdateMetadataFunc(); } -$stub = new grpc\testing\TestServiceClient( - new Grpc\BaseStub( - $server_address, - $opts)); +$internal_stub = new Grpc\BaseStub($server_address, $opts); +hardAssert($internal_stub->getTarget() == $server_address, + 'Unexpected target URI value'); + +$stub = new grpc\testing\TestServiceClient($internal_stub); echo "Connecting to $server_address\n"; echo "Running test case $args[test_case]\n"; diff --git a/src/php/tests/unit_tests/CallTest.php b/src/php/tests/unit_tests/CallTest.php index 77a2d86ce4c..3d50da32512 100755 --- a/src/php/tests/unit_tests/CallTest.php +++ b/src/php/tests/unit_tests/CallTest.php @@ -79,4 +79,8 @@ class CallTest extends PHPUnit_Framework_TestCase{ $result = $this->call->startBatch($batch); $this->assertTrue($result->send_metadata); } + + public function testGetPeer() { + $this->assertTrue($this->call->getPeer() == 'localhost:' . self::$port); + } } diff --git a/src/php/tests/unit_tests/EndToEndTest.php b/src/php/tests/unit_tests/EndToEndTest.php index 2980dca4a75..9a1ab81acc4 100755 --- a/src/php/tests/unit_tests/EndToEndTest.php +++ b/src/php/tests/unit_tests/EndToEndTest.php @@ -34,8 +34,8 @@ class EndToEndTest extends PHPUnit_Framework_TestCase{ public function setUp() { $this->server = new Grpc\Server([]); - $port = $this->server->addHttp2Port('0.0.0.0:0'); - $this->channel = new Grpc\Channel('localhost:' . $port, []); + $this->port = $this->server->addHttp2Port('0.0.0.0:0'); + $this->channel = new Grpc\Channel('localhost:' . $this->port, []); $this->server->start(); } @@ -149,4 +149,8 @@ class EndToEndTest extends PHPUnit_Framework_TestCase{ unset($call); unset($server_call); } + + public function testGetTarget() { + $this->assertTrue($this->channel->getTarget() == 'localhost:' . $this->port); + } } From 9cd90a64caaac14411b8c3d7a5431e39b94f5aaa Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Mon, 27 Jul 2015 11:23:13 -0700 Subject: [PATCH 31/47] Rearranged some code for jsdoc, added some documentation --- src/node/index.js | 32 +++++++++++--------------- src/node/src/client.js | 25 +++++++++++--------- src/node/src/common.js | 52 ++++++++++++++++-------------------------- src/node/src/server.js | 45 ++++++++++++++++++++++++++++++++++-- 4 files changed, 90 insertions(+), 64 deletions(-) diff --git a/src/node/index.js b/src/node/index.js index d81e780443f..b26ab35f2c5 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -48,7 +48,7 @@ var grpc = require('bindings')('grpc'); * @param {ProtoBuf.Reflect.Namespace} value The ProtoBuf object to load. * @return {Object} The resulting gRPC object */ -function loadObject(value) { +exports.loadObject = function loadObject(value) { var result = {}; if (value.className === 'Namespace') { _.each(value.children, function(child) { @@ -62,7 +62,9 @@ function loadObject(value) { } else { return value; } -} +}; + +var loadObject = exports.loadObject; /** * Load a gRPC object from a .proto file. @@ -71,7 +73,7 @@ function loadObject(value) { * 'json'. Defaults to 'proto' * @return {Object} The resulting gRPC object */ -function load(filename, format) { +exports.load = function load(filename, format) { if (!format) { format = 'proto'; } @@ -88,7 +90,7 @@ function load(filename, format) { } return loadObject(builder.ns); -} +}; /** * Get a function that a client can use to update metadata with authentication @@ -97,7 +99,7 @@ function load(filename, format) { * @param {Object} credential The credential object to use * @return {function(Object, callback)} Metadata updater function */ -function getGoogleAuthDelegate(credential) { +exports.getGoogleAuthDelegate = function getGoogleAuthDelegate(credential) { /** * Update a metadata object with authentication information. * @param {string} authURI The uri to authenticate to @@ -120,20 +122,10 @@ function getGoogleAuthDelegate(credential) { callback(null, metadata); }); }; -} - -/** - * See docs for loadObject - */ -exports.loadObject = loadObject; +}; /** - * See docs for load - */ -exports.load = load; - -/** - * See docs for Server + * @see module:src/server.Server */ exports.Server = server.Server; @@ -141,6 +133,7 @@ exports.Server = server.Server; * Status name to code number mapping */ exports.status = grpc.status; + /** * Call error name to code number mapping */ @@ -156,6 +149,7 @@ exports.Credentials = grpc.Credentials; */ exports.ServerCredentials = grpc.ServerCredentials; -exports.getGoogleAuthDelegate = getGoogleAuthDelegate; - +/** + * @see module:src/client.makeClientConstructor + */ exports.makeGenericClientConstructor = client.makeClientConstructor; diff --git a/src/node/src/client.js b/src/node/src/client.js index da6327b4320..a7cfd0e6208 100644 --- a/src/node/src/client.js +++ b/src/node/src/client.js @@ -31,6 +31,11 @@ * */ +/** + * Server module + * @module + */ + 'use strict'; var _ = require('lodash'); @@ -72,6 +77,7 @@ function ClientWritableStream(call, serialize) { /** * Attempt to write the given chunk. Calls the callback when done. This is an * implementation of a method needed for implementing stream.Writable. + * @access private * @param {Buffer} chunk The chunk to write * @param {string} encoding Ignored * @param {function(Error=)} callback Called when the write is complete @@ -110,6 +116,7 @@ function ClientReadableStream(call, deserialize) { /** * Read the next object from the stream. + * @access private * @param {*} size Ignored because we use objectMode=true */ function _read(size) { @@ -503,7 +510,7 @@ var requester_makers = { * @param {string} serviceName The name of the service * @return {function(string, Object)} New client constructor */ -function makeClientConstructor(methods, serviceName) { +exports.makeClientConstructor = function(methods, serviceName) { /** * Create a client with the given methods * @constructor @@ -552,7 +559,7 @@ function makeClientConstructor(methods, serviceName) { }); return Client; -} +}; /** * Creates a constructor for clients for the given service @@ -560,22 +567,18 @@ function makeClientConstructor(methods, serviceName) { * for * @return {function(string, Object)} New client constructor */ -function makeProtobufClientConstructor(service) { +exports.makeProtobufClientConstructor = function(service) { var method_attrs = common.getProtobufServiceAttrs(service, service.name); - var Client = makeClientConstructor(method_attrs); + var Client = exports.makeClientConstructor(method_attrs); Client.service = service; - return Client; -} - -exports.makeClientConstructor = makeClientConstructor; - -exports.makeProtobufClientConstructor = makeProtobufClientConstructor; +}; /** - * See docs for client.status + * Map of status code names to status codes */ exports.status = grpc.status; + /** * See docs for client.callError */ diff --git a/src/node/src/common.js b/src/node/src/common.js index feaa859a4f4..5551ebeec84 100644 --- a/src/node/src/common.js +++ b/src/node/src/common.js @@ -31,6 +31,10 @@ * */ +/** + * @module + */ + 'use strict'; var _ = require('lodash'); @@ -40,7 +44,7 @@ var _ = require('lodash'); * @param {function()} cls The constructor of the message type to deserialize * @return {function(Buffer):cls} The deserialization function */ -function deserializeCls(cls) { +exports.deserializeCls = function deserializeCls(cls) { /** * Deserialize a buffer to a message object * @param {Buffer} arg_buf The buffer to deserialize @@ -51,14 +55,16 @@ function deserializeCls(cls) { // and longs as strings (second argument) return cls.decode(arg_buf).toRaw(false, true); }; -} +}; + +var deserializeCls = exports.deserializeCls; /** * Get a function that serializes objects to a buffer by protobuf class. * @param {function()} Cls The constructor of the message type to serialize * @return {function(Cls):Buffer} The serialization function */ -function serializeCls(Cls) { +exports.serializeCls = function serializeCls(Cls) { /** * Serialize an object to a Buffer * @param {Object} arg The object to serialize @@ -67,14 +73,16 @@ function serializeCls(Cls) { return function serialize(arg) { return new Buffer(new Cls(arg).encode().toBuffer()); }; -} +}; + +var serializeCls = exports.serializeCls; /** * Get the fully qualified (dotted) name of a ProtoBuf.Reflect value. * @param {ProtoBuf.Reflect.Namespace} value The value to get the name of * @return {string} The fully qualified name of the value */ -function fullyQualifiedName(value) { +exports.fullyQualifiedName = function fullyQualifiedName(value) { if (value === null || value === undefined) { return ''; } @@ -89,7 +97,9 @@ function fullyQualifiedName(value) { } } return name; -} +}; + +var fullyQualifiedName = exports.fullyQualifiedName; /** * Wrap a function to pass null-like values through without calling it. If no @@ -97,7 +107,7 @@ function fullyQualifiedName(value) { * @param {?function} func The function to wrap * @return {function} The wrapped function */ -function wrapIgnoreNull(func) { +exports.wrapIgnoreNull = function wrapIgnoreNull(func) { if (!func) { return _.identity; } @@ -107,14 +117,14 @@ function wrapIgnoreNull(func) { } return func(arg); }; -} +}; /** * Return a map from method names to method attributes for the service. * @param {ProtoBuf.Reflect.Service} service The service to get attributes for * @return {Object} The attributes map */ -function getProtobufServiceAttrs(service) { +exports.getProtobufServiceAttrs = function getProtobufServiceAttrs(service) { var prefix = '/' + fullyQualifiedName(service) + '/'; return _.object(_.map(service.children, function(method) { return [_.camelCase(method.name), { @@ -127,26 +137,4 @@ function getProtobufServiceAttrs(service) { responseDeserialize: deserializeCls(method.resolvedResponseType.build()) }]; })); -} - -/** - * See docs for deserializeCls - */ -exports.deserializeCls = deserializeCls; - -/** - * See docs for serializeCls - */ -exports.serializeCls = serializeCls; - -/** - * See docs for fullyQualifiedName - */ -exports.fullyQualifiedName = fullyQualifiedName; - -/** - * See docs for wrapIgnoreNull - */ -exports.wrapIgnoreNull = wrapIgnoreNull; - -exports.getProtobufServiceAttrs = getProtobufServiceAttrs; +}; diff --git a/src/node/src/server.js b/src/node/src/server.js index 0a3a0031bdf..1c8a53e09ce 100644 --- a/src/node/src/server.js +++ b/src/node/src/server.js @@ -31,6 +31,11 @@ * */ +/** + * Server module + * @module + */ + 'use strict'; var _ = require('lodash'); @@ -50,6 +55,7 @@ var EventEmitter = require('events').EventEmitter; /** * Handle an error on a call by sending it as a status + * @access private * @param {grpc.Call} call The call to send the error on * @param {Object} error The error object */ @@ -82,6 +88,7 @@ function handleError(call, error) { /** * Wait for the client to close, then emit a cancelled event if the client * cancelled. + * @access private * @param {grpc.Call} call The call object to wait on * @param {EventEmitter} emitter The event emitter to emit the cancelled event * on @@ -102,6 +109,7 @@ function waitForCancel(call, emitter) { /** * Send a response to a unary or client streaming call. + * @access private * @param {grpc.Call} call The call to respond on * @param {*} value The value to respond with * @param {function(*):Buffer=} serialize Serialization function for the @@ -130,6 +138,7 @@ function sendUnaryResponse(call, value, serialize, metadata) { /** * Initialize a writable stream. This is used for both the writable and duplex * stream constructors. + * @access private * @param {Writable} stream The stream to set up * @param {function(*):Buffer=} Serialization function for responses */ @@ -203,6 +212,7 @@ function setUpWritable(stream, serialize) { /** * Initialize a readable stream. This is used for both the readable and duplex * stream constructors. + * @access private * @param {Readable} stream The stream to initialize * @param {function(Buffer):*=} deserialize Deserialization function for * incoming data. @@ -242,6 +252,7 @@ function ServerWritableStream(call, serialize) { /** * Start writing a chunk of data. This is an implementation of a method required * for implementing stream.Writable. + * @access private * @param {Buffer} chunk The chunk of data to write * @param {string} encoding Ignored * @param {function(Error=)} callback Callback to indicate that the write is @@ -266,6 +277,11 @@ function _write(chunk, encoding, callback) { ServerWritableStream.prototype._write = _write; +/** + * Send the initial metadata for a writable stream. + * @param {Object>} responseMetadata Metadata + * to send + */ function sendMetadata(responseMetadata) { /* jshint validthis: true */ if (!this.call.metadataSent) { @@ -281,6 +297,10 @@ function sendMetadata(responseMetadata) { } } +/** + * @inheritdoc + * @alias module:src/server~ServerWritableStream#sendMetadata + */ ServerWritableStream.prototype.sendMetadata = sendMetadata; util.inherits(ServerReadableStream, Readable); @@ -301,6 +321,7 @@ function ServerReadableStream(call, deserialize) { /** * Start reading from the gRPC data source. This is an implementation of a * method required for implementing stream.Readable + * @access private * @param {number} size Ignored */ function _read(size) { @@ -375,6 +396,7 @@ ServerDuplexStream.prototype.sendMetadata = sendMetadata; /** * Fully handle a unary call + * @access private * @param {grpc.Call} call The call to handle * @param {Object} handler Request handler object for the method that was called * @param {Object} metadata Metadata from the client @@ -426,6 +448,7 @@ function handleUnary(call, handler, metadata) { /** * Fully handle a server streaming call + * @access private * @param {grpc.Call} call The call to handle * @param {Object} handler Request handler object for the method that was called * @param {Object} metadata Metadata from the client @@ -454,6 +477,7 @@ function handleServerStreaming(call, handler, metadata) { /** * Fully handle a client streaming call + * @access private * @param {grpc.Call} call The call to handle * @param {Object} handler Request handler object for the method that was called * @param {Object} metadata Metadata from the client @@ -488,6 +512,7 @@ function handleClientStreaming(call, handler, metadata) { /** * Fully handle a bidirectional streaming call + * @access private * @param {grpc.Call} call The call to handle * @param {Object} handler Request handler object for the method that was called * @param {Object} metadata Metadata from the client @@ -571,7 +596,8 @@ function Server(options) { } server.requestCall(handleNewCall); }; - /** Shuts down the server. + /** + * Shuts down the server. */ this.shutdown = function() { server.shutdown(); @@ -605,6 +631,15 @@ Server.prototype.register = function(name, handler, serialize, deserialize, return true; }; +/** + * Add a service to the server, with a corresponding implementation. If you are + * generating this from a proto file, you should instead use + * addProtoService. + * @param {Object} service The service descriptor, as + * {@link module:src/common.getProtobufServiceAttrs} returns + * @param {Object} implementation Map of method names to + * method implementation for the provided service. + */ Server.prototype.addService = function(service, implementation) { if (this.started) { throw new Error('Can\'t add a service to a started server.'); @@ -642,6 +677,12 @@ Server.prototype.addService = function(service, implementation) { }); }; +/** + * Add a proto service to the server, with a corresponding implementation + * @param {Protobuf.Reflect.Service} service The proto service descriptor + * @param {Object} implementation Map of method names to + * method implementation for the provided service. + */ Server.prototype.addProtoService = function(service, implementation) { this.addService(common.getProtobufServiceAttrs(service), implementation); }; @@ -665,6 +706,6 @@ Server.prototype.bind = function(port, creds) { }; /** - * See documentation for Server + * @see module:src/server~Server */ exports.Server = Server; From c0034dd10b93dccef33d92f4b1990a6ac30b06aa Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 27 Jul 2015 12:26:15 -0700 Subject: [PATCH 32/47] review feedback: unit test of uri value too strict --- src/php/tests/interop/interop_client.php | 2 +- src/php/tests/unit_tests/CallTest.php | 2 +- src/php/tests/unit_tests/EndToEndTest.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php index 4bcf7df7f72..44e6242c299 100755 --- a/src/php/tests/interop/interop_client.php +++ b/src/php/tests/interop/interop_client.php @@ -333,7 +333,7 @@ if (in_array($args['test_case'], array( } $internal_stub = new Grpc\BaseStub($server_address, $opts); -hardAssert($internal_stub->getTarget() == $server_address, +hardAssert(is_string($internal_stub->getTarget()), 'Unexpected target URI value'); $stub = new grpc\testing\TestServiceClient($internal_stub); diff --git a/src/php/tests/unit_tests/CallTest.php b/src/php/tests/unit_tests/CallTest.php index 3d50da32512..caff15ee110 100755 --- a/src/php/tests/unit_tests/CallTest.php +++ b/src/php/tests/unit_tests/CallTest.php @@ -81,6 +81,6 @@ class CallTest extends PHPUnit_Framework_TestCase{ } public function testGetPeer() { - $this->assertTrue($this->call->getPeer() == 'localhost:' . self::$port); + $this->assertTrue(is_string($this->call->getPeer())); } } diff --git a/src/php/tests/unit_tests/EndToEndTest.php b/src/php/tests/unit_tests/EndToEndTest.php index 9a1ab81acc4..27e27cdfdf3 100755 --- a/src/php/tests/unit_tests/EndToEndTest.php +++ b/src/php/tests/unit_tests/EndToEndTest.php @@ -151,6 +151,6 @@ class EndToEndTest extends PHPUnit_Framework_TestCase{ } public function testGetTarget() { - $this->assertTrue($this->channel->getTarget() == 'localhost:' . $this->port); + $this->assertTrue(is_string($this->channel->getTarget())); } } From 7b042c5793b1477f4a0a1a1ee72796ea53bf88ae Mon Sep 17 00:00:00 2001 From: Nathaniel Manista Date: Mon, 27 Jul 2015 20:59:36 +0000 Subject: [PATCH 33/47] Add STREAM_LENGTH and POOL_SIZE to test_constants Only one of these is used, and in only one place, as of this commit, but they will be used widely after further development. --- src/python/src/grpc/_links/_proto_scenarios.py | 4 ++-- src/python/src/grpc/framework/common/test_constants.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/python/src/grpc/_links/_proto_scenarios.py b/src/python/src/grpc/_links/_proto_scenarios.py index ccf3c297824..320c0e0f506 100644 --- a/src/python/src/grpc/_links/_proto_scenarios.py +++ b/src/python/src/grpc/_links/_proto_scenarios.py @@ -33,6 +33,7 @@ import abc import threading from grpc._junkdrawer import math_pb2 +from grpc.framework.common import test_constants class ProtoScenario(object): @@ -219,10 +220,9 @@ class BidirectionallyUnaryScenario(ProtoScenario): class BidirectionallyStreamingScenario(ProtoScenario): """A scenario that transmits no protocol buffers in either direction.""" - _STREAM_LENGTH = 200 _REQUESTS = tuple( math_pb2.DivArgs(dividend=59 + index, divisor=7 + index) - for index in range(_STREAM_LENGTH)) + for index in range(test_constants.STREAM_LENGTH)) def __init__(self): self._lock = threading.Lock() diff --git a/src/python/src/grpc/framework/common/test_constants.py b/src/python/src/grpc/framework/common/test_constants.py index 237b8754ed2..3126d0d82ce 100644 --- a/src/python/src/grpc/framework/common/test_constants.py +++ b/src/python/src/grpc/framework/common/test_constants.py @@ -35,3 +35,9 @@ SHORT_TIMEOUT = 4 # Absurdly large value for maximum duration in seconds for should-not-time-out # RPCs made during tests. LONG_TIMEOUT = 3000 + +# The number of payloads to transmit in streaming tests. +STREAM_LENGTH = 200 + +# The size of thread pools to use in tests. +POOL_SIZE = 10 From c089743f49073140fb73dd2ebe6785e82f758120 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 27 Jul 2015 22:18:13 +0000 Subject: [PATCH 34/47] Remove explicit reference to nullptr since this isn't within grpc namespace and not supported by older compilers. Not sure why I didn't catch this earlier. --- src/compiler/python_generator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 724b69c703f..2982a89fad1 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -249,7 +249,7 @@ bool GetModuleAndMessagePath(const Descriptor* type, do { message_path.push_back(path_elem_type); path_elem_type = path_elem_type->containing_type(); - } while (path_elem_type != nullptr); + } while (path_elem_type); // implicit nullptr comparison; don't be explicit grpc::string file_name = type->file()->name(); static const int proto_suffix_length = strlen(".proto"); if (!(file_name.size() > static_cast(proto_suffix_length) && From 1b0e3437be035526a397e69ce6ef9397e61b2eb1 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Mon, 27 Jul 2015 16:02:52 -0700 Subject: [PATCH 35/47] Adding c++ interop (cloud to prod) to jenkins work flow. First effort in adding interop to jenkins work flow, more languages will follow. --- tools/jenkins/run_jenkins.sh | 4 ++- tools/run_tests/jobset.py | 2 +- tools/run_tests/run_interops.py | 36 +++++++++++++++++++++ tools/run_tests/run_interops_build.sh | 46 +++++++++++++++++++++++++++ tools/run_tests/run_interops_test.sh | 43 +++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 2 deletions(-) create mode 100755 tools/run_tests/run_interops.py create mode 100755 tools/run_tests/run_interops_build.sh create mode 100755 tools/run_tests/run_interops_test.sh diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 56f9e82ca58..f7dfe719927 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -90,7 +90,9 @@ then docker cp $DOCKER_CID:/var/local/git/grpc/report.xml $git_root sleep 4 docker rm $DOCKER_CID || true - +elif [ "$platform" == "interop" ] +then + python tools/run_tests/run_interops.py --language=$language elif [ "$platform" == "windows" ] then echo "building $language on Windows" diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 46137f01f40..ec25b476102 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -206,7 +206,7 @@ class Job(object): do_newline=self._newline_on_success or self._travis) if self._bin_hash: update_cache.finished(self._spec.identity(), self._bin_hash) - elif self._state == _RUNNING and time.time() - self._start > 600: + elif self._state == _RUNNING and time.time() - self._start > 900: self._tempfile.seek(0) stdout = self._tempfile.read() filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore')) diff --git a/tools/run_tests/run_interops.py b/tools/run_tests/run_interops.py new file mode 100755 index 00000000000..1cf268526dc --- /dev/null +++ b/tools/run_tests/run_interops.py @@ -0,0 +1,36 @@ +import argparse +import xml.etree.cElementTree as ET +import jobset + +argp = argparse.ArgumentParser(description='Run interop tests.') +argp.add_argument('-l', '--language', + choices=['build_only', 'c++'], + nargs='+', + default=['build_only']) +args = argp.parse_args() + +# build job +build_steps = 'tools/run_tests/run_interops_build.sh' +build_job = jobset.JobSpec(cmdline=build_steps, shortname='build') + +# test jobs +_TESTS = ['large_unary', 'empty_unary', 'ping_pong', 'client_streaming', 'server_streaming'] +jobs = [] +jobNumber = 0 +for lang in args.language: + for test in _TESTS: + test_job = jobset.JobSpec(cmdline=['tools/run_tests/run_interops_test.sh', '%s' % lang, '%s' % test], shortname=test) + jobs.append(test_job) + jobNumber+=1 + +root = ET.Element('testsuites') +testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') + +# always do the build of docker first, and then all the tests can run in parallel +jobset.run([build_job], maxjobs=1, xml_report=testsuite) +jobset.run(jobs, maxjobs=jobNumber, xml_report=testsuite) + +tree = ET.ElementTree(root) +tree.write('report.xml', encoding='UTF-8') + + diff --git a/tools/run_tests/run_interops_build.sh b/tools/run_tests/run_interops_build.sh new file mode 100755 index 00000000000..bbe36eed620 --- /dev/null +++ b/tools/run_tests/run_interops_build.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -e + +#clean up any old docker files +sudo docker rmi -f grpc/cxx || true +sudo docker rmi -f grpc/base || true +sudo docker rmi -f 0.0.0.0:5000/grpc/base || true + +#prepare building by pulling down base images and necessary files +sudo docker pull 0.0.0.0:5000/grpc/base +sudo docker tag -f 0.0.0.0:5000/grpc/base grpc/base +gsutil cp -R gs://docker-interop-images/admin/service_account tools/dockerfile/grpc_cxx +gsutil cp -R gs://docker-interop-images/admin/cacerts tools/dockerfile/grpc_cxx + +#build docker file, add more languages later +sudo docker build --no-cache -t grpc/cxx tools/dockerfile/grpc_cxx diff --git a/tools/run_tests/run_interops_test.sh b/tools/run_tests/run_interops_test.sh new file mode 100755 index 00000000000..1d0eedad85a --- /dev/null +++ b/tools/run_tests/run_interops_test.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +language=$1 +test_case=$2 + +set -e +if [ "$language" = "c++" ] +then + sudo docker run grpc/cxx /var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots --server_host_override=grpc-test.sandbox.google.com --server_host=grpc-test.sandbox.google.com --server_port=443 --test_case=$test_case +else + echo "interop testss not added for $language" + exit 1 +fi + From eabb4f53b975340250676d88cac572e34c862cce Mon Sep 17 00:00:00 2001 From: Julien Boeuf Date: Mon, 27 Jul 2015 16:28:09 -0700 Subject: [PATCH 36/47] Addressing comments. --- include/grpc++/server_credentials.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/grpc++/server_credentials.h b/include/grpc++/server_credentials.h index 94041c7757c..11acd67e8ad 100644 --- a/include/grpc++/server_credentials.h +++ b/include/grpc++/server_credentials.h @@ -58,6 +58,8 @@ class ServerCredentials { // Options to create ServerCredentials with SSL struct SslServerCredentialsOptions { + SslServerCredentialsOptions() : force_client_auth(false) {} + struct PemKeyCertPair { grpc::string private_key; grpc::string cert_chain; From 5f8d9dccf45ed5633a2b980f8a11919183083c2c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 27 Jul 2015 16:34:35 -0700 Subject: [PATCH 37/47] only use lowercase metadata keys in headers --- src/csharp/Grpc.Core.Tests/ClientServerTest.cs | 4 ++-- src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs index e286ea519f9..540fe756c07 100644 --- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs +++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs @@ -209,8 +209,8 @@ namespace Grpc.Core.Tests { var headers = new Metadata { - new Metadata.Entry("asciiHeader", "abcdefg"), - new Metadata.Entry("binaryHeader-bin", new byte[] { 1, 2, 3, 0, 0xff }), + new Metadata.Entry("ascii-header", "abcdefg"), + new Metadata.Entry("binary-header-bin", new byte[] { 1, 2, 3, 0, 0xff }), }; var internalCall = new Call(ServiceName, EchoMethod, channel, headers); var call = Calls.AsyncUnaryCall(internalCall, "ABC", CancellationToken.None); diff --git a/src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs b/src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs index 26f332c1cf7..242d29a9a5f 100644 --- a/src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs +++ b/src/csharp/Grpc.Examples.Tests/MathClientServerTests.cs @@ -65,7 +65,7 @@ namespace math.Tests // for header support. client.HeaderInterceptor = (metadata) => { - metadata.Add(new Metadata.Entry("customHeader", "abcdef")); + metadata.Add(new Metadata.Entry("custom-header", "abcdef")); }; } From c463c5362b906cc85d486626b4ca65685c3645c1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Sat, 25 Jul 2015 00:06:12 -0700 Subject: [PATCH 38/47] Add ReleaseSigned configuration for Grpc.sln --- src/csharp/Grpc.Auth/Grpc.Auth.csproj | 11 ++- src/csharp/Grpc.Auth/Grpc.Auth.nuspec | 6 +- .../Grpc.Auth/Properties/AssemblyInfo.cs | 2 - .../Grpc.Core.Tests/Grpc.Core.Tests.csproj | 13 ++- src/csharp/Grpc.Core/Grpc.Core.csproj | 14 ++- src/csharp/Grpc.Core/Grpc.Core.nuspec | 6 +- .../Grpc.Core/Properties/AssemblyInfo.cs | 8 ++ .../Grpc.Examples.MathClient.csproj | 15 ++-- .../Grpc.Examples.MathServer.csproj | 15 ++-- .../Grpc.Examples.Tests.csproj | 13 ++- src/csharp/Grpc.Examples/Grpc.Examples.csproj | 13 ++- .../Grpc.HealthCheck.Tests.csproj | 11 ++- .../Grpc.HealthCheck/Grpc.HealthCheck.csproj | 11 ++- .../Grpc.HealthCheck/Grpc.HealthCheck.nuspec | 6 +- .../Grpc.IntegrationTesting.Client.csproj | 13 ++- .../Grpc.IntegrationTesting.Server.csproj | 13 ++- .../Grpc.IntegrationTesting.csproj | 13 ++- src/csharp/Grpc.sln | 83 +++++++++++------- src/csharp/build_packages.bat | 2 +- src/csharp/buildall.bat | 5 ++ src/csharp/keys/Grpc.public.snk | Bin 0 -> 160 bytes src/csharp/keys/README.md | 5 ++ 22 files changed, 200 insertions(+), 78 deletions(-) create mode 100644 src/csharp/keys/Grpc.public.snk create mode 100644 src/csharp/keys/README.md diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.csproj b/src/csharp/Grpc.Auth/Grpc.Auth.csproj index 5615ffd620b..8e5036832d5 100644 --- a/src/csharp/Grpc.Auth/Grpc.Auth.csproj +++ b/src/csharp/Grpc.Auth/Grpc.Auth.csproj @@ -24,13 +24,22 @@ false - full + pdbonly true bin\Release prompt 4 false + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk + ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll diff --git a/src/csharp/Grpc.Auth/Grpc.Auth.nuspec b/src/csharp/Grpc.Auth/Grpc.Auth.nuspec index eeaa49aa92a..2dc10d24c27 100644 --- a/src/csharp/Grpc.Auth/Grpc.Auth.nuspec +++ b/src/csharp/Grpc.Auth/Grpc.Auth.nuspec @@ -20,9 +20,9 @@ - - - + + + diff --git a/src/csharp/Grpc.Auth/Properties/AssemblyInfo.cs b/src/csharp/Grpc.Auth/Properties/AssemblyInfo.cs index 70cb32d5b2b..83396c546b6 100644 --- a/src/csharp/Grpc.Auth/Properties/AssemblyInfo.cs +++ b/src/csharp/Grpc.Auth/Properties/AssemblyInfo.cs @@ -9,5 +9,3 @@ using System.Runtime.CompilerServices; [assembly: AssemblyCopyright("Google Inc. All rights reserved.")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] - -[assembly: InternalsVisibleTo("Grpc.Auth.Tests")] diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 1c4cca8b698..242a60d098d 100644 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -17,15 +17,22 @@ DEBUG; prompt 4 - false - full + pdbonly true bin\Release prompt 4 - false + + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 0d879e9b1e0..940a6b8ac09 100644 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -21,15 +21,23 @@ DEBUG; prompt 4 - false - full + pdbonly true bin\Release prompt 4 - false + + + pdbonly + true + bin\ReleaseSigned + SIGNED + prompt + 4 + True + C:\keys\Grpc.snk diff --git a/src/csharp/Grpc.Core/Grpc.Core.nuspec b/src/csharp/Grpc.Core/Grpc.Core.nuspec index 5ace6dcf894..086776f69db 100644 --- a/src/csharp/Grpc.Core/Grpc.Core.nuspec +++ b/src/csharp/Grpc.Core/Grpc.Core.nuspec @@ -21,9 +21,9 @@ - - - + + + diff --git a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs index 2b3d7530f2c..29db85d7aae 100644 --- a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs +++ b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs @@ -10,4 +10,12 @@ using System.Runtime.CompilerServices; [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] +#if SIGNED +[assembly: InternalsVisibleTo("Grpc.Core.Tests,PublicKey=" + + "00240000048000009400000006020000002400005253413100040000010001002f5797a92c6fcde81bd4098f43" + + "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" + + "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" + + "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")] +#else [assembly: InternalsVisibleTo("Grpc.Core.Tests")] +#endif \ No newline at end of file diff --git a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj index 85996a570cd..b603e3af3c3 100644 --- a/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj +++ b/src/csharp/Grpc.Examples.MathClient/Grpc.Examples.MathClient.csproj @@ -19,17 +19,22 @@ DEBUG; prompt 4 - true - AnyCPU - full + pdbonly true bin\Release prompt 4 - true - AnyCPU + + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk diff --git a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj index 6c8856cc921..5f74b587736 100644 --- a/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj +++ b/src/csharp/Grpc.Examples.MathServer/Grpc.Examples.MathServer.csproj @@ -19,17 +19,22 @@ DEBUG; prompt 4 - true - AnyCPU - full + pdbonly true bin\Release prompt 4 - true - AnyCPU + + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index d59d7515d1f..9a8f780b245 100644 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -19,15 +19,22 @@ DEBUG; prompt 4 - false - full + pdbonly true bin\Release prompt 4 - false + + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk diff --git a/src/csharp/Grpc.Examples/Grpc.Examples.csproj b/src/csharp/Grpc.Examples/Grpc.Examples.csproj index eaf24a253c0..c1aa40500e4 100644 --- a/src/csharp/Grpc.Examples/Grpc.Examples.csproj +++ b/src/csharp/Grpc.Examples/Grpc.Examples.csproj @@ -19,15 +19,22 @@ DEBUG; prompt 4 - false - full + pdbonly true bin\Release prompt 4 - false + + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj index 72e110302b6..c922ddfb9e7 100644 --- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj +++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj @@ -17,7 +17,6 @@ full false bin\Debug\ - DEBUG;TRACE prompt 4 @@ -25,10 +24,18 @@ pdbonly true bin\Release\ - TRACE prompt 4 + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk + ..\packages\Google.ProtocolBuffers.2.4.1.555\lib\net40\Google.ProtocolBuffers.dll diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj index 4ebb6446dd2..0b7a7b91c6b 100644 --- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj +++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.csproj @@ -18,7 +18,6 @@ full false bin\Debug\ - DEBUG;TRACE prompt 4 @@ -26,10 +25,18 @@ pdbonly true bin\Release\ - TRACE prompt 4 + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk + ..\packages\Google.ProtocolBuffers.2.4.1.555\lib\net40\Google.ProtocolBuffers.dll diff --git a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec index ca35b368053..acdfba42c81 100644 --- a/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec +++ b/src/csharp/Grpc.HealthCheck/Grpc.HealthCheck.nuspec @@ -20,9 +20,9 @@ - - - + + + diff --git a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj index 37d53d61e0d..2c38c9645c5 100644 --- a/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Client/Grpc.IntegrationTesting.Client.csproj @@ -18,18 +18,25 @@ DEBUG; prompt 4 - true AnyCPU - full + pdbonly true bin\Release prompt 4 - true AnyCPU + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk + diff --git a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj index 0f3b9eb5101..949ad61375c 100644 --- a/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj +++ b/src/csharp/Grpc.IntegrationTesting.Server/Grpc.IntegrationTesting.Server.csproj @@ -18,18 +18,25 @@ DEBUG; prompt 4 - true AnyCPU - full + pdbonly true bin\Release prompt 4 - true AnyCPU + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk + diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index db2e304d373..abc27f811ed 100644 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -18,18 +18,25 @@ DEBUG; prompt 4 - true AnyCPU - full + pdbonly true bin\Release prompt 4 - true AnyCPU + + pdbonly + true + bin\ReleaseSigned + prompt + 4 + True + C:\keys\Grpc.snk + ..\packages\BouncyCastle.1.7.0\lib\Net40-Client\BouncyCastle.Crypto.dll diff --git a/src/csharp/Grpc.sln b/src/csharp/Grpc.sln index 0cd8aaef6d2..f19f29c6a27 100644 --- a/src/csharp/Grpc.sln +++ b/src/csharp/Grpc.sln @@ -36,58 +36,81 @@ Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU + ReleaseSigned|Any CPU = ReleaseSigned|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Release|Any CPU.Build.0 = Release|Any CPU - {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.Release|Any CPU.Build.0 = Release|Any CPU - {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Release|Any CPU.Build.0 = Release|Any CPU {7DC1433E-3225-42C7-B7EA-546D56E27A4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7DC1433E-3225-42C7-B7EA-546D56E27A4B}.Debug|Any CPU.Build.0 = Debug|Any CPU {7DC1433E-3225-42C7-B7EA-546D56E27A4B}.Release|Any CPU.ActiveCfg = Release|Any CPU {7DC1433E-3225-42C7-B7EA-546D56E27A4B}.Release|Any CPU.Build.0 = Release|Any CPU + {7DC1433E-3225-42C7-B7EA-546D56E27A4B}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {7DC1433E-3225-42C7-B7EA-546D56E27A4B}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU + {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Release|Any CPU.Build.0 = Release|Any CPU + {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU {86EC5CB4-4EA2-40A2-8057-86542A0353BB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {86EC5CB4-4EA2-40A2-8057-86542A0353BB}.Debug|Any CPU.Build.0 = Debug|Any CPU {86EC5CB4-4EA2-40A2-8057-86542A0353BB}.Release|Any CPU.ActiveCfg = Release|Any CPU {86EC5CB4-4EA2-40A2-8057-86542A0353BB}.Release|Any CPU.Build.0 = Release|Any CPU + {86EC5CB4-4EA2-40A2-8057-86542A0353BB}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {86EC5CB4-4EA2-40A2-8057-86542A0353BB}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU + {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.Release|Any CPU.Build.0 = Release|Any CPU + {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {143B1C29-C442-4BE0-BF3F-A8F92288AC9F}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU + {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.Release|Any CPU.Build.0 = Release|Any CPU + {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {61ECB8EE-0C96-4F8E-B187-8E4D227417C0}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU + {C61154BA-DD4A-4838-8420-0162A28925E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C61154BA-DD4A-4838-8420-0162A28925E0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C61154BA-DD4A-4838-8420-0162A28925E0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C61154BA-DD4A-4838-8420-0162A28925E0}.Release|Any CPU.Build.0 = Release|Any CPU + {C61154BA-DD4A-4838-8420-0162A28925E0}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {C61154BA-DD4A-4838-8420-0162A28925E0}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU + {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.Release|Any CPU.Build.0 = Release|Any CPU + {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {3D166931-BA2D-416E-95A3-D36E8F6E90B9}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU {A654F3B8-E859-4E6A-B30D-227527DBEF0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A654F3B8-E859-4E6A-B30D-227527DBEF0D}.Debug|Any CPU.Build.0 = Debug|Any CPU {A654F3B8-E859-4E6A-B30D-227527DBEF0D}.Release|Any CPU.ActiveCfg = Release|Any CPU {A654F3B8-E859-4E6A-B30D-227527DBEF0D}.Release|Any CPU.Build.0 = Release|Any CPU - {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.Release|Any CPU.Build.0 = Release|Any CPU - {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.Release|Any CPU.Build.0 = Release|Any CPU + {A654F3B8-E859-4E6A-B30D-227527DBEF0D}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {A654F3B8-E859-4E6A-B30D-227527DBEF0D}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU {BF62FE08-373A-43D6-9D73-41CAA38B7011}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BF62FE08-373A-43D6-9D73-41CAA38B7011}.Debug|Any CPU.Build.0 = Debug|Any CPU {BF62FE08-373A-43D6-9D73-41CAA38B7011}.Release|Any CPU.ActiveCfg = Release|Any CPU {BF62FE08-373A-43D6-9D73-41CAA38B7011}.Release|Any CPU.Build.0 = Release|Any CPU - {C61154BA-DD4A-4838-8420-0162A28925E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C61154BA-DD4A-4838-8420-0162A28925E0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C61154BA-DD4A-4838-8420-0162A28925E0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C61154BA-DD4A-4838-8420-0162A28925E0}.Release|Any CPU.Build.0 = Release|Any CPU - {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CCC4440E-49F7-4790-B0AF-FEABB0837AE7}.Release|Any CPU.Build.0 = Release|Any CPU + {BF62FE08-373A-43D6-9D73-41CAA38B7011}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {BF62FE08-373A-43D6-9D73-41CAA38B7011}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU + {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.Release|Any CPU.Build.0 = Release|Any CPU + {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {AE21D0EE-9A2C-4C15-AB7F-5224EED5B0EA}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU + {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.Release|Any CPU.Build.0 = Release|Any CPU + {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {AA5E328A-8835-49D7-98ED-C29F2B3049F0}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU {F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.Debug|Any CPU.Build.0 = Debug|Any CPU {F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.Release|Any CPU.ActiveCfg = Release|Any CPU {F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(NestedProjects) = preSolution + {F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.ReleaseSigned|Any CPU.ActiveCfg = ReleaseSigned|Any CPU + {F8C6D937-C44B-4EE3-A431-B0FBAEACE47D}.ReleaseSigned|Any CPU.Build.0 = ReleaseSigned|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/csharp/build_packages.bat b/src/csharp/build_packages.bat index c3e5fe8817d..9e1253bf0bd 100644 --- a/src/csharp/build_packages.bat +++ b/src/csharp/build_packages.bat @@ -12,7 +12,7 @@ cd ..\..\vsprojects\nuget_package @call buildall.bat || goto :error endlocal -@call buildall.bat || goto :error +@call buildall.bat BUILD_SIGNED || goto :error %NUGET% pack ..\..\vsprojects\nuget_package\grpc.native.csharp_ext.nuspec -Version %CORE_VERSION% || goto :error %NUGET% pack Grpc.Auth\Grpc.Auth.nuspec -Symbols -Version %VERSION% || goto :error diff --git a/src/csharp/buildall.bat b/src/csharp/buildall.bat index 16860aec3c3..e73feb87b9d 100644 --- a/src/csharp/buildall.bat +++ b/src/csharp/buildall.bat @@ -13,6 +13,11 @@ msbuild ..\..\vsprojects\grpc.sln /t:grpc_csharp_ext /p:PlatformToolset=v120 || msbuild Grpc.sln /p:Configuration=Debug || goto :error msbuild Grpc.sln /p:Configuration=Release || goto :error + +if "%1" == "BUILD_SIGNED" ( +msbuild Grpc.sln /p:Configuration=ReleaseSigned || goto :error +) + endlocal goto :EOF diff --git a/src/csharp/keys/Grpc.public.snk b/src/csharp/keys/Grpc.public.snk new file mode 100644 index 0000000000000000000000000000000000000000..bac3046b367212f394cd5ef7e9b802aeccc0d238 GIT binary patch literal 160 zcmV;R0AK$ABme*efB*oL000060ssI2Bme+XQ$aBR1ONa50096mSC^?QZ_Vf%)CrG6 z1VX!x5_X3o-U|$|v0jy1GNBVvbPna8ER@8K+>RH05K>cXipEjA;T+}KIw$; Date: Mon, 27 Jul 2015 18:22:36 -0700 Subject: [PATCH 39/47] Adding new interop tests --- tools/gce_setup/cloud_prod_runner.sh | 4 ++-- tools/gce_setup/grpc_docker.sh | 7 ++++++- tools/gce_setup/interop_test_runner.sh | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/gce_setup/cloud_prod_runner.sh b/tools/gce_setup/cloud_prod_runner.sh index 4206a66db78..904e8d65ad9 100755 --- a/tools/gce_setup/cloud_prod_runner.sh +++ b/tools/gce_setup/cloud_prod_runner.sh @@ -34,8 +34,8 @@ log_link=https://pantheon.corp.google.com/m/cloudstorage/b/stoked-keyword-656-ou main() { source grpc_docker.sh - test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response) - auth_test_cases=(service_account_creds compute_engine_creds jwt_token_creds) + test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response empty_stream timeout_on_sleeping_server slower_consumer half_duplex) + auth_test_cases=(service_account_creds compute_engine_creds jwt_token_creds oauth2_auth_token per_rpc_creds) clients=(cxx java go ruby node csharp_mono csharp_dotnet python php) for test_case in "${test_cases[@]}" do diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index b53aa98aab9..c06780a6993 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -508,7 +508,12 @@ grpc_cloud_prod_auth_test_args() { grpc_gen_test_cmd="grpc_cloud_prod_auth_" [[ -n $1 ]] && { # test_case test_case=$1 - grpc_gen_test_cmd+="$1" + test_command="service_account_creds" + if [ "$test_case" == "compute_engine_creds" ] + then + test_command="compute_engine_creds" + fi + grpc_gen_test_cmd+=$test_command shift } || { echo "$FUNCNAME: missing arg: test_case" 1>&2 diff --git a/tools/gce_setup/interop_test_runner.sh b/tools/gce_setup/interop_test_runner.sh index 6ff3731985d..1825694b319 100755 --- a/tools/gce_setup/interop_test_runner.sh +++ b/tools/gce_setup/interop_test_runner.sh @@ -37,7 +37,7 @@ fail_log_link=https://pantheon.corp.google.com/m/cloudstorage/b/stoked-keyword-6 main() { source grpc_docker.sh - test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response) + test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response empty_stream timeout_on_sleeping_server slower_consumer half_duplex) clients=(cxx java go ruby node python csharp_mono php) servers=(cxx java go ruby node python csharp_mono) for test_case in "${test_cases[@]}" From 8a67d39ada064b65e10a918d30be1e8bd21e84ac Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 28 Jul 2015 07:41:54 -0700 Subject: [PATCH 40/47] Integration compile fix --- .../end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c b/test/core/end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c index 03d3be5a7c3..a2ab25d886f 100644 --- a/test/core/end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c +++ b/test/core/end2end/fixtures/chttp2_fullstack_uds_posix_with_poll.c @@ -78,7 +78,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_fullstack( void chttp2_init_client_fullstack(grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { fullstack_fixture_data *ffd = f->fixture_data; - f->client = grpc_channel_create(ffd->localaddr, client_args); + f->client = grpc_insecure_channel_create(ffd->localaddr, client_args); } void chttp2_init_server_fullstack(grpc_end2end_test_fixture *f, From a4c4d3c446ea75506d9343b07e59942616c522a2 Mon Sep 17 00:00:00 2001 From: Alistair Veitch Date: Tue, 28 Jul 2015 14:36:22 -0700 Subject: [PATCH 41/47] fix feature/function typo --- include/grpc/census.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/grpc/census.h b/include/grpc/census.h index 4b5fd1b2116..7603dfdce1c 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -54,7 +54,7 @@ enum census_features { CENSUS_FEATURE_TRACING | CENSUS_FEATURE_STATS | CENSUS_FEATURE_CPU }; -/** Shutdown and startup census subsystem. The 'functions' argument should be +/** Shutdown and startup census subsystem. The 'features' argument should be * the OR (|) of census_features values. If census fails to initialize, then * census_initialize() will return a non-zero value. It is an error to call * census_initialize() more than once (without an intervening From f29e9db90c0ab443ff9836c9946f2fdfd067cf8e Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Wed, 29 Jul 2015 06:00:22 +0200 Subject: [PATCH 42/47] Fixing Windows's missing pollset_set. --- src/core/iomgr/tcp_windows.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/tcp_windows.c b/src/core/iomgr/tcp_windows.c index 38ae74abecd..5e0457a37b1 100644 --- a/src/core/iomgr/tcp_windows.c +++ b/src/core/iomgr/tcp_windows.c @@ -368,7 +368,14 @@ static grpc_endpoint_write_status win_write(grpc_endpoint *ep, return GRPC_ENDPOINT_WRITE_PENDING; } -static void win_add_to_pollset(grpc_endpoint *ep, grpc_pollset *pollset) { +static void win_add_to_pollset(grpc_endpoint *ep, grpc_pollset *ps) { + (void) ps; + grpc_tcp *tcp = (grpc_tcp *) ep; + grpc_iocp_add_socket(tcp->socket); +} + +static void win_add_to_pollset_set(grpc_endpoint *ep, grpc_pollset_set *pss) { + (void) pss; grpc_tcp *tcp = (grpc_tcp *) ep; grpc_iocp_add_socket(tcp->socket); } @@ -402,8 +409,9 @@ static char *win_get_peer(grpc_endpoint *ep) { } static grpc_endpoint_vtable vtable = {win_notify_on_read, win_write, - win_add_to_pollset, win_shutdown, - win_destroy, win_get_peer}; + win_add_to_pollset, win_add_to_pollset_set, + win_shutdown, win_destroy, + win_get_peer}; grpc_endpoint *grpc_tcp_create(grpc_winsocket *socket, char *peer_string) { grpc_tcp *tcp = (grpc_tcp *) gpr_malloc(sizeof(grpc_tcp)); From b3d69c3c0fbd6a833214898fd172b8e1d2102b56 Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Wed, 29 Jul 2015 06:12:51 +0200 Subject: [PATCH 43/47] Fixing Windows build --- include/grpc++/dynamic_thread_pool.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/grpc++/dynamic_thread_pool.h b/include/grpc++/dynamic_thread_pool.h index bc4e2d4d749..f0cd35940f5 100644 --- a/include/grpc++/dynamic_thread_pool.h +++ b/include/grpc++/dynamic_thread_pool.h @@ -41,6 +41,7 @@ #include #include +#include #include namespace grpc { From eab5c3355dc11ffe8f20747ef2d60af36601c5f3 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 29 Jul 2015 11:49:31 -0700 Subject: [PATCH 44/47] fix race in server shutdown --- src/core/iomgr/tcp_server_windows.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/core/iomgr/tcp_server_windows.c b/src/core/iomgr/tcp_server_windows.c index cc680507ffc..bcd2aa8536f 100644 --- a/src/core/iomgr/tcp_server_windows.c +++ b/src/core/iomgr/tcp_server_windows.c @@ -79,6 +79,8 @@ struct grpc_tcp_server { /* active port count: how many ports are actually still listening */ int active_ports; + /* number of iomgr callbacks that have been explicitly scheduled during shutdown */ + int iomgr_callbacks_pending; /* all listening ports */ server_port *ports; @@ -93,6 +95,7 @@ grpc_tcp_server *grpc_tcp_server_create(void) { gpr_mu_init(&s->mu); gpr_cv_init(&s->cv); s->active_ports = 0; + s->iomgr_callbacks_pending = 0; s->cb = NULL; s->cb_arg = NULL; s->ports = gpr_malloc(sizeof(server_port) * INIT_PORT_CAP); @@ -112,10 +115,10 @@ void grpc_tcp_server_destroy(grpc_tcp_server *s, for (i = 0; i < s->nports; i++) { server_port *sp = &s->ports[i]; sp->shutting_down = 1; - grpc_winsocket_shutdown(sp->socket); + s->iomgr_callbacks_pending += grpc_winsocket_shutdown(sp->socket); } /* This happens asynchronously. Wait while that happens. */ - while (s->active_ports) { + while (s->active_ports || s->iomgr_callbacks_pending) { gpr_cv_wait(&s->cv, &s->mu, gpr_inf_future(GPR_CLOCK_REALTIME)); } gpr_mu_unlock(&s->mu); @@ -254,8 +257,16 @@ static void on_accept(void *arg, int from_iocp) { /* The general mechanism for shutting down is to queue abortion calls. While this is necessary in the read/write case, it's useless for the accept - case. Let's do nothing. */ - if (!from_iocp) return; + case. We only need to adjust the pending callback count */ + if (!from_iocp) { + gpr_mu_lock(&sp->server->mu); + GPR_ASSERT(sp->server->iomgr_callbacks_pending > 0); + if (0 == --sp->server->iomgr_callbacks_pending) { + gpr_cv_broadcast(&sp->server->cv); + } + gpr_mu_unlock(&sp->server->mu); + return; + } /* The IOCP notified us of a completed operation. Let's grab the results, and act accordingly. */ @@ -264,11 +275,12 @@ static void on_accept(void *arg, int from_iocp) { &transfered_bytes, FALSE, &flags); if (!wsa_success) { if (sp->shutting_down) { - /* During the shutdown case, we ARE expecting an error. So that's swell, + /* During the shutdown case, we ARE expecting an error. So that's well, and we can wake up the shutdown thread. */ sp->shutting_down = 0; sp->socket->read_info.outstanding = 0; gpr_mu_lock(&sp->server->mu); + GPR_ASSERT(sp->server->active_ports > 0); if (0 == --sp->server->active_ports) { gpr_cv_broadcast(&sp->server->cv); } From 206b6866a4106f673249e7486879114925f9df1b Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Wed, 29 Jul 2015 14:13:09 -0700 Subject: [PATCH 45/47] removing unnecessary tests. --- tools/gce_setup/cloud_prod_runner.sh | 2 +- tools/gce_setup/interop_test_runner.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/gce_setup/cloud_prod_runner.sh b/tools/gce_setup/cloud_prod_runner.sh index 904e8d65ad9..73437028837 100755 --- a/tools/gce_setup/cloud_prod_runner.sh +++ b/tools/gce_setup/cloud_prod_runner.sh @@ -34,7 +34,7 @@ log_link=https://pantheon.corp.google.com/m/cloudstorage/b/stoked-keyword-656-ou main() { source grpc_docker.sh - test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response empty_stream timeout_on_sleeping_server slower_consumer half_duplex) + test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response empty_stream timeout_on_sleeping_server) auth_test_cases=(service_account_creds compute_engine_creds jwt_token_creds oauth2_auth_token per_rpc_creds) clients=(cxx java go ruby node csharp_mono csharp_dotnet python php) for test_case in "${test_cases[@]}" diff --git a/tools/gce_setup/interop_test_runner.sh b/tools/gce_setup/interop_test_runner.sh index 1825694b319..506471411be 100755 --- a/tools/gce_setup/interop_test_runner.sh +++ b/tools/gce_setup/interop_test_runner.sh @@ -37,7 +37,7 @@ fail_log_link=https://pantheon.corp.google.com/m/cloudstorage/b/stoked-keyword-6 main() { source grpc_docker.sh - test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response empty_stream timeout_on_sleeping_server slower_consumer half_duplex) + test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response empty_stream timeout_on_sleeping_server) clients=(cxx java go ruby node python csharp_mono php) servers=(cxx java go ruby node python csharp_mono) for test_case in "${test_cases[@]}" From 766d9ee14855c4fe345dd2f241ea646df3308f05 Mon Sep 17 00:00:00 2001 From: yang-g Date: Wed, 29 Jul 2015 15:51:25 -0700 Subject: [PATCH 46/47] fix build --- include/grpc++/impl/sync_no_cxx11.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/grpc++/impl/sync_no_cxx11.h b/include/grpc++/impl/sync_no_cxx11.h index fda668957eb..5869b04c765 100644 --- a/include/grpc++/impl/sync_no_cxx11.h +++ b/include/grpc++/impl/sync_no_cxx11.h @@ -87,7 +87,7 @@ class condition_variable { ~condition_variable() { gpr_cv_destroy(&cv_); } void wait(lock_guard &mu) { mu.locked = false; - gpr_cv_wait(&cv_, &mu.mu_.mu_, gpr_inf_future(GPR_CLOCK_REALTIME); + gpr_cv_wait(&cv_, &mu.mu_.mu_, gpr_inf_future(GPR_CLOCK_REALTIME)); mu.locked = true; } void notify_one() { gpr_cv_signal(&cv_); } From 95099bb934ec8a50fbf5cb25cd6884a8b6464d33 Mon Sep 17 00:00:00 2001 From: Donna Dionne Date: Wed, 29 Jul 2015 16:00:13 -0700 Subject: [PATCH 47/47] adding one more prepare command to mirror repositories. --- tools/run_tests/run_interops_build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_interops_build.sh b/tools/run_tests/run_interops_build.sh index bbe36eed620..23441a5300d 100755 --- a/tools/run_tests/run_interops_build.sh +++ b/tools/run_tests/run_interops_build.sh @@ -31,10 +31,11 @@ set -e -#clean up any old docker files +#clean up any old docker files and start mirroring repository if not started already sudo docker rmi -f grpc/cxx || true sudo docker rmi -f grpc/base || true sudo docker rmi -f 0.0.0.0:5000/grpc/base || true +sudo docker run -d -e GCS_BUCKET=docker-interop-images -e STORAGE_PATH=/admin/docker_images -p 5000:5000 google/docker-registry || true #prepare building by pulling down base images and necessary files sudo docker pull 0.0.0.0:5000/grpc/base