diff --git a/src/php/bin/generate_proto_php.sh b/src/php/bin/generate_proto_php.sh index 1c943457ba1..5dd22f665f5 100755 --- a/src/php/bin/generate_proto_php.sh +++ b/src/php/bin/generate_proto_php.sh @@ -46,14 +46,6 @@ $PROTOC --proto_path=. \ src/proto/grpc/testing/empty.proto \ src/proto/grpc/testing/test.proto -# qps test protos -$PROTOC --proto_path=. \ - --php_out=src/php/tests/qps/generated_code \ - --grpc_out=src/php/tests/qps/generated_code \ - --plugin=$PLUGIN \ - src/proto/grpc/core/stats.proto \ - src/proto/grpc/testing/{benchmark_service,compiler_test,control,echo_messages,empty,empty_service,messages,payloads,proxy-service,report_qps_scenario_service,stats,test,worker_service}.proto - # change it back sed 's/message EmptyMessage/message Empty/g' \ src/proto/grpc/testing/empty.proto > $output_file @@ -61,3 +53,12 @@ mv $output_file ./src/proto/grpc/testing/empty.proto sed 's/grpc\.testing\.EmptyMessage/grpc\.testing\.Empty/g' \ src/proto/grpc/testing/test.proto > $output_file mv $output_file ./src/proto/grpc/testing/test.proto + +# Hack for xDS interop: need this to be a separate file in the correct namespace. +# To be removed when grpc_php_plugin generates service stubs. +echo ' ./src/php/tests/interop/Grpc/Testing/LoadBalancerStatsServiceStub.php diff --git a/src/php/lib/Grpc/RpcServer.php b/src/php/lib/Grpc/RpcServer.php new file mode 100644 index 00000000000..4c88b2e1849 --- /dev/null +++ b/src/php/lib/Grpc/RpcServer.php @@ -0,0 +1,168 @@ + => [ + // 'service' => , + // 'method' => , + // 'request' => , + // ] ] + protected $paths_map; + + private function waitForNextEvent() { + return $this->requestCall(); + } + + private function loadRequest($request) { + if (!$this->call) { + throw new Exception("serverCall is not ready"); + } + $event = $this->call->startBatch([ + OP_RECV_MESSAGE => true, + ]); + if (!$event->message) { + throw new Exception("Did not receive a proper message"); + } + $request->mergeFromString($event->message); + return $request; + } + + protected function sendOkResponse($response) { + if (!$this->call) { + throw new Exception("serverCall is not ready"); + } + $this->call->startBatch([ + OP_SEND_INITIAL_METADATA => [], + OP_SEND_MESSAGE => ['message' => + $response->serializeToString()], + OP_SEND_STATUS_FROM_SERVER => [ + 'metadata' => [], + 'code' => STATUS_OK, + 'details' => 'OK', + ], + OP_RECV_CLOSE_ON_SERVER => true, + ]); + } + + /** + * Add a service to this server + * + * @param Object $service The service to be added + */ + public function handle($service) { + $rf = new \ReflectionClass($service); + + // If input does not have a parent class, which should be the + // generated stub, don't proceeed. This might change in the + // future. + if (!$rf->getParentClass()) return; + + // The input class name needs to match the service name + $service_name = $rf->getName(); + $namespace = $rf->getParentClass()->getNamespaceName(); + $prefix = ""; + if ($namespace) { + $parts = explode("\\", $namespace); + foreach ($parts as $part) { + $prefix .= lcfirst($part) . "."; + } + } + $base_path = "/" . $prefix . $service_name; + + // Right now, assume all the methods in the class are RPC method + // implementations. Might change in the future. + $methods = $rf->getMethods(); + foreach ($methods as $method) { + $method_name = $method->getName(); + $full_path = $base_path . "/" . ucfirst($method_name); + + $method_params = $method->getParameters(); + // RPC should have exactly 1 request param + if (count($method_params) != 1) continue; + $request_param = $method_params[0]; + // Method implementation must have type hint for request param + if (!$request_param->getType()) continue; + $request_type = $request_param->getType()->getName(); + + // $full_path needs to match the incoming event->method + // from requestCall() for us to know how to handle the request + $this->paths_map[$full_path] = [ + 'service' => $service, + 'method' => $method_name, + 'request' => new $request_type(), + ]; + } + } + + public function run() { + $this->start(); + while (true) { + // This blocks until the server receives a request + $event = $this->waitForNextEvent(); + if (!$event) { + throw new Exception( + "Unexpected error: server->waitForNextEvent delivers" + . " an empty event"); + } + if (!$event->call) { + throw new Exception( + "Unexpected error: server->waitForNextEvent delivers" + . " an event without a call"); + } + $this->call = $event->call; + $full_path = $event->method; + + // TODO: Can send a proper UNIMPLEMENTED response in the future + if (!array_key_exists($full_path, $this->paths_map)) continue; + + $service = $this->paths_map[$full_path]['service']; + $method = $this->paths_map[$full_path]['method']; + $request = $this->paths_map[$full_path]['request']; + + $request = $this->loadRequest($request); + if (!$request) { + throw new Exception("Unexpected error: fail to parse request"); + } + if (!method_exists($service, $method)) { + // TODO: Can send a proper UNIMPLEMENTED response in the future + throw new Exception("Method not implemented"); + } + + // Dispatch to actual server logic + $response = $service->$method($request); + $this->sendOkResponse($response); + $this->call = null; + } + } +} diff --git a/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsServiceStub.php b/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsServiceStub.php new file mode 100644 index 00000000000..529f5b0c4a7 --- /dev/null +++ b/src/php/tests/interop/Grpc/Testing/LoadBalancerStatsServiceStub.php @@ -0,0 +1,6 @@ +getNumRpcs(); + $timeout_sec = $request->getTimeoutSec(); + $rpcs_by_peer = []; + $num_failures = $num_rpcs; + + // Heavy limitation now: the server is blocking, until all + // the necessary num_rpcs are finished, or timeout is reached + global $client_thread; + $start_id = count($client_thread->results) + 1; + $end_id = $start_id + $num_rpcs; + $now = hrtime(true); + $timeout = $now[0] + ($now[1] / 1e9) + $timeout_sec; + while (true) { + $curr_hr = hrtime(true); + $curr_time = $curr_hr[0] + ($curr_hr[1] / 1e9); + if ($curr_time > $timeout) { + break; + } + // Thread variable seems to be read-only + $curr_id = count($client_thread->results); + if ($curr_id >= $end_id) { + break; + } + usleep(50000); + } + + // Tally up results + $end_id = min($end_id, count($client_thread->results)); + for ($i = $start_id; $i < $end_id; $i++) { + $hostname = $client_thread->results[$i]; + if ($hostname) { + $num_failures -= 1; + if (!array_key_exists($hostname, $rpcs_by_peer)) { + $rpcs_by_peer[$hostname] = 0; + } + $rpcs_by_peer[$hostname] += 1; + } + } + $response = new Grpc\Testing\LoadBalancerStatsResponse(); + $response->setRpcsByPeer($rpcs_by_peer); + $response->setNumFailures($num_failures); + return $response; + } +} + +// This client thread blindly sends a unary RPC to the server once +// every 1 / qps seconds. +class ClientThread extends Thread { + private $server_address_; + private $target_seconds_between_rpcs_; + private $fail_on_failed_rpcs_; + private $autoload_path_; + public $results; + + public function __construct($server_address, $qps, $fail_on_failed_rpcs, + $autoload_path) { + $this->server_address_ = $server_address; + $this->target_seconds_between_rpcs_ = 1.0 / $qps; + $this->fail_on_failed_rpcs_ = $fail_on_failed_rpcs; + $this->autoload_path_ = $autoload_path; + $this->results = []; + } + + public function run() { + // Autoloaded classes do not get inherited in threads. + // Hence we need to do this. + require_once($this->autoload_path_); + + $stub = new Grpc\Testing\TestServiceClient($this->server_address_, [ + 'credentials' => Grpc\ChannelCredentials::createInsecure() + ]); + $request = new Grpc\Testing\SimpleRequest(); + $target_next_start_us = hrtime(true) / 1000; + while (true) { + $now_us = hrtime(true) / 1000; + $sleep_us = $target_next_start_us - $now_us; + if ($sleep_us < 0) { + echo "php xds: warning, rpc takes too long to finish. " + . "If you consistently see this, the qps is too high.\n"; + } else { + usleep($sleep_us); + } + $target_next_start_us + += ($this->target_seconds_between_rpcs_ * 1000000); + list($response, $status) + = $stub->UnaryCall($request)->wait(); + if ($status->code == Grpc\STATUS_OK) { + $this->results[] = $response->getHostname(); + } else { + if ($this->fail_on_failed_rpcs_) { + throw new Exception('UnaryCall failed with status ' + . $status->code); + } + $this->results[] = ""; + } + } + } + + // This is needed for loading autoload_path in the child thread + public function start(int $options = PTHREADS_INHERIT_ALL) { + return parent::start(PTHREADS_INHERIT_NONE); + } +} + + +// Note: num_channels are currently ignored for now +$args = getopt('', ['fail_on_failed_rpcs:', 'num_channels:', + 'server:', 'stats_port:', 'qps:']); + +$client_thread = new ClientThread($args['server'], $args['qps'], + $args['fail_on_failed_rpcs'], + $autoload_path); +$client_thread->start(); + +$server = new Grpc\RpcServer(); +$server->addHttp2Port('0.0.0.0:'.$args['stats_port']); +$server->handle(new LoadBalancerStatsService()); +$server->run(); diff --git a/tools/dockerfile/test/php73_zts_stretch_x64/Dockerfile b/tools/dockerfile/test/php73_zts_stretch_x64/Dockerfile new file mode 100644 index 00000000000..70e6af733c9 --- /dev/null +++ b/tools/dockerfile/test/php73_zts_stretch_x64/Dockerfile @@ -0,0 +1,50 @@ +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM php:7.3-zts-stretch + +RUN apt-get -qq update && apt-get -qq install -y \ + autoconf automake build-essential git libtool curl \ + python-all-dev \ + python3-all-dev \ + python-setuptools + +WORKDIR /tmp + +RUN git clone https://github.com/grpc/grpc +RUN git clone https://github.com/krakjoe/pthreads + +RUN cd grpc && \ + git submodule update --init --recursive && \ + make && \ + make install && \ + cd third_party/protobuf && \ + make install && \ + ldconfig + +RUN cd pthreads && \ + phpize && \ + ./configure && \ + make && \ + make install + +RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7 +RUN pip install --upgrade pip==19.3.1 +RUN pip install virtualenv==16.7.9 +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0 + +RUN curl -sS https://getcomposer.org/installer | php +RUN mv composer.phar /usr/local/bin/composer + +WORKDIR /var/local/git/grpc diff --git a/tools/internal_ci/linux/grpc_experiment.cfg b/tools/internal_ci/linux/grpc_experiment.cfg new file mode 100644 index 00000000000..5bf9f931675 --- /dev/null +++ b/tools/internal_ci/linux/grpc_experiment.cfg @@ -0,0 +1,25 @@ +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_xds_php.sh" +timeout_mins: 90 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/linux/grpc_xds_php.cfg b/tools/internal_ci/linux/grpc_xds_php.cfg new file mode 100644 index 00000000000..5bf9f931675 --- /dev/null +++ b/tools/internal_ci/linux/grpc_xds_php.cfg @@ -0,0 +1,25 @@ +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_xds_php.sh" +timeout_mins: 90 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/linux/grpc_xds_php.sh b/tools/internal_ci/linux/grpc_xds_php.sh new file mode 100755 index 00000000000..8b643430ea3 --- /dev/null +++ b/tools/internal_ci/linux/grpc_xds_php.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +export DOCKERFILE_DIR=tools/dockerfile/test/php73_zts_stretch_x64 +export DOCKER_RUN_SCRIPT=tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh +export OUTPUT_DIR=reports +exec tools/run_tests/dockerize/build_and_run_docker.sh diff --git a/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh new file mode 100755 index 00000000000..0d971d7b69b --- /dev/null +++ b/tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex -o igncr || set -ex + +mkdir -p /var/local/git +git clone /var/local/jenkins/grpc /var/local/git/grpc +(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ +&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ +${name}') +cd /var/local/git/grpc + +VIRTUAL_ENV=$(mktemp -d) +virtualenv "$VIRTUAL_ENV" +PYTHON="$VIRTUAL_ENV"/bin/python +"$PYTHON" -m pip install --upgrade pip +"$PYTHON" -m pip install --upgrade grpcio-tools google-api-python-client google-auth-httplib2 oauth2client + +# Prepare generated Python code. +TOOLS_DIR=tools/run_tests +PROTO_SOURCE_DIR=src/proto/grpc/testing +PROTO_DEST_DIR="$TOOLS_DIR"/"$PROTO_SOURCE_DIR" +mkdir -p "$PROTO_DEST_DIR" +touch "$TOOLS_DIR"/src/__init__.py +touch "$TOOLS_DIR"/src/proto/__init__.py +touch "$TOOLS_DIR"/src/proto/grpc/__init__.py +touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py + +"$PYTHON" -m grpc_tools.protoc \ + --proto_path=. \ + --python_out="$TOOLS_DIR" \ + --grpc_python_out="$TOOLS_DIR" \ + "$PROTO_SOURCE_DIR"/test.proto \ + "$PROTO_SOURCE_DIR"/messages.proto \ + "$PROTO_SOURCE_DIR"/empty.proto + +# Compile the PHP extension. +(cd src/php/ext/grpc && \ + phpize && \ + ./configure && \ + make && \ + make install) + +# Prepare generated PHP code. +export CC=/usr/bin/gcc +./tools/bazel build @com_google_protobuf//:protoc +./tools/bazel build src/compiler:grpc_php_plugin +(cd src/php && \ + composer install && \ + ./bin/generate_proto_php.sh) + +GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ + tools/run_tests/run_xds_tests.py \ + --test_case=all \ + --project_id=grpc-testing \ + --source_image=projects/grpc-testing/global/images/xds-test-server \ + --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ + --gcp_suffix=$(date '+%s') \ + --only_stable_gcp_apis \ + --verbose \ + --client_cmd='php -d extension=grpc.so -d extension=pthreads.so src/php/tests/interop/xds_client.php --server=xds-experimental:///{server_uri} --stats_port={stats_port} --qps={qps}'