First pass IWYU tooling (#27857)

* First pass IWYU tooling

* fix

* Automated change: Fix sanity tests

* Update iwyu.sh

* Update iwyu.sh

* Update Dockerfile.template

* Update iwyu.sh

* Automated change: Fix sanity tests

Co-authored-by: ctiller <ctiller@users.noreply.github.com>

.gitignore

@@ -157,3 +157,8 @@ BenchmarkDotNet.Artifacts/
 
 # clang JSON compilation database file
 compile_commands.json
+
+# IWYU byproducts
+compile_commands_for_iwyu.json
+iwyu.out
+iwyu_files.txt

BUILD

@@ -1246,6 +1246,9 @@ grpc_cc_library(
     srcs = [
         "src/core/lib/promise/activity.cc",
     ],
+    external_deps = [
+        "absl/base:core_headers",
+    ],
     language = "c++",
     public_hdrs = [
         "src/core/lib/promise/activity.h",

src/core/lib/promise/activity.cc

@@ -16,6 +16,8 @@
 
 #include "src/core/lib/promise/activity.h"
 
+#include "absl/base/attributes.h"
+
 #include "src/core/lib/gprpp/atomic_utils.h"
 
 namespace grpc_core {

src/core/lib/promise/activity.h

@@ -17,10 +17,24 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stddef.h>
+#include <stdint.h>
+
+#include <algorithm>
+#include <atomic>
 #include <functional>
+#include <memory>
+#include <utility>
+
+#include "absl/base/thread_annotations.h"
+#include "absl/status/status.h"
+#include "absl/types/optional.h"
+#include "absl/types/variant.h"
+#include "absl/utility/utility.h"
 
 #include <grpc/support/log.h>
 
+#include "src/core/lib/gpr/tls.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/gprpp/sync.h"
 #include "src/core/lib/promise/context.h"

src/core/lib/promise/context.h

@@ -17,6 +17,7 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <type_traits>
 #include <utility>
 
 #include "src/core/lib/gpr/tls.h"

src/core/lib/promise/detail/basic_join.h

@@ -17,14 +17,21 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <assert.h>
+#include <stddef.h>
+
+#include <array>
 #include <tuple>
+#include <type_traits>
 #include <utility>
 
+#include "absl/types/variant.h"
 #include "absl/utility/utility.h"
 
 #include "src/core/lib/gprpp/bitset.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
+#include "src/core/lib/promise/detail/promise_like.h"
 #include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {

src/core/lib/promise/detail/basic_seq.h

@@ -17,8 +17,13 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <array>
 #include <cassert>
+#include <new>
+#include <tuple>
+#include <utility>
 
+#include "absl/meta/type_traits.h"
 #include "absl/types/variant.h"
 #include "absl/utility/utility.h"
 
@@ -29,6 +34,8 @@
 
 namespace grpc_core {
 namespace promise_detail {
+template <typename F>
+class PromiseLike;
 
 // Helper for SeqState to evaluate some common types to all partial
 // specializations.

src/core/lib/promise/detail/promise_factory.h

@@ -17,6 +17,9 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <type_traits>
+#include <utility>
+
 #include "absl/meta/type_traits.h"
 
 #include "src/core/lib/promise/detail/promise_like.h"
@@ -65,6 +68,7 @@ struct IsVoidCallable<F, absl::void_t<decltype(std::declval<F>()())>> {
 // Given F(A,B,C,...), what's the return type?
 template <typename T, typename Ignored = void>
 struct ResultOfT;
+
 template <typename F, typename... Args>
 struct ResultOfT<F(Args...),
                  absl::void_t<decltype(std::declval<RemoveCVRef<F>>()(

src/core/lib/promise/detail/promise_like.h

@@ -19,6 +19,8 @@
 
 #include <utility>
 
+#include "absl/meta/type_traits.h"
+
 #include "src/core/lib/promise/poll.h"
 
 // A Promise is a callable object that returns Poll<T> for some T.

src/core/lib/promise/detail/status.h

@@ -17,6 +17,8 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <utility>
+
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 

src/core/lib/promise/exec_ctx_wakeup_scheduler.h

@@ -17,6 +17,9 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "src/core/lib/gprpp/debug_location.h"
+#include "src/core/lib/iomgr/closure.h"
+#include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/iomgr/exec_ctx.h"
 
 namespace grpc_core {

src/core/lib/promise/for_each.h

@@ -17,6 +17,9 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <type_traits>
+#include <utility>
+
 #include "absl/status/status.h"
 #include "absl/types/variant.h"
 
@@ -45,6 +48,7 @@ Poll<absl::Status> FinishIteration(absl::Status* r, Reader* reader,
 // easily.
 template <typename T>
 struct Done;
+
 template <>
 struct Done<absl::Status> {
   static absl::Status Make() { return absl::OkStatus(); }

src/core/lib/promise/if.h

@@ -17,10 +17,13 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <type_traits>
+
 #include "absl/status/statusor.h"
 #include "absl/types/variant.h"
 
 #include "src/core/lib/promise/detail/promise_factory.h"
+#include "src/core/lib/promise/detail/promise_like.h"
 #include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {

src/core/lib/promise/intra_activity_waiter.h

@@ -18,6 +18,7 @@
 #include <grpc/support/port_platform.h>
 
 #include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {
 

src/core/lib/promise/join.h

@@ -17,6 +17,8 @@
 
 #include <grpc/support/port_platform.h>
 
+#include "absl/meta/type_traits.h"
+
 #include "src/core/lib/promise/detail/basic_join.h"
 
 namespace grpc_core {

src/core/lib/promise/loop.h

@@ -17,9 +17,13 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <new>
+#include <type_traits>
+
 #include "absl/types/variant.h"
 
 #include "src/core/lib/promise/detail/promise_factory.h"
+#include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {
 
@@ -36,6 +40,7 @@ namespace promise_detail {
 
 template <typename T>
 struct LoopTraits;
+
 template <typename T>
 struct LoopTraits<LoopCtl<T>> {
   using Result = T;

src/core/lib/promise/map.h

@@ -17,7 +17,11 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stddef.h>
+
 #include <tuple>
+#include <type_traits>
+#include <utility>
 
 #include "absl/types/variant.h"
 

src/core/lib/promise/observable.h

@@ -17,11 +17,20 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stdint.h>
+
 #include <limits>
+#include <memory>
+#include <type_traits>
 
+#include "absl/base/thread_annotations.h"
+#include "absl/synchronization/mutex.h"
 #include "absl/types/optional.h"
+#include "absl/types/variant.h"
 
 #include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/detail/promise_like.h"
+#include "src/core/lib/promise/poll.h"
 #include "src/core/lib/promise/wait_set.h"
 
 namespace grpc_core {

src/core/lib/promise/pipe.h

@@ -17,9 +17,19 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <new>
+#include <type_traits>
+#include <utility>
+
+#include "absl/container/inlined_vector.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/types/optional.h"
+#include "absl/types/variant.h"
 
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/detail/promise_factory.h"
@@ -30,10 +40,8 @@ namespace grpc_core {
 
 template <typename T>
 struct Pipe;
-
 template <typename T>
 class PipeSender;
-
 template <typename T>
 class PipeReceiver;
 
@@ -41,7 +49,6 @@ namespace pipe_detail {
 
 template <typename T>
 class Push;
-
 template <typename T>
 class Next;
 

src/core/lib/promise/poll.h

@@ -17,6 +17,8 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stddef.h>
+
 #include "absl/types/variant.h"
 
 namespace grpc_core {

src/core/lib/promise/promise.h

@@ -18,8 +18,10 @@
 #include <grpc/support/port_platform.h>
 
 #include <functional>
+#include <type_traits>
 
 #include "absl/types/optional.h"
+#include "absl/types/variant.h"
 
 #include "src/core/lib/promise/detail/promise_like.h"
 #include "src/core/lib/promise/poll.h"

src/core/lib/promise/race.h

@@ -18,6 +18,9 @@
 #include <grpc/support/port_platform.h>
 
 #include <tuple>
+#include <type_traits>
+
+#include "absl/types/variant.h"
 
 #include "src/core/lib/promise/poll.h"
 

src/core/lib/promise/seq.h

@@ -17,6 +17,8 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <utility>
+
 #include "absl/types/variant.h"
 
 #include "src/core/lib/promise/detail/basic_seq.h"

src/core/lib/promise/try_join.h

@@ -17,8 +17,15 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <type_traits>
+
+#include "absl/meta/type_traits.h"
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+
 #include "src/core/lib/promise/detail/basic_join.h"
 #include "src/core/lib/promise/detail/status.h"
+#include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {
 

src/core/lib/promise/try_seq.h

@@ -18,7 +18,9 @@
 #include <grpc/support/port_platform.h>
 
 #include <tuple>
+#include <utility>
 
+#include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/types/variant.h"
 

src/core/lib/promise/wait_set.h

@@ -17,9 +17,12 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <utility>
+
 #include "absl/container/flat_hash_set.h"
 
 #include "src/core/lib/promise/activity.h"
+#include "src/core/lib/promise/poll.h"
 
 namespace grpc_core {
 

templates/tools/dockerfile/grpc_iwyu/Dockerfile.template

@@ -0,0 +1,35 @@
+%YAML 1.2
+--- |
+  # Copyright 2021 gRPC authors.
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+  
+  FROM debian:bullseye
+  
+  RUN apt-get update && apt-get install -y clang-11 llvm-11-dev libclang-11-dev clang-format-11 jq git cmake python
+  
+  ENV CLANG_FORMAT=clang-format-11
+
+  RUN git clone https://github.com/include-what-you-use/include-what-you-use.git /iwyu
+  # latest commit on the clang 11 branch
+  RUN cd /iwyu && git checkout 5db414ac448004fe019871c977905cb7c2cff23f
+
+  RUN mkdir /iwyu_build && cd /iwyu_build && cmake -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 /iwyu && make
+
+  ADD iwyu.sh /
+  
+  # When running locally, we'll be impersonating the current user, so we need
+  # to make the script runnable by everyone.
+  RUN chmod a+rx /iwyu.sh
+  
+  CMD ["echo 'Run with tools/distrib/iwyu.sh'"]

tools/distrib/iwyu.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright 2021 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo "NOTE: to automagically apply fixes, invoke with --fix"
+
+set -ex
+
+# change to root directory
+cd $(dirname $0)/../..
+REPO_ROOT=$(pwd)
+
+# grep targets with manual tag, which is not included in a result of bazel build using ...
+# let's get a list of them using query command and pass it to gen_compilation_database.py
+export MANUAL_TARGETS=$(bazel query 'attr("tags", "manual", tests(//test/cpp/...))' | grep -v _on_ios)
+
+# generate a clang compilation database for all C/C++ sources in the repo.
+tools/distrib/gen_compilation_database.py \
+  --include_headers \
+  --ignore_system_headers \
+  --dedup_targets \
+  "//:*" \
+  "//src/core/..." \
+  "//src/compiler/..." \
+  "//test/core/..." \
+  "//test/cpp/..." \
+  $MANUAL_TARGETS
+
+# build clang-tidy docker image
+docker build -t grpc_iwyu tools/dockerfile/grpc_iwyu
+
+# run clang-tidy against the checked out codebase
+# when modifying the checked-out files, the current user will be impersonated
+# so that the updated files don't end up being owned by "root".
+docker run \
+  -e TEST="$TEST" \
+  -e CHANGED_FILES="$CHANGED_FILES" \
+  -e IWYU_ROOT="/local-code" \
+  --rm=true \
+  -v "${REPO_ROOT}":/local-code \
+  -v "${HOME/.cache/bazel}":"${HOME/.cache/bazel}" \
+  --user "$(id -u):$(id -g)" \
+  -t grpc_iwyu /iwyu.sh "$@"
+

tools/dockerfile/grpc_iwyu/Dockerfile

@@ -0,0 +1,33 @@
+# Copyright 2021 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM debian:bullseye
+
+RUN apt-get update && apt-get install -y clang-11 llvm-11-dev libclang-11-dev clang-format-11 jq git cmake python
+
+ENV CLANG_FORMAT=clang-format-11
+
+RUN git clone https://github.com/include-what-you-use/include-what-you-use.git /iwyu
+# latest commit on the clang 11 branch
+RUN cd /iwyu && git checkout 5db414ac448004fe019871c977905cb7c2cff23f
+
+RUN mkdir /iwyu_build && cd /iwyu_build && cmake -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/lib/llvm-11 /iwyu && make
+
+ADD iwyu.sh /
+
+# When running locally, we'll be impersonating the current user, so we need
+# to make the script runnable by everyone.
+RUN chmod a+rx /iwyu.sh
+
+CMD ["echo 'Run with tools/distrib/iwyu.sh'"]

tools/dockerfile/grpc_iwyu/iwyu.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+cd ${IWYU_ROOT}
+
+export PATH=${PATH}:/iwyu_build/bin
+
+cat compile_commands.json | sed "s,\"file\": \",\"file\": \"${IWYU_ROOT}/,g" > compile_commands_for_iwyu.json
+
+# figure out which files to include
+cat compile_commands.json | jq -r '.[].file' \
+  | grep -E "^src/core/lib/promise/" \
+  | grep -v -E "/upb-generated/|/upbdefs-generated/" \
+  | sort \
+  | tee iwyu_files.txt
+
+# run iwyu, filtering out changes to port_platform.h
+xargs -a iwyu_files.txt /iwyu/iwyu_tool.py -p compile_commands_for_iwyu.json -j 16 \
+  | grep -v -E "port_platform.h" \
+  | tee iwyu.out
+
+# apply the suggested changes
+/iwyu/fix_includes.py --nocomments < iwyu.out || true
+
+# reformat sources, since iwyu gets this wrong
+xargs -a iwyu_files.txt $CLANG_FORMAT -i
+
+# TODO(ctiller): expand this to match the clang-tidy directories:
+#  | grep -E "(^include/|^src/core/|^src/cpp/|^test/core/|^test/cpp/)"