[build] Speed up clang_format (#34839)

This reduces clang-format duration to under 10s on my 128 core machine.
Previously I stopped the process at 10 minutes. Major changes:

Eliminates 18,000 files
```
find ... \
    -and -not -path '*/cmake/build/*' \
    -and -not -path '*/huffman_geometries/*' \
```

Also, this now runs `$CPU_COUNT` number of parallel clang-format
processes with a roughly equal number of files. Previously, this script
was formatting one file at a time in separate processes (`xargs -n 1`).
pull/34860/head
AJ Heller 1 year ago committed by GitHub
parent fb5a3fad57
commit 50b654894a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 26
      tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh

@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
set -ex
# directories to run against
DIRS="examples/cpp examples/android/binder src/core/lib src/core/tsi src/core/ext src/cpp test/core test/cpp include src/compiler src/ruby src/objective-c tools/distrib/python src/python/grpcio_observability"
@ -32,7 +32,25 @@ for dir in $DIRS
do
for glob in $GLOB
do
files="$files `find ${CLANG_FORMAT_ROOT}/$dir -name $glob -and -not -name '*.generated.*' -and -not -name '*.upb.h' -and -not -name '*.upb.c' -and -not -name '*.upbdefs.h' -and -not -name '*.upbdefs.c' -and -not -name '*.pb.h' -and -not -name '*.pb.c' -and -not -name '*.pb.cc' -and -not -name '*.pbobjc.h' -and -not -name '*.pbobjc.m' -and -not -name '*.pbrpc.h' -and -not -name '*.pbrpc.m' -and -not -name end2end_tests.cc -and -not -name grpc_shadow_boringssl.h -and -not -name grpc_tls_credentials_options.h -and -not -name grpc_tls_credentials_options_comparator_test.cc`"
files="$files `find ${CLANG_FORMAT_ROOT}/$dir -name $glob \
-and -not -name '*.generated.*' \
-and -not -name '*.upb.h' \
-and -not -name '*.upb.c' \
-and -not -name '*.upbdefs.h' \
-and -not -name '*.upbdefs.c' \
-and -not -name '*.pb.h' \
-and -not -name '*.pb.c' \
-and -not -name '*.pb.cc' \
-and -not -name '*.pbobjc.h' \
-and -not -name '*.pbobjc.m' \
-and -not -name '*.pbrpc.h' \
-and -not -name '*.pbrpc.m' \
-and -not -name end2end_tests.cc \
-and -not -name grpc_shadow_boringssl.h \
-and -not -name grpc_tls_credentials_options.h \
-and -not -name grpc_tls_credentials_options_comparator_test.cc \
-and -not -path '*/cmake/build/*' \
`"
done
done
@ -42,9 +60,11 @@ if [ -n "$CHANGED_FILES" ]; then
files=$(comm -12 <(echo $files | tr ' ' '\n' | sort -u) <(echo $CHANGED_FILES | tr ' ' '\n' | sort -u))
fi
FILES_PER_PROCESS="$(expr $(echo "$files" | grep -o '\n' | wc -l) / $CPU_COUNT + 1)"
if [ "$TEST" == "" ]
then
echo $files | xargs -P $CPU_COUNT -n 1 $CLANG_FORMAT -i
echo $files | xargs -P $CPU_COUNT -n $FILES_PER_PROCESS $CLANG_FORMAT -i --verbose
else
ok=yes
for file in $files

Loading…
Cancel
Save