From 2680c5ba71a7b4c5cd2d633b0c1a6c117b313f6c Mon Sep 17 00:00:00 2001 From: "data-plane-api(Azure Pipelines)" Date: Tue, 13 Oct 2020 21:58:15 +0000 Subject: [PATCH] dependencies: refactor repository location schema utils, cleanups. (#13452) - Refactor code responsible for processing repository location specs, i.e. checking for the presence of fields like last_updated and interpolation of version. The same code is now usable by both API repository_locations.bzl and bazel/repository_locations.bzl. - Cleanup reference to repo locations in repository_locations.bzl, now using a consistent set of macros. - Add API dependencies to dependency dashboard. Risk level: Low Testing: Docs build. Part of #12673 Signed-off-by: Harvey Tuch Mirrored from https://github.com/envoyproxy/envoy @ 9181790e3ab21b53ec268470202986b9517c3723 --- bazel/envoy_http_archive.bzl | 3 +- bazel/external_deps.bzl | 116 +++++++++++++++++++++++++++ bazel/repositories.bzl | 39 ++++----- bazel/repository_locations.bzl | 23 +----- bazel/repository_locations_utils.bzl | 20 +++++ 5 files changed, 160 insertions(+), 41 deletions(-) create mode 100644 bazel/external_deps.bzl create mode 100644 bazel/repository_locations_utils.bzl diff --git a/bazel/envoy_http_archive.bzl b/bazel/envoy_http_archive.bzl index 13b98f77..15fd65b2 100644 --- a/bazel/envoy_http_archive.bzl +++ b/bazel/envoy_http_archive.bzl @@ -10,8 +10,7 @@ def envoy_http_archive(name, locations, **kwargs): # This repository has already been defined, probably because the user # wants to override the version. Do nothing. return - loc_key = kwargs.pop("repository_key", name) - location = locations[loc_key] + location = locations[name] # HTTP tarball at a given URL. Add a BUILD file if requested. http_archive( diff --git a/bazel/external_deps.bzl b/bazel/external_deps.bzl new file mode 100644 index 00000000..cd9b6759 --- /dev/null +++ b/bazel/external_deps.bzl @@ -0,0 +1,116 @@ +load("@envoy_api//bazel:repository_locations_utils.bzl", "load_repository_locations_spec") + +# Envoy dependencies may be annotated with the following attributes: +DEPENDENCY_ANNOTATIONS = [ + # List of the categories describing how the dependency is being used. This attribute is used + # for automatic tracking of security posture of Envoy's dependencies. + # Possible values are documented in the USE_CATEGORIES list below. + # This attribute is mandatory for each dependecy. + "use_category", + + # Attribute specifying CPE (Common Platform Enumeration, see https://nvd.nist.gov/products/cpe) ID + # of the dependency. The ID may be in v2.3 or v2.2 format, although v2.3 is prefferred. See + # https://nvd.nist.gov/products/cpe for CPE format. Use single wildcard '*' for version and vector elements + # i.e. 'cpe:2.3:a:nghttp2:nghttp2:*'. Use "N/A" for dependencies without CPE assigned. + # This attribute is optional for components with use categories listed in the + # USE_CATEGORIES_WITH_CPE_OPTIONAL + "cpe", +] + +# NOTE: If a dependency use case is either dataplane or controlplane, the other uses are not needed +# to be declared. +USE_CATEGORIES = [ + # This dependency is used in API protos. + "api", + # This dependency is used in build process. + "build", + # This dependency is used to process xDS requests. + "controlplane", + # This dependency is used in processing downstream or upstream requests (core). + "dataplane_core", + # This dependency is used in processing downstream or upstream requests (extensions). + "dataplane_ext", + # This dependecy is used for logging, metrics or tracing (core). It may process unstrusted input. + "observability_core", + # This dependecy is used for logging, metrics or tracing (extensions). It may process unstrusted input. + "observability_ext", + # This dependency does not handle untrusted data and is used for various utility purposes. + "other", + # This dependency is used only in tests. + "test_only", +] + +# Components with these use categories are not required to specify the 'cpe' +# and 'last_updated' annotation. +USE_CATEGORIES_WITH_CPE_OPTIONAL = ["build", "other", "test_only", "api"] + +def _fail_missing_attribute(attr, key): + fail("The '%s' attribute must be defined for external dependecy " % attr + key) + +# Method for verifying content of the repository location specifications. +# +# We also remove repository metadata attributes so that further consumers, e.g. +# http_archive, are not confused by them. +def load_repository_locations(repository_locations_spec): + locations = {} + for key, location in load_repository_locations_spec(repository_locations_spec).items(): + mutable_location = dict(location) + locations[key] = mutable_location + + if "sha256" not in location or len(location["sha256"]) == 0: + _fail_missing_attribute("sha256", key) + + if "project_name" not in location: + _fail_missing_attribute("project_name", key) + mutable_location.pop("project_name") + + if "project_desc" not in location: + _fail_missing_attribute("project_desc", key) + mutable_location.pop("project_desc") + + if "project_url" not in location: + _fail_missing_attribute("project_url", key) + project_url = mutable_location.pop("project_url") + if not project_url.startswith("https://") and not project_url.startswith("http://"): + fail("project_url must start with https:// or http://: " + project_url) + + if "version" not in location: + _fail_missing_attribute("version", key) + mutable_location.pop("version") + + if "use_category" not in location: + _fail_missing_attribute("use_category", key) + use_category = mutable_location.pop("use_category") + + if "dataplane_ext" in use_category or "observability_ext" in use_category: + if "extensions" not in location: + _fail_missing_attribute("extensions", key) + mutable_location.pop("extensions") + + if "last_updated" not in location: + _fail_missing_attribute("last_updated", key) + last_updated = mutable_location.pop("last_updated") + + # Starlark doesn't have regexes. + if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-": + fail("last_updated must match YYYY-DD-MM: " + last_updated) + + if "cpe" in location: + cpe = mutable_location.pop("cpe") + + # Starlark doesn't have regexes. + cpe_components = len(cpe.split(":")) + + # We allow cpe:2.3:a:foo:* and cpe:2.3.:a:foo:bar:* only. + cpe_components_valid = cpe_components in [5, 6] + cpe_matches = (cpe == "N/A" or (cpe.startswith("cpe:2.3:a:") and cpe.endswith(":*") and cpe_components_valid)) + if not cpe_matches: + fail("CPE must match cpe:2.3:a:::*: " + cpe) + elif not [category for category in USE_CATEGORIES_WITH_CPE_OPTIONAL if category in location["use_category"]]: + _fail_missing_attribute("cpe", key) + + for category in location["use_category"]: + if category not in USE_CATEGORIES: + fail("Unknown use_category value '" + category + "' for dependecy " + key) + + return locations diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index a64e733c..a12a0ea9 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -1,40 +1,43 @@ load(":envoy_http_archive.bzl", "envoy_http_archive") -load(":repository_locations.bzl", "REPOSITORY_LOCATIONS") +load(":external_deps.bzl", "load_repository_locations") +load(":repository_locations.bzl", "REPOSITORY_LOCATIONS_SPEC") -def api_dependencies(): +REPOSITORY_LOCATIONS = load_repository_locations(REPOSITORY_LOCATIONS_SPEC) + +# Use this macro to reference any HTTP archive from bazel/repository_locations.bzl. +def external_http_archive(name, **kwargs): envoy_http_archive( - "bazel_skylib", + name, locations = REPOSITORY_LOCATIONS, + **kwargs ) - envoy_http_archive( - "com_envoyproxy_protoc_gen_validate", - locations = REPOSITORY_LOCATIONS, + +def api_dependencies(): + external_http_archive( + name = "bazel_skylib", ) - envoy_http_archive( + external_http_archive( + name = "com_envoyproxy_protoc_gen_validate", + ) + external_http_archive( name = "com_google_googleapis", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "com_github_cncf_udpa", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "prometheus_metrics_model", - locations = REPOSITORY_LOCATIONS, build_file_content = PROMETHEUSMETRICS_BUILD_CONTENT, ) - envoy_http_archive( + external_http_archive( name = "opencensus_proto", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "rules_proto", - locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( + external_http_archive( name = "com_github_openzipkin_zipkinapi", - locations = REPOSITORY_LOCATIONS, build_file_content = ZIPKINAPI_BUILD_CONTENT, ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 152d4be5..bdcf31e8 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1,4 +1,5 @@ -DEPENDENCY_REPOSITORIES_SPEC = dict( +# This should match the schema defined in external_deps.bzl. +REPOSITORY_LOCATIONS_SPEC = dict( bazel_skylib = dict( project_name = "bazel-skylib", project_desc = "Common useful functions and rules for Bazel", @@ -88,23 +89,3 @@ DEPENDENCY_REPOSITORIES_SPEC = dict( use_category = ["api"], ), ) - -def _format_version(s, version): - return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_")) - -# Interpolate {version} in the above dependency specs. This code should be capable of running in both Python -# and Starlark. -def _dependency_repositories(): - locations = {} - for key, location in DEPENDENCY_REPOSITORIES_SPEC.items(): - mutable_location = dict(location) - locations[key] = mutable_location - - # Fixup with version information. - if "version" in location: - if "strip_prefix" in location: - mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"]) - mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]] - return locations - -REPOSITORY_LOCATIONS = _dependency_repositories() diff --git a/bazel/repository_locations_utils.bzl b/bazel/repository_locations_utils.bzl new file mode 100644 index 00000000..3b984e1b --- /dev/null +++ b/bazel/repository_locations_utils.bzl @@ -0,0 +1,20 @@ +def _format_version(s, version): + return s.format(version = version, dash_version = version.replace(".", "-"), underscore_version = version.replace(".", "_")) + +# Generate a "repository location specification" from raw repository +# specification. The information should match the format required by +# external_deps.bzl. This function mostly does interpolation of {version} in +# the repository info fields. This code should be capable of running in both +# Python and Starlark. +def load_repository_locations_spec(repository_locations_spec): + locations = {} + for key, location in repository_locations_spec.items(): + mutable_location = dict(location) + locations[key] = mutable_location + + # Fixup with version information. + if "version" in location: + if "strip_prefix" in location: + mutable_location["strip_prefix"] = _format_version(location["strip_prefix"], location["version"]) + mutable_location["urls"] = [_format_version(url, location["version"]) for url in location["urls"]] + return locations