dependencies: track untracked implied dependencies, wrapup dashboard. (#13571)

A few changes that wrapup #12673.

* Python/Go dependencies that aren't part of the Envoy binary build
  don't make sense to track in repository_locations.bzl, since they
  have their own language specific metadata (e.g. requirements.txt)
  or are in many cases transitively implied.

* Ensure that the full set of dependencies visible to bazel query
  is now validated. This requires that we explicitly call out
  transitive dependencies that are implied by direct dependencies
  in repository_locations.bzl. A new annotation `implied_untracked_deps`
  is used.

Fixes #12673

Risk level: Low
Testing: validate.py.

Signed-off-by: Harvey Tuch <htuch@google.com>

Mirrored from https://github.com/envoyproxy/envoy @ 499f46a24f59eeb4fc3b28e24f6b9191b009580d

bazel/external_deps.bzl

@@ -2,12 +2,6 @@ load("@envoy_api//bazel:repository_locations_utils.bzl", "load_repository_locati
 
 # Envoy dependencies may be annotated with the following attributes:
 DEPENDENCY_ANNOTATIONS = [
-    # List of the categories describing how the dependency is being used. This attribute is used
-    # for automatic tracking of security posture of Envoy's dependencies.
-    # Possible values are documented in the USE_CATEGORIES list below.
-    # This attribute is mandatory for each dependecy.
-    "use_category",
-
     # Attribute specifying CPE (Common Platform Enumeration, see https://nvd.nist.gov/products/cpe) ID
     # of the dependency. The ID may be in v2.3 or v2.2 format, although v2.3 is prefferred. See
     # https://nvd.nist.gov/products/cpe for CPE format. Use single wildcard '*' for version and vector elements
@@ -15,6 +9,31 @@ DEPENDENCY_ANNOTATIONS = [
     # This attribute is optional for components with use categories listed in the
     # USE_CATEGORIES_WITH_CPE_OPTIONAL
     "cpe",
+
+    # A list of extensions when 'use_category' contains 'dataplane_ext' or 'observability_ext'.
+    "extensions",
+
+    # Additional dependencies loaded transitively via this dependency that are not tracked in
+    # Envoy (see the external dependency at the given version for information).
+    "implied_untracked_deps",
+
+    # When the dependency was last updated in Envoy.
+    "last_updated",
+
+    # Project metadata.
+    "project_desc",
+    "project_name",
+    "project_url",
+
+    # List of the categories describing how the dependency is being used. This attribute is used
+    # for automatic tracking of security posture of Envoy's dependencies.
+    # Possible values are documented in the USE_CATEGORIES list below.
+    # This attribute is mandatory for each dependecy.
+    "use_category",
+
+    # The dependency version. This may be either a tagged release (preferred)
+    # or git SHA (as an exception when no release tagged version is suitable).
+    "version",
 ]
 
 # NOTE: If a dependency use case is either dataplane or controlplane, the other uses are not needed
@@ -38,6 +57,10 @@ USE_CATEGORIES = [
     "other",
     # This dependency is used only in tests.
     "test_only",
+    # Documentation generation
+    "docs",
+    # Developer tools (not used in build or docs)
+    "devtools",
 ]
 
 # Components with these use categories are not required to specify the 'cpe'
@@ -62,41 +85,37 @@ def load_repository_locations(repository_locations_spec):
 
         if "project_name" not in location:
             _fail_missing_attribute("project_name", key)
-        mutable_location.pop("project_name")
 
         if "project_desc" not in location:
             _fail_missing_attribute("project_desc", key)
-        mutable_location.pop("project_desc")
 
         if "project_url" not in location:
             _fail_missing_attribute("project_url", key)
-        project_url = mutable_location.pop("project_url")
+        project_url = location["project_url"]
         if not project_url.startswith("https://") and not project_url.startswith("http://"):
             fail("project_url must start with https:// or http://: " + project_url)
 
         if "version" not in location:
             _fail_missing_attribute("version", key)
-        mutable_location.pop("version")
 
         if "use_category" not in location:
             _fail_missing_attribute("use_category", key)
-        use_category = mutable_location.pop("use_category")
+        use_category = location["use_category"]
 
         if "dataplane_ext" in use_category or "observability_ext" in use_category:
             if "extensions" not in location:
                 _fail_missing_attribute("extensions", key)
-            mutable_location.pop("extensions")
 
         if "last_updated" not in location:
             _fail_missing_attribute("last_updated", key)
-        last_updated = mutable_location.pop("last_updated")
+        last_updated = location["last_updated"]
 
         # Starlark doesn't have regexes.
         if len(last_updated) != 10 or last_updated[4] != "-" or last_updated[7] != "-":
             fail("last_updated must match YYYY-DD-MM: " + last_updated)
 
         if "cpe" in location:
-            cpe = mutable_location.pop("cpe")
+            cpe = location["cpe"]
 
             # Starlark doesn't have regexes.
             cpe_components = len(cpe.split(":"))
@@ -113,4 +132,9 @@ def load_repository_locations(repository_locations_spec):
             if category not in USE_CATEGORIES:
                 fail("Unknown use_category value '" + category + "' for dependecy " + key)
 
+        # Remove any extra annotations that we add, so that we don't confuse http_archive etc.
+        for annotation in DEPENDENCY_ANNOTATIONS:
+            if annotation in mutable_location:
+                mutable_location.pop(annotation)
+
     return locations