acvp: add tests

This change causes the run_tests target, in FIPS builds, to run ACVP
for each supported algorithm. For most of them the output can be compared
against a known result. For some of them the output is too variable and
they are only run to ensure that they don't have local errors.

The ACVP test vectors have been trimmed significantly because they're
often huge. Firstly an included tool drops all but one test from each
group. Some vector sets have been manually trimmed to remove tests that
cause variable output.

Change-Id: Iff73851e3d47813041cc7ea6d881282750274940
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/44746
Commit-Queue: Adam Langley <agl@google.com>
Reviewed-by: David Benjamin <davidben@google.com>

CMakeLists.txt

@@ -617,6 +617,30 @@ if(UNIX AND NOT APPLE AND NOT ANDROID)
   set(HANDSHAKER_ARGS "-handshaker-path" $<TARGET_FILE:handshaker>)
 endif()
 
+if(FIPS)
+  add_custom_target(
+    acvp_tests
+    COMMAND ${GO_EXECUTABLE} build -o ${CMAKE_BINARY_DIR}/acvptool
+            boringssl.googlesource.com/boringssl/util/fipstools/acvp/acvptool
+    COMMAND ${GO_EXECUTABLE} build -o ${CMAKE_BINARY_DIR}/testmodulewrapper
+            boringssl.googlesource.com/boringssl/util/fipstools/acvp/acvptool/testmodulewrapper
+    COMMAND cd util/fipstools/acvp/acvptool/test &&
+            ${GO_EXECUTABLE} run check_expected.go
+            -tool ${CMAKE_BINARY_DIR}/acvptool
+            -module-wrappers modulewrapper:$<TARGET_FILE:modulewrapper>,testmodulewrapper:${CMAKE_BINARY_DIR}/testmodulewrapper
+            -tests tests.json
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+    DEPENDS modulewrapper
+    USES_TERMINAL)
+
+  add_custom_target(
+    fips_specific_tests_if_any
+    DEPENDS acvp_tests
+  )
+else()
+  add_custom_target(fips_specific_tests_if_any)
+endif()
+
 add_custom_target(
     run_tests
     COMMAND ${GO_EXECUTABLE} run util/all_tests.go -build-dir
@@ -625,5 +649,5 @@ add_custom_target(
             ${GO_EXECUTABLE} test -shim-path $<TARGET_FILE:bssl_shim>
               ${HANDSHAKER_ARGS} ${RUNNER_ARGS}
     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-    DEPENDS all_tests bssl_shim handshaker
+    DEPENDS all_tests bssl_shim handshaker fips_specific_tests_if_any
     USES_TERMINAL)

util/fipstools/acvp/acvptool/test/check_expected.go

@@ -0,0 +1,198 @@
+// Copyright (c) 2021, Google Inc.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+package main
+
+import (
+	"bytes"
+	"compress/bzip2"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+)
+
+var (
+	toolPath       *string = flag.String("tool", "", "Path to acvptool binary")
+	moduleWrappers *string = flag.String("module-wrappers", "", "Comma-separated list of name:path pairs for known module wrappers")
+	testsPath      *string = flag.String("tests", "", "Path to JSON file listing tests")
+	update         *bool   = flag.Bool("update", false, "If true then write updated outputs")
+)
+
+type invocation struct {
+	toolPath     string
+	wrapperPath  string
+	inPath       string
+	expectedPath string
+}
+
+func main() {
+	flag.Parse()
+
+	if len(*toolPath) == 0 {
+		log.Fatal("-tool must be given")
+	}
+
+	if len(*moduleWrappers) == 0 {
+		log.Fatal("-module-wrappers must be given")
+	}
+
+	wrappers := make(map[string]string)
+	pairs := strings.Split(*moduleWrappers, ",")
+	for _, pair := range pairs {
+		parts := strings.SplitN(pair, ":", 2)
+		if _, ok := wrappers[parts[0]]; ok {
+			log.Fatalf("wrapper %q defined twice", parts[0])
+		}
+		wrappers[parts[0]] = parts[1]
+	}
+
+	if len(*testsPath) == 0 {
+		log.Fatal("-tests must be given")
+	}
+
+	testsFile, err := os.Open(*testsPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer testsFile.Close()
+
+	decoder := json.NewDecoder(testsFile)
+	var tests []struct {
+		Wrapper string
+		In      string
+		Out     string // Optional, may be empty.
+	}
+	if err := decoder.Decode(&tests); err != nil {
+		log.Fatal(err)
+	}
+
+	work := make(chan invocation, runtime.NumCPU())
+	var numFailed uint32
+
+	var wg sync.WaitGroup
+	for i := 0; i < runtime.NumCPU(); i++ {
+		wg.Add(1)
+		go worker(&wg, work, &numFailed)
+	}
+
+	for _, test := range tests {
+		wrapper, ok := wrappers[test.Wrapper]
+		if !ok {
+			log.Fatalf("wrapper %q not specified on command line", test.Wrapper)
+		}
+		work <- invocation{
+			toolPath:     *toolPath,
+			wrapperPath:  wrapper,
+			inPath:       test.In,
+			expectedPath: test.Out,
+		}
+	}
+
+	close(work)
+	wg.Wait()
+
+	n := atomic.LoadUint32(&numFailed)
+	if n > 0 {
+		log.Printf("Failed %d tests", n)
+		os.Exit(1)
+	} else {
+		log.Printf("%d ACVP tests matched expectations", len(tests))
+	}
+}
+
+func worker(wg *sync.WaitGroup, work <-chan invocation, numFailed *uint32) {
+	defer wg.Done()
+
+	for test := range work {
+		if err := doTest(test); err != nil {
+			log.Printf("Test failed for %q: %s", test.inPath, err)
+			atomic.AddUint32(numFailed, 1)
+		}
+	}
+}
+
+func doTest(test invocation) error {
+	input, err := os.Open(test.inPath)
+	if err != nil {
+		return fmt.Errorf("Failed to open %q: %s", test.inPath, err)
+	}
+	defer input.Close()
+
+	tempFile, err := ioutil.TempFile("", "boringssl-check_expected-")
+	if err != nil {
+		return fmt.Errorf("Failed to create temp file: %s", err)
+	}
+	defer os.Remove(tempFile.Name())
+	defer tempFile.Close()
+
+	decompressor := bzip2.NewReader(input)
+	if _, err := io.Copy(tempFile, decompressor); err != nil {
+		return fmt.Errorf("Failed to decompress %q: %s", test.inPath, err)
+	}
+
+	cmd := exec.Command(test.toolPath, "-wrapper", test.wrapperPath, "-json", tempFile.Name())
+	result, err := cmd.CombinedOutput()
+	if err != nil {
+		os.Stderr.Write(result)
+		return fmt.Errorf("Failed to process %q", test.inPath)
+	}
+
+	if len(test.expectedPath) == 0 {
+		// This test has variable output and thus cannot be compared against a fixed
+		// result.
+		return nil
+	}
+
+	expected, err := os.Open(test.expectedPath)
+	if err != nil {
+		if *update {
+			writeUpdate(test.expectedPath, result)
+		}
+		return fmt.Errorf("Failed to open %q: %s", test.expectedPath, err)
+	}
+	defer expected.Close()
+
+	decompressor = bzip2.NewReader(expected)
+
+	var expectedBuf bytes.Buffer
+	if _, err := io.Copy(&expectedBuf, decompressor); err != nil {
+		return fmt.Errorf("Failed to decompress %q: %s", test.expectedPath, err)
+	}
+
+	if !bytes.Equal(expectedBuf.Bytes(), result) {
+		if *update {
+			writeUpdate(test.expectedPath, result)
+		}
+		return fmt.Errorf("Mismatch for %q", test.expectedPath)
+	}
+
+	return nil
+}
+
+func writeUpdate(path string, contents []byte) {
+	if err := ioutil.WriteFile(path, contents, 0644); err != nil {
+		log.Printf("Failed to create missing file %q: %s", path, err)
+	} else {
+		log.Printf("Wrote %q", path)
+	}
+}

util/fipstools/acvp/acvptool/test/tests.json

@@ -0,0 +1,29 @@
+[
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-AES-CBC.bz2", "Out": "expected/ACVP-AES-CBC.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-AES-CCM.bz2", "Out": "expected/ACVP-AES-CCM.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-AES-CTR.bz2", "Out": "expected/ACVP-AES-CTR.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-AES-ECB.bz2", "Out": "expected/ACVP-AES-ECB.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-AES-GCM.bz2", "Out": "expected/ACVP-AES-GCM.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-AES-KW.bz2", "Out": "expected/ACVP-AES-KW.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-AES-KWP.bz2", "Out": "expected/ACVP-AES-KWP.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-TDES-CBC.bz2", "Out": "expected/ACVP-TDES-CBC.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ACVP-TDES-ECB.bz2", "Out": "expected/ACVP-TDES-ECB.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/CMAC-AES.bz2", "Out": "expected/CMAC-AES.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ctrDRBG.bz2", "Out": "expected/ctrDRBG.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/ECDSA.bz2", "Out": "expected/ECDSA.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/HMAC-SHA-1.bz2", "Out": "expected/HMAC-SHA-1.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/HMAC-SHA2-224.bz2", "Out": "expected/HMAC-SHA2-224.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/HMAC-SHA2-256.bz2", "Out": "expected/HMAC-SHA2-256.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/HMAC-SHA2-384.bz2", "Out": "expected/HMAC-SHA2-384.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/HMAC-SHA2-512.bz2", "Out": "expected/HMAC-SHA2-512.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/KAS-ECC-SSC.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/KAS-FFC.bz2"},
+{"Wrapper": "testmodulewrapper", "In": "vectors/KDF.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/kdf-components.bz2", "Out": "expected/kdf-components.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/RSA.bz2", "Out": "expected/RSA.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/SHA-1.bz2", "Out": "expected/SHA-1.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/SHA2-224.bz2", "Out": "expected/SHA2-224.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/SHA2-256.bz2", "Out": "expected/SHA2-256.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/SHA2-384.bz2", "Out": "expected/SHA2-384.bz2"},
+{"Wrapper": "modulewrapper", "In": "vectors/SHA2-512.bz2", "Out": "expected/SHA2-512.bz2"}
+]

util/fipstools/acvp/acvptool/test/trim_vectors.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2021, Google Inc.
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+// trimvectors takes an ACVP vector set file and discards all but a single test
+// from each test group. This hope is that this achieves good coverage without
+// having to check in megabytes worth of JSON files.
+package main
+
+import (
+	"encoding/json"
+	"os"
+)
+
+func main() {
+	var vectorSets []interface{}
+	decoder := json.NewDecoder(os.Stdin)
+	if err := decoder.Decode(&vectorSets); err != nil {
+		panic(err)
+	}
+
+	// The first element is the metadata which is left unmodified.
+	for i := 1; i < len(vectorSets); i++ {
+		vectorSet := vectorSets[i].(map[string]interface{})
+		testGroups := vectorSet["testGroups"].([]interface{})
+		for _, testGroupInterface := range testGroups {
+			testGroup := testGroupInterface.(map[string]interface{})
+			tests := testGroup["tests"].([]interface{})
+
+			keepIndex := 10
+			if keepIndex >= len(tests) {
+				keepIndex = len(tests) - 1
+			}
+
+			testGroup["tests"] = []interface{}{tests[keepIndex]}
+		}
+	}
+
+	encoder := json.NewEncoder(os.Stdout)
+	encoder.SetIndent("", "  ")
+	if err := encoder.Encode(vectorSets); err != nil {
+		panic(err)
+	}
+}