Remove IMPORT and related importing tools

Change-Id: I553dc69083878bb33d0a62f512622d77be9cdee9
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/64068
Reviewed-by: David Benjamin <davidben@google.com>
Commit-Queue: Bob Beck <bbe@google.com>
chromium-stable
Bob Beck 1 year ago committed by Boringssl LUCI CQ
parent 6beabf3f44
commit 71fb2a4e07
  1. 30
      pki/IMPORT
  2. 349
      pki/import_spec.json
  3. 187
      pki/import_tool.go

@ -1,30 +0,0 @@
#!/bin/sh
# Set this to be the location of a chromium checkout, and
# apply the patches in ./patches with "git am" first
# before running this script.
CHROMIUM_SRC=~/chromium/src
mkdir -p ./testdata
cp $CHROMIUM_SRC/net/test/test_certificate_data.h ./testdata
tar -C $CHROMIUM_SRC/net/third_party -cf - nist-pkits | tar -C ./testdata -xf -
tar -C $CHROMIUM_SRC/net/data -cf - cert_issuer_source_static_unittest \
ssl/certificates \
certificate_policies_unittest \
name_constraints_unittest \
ocsp_unittest \
crl_unittest \
parse_certificate_unittest \
path_builder_unittest \
verify_certificate_chain_unittest \
verify_name_match_unittest \
verify_signed_data_unittest | tar -C ./testdata -xf -
go run ./import_tool.go -spec import_spec.json --source-base $CHROMIUM_SRC -dest-base .
sed -i "s/#include \"/#include \"..\/pki\//g" *fuzzer.cc
mv *fuzzer.cc ../fuzz

@ -1,349 +0,0 @@
{
"replacements": [
{"match": "^#include \"base/supports_user_data.h\"",
"replace": ""},
{"match": ": public base::SupportsUserData",
"replace": ""},
{"match": "~Result\\(\\) override;",
"replace": "~Result();"},
{"match": "base::SupportsUserData",
"replace": "void"},
{"match": "^#include \"net/dns/dns_util.h\"",
"replace": ""},
{"match": "^#include \"base/gtest_prod_util.h\"",
"replace": ""},
{"match": "^#include \"base/pickle.h\"",
"replace": ""},
{"match": "^#include \"base/check.h\"",
"replace": "#include \"fillins/check.h\""},
{"match": "^#include \"base/notreached.h\"",
"replace": ""},
{"match": "^#include \"base/check_op.h\"",
"replace": "#include \"fillins/check.h\""},
{"match": "^#include \"net/base/hash_value.h\"",
"replace": "#include \"fillins/hash_value.h\""},
{"match": "^#include \"net/cert/x509_util.h\"",
"replace": "#include \"fillins/x509_util.h\""},
{"match": "^#include \"url/gurl.h\"",
"replace": "#include \"webutil/url/url.h\""},
{"match": "^#include \"build/build_config.h\"",
"replace": ""},
{"match": "^#include \"base/numerics/clamped_math.h\"",
"replace": "#include \"fillins/clamped_math.h\""},
{"match": "^#include \"base/numerics/safe_conversions.h\"",
"replace": ""},
{"match": "^#include \"net/base/net_export.h\"",
"replace": ""},
{"match": "^#include \"base/strings/string_util.h\"",
"replace": "#include \"fillins/fillins_string_util.h\""},
{"match": "^#include \"base/base_paths.h\"",
"replace": "#include \"fillins/path_service.h\"",
"using": ["bssl::fillins::PathService"]},
{"match": "base::PathService",
"replace": "bssl::fillins::PathService"},
{"match": "base::ClampAdd",
"replace": "bssl::fillins::ClampAdd"},
{"match": "base::ClampMul",
"replace": "bssl::fillins::ClampAdd"},
{"match": "^#include \"base/files/file_util.h\"",
"replace": "#include \"fillins/file_util.h\""},
{"match": "^#include \"base/path_service.h\"",
"replace": ""},
{"match": "^#include \"crypto/openssl_util.h\"",
"replace": "#include \"fillins/openssl_util.h\""},
{"match": "\"net/data/",
"replace": "\"testdata/"},
{"match": "\"net/third_party/nist-pkits",
"replace": "\"testdata/nist-pkits"},
{"match": "^#include \"net/base/net_errors.h\"",
"replace": "#include \"fillins/log.h\"\n#include \"fillins/net_errors.h\"\n"},
{"match": "^#include \"net/test/test_certificate_data.h\"",
"replace": "#include \"testdata/test_certificate_data.h\""},
{"match": "^#include \"net/third_party/nist-pkits/pkits_testcases-inl.h\"",
"replace": "#include \"testdata/nist-pkits/pkits_testcases-inl.h\""},
{"match": "^#include \"base/sys_byteorder.h\"",
"replace": "#include \"fillins/inet.h\""},
{"match": "^#include \"base/third_party/icu/icu_utf.h\"",
"replace": "#include \"fillins/utf_string_conversions.h\""},
{"match": "^#include \"base/strings/utf_string_conversions.h\"",
"replace": "#include \"fillins/utf_string_conversions.h\""},
{"match": "^#include \"base/strings/utf_string_conversion_utils.h\"",
"replace": ""},
{"match": "^#include \"base/memory/ref_counted.h\"",
"replace": "#include <memory>"},
{"match": "^#include \"base/base64.h\"",
"replace": "#include \"fillins/fillins_base64.h\""},
{"match": "^#include \"base/strings/stringprintf.h\"",
"replace": ""},
{"match": "^#include \"third_party/boringssl/src/include/openssl/(.*).h\"",
"replace": "#include <openssl/$1.h>"},
{"match": "^#include \"net/cert/pki/",
"replace": "#include \""},
{"match": "^#include \"net/cert/",
"replace": "#include \""},
{"match": "^#include \"net/der/",
"replace": "#include \""},
{"match": "^#include \"net/",
"replace": "#include \""},
{"match": "^#include \"net_buildflags.h\"",
"replace": ""},
{"match": "^#include \"base/time/time.h\"",
"replace": ""},
{"match": "^#include \"base/strings/string_piece.h\"",
"replace": "#include <string_view>\n"},
{"match": "^#include \"testing/gtest/include/gtest/gtest.h\"",
"replace": "#include <gtest/gtest.h>"},
{"match": "^#include \"testing/gmock/include/gmock/gmock.h\"",
"replace": "#include <gmock/gmock.h>"},
{"match": "^#include \"base/containers/span.h\"",
"replace": "#include <openssl/span.h>"},
{"match": "^#include \"third_party/abseil-cpp/absl/types/optional.h\"",
"replace": "#include <optional>"},
{"match": "^#include \"base/containers/contains.h\"",
"replace": ""},
{"match": "LOG(ERROR)",
"replace": "std::cerr"},
{"match": "GURL",
"replace": "URL",
"include": "webutil/url/url.h"},
{"match": "absl::nullopt",
"replace": "std::nullopt" },
{"match": "absl::optional",
"replace": "std::optional" },
{"match": "absl::make_optional",
"replace": "std::make_optional" },
{"match": "base::span",
"replace": "bssl::Span" },
{"match": "base::make_span",
"replace": "bssl::MakeSpan" },
{"match": "base::as_bytes",
"replace": "fillins::as_bytes",
"include": "fillins/bits.h"},
{"match": "^namespace net {",
"replace": "namespace bssl {"},
{"match": "namespace net::([^ ]+) {",
"replace": "namespace bssl::$1 {"},
{"match": "NET_EXPORT_PRIVATE ",
"replace": "OPENSSL_EXPORT ",
"include": "fillins/openssl_util.h"},
{"match": "NET_EXPORT ",
"replace": "OPENSSL_EXPORT ",
"include": "fillins/openssl_util.h"},
{"match": "NOTREACHED\\(\\)",
"replace": "abort(); //NOTREACHED" },
{"match": "NOTREACHED_NORETURN\\(\\)",
"replace": "abort(); //NOTREACHED_NORETURN" },
{"match": "FRIEND_TEST_ALL_PREFIXES\\(.+;",
"replace": ""},
{"match": " NET_DER",
"replace": " BSSL_DER"},
{"match": " NET_CERT_PKI",
"replace": " BSSL_PKI"},
{"match": " NET_CERT",
"replace": " BSSL_PKI"},
{"match": "^using base::StringPiece;",
"replace": ""},
{"match": "base::StringPiece",
"replace": "std::string_view"},
{"match": "base::StartsWith\\(",
"replace": "bssl::string_util::StartsWith(",
"include": "string_util.h"},
{"match": "base::StringPrintf",
"replace": "absl::StrFormat",
"include": "third_party/absl/strings/str_format.h"},
{"match": "base::Base64Encode",
"replace": "fillins::Base64Encode"},
{"match": "base::Base64Decode",
"replace": "fillins::Base64Decode"},
{"match": "base::ReadFileToString",
"replace": "fillins::ReadFileToString"},
{"match": "base::CollapseWhitespaceASCII",
"replace": "fillins::CollapseWhitespaceASCII"},
{"match": "base::FilePath",
"replace": "fillins::FilePath"},
{"match": "base::DIR_SRC_TEST_DATA_ROOT",
"replace": "fillins::BSSL_TEST_DATA_ROOT"},
{"match": "base::NetToHost16\\(",
"replace": "ntohs("},
{"match": "base::NetToHost32\\(",
"replace": "ntohl("},
{ "match": "base_icu::UChar32",
"replace": "uint32_t"},
{"match": "base::WriteUnicodeCharacter\\(",
"replace": "fillins::WriteUnicodeCharacter("},
{"match": "base::IsAsciiAlpha\\(",
"replace": "fillins::IsAsciiAlpha("},
{"match": "scoped_refptr<",
"replace": "std::shared_ptr<"},
{"match": ": public base::RefCountedThreadSafe<.+>",
"replace": ""},
{"match": "friend class base::RefCountedThreadSafe<.+>;",
"replace": ""},
{"match": "\\bnet::",
"replace": "bssl::"},
{"match": "base::Time::Exploded",
"replace": "fillins::Exploded",
"include": "fillins/time.h"},
{"match": "([a-zA-Z_0-9]+)\\.UTCExplode\\(&([^)]*)\\)",
"replace": "fillins::UTCExplode($1, &$2)"},
{"match": "net::ReadTestFileToString\\(",
"replace": "ReadTestFileToString("},
{"match": "base::Seconds\\(",
"replace": "absl::Seconds("},
{"match": "base::Time::UnixEpoch\\(",
"replace": "absl::UnixEpoch("},
{"match": "base::Time::FromUTCExploded\\(",
"replace": "fillins::FromUTCExploded(",
"include": "fillins/time.h"},
{"match": "base::Time::Now\\(\\)",
"replace": "absl::Now()"},
{"match": "base::Time::Min\\(\\)",
"replace": "absl::InfinitePast()"},
{"match": "base::Time::Max\\(\\)",
"replace": "absl::InfiniteFuture()"},
{"match": "base::Time",
"replace": "absl::Time",
"include": "fillins/time.h"},
{"match": "constexpr absl::Time",
"replace": "const absl::Time"},
{"match": "^ // Map from OID to ParsedExtension.$",
"replace": "~ParsedCertificate();\n$0"},
{"match": "^ ~ParsedCertificate\\(\\);$",
"replace": " "},
{"match": "crypto::OpenSSLErrStackTracer",
"replace": "fillins::OpenSSLErrStackTracer"},
{"match": "\\(FROM_HERE\\)",
"replace": ""},
{"match": "([^a-zA-Z])StringPiece([^a-zA-Z])",
"replace": "${1}std::string_view$2"},
{"match": "crypto::kSHA256Length",
"replace": "SHA256_DIGEST_LENGTH"},
{"match": "raw_ptr<([^>]*)>",
"replace": "$1 *"}
],
"files": [
"net/cert/asn1_util.h",
"net/cert/asn1_util.cc",
"net/cert/cert_status_flags.h",
"net/cert/cert_status_flags_list.h",
"net/cert/cert_verify_proc_blocklist.inc",
"net/cert/pki/cert_error_id.cc",
"net/cert/pki/cert_error_id.h",
"net/cert/pki/cert_error_params.cc",
"net/cert/pki/cert_error_params.h",
"net/cert/pki/cert_errors.cc",
"net/cert/pki/cert_errors.h",
"net/cert/pki/certificate_policies.cc",
"net/cert/pki/certificate_policies.h",
"net/cert/pki/certificate_policies_unittest.cc",
"net/cert/pki/cert_issuer_source.h",
"net/cert/pki/cert_issuer_source_static.cc",
"net/cert/pki/cert_issuer_source_static.h",
"net/cert/pki/cert_issuer_source_static_unittest.cc",
"net/cert/pki/cert_issuer_source_sync_unittest.h",
"net/cert/pki/common_cert_errors.cc",
"net/cert/pki/common_cert_errors.h",
"net/cert/pki/crl.h",
"net/cert/pki/crl.cc",
"net/cert/pki/crl_unittest.cc",
"net/cert/pki/crl_parse_crl_certificatelist_fuzzer.cc",
"net/cert/pki/crl_parse_crl_tbscertlist_fuzzer.cc",
"net/cert/pki/crl_parse_issuing_distribution_point_fuzzer.cc",
"net/cert/pki/crl_getcrlstatusforcert_fuzzer.cc",
"net/cert/pki/extended_key_usage.cc",
"net/cert/pki/extended_key_usage.h",
"net/cert/pki/extended_key_usage_unittest.cc",
"net/cert/pki/general_names.h",
"net/cert/pki/general_names.cc",
"net/cert/pki/general_names_unittest.cc",
"net/cert/pki/ip_util.h",
"net/cert/pki/ip_util.cc",
"net/cert/pki/ip_util_unittest.cc",
"net/cert/pki/mock_signature_verify_cache.h",
"net/cert/pki/mock_signature_verify_cache.cc",
"net/cert/pki/name_constraints.cc",
"net/cert/pki/name_constraints.h",
"net/cert/pki/name_constraints_unittest.cc",
"net/cert/pki/nist_pkits_unittest.cc",
"net/cert/pki/nist_pkits_unittest.h",
"net/cert/pki/ocsp.cc",
"net/cert/pki/ocsp.h",
"net/cert/pki/ocsp_parse_ocsp_cert_id_fuzzer.cc",
"net/cert/pki/ocsp_parse_ocsp_response_data_fuzzer.cc",
"net/cert/pki/ocsp_parse_ocsp_response_fuzzer.cc",
"net/cert/pki/ocsp_parse_ocsp_single_response_fuzzer.cc",
"net/cert/pki/ocsp_unittest.cc",
"net/cert/pki/parse_authority_key_identifier_fuzzer.cc",
"net/cert/pki/parse_certificate.cc",
"net/cert/pki/parse_certificate.h",
"net/cert/pki/parse_certificate_unittest.cc",
"net/cert/pki/parsed_certificate.cc",
"net/cert/pki/parsed_certificate.h",
"net/cert/pki/parse_certificate_fuzzer.cc",
"net/cert/pki/parsed_certificate_unittest.cc",
"net/cert/pki/parse_crldp_fuzzer.cc",
"net/cert/pki/parse_name.cc",
"net/cert/pki/parse_name.h",
"net/cert/pki/parse_name_unittest.cc",
"net/cert/pki/path_builder.cc",
"net/cert/pki/path_builder.h",
"net/cert/pki/path_builder_pkits_unittest.cc",
"net/cert/pki/path_builder_unittest.cc",
"net/cert/pki/path_builder_verify_certificate_chain_unittest.cc",
"net/cert/pki/revocation_util.h",
"net/cert/pki/revocation_util.cc",
"net/cert/pki/signature_algorithm.cc",
"net/cert/pki/signature_algorithm.h",
"net/cert/pki/signature_algorithm_unittest.cc",
"net/cert/pki/simple_path_builder_delegate.cc",
"net/cert/pki/simple_path_builder_delegate.h",
"net/cert/pki/simple_path_builder_delegate_unittest.cc",
"net/cert/pki/string_util.cc",
"net/cert/pki/string_util_unittest.cc",
"net/cert/pki/string_util.h",
"net/cert/pki/signature_verify_cache.h",
"net/cert/pki/test_helpers.cc",
"net/cert/pki/test_helpers.h",
"net/cert/pki/trust_store.cc",
"net/cert/pki/trust_store_collection.cc",
"net/cert/pki/trust_store_collection.h",
"net/cert/pki/trust_store_collection_unittest.cc",
"net/cert/pki/trust_store.h",
"net/cert/pki/trust_store_in_memory.cc",
"net/cert/pki/trust_store_in_memory.h",
"net/cert/pki/verify_certificate_chain.cc",
"net/cert/pki/verify_certificate_chain.h",
"net/cert/pki/verify_certificate_chain_pkits_unittest.cc",
"net/cert/pki/verify_certificate_chain_typed_unittest.h",
"net/cert/pki/verify_certificate_chain_unittest.cc",
"net/cert/pki/verify_name_match.cc",
"net/cert/pki/verify_name_match.h",
"net/cert/pki/verify_name_match_unittest.cc",
"net/cert/pki/verify_name_match_fuzzer.cc",
"net/cert/pki/verify_name_match_normalizename_fuzzer.cc",
"net/cert/pki/verify_name_match_verifynameinsubtree_fuzzer.cc",
"net/cert/pki/verify_signed_data.cc",
"net/cert/pki/verify_signed_data.h",
"net/cert/pki/verify_signed_data_unittest.cc",
"net/cert/ocsp_revocation_status.h",
"net/cert/ocsp_verify_result.h",
"net/cert/ocsp_verify_result.cc",
"net/cert/pem.cc",
"net/cert/pem.h",
"net/cert/pem_unittest.cc",
"net/der/encode_values.cc",
"net/der/encode_values.h",
"net/der/encode_values_unittest.cc",
"net/der/input.cc",
"net/der/input.h",
"net/der/input_unittest.cc",
"net/der/parser.cc",
"net/der/parser.h",
"net/der/parser_unittest.cc",
"net/der/parse_values.cc",
"net/der/parse_values.h",
"net/der/parse_values_unittest.cc",
"net/der/tag.cc",
"net/der/tag.h"
]
}

@ -1,187 +0,0 @@
// import_tool is a quick tool for importing Chromium's certificate verifier
// code into google3. In time it might be replaced by Copybara, but this is a
// lighter-weight solution while we're quickly iterating and only going in one
// direction.
//
// Usage: ./import_tool -spec import_spec.json\
// -source-base ~/src/chromium/src/net\
// -dest-base .
package main
import (
"bufio"
"encoding/json"
"errors"
"flag"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"sync/atomic"
)
type specification struct {
Replacements []replacement `json:"replacements"`
Files []string `json:"files"`
}
type replacement struct {
Match string `json:"match"`
matchRE *regexp.Regexp `json:"-"`
Replace string `json:"replace"`
Include string `json:"include"`
Using []string `json:"using"`
used uint32
}
var (
specFile *string = flag.String("spec", "", "Location of spec JSON")
sourceBase *string = flag.String("source-base", "", "Path of the source files")
destBase *string = flag.String("dest-base", "", "Path of the destination files")
)
func transformFile(spec *specification, filename string) error {
const newLine = "\n"
sourcePath := filepath.Join(*sourceBase, filename)
destPath := filename
destPath = strings.TrimPrefix(destPath, "net/")
destPath = strings.TrimPrefix(destPath, "cert/")
destPath = strings.TrimPrefix(destPath, "der/")
destPath = strings.TrimPrefix(destPath, "pki/")
destPath = filepath.Join(*destBase, destPath)
destDir := filepath.Dir(destPath)
if err := os.MkdirAll(destDir, 0755); err != nil {
return err
}
source, err := os.Open(sourcePath)
if err != nil {
return err
}
defer source.Close()
dest, err := os.OpenFile(destPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer dest.Close()
var using []string
var includeInsertionPoint int
includes := make(map[string]struct{})
scanner := bufio.NewScanner(source)
out := ""
for scanner.Scan() {
line := scanner.Text()
if includeInsertionPoint == 0 && len(line) > 0 &&
!strings.HasPrefix(line, "// ") &&
!strings.HasPrefix(line, "#if") &&
!strings.HasPrefix(line, "#define ") {
includeInsertionPoint = len(out)
}
for i, repl := range spec.Replacements {
if !repl.matchRE.MatchString(line) {
continue
}
line = repl.matchRE.ReplaceAllString(line, repl.Replace)
atomic.StoreUint32(&spec.Replacements[i].used, 1)
using = append(using, repl.Using...)
if repl.Include != "" {
includes[repl.Include] = struct{}{}
}
}
for _, u := range using {
line = strings.Replace(
line, "namespace chromium_certificate_verifier {",
"namespace chromium_certificate_verifier {\nusing "+u+";", 1)
}
out += line
out += newLine
}
if len(includes) > 0 {
if includeInsertionPoint == 0 {
panic("failed to find include insertion point for " + filename)
}
var s string
for include := range includes {
s = s + "#include \"" + include + "\"\n"
}
out = out[:includeInsertionPoint] + s + out[includeInsertionPoint:]
}
dest.WriteString(out)
fmt.Printf("%s\n", filename)
return nil
}
func do() error {
flag.Parse()
specBytes, err := ioutil.ReadFile(*specFile)
if err != nil {
return err
}
var spec specification
if err := json.Unmarshal(specBytes, &spec); err != nil {
if jsonError, ok := err.(*json.SyntaxError); ok {
return fmt.Errorf("JSON parse error at offset %v: %v", jsonError.Offset, err.Error())
}
return errors.New("JSON parse error: " + err.Error())
}
for i, repl := range spec.Replacements {
var err error
spec.Replacements[i].matchRE, err = regexp.Compile(repl.Match)
if err != nil {
return fmt.Errorf("Failed to parse %q: %s", repl.Match, err)
}
}
errors := make(chan error, len(spec.Files))
var wg sync.WaitGroup
for _, filename := range spec.Files {
wg.Add(1)
go func(filename string) {
if err := transformFile(&spec, filename); err != nil {
errors <- err
}
wg.Done()
}(filename)
}
wg.Wait()
select {
case err := <-errors:
return err
default:
break
}
for _, repl := range spec.Replacements {
if repl.used == 0 {
fmt.Fprintf(os.Stderr, "replacement for \"%s\" not used\n", repl.Match)
}
}
return nil
}
func main() {
if err := do(); err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(1)
}
}
Loading…
Cancel
Save