Always use an ASN1_STRING_TABLE global mask of UTF8String.

ASN1_STRING_set_by_NID is very complex and depends on a "global mask"
for most NIDs. (Some NIDs use a single type and use STABLE_NO_MASK to
disable the global mask.) Historically, it defaulted to allowing all
types, but it switched to UTF8String in OpenSSL 1.0.2.

Updating the global mask is not thread-safe, and it's 2021. Let's just
always use UTF-8. The only callers I found set it to UTF-8 anyway (with
the exception of some test script we don't use, and some code that
wasn't compiled). No-op writes in the C/C++ memory model are still race
conditions, so this CL fixes some bugs in those callers.

Update-Note: The global mask for ASN1_STRING_set_by_NID is now always
UTF-8. Callers that want another type should reconsider and, if UTF-8 is
still unsuitable, just pass the actual desired type into
ASN1_mbstring_copy, X509_NAME_ENTRY_set_data, etc

Change-Id: I679e99c57da9a48c805460abcb3af5b2f938c93f
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/48766
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: Adam Langley <agl@google.com>

crypto/asn1/a_strnid.c

@@ -69,53 +69,17 @@ DEFINE_STACK_OF(ASN1_STRING_TABLE)
 static STACK_OF(ASN1_STRING_TABLE) *stable = NULL;
 static void st_free(ASN1_STRING_TABLE *tbl);
 
-/*
- * This is the global mask for the mbstring functions: this is use to mask
- * out certain types (such as BMPString and UTF8String) because certain
- * software (e.g. Netscape) has problems with them.
- */
-
-static unsigned long global_mask = B_ASN1_UTF8STRING;
-
 void ASN1_STRING_set_default_mask(unsigned long mask)
 {
-    global_mask = mask;
 }
 
 unsigned long ASN1_STRING_get_default_mask(void)
 {
-    return global_mask;
+    return B_ASN1_UTF8STRING;
 }
 
-/*
- * This function sets the default to various "flavours" of configuration.
- * based on an ASCII string. Currently this is: MASK:XXXX : a numerical mask
- * value. nobmp : Don't use BMPStrings (just Printable, T61). pkix : PKIX
- * recommendation in RFC2459. utf8only : only use UTF8Strings (RFC2459
- * recommendation for 2004). default: the default value, Printable, T61, BMP.
- */
-
 int ASN1_STRING_set_default_mask_asc(const char *p)
 {
-    unsigned long mask;
-    char *end;
-    if (!strncmp(p, "MASK:", 5)) {
-        if (!p[5])
-            return 0;
-        mask = strtoul(p + 5, &end, 0);
-        if (*end)
-            return 0;
-    } else if (!strcmp(p, "nombstr"))
-        mask = ~((unsigned long)(B_ASN1_BMPSTRING | B_ASN1_UTF8STRING));
-    else if (!strcmp(p, "pkix"))
-        mask = ~((unsigned long)B_ASN1_T61STRING);
-    else if (!strcmp(p, "utf8only"))
-        mask = B_ASN1_UTF8STRING;
-    else if (!strcmp(p, "default"))
-        mask = 0xFFFFFFFFL;
-    else
-        return 0;
-    ASN1_STRING_set_default_mask(mask);
     return 1;
 }
 
@@ -139,13 +103,12 @@ ASN1_STRING *ASN1_STRING_set_by_NID(ASN1_STRING **out,
     if (tbl) {
         mask = tbl->mask;
         if (!(tbl->flags & STABLE_NO_MASK))
-            mask &= global_mask;
+            mask &= B_ASN1_UTF8STRING;
         ret = ASN1_mbstring_ncopy(out, in, inlen, inform, mask,
                                   tbl->minsize, tbl->maxsize);
-    } else
-        ret =
-            ASN1_mbstring_copy(out, in, inlen, inform,
-                               DIRSTRING_TYPE & global_mask);
+    } else {
+        ret = ASN1_mbstring_copy(out, in, inlen, inform, B_ASN1_UTF8STRING);
+    }
     if (ret <= 0)
         return NULL;
     return *out;

include/openssl/asn1.h

@@ -1155,8 +1155,13 @@ OPENSSL_EXPORT void *ASN1_item_unpack(const ASN1_STRING *oct,
 OPENSSL_EXPORT ASN1_STRING *ASN1_item_pack(void *obj, const ASN1_ITEM *it,
                                            ASN1_OCTET_STRING **oct);
 
+// ASN1_STRING_set_default_mask does nothing.
 OPENSSL_EXPORT void ASN1_STRING_set_default_mask(unsigned long mask);
+
+// ASN1_STRING_set_default_mask_asc returns one.
 OPENSSL_EXPORT int ASN1_STRING_set_default_mask_asc(const char *p);
+
+// ASN1_STRING_get_default_mask returns |B_ASN1_UTF8STRING|.
 OPENSSL_EXPORT unsigned long ASN1_STRING_get_default_mask(void);
 
 OPENSSL_EXPORT ASN1_STRING *ASN1_STRING_set_by_NID(ASN1_STRING **out,