Remove ASN1_TFLG_SET_ORDER.

ASN1_TFLG_SET_ORDER was used in OpenSSL's CMS and PKCS#7
implementations, which we've removed. Fields that use it not only get
the DER SET sorting but, when serialized, go back and mutate the
original object to match.

This is unused, so remove it. This removes one of the sources of
non-const behavior in i2d functions.

Bug: 407
Change-Id: I6b2bf8d11c30a41b53d14ad475c26a1a30dfd31f
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/48786
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: Adam Langley <agl@google.com>

crypto/asn1/tasn_enc.c

@@ -295,11 +295,12 @@ static int asn1_template_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
 
         if (flags & ASN1_TFLG_SET_OF) {
             isset = 1;
-            /* 2 means we reorder */
-            if (flags & ASN1_TFLG_SEQUENCE_OF)
-                isset = 2;
-        } else
+            /* Historically, types with both bits set were mutated when
+             * serialized to apply the sort. We no longer support this. */
+            assert((flags & ASN1_TFLG_SEQUENCE_OF) == 0);
+        } else {
             isset = 0;
+        }
 
         /*
          * Work out inner tag value: if EXPLICIT or no tagging use underlying
@@ -378,7 +379,6 @@ static int asn1_template_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
 typedef struct {
     unsigned char *data;
     int length;
-    ASN1_VALUE *field;
 } DER_ENC;
 
 static int der_cmp(const void *a, const void *b)
@@ -433,7 +433,6 @@ static int asn1_set_seq_out(STACK_OF(ASN1_VALUE) *sk, unsigned char **out,
         skitem = sk_ASN1_VALUE_value(sk, i);
         tder->data = p;
         tder->length = ASN1_item_ex_i2d(&skitem, &p, item, -1, iclass);
-        tder->field = skitem;
     }
 
     /* Now sort them */
@@ -445,11 +444,6 @@ static int asn1_set_seq_out(STACK_OF(ASN1_VALUE) *sk, unsigned char **out,
         p += tder->length;
     }
     *out = p;
-    /* If do_sort is 2 then reorder the STACK */
-    if (do_sort == 2) {
-        for (i = 0, tder = derlst; i < sk_ASN1_VALUE_num(sk); i++, tder++)
-            (void)sk_ASN1_VALUE_set(sk, i, tder->field);
-    }
     OPENSSL_free(derlst);
     OPENSSL_free(tmpdat);
     return 1;

include/openssl/asn1t.h

@@ -389,13 +389,6 @@ struct ASN1_ADB_TABLE_st {
 /* Field is a SEQUENCE OF */
 #define ASN1_TFLG_SEQUENCE_OF	(0x2 << 1)
 
-/* Special case: this refers to a SET OF that
- * will be sorted into DER order when encoded *and*
- * the corresponding STACK will be modified to match
- * the new order.
- */
-#define ASN1_TFLG_SET_ORDER	(0x3 << 1)
-
 /* Mask for SET OF or SEQUENCE OF */
 #define ASN1_TFLG_SK_MASK	(0x3 << 1)