openssl: Handle BoringSSL-style ASN1_INTEGERs in cert serials

OpenSSL stores the serial number for an X509 certificate as an
`ASN1_INTEGER` type. Within BoringSSL (and AWS-LC), the library
represents the value of zero as an empty array [1] which is different
from OpenSSL which represents it as the 1-byte array [0x00]. Though the
value of zero for the certificate serial number is illegal under
X.509 [2], we need to handle/encode it consistently within strongSwan.
From 18082ce2b0 ("certificates: Retrieve serial numbers in canonical
form"), we infer that the canonical representation of the zero serial
is [0x00]. To do this, we introduce `openssl_asn1_int2chunk` to
complement the existing string version that allows us to handle the
special case for zero instead of always returning a reference to the
library-dependent encodings.

References strongswan/strongswan#1907
Closes strongswan/strongswan#2138

[1] https://github.com/google/boringssl/commit/bdc35b63617f78037768f4897d8835696f02181a
[2] https://datatracker.ietf.org/doc/html/rfc5280#section-4.1.2.2

src/libstrongswan/plugins/openssl/openssl_crl.c

@@ -177,7 +177,7 @@ METHOD(enumerator_t, crl_enumerate, bool,
 		revoked = sk_X509_REVOKED_value(this->stack, this->i);
 		if (serial)
 		{
-			*serial = openssl_asn1_str2chunk(
+			*serial = openssl_asn1_int2chunk(
 									X509_REVOKED_get0_serialNumber(revoked));
 		}
 		if (date)

src/libstrongswan/plugins/openssl/openssl_util.c

@@ -35,6 +35,13 @@
 #define ASN1_STRING_get0_data(a) ASN1_STRING_data((ASN1_STRING*)a)
 #endif
 
+#if defined(OPENSSL_IS_BORINGSSL) || defined(OPENSSL_IS_AWSLC)
+/**
+ * Chunk for an ASN.1 Integer with a value of 0.
+ */
+static const chunk_t asn1_zero_int = chunk_from_chars(0x00);
+#endif
+
 /*
  * Described in header
  */
@@ -230,6 +237,22 @@ chunk_t openssl_asn1_str2chunk(const ASN1_STRING *asn1)
 	return chunk_empty;
 }
 
+/**
+ * Described in header.
+ */
+chunk_t openssl_asn1_int2chunk(const ASN1_INTEGER *asn1)
+{
+#if defined(OPENSSL_IS_BORINGSSL) || defined(OPENSSL_IS_AWSLC)
+	/* BoringSSL and AWS-LC use an empty chunk for 0 so return a properly
+	 * encoded chunk here. */
+	if (asn1 && ASN1_STRING_length(asn1) == 0)
+	{
+		return asn1_zero_int;
+	}
+#endif
+	return openssl_asn1_str2chunk(asn1);
+}
+
 /**
  * Convert a X509 name to a ID_DER_ASN1_DN identification_t
  */

src/libstrongswan/plugins/openssl/openssl_util.h

@@ -125,6 +125,14 @@ chunk_t openssl_asn1_obj2chunk(const ASN1_OBJECT *asn1);
  */
 chunk_t openssl_asn1_str2chunk(const ASN1_STRING *asn1);
 
+/**
+ * Convert an OpenSSL ASN1_INTEGER to a chunk.
+ *
+ * @param asn1		asn1 integer to convert
+ * @return			chunk, pointing into asn1 integer
+ */
+chunk_t openssl_asn1_int2chunk(const ASN1_INTEGER *asn1);
+
 /**
  * Convert an openssl X509_NAME to a identification_t of type ID_DER_ASN1_DN.
  *

src/libstrongswan/plugins/openssl/openssl_x509.c

@@ -265,7 +265,7 @@ METHOD(x509_t, get_flags, x509_flag_t,
 METHOD(x509_t, get_serial, chunk_t,
 	private_openssl_x509_t *this)
 {
-	return openssl_asn1_str2chunk(X509_get_serialNumber(this->x509));
+	return openssl_asn1_int2chunk(X509_get_serialNumber(this->x509));
 }
 
 METHOD(x509_t, get_subjectKeyIdentifier, chunk_t,