pki: Ensure allocated serial numbers don't start with a zero byte

If the randomly allocated serial starts with 0x80, the fix added with
e49197f15e ("pki: Don't generate negative random serial numbers in
X.509 certificates") causes the value to become zero.  So the encoded
ASN.1 integer will start with a zero byte, which is only correct if the
integer would otherwise be interpreted as negative, i.e. the next byte
starts with the most significant bit set.  If that isn't the case, the
encoding is technically invalid and might get rejected by strict parsers.

Note that e49197f15e did not actually fix a violation of RFC 5280 as
asn1_integer() assumes all passed numbers are positive and automatically
adds a zero prefix if the MSB is set.  What it did instead (or at least
attempted to) is ensure that the generated serial is a positive 64-bit
number in two's complement. The difference can be seen in the output of
`openssl x509 -text`.  While 8-byte serials with the MSB set are printed
as hex dump:

    Serial Number:
            af:e2:e1:47:0f:66:b5:a4

    (The encoding is 02:09:00:af:e2:e1:47:0f:66:b5:a4)

those without MSB set are actually printed as number:

    Serial Number: 289805014144645117 (0x405981ffa37f7fd)

    (The encoding is 02:08:04:05:98:1F:FA:37:F7:FD)

The reason is that OpenSSL only does the latter if the number fits into a
signed `long` variable, which isn't the case if a positive 64-bit number
has the MSB set (i.e. has a zero prefix) as it would be interpreted as
negative number in two's complement.  OpenSSL does print negative serial
numbers (even if it's a violation of the RFC), but only if they were
encoded as such, i.e. if there was no zero prefix:

    Serial Number: -5492225205882687294 (-0x4c384db9c7158f3e)

    (The encoding is 02:08:b3:c7:b2:46:38:ea:70:c2)

Fixes: e49197f15e ("pki: Don't generate negative random serial numbers in X.509 certificates")
Closes strongswan/strongswan#631

src/pki/commands/acert.c

@@ -43,7 +43,6 @@ static int acert()
 	chunk_t serial = chunk_empty, encoding = chunk_empty;
 	time_t not_before, not_after, lifetime = 24 * 60 * 60;
 	char *datenb = NULL, *datena = NULL, *dateform = NULL;
-	rng_t *rng;
 	char *arg;
 	bool pss = lib->settings->get_bool(lib->settings, "%s.rsa_pss", FALSE,
 									   lib->ns);
@@ -186,22 +185,10 @@ static int acert()
 	{
 		serial = chunk_from_hex(chunk_create(hex, strlen(hex)), NULL);
 	}
-	else
+	else if (!allocate_serial(8, &serial))
 	{
-		rng = lib->crypto->create_rng(lib->crypto, RNG_WEAK);
-		if (!rng)
-		{
-			error = "no random number generator found";
-			goto end;
-		}
-		if (!rng_allocate_bytes_not_zero(rng, 8, &serial, FALSE))
-		{
-			error = "failed to generate serial number";
-			rng->destroy(rng);
-			goto end;
-		}
-		serial.ptr[0] &= 0x7F;
-		rng->destroy(rng);
+		error = "failed to generate serial number";
+		goto end;
 	}
 
 	if (file)

src/pki/commands/issue.c

@@ -432,23 +432,10 @@ static int issue()
 	{
 		serial = chunk_from_hex(chunk_create(hex, strlen(hex)), NULL);
 	}
-	else
+	else if (!allocate_serial(8, &serial))
 	{
-		rng_t *rng = lib->crypto->create_rng(lib->crypto, RNG_WEAK);
-
-		if (!rng)
-		{
-			error = "no random number generator found";
-			goto end;
-		}
-		if (!rng_allocate_bytes_not_zero(rng, 8, &serial, FALSE))
-		{
-			error = "failed to generate serial number";
-			rng->destroy(rng);
-			goto end;
-		}
-		serial.ptr[0] &= 0x7F;
-		rng->destroy(rng);
+		error = "failed to generate serial number";
+		goto end;
 	}
 
 	if (pkcs10)

src/pki/commands/self.c

@@ -368,23 +368,10 @@ static int self()
 	{
 		serial = chunk_from_hex(chunk_create(hex, strlen(hex)), NULL);
 	}
-	else
+	else if (!allocate_serial(8, &serial))
 	{
-		rng_t *rng = lib->crypto->create_rng(lib->crypto, RNG_WEAK);
-
-		if (!rng)
-		{
-			error = "no random number generator found";
-			goto end;
-		}
-		if (!rng_allocate_bytes_not_zero(rng, 8, &serial, FALSE))
-		{
-			error = "failed to generate serial number";
-			rng->destroy(rng);
-			goto end;
-		}
-		serial.ptr[0] &= 0x7F;
-		rng->destroy(rng);
+		error = "failed to generate serial number";
+		goto end;
 	}
 	scheme = get_signature_scheme(private, digest, pss);
 	if (!scheme)

src/pki/pki.c

@@ -340,6 +340,41 @@ traffic_selector_t* parse_ts(char *str)
 	return traffic_selector_create_from_cidr(str, 0, 0, 65535);
 }
 
+/*
+ * Described in header
+ */
+bool allocate_serial(size_t len, chunk_t *serial)
+{
+	rng_t *rng;
+
+	if (!len)
+	{
+		len = 1;
+	}
+	rng = lib->crypto->create_rng(lib->crypto, RNG_WEAK);
+	if (!rng)
+	{
+		fprintf(stderr, "no random number generator found\n");
+		return FALSE;
+	}
+	if (!rng_allocate_bytes_not_zero(rng, len, serial, FALSE))
+	{
+		rng->destroy(rng);
+		return FALSE;
+	}
+	/* ensure the serial is positive but doesn't start with 0 */
+	while (!(serial->ptr[0] &= 0x7F))
+	{
+		if (!rng->get_bytes(rng, 1, serial->ptr))
+		{
+			rng->destroy(rng);
+			return FALSE;
+		}
+	}
+	rng->destroy(rng);
+	return TRUE;
+}
+
 /**
  * Callback credential set pki uses
  */

src/pki/pki.h

@@ -79,4 +79,13 @@ signature_params_t *get_signature_scheme(private_key_t *private,
  */
 traffic_selector_t* parse_ts(char *str);
 
+/**
+ * Generate a random serial number for certificates.
+ *
+ * @param len       length of the generated serial number
+ * @param serial    allocated serial number
+ * @return          TRUE if allocation was successful
+ */
+bool allocate_serial(size_t len, chunk_t *serial);
+
 #endif /** PKI_H_ @}*/