signature-params: Provide option for maximum RSA/PSS salt length

However, the length now has to be resolved early, so we don't operate on
the negative constant values e.g. when generating the encoding.

src/libstrongswan/credentials/auth_cfg.c

@@ -551,6 +551,7 @@ static signature_params_t *create_rsa_pss_constraint(char *token)
 			.scheme = SIGN_RSA_EMSA_PSS,
 			.params = &pss,
 		};
+		rsa_pss_params_set_salt_len(&pss, 0);
 		params = signature_params_clone(&pss_params);
 	}
 	return params;

src/libstrongswan/credentials/keys/public_key.c

@@ -250,7 +250,7 @@ int signature_scheme_to_oid(signature_scheme_t scheme)
 #define PSS_PARAMS(bits) static rsa_pss_params_t pss_params_sha##bits = { \
 	.hash = HASH_SHA##bits, \
 	.mgf1_hash = HASH_SHA##bits, \
-	.salt_len = RSA_PSS_SALT_LEN_DEFAULT, \
+	.salt_len = HASH_SIZE_SHA##bits, \
 }
 
 PSS_PARAMS(256);

src/libstrongswan/credentials/keys/signature_params.c

@@ -18,22 +18,43 @@
 #include <asn1/oid.h>
 #include <asn1/asn1_parser.h>
 
-/**
- * Determine the salt length in case it is not configured
+/*
+ * Described in header
  */
-static ssize_t rsa_pss_salt_length(rsa_pss_params_t *pss)
+bool rsa_pss_params_set_salt_len(rsa_pss_params_t *params, size_t modbits)
 {
-	ssize_t salt_len = pss->salt_len;
+	size_t hash_len;
 
-	if (salt_len <= RSA_PSS_SALT_LEN_DEFAULT)
+	if (params->salt_len < 0)
 	{
-		salt_len = hasher_hash_size(pss->hash);
-		if (!salt_len)
+		hash_len = hasher_hash_size(params->hash);
+		if (!hash_len)
 		{
-			return -1;
+			return FALSE;
+		}
+
+		switch (params->salt_len)
+		{
+			case RSA_PSS_SALT_LEN_DEFAULT:
+				params->salt_len = hash_len;
+				break;
+			case RSA_PSS_SALT_LEN_MAX:
+				if (modbits)
+				{
+					/* emBits = modBits - 1 */
+					modbits -= 1;
+					/* emLen = ceil(emBits/8) */
+					modbits = (modbits+7) / BITS_PER_BYTE;
+					/* account for 0x01 separator in DB, 0xbc trailing byte */
+					params->salt_len = max(0, (ssize_t)(modbits - hash_len - 2));
+					break;
+				}
+				return FALSE;
+			default:
+				return FALSE;
 		}
 	}
-	return salt_len;
+	return TRUE;
 }
 
 /**
@@ -68,8 +89,7 @@ static bool compare_params(signature_params_t *a, signature_params_t *b,
 
 				return pss_a->hash == pss_b->hash &&
 					   pss_a->mgf1_hash == pss_b->mgf1_hash &&
-					   (!strict ||
-						rsa_pss_salt_length(pss_a) == rsa_pss_salt_length(pss_b));
+					   (!strict || pss_a->salt_len == pss_b->salt_len);
 			}
 			default:
 				break;
@@ -328,7 +348,6 @@ end:
 bool rsa_pss_params_build(rsa_pss_params_t *params, chunk_t *asn1)
 {
 	chunk_t hash = chunk_empty, mgf = chunk_empty, slen = chunk_empty;
-	ssize_t salt_len;
 	int alg;
 
 	if (params->hash != HASH_SHA1)
@@ -351,16 +370,15 @@ bool rsa_pss_params_build(rsa_pss_params_t *params, chunk_t *asn1)
 		mgf = asn1_algorithmIdentifier_params(OID_MGF1,
 											  asn1_algorithmIdentifier(alg));
 	}
-	salt_len = rsa_pss_salt_length(params);
-	if (salt_len < 0)
+	if (params->salt_len < 0)
 	{
 		chunk_free(&hash);
 		chunk_free(&mgf);
 		return FALSE;
 	}
-	else if (salt_len != HASH_SIZE_SHA1)
+	else if (params->salt_len != HASH_SIZE_SHA1)
 	{
-		slen = asn1_integer("m", asn1_integer_from_uint64(salt_len));
+		slen = asn1_integer("m", asn1_integer_from_uint64(params->salt_len));
 	}
 	*asn1 = asn1_wrap(ASN1_SEQUENCE, "mmm",
 				hash.len ? asn1_wrap(ASN1_CONTEXT_C_0, "m", hash) : chunk_empty,

src/libstrongswan/credentials/keys/signature_params.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017 Tobias Brunner
+ * Copyright (C) 2017-2018 Tobias Brunner
  * HSR Hochschule fuer Technik Rapperswil
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -100,11 +100,15 @@ struct rsa_pss_params_t {
 	hash_algorithm_t hash;
 	/** Hash for the MGF1 function */
 	hash_algorithm_t mgf1_hash;
-	/** Salt length, use RSA_PSS_SALT_LEN_DEFAULT for length equal to hash */
+	/** Salt length, use the constants below for special lengths resolved
+	 * via rsa_pss_params_set_salt_len() */
 	ssize_t salt_len;
 	/** Salt value, for unit tests (not all implementations support this) */
 	chunk_t salt;
+/** Use a salt length equal to the length of the hash */
 #define RSA_PSS_SALT_LEN_DEFAULT -1
+/** Use the maximum salt length depending on the hash and key length */
+#define RSA_PSS_SALT_LEN_MAX -2
 };
 
 /**
@@ -126,4 +130,15 @@ bool rsa_pss_params_parse(chunk_t asn1, int level0, rsa_pss_params_t *params);
  */
 bool rsa_pss_params_build(rsa_pss_params_t *params, chunk_t *asn1);
 
+/**
+ * Determine and set the salt length for the given params in case constants
+ * are used
+ *
+ * @param params	parameters to update
+ * @param modbits	RSA modulus length in bits (required if RSA_PSS_SALT_LEN_MAX
+ *					is used)
+ * @return			salt length to use, negative on error
+ */
+bool rsa_pss_params_set_salt_len(rsa_pss_params_t *params, size_t modbits);
+
 #endif /** SIGNATURE_PARAMS_H_ @}*/