Select private key based on received cipher suites

src/libtls/tls_crypto.c

@@ -397,9 +397,10 @@ struct private_tls_crypto_t {
 
 typedef struct {
 	tls_cipher_suite_t suite;
+	key_type_t key;
+	diffie_hellman_group_t dh;
 	hash_algorithm_t hash;
 	pseudo_random_function_t prf;
-	diffie_hellman_group_t dh;
 	integrity_algorithm_t mac;
 	encryption_algorithm_t encr;
 	size_t encr_size;
@@ -410,87 +411,108 @@ typedef struct {
  */
 static suite_algs_t suite_algs[] = {
 	{ TLS_DHE_RSA_WITH_AES_128_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_2048_BIT,
+		KEY_RSA, MODP_2048_BIT,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_AES_CBC, 16
 	},
 	{ TLS_DHE_RSA_WITH_AES_128_CBC_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_3072_BIT,
+		KEY_RSA, MODP_3072_BIT,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_AES_CBC, 16
 	},
 	{ TLS_DHE_RSA_WITH_AES_256_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_3072_BIT,
+		KEY_RSA, MODP_3072_BIT,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_AES_CBC, 32
 	},
 	{ TLS_DHE_RSA_WITH_AES_256_CBC_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_4096_BIT,
+		KEY_RSA, MODP_4096_BIT,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_AES_CBC, 32
 	},
 	{ TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_2048_BIT,
+		KEY_RSA, MODP_2048_BIT,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_CAMELLIA_CBC, 16
 	},
 	{ TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_3072_BIT,
+		KEY_RSA, MODP_3072_BIT,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_CAMELLIA_CBC, 16
 	},
 	{ TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_3072_BIT,
+		KEY_RSA, MODP_3072_BIT,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_CAMELLIA_CBC, 32
 	},
 	{ TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_4096_BIT,
+		KEY_RSA, MODP_4096_BIT,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_CAMELLIA_CBC, 32
 	},
 	{ TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_2048_BIT,
+		KEY_RSA, MODP_2048_BIT,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_3DES, 0
 	},
 	{ TLS_RSA_WITH_AES_128_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_AES_CBC, 16
 	},
 	{ TLS_RSA_WITH_AES_128_CBC_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_AES_CBC, 16
 	},
 	{ TLS_RSA_WITH_AES_256_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_AES_CBC, 32
 	},
 	{ TLS_RSA_WITH_AES_256_CBC_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_AES_CBC, 32
 	},
 	{ TLS_RSA_WITH_CAMELLIA_128_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_CAMELLIA_CBC, 16
 	},
 	{ TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_CAMELLIA_CBC, 16
 	},
 	{ TLS_RSA_WITH_CAMELLIA_256_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_CAMELLIA_CBC, 32
 	},
 	{ TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_CAMELLIA_CBC, 32
 	},
 	{ TLS_RSA_WITH_3DES_EDE_CBC_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_3DES, 0
 	},
 	{ TLS_RSA_WITH_NULL_SHA,
-		HASH_SHA1, PRF_HMAC_SHA1, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA1, PRF_HMAC_SHA1,
 		AUTH_HMAC_SHA1_160, ENCR_NULL, 0
 	},
 	{ TLS_RSA_WITH_NULL_SHA256,
-		HASH_SHA256, PRF_HMAC_SHA2_256, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_SHA256, PRF_HMAC_SHA2_256,
 		AUTH_HMAC_SHA2_256_256, ENCR_NULL, 0
 	},
 	{ TLS_RSA_WITH_NULL_MD5,
-		HASH_MD5, PRF_HMAC_MD5, MODP_NONE,
+		KEY_RSA, MODP_NONE,
+		HASH_MD5, PRF_HMAC_MD5,
 		AUTH_HMAC_MD5_128, ENCR_NULL, 0
 	},
 };
@@ -614,17 +636,8 @@ METHOD(tls_crypto_t, get_cipher_suites, int,
 /**
  * Create crypto primitives
  */
-static bool create_ciphers(private_tls_crypto_t *this, tls_cipher_suite_t suite)
+static bool create_ciphers(private_tls_crypto_t *this, suite_algs_t *algs)
 {
-	suite_algs_t *algs;
-
-	algs = find_suite(suite);
-	if (!algs)
-	{
-		DBG1(DBG_TLS, "selected TLS suite not supported");
-		return FALSE;
-	}
-
 	DESTROY_IF(this->prf);
 	if (this->tls->get_version(this->tls) < TLS_1_2)
 	{
@@ -674,8 +687,10 @@ static bool create_ciphers(private_tls_crypto_t *this, tls_cipher_suite_t suite)
 }
 
 METHOD(tls_crypto_t, select_cipher_suite, tls_cipher_suite_t,
-	private_tls_crypto_t *this, tls_cipher_suite_t *suites, int count)
+	private_tls_crypto_t *this, tls_cipher_suite_t *suites, int count,
+	key_type_t key)
 {
+	suite_algs_t *algs;
 	int i, j;
 
 	for (i = 0; i < this->suite_count; i++)
@@ -684,10 +699,17 @@ METHOD(tls_crypto_t, select_cipher_suite, tls_cipher_suite_t,
 		{
 			if (this->suites[i] == suites[j])
 			{
-				if (create_ciphers(this, this->suites[i]))
+				algs = find_suite(this->suites[i]);
+				if (algs)
 				{
-					this->suite = this->suites[i];
+					if (key == KEY_ANY || key == algs->key)
-					return this->suite;
+					{
+						if (create_ciphers(this, algs))
+						{
+							this->suite = this->suites[i];
+							return this->suite;
+						}
+					}
 				}
 			}
 		}

src/libtls/tls_crypto.h

@@ -376,10 +376,12 @@ struct tls_crypto_t {
 	 *
 	 * @param suites		list of candidates to select from
 	 * @param count			number of suites
+	 * @param key			key type used, or KEY_ANY
 	 * @return				selected suite, 0 if none acceptable
 	 */
 	tls_cipher_suite_t (*select_cipher_suite)(tls_crypto_t *this,
-										tls_cipher_suite_t *suites, int count);
+										tls_cipher_suite_t *suites, int count,
+										key_type_t key);
 
 	/**
 	 * Get the Diffie-Hellman group to use, if any.

src/libtls/tls_peer.c

@@ -156,7 +156,7 @@ static status_t process_server_hello(private_tls_peer_t *this,
 		return NEED_MORE;
 	}
 	suite = cipher;
-	if (!this->crypto->select_cipher_suite(this->crypto, &suite, 1))
+	if (!this->crypto->select_cipher_suite(this->crypto, &suite, 1, KEY_ANY))
 	{
 		DBG1(DBG_TLS, "received TLS cipher suite %N inacceptable",
 			 tls_cipher_suite_names, suite);

src/libtls/tls_server.c

@@ -136,6 +136,58 @@ struct private_tls_server_t {
 	bool curves_received;
 };
 
+/**
+ * Find a cipher suite and a server key
+ */
+static bool select_suite_and_key(private_tls_server_t *this,
+								 tls_cipher_suite_t *suites, int count)
+{
+	private_key_t *key;
+	key_type_t type;
+
+	key = lib->credmgr->get_private(lib->credmgr, KEY_ANY, this->server,
+									this->server_auth);
+	if (!key)
+	{
+		DBG1(DBG_TLS, "no usable TLS server certificate found for '%Y'",
+			 this->server);
+		return FALSE;
+	}
+	this->suite = this->crypto->select_cipher_suite(this->crypto,
+											suites, count, key->get_type(key));
+	if (!this->suite)
+	{	/* no match for this key, try to find another type */
+		if (key->get_type(key) == KEY_ECDSA)
+		{
+			type = KEY_RSA;
+		}
+		else
+		{
+			type = KEY_ECDSA;
+		}
+		key->destroy(key);
+
+		this->suite = this->crypto->select_cipher_suite(this->crypto,
+											suites, count, type);
+		if (!this->suite)
+		{
+			DBG1(DBG_TLS, "received cipher suites inacceptable");
+			return FALSE;
+		}
+		this->server_auth->destroy(this->server_auth);
+		this->server_auth = auth_cfg_create();
+		key = lib->credmgr->get_private(lib->credmgr, type, this->server,
+										this->server_auth);
+		if (!key)
+		{
+			DBG1(DBG_TLS, "received cipher suites inacceptable");
+			return FALSE;
+		}
+	}
+	this->private = key;
+	return TRUE;
+}
+
 /**
  * Process client hello message
  */
@@ -216,10 +268,9 @@ static status_t process_client_hello(private_tls_server_t *this,
 		suites[i] = untoh16(&ciphers.ptr[i * sizeof(u_int16_t)]);
 		DBG2(DBG_TLS, "  %N", tls_cipher_suite_names, suites[i]);
 	}
-	this->suite = this->crypto->select_cipher_suite(this->crypto, suites, count);
+
-	if (!this->suite)
+	if (!select_suite_and_key(this, suites, count))
 	{
-		DBG1(DBG_TLS, "received cipher suites inacceptable");
 		this->alert->add(this->alert, TLS_FATAL, TLS_HANDSHAKE_FAILURE);
 		return NEED_MORE;
 	}
@@ -603,15 +654,6 @@ static status_t send_certificate(private_tls_server_t *this,
 	tls_writer_t *certs;
 	chunk_t data;
 
-	this->private = lib->credmgr->get_private(lib->credmgr,
-									KEY_ANY, this->server, this->server_auth);
-	if (!this->private)
-	{
-		DBG1(DBG_TLS, "no TLS server certificate found for '%Y'", this->server);
-		this->alert->add(this->alert, TLS_FATAL, TLS_INTERNAL_ERROR);
-		return FAILED;
-	}
-
 	/* generate certificate payload */
 	certs = tls_writer_create(256);
 	cert = this->server_auth->get(this->server_auth, AUTH_RULE_SUBJECT_CERT);