crypto: Define MODP_CUSTOM outside of IKE DH range

Before this fix it was possible to crash charon with an IKE_SA_INIT
message containing a KE payload with DH group MODP_CUSTOM(1025).
Defining MODP_CUSTOM outside of the two byte IKE DH identifier range
prevents it from getting negotiated.

Fixes CVE-2014-9221.

src/charon-tkm/src/tkm/tkm_diffie_hellman.c

@@ -41,7 +41,7 @@ struct private_tkm_diffie_hellman_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * Diffie Hellman public value.

src/libstrongswan/crypto/diffie_hellman.c

@@ -42,15 +42,16 @@ ENUM_NEXT(diffie_hellman_group_names, MODP_1024_160, ECP_512_BP, ECP_521_BIT,
 	"ECP_256_BP",
 	"ECP_384_BP",
 	"ECP_512_BP");
-ENUM_NEXT(diffie_hellman_group_names, MODP_NULL, MODP_CUSTOM, ECP_512_BP,
-	"MODP_NULL",
-	"MODP_CUSTOM");
-ENUM_NEXT(diffie_hellman_group_names, NTRU_112_BIT, NTRU_256_BIT, MODP_CUSTOM,
+ENUM_NEXT(diffie_hellman_group_names, MODP_NULL, MODP_NULL, ECP_512_BP,
+	"MODP_NULL");
+ENUM_NEXT(diffie_hellman_group_names, NTRU_112_BIT, NTRU_256_BIT, MODP_NULL,
 	"NTRU_112",
 	"NTRU_128",
 	"NTRU_192",
 	"NTRU_256");
-ENUM_END(diffie_hellman_group_names, NTRU_256_BIT);
+ENUM_NEXT(diffie_hellman_group_names, MODP_CUSTOM, MODP_CUSTOM, NTRU_256_BIT,
+	"MODP_CUSTOM");
+ENUM_END(diffie_hellman_group_names, MODP_CUSTOM);
 
 
 /**

src/libstrongswan/crypto/diffie_hellman.h

@@ -63,12 +63,14 @@ enum diffie_hellman_group_t {
 	/** insecure NULL diffie hellman group for testing, in PRIVATE USE */
 	MODP_NULL = 1024,
 	/** MODP group with custom generator/prime */
-	MODP_CUSTOM = 1025,
 	/** Parameters defined by IEEE 1363.1, in PRIVATE USE */
 	NTRU_112_BIT = 1030,
 	NTRU_128_BIT = 1031,
 	NTRU_192_BIT = 1032,
-	NTRU_256_BIT = 1033
+	NTRU_256_BIT = 1033,
+	/** internally used DH group with additional parameters g and p, outside
+	 * of PRIVATE USE (i.e. IKEv2 DH group range) so it can't be negotiated */
+	MODP_CUSTOM = 65536,
 };
 
 /**

src/libstrongswan/plugins/gcrypt/gcrypt_dh.c

@@ -35,7 +35,7 @@ struct private_gcrypt_dh_t {
 	/**
 	 * Diffie Hellman group number
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/*
 	 * Generator value

src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c

@@ -42,7 +42,7 @@ struct private_gmp_diffie_hellman_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/*
 	 * Generator value.

src/libstrongswan/plugins/ntru/ntru_ke.c

@@ -56,7 +56,7 @@ struct private_ntru_ke_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * NTRU Parameter Set

src/libstrongswan/plugins/openssl/openssl_diffie_hellman.c

@@ -38,7 +38,7 @@ struct private_openssl_diffie_hellman_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * Diffie Hellman object

src/libstrongswan/plugins/openssl/openssl_ec_diffie_hellman.c

@@ -40,7 +40,7 @@ struct private_openssl_ec_diffie_hellman_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * EC private (public) key

src/libstrongswan/plugins/pkcs11/pkcs11_dh.c

@@ -47,7 +47,7 @@ struct private_pkcs11_dh_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * Handle for own private value