encoding: Allow ke_payload_create_from_diffie_hellman() to fail

src/libcharon/encoding/payloads/ke_payload.h

@@ -73,7 +73,7 @@ ke_payload_t *ke_payload_create(payload_type_t type);
  *
  * @param type		PLV2_KEY_EXCHANGE or PLV1_KEY_EXCHANGE
  * @param dh		diffie hellman object containing group and key
- * @return 			ke_payload_t object
+ * @return 			ke_payload_t object, NULL on error
  */
 ke_payload_t *ke_payload_create_from_diffie_hellman(payload_type_t type,
 													diffie_hellman_t *dh);

src/libcharon/sa/ikev1/phase1.c

@@ -694,7 +694,13 @@ METHOD(phase1_t, add_nonce_ke, bool,
 	nonce_gen_t *nonceg;
 	chunk_t nonce;
 
-	ke_payload = ke_payload_create_from_diffie_hellman(PLV1_KEY_EXCHANGE, this->dh);
+	ke_payload = ke_payload_create_from_diffie_hellman(PLV1_KEY_EXCHANGE,
+													   this->dh);
+	if (!ke_payload)
+	{
+		DBG1(DBG_IKE, "creating KE payload failed");
+		return FALSE;
+	}
 	message->add_payload(message, &ke_payload->payload_interface);
 
 	nonceg = this->keymat->keymat.create_nonce_gen(&this->keymat->keymat);

src/libcharon/sa/ikev1/tasks/quick_mode.c

@@ -465,12 +465,19 @@ static bool get_nonce(private_quick_mode_t *this, chunk_t *nonce,
 /**
  * Add KE payload to message
  */
-static void add_ke(private_quick_mode_t *this, message_t *message)
+static bool add_ke(private_quick_mode_t *this, message_t *message)
 {
 	ke_payload_t *ke_payload;
 
-	ke_payload = ke_payload_create_from_diffie_hellman(PLV1_KEY_EXCHANGE, this->dh);
+	ke_payload = ke_payload_create_from_diffie_hellman(PLV1_KEY_EXCHANGE,
+													   this->dh);
+	if (!ke_payload)
+	{
+		DBG1(DBG_IKE, "creating KE payload failed");
+		return FALSE;
+	}
 	message->add_payload(message, &ke_payload->payload_interface);
+	return TRUE;
 }
 
 /**
@@ -880,7 +887,10 @@ METHOD(task_t, build_i, status_t,
 			}
 			if (group != MODP_NONE)
 			{
-				add_ke(this, message);
+				if (!add_ke(this, message))
+				{
+					return FAILED;
+				}
 			}
 			if (!this->tsi)
 			{
@@ -1218,7 +1228,10 @@ METHOD(task_t, build_r, status_t,
 			}
 			if (this->dh)
 			{
-				add_ke(this, message);
+				if (!add_ke(this, message))
+				{
+					return FAILED;
+				}
 			}
 
 			add_ts(this, message);

src/libcharon/sa/ikev2/tasks/child_create.c

@@ -716,7 +716,7 @@ static status_t select_and_install(private_child_create_t *this,
 /**
  * build the payloads for the message
  */
-static void build_payloads(private_child_create_t *this, message_t *message)
+static bool build_payloads(private_child_create_t *this, message_t *message)
 {
 	sa_payload_t *sa_payload;
 	nonce_payload_t *nonce_payload;
@@ -748,6 +748,11 @@ static void build_payloads(private_child_create_t *this, message_t *message)
 	{
 		ke_payload = ke_payload_create_from_diffie_hellman(PLV2_KEY_EXCHANGE,
 														   this->dh);
+		if (!ke_payload)
+		{
+			DBG1(DBG_IKE, "creating KE payload failed");
+			return FALSE;
+		}
 		message->add_payload(message, (payload_t*)ke_payload);
 	}
 
@@ -776,6 +781,7 @@ static void build_payloads(private_child_create_t *this, message_t *message)
 		message->add_notify(message, FALSE, ESP_TFC_PADDING_NOT_SUPPORTED,
 							chunk_empty);
 	}
+	return TRUE;
 }
 
 /**
@@ -1035,7 +1041,10 @@ METHOD(task_t, build_i, status_t,
 							NARROW_INITIATOR_PRE_AUTH, this->tsi, this->tsr);
 	}
 
-	build_payloads(this, message);
+	if (!build_payloads(this, message))
+	{
+		return FAILED;
+	}
 
 	this->tsi->destroy_offset(this->tsi, offsetof(traffic_selector_t, destroy));
 	this->tsr->destroy_offset(this->tsr, offsetof(traffic_selector_t, destroy));
@@ -1288,7 +1297,12 @@ METHOD(task_t, build_r, status_t,
 			return SUCCESS;
 	}
 
-	build_payloads(this, message);
+	if (!build_payloads(this, message))
+	{
+		message->add_notify(message, FALSE, NO_PROPOSAL_CHOSEN, chunk_empty);
+		handle_child_sa_failure(this, message);
+		return SUCCESS;
+	}
 
 	if (!this->rekey)
 	{	/* invoke the child_up() hook if we are not rekeying */

src/libcharon/sa/ikev2/tasks/ike_init.c

@@ -210,7 +210,7 @@ static void handle_supported_hash_algorithms(private_ike_init_t *this,
 /**
  * build the payloads for the message
  */
-static void build_payloads(private_ike_init_t *this, message_t *message)
+static bool build_payloads(private_ike_init_t *this, message_t *message)
 {
 	sa_payload_t *sa_payload;
 	ke_payload_t *ke_payload;
@@ -254,7 +254,13 @@ static void build_payloads(private_ike_init_t *this, message_t *message)
 
 	nonce_payload = nonce_payload_create(PLV2_NONCE);
 	nonce_payload->set_nonce(nonce_payload, this->my_nonce);
-	ke_payload = ke_payload_create_from_diffie_hellman(PLV2_KEY_EXCHANGE, this->dh);
+	ke_payload = ke_payload_create_from_diffie_hellman(PLV2_KEY_EXCHANGE,
+													   this->dh);
+	if (!ke_payload)
+	{
+		DBG1(DBG_IKE, "creating KE payload failed");
+		return FALSE;
+	}
 
 	if (this->old_sa)
 	{	/* payload order differs if we are rekeying */
@@ -289,6 +295,7 @@ static void build_payloads(private_ike_init_t *this, message_t *message)
 			send_supported_hash_algorithms(this, message);
 		}
 	}
+	return TRUE;
 }
 
 /**
@@ -438,7 +445,10 @@ METHOD(task_t, build_i, status_t,
 		message->add_notify(message, FALSE, COOKIE, this->cookie);
 	}
 
-	build_payloads(this, message);
+	if (!build_payloads(this, message))
+	{
+		return FAILED;
+	}
 
 #ifdef ME
 	{
@@ -572,7 +582,10 @@ METHOD(task_t, build_r, status_t,
 		message->add_notify(message, TRUE, NO_PROPOSAL_CHOSEN, chunk_empty);
 		return FAILED;
 	}
-	build_payloads(this, message);
+	if (!build_payloads(this, message))
+	{
+		return FAILED;
+	}
 	return SUCCESS;
 }