Cleaned up memory management and return values for encryption payload

src/libcharon/encoding/payloads/encryption_payload.c

@@ -308,7 +308,7 @@ static chunk_t append_header(private_encryption_payload_t *this, chunk_t assoc)
 	return chunk_cat("cc", assoc, chunk_from_thing(header));
 }
 
-METHOD(encryption_payload_t, encrypt, bool,
+METHOD(encryption_payload_t, encrypt, status_t,
 	private_encryption_payload_t *this, chunk_t assoc)
 {
 	chunk_t iv, plain, padding, icv, crypt;
@@ -319,14 +319,14 @@ METHOD(encryption_payload_t, encrypt, bool,
 	if (this->aead == NULL)
 	{
 		DBG1(DBG_ENC, "encrypting encryption payload failed, transform missing");
-		return FALSE;
+		return INVALID_STATE;
 	}
 
 	rng = lib->crypto->create_rng(lib->crypto, RNG_WEAK);
 	if (!rng)
 	{
 		DBG1(DBG_ENC, "encrypting encryption payload failed, no RNG found");
-		return FALSE;
+		return NOT_SUPPORTED;
 	}
 
 	assoc = append_header(this, assoc);
@@ -362,7 +362,7 @@ METHOD(encryption_payload_t, encrypt, bool,
 		DBG1(DBG_ENC, "encrypting encryption payload failed, no IV or padding");
 		rng->destroy(rng);
 		free(assoc.ptr);
-		return FALSE;
+		return FAILED;
 	}
 	padding.ptr[padding.len - 1] = padding.len - 1;
 	rng->destroy(rng);
@@ -376,7 +376,7 @@ METHOD(encryption_payload_t, encrypt, bool,
 	if (!this->aead->encrypt(this->aead, crypt, assoc, iv, NULL))
 	{
 		free(assoc.ptr);
-		return FALSE;
+		return FAILED;
 	}
 
 	DBG3(DBG_ENC, "encrypted %B", &crypt);
@@ -384,10 +384,10 @@ METHOD(encryption_payload_t, encrypt, bool,
 
 	free(assoc.ptr);
 
-	return TRUE;
+	return SUCCESS;
 }
 
-METHOD(encryption_payload_t, encrypt_v1, bool,
+METHOD(encryption_payload_t, encrypt_v1, status_t,
 	private_encryption_payload_t *this, chunk_t iv)
 {
 	generator_t *generator;
@@ -397,8 +397,7 @@ METHOD(encryption_payload_t, encrypt_v1, bool,
 	if (this->aead == NULL)
 	{
 		DBG1(DBG_ENC, "encryption failed, transform missing");
-		chunk_free(&iv);
-		return FALSE;
+		return INVALID_STATE;
 	}
 
 	generator = generator_create();
@@ -422,14 +421,12 @@ METHOD(encryption_payload_t, encrypt_v1, bool,
 
 	if (!this->aead->encrypt(this->aead, this->encrypted, chunk_empty, iv, NULL))
 	{
-		chunk_free(&iv);
-		return FALSE;
+		return FAILED;
 	}
-	chunk_free(&iv);
 
 	DBG3(DBG_ENC, "encrypted %B", &this->encrypted);
 
-	return TRUE;
+	return SUCCESS;
 }
 
 /**
@@ -548,7 +545,6 @@ METHOD(encryption_payload_t, decrypt_v1, status_t,
 	if (this->aead == NULL)
 	{
 		DBG1(DBG_ENC, "decryption failed, transform missing");
-		chunk_free(&iv);
 		return INVALID_STATE;
 	}
 
@@ -557,15 +553,16 @@ METHOD(encryption_payload_t, decrypt_v1, status_t,
 		this->encrypted.len < iv.len || this->encrypted.len % iv.len)
 	{
 		DBG1(DBG_ENC, "decryption failed, invalid length");
-		chunk_free(&iv);
 		return FAILED;
 	}
 
 	DBG3(DBG_ENC, "decrypting payloads:");
 	DBG3(DBG_ENC, "encrypted %B", &this->encrypted);
 
-	this->aead->decrypt(this->aead, this->encrypted, chunk_empty, iv, NULL);
-	chunk_free(&iv);
+	if (!this->aead->decrypt(this->aead, this->encrypted, chunk_empty, iv, NULL))
+	{
+		return FAILED;
+	}
 
 	DBG3(DBG_ENC, "plain %B", &this->encrypted);
 

src/libcharon/encoding/payloads/encryption_payload.h

@@ -72,14 +72,18 @@ struct encryption_payload_t {
 	 * Generate, encrypt and sign contained payloads.
 	 *
 	 * @param assoc			associated data
-	 * @return				TRUE if encrypted
+	 * @return
+	 * 						- SUCCESS if encryption successful
+	 * 						- FAILED if encryption failed
+	 * 						- INVALID_STATE if aead not supplied, but needed
 	 */
-	bool (*encrypt) (encryption_payload_t *this, chunk_t assoc);
+	status_t (*encrypt) (encryption_payload_t *this, chunk_t assoc);
 
 	/**
 	 * Decrypt, verify and parse contained payloads.
 	 *
 	 * @param assoc			associated data
+	 * @return
 	 * 						- SUCCESS if parsing successful
 	 *						- PARSE_ERROR if sub-payload parsing failed
 	 * 						- VERIFY_ERROR if sub-payload verification failed