libipsec: Reject policies for anything but ESP tunnel mode

This is only relevant for trap policies as similar SAs are already
rejected when negotiated.

src/libipsec/ipsec_policy.c

@@ -202,6 +202,23 @@ ipsec_policy_t *ipsec_policy_create(host_t *src, host_t *dst,
 {
 	private_ipsec_policy_t *this;
 
+
+	if (!sa->esp.use)
+	{
+		DBG1(DBG_ESP, "  IPsec policy: protocol not supported");
+		return NULL;
+	}
+	if (sa->ipcomp.transform != IPCOMP_NONE)
+	{
+		DBG1(DBG_ESP, "  IPsec policy: compression not supported");
+		return NULL;
+	}
+	if (sa->mode != MODE_TUNNEL)
+	{
+		DBG1(DBG_ESP, "  IPsec policy: unsupported mode");
+		return NULL;
+	}
+
 	INIT(this,
 		.public = {
 			.match = _match,

src/libipsec/ipsec_policy_mgr.c

@@ -158,6 +158,11 @@ METHOD(ipsec_policy_mgr_t, add_policy, status_t,
 
 	policy = ipsec_policy_create(src, dst, src_ts, dst_ts, direction, type, sa,
 								 mark, priority);
+	if (!policy)
+	{
+		DBG1(DBG_ESP, "failed to create policy");
+		return FAILED;
+	}
 	entry = policy_entry_create(policy);
 
 	this->lock->write_lock(this->lock);