feat: enhance DoH profile management and resolver policy in modules

- Added `DohResolverPolicy` schema to OpenAPI documentation, defining policies for domain resolution.
- Updated module handling to support multiple DoH profiles via `doh_profile_ids` and introduced `doh_resolver_policy` in the API.
- Refactored related functions to accommodate the new DoH profile structure, ensuring backward compatibility with existing `doh_profile_id`.
- Enhanced UI components to allow selection and management of DoH profiles and policies in the web interface.
- Updated database interactions to handle new fields and ensure proper data normalization.

internal/pipeline/collect_parallel.go

@@ -175,7 +175,7 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
 	return out, nil
 }
 
-func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profile *store.DohProfile, entries []*store.DomainEntry) ([]store.PrefixRow, error) {
+func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry) ([]store.PrefixRow, error) {
 	var validDom []*store.DomainEntry
 	for _, e := range entries {
 		if e != nil {
@@ -202,7 +202,7 @@ func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Mo
 				c := *mod.DefaultCommunityID
 				comm = &c
 			}
-			addrs, err := resolveDomainIPs(ctx, hc, profile, entry.FQDN)
+			addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, entry.FQDN)
 			if err != nil {
 				results[idx] = domResult{err: fmt.Errorf("resolve domain %q: %w", entry.FQDN, err)}
 				return

internal/pipeline/doh_resolve.go

@@ -0,0 +1,131 @@
+package pipeline
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/netip"
+	"strings"
+	"time"
+
+	"evobgp/internal/store"
+
+	"github.com/miekg/dns"
+)
+
+func loadModuleDohProfiles(st store.Backend, tenantID string, mod *store.Module) ([]*store.DohProfile, string, error) {
+	if mod == nil {
+		return nil, store.DohPolicyPrimaryOnly, nil
+	}
+	policy := store.NormalizeDohResolverPolicy(mod.DohResolverPolicy)
+	var profiles []*store.DohProfile
+	for _, id := range mod.EffectiveDohProfileIDs() {
+		prof, err := st.GetDohProfile(tenantID, id)
+		if err != nil {
+			return nil, "", fmt.Errorf("get doh profile %s: %w", id, err)
+		}
+		profiles = append(profiles, prof)
+	}
+	return profiles, policy, nil
+}
+
+func resolveDomainIPsWithPolicy(ctx context.Context, hc *http.Client, profiles []*store.DohProfile, policy, fqdn string) ([]netip.Addr, error) {
+	policy = store.NormalizeDohResolverPolicy(policy)
+	if len(profiles) == 0 {
+		return resolveDomainIPs(ctx, hc, nil, fqdn)
+	}
+	if len(profiles) == 1 {
+		return resolveDomainIPs(ctx, hc, profiles[0], fqdn)
+	}
+
+	switch policy {
+	case store.DohPolicyUnion:
+		return resolveDomainIPsUnion(ctx, hc, profiles, fqdn)
+	case store.DohPolicyFailover:
+		return resolveDomainIPsFailover(ctx, hc, profiles, fqdn)
+	default:
+		return resolveDomainIPs(ctx, hc, profiles[0], fqdn)
+	}
+}
+
+func resolveDomainIPsUnion(ctx context.Context, hc *http.Client, profiles []*store.DohProfile, fqdn string) ([]netip.Addr, error) {
+	var merged []netip.Addr
+	var errs []error
+	for _, prof := range profiles {
+		if prof == nil {
+			continue
+		}
+		ips, err := resolveDomainIPsNoSystemFallback(ctx, hc, prof, fqdn)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", strings.TrimSpace(prof.URL), err))
+			continue
+		}
+		merged = append(merged, ips...)
+	}
+	merged = uniqAddrs(merged)
+	if len(merged) > 0 {
+		return merged, nil
+	}
+	if len(errs) > 0 {
+		return nil, fmt.Errorf("doh union failed: %v", errs)
+	}
+	return nil, nil
+}
+
+func resolveDomainIPsFailover(ctx context.Context, hc *http.Client, profiles []*store.DohProfile, fqdn string) ([]netip.Addr, error) {
+	var lastErr error
+	for _, prof := range profiles {
+		if prof == nil {
+			continue
+		}
+		ips, err := resolveDomainIPsNoSystemFallback(ctx, hc, prof, fqdn)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		if len(ips) > 0 {
+			return ips, nil
+		}
+	}
+	if lastErr != nil {
+		return nil, lastErr
+	}
+	return resolveDomainIPs(ctx, hc, nil, fqdn)
+}
+
+// resolveDomainIPsNoSystemFallback queries one DoH profile without falling back to OS resolver.
+func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, profile *store.DohProfile, fqdn string) ([]netip.Addr, error) {
+	host := strings.TrimSpace(strings.TrimSuffix(fqdn, "."))
+	if host == "" {
+		return nil, nil
+	}
+	if profile == nil || strings.TrimSpace(profile.URL) == "" {
+		return nil, fmt.Errorf("empty doh profile")
+	}
+
+	timeout := dohProfileTimeout(profile)
+	dctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	baseURL := strings.TrimSpace(profile.URL)
+	v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA)
+	v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA)
+	if err4 != nil {
+		v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
+	}
+	if err6 != nil {
+		v6, err6 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "AAAA")
+	}
+	if err4 != nil && err6 != nil {
+		return nil, fmt.Errorf("doh failed for A and AAAA: %v; %v", err4, err6)
+	}
+	return uniqAddrs(append(v4, v6...)), nil
+}
+
+func dohProfileTimeout(profile *store.DohProfile) time.Duration {
+	timeout := 10 * time.Second
+	if profile != nil && profile.TimeoutMs != nil && *profile.TimeoutMs > 0 {
+		timeout = time.Duration(*profile.TimeoutMs) * time.Millisecond
+	}
+	return timeout
+}

internal/pipeline/doh_resolve_test.go

@@ -0,0 +1,148 @@
+package pipeline
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"evobgp/internal/store"
+)
+
+func TestResolveDomainIPsWithPolicy_Union(t *testing.T) {
+	srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.1"}]}`))
+	}))
+	defer srvRU.Close()
+	srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.1"}]}`))
+	}))
+	defer srvEU.Close()
+
+	profiles := []*store.DohProfile{
+		{URL: srvRU.URL},
+		{URL: srvEU.URL},
+	}
+	ips, err := resolveDomainIPsWithPolicy(context.Background(), srvRU.Client(), profiles, store.DohPolicyUnion, "example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ips) != 2 {
+		t.Fatalf("want 2 ips, got %v", ips)
+	}
+	seen := map[string]bool{ips[0].String(): true, ips[1].String(): true}
+	if !seen["198.51.100.1"] || !seen["203.0.113.1"] {
+		t.Fatalf("unexpected ips: %v", ips)
+	}
+}
+
+func TestResolveDomainIPsWithPolicy_Failover(t *testing.T) {
+	var calls int
+	srvBad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls++
+		http.Error(w, "fail", http.StatusBadGateway)
+	}))
+	defer srvBad.Close()
+	srvOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls++
+		_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.5"}]}`))
+	}))
+	defer srvOK.Close()
+
+	profiles := []*store.DohProfile{
+		{URL: srvBad.URL},
+		{URL: srvOK.URL},
+	}
+	ips, err := resolveDomainIPsWithPolicy(context.Background(), srvBad.Client(), profiles, store.DohPolicyFailover, "example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ips) != 1 || ips[0].String() != "198.51.100.5" {
+		t.Fatalf("unexpected ips: %v", ips)
+	}
+	if calls < 2 {
+		t.Fatalf("want at least 2 resolver calls, got %d", calls)
+	}
+}
+
+func TestResolveDomainIPsWithPolicy_PrimaryOnly(t *testing.T) {
+	var secondCalled bool
+	srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.9"}]}`))
+	}))
+	defer srv1.Close()
+	srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		secondCalled = true
+		_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.9"}]}`))
+	}))
+	defer srv2.Close()
+
+	profiles := []*store.DohProfile{
+		{URL: srv1.URL},
+		{URL: srv2.URL},
+	}
+	ips, err := resolveDomainIPsWithPolicy(context.Background(), srv1.Client(), profiles, store.DohPolicyPrimaryOnly, "example.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ips) != 1 || ips[0].String() != "198.51.100.9" {
+		t.Fatalf("unexpected ips: %v", ips)
+	}
+	if secondCalled {
+		t.Fatal("secondary resolver must not be queried in primary_only mode")
+	}
+}
+
+func TestCollectModulePrefixRows_DohUnion(t *testing.T) {
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, _, _ := m.DemoIDs()
+
+	mod, err := m.CreateModule(tenant, &store.Module{
+		Type:              "DOMAINS",
+		Name:              "domains-union",
+		Enabled:           true,
+		DohResolverPolicy: store.DohPolicyUnion,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.2"}]}`))
+	}))
+	defer srvRU.Close()
+	srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.2"}]}`))
+	}))
+	defer srvEU.Close()
+
+	ru, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "ru", URL: srvRU.URL})
+	if err != nil {
+		t.Fatal(err)
+	}
+	eu, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "eu", URL: srvEU.URL})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.UpdateModule(tenant, mod.ID, &store.ModulePatch{
+		DohProfileIDs: &[]string{ru.ID, eu.ID},
+	}); err != nil {
+		t.Fatal(err)
+	}
+	mod, err = m.GetModule(tenant, mod.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := m.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "svc.example.com"}); err != nil {
+		t.Fatal(err)
+	}
+
+	rows, err := collectModulePrefixRows(context.Background(), m, srvRU.Client(), tenant, mod, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 {
+		t.Fatalf("want 2 prefix rows, got %+v", rows)
+	}
+}

internal/pipeline/module_hash.go

@@ -21,14 +21,13 @@ func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module)
 	if mod.DefaultCommunityID != nil {
 		_, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID))
 	}
-	if mod.DohProfileID != nil {
-		_, _ = fmt.Fprintf(h, "doh_profile=%s\n", strings.TrimSpace(*mod.DohProfileID))
-		if pid := strings.TrimSpace(*mod.DohProfileID); pid != "" {
-			if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil {
-				_, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL))
-				if prof.TimeoutMs != nil {
-					_, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs)
-				}
+	_, _ = fmt.Fprintf(h, "doh_policy=%s\n", store.NormalizeDohResolverPolicy(mod.DohResolverPolicy))
+	for _, pid := range mod.EffectiveDohProfileIDs() {
+		_, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid)
+		if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil {
+			_, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL))
+			if prof.TimeoutMs != nil {
+				_, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs)
 			}
 		}
 	}

internal/pipeline/refresh.go

@@ -171,14 +171,11 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
 		if err != nil {
 			return nil, err
 		}
-		var profile *store.DohProfile
-		if mod.DohProfileID != nil && strings.TrimSpace(*mod.DohProfileID) != "" {
-			profile, err = st.GetDohProfile(tenantID, strings.TrimSpace(*mod.DohProfileID))
-			if err != nil {
-				return nil, fmt.Errorf("get doh profile: %w", err)
-			}
+		profiles, policy, err := loadModuleDohProfiles(st, tenantID, mod)
+		if err != nil {
+			return nil, err
 		}
-		return collectDomainPrefixRows(ctx, hc, mod, profile, entries)
+		return collectDomainPrefixRows(ctx, hc, mod, profiles, policy, entries)
 	default:
 		return nil, fmt.Errorf("unknown module type %q", mod.Type)
 	}