diff --git a/go.mod b/go.mod index f6523f9..2181546 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module evobgp -go 1.22 +go 1.24.0 require ( github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.7.2 + github.com/miekg/dns v1.1.72 github.com/prometheus/client_golang v1.20.5 modernc.org/sqlite v1.34.5 ) @@ -24,10 +25,13 @@ require ( github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + golang.org/x/tools v0.40.0 // indirect google.golang.org/protobuf v1.34.2 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/go.sum b/go.sum index f81e26f..eae2bcf 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= @@ -49,19 +51,21 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/pipeline/refresh.go b/internal/pipeline/refresh.go index 06aecdb..5a5e0fd 100644 --- a/internal/pipeline/refresh.go +++ b/internal/pipeline/refresh.go @@ -1,13 +1,16 @@ package pipeline import ( + "bytes" "context" "crypto/sha256" + "encoding/base64" "encoding/json" "fmt" "io" - "net/netip" + "net" "net/http" + "net/netip" "net/url" "os" "sort" @@ -20,6 +23,7 @@ import ( "evobgp/internal/store" "github.com/google/uuid" + "github.com/miekg/dns" ) const ( @@ -264,7 +268,7 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr return nil, nil } if profile == nil || strings.TrimSpace(profile.URL) == "" { - return nil, nil + return resolveDomainWithSystemDNS(ctx, host) } timeout := 10 * time.Second @@ -274,13 +278,113 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr dctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - // RFC8484 endpoint with JSON mode: ?name=&type=A/AAAA - v4, err4 := resolveDomainWithDOHJSON(dctx, hc, strings.TrimSpace(profile.URL), host, "A") - v6, err6 := resolveDomainWithDOHJSON(dctx, hc, strings.TrimSpace(profile.URL), host, "AAAA") - if err4 != nil && err6 != nil { - return nil, fmt.Errorf("doh failed for A and AAAA: %v; %v", err4, err6) + baseURL := strings.TrimSpace(profile.URL) + // Prefer RFC8484 dns-message transport. Some providers don't support dns-json. + v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA) + v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA) + if err4 != nil { + // Fallback to JSON mode for providers that only expose dns-json. + v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A") } - return uniqAddrs(append(v4, v6...)), nil + if err6 != nil { + v6, err6 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "AAAA") + } + if err4 != nil && err6 != nil { + // Some DoH providers return non-JSON responses (RFC8484 dns-message, HTML error pages, etc.). + // Fall back to system resolver to avoid failing the whole module refresh. + ips, fallbackErr := resolveDomainWithSystemDNS(dctx, host) + if fallbackErr != nil { + return nil, fmt.Errorf("doh failed for A and AAAA: %v; %v; fallback dns failed: %w", err4, err6, fallbackErr) + } + return ips, nil + } + out := uniqAddrs(append(v4, v6...)) + if len(out) > 0 { + return out, nil + } + // If DoH succeeds but returns no A/AAAA records, attempt system resolver as best-effort fallback. + ips, err := resolveDomainWithSystemDNS(dctx, host) + if err != nil { + return nil, err + } + return ips, nil +} + +func resolveDomainWithDOHMessage(ctx context.Context, hc *http.Client, baseURL, host string, qtype uint16) ([]netip.Addr, error) { + msg := new(dns.Msg) + msg.SetQuestion(dns.Fqdn(host), qtype) + wire, err := msg.Pack() + if err != nil { + return nil, err + } + + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + q := u.Query() + q.Set("dns", base64.RawURLEncoding.EncodeToString(wire)) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/dns-message") + resp, err := hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return nil, fmt.Errorf("doh dns-message status %s: %s", resp.Status, strings.TrimSpace(string(body))) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + return nil, err + } + parsed := new(dns.Msg) + if err := parsed.Unpack(raw); err != nil { + return nil, err + } + if parsed.Rcode != dns.RcodeSuccess { + return nil, fmt.Errorf("doh rcode=%s", dns.RcodeToString[parsed.Rcode]) + } + var out []netip.Addr + for _, rr := range parsed.Answer { + switch x := rr.(type) { + case *dns.A: + if qtype == dns.TypeA { + if ip, ok := netip.AddrFromSlice(x.A.To4()); ok { + out = append(out, ip.Unmap()) + } + } + case *dns.AAAA: + if qtype == dns.TypeAAAA { + if ip, ok := netip.AddrFromSlice(x.AAAA.To16()); ok { + out = append(out, ip.Unmap()) + } + } + } + } + // Some providers may return JSON even on dns-message request. + if len(out) == 0 && bytes.Contains(bytes.ToLower(raw), []byte(`"answer"`)) { + qs := "A" + if qtype == dns.TypeAAAA { + qs = "AAAA" + } + return resolveDomainWithDOHJSON(ctx, hc, baseURL, host, qs) + } + return uniqAddrs(out), nil +} + +func resolveDomainWithSystemDNS(ctx context.Context, host string) ([]netip.Addr, error) { + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) + if err != nil { + return nil, err + } + return uniqAddrs(ips), nil } func resolveDomainWithDOHJSON(ctx context.Context, hc *http.Client, baseURL, host, qtype string) ([]netip.Addr, error) { diff --git a/web/src/routes/operations/+page.svelte b/web/src/routes/operations/+page.svelte index c3101c3..cc8983b 100644 --- a/web/src/routes/operations/+page.svelte +++ b/web/src/routes/operations/+page.svelte @@ -756,21 +756,37 @@
{#each logEntries as entry, idx (`${job.job_id}-${idx}`)} -
-

{entry.message}

-

- source: {entry.source} - · - kind: {entry.kind} - · - community: {entry.community} - · - prefixes: {entry.prefix_count} -

+
+

{entry.message}

+
+
+ source + {entry.source} +
+
+ kind + {entry.kind} +
+
+ community + {entry.community} +
+
+ prefixes + {entry.prefix_count} +
+
{#if entry.sample && entry.sample.length > 0} -

- {entry.sample.join(', ')} -

+
+

sample

+ +
+ {#each entry.sample as sampleValue, sampleIdx (`${job.job_id}-${idx}-sample-${sampleIdx}`)} +

{sampleValue}

+ {/each} +
+ +
{/if}
{/each}