From 1c5a9c91bbe7b02311016075ef28c51257b54936 Mon Sep 17 00:00:00 2001 From: Denozordec Date: Mon, 6 Apr 2026 20:20:52 +0700 Subject: [PATCH] feat: refactor materialization logic to group static routes by community in the refresh pipeline. Introduce a new structure for managing static routes, enhancing the handling of IPv4 and IPv6 routes while preserving community associations. Update related functions to support the new grouping mechanism. --- internal/pipeline/bird_materialize.go | 89 ++++++++++++++---- .../pipeline/materialize_regression_test.go | 94 +++++++++++++++++++ internal/pipeline/refresh.go | 48 ++++++++-- 3 files changed, 204 insertions(+), 27 deletions(-) create mode 100644 internal/pipeline/materialize_regression_test.go diff --git a/internal/pipeline/bird_materialize.go b/internal/pipeline/bird_materialize.go index cebd228..cc37182 100644 --- a/internal/pipeline/bird_materialize.go +++ b/internal/pipeline/bird_materialize.go @@ -3,6 +3,7 @@ package pipeline import ( "fmt" "net/netip" + "sort" "strconv" "strings" @@ -10,12 +11,18 @@ import ( "evobgp/internal/store" ) -// materializeRowsForBird turns prefix rows into export-filter CIDR lists, optional AS_PATH rules, and static routes with per-prefix communities. -// Duplicate CIDRs: first occurrence wins for static community; filters use unique CIDRs. +type staticCommunityRoutes struct { + CommunityID string + RoutesV4 []birdfmt.StaticRoute + RoutesV6 []birdfmt.StaticRoute +} + +// materializeRowsForBird turns prefix rows into export-filter CIDR lists, optional AS_PATH rules, +// and static routes grouped by community. The same prefix can appear in multiple community groups. func materializeRowsForBird(st store.Backend, tenantID string, rows []store.PrefixRow) ( filterV4, filterV6 []netip.Prefix, pathASNs []int64, - staticV4, staticV6 []birdfmt.StaticRoute, + staticByCommunity []staticCommunityRoutes, err error, ) { var pathSeen map[int64]struct{} @@ -31,7 +38,12 @@ func materializeRowsForBird(st store.Backend, tenantID string, rows []store.Pref } filterSeen := make(map[string]struct{}) - staticChosen := make(map[string]birdfmt.StaticRoute) + type staticGroup struct { + v4 map[string]birdfmt.StaticRoute + v6 map[string]birdfmt.StaticRoute + } + staticGroups := make(map[string]*staticGroup) + routeBodyByCommunity := make(map[string]string) for _, pr := range rows { p := strings.TrimSpace(pr.Prefix) @@ -58,24 +70,67 @@ func materializeRowsForBird(st store.Backend, tenantID string, rows []store.Pref } } - if _, ok := staticChosen[key]; ok { - continue + communityID := "" + if pr.CommunityID != nil { + communityID = strings.TrimSpace(*pr.CommunityID) } - body, err := communityRouteBody(st, tenantID, pr.CommunityID) - if err != nil { - return nil, nil, nil, nil, nil, err + body, ok := routeBodyByCommunity[communityID] + if !ok { + body, err = communityRouteBody(st, tenantID, pr.CommunityID) + if err != nil { + return nil, nil, nil, nil, err + } + routeBodyByCommunity[communityID] = body + } + grp, ok := staticGroups[communityID] + if !ok { + grp = &staticGroup{ + v4: make(map[string]birdfmt.StaticRoute), + v6: make(map[string]birdfmt.StaticRoute), + } + staticGroups[communityID] = grp + } + if pfx.Addr().Is4() { + if _, exists := grp.v4[key]; !exists { + grp.v4[key] = birdfmt.StaticRoute{Prefix: pfx, RouteBody: body} + } + } else if pfx.Addr().Is6() { + if _, exists := grp.v6[key]; !exists { + grp.v6[key] = birdfmt.StaticRoute{Prefix: pfx, RouteBody: body} + } } - staticChosen[key] = birdfmt.StaticRoute{Prefix: pfx, RouteBody: body} } - for _, sr := range staticChosen { - if sr.Prefix.Addr().Is4() { - staticV4 = append(staticV4, sr) - } else if sr.Prefix.Addr().Is6() { - staticV6 = append(staticV6, sr) - } + groupKeys := make([]string, 0, len(staticGroups)) + for k := range staticGroups { + groupKeys = append(groupKeys, k) } - return filterV4, filterV6, pathASNs, staticV4, staticV6, nil + sort.Strings(groupKeys) + + for _, gid := range groupKeys { + grp := staticGroups[gid] + out := staticCommunityRoutes{CommunityID: gid} + + v4Keys := make([]string, 0, len(grp.v4)) + for k := range grp.v4 { + v4Keys = append(v4Keys, k) + } + sort.Strings(v4Keys) + for _, k := range v4Keys { + out.RoutesV4 = append(out.RoutesV4, grp.v4[k]) + } + + v6Keys := make([]string, 0, len(grp.v6)) + for k := range grp.v6 { + v6Keys = append(v6Keys, k) + } + sort.Strings(v6Keys) + for _, k := range v6Keys { + out.RoutesV6 = append(out.RoutesV6, grp.v6[k]) + } + staticByCommunity = append(staticByCommunity, out) + } + return filterV4, filterV6, pathASNs, staticByCommunity, nil } func communityRouteBody(st store.Backend, tenantID string, cid *string) (string, error) { diff --git a/internal/pipeline/materialize_regression_test.go b/internal/pipeline/materialize_regression_test.go new file mode 100644 index 0000000..cfffc68 --- /dev/null +++ b/internal/pipeline/materialize_regression_test.go @@ -0,0 +1,94 @@ +package pipeline + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "evobgp/internal/birdfmt" + "evobgp/internal/store" +) + +func TestBuildPreviewFragments_SamePrefixDifferentCommunity(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, moduleIP, _, _ := m.DemoIDs() + + c1, err := m.CreateCommunity(tenant, &store.Community{Community: "65000:1", Title: "A", ValueJSON: "{}"}) + if err != nil { + t.Fatal(err) + } + c2, err := m.CreateCommunity(tenant, &store.Community{Community: "65000:2", Title: "B", ValueJSON: "{}"}) + if err != nil { + t.Fatal(err) + } + + rows := []store.PrefixRow{ + {Prefix: "203.0.113.0/24", CommunityID: &c1.ID, Source: "ip_range"}, + {Prefix: "203.0.113.0/24", CommunityID: &c2.ID, Source: "cdn:test"}, + } + out, err := buildPreviewFragments(m, tenant, moduleIP, "test-revision", rows) + if err != nil { + t.Fatal(err) + } + staticV4 := out[birdfmt.FragmentIncludePath(birdfmt.FragmentPrefixesV4)] + if !strings.Contains(staticV4, "protocol static evobgp_prefixes_v4_"+communityProtocolSuffix(c1.ID)) { + t.Fatalf("expected dedicated static protocol for first community, got:\n%s", staticV4) + } + if !strings.Contains(staticV4, "protocol static evobgp_prefixes_v4_"+communityProtocolSuffix(c2.ID)) { + t.Fatalf("expected dedicated static protocol for second community, got:\n%s", staticV4) + } + out2, err := buildPreviewFragments(m, tenant, moduleIP, "test-revision", rows) + if err != nil { + t.Fatal(err) + } + staticV4Second := out2[birdfmt.FragmentIncludePath(birdfmt.FragmentPrefixesV4)] + if staticV4 != staticV4Second { + t.Fatalf("expected deterministic static preview text, got first:\n%s\nsecond:\n%s", staticV4, staticV4Second) + } +} + +func TestCollectModulePrefixRows_CDNRefreshForcesFullGet(t *testing.T) { + m := store.NewMemory() + m.SeedDemo() + tenant, _, _, _, _ := m.DemoIDs() + + mod, err := m.CreateModule(tenant, &store.Module{ + Type: "CDN_CIDRS", + Name: "cdn-refresh-force-full", + Enabled: true, + }) + if err != nil { + t.Fatal(err) + } + + var gotIfNoneMatch string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotIfNoneMatch = strings.TrimSpace(r.Header.Get("If-None-Match")) + w.Header().Set("ETag", "etag-new") + _, _ = w.Write([]byte("198.51.100.0/24\n")) + })) + defer srv.Close() + + etag := "etag-old" + if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{ + SourceKind: "txt", + URL: srv.URL, + Etag: etag, + }); err != nil { + t.Fatal(err) + } + + collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod) + if err != nil { + t.Fatal(err) + } + if gotIfNoneMatch != "" { + t.Fatalf("refresh path must not send If-None-Match, got %q", gotIfNoneMatch) + } + if len(collected) != 1 || collected[0].Prefix != "198.51.100.0/24" { + t.Fatalf("unexpected collected rows: %+v", collected) + } +} diff --git a/internal/pipeline/refresh.go b/internal/pipeline/refresh.go index 3567828..78854e2 100644 --- a/internal/pipeline/refresh.go +++ b/internal/pipeline/refresh.go @@ -163,17 +163,10 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli if err != nil { return nil, err } - if strings.TrimSpace(src.Etag) != "" { - req.Header.Set("If-None-Match", strings.TrimSpace(src.Etag)) - } resp, err := hc.Do(req) if err != nil { return nil, fmt.Errorf("cdn fetch %s: %w", u, err) } - if resp.StatusCode == http.StatusNotModified { - _ = resp.Body.Close() - continue - } if resp.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, resp.Body) _ = resp.Body.Close() @@ -674,7 +667,7 @@ func hashAggregatedMaterialization(tenantID string, rows []store.PrefixRow) stri } func buildPreviewFragments(st store.Backend, tenantID, moduleID, revisionID string, rows []store.PrefixRow) (map[string]string, error) { - v4, v6, pathASNs, sr4, sr6, err := materializeRowsForBird(st, tenantID, rows) + v4, v6, pathASNs, staticGroups, err := materializeRowsForBird(st, tenantID, rows) if err != nil { return nil, err } @@ -686,8 +679,7 @@ func buildPreviewFragments(st store.Backend, tenantID, moduleID, revisionID stri if err != nil { return nil, err } - staticV4 := birdfmt.RenderStaticIPv4Routes("evobgp_prefixes_v4", sr4) - staticV6 := birdfmt.RenderStaticIPv6Routes("evobgp_prefixes_v6", sr6) + staticV4, staticV6 := renderStaticProtocolsByCommunity(staticGroups) locals := birdLocalsFromStore(st, tenantID) tplBody, err := birdfmt.RenderBGPTemplates(birdfmt.BGPTemplatesOptions{ @@ -732,6 +724,42 @@ func buildPreviewFragments(st store.Backend, tenantID, moduleID, revisionID stri return out, nil } +func renderStaticProtocolsByCommunity(groups []staticCommunityRoutes) (string, string) { + var b4 strings.Builder + var b6 strings.Builder + for _, grp := range groups { + nameSuffix := communityProtocolSuffix(grp.CommunityID) + if len(grp.RoutesV4) > 0 { + b4.WriteString(birdfmt.RenderStaticIPv4Routes("evobgp_prefixes_v4_"+nameSuffix, grp.RoutesV4)) + } + if len(grp.RoutesV6) > 0 { + b6.WriteString(birdfmt.RenderStaticIPv6Routes("evobgp_prefixes_v6_"+nameSuffix, grp.RoutesV6)) + } + } + return b4.String(), b6.String() +} + +func communityProtocolSuffix(communityID string) string { + raw := strings.TrimSpace(communityID) + if raw == "" { + return "default" + } + var b strings.Builder + b.Grow(len(raw)) + for _, r := range raw { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + continue + } + b.WriteByte('_') + } + out := strings.Trim(b.String(), "_") + if out == "" { + return "default" + } + return "c_" + out +} + type birdLocals struct { routerID string localV4 string