feat: implement CDN source preview functionality and enhance data handling. Add a new endpoint for previewing CDN sources, allowing users to fetch and parse CIDR prefixes from specified URLs. Update OpenAPI documentation to include new request and response schemas, and modify internal logic to support JSON parsing with prefix path traversal. Enhance UI components to accommodate new preview features, improving user experience and data management.

internal/httpapi/routes_crud.go

@@ -2,11 +2,14 @@ package httpapi
 
 import (
 	"encoding/json"
+	"fmt"
+	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
+	"evobgp/internal/pipeline"
 	"evobgp/internal/store"
 )
 
@@ -17,6 +20,7 @@ func (s *Server) registerCRUDRoutes(m *http.ServeMux) {
 
 	m.HandleFunc("GET /modules/{module_id}/cdn-sources", s.handleListCDNSources)
 	m.HandleFunc("POST /modules/{module_id}/cdn-sources", s.handlePostCDNSource)
+	m.HandleFunc("POST /modules/{module_id}/cdn-sources/preview", s.handlePreviewCDNSource)
 	m.HandleFunc("PATCH /modules/{module_id}/cdn-sources/{source_id}", s.handlePatchCDNSource)
 	m.HandleFunc("DELETE /modules/{module_id}/cdn-sources/{source_id}", s.handleDeleteCDNSource)
 
@@ -68,14 +72,14 @@ func (s *Server) handlePostModule(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	var body struct {
-		Type                 string  `json:"type"`
-		Name                 string  `json:"name"`
-		Enabled              bool    `json:"enabled"`
-		Priority             int     `json:"priority"`
-		RefreshIntervalSec   int     `json:"refresh_interval_sec"`
-		CronExpr             string  `json:"cron_expr"`
-		DefaultCommunityID   *string `json:"default_community_id"`
-		DohProfileID         *string `json:"doh_profile_id"`
+		Type               string  `json:"type"`
+		Name               string  `json:"name"`
+		Enabled            bool    `json:"enabled"`
+		Priority           int     `json:"priority"`
+		RefreshIntervalSec int     `json:"refresh_interval_sec"`
+		CronExpr           string  `json:"cron_expr"`
+		DefaultCommunityID *string `json:"default_community_id"`
+		DohProfileID       *string `json:"doh_profile_id"`
 	}
 	if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
 		writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid json")
@@ -153,7 +157,7 @@ func (s *Server) handleListCDNSources(w http.ResponseWriter, r *http.Request) {
 }
 
 func cdnSourceJSON(x *store.CDNSource) map[string]any {
-	m := map[string]any{"id": x.ID, "source_kind": x.SourceKind, "url": x.URL, "etag": x.Etag}
+	m := map[string]any{"id": x.ID, "source_kind": x.SourceKind, "url": x.URL, "prefix_path": x.PrefixPath, "etag": x.Etag}
 	if x.RefreshIntervalSec != nil {
 		m["refresh_interval_sec"] = *x.RefreshIntervalSec
 	} else {
@@ -167,6 +171,76 @@ func cdnSourceJSON(x *store.CDNSource) map[string]any {
 	return m
 }
 
+func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request) {
+	a, ok := authFromContext(r.Context())
+	if !ok || !s.requireAtLeast(w, a, "editor") {
+		return
+	}
+	var body struct {
+		URL        string `json:"url"`
+		SourceKind string `json:"source_kind"`
+		PrefixPath string `json:"prefix_path"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+		writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid json")
+		return
+	}
+	u := strings.TrimSpace(body.URL)
+	if u == "" {
+		writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "url is required")
+		return
+	}
+	mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
+	if err != nil {
+		writeStoreErr(w, err)
+		return
+	}
+	if mod.Type != "CDN_CIDRS" {
+		writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS")
+		return
+	}
+	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil)
+	if err != nil {
+		writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url")
+		return
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "Bad Gateway", err.Error())
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		_, _ = io.Copy(io.Discard, resp.Body)
+		writeProblem(w, http.StatusBadGateway, "Bad Gateway", fmt.Sprintf("upstream status: %s", resp.Status))
+		return
+	}
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
+	if err != nil {
+		writeProblem(w, http.StatusBadGateway, "Bad Gateway", err.Error())
+		return
+	}
+	pfxs, err := pipeline.ExtractCIDRs(string(raw), body.SourceKind, body.PrefixPath)
+	if err != nil {
+		writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", err.Error())
+		return
+	}
+	items := make([]string, 0, len(pfxs))
+	const previewLimit = 100
+	for i, p := range pfxs {
+		if i >= previewLimit {
+			break
+		}
+		items = append(items, p.String())
+	}
+	writeJSON(w, http.StatusOK, map[string]any{
+		"items":      items,
+		"total":      len(pfxs),
+		"truncated":  len(pfxs) > previewLimit,
+		"source_url": u,
+	})
+}
+
 func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) {
 	a, ok := authFromContext(r.Context())
 	if !ok || !s.requireAtLeast(w, a, "editor") {

internal/pipeline/parse.go

@@ -2,6 +2,7 @@ package pipeline
 
 import (
 	"bufio"
+	"encoding/json"
 	"net/netip"
 	"strings"
 )
@@ -31,6 +32,105 @@ func ParseCIDRLines(body string) []netip.Prefix {
 	return out
 }
 
+// ExtractCIDRs parses CIDRs from either plaintext lines or JSON payload.
+// For sourceKind="json", prefixPath supports dotted traversal, with [] for arrays:
+// e.g. "prefixes[]", "data.items[].cidr".
+func ExtractCIDRs(body, sourceKind, prefixPath string) ([]netip.Prefix, error) {
+	if strings.EqualFold(strings.TrimSpace(sourceKind), "json") {
+		return parseCIDRsFromJSON(body, prefixPath)
+	}
+	return ParseCIDRLines(body), nil
+}
+
+func parseCIDRsFromJSON(body, prefixPath string) ([]netip.Prefix, error) {
+	var root any
+	if err := json.Unmarshal([]byte(body), &root); err != nil {
+		return nil, err
+	}
+	values := jsonValuesAtPath(root, prefixPath)
+	seen := make(map[string]struct{})
+	var out []netip.Prefix
+	for _, raw := range values {
+		pfx := parseOneCIDR(raw)
+		if !pfx.IsValid() {
+			continue
+		}
+		m := pfx.Masked()
+		s := m.String()
+		if _, ok := seen[s]; ok {
+			continue
+		}
+		seen[s] = struct{}{}
+		out = append(out, m)
+	}
+	return out, nil
+}
+
+func jsonValuesAtPath(root any, prefixPath string) []string {
+	path := strings.TrimSpace(prefixPath)
+	if path == "" {
+		return flattenJSONStrings(root)
+	}
+	parts := strings.Split(path, ".")
+	nodes := []any{root}
+	for _, p := range parts {
+		part := strings.TrimSpace(p)
+		if part == "" {
+			continue
+		}
+		iter := strings.HasSuffix(part, "[]")
+		key := strings.TrimSuffix(part, "[]")
+		var next []any
+		for _, n := range nodes {
+			obj, ok := n.(map[string]any)
+			if !ok {
+				continue
+			}
+			child, ok := obj[key]
+			if !ok {
+				continue
+			}
+			if iter {
+				if arr, ok := child.([]any); ok {
+					next = append(next, arr...)
+				}
+				continue
+			}
+			next = append(next, child)
+		}
+		nodes = next
+		if len(nodes) == 0 {
+			return nil
+		}
+	}
+	var out []string
+	for _, n := range nodes {
+		out = append(out, flattenJSONStrings(n)...)
+	}
+	return out
+}
+
+func flattenJSONStrings(v any) []string {
+	switch x := v.(type) {
+	case string:
+		return []string{strings.TrimSpace(x)}
+	case []any:
+		var out []string
+		for _, item := range x {
+			out = append(out, flattenJSONStrings(item)...)
+		}
+		return out
+	case map[string]any:
+		var out []string
+		for _, item := range x {
+			out = append(out, flattenJSONStrings(item)...)
+		}
+		return out
+	default:
+		return nil
+	}
+}
+
 func parseOneCIDR(s string) netip.Prefix {
 	if p, err := netip.ParsePrefix(s); err == nil {
 		return p

internal/pipeline/refresh.go

@@ -182,7 +182,11 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
 				e := etag
 				_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, &store.CDNSourcePatch{Etag: &e})
 			}
-			for _, pfx := range ParseCIDRLines(string(body)) {
+			pfxs, err := ExtractCIDRs(string(body), src.SourceKind, src.PrefixPath)
+			if err != nil {
+				return nil, fmt.Errorf("cdn parse %s: %w", u, err)
+			}
+			for _, pfx := range pfxs {
 				comm := src.CommunityID
 				if comm == nil && mod.DefaultCommunityID != nil {
 					c := *mod.DefaultCommunityID

internal/repository/postgres_entities.go

@@ -23,7 +23,7 @@ func (p *Postgres) ListCDNSources(tenantID, moduleID string) ([]*store.CDNSource
 	}
 	ctx := context.Background()
 	rows, err := p.pool.Query(ctx, `
-		SELECT id::text, source_kind, url, COALESCE(etag,''), refresh_interval_sec, community_id::text
+		SELECT id::text, source_kind, url, COALESCE(prefix_path,''), COALESCE(etag,''), refresh_interval_sec, community_id::text
 		FROM module_cdn_source WHERE module_id=$1 ORDER BY url`, moduleID)
 	if err != nil {
 		return nil, err
@@ -35,7 +35,7 @@ func (p *Postgres) ListCDNSources(tenantID, moduleID string) ([]*store.CDNSource
 		s.ModuleID = moduleID
 		var ri *int32
 		var comm *string
-		if err := rows.Scan(&s.ID, &s.SourceKind, &s.URL, &s.Etag, &ri, &comm); err != nil {
+		if err := rows.Scan(&s.ID, &s.SourceKind, &s.URL, &s.PrefixPath, &s.Etag, &ri, &comm); err != nil {
 			continue
 		}
 		if ri != nil {
@@ -62,9 +62,9 @@ func (p *Postgres) CreateCDNSource(tenantID, moduleID string, in *store.CDNSourc
 	ctx := context.Background()
 	id := uuid.NewString()
 	_, err = p.pool.Exec(ctx, `
-		INSERT INTO module_cdn_source (id, module_id, source_kind, url, etag, refresh_interval_sec, community_id)
-		VALUES ($1,$2,$3,$4,$5,$6, NULLIF($7::uuid, '00000000-0000-0000-0000-000000000000'::uuid))`,
-		id, moduleID, in.SourceKind, strings.TrimSpace(in.URL), in.Etag, nullInt32Ptr(in.RefreshIntervalSec), uuidOrNilPtr(in.CommunityID))
+		INSERT INTO module_cdn_source (id, module_id, source_kind, url, prefix_path, etag, refresh_interval_sec, community_id)
+		VALUES ($1,$2,$3,$4,$5,$6,$7, NULLIF($8::uuid, '00000000-0000-0000-0000-000000000000'::uuid))`,
+		id, moduleID, in.SourceKind, strings.TrimSpace(in.URL), strings.TrimSpace(in.PrefixPath), in.Etag, nullInt32Ptr(in.RefreshIntervalSec), uuidOrNilPtr(in.CommunityID))
 	if err != nil {
 		return nil, err
 	}
@@ -77,8 +77,8 @@ func (p *Postgres) getCDNSource(ctx context.Context, moduleID, id string) (*stor
 	var ri *int32
 	var comm *string
 	err := p.pool.QueryRow(ctx, `
-		SELECT id::text, source_kind, url, COALESCE(etag,''), refresh_interval_sec, community_id::text
-		FROM module_cdn_source WHERE id=$1 AND module_id=$2`, id, moduleID).Scan(&s.ID, &s.SourceKind, &s.URL, &s.Etag, &ri, &comm)
+		SELECT id::text, source_kind, url, COALESCE(prefix_path,''), COALESCE(etag,''), refresh_interval_sec, community_id::text
+		FROM module_cdn_source WHERE id=$1 AND module_id=$2`, id, moduleID).Scan(&s.ID, &s.SourceKind, &s.URL, &s.PrefixPath, &s.Etag, &ri, &comm)
 	if err != nil {
 		return nil, err
 	}
@@ -114,6 +114,9 @@ func (p *Postgres) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *s
 	if patch.URL != nil {
 		cur.URL = strings.TrimSpace(*patch.URL)
 	}
+	if patch.PrefixPath != nil {
+		cur.PrefixPath = strings.TrimSpace(*patch.PrefixPath)
+	}
 	if patch.Etag != nil {
 		cur.Etag = *patch.Etag
 	}
@@ -130,10 +133,10 @@ func (p *Postgres) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *s
 	}
 	ctx := context.Background()
 	_, err = p.pool.Exec(ctx, `
-		UPDATE module_cdn_source SET source_kind=$3, url=$4, etag=$5, refresh_interval_sec=$6,
-			community_id=NULLIF($7::uuid, '00000000-0000-0000-0000-000000000000'::uuid), updated_at=now()
+		UPDATE module_cdn_source SET source_kind=$3, url=$4, prefix_path=$5, etag=$6, refresh_interval_sec=$7,
+			community_id=NULLIF($8::uuid, '00000000-0000-0000-0000-000000000000'::uuid), updated_at=now()
 		WHERE id=$1 AND module_id=$2`,
-		sourceID, moduleID, cur.SourceKind, cur.URL, cur.Etag, nullInt32Ptr(cur.RefreshIntervalSec), uuidOrNilPtr(cur.CommunityID))
+		sourceID, moduleID, cur.SourceKind, cur.URL, cur.PrefixPath, cur.Etag, nullInt32Ptr(cur.RefreshIntervalSec), uuidOrNilPtr(cur.CommunityID))
 	if err != nil {
 		return nil, err
 	}

internal/store/backend.go

@@ -82,29 +82,31 @@ type Backend interface {
 
 // ModulePatch is a partial update for module.
 type ModulePatch struct {
-	Name                 *string `json:"name,omitempty"`
-	Enabled              *bool   `json:"enabled,omitempty"`
-	Priority             *int    `json:"priority,omitempty"`
-	RefreshIntervalSec   *int    `json:"refresh_interval_sec,omitempty"`
-	CronExpr             *string `json:"cron_expr,omitempty"`
-	DefaultCommunityID   *string `json:"default_community_id,omitempty"`
-	DohProfileID         *string `json:"doh_profile_id,omitempty"`
+	Name               *string `json:"name,omitempty"`
+	Enabled            *bool   `json:"enabled,omitempty"`
+	Priority           *int    `json:"priority,omitempty"`
+	RefreshIntervalSec *int    `json:"refresh_interval_sec,omitempty"`
+	CronExpr           *string `json:"cron_expr,omitempty"`
+	DefaultCommunityID *string `json:"default_community_id,omitempty"`
+	DohProfileID       *string `json:"doh_profile_id,omitempty"`
 }
 
 // CDNSource is a row under a CDN module.
 type CDNSource struct {
-	ID                   string  `json:"id,omitempty"`
-	ModuleID             string  `json:"module_id,omitempty"`
-	SourceKind           string  `json:"source_kind"`
-	URL                  string  `json:"url"`
-	Etag                 string  `json:"etag"`
-	RefreshIntervalSec   *int    `json:"refresh_interval_sec"`
-	CommunityID          *string `json:"community_id"`
+	ID                 string  `json:"id,omitempty"`
+	ModuleID           string  `json:"module_id,omitempty"`
+	SourceKind         string  `json:"source_kind"`
+	URL                string  `json:"url"`
+	PrefixPath         string  `json:"prefix_path,omitempty"`
+	Etag               string  `json:"etag"`
+	RefreshIntervalSec *int    `json:"refresh_interval_sec"`
+	CommunityID        *string `json:"community_id"`
 }
 
 type CDNSourcePatch struct {
 	SourceKind         *string `json:"source_kind,omitempty"`
 	URL                *string `json:"url,omitempty"`
+	PrefixPath         *string `json:"prefix_path,omitempty"`
 	Etag               *string `json:"etag,omitempty"`
 	RefreshIntervalSec *int    `json:"refresh_interval_sec,omitempty"`
 	CommunityID        *string `json:"community_id,omitempty"`
@@ -131,10 +133,10 @@ func ValidASN(n int64) bool {
 }
 
 type DomainEntry struct {
-	ID           string  `json:"id,omitempty"`
-	ModuleID     string  `json:"module_id,omitempty"`
-	FQDN         string  `json:"fqdn"`
-	CommunityID  *string `json:"community_id"`
+	ID          string  `json:"id,omitempty"`
+	ModuleID    string  `json:"module_id,omitempty"`
+	FQDN        string  `json:"fqdn"`
+	CommunityID *string `json:"community_id"`
 }
 
 type DomainEntryPatch struct {
@@ -143,10 +145,10 @@ type DomainEntryPatch struct {
 }
 
 type IPRangeEntry struct {
-	ID           string  `json:"id,omitempty"`
-	ModuleID     string  `json:"module_id,omitempty"`
-	Prefix       string  `json:"prefix"`
-	CommunityID  *string `json:"community_id"`
+	ID          string  `json:"id,omitempty"`
+	ModuleID    string  `json:"module_id,omitempty"`
+	Prefix      string  `json:"prefix"`
+	CommunityID *string `json:"community_id"`
 }
 
 type IPRangePatch struct {
@@ -155,12 +157,12 @@ type IPRangePatch struct {
 }
 
 type DohProfile struct {
-	ID         string  `json:"id,omitempty"`
-	TenantID   string  `json:"tenant_id,omitempty"`
-	Name       string  `json:"name"`
-	URL        string  `json:"url"`
-	TimeoutMs  *int    `json:"timeout_ms"`
-	SecretRef  *string `json:"vault_secret_ref"`
+	ID        string  `json:"id,omitempty"`
+	TenantID  string  `json:"tenant_id,omitempty"`
+	Name      string  `json:"name"`
+	URL       string  `json:"url"`
+	TimeoutMs *int    `json:"timeout_ms"`
+	SecretRef *string `json:"vault_secret_ref"`
 }
 
 type DohProfilePatch struct {
@@ -202,7 +204,7 @@ type SpeakerPatch struct {
 
 // PrefixRow is one materialized prefix for GET /revisions/.../prefixes.
 type PrefixRow struct {
-	Prefix       string
-	CommunityID  *string
-	Source       string
+	Prefix      string
+	CommunityID *string
+	Source      string
 }

internal/store/memory_crud.go

@@ -22,16 +22,16 @@ func (m *Memory) CreateModule(tenantID string, in *Module) (*Module, error) {
 	}
 	id := uuid.NewString()
 	mod := &Module{
-		ID:                   id,
-		TenantID:             tenantID,
-		Type:                 in.Type,
-		Name:                 strings.TrimSpace(in.Name),
-		Enabled:              in.Enabled,
-		Priority:             in.Priority,
-		RefreshIntervalSec:   in.RefreshIntervalSec,
-		CronExpr:             in.CronExpr,
-		DefaultCommunityID:   in.DefaultCommunityID,
-		DohProfileID:         in.DohProfileID,
+		ID:                 id,
+		TenantID:           tenantID,
+		Type:               in.Type,
+		Name:               strings.TrimSpace(in.Name),
+		Enabled:            in.Enabled,
+		Priority:           in.Priority,
+		RefreshIntervalSec: in.RefreshIntervalSec,
+		CronExpr:           in.CronExpr,
+		DefaultCommunityID: in.DefaultCommunityID,
+		DohProfileID:       in.DohProfileID,
 	}
 	m.modules[id] = mod
 	return mod, nil
@@ -139,6 +139,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN
 		ModuleID:           moduleID,
 		SourceKind:         in.SourceKind,
 		URL:                strings.TrimSpace(in.URL),
+		PrefixPath:         strings.TrimSpace(in.PrefixPath),
 		Etag:               in.Etag,
 		RefreshIntervalSec: in.RefreshIntervalSec,
 		CommunityID:        in.CommunityID,
@@ -166,6 +167,9 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
 	if patch.URL != nil {
 		s.URL = strings.TrimSpace(*patch.URL)
 	}
+	if patch.PrefixPath != nil {
+		s.PrefixPath = strings.TrimSpace(*patch.PrefixPath)
+	}
 	if patch.Etag != nil {
 		s.Etag = *patch.Etag
 	}
@@ -654,7 +658,7 @@ func (m *Memory) CreatePeer(tenantID string, in *BGPPeer) (*BGPPeer, error) {
 	p := &BGPPeer{
 		ID: id, TenantID: tenantID, SpeakerID: in.SpeakerID, Name: in.Name,
 		Neighbor: neighbor, RemoteASN: in.RemoteASN,
-		Enabled:  EffectivePeerEnabledOnCreate(in.Enabled, in.SessionState),
+		Enabled:      EffectivePeerEnabledOnCreate(in.Enabled, in.SessionState),
 		SessionState: in.SessionState, PoliciesJSON: in.PoliciesJSON,
 	}
 	m.peers[id] = p