feat: add CSV import and export functionality for module entries. Implement endpoints for exporting and importing module entries in CSV format, supporting types AS_PREFIXES, DOMAINS, and IP_RANGES. Enhance UI with buttons for CSV operations, improving user experience in managing module data.
CI / changes (push) Successful in 6s
CI / openapi (push) Successful in 22s
CI / go (push) Successful in 50s
CI / docker-web (deploy/docker/evobgp-web/Dockerfile, , evobgp-web) (push) Successful in 1m4s
CI / docker-web (deploy/docker/evobgp-web/Dockerfile, evobgp-all, evobgp-web-all) (push) Successful in 1m5s
CI / docker-bird (push) Has been skipped
CI / bird2 (push) Successful in 15s
CI / docker-go-prime (push) Successful in 24s
CI / docker-go (deploy/docker/evobgp-agent/Dockerfile, , evobgp-agent) (push) Successful in 1m1s
CI / docker-go (evobgp-all, 1, deploy/docker/gobinary/Dockerfile, , evobgp-all) (push) Successful in 2m7s
CI / docker-go (evobgp-api, 1, deploy/docker/gobinary/Dockerfile, , evobgp-api) (push) Successful in 1m22s
CI / docker-go (evobgp-deploy, 0, deploy/docker/gobinary/Dockerfile, , evobgp-deploy) (push) Successful in 1m26s
CI / docker-go (evobgp-ingest, 0, deploy/docker/gobinary/Dockerfile, , evobgp-ingest) (push) Successful in 1m23s
CI / docker-go (evobgp-node, 0, deploy/docker/gobinary/Dockerfile, , evobgp-node) (push) Successful in 1m6s
CI / docker-go (evobgp-render, 0, deploy/docker/gobinary/Dockerfile, , evobgp-render) (push) Successful in 1m28s
CI / docker-go (evobgp-scheduler, 0, deploy/docker/gobinary/Dockerfile, , evobgp-scheduler) (push) Successful in 1m21s

This commit is contained in:
Denozordec
2026-04-06 18:58:52 +07:00
parent 233f29b437
commit bae2d68803
4 changed files with 575 additions and 4 deletions
+68
View File
@@ -1515,6 +1515,74 @@ paths:
default:
$ref: "#/components/responses/DefaultProblem"
/v1/modules/{module_id}/entries.csv:
parameters:
- $ref: "#/components/parameters/TenantId"
- $ref: "#/components/parameters/ModuleId"
get:
tags: [Modules]
summary: Экспорт записей модуля в CSV
description: |
Доступно для типов модулей `AS_PREFIXES`, `DOMAINS`, `IP_RANGES`.
Возвращает CSV с колонками:
- AS: `asn,community`
- Домены: `domain,community`
- IP ranges: `ipRange,community`
operationId: exportModuleEntriesCsv
responses:
"200":
description: CSV-файл записей модуля.
content:
text/csv:
schema:
type: string
"404":
$ref: "#/components/responses/NotFound"
"422":
$ref: "#/components/responses/UnprocessableEntity"
default:
$ref: "#/components/responses/DefaultProblem"
post:
tags: [Modules]
summary: Импорт записей модуля из CSV
description: |
Импортирует CSV в модуль типов `AS_PREFIXES`, `DOMAINS`, `IP_RANGES`.
Поддерживаемые заголовки:
- `asn,community`
- `domain,community`
- `ipRange,community`
В поле `community` можно передавать либо ID community, либо её значение.
operationId: importModuleEntriesCsv
parameters:
- $ref: "#/components/parameters/IdempotencyKey"
requestBody:
required: true
content:
text/csv:
schema:
type: string
responses:
"200":
description: Импорт завершён.
content:
application/json:
schema:
type: object
required: [imported, module_type]
properties:
imported:
type: integer
minimum: 0
module_type:
type: string
"404":
$ref: "#/components/responses/NotFound"
"422":
$ref: "#/components/responses/UnprocessableEntity"
default:
$ref: "#/components/responses/DefaultProblem"
/v1/doh-profiles:
get:
tags: [DoH profiles]
+246
View File
@@ -1,6 +1,7 @@
package httpapi
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
@@ -38,6 +39,8 @@ func (s *Server) registerCRUDRoutes(m *http.ServeMux) {
m.HandleFunc("POST /modules/{module_id}/ip-range-entries", s.handlePostIPRange)
m.HandleFunc("PATCH /modules/{module_id}/ip-range-entries/{entry_id}", s.handlePatchIPRange)
m.HandleFunc("DELETE /modules/{module_id}/ip-range-entries/{entry_id}", s.handleDeleteIPRange)
m.HandleFunc("GET /modules/{module_id}/entries.csv", s.handleExportModuleEntriesCSV)
m.HandleFunc("POST /modules/{module_id}/entries.csv", s.handleImportModuleEntriesCSV)
m.HandleFunc("GET /doh-profiles", s.handleListDoh)
m.HandleFunc("POST /doh-profiles", s.handlePostDoh)
@@ -535,6 +538,249 @@ func (s *Server) handleDeleteIPRange(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
func (s *Server) handleExportModuleEntriesCSV(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok || !s.requireAtLeast(w, a, "viewer") {
return
}
moduleID := r.PathValue("module_id")
mod, err := s.store.GetModule(a.TenantID, moduleID)
if err != nil {
writeStoreErr(w, err)
return
}
communities, err := s.store.ListCommunities(a.TenantID)
if err != nil {
writeStoreErr(w, err)
return
}
communityByID := make(map[string]string, len(communities))
for _, c := range communities {
communityByID[c.ID] = strings.TrimSpace(c.Community)
}
records := make([][]string, 0, 64)
switch mod.Type {
case "AS_PREFIXES":
records = append(records, []string{"asn", "community"})
list, err := s.store.ListASEntries(a.TenantID, moduleID)
if err != nil {
writeStoreErr(w, err)
return
}
for _, x := range list {
community := ""
if x.CommunityID != nil {
community = communityByID[*x.CommunityID]
}
records = append(records, []string{strconv.FormatInt(x.ASN, 10), community})
}
case "DOMAINS":
records = append(records, []string{"domain", "community"})
list, err := s.store.ListDomainEntries(a.TenantID, moduleID)
if err != nil {
writeStoreErr(w, err)
return
}
for _, x := range list {
community := ""
if x.CommunityID != nil {
community = communityByID[*x.CommunityID]
}
records = append(records, []string{x.FQDN, community})
}
case "IP_RANGES":
records = append(records, []string{"ipRange", "community"})
list, err := s.store.ListIPRangeEntries(a.TenantID, moduleID)
if err != nil {
writeStoreErr(w, err)
return
}
for _, x := range list {
community := ""
if x.CommunityID != nil {
community = communityByID[*x.CommunityID]
}
records = append(records, []string{x.Prefix, community})
}
default:
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv import/export is supported only for AS_PREFIXES, DOMAINS, IP_RANGES")
return
}
w.Header().Set("Content-Type", "text/csv; charset=utf-8")
w.Header().Set("Content-Disposition", `attachment; filename="module-entries.csv"`)
w.WriteHeader(http.StatusOK)
cw := csv.NewWriter(w)
for _, rec := range records {
if err := cw.Write(rec); err != nil {
return
}
}
cw.Flush()
}
func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok || !s.requireAtLeast(w, a, "editor") {
return
}
moduleID := r.PathValue("module_id")
mod, err := s.store.GetModule(a.TenantID, moduleID)
if err != nil {
writeStoreErr(w, err)
return
}
cr := csv.NewReader(io.LimitReader(r.Body, 8<<20))
cr.TrimLeadingSpace = true
cr.FieldsPerRecord = -1
rows, err := cr.ReadAll()
if err != nil {
writeProblem(w, http.StatusBadRequest, "Bad Request", "invalid csv")
return
}
if len(rows) == 0 {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty")
return
}
communities, err := s.store.ListCommunities(a.TenantID)
if err != nil {
writeStoreErr(w, err)
return
}
communityIDByID := make(map[string]string, len(communities))
communityIDByValue := make(map[string]string, len(communities))
for _, c := range communities {
communityIDByID[c.ID] = c.ID
communityIDByValue[strings.TrimSpace(c.Community)] = c.ID
}
resolveCommunity := func(raw string, required bool) (*string, error) {
v := strings.TrimSpace(raw)
if v == "" {
if required {
return nil, fmt.Errorf("community is required")
}
return nil, nil
}
if id, ok := communityIDByID[v]; ok {
return &id, nil
}
if id, ok := communityIDByValue[v]; ok {
return &id, nil
}
return nil, fmt.Errorf("unknown community %q", v)
}
start := 0
if len(rows[0]) >= 2 {
key := strings.ToLower(strings.TrimSpace(rows[0][0]))
switch key {
case "asn", "domain", "iprange":
start = 1
}
}
imported := 0
switch mod.Type {
case "AS_PREFIXES":
for i := start; i < len(rows); i++ {
rec := rows[i]
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
continue
}
if len(rec) < 2 {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: expected 2 columns", i+1))
return
}
asn, err := strconv.ParseInt(strings.TrimSpace(rec[0]), 10, 64)
if err != nil || asn <= 0 {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: invalid asn", i+1))
return
}
cid, err := resolveCommunity(rec[1], false)
if err != nil {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: %v", i+1, err))
return
}
_, err = s.store.CreateASEntry(a.TenantID, moduleID, &store.ASEntry{ASN: asn, CommunityID: cid})
if err != nil {
writeStoreErr(w, err)
return
}
imported++
}
case "DOMAINS":
for i := start; i < len(rows); i++ {
rec := rows[i]
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
continue
}
if len(rec) < 2 {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: expected 2 columns", i+1))
return
}
fqdn := strings.TrimSpace(rec[0])
if fqdn == "" {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: domain is required", i+1))
return
}
cid, err := resolveCommunity(rec[1], false)
if err != nil {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: %v", i+1, err))
return
}
_, err = s.store.CreateDomainEntry(a.TenantID, moduleID, &store.DomainEntry{FQDN: fqdn, CommunityID: cid})
if err != nil {
writeStoreErr(w, err)
return
}
imported++
}
case "IP_RANGES":
for i := start; i < len(rows); i++ {
rec := rows[i]
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
continue
}
if len(rec) < 2 {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: expected 2 columns", i+1))
return
}
prefix := strings.TrimSpace(rec[0])
if prefix == "" {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: ipRange is required", i+1))
return
}
cid, err := resolveCommunity(rec[1], true)
if err != nil {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", fmt.Sprintf("line %d: %v", i+1, err))
return
}
_, err = s.store.CreateIPRangeEntry(a.TenantID, moduleID, &store.IPRangeEntry{Prefix: prefix, CommunityID: cid})
if err != nil {
writeStoreErr(w, err)
return
}
imported++
}
if imported > 0 {
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "ip_range_import_csv")
}
default:
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv import/export is supported only for AS_PREFIXES, DOMAINS, IP_RANGES")
return
}
writeJSON(w, http.StatusOK, map[string]any{
"imported": imported,
"module_type": mod.Type,
})
}
func (s *Server) handleListDoh(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok || !s.requireAtLeast(w, a, "viewer") {
+105
View File
@@ -0,0 +1,105 @@
package httpapi
import (
"encoding/csv"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) {
srv, err := New(Options{
InsecureDev: true,
SeedDemo: true,
})
if err != nil {
t.Fatal(err)
}
defer srv.Close()
tenant, _, modIP, _, _ := srv.Store().DemoIDs()
srv.apiKeys = parseAPIKeysSpec("opkey|" + tenant + "|operator")
ts := httptest.NewServer(srv.Handler())
defer ts.Close()
client := ts.Client()
base := ts.URL
reqList, _ := http.NewRequest(http.MethodGet, base+"/v1/communities?limit=10", nil)
reqList.Header.Set("Authorization", "Bearer opkey")
respList, err := client.Do(reqList)
if err != nil {
t.Fatal(err)
}
defer respList.Body.Close()
if respList.StatusCode != http.StatusOK {
b, _ := io.ReadAll(respList.Body)
t.Fatalf("communities status %d: %s", respList.StatusCode, b)
}
var listBody struct {
Items []struct {
Community string `json:"community"`
} `json:"items"`
}
if err := json.NewDecoder(respList.Body).Decode(&listBody); err != nil {
t.Fatal(err)
}
if len(listBody.Items) == 0 {
t.Fatal("expected seeded community")
}
csvBody := "\"ipRange\",\"community\"\n\"10.254.1.254/32\",\"" + listBody.Items[0].Community + "\"\n\"160.79.104.0/23\",\"" + listBody.Items[0].Community + "\"\n"
reqImport, _ := http.NewRequest(http.MethodPost, base+"/v1/modules/"+modIP+"/entries.csv", strings.NewReader(csvBody))
reqImport.Header.Set("Authorization", "Bearer opkey")
reqImport.Header.Set("Content-Type", "text/csv")
respImport, err := client.Do(reqImport)
if err != nil {
t.Fatal(err)
}
defer respImport.Body.Close()
if respImport.StatusCode != http.StatusOK {
b, _ := io.ReadAll(respImport.Body)
t.Fatalf("import status %d: %s", respImport.StatusCode, b)
}
var importBody struct {
Imported int `json:"imported"`
}
if err := json.NewDecoder(respImport.Body).Decode(&importBody); err != nil {
t.Fatal(err)
}
if importBody.Imported != 2 {
t.Fatalf("expected imported=2, got %d", importBody.Imported)
}
reqExport, _ := http.NewRequest(http.MethodGet, base+"/v1/modules/"+modIP+"/entries.csv", nil)
reqExport.Header.Set("Authorization", "Bearer opkey")
respExport, err := client.Do(reqExport)
if err != nil {
t.Fatal(err)
}
defer respExport.Body.Close()
if respExport.StatusCode != http.StatusOK {
b, _ := io.ReadAll(respExport.Body)
t.Fatalf("export status %d: %s", respExport.StatusCode, b)
}
if got := respExport.Header.Get("Content-Type"); !strings.Contains(got, "text/csv") {
t.Fatalf("unexpected content-type: %q", got)
}
raw, err := io.ReadAll(respExport.Body)
if err != nil {
t.Fatal(err)
}
records, err := csv.NewReader(strings.NewReader(string(raw))).ReadAll()
if err != nil {
t.Fatal(err)
}
if len(records) < 3 {
t.Fatalf("expected at least 3 csv rows, got %d", len(records))
}
if records[0][0] != "ipRange" || records[0][1] != "community" {
t.Fatalf("unexpected header: %#v", records[0])
}
}
+156 -4
View File
@@ -3,7 +3,7 @@
import { page } from '$app/state';
import { goto } from '$app/navigation';
import { resolve } from '$app/paths';
import { apiJSON, apiMutate } from '$lib/api/client.js';
import { apiFetch, apiJSON, apiMutate } from '$lib/api/client.js';
import type {
ModuleRow,
ModulePatch,
@@ -72,6 +72,8 @@
import Trash2 from '@lucide/svelte/icons/trash-2';
import RefreshCw from '@lucide/svelte/icons/refresh-cw';
import Save from '@lucide/svelte/icons/save';
import Upload from '@lucide/svelte/icons/upload';
import Download from '@lucide/svelte/icons/download';
const moduleId = $derived(page.params.moduleId);
@@ -132,6 +134,28 @@
// Refresh
let refreshing = $state(false);
let csvImporting = $state(false);
let csvExporting = $state(false);
let csvFileInput = $state<HTMLInputElement | null>(null);
function supportsCsvIO(type: ModuleRow['type'] | null | undefined): boolean {
return type === 'AS_PREFIXES' || type === 'DOMAINS' || type === 'IP_RANGES';
}
function sanitizeFilenamePart(v: string): string {
const cleaned = v
.trim()
.toLowerCase()
.replace(/[^a-z0-9._-]+/g, '-')
.replace(/-+/g, '-')
.replace(/^[-_.]+|[-_.]+$/g, '');
return cleaned || 'module';
}
async function readErrorText(res: Response): Promise<string> {
const body = (await res.text()).trim();
return body || `HTTP ${res.status}`;
}
async function loadMod() {
loadingMod = true;
@@ -245,6 +269,67 @@
}
}
async function exportEntriesCsv() {
if (!mod || !supportsCsvIO(mod.type) || csvExporting) return;
csvExporting = true;
try {
const res = await apiFetch(`/v1/modules/${moduleId}/entries.csv`, {
method: 'GET',
headers: { Accept: 'text/csv' }
});
if (!res.ok) {
toast.error(await readErrorText(res));
return;
}
const blob = await res.blob();
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
const safeModuleName = sanitizeFilenamePart(mod.name);
a.href = url;
a.download = `${safeModuleName}-${mod.type.toLowerCase()}-entries.csv`;
document.body.appendChild(a);
a.click();
a.remove();
URL.revokeObjectURL(url);
} catch (e) {
toast.error(e instanceof Error ? e.message : String(e));
} finally {
csvExporting = false;
}
}
function openImportCsvPicker() {
if (!mod || !supportsCsvIO(mod.type) || csvImporting) return;
csvFileInput?.click();
}
async function handleImportCsvChange(event: Event) {
const input = event.currentTarget as HTMLInputElement | null;
const file = input?.files?.[0];
if (!mod || !supportsCsvIO(mod.type) || !file || csvImporting) return;
csvImporting = true;
try {
const fileText = await file.text();
const res = await apiFetch(`/v1/modules/${moduleId}/entries.csv`, {
method: 'POST',
headers: { 'Content-Type': 'text/csv' },
body: fileText
});
if (!res.ok) {
toast.error(await readErrorText(res));
return;
}
const payload = (await res.json()) as { imported?: number };
toast.success(`Импортировано записей: ${payload.imported ?? 0}`);
await loadEntries();
} catch (e) {
toast.error(e instanceof Error ? e.message : String(e));
} finally {
csvImporting = false;
if (input) input.value = '';
}
}
// --- AS Entries ---
function openAsCreate() {
asEdit = null;
@@ -497,6 +582,13 @@
<div class="text-muted-foreground flex h-32 items-center justify-center">Загрузка…</div>
{:else if mod}
<div class="space-y-6">
<input
class="hidden"
type="file"
accept=".csv,text/csv"
bind:this={csvFileInput}
onchange={handleImportCsvChange}
/>
<!-- Header -->
<div class="flex flex-col gap-3 xl:flex-row xl:items-start xl:justify-between">
<div class="flex min-w-0 items-start gap-3">
@@ -564,7 +656,27 @@
Номер AS и community; имя, число префиксов и дата обновляются при успешном обновлении модуля (RIPEstat)
</CardDescription>
</div>
<Button size="sm" class="shrink-0 self-start sm:self-auto" onclick={openAsCreate}><Plus />Добавить</Button>
<div class="flex shrink-0 flex-wrap items-center justify-end gap-2 self-start sm:self-auto">
<Button
variant="outline"
size="sm"
onclick={openImportCsvPicker}
disabled={!supportsCsvIO(mod.type) || csvImporting || csvExporting}
>
<Upload />
{csvImporting ? 'Импорт…' : 'Импорт CSV'}
</Button>
<Button
variant="outline"
size="sm"
onclick={exportEntriesCsv}
disabled={!supportsCsvIO(mod.type) || csvExporting || csvImporting}
>
<Download />
{csvExporting ? 'Экспорт…' : 'Экспорт CSV'}
</Button>
<Button size="sm" onclick={openAsCreate}><Plus />Добавить</Button>
</div>
</CardHeader>
<CardContent class="p-0">
<Table>
@@ -672,7 +784,27 @@
<CardTitle class="text-base">Домены</CardTitle>
<CardDescription>FQDN для резолвинга через DoH</CardDescription>
</div>
<Button size="sm" class="shrink-0 self-start sm:self-auto" onclick={openDomainCreate}><Plus />Добавить</Button>
<div class="flex shrink-0 flex-wrap items-center justify-end gap-2 self-start sm:self-auto">
<Button
variant="outline"
size="sm"
onclick={openImportCsvPicker}
disabled={!supportsCsvIO(mod.type) || csvImporting || csvExporting}
>
<Upload />
{csvImporting ? 'Импорт…' : 'Импорт CSV'}
</Button>
<Button
variant="outline"
size="sm"
onclick={exportEntriesCsv}
disabled={!supportsCsvIO(mod.type) || csvExporting || csvImporting}
>
<Download />
{csvExporting ? 'Экспорт…' : 'Экспорт CSV'}
</Button>
<Button size="sm" onclick={openDomainCreate}><Plus />Добавить</Button>
</div>
</CardHeader>
<CardContent class="p-0">
<Table>
@@ -711,7 +843,27 @@
<CardTitle class="text-base">IP-диапазоны</CardTitle>
<CardDescription>Статические CIDR для анонса</CardDescription>
</div>
<Button size="sm" class="shrink-0 self-start sm:self-auto" onclick={openIpCreate}><Plus />Добавить</Button>
<div class="flex shrink-0 flex-wrap items-center justify-end gap-2 self-start sm:self-auto">
<Button
variant="outline"
size="sm"
onclick={openImportCsvPicker}
disabled={!supportsCsvIO(mod.type) || csvImporting || csvExporting}
>
<Upload />
{csvImporting ? 'Импорт…' : 'Импорт CSV'}
</Button>
<Button
variant="outline"
size="sm"
onclick={exportEntriesCsv}
disabled={!supportsCsvIO(mod.type) || csvExporting || csvImporting}
>
<Download />
{csvExporting ? 'Экспорт…' : 'Экспорт CSV'}
</Button>
<Button size="sm" onclick={openIpCreate}><Plus />Добавить</Button>
</div>
</CardHeader>
<CardContent class="p-0">
<Table>