denozord/EvoBGP
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 131 Wiki Activity

refactor(web): remove deprecated dashboard components and enhance KPI grid
quality / commitlint (push) Skipped
Details
quality / changes (push) Successful in 9s
Details
quality / docker-check (push) Skipped
Details
quality / openapi (push) Successful in 46s
Details
quality / web (push) Successful in 1m16s
Details
quality / go (push) Successful in 2m42s
Details
quality / bird2 (push) Successful in 16s
Details
CD / quality (push) Successful in 5m19s
Details
CD / publish (push) Successful in 7m19s
Details

Browse Source 
- Deleted unused components: `DashboardActivityTimeline`, `DashboardFramePanel`, `DashboardModulesGrid`, `DashboardRecentJobsGrid`, and `DashboardRecentRevisionsGrid` to streamline the dashboard.
- Updated `DashboardKpiGrid` to improve KPI display logic, including progress indicators and enhanced badge functionality.
- Refactored `DashboardNetworkHealth` to provide better status representation based on loading states and network conditions.
- Introduced new properties for KPI cards to support progress tracking and improved visual feedback.

This cleanup aims to enhance performance and maintainability of the dashboard while providing a better user experience.
This commit is contained in:
Denozordec
2026-08-31 10:15:59 +07:00
parent e469c421ca
commit dc803bcb34
51 changed files with 1895 additions and 1078 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/web/src/components/dashboard/dashboard-activity-timeline.tsx
-102
View File
@@ -1,102 +0,0 @@
import { AlertTriangle, CheckCircle, Info } from 'lucide-react'
import { DashboardFramePanel } from '@/components/dashboard/dashboard-frame-panel'
import { Badge } from '@/components/reui/badge'
import {
Timeline,
TimelineContent,
TimelineHeader,
TimelineIndicator,
TimelineItem,
TimelineSeparator,
TimelineTitle,
} from '@/components/reui/timeline'
import { cn } from '@evobgp/ui/lib/utils'
import { recentPlatformActivity } from '@/lib/metrics'
import type { JobRow, PeerRow, RevisionRow, SpeakerRow } from '@/types/api'
const KIND_META = {
job: { icon: Info, className: 'text-info' },
revision: { icon: CheckCircle, className: 'text-success' },
network: { icon: AlertTriangle, className: 'text-warning' },
} as const
function statusBadgeVariant(status: string) {
const s = status.toLowerCase()
if (['ok', 'success', 'completed', 'done'].includes(s)) return 'success-light' as const
if (['running', 'queued', 'pending'].includes(s)) return 'info-light' as const
if (['warning', 'mismatch'].includes(s)) return 'warning-light' as const
if (['failed', 'error', 'cancelled'].includes(s)) return 'destructive-light' as const
return 'outline' as const
}
export function DashboardActivityTimeline({
jobs,
revisions,
peers,
speakers,
loading,
}: {
jobs: JobRow[]
revisions: RevisionRow[]
peers: PeerRow[]
speakers: SpeakerRow[]
loading?: boolean
}) {
const items = recentPlatformActivity(jobs, revisions, peers, speakers, 6)
return (
<DashboardFramePanel
title="Недавняя активность"
description="Задачи, ревизии и сетевые события"
className="h-full min-w-0"
>
{loading ? (
<p className="text-muted-foreground px-4 py-6 text-sm">Загрузка…</p>
) : items.length === 0 ? (
<p className="text-muted-foreground px-4 py-6 text-sm">Нет недавних событий</p>
) : (
<div className="px-4 py-4">
<Timeline defaultValue={items.length}>
{items.map((item, index) => {
const meta = KIND_META[item.kind]
const Icon = meta.icon
return (
<TimelineItem
key={item.id}
step={index + 1}
className="group-data-[orientation=vertical]/timeline:ms-8 group-data-[orientation=vertical]/timeline:not-last:pb-4"
>
<TimelineHeader>
<TimelineSeparator className="bg-border! group-data-[orientation=vertical]/timeline:-left-6 group-data-[orientation=vertical]/timeline:top-2 group-data-[orientation=vertical]/timeline:h-[calc(100%-1.5rem)] group-data-[orientation=vertical]/timeline:translate-y-5" />
<TimelineIndicator className="border-none bg-transparent group-data-[orientation=vertical]/timeline:-left-6">
<span
className={cn(
'bg-muted/70 flex size-7 items-center justify-center rounded-full',
meta.className,
)}
>
<Icon className="size-3.5" aria-hidden />
</span>
</TimelineIndicator>
</TimelineHeader>
<TimelineContent className="min-w-0 pb-1 text-foreground">
<TimelineTitle className="text-sm leading-snug font-normal break-words">
{item.message}
</TimelineTitle>
<div className="mt-2">
<Badge variant={statusBadgeVariant(item.status)} size="sm">
{item.statusLabel ?? item.status}
</Badge>
</div>
</TimelineContent>
</TimelineItem>
)
})}
</Timeline>
</div>
)}
</DashboardFramePanel>
)
}
apps/web/src/components/dashboard/dashboard-frame-panel.tsx
-36
View File
@@ -1,36 +0,0 @@
import type { ReactNode } from 'react'
import {
FrameSection,
panelCardContentFlushClassName,
} from '@/components/reui-kit'
import { cn } from '@evobgp/ui/lib/utils'
/** Frame panel for dashboard sections. Preview: https://reui.io/docs/components/base/frame */
export function DashboardFramePanel({
title,
description,
actions,
children,
className,
contentClassName,
}: {
title?: string
description?: string
actions?: ReactNode
children: ReactNode
className?: string
contentClassName?: string
}) {
return (
<FrameSection
title={title}
description={description}
actions={actions}
className={cn('h-full', className)}
contentClassName={cn(panelCardContentFlushClassName, contentClassName)}
>
{children}
</FrameSection>
)
}
apps/web/src/components/dashboard/dashboard-kpi-grid.tsx
+30 -34
View File
@@ -3,8 +3,6 @@ import {
Boxes, Boxes,
ListChecks, ListChecks,
Network, Network,
ServerCog,
Share2,
} from 'lucide-react' } from 'lucide-react'
import type { ReactNode } from 'react' import type { ReactNode } from 'react'
@@ -16,6 +14,11 @@ import type { JobRow, ModuleRow, PeerRow, SpeakerRow } from '@/types/api'
type KpiCard = KpiStatItem & { icon: ReactNode } type KpiCard = KpiStatItem & { icon: ReactNode }
function ratioPercent(part: number, total: number): number | undefined {
if (total <= 0) return undefined
return Math.round((part / total) * 100)
}
function buildKpis({ function buildKpis({
modules, modules,
peers, peers,
@@ -40,17 +43,34 @@ function buildKpis({
).length ).length
const offlineSpeakers = Math.max(0, speakers.length - network.speakersOnline) const offlineSpeakers = Math.max(0, speakers.length - network.speakersOnline)
const riskCount = network.peersMismatch + failedJobs + offlineSpeakers const riskCount = network.peersMismatch + failedJobs + offlineSpeakers
const disabledModules = Math.max(0, modules.length - enabledModules)
return [ return [
{ {
id: 'modules', id: 'modules',
icon: <Boxes aria-hidden />, icon: <Boxes aria-hidden />,
iconClassName: 'text-primary', iconClassName: 'text-primary',
value: loading ? '—' : `${enabledModules}/${modules.length || 0}`, value: loading ? '—' : enabledModules,
label: 'Модули активны', label: 'Модули активны',
progress: loading ? undefined : ratioPercent(enabledModules, modules.length),
footer: ( footer: (
<Badge variant="primary-light" size="sm"> <Badge
{loading ? '…' : `${modules.length} всего`} variant={
loading || modules.length === 0
? 'outline'
: disabledModules === 0
? 'success-light'
: 'warning-light'
}
size="sm"
>
{loading
? '…'
: modules.length === 0
? 'нет модулей'
: disabledModules === 0
? 'все активны'
: `${disabledModules} выкл`}
</Badge> </Badge>
), ),
}, },
@@ -60,6 +80,7 @@ function buildKpis({
iconClassName: 'text-info', iconClassName: 'text-info',
value: loading || bgpPct === null ? '—' : `${bgpPct}%`, value: loading || bgpPct === null ? '—' : `${bgpPct}%`,
label: 'BGP готовность', label: 'BGP готовность',
progress: loading || bgpPct === null ? undefined : bgpPct,
footer: ( footer: (
<Badge <Badge
variant={ variant={
@@ -73,34 +94,9 @@ function buildKpis({
> >
{loading || bgpPct === null {loading || bgpPct === null
? 'нет включённых пиров' ? 'нет включённых пиров'
: `${network.peersEstablished} установлено`} : bgpPct >= 90
</Badge> ? 'сессии в норме'
), : `${network.peersEstablished} установлено`}
},
{
id: 'peers',
icon: <Share2 aria-hidden />,
iconClassName: 'text-success',
value: loading ? '—' : `${network.peersEstablished}/${peersEnabled}`,
label: 'Пиры установлены',
footer: (
<Badge variant="success-light" size="sm">
{loading ? '…' : `${network.peersTotal} в каталоге`}
</Badge>
),
},
{
id: 'speakers',
icon: <ServerCog aria-hidden />,
iconClassName: 'text-warning',
value: loading ? '—' : `${network.speakersOnline}/${network.speakersTotal}`,
label: 'Спикеры в сети',
footer: (
<Badge
variant={network.speakersOnline === network.speakersTotal ? 'success-light' : 'warning-light'}
size="sm"
>
{loading ? '…' : 'в сети'}
</Badge> </Badge>
), ),
}, },
@@ -112,7 +108,7 @@ function buildKpis({
label: 'Активные задачи', label: 'Активные задачи',
footer: ( footer: (
<Badge variant={running > 0 ? 'info-light' : 'outline'} size="sm"> <Badge variant={running > 0 ? 'info-light' : 'outline'} size="sm">
{loading ? '…' : `${jobs.length} в выборке`} {loading ? '…' : running > 0 ? 'выполняются' : 'очередь пуста'}
</Badge> </Badge>
), ),
}, },
apps/web/src/components/dashboard/dashboard-modules-grid.tsx
-117
View File
@@ -1,117 +0,0 @@
import { useMemo, useState } from 'react'
import { Link, useNavigate } from '@tanstack/react-router'
import { BoxesIcon, PlusIcon, SearchIcon } from 'lucide-react'
import { CategoryBadge } from '@/components/category-badge'
import { DataGridNameCell } from '@/components/data-grid-cell'
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
import type { FilterField, FilterQuery } from '@/components/reui/filters/filters-types'
import {
ResourcePage,
createTextFilterQuery,
type DataGridColumnDef,
} from '@/components/reui-kit'
import { Button } from '@evobgp/ui/components/button'
import { moduleTypeRu } from '@/lib/ui-labels'
import type { ModuleRow } from '@/types/api'
const MODULE_TABS = [
{ id: 'all', label: 'Все' },
{ id: 'enabled', label: 'Вкл' },
{ id: 'disabled', label: 'Выкл' },
]
const filterFields: FilterField[] = [
{
id: 'search',
label: 'Поиск',
icon: <SearchIcon className="size-3.5" aria-hidden />,
type: 'text',
placeholder: 'Поиск модулей…',
},
]
function getFilterFieldValue(item: ModuleRow, field: string): unknown {
if (field === 'search') {
return `${item.name} ${item.type} ${moduleTypeRu(item.type)}`
}
return undefined
}
function tabFilter(item: ModuleRow, tabId: string): boolean {
if (tabId === 'enabled') return item.enabled !== false
if (tabId === 'disabled') return item.enabled === false
return true
}
export function DashboardModulesGrid({
modules,
isLoading = false,
}: {
modules: ModuleRow[]
isLoading?: boolean
}) {
const navigate = useNavigate()
const [filterQuery, setFilterQuery] = useState<FilterQuery>(() =>
createTextFilterQuery('search'),
)
const columns = useMemo<DataGridColumnDef<ModuleRow>[]>(
() => [
{
accessorKey: 'name',
id: 'name',
header: ({ column }) => <DataGridColumnHeader column={column} title="Модуль" />,
cell: ({ row }) => <DataGridNameCell icon={BoxesIcon} title={row.original.name} />,
minSize: 180,
meta: { headerTitle: 'Модуль' },
},
{
accessorKey: 'type',
id: 'type',
header: ({ column }) => <DataGridColumnHeader column={column} title="Тип" />,
cell: ({ row }) => <CategoryBadge>{moduleTypeRu(row.original.type)}</CategoryBadge>,
meta: { headerTitle: 'Тип' },
},
{
accessorKey: 'priority',
id: 'priority',
header: ({ column }) => <DataGridColumnHeader column={column} title="Приоритет" />,
cell: ({ row }) => (
<span className="font-mono text-sm tabular-nums">{row.original.priority}</span>
),
size: 88,
meta: { headerTitle: 'Приоритет' },
},
],
[],
)
return (
<ResourcePage
title="Модули"
description="Поиск и быстрый переход к настройке"
tabs={MODULE_TABS}
tabFilter={tabFilter}
filterFields={filterFields}
filterQuery={filterQuery}
onFilterQueryChange={setFilterQuery}
onClearFilters={() => setFilterQuery(createTextFilterQuery('search'))}
getFilterFieldValue={getFilterFieldValue}
columns={columns}
data={modules}
getRowId={(row) => row.id}
isLoading={isLoading}
primaryAction={
<Button variant="outline" size="sm" render={<Link to="/modules" search={{ create: true }} />}>
<PlusIcon />
Создать
</Button>
}
onRowClick={(row) =>
void navigate({ to: '/modules/$moduleId', params: { moduleId: row.id } })
}
emptyState={{ title: 'Нет модулей по выбранным фильтрам.' }}
/>
)
}
apps/web/src/components/dashboard/dashboard-network-health.tsx
+22 -2
View File
@@ -60,8 +60,28 @@ export function DashboardNetworkHealth({
label: mode === 'peers' ? 'Утилизация пиров' : 'Спикеры в сети', label: mode === 'peers' ? 'Утилизация пиров' : 'Спикеры в сети',
percent: loading ? 0 : utilization, percent: loading ? 0 : utilization,
badge: ( badge: (
<Badge variant="outline" radius="full" className="h-6 px-2 text-[10px]"> <Badge
{loading ? '…' : `${established}/${total}`} variant={
loading || total === 0
? 'outline'
: offline === 0
? 'success-light'
: 'warning-light'
}
size="sm"
radius="full"
>
{loading
? '…'
: total === 0
? 'нет данных'
: offline === 0
? mode === 'peers'
? 'все установлены'
: 'все в сети'
: mode === 'peers'
? `${offline} не установлены`
: `${offline} офлайн`}
</Badge> </Badge>
), ),
}} }}
apps/web/src/components/dashboard/dashboard-recent-jobs-grid.tsx
-62
View File
@@ -1,62 +0,0 @@
import { useMemo } from 'react'
import { ListTodo } from 'lucide-react'
import { DataGridNameCell } from '@/components/data-grid-cell'
import { StatusBadge } from '@/components/status-badge'
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
import { FrameDataGrid, type DataGridColumnDef } from '@/components/reui-kit'
import { jobKindRu } from '@/lib/ui-labels'
import type { JobRow } from '@/types/api'
export function DashboardRecentJobsGrid({
jobs,
nameById,
isLoading = false,
}: {
jobs: JobRow[]
nameById: Map<string, string>
isLoading?: boolean
}) {
const data = useMemo(() => jobs.slice(0, 8), [jobs])
const columns = useMemo<DataGridColumnDef<JobRow>[]>(
() => [
{
accessorKey: 'kind',
header: ({ column }) => <DataGridColumnHeader column={column} title="Вид" />,
cell: ({ row }) => (
<DataGridNameCell
icon={ListTodo}
title={jobKindRu(row.original.kind)}
subtitle={
row.original.meta?.module_id
? (nameById.get(String(row.original.meta.module_id)) ?? undefined)
: undefined
}
/>
),
meta: { headerTitle: 'Вид' },
},
{
accessorKey: 'status',
header: ({ column }) => <DataGridColumnHeader column={column} title="Статус" />,
cell: ({ row }) => <StatusBadge status={row.original.status} />,
meta: { headerTitle: 'Статус' },
},
],
[nameById],
)
return (
<FrameDataGrid
title="Недавние задачи"
description="Последние фоновые операции"
columns={columns}
data={data}
rowId={(row) => row.job_id}
emptyTitle="Нет задач"
pagination={false}
isLoading={isLoading}
/>
)
}
apps/web/src/components/dashboard/dashboard-recent-revisions-grid.tsx
-55
View File
@@ -1,55 +0,0 @@
import { useMemo } from 'react'
import { GitCommitHorizontal } from 'lucide-react'
import { DataGridMutedCell, DataGridNameCell } from '@/components/data-grid-cell'
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
import { FrameDataGrid, type DataGridColumnDef } from '@/components/reui-kit'
import type { RevisionRow } from '@/types/api'
export function DashboardRecentRevisionsGrid({
revisions,
isLoading = false,
}: {
revisions: RevisionRow[]
isLoading?: boolean
}) {
const data = useMemo(() => revisions.slice(0, 8), [revisions])
const columns = useMemo<DataGridColumnDef<RevisionRow>[]>(
() => [
{
id: 'id',
accessorFn: (row) => row.id,
header: ({ column }) => <DataGridColumnHeader column={column} title="ID" />,
cell: ({ row }) => (
<DataGridNameCell icon={GitCommitHorizontal} title={`${row.original.id.slice(0, 10)}…`} />
),
meta: { headerTitle: 'ID' },
},
{
accessorKey: 'created_at',
header: ({ column }) => <DataGridColumnHeader column={column} title="Создана" />,
cell: ({ row }) => (
<DataGridMutedCell>
{new Date(row.original.created_at).toLocaleString('ru-RU')}
</DataGridMutedCell>
),
meta: { headerTitle: 'Создана' },
},
],
[],
)
return (
<FrameDataGrid
title="Последние ревизии"
description="История конфигураций"
columns={columns}
data={data}
rowId={(row) => row.id}
emptyTitle="Нет ревизий"
pagination={false}
isLoading={isLoading}
/>
)
}
apps/web/src/components/reui-kit/kpi-stat-grid.tsx
+28
View File
@@ -3,6 +3,7 @@ import { Link } from '@tanstack/react-router'
import { Frame, FramePanel } from '@/components/reui/frame' import { Frame, FramePanel } from '@/components/reui/frame'
import { Badge } from '@/components/reui/badge' import { Badge } from '@/components/reui/badge'
import { Progress } from '@evobgp/ui/components/progress'
import { cn } from '@evobgp/ui/lib/utils' import { cn } from '@evobgp/ui/lib/utils'
import { kpiCols } from './kpi-cols' import { kpiCols } from './kpi-cols'
import { IconTile } from '@/components/reui/icon-tile' import { IconTile } from '@/components/reui/icon-tile'
@@ -29,6 +30,8 @@ export type KpiStatItem = {
iconClassName?: string iconClassName?: string
variant?: KpiStatVariant variant?: KpiStatVariant
footer?: ReactNode footer?: ReactNode
/** 0–100 completion bar under the value (stats-4). Omit to hide. */
progress?: number
} }
/** CFDM-compatible card shape (id required). */ /** CFDM-compatible card shape (id required). */
@@ -48,6 +51,17 @@ const VALUE_VARIANT_CLASS: Record<KpiStatVariant, string> = {
destructive: 'text-destructive', destructive: 'text-destructive',
} }
const PROGRESS_TONE_CLASS: Record<KpiStatVariant, string> = {
default: '',
warning: '[&_[data-slot=progress-indicator]]:bg-warning',
destructive: '[&_[data-slot=progress-indicator]]:bg-destructive',
}
function clampProgress(value: number): number {
if (Number.isNaN(value)) return 0
return Math.min(100, Math.max(0, value))
}
function handleCardKeyDown(onActivate: () => void, event: KeyboardEvent<HTMLDivElement>) { function handleCardKeyDown(onActivate: () => void, event: KeyboardEvent<HTMLDivElement>) {
if (event.key === 'Enter' || event.key === ' ') { if (event.key === 'Enter' || event.key === ' ') {
event.preventDefault() event.preventDefault()
@@ -111,6 +125,20 @@ function KpiStatCardBody({ item }: { item: KpiStatItem }) {
> >
{item.value} {item.value}
</div> </div>
{item.progress !== undefined ? (
<Progress
value={clampProgress(item.progress)}
className={cn(
'mt-1.5 w-full gap-0 **:data-[slot=progress-track]:h-1.5',
PROGRESS_TONE_CLASS[valueVariant],
)}
aria-label={
typeof item.label === 'string'
? `${item.label}: ${clampProgress(item.progress)}%`
: undefined
}
/>
) : null}
{footer ? ( {footer ? (
<div className="min-w-0 max-w-full @[20rem]:hidden">{footer}</div> <div className="min-w-0 max-w-full @[20rem]:hidden">{footer}</div>
) : null} ) : null}
apps/web/src/components/reui-kit/ops-dashboard.tsx
+17 -13
View File
@@ -12,7 +12,7 @@ interface OpsDashboardProps {
/** Slot after KPI (QuickActionGrid). Preview: stats-12 · card-12 */ /** Slot after KPI (QuickActionGrid). Preview: stats-12 · card-12 */
afterKpi?: ReactNode afterKpi?: ReactNode
charts: ReactNode charts: ReactNode
queue: ReactNode queue?: ReactNode
queueTitle?: string queueTitle?: string
queueDescription?: string queueDescription?: string
headerActions?: ReactNode headerActions?: ReactNode
@@ -31,10 +31,12 @@ function OpsDashboardSkeleton() {
</header> </header>
<KpiStatGrid cards={[]} isLoading skeletonCount={4} /> <KpiStatGrid cards={[]} isLoading skeletonCount={4} />
<div className="flex min-w-0 flex-col gap-4"> <div className="flex min-w-0 flex-col gap-4">
<Skeleton className="h-64 w-full rounded-xl" /> <Skeleton className="h-40 w-full rounded-xl" />
<Skeleton className="h-64 w-full rounded-xl" /> <div className="grid min-w-0 grid-cols-1 gap-4 @5xl:grid-cols-2">
<Skeleton className="h-56 w-full rounded-xl" />
<Skeleton className="h-56 w-full rounded-xl" />
</div>
</div> </div>
<Skeleton className="h-48 w-full rounded-xl" />
</div> </div>
) )
} }
@@ -76,15 +78,17 @@ export function OpsDashboard({
{charts} {charts}
</section> </section>
<section aria-label={queueTitle} className="flex min-w-0 flex-col gap-4"> {queue ? (
<div className="flex min-w-0 flex-col gap-1"> <section aria-label={queueTitle} className="flex min-w-0 flex-col gap-4">
<h2 className="text-sm font-semibold tracking-tight">{queueTitle}</h2> <div className="flex min-w-0 flex-col gap-1">
{queueDescription ? ( <h2 className="text-sm font-semibold tracking-tight">{queueTitle}</h2>
<p className="text-muted-foreground max-w-prose text-sm">{queueDescription}</p> {queueDescription ? (
) : null} <p className="text-muted-foreground max-w-prose text-sm">{queueDescription}</p>
</div> ) : null}
{queue} </div>
</section> {queue}
</section>
) : null}
</div> </div>
) )
} }
apps/web/src/routes/_auth/dashboard.tsx
+18 -49
View File
@@ -4,24 +4,17 @@ import { RefreshCw } from 'lucide-react'
import { useState } from 'react' import { useState } from 'react'
import { Button } from '@evobgp/ui/components/button' import { Button } from '@evobgp/ui/components/button'
import { Skeleton } from '@evobgp/ui/components/skeleton'
import { DashboardActivityTimeline } from '@/components/dashboard/dashboard-activity-timeline'
import { buildDashboardKpiCards } from '@/components/dashboard/dashboard-kpi-grid' import { buildDashboardKpiCards } from '@/components/dashboard/dashboard-kpi-grid'
import { DashboardModulesGrid } from '@/components/dashboard/dashboard-modules-grid'
import { DashboardNetworkHealth } from '@/components/dashboard/dashboard-network-health' import { DashboardNetworkHealth } from '@/components/dashboard/dashboard-network-health'
import { DashboardOperationsBreakdown } from '@/components/dashboard/dashboard-operations-breakdown' import { DashboardOperationsBreakdown } from '@/components/dashboard/dashboard-operations-breakdown'
import { DashboardQuickLinks } from '@/components/dashboard/dashboard-quick-links' import { DashboardQuickLinks } from '@/components/dashboard/dashboard-quick-links'
import { DashboardRecentJobsGrid } from '@/components/dashboard/dashboard-recent-jobs-grid'
import { DashboardRecentRevisionsGrid } from '@/components/dashboard/dashboard-recent-revisions-grid'
import { OpsDashboard } from '@/components/reui-kit' import { OpsDashboard } from '@/components/reui-kit'
import { chartPanelGridClassName, dashboardMainSidebarClassName } from '@/lib/ui-surface' import { chartPanelGridClassName } from '@/lib/ui-surface'
import { import {
moduleNameById,
overviewJobsQueryOptions, overviewJobsQueryOptions,
overviewModulesQueryOptions, overviewModulesQueryOptions,
overviewPeersQueryOptions, overviewPeersQueryOptions,
overviewRevisionsQueryOptions,
overviewSpeakersQueryOptions, overviewSpeakersQueryOptions,
} from '@/queries/overview' } from '@/queries/overview'
import { settingsQueryOptions } from '@/queries/settings' import { settingsQueryOptions } from '@/queries/settings'
@@ -36,11 +29,12 @@ function parseShowQuickActions(value: unknown): boolean {
} }
/** /**
* Dashboard — OpsDashboard kit (KPI → QuickActions → modules → activity/health → queue). * Dashboard — KPI infographic + Quick Actions + charts (no list duplicates).
* Charts slot is a vertical stack: modules stay full-width; activity shares a row
* with BGP widgets only at @5xl (container), never nested 8+4 inside a 2-col parent.
* @see https://reui.io/preview/base/dashboard-1 * @see https://reui.io/preview/base/dashboard-1
* @see https://reui.io/preview/base/stats-12 * @see https://reui.io/preview/base/stats-12
* @see https://reui.io/preview/base/stats-4
* @see https://reui.io/preview/base/card-12
* @see https://reui.io/preview/base/chart-27
*/ */
function DashboardComponent() { function DashboardComponent() {
const [lastUpdated, setLastUpdated] = useState<Date | null>(null) const [lastUpdated, setLastUpdated] = useState<Date | null>(null)
@@ -52,14 +46,13 @@ function DashboardComponent() {
overviewModulesQueryOptions(), overviewModulesQueryOptions(),
overviewPeersQueryOptions(), overviewPeersQueryOptions(),
overviewSpeakersQueryOptions(), overviewSpeakersQueryOptions(),
overviewRevisionsQueryOptions(),
overviewJobsQueryOptions(), overviewJobsQueryOptions(),
], ],
}) })
const [modulesQ, peersQ, speakersQ, revisionsQ, jobsQ] = results const [modulesQ, peersQ, speakersQ, jobsQ] = results
const initialLoading = const initialLoading =
modulesQ.isLoading || peersQ.isLoading || speakersQ.isLoading || revisionsQ.isLoading || jobsQ.isLoading modulesQ.isLoading || peersQ.isLoading || speakersQ.isLoading || jobsQ.isLoading
const refreshing = results.some((r) => r.isFetching && !r.isLoading) const refreshing = results.some((r) => r.isFetching && !r.isLoading)
if (!lastUpdated && !initialLoading && results.every((r) => r.isSuccess || r.isError)) { if (!lastUpdated && !initialLoading && results.every((r) => r.isSuccess || r.isError)) {
@@ -74,11 +67,7 @@ function DashboardComponent() {
const modules = modulesQ.data?.items ?? [] const modules = modulesQ.data?.items ?? []
const peers = peersQ.data?.items ?? [] const peers = peersQ.data?.items ?? []
const speakers = speakersQ.data?.items ?? [] const speakers = speakersQ.data?.items ?? []
const revisions = revisionsQ.data?.items ?? []
const jobs = jobsQ.data?.items ?? [] const jobs = jobsQ.data?.items ?? []
const nameById = moduleNameById(modules)
const activityLoading = refreshing && jobs.length === 0 && revisions.length === 0
const kpiCards = buildDashboardKpiCards({ modules, peers, speakers, jobs }) const kpiCards = buildDashboardKpiCards({ modules, peers, speakers, jobs })
return ( return (
@@ -99,38 +88,18 @@ function DashboardComponent() {
isLoading={initialLoading} isLoading={initialLoading}
afterKpi={showQuickActions ? <DashboardQuickLinks /> : null} afterKpi={showQuickActions ? <DashboardQuickLinks /> : null}
charts={ charts={
<>
<div className="min-w-0">
<DashboardModulesGrid modules={modules} isLoading={refreshing} />
</div>
<div className={dashboardMainSidebarClassName}>
<DashboardActivityTimeline
jobs={jobs}
revisions={revisions}
peers={peers}
speakers={speakers}
/>
<div className="grid min-w-0 items-start gap-4">
<DashboardNetworkHealth peers={peers} speakers={speakers} jobs={jobs} />
<DashboardOperationsBreakdown jobs={jobs} modules={modules} />
</div>
</div>
</>
}
queueTitle="Задачи и ревизии"
queueDescription="Последние фоновые операции и история конфигураций"
queue={
<div className={chartPanelGridClassName}> <div className={chartPanelGridClassName}>
{activityLoading ? ( <DashboardNetworkHealth
<Skeleton className="m-4 h-24 w-auto" /> peers={peers}
) : ( speakers={speakers}
<DashboardRecentJobsGrid jobs={jobs.slice(0, 8)} nameById={nameById} isLoading={refreshing} /> jobs={jobs}
)} loading={refreshing && peers.length === 0 && speakers.length === 0}
{activityLoading ? ( />
<Skeleton className="m-4 h-24 w-auto" /> <DashboardOperationsBreakdown
) : ( jobs={jobs}
<DashboardRecentRevisionsGrid revisions={revisions} isLoading={refreshing} /> modules={modules}
)} loading={refreshing && jobs.length === 0 && modules.length === 0}
/>
</div> </div>
} }
/> />
apps/web/tsconfig.tsbuildinfo
+1 -1
View File
File diff suppressed because one or more lines are too long
docs/access.md
+10
View File
@@ -168,6 +168,16 @@ http://localhost:5173,http://127.0.0.1:5173,https://ui.example.com
Разрешённые заголовки включают `Authorization`, `Content-Type`, `Idempotency-Key`, `Accept`, `X-Tenant-Id` (см. `internal/httpapi/cors.go`). Разрешённые заголовки включают `Authorization`, `Content-Type`, `Idempotency-Key`, `Accept`, `X-Tenant-Id` (см. `internal/httpapi/cors.go`).
## Pipeline: TTL и внешние вызовы
| Переменная | Назначение |
|------------|------------|
| `EVOBGP_ASN_CACHE_TTL_SEC` | TTL кэша объявленных префиксов RIPEstat (default 1800) |
| `EVOBGP_ASN_HOLDER_TTL_SEC` | TTL имени holder AS (default 7 суток) |
| `EVOBGP_CDN_DNS_CACHE_TTL_SEC` | TTL DNS при проверке CDN URL (default 300) |
| `EVOBGP_DOMAIN_CACHE_TTL_SEC` | TTL кэша DoH A/AAAA (default 300) |
| `EVOBGP_CDN_PARTIAL_OK` | Сохранять prior-строки skipped CDN-источников при частичном сбое |
## Заголовок `X-Tenant-Id` (решение: не реализован) ## Заголовок `X-Tenant-Id` (решение: не реализован)
**Решение (done):** заголовок **`X-Tenant-Id` не переключает tenant** в handlers и **не планируется** без отдельного ADR на супер-роли. **Решение (done):** заголовок **`X-Tenant-Id` не переключает tenant** в handlers и **не планируется** без отдельного ADR на супер-роли.
docs/architecture.md
+12 -1
View File
@@ -39,7 +39,7 @@
| `config` | Переменные окружения `EVOBGP_*`. | | `config` | Переменные окружения `EVOBGP_*`. |
| `observability` | Метрики Prometheus, HTTP middleware. | | `observability` | Метрики Prometheus, HTTP middleware. |
| `broker` | Опциональный `EVOBGP_BROKER_URL` для будущей шины; сейчас задачи только in-process (`jobs.Registry`), пакет лишь логирует факт настройки URL. | | `broker` | Опциональный `EVOBGP_BROKER_URL` для будущей шины; сейчас задачи только in-process (`jobs.Registry`), пакет лишь логирует факт настройки URL. |
| `pipeline` | Ingest+render в одном шаге для `module_refresh`: выборка префиксов (CDN/AS/IP/пустые DOMAINS), `CreateRenderRevision`, превью BIRD через `birdfmt`. | | `pipeline` | Ingest+render для `module_refresh`: CDN/AS/IP/DOMAINS → `module_prefix_snapshot` (batch `COPY`, per-module lock) → агрегация CIDR O(n log n) → `CreateRenderRevision`. Fast-path снапшота — `module.input_hash`; DoH — TTL-кэш `domain_resolve_cache`; scheduler — jitter границ интервала. |
| `nodedispatch` | Panel→Node HTTP wake-up (`POST /v1/agent/sync`) после `deploy_apply`. | | `nodedispatch` | Panel→Node HTTP wake-up (`POST /v1/agent/sync`) после `deploy_apply`. |
| `agentserver` | HTTP API на реплике (`serve`): sync + health для Traefik; опционально firewall failover (`/v1/firewall/*`). | | `agentserver` | HTTP API на реплике (`serve`): sync + health для Traefik; опционально firewall failover (`/v1/firewall/*`). |
| `firewall` | Вычисление policy block/accept → плоский CIDR blocklist. | | `firewall` | Вычисление policy block/accept → плоский CIDR blocklist. |
@@ -98,6 +98,17 @@ flowchart LR
Профиль Compose **`microvps-full`** добавляет к этому стеку **Web UI** (nginx → `evobgp-all`), **NATS** и **Prometheus** без отдельных контейнеров воркеров (функционально то же, что отдельные `scheduler`/`ingest`/… в reference). Запуск и лимиты под ~1 ГиБ RAM — в [quickstart.md](quickstart.md). Профиль Compose **`microvps-full`** добавляет к этому стеку **Web UI** (nginx → `evobgp-all`), **NATS** и **Prometheus** без отдельных контейнеров воркеров (функционально то же, что отдельные `scheduler`/`ingest`/… в reference). Запуск и лимиты под ~1 ГиБ RAM — в [quickstart.md](quickstart.md).
## Поток pipeline (ingest → render)
1. **Scheduler** (`evobgp-scheduler` / in-process в `evobgp-all`) ставит `tenant_refresh`, если `ModuleDueForScheduler`: граница окна `refresh_interval_sec` со **сдвигом `fnv32(module.ID) % interval`**, чтобы модули с одним интервалом не били внешние API одновременно.
2. **Ingest** (`RefreshModuleIngest`):
- `AS_PREFIXES` — RIPEstat prefixes + holder параллельно, кэш `asn_prefix_cache`; дедуп строк по `prefix + community`.
- `CDN_CIDRS` — единый `fetchCDNSourceRows` (conditional GET); prefetch уважает `RefreshIntervalSec`; merge снапшота под `LockModuleSnapshot` (пропущенные по ошибке источники сохраняют prior-строки при `EVOBGP_CDN_PARTIAL_OK`).
- `DOMAINS` — DoH A+AAAA параллельно; попадания в `domain_resolve_cache` с TTL `EVOBGP_DOMAIN_CACHE_TTL_SEC` (default 300).
- `IP_RANGES` — напрямую из записей модуля.
3. Снапшот пишется **batch** (`pgx.CopyFrom` в PostgreSQL). Совпадение `module.input_hash` со снапшотом — O(1) пропуск повторного ingest при render; CRUD entries обнуляет hash.
4. **Render** (`RenderTenantRevision`): `smartAggregatePrefixRows` (стек-схлопывание O(n log n), IPv6 без `math/big`) → ревизия, если набор префиксов изменился.
## Поток: ревизия и бандл для ноды ## Поток: ревизия и бандл для ноды
1. Оператор (роль `operator` или выше по политике) изменяет модули и запускает цепочку, приводящую к новой **ревизии** (часть шагов может быть асинхронной через jobs — см. OpenAPI). 1. Оператор (роль `operator` или выше по политике) изменяет модули и запускает цепочку, приводящую к новой **ревизии** (часть шагов может быть асинхронной через jobs — см. OpenAPI).
docs/production-checklist.md
+3
View File
@@ -12,6 +12,9 @@
- `EVOBGP_CORS_ORIGINS` — явный whitelist origin веб-панели. - `EVOBGP_CORS_ORIGINS` — явный whitelist origin веб-панели.
- `EVOBGP_STALE_ON_UPSTREAM_ERROR=1` (по умолчанию) — stale snapshot при сбоях CDN/ASN/DoH. - `EVOBGP_STALE_ON_UPSTREAM_ERROR=1` (по умолчанию) — stale snapshot при сбоях CDN/ASN/DoH.
- Опционально `EVOBGP_CDN_PARTIAL_OK=1` — при сбое одного CDN source без stale cache продолжать refresh остальных (иначе fail модуля). - Опционально `EVOBGP_CDN_PARTIAL_OK=1` — при сбое одного CDN source без stale cache продолжать refresh остальных (иначе fail модуля).
- `EVOBGP_ASN_HOLDER_TTL_SEC` — TTL имени holder AS (default 7 суток); prefixes кэшируются отдельно (`EVOBGP_ASN_CACHE_TTL_SEC`, default 1800).
- `EVOBGP_CDN_DNS_CACHE_TTL_SEC` — TTL кэша DNS при SSRF-проверке CDN URL (default 300).
- `EVOBGP_DOMAIN_CACHE_TTL_SEC` — TTL кэша DoH A/AAAA (`domain_resolve_cache`, default 300).
## Рекомендуется ## Рекомендуется
internal/asnresolve/ripestat.go
-21
View File
@@ -10,9 +10,7 @@ import (
"net/netip" "net/netip"
"os" "os"
"sort" "sort"
"strconv"
"strings" "strings"
"time"
"evobgp/internal/httpclient" "evobgp/internal/httpclient"
) )
@@ -137,22 +135,3 @@ func truncateForErr(b []byte, n int) string {
} }
return s return s
} }
// PolitePause is a short delay between upstream ASN lookups (same refresh).
func PolitePause() {
d := 150 * time.Millisecond
if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE_PAUSE_MS")); s != "" {
if ms, err := parsePositiveInt(s); err == nil && ms > 0 {
d = time.Duration(ms) * time.Millisecond
}
}
time.Sleep(d)
}
func parsePositiveInt(s string) (int, error) {
n, err := strconv.Atoi(s)
if err != nil || n <= 0 {
return 0, fmt.Errorf("invalid")
}
return n, nil
}
internal/pipeline/aggregate_collapse.go
+173
View File
@@ -0,0 +1,173 @@
package pipeline
import (
"net/netip"
"sort"
"evobgp/internal/store"
)
// Prefix collapse replaces the former prune+merge fixed-point loop (O(n²) per pass,
// multiple passes) with a single sort + linear stack pass: O(n log n) overall.
//
// Classic trie-collapse without building a trie:
// 1. Sort prefixes by (address, mask len ascending).
// 2. Walk left to right keeping a stack of "open" prefixes:
// - while the stack top contains the new prefix -> the top covers it, drop the new one (prune);
// - else, while the stack top is a sibling of the new prefix (same mask, XOR of the
// network bit equals the parent block) and merging is allowed for that mask length
// -> pop the sibling, replace the new prefix with the parent and re-check;
// - otherwise push the new prefix.
//
// A parent absorbs its sibling children only when both halves are present, which matches
// the previous merge-to-fixed-point semantics, including the guard that forbids merging
// above a floor mask (IPv4 /8, IPv6 /16).
type collapseItem struct {
row store.PrefixRow
pfx netip.Prefix
}
// collapsePrefixGroup collapses one (community, source) group of same-family prefixes.
// All rows must be masked; family and floor are enforced by minBits.
func collapsePrefixGroup(rows []store.PrefixRow, is4 bool) []store.PrefixRow {
if len(rows) <= 1 {
return rows
}
items := make([]collapseItem, 0, len(rows))
seen := make(map[string]struct{}, len(rows))
for _, row := range rows {
if _, dup := seen[row.Prefix]; dup {
continue
}
seen[row.Prefix] = struct{}{}
pfx, err := netip.ParsePrefix(row.Prefix)
if err != nil {
continue
}
pfx = pfx.Masked()
if is4 != pfx.Addr().Is4() {
continue
}
items = append(items, collapseItem{row: row, pfx: pfx})
}
if len(items) <= 1 {
out := make([]store.PrefixRow, 0, len(items))
for _, it := range items {
out = append(out, it.row)
}
return out
}
// Sort by (address, mask len): a parent always sorts before its children, and a
// shorter sibling sorts before a longer one within the same parent block.
sort.Slice(items, func(i, j int) bool {
a, b := items[i].pfx, items[j].pfx
if a.Addr() != b.Addr() {
return lessAddr(a.Addr(), b.Addr())
}
return a.Bits() < b.Bits()
})
minBits := 8
if !is4 {
minBits = 16
}
stack := make([]collapseItem, 0, len(items))
for _, it := range items {
cur := it
dropped := false
for len(stack) > 0 {
top := stack[len(stack)-1]
if top.pfx.Contains(cur.pfx.Addr()) && top.pfx.Bits() <= cur.pfx.Bits() {
// Covered by an existing prefix: drop (prune).
dropped = true
break
}
if cur.pfx.Bits() == top.pfx.Bits() && cur.pfx.Bits() > minBits && areSiblings(top.pfx, cur.pfx) {
// Merge siblings into the parent (parent keeps the lower sibling's attributes),
// then re-check the parent against the new stack top.
stack = stack[:len(stack)-1]
parentBits := cur.pfx.Bits() - 1
parent := netip.PrefixFrom(maskAddr(cur.pfx.Addr(), parentBits, is4), parentBits).Masked()
cur = collapseItem{row: top.row, pfx: parent}
cur.row.Prefix = parent.String()
continue
}
break
}
if !dropped {
stack = append(stack, cur)
}
}
out := make([]store.PrefixRow, 0, len(stack))
for _, it := range stack {
out = append(out, it.row)
}
sortPrefixRows(out)
return out
}
// areSiblings reports whether two same-length prefixes combine into their common parent.
func areSiblings(a, b netip.Prefix) bool {
if a.Bits() != b.Bits() || a.Bits() == 0 {
return false
}
parentBits := a.Bits() - 1
pa := maskAddr(a.Addr(), parentBits, a.Addr().Is4())
pb := maskAddr(b.Addr(), parentBits, b.Addr().Is4())
return pa == pb
}
// maskAddr clears the host bits below prefixLen (IPv4: 32-bit space; IPv6: 128-bit).
func maskAddr(a netip.Addr, prefixLen int, is4 bool) netip.Addr {
if is4 {
v := uint32FromIPv4(a)
if prefixLen <= 0 {
v = 0
} else if prefixLen < 32 {
v &= ^(uint32(1)<<(32-prefixLen) - 1)
}
return u32ToIPv4(v)
}
b := a.As16()
hostBits := 128 - prefixLen
fullBytes := hostBits / 8
for i := 15; i > 15-fullBytes; i-- {
b[i] = 0
}
if rem := hostBits % 8; rem > 0 {
idx := 15 - fullBytes
if idx >= 0 && idx < 16 {
b[idx] &= byte(0xFF << rem)
}
}
return netip.AddrFrom16(b)
}
func uint32FromIPv4(a netip.Addr) uint32 {
o := a.As4()
return uint32(o[0])<<24 | uint32(o[1])<<16 | uint32(o[2])<<8 | uint32(o[3])
}
func u32ToIPv4(v uint32) netip.Addr {
return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
}
func lessAddr(a, b netip.Addr) bool {
if a.Is4() != b.Is4() {
return a.Is4()
}
if a.Is4() {
return uint32FromIPv4(a) < uint32FromIPv4(b)
}
a16, b16 := a.As16(), b.As16()
for i := 0; i < 16; i++ {
if a16[i] != b16[i] {
return a16[i] < b16[i]
}
}
return false
}
internal/pipeline/aggregate_collapse_test.go
+294
View File
@@ -0,0 +1,294 @@
package pipeline
import (
"fmt"
"math/big"
"math/rand"
"net/netip"
"sort"
"strings"
"testing"
"evobgp/internal/store"
)
// The functions below are a verbatim copy of the pre-2.1 prune+merge loop
// (from git HEAD internal/pipeline/refresh.go). They exist only as an
// equivalence oracle for collapsePrefixGroup.
func legacyAggregateCIDRGroup(rows []store.PrefixRow, mergeFn func(map[string]store.PrefixRow) bool) []store.PrefixRow {
if len(rows) <= 1 {
return rows
}
set := make(map[string]store.PrefixRow, len(rows))
for _, row := range rows {
set[row.Prefix] = row
}
legacyPruneCoveredPrefixes(set)
for {
if !mergeFn(set) {
break
}
legacyPruneCoveredPrefixes(set)
}
out := make([]store.PrefixRow, 0, len(set))
for _, row := range set {
out = append(out, row)
}
sortPrefixRows(out)
return out
}
func legacyPruneCoveredPrefixes(set map[string]store.PrefixRow) {
type item struct {
key string
pfx netip.Prefix
bits int
}
items := make([]item, 0, len(set))
for k := range set {
p, err := netip.ParsePrefix(k)
if err != nil {
continue
}
items = append(items, item{key: k, pfx: p, bits: p.Bits()})
}
sort.Slice(items, func(i, j int) bool {
if items[i].bits != items[j].bits {
return items[i].bits < items[j].bits
}
return items[i].key < items[j].key
})
for i := 0; i < len(items); i++ {
for j := i + 1; j < len(items); j++ {
if items[j].bits <= items[i].bits {
continue
}
if items[i].pfx.Contains(items[j].pfx.Addr()) {
delete(set, items[j].key)
}
}
}
}
func legacyMergeSiblingPrefixesIPv4(set map[string]store.PrefixRow) bool {
merged := false
seen := make(map[string]struct{}, len(set))
for key, row := range set {
if _, done := seen[key]; done {
continue
}
pfx, err := netip.ParsePrefix(key)
if err != nil || !pfx.Addr().Is4() {
continue
}
bits := pfx.Bits()
if bits <= 8 {
continue
}
netNum := ipv4PrefixNetworkLegacy(pfx)
blockSize := uint32(1) << (32 - bits)
siblingNet := netNum ^ blockSize
siblingPfx := netip.PrefixFrom(u32ToIPv4(siblingNet), bits).Masked().String()
if _, ok := set[siblingPfx]; !ok {
continue
}
parentBits := bits - 1
parentBlock := uint32(1) << (32 - parentBits)
parentNet := netNum & ^(parentBlock - 1)
parentPfx := netip.PrefixFrom(u32ToIPv4(parentNet), parentBits).Masked().String()
delete(set, key)
delete(set, siblingPfx)
parentRow := row
parentRow.Prefix = parentPfx
set[parentPfx] = parentRow
seen[key] = struct{}{}
seen[siblingPfx] = struct{}{}
merged = true
}
return merged
}
func ipv4PrefixNetworkLegacy(p netip.Prefix) uint32 {
a := p.Masked().Addr().As4()
return uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3])
}
func legacyMergeSiblingPrefixesIPv6(set map[string]store.PrefixRow) bool {
merged := false
seen := make(map[string]struct{}, len(set))
for key, row := range set {
if _, done := seen[key]; done {
continue
}
pfx, err := netip.ParsePrefix(key)
if err != nil || !pfx.Addr().Is6() {
continue
}
bits := pfx.Bits()
if bits <= 16 {
continue
}
netNum := ipv6PrefixNetworkLegacy(pfx)
blockSize := new(big.Int).Lsh(big.NewInt(1), uint(128-bits))
siblingNet := new(big.Int).Xor(netNum, blockSize)
siblingPfx := ipv6PrefixFromBigIntLegacy(siblingNet, bits).String()
if _, ok := set[siblingPfx]; !ok {
continue
}
parentBits := bits - 1
parentBlock := new(big.Int).Lsh(big.NewInt(1), uint(128-parentBits))
mask := new(big.Int).Sub(parentBlock, big.NewInt(1))
mask.Not(mask)
parentNet := new(big.Int).And(netNum, mask)
parentPfx := ipv6PrefixFromBigIntLegacy(parentNet, parentBits).String()
delete(set, key)
delete(set, siblingPfx)
parentRow := row
parentRow.Prefix = parentPfx
set[parentPfx] = parentRow
seen[key] = struct{}{}
seen[siblingPfx] = struct{}{}
merged = true
}
return merged
}
func ipv6PrefixNetworkLegacy(p netip.Prefix) *big.Int {
a := p.Masked().Addr().As16()
n := new(big.Int)
n.SetBytes(a[:])
return n
}
func ipv6PrefixFromBigIntLegacy(n *big.Int, bits int) netip.Prefix {
b := n.Bytes()
var a [16]byte
copy(a[16-len(b):], b)
return netip.PrefixFrom(netip.AddrFrom16(a), bits).Masked()
}
func normalizeForCompare(rows []store.PrefixRow) []string {
type line struct{ p, c, s string }
lines := make([]line, 0, len(rows))
for _, r := range rows {
p := strings.TrimSpace(r.Prefix)
if p != "" {
if pfx, err := netip.ParsePrefix(p); err == nil {
p = pfx.Masked().String()
}
}
lines = append(lines, line{p, prefixRowCommunity(r), r.Source})
}
sort.Slice(lines, func(i, j int) bool {
if lines[i].p != lines[j].p {
return lines[i].p < lines[j].p
}
if lines[i].c != lines[j].c {
return lines[i].c < lines[j].c
}
return lines[i].s < lines[j].s
})
out := make([]string, 0, len(lines))
for _, l := range lines {
out = append(out, fmt.Sprintf("%s|%s|%s", l.p, l.c, l.s))
}
return out
}
func TestCollapsePrefixGroup_EquivalenceWithLegacy(t *testing.T) {
r := rand.New(rand.NewSource(7))
for iter := 0; iter < 200; iter++ {
n := 1 + r.Intn(60)
v4rows := make([]store.PrefixRow, 0, n)
for i := 0; i < n; i++ {
addr := netip.AddrFrom4([4]byte{203, byte(r.Intn(4)), byte(r.Intn(256)), byte(r.Intn(256))})
bits := 16 + r.Intn(9)
pfx := netip.PrefixFrom(addr, bits).Masked()
v4rows = append(v4rows, store.PrefixRow{Prefix: pfx.String(), Source: "ip_range"})
}
legacy := legacyAggregateCIDRGroup(append([]store.PrefixRow(nil), v4rows...), legacyMergeSiblingPrefixesIPv4)
got := collapsePrefixGroup(append([]store.PrefixRow(nil), v4rows...), true)
if fmt.Sprint(normalizeForCompare(legacy)) != fmt.Sprint(normalizeForCompare(got)) {
t.Fatalf("iter %d mismatch:\nlegacy=%v\ngot =%v", iter, normalizeForCompare(legacy), normalizeForCompare(got))
}
}
}
func TestCollapsePrefixGroup_EquivalenceWithLegacyIPv6(t *testing.T) {
r := rand.New(rand.NewSource(11))
for iter := 0; iter < 200; iter++ {
n := 1 + r.Intn(60)
rows := make([]store.PrefixRow, 0, n)
for i := 0; i < n; i++ {
var a [16]byte
a[0], a[1] = 0x20, 0x01
a[2], a[3] = 0x0d, 0xb8
a[4] = byte(r.Intn(2))
a[5] = byte(r.Intn(256))
a[6] = byte(r.Intn(256))
addr := netip.AddrFrom16(a)
bits := 32 + r.Intn(17)
pfx := netip.PrefixFrom(addr, bits).Masked()
rows = append(rows, store.PrefixRow{Prefix: pfx.String(), Source: "ip_range"})
}
legacy := legacyAggregateCIDRGroup(append([]store.PrefixRow(nil), rows...), legacyMergeSiblingPrefixesIPv6)
got := collapsePrefixGroup(append([]store.PrefixRow(nil), rows...), false)
if fmt.Sprint(normalizeForCompare(legacy)) != fmt.Sprint(normalizeForCompare(got)) {
t.Fatalf("iter %d mismatch:\nlegacy=%v\ngot =%v", iter, normalizeForCompare(legacy), normalizeForCompare(got))
}
}
}
func TestCollapsePrefixGroup_CoversAndSiblingChain(t *testing.T) {
rows := []store.PrefixRow{
{Prefix: "10.0.0.0/16", Source: "ip_range"},
{Prefix: "10.0.0.0/24", Source: "ip_range"},
{Prefix: "10.0.1.0/24", Source: "ip_range"},
{Prefix: "10.0.2.0/24", Source: "ip_range"},
{Prefix: "10.0.3.0/24", Source: "ip_range"},
{Prefix: "10.1.0.0/24", Source: "ip_range"},
}
got := collapsePrefixGroup(rows, true)
if len(got) != 2 {
t.Fatalf("want 2 rows (/16 + /24), got %d: %+v", len(got), got)
}
set := map[string]bool{}
for _, r := range got {
set[r.Prefix] = true
}
if !set["10.0.0.0/16"] || !set["10.1.0.0/24"] {
t.Fatalf("unexpected rows: %+v", got)
}
}
func TestCollapsePrefixGroup_RespectsMinBitsFloor(t *testing.T) {
rows := []store.PrefixRow{
{Prefix: "10.0.0.0/8", Source: "ip_range"},
{Prefix: "11.0.0.0/8", Source: "ip_range"},
}
got := collapsePrefixGroup(rows, true)
if len(got) != 2 {
t.Fatalf("floor must prevent /8+/8 -> /7, got %+v", got)
}
rows9 := []store.PrefixRow{
{Prefix: "10.0.0.0/9", Source: "ip_range"},
{Prefix: "10.128.0.0/9", Source: "ip_range"},
}
got9 := collapsePrefixGroup(rows9, true)
if len(got9) != 1 || got9[0].Prefix != "10.0.0.0/8" {
t.Fatalf("expected /9+/9 -> /8, got %+v", got9)
}
}
func TestCollapsePrefixGroup_PrunesCovered(t *testing.T) {
rows := []store.PrefixRow{
{Prefix: "10.0.0.0/16", Source: "ip_range"},
{Prefix: "10.0.0.0/24", Source: "ip_range"},
}
got := collapsePrefixGroup(rows, true)
if len(got) != 1 || got[0].Prefix != "10.0.0.0/16" {
t.Fatalf("want covered prune to /16, got %+v", got)
}
}
internal/pipeline/asn_cache.go
+83 -16
View File
@@ -24,28 +24,43 @@ func asnCacheTTL() time.Duration {
return time.Duration(sec) * time.Second return time.Duration(sec) * time.Second
} }
// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache). // asnHolderTTL is how long a holder name stays authoritative between refreshes;
func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) { // holder text changes rarely, so it survives short prefix-cache TTLs.
ttl := asnCacheTTL() func asnHolderTTL() time.Duration {
if st != nil { sec := 7 * 24 * 3600
if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil && time.Since(ent.FetchedAt) < ttl { if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_HOLDER_TTL_SEC")); s != "" {
out := make([]netip.Prefix, 0, len(ent.Prefixes)) if v, err := strconv.Atoi(s); err == nil && v > 0 {
for _, p := range ent.Prefixes { sec = v
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
if perr != nil {
continue
}
out = append(out, pfx.Masked())
}
return out, ent.Holder, nil
} }
} }
return time.Duration(sec) * time.Second
}
// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache).
// The holder lookup starts concurrently with the prefix fetch (one RTT instead of two);
// a holder failure is non-fatal — the previously cached holder name is kept.
func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) {
ttl := asnCacheTTL()
var prevHolder string
if st != nil {
if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil {
prevHolder = ent.Holder
if time.Since(ent.FetchedAt) < ttl {
return parseASNCachePrefixes(ent.Prefixes), ent.Holder, nil
}
}
}
holderCh := startASNHolderFetch(ctx, hc, st, asn, prevHolder)
pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, asn) pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, asn)
if err != nil { if err != nil {
holder := <-holderCh // drain to avoid leaking the goroutine's channel send
_ = holder
return nil, "", err return nil, "", err
} }
asnresolve.PolitePause()
holder, _ := asnresolve.ASHolderName(ctx, hc, asn) holder := <-holderCh
if st != nil { if st != nil {
strs := make([]string, len(pfxs)) strs := make([]string, len(pfxs))
for i, p := range pfxs { for i, p := range pfxs {
@@ -57,3 +72,55 @@ func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client,
} }
return pfxs, holder, nil return pfxs, holder, nil
} }
// startASNHolderFetch launches the holder lookup concurrently. The returned buffered
// channel always yields exactly one value, so callers may abandon it without leaking.
func startASNHolderFetch(ctx context.Context, hc *http.Client, st store.Backend, asn int64, prevHolder string) <-chan string {
ch := make(chan string, 1)
go func() {
if prevHolder != "" && holderStillFresh(st, asn, prevHolder) {
ch <- prevHolder
return
}
holder, err := asnresolve.ASHolderName(ctx, hc, asn)
if err != nil || strings.TrimSpace(holder) == "" {
ch <- prevHolder
return
}
ch <- strings.TrimSpace(holder)
}()
return ch
}
// holderStillFresh reports whether the cached holder name is within its own (long) TTL.
func holderStillFresh(st store.Backend, asn int64, prevHolder string) bool {
if st == nil {
return false
}
ent, ok, err := st.GetASNPrefixCache(asn)
return err == nil && ok && ent != nil && ent.Holder == prevHolder && time.Since(ent.FetchedAt) < asnHolderTTL()
}
func parseASNCachePrefixes(raw []string) []netip.Prefix {
out := make([]netip.Prefix, 0, len(raw))
for _, p := range raw {
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
if perr != nil {
continue
}
out = append(out, pfx.Masked())
}
return out
}
func parseCachedASNCachedPrefixes(raw []string) []netip.Prefix {
out := make([]netip.Prefix, 0, len(raw))
for _, p := range raw {
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
if perr != nil {
continue
}
out = append(out, pfx.Masked())
}
return out
}
internal/pipeline/asn_cache_test.go
+82 -3
View File
@@ -51,13 +51,92 @@ func TestResolveASNForEntry_UsesTTLCache(t *testing.T) {
} }
} }
func TestResolveASNForEntry_HolderFailureIsNonFatal(t *testing.T) {
m := store.NewMemory()
t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "1")
var holderCalls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.URL.Path, "/announced") {
_, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`))
return
}
holderCalls.Add(1)
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced")
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview")
ctx := context.Background()
hc := srv.Client()
pfxs, holder, err := resolveASNForEntry(ctx, m, hc, 64512)
if err != nil {
t.Fatalf("holder failure must not fail ingest: %v", err)
}
if len(pfxs) != 1 {
t.Fatalf("expected 1 prefix, got %+v", pfxs)
}
if holder != "" {
t.Fatalf("expected empty holder on upstream failure, got %q", holder)
}
if holderCalls.Load() == 0 {
t.Fatal("expected holder endpoint to be attempted")
}
}
func TestResolveASNForEntry_KeepsPreviousHolderOnFailure(t *testing.T) {
m := store.NewMemory()
t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "1")
t.Setenv("EVOBGP_ASN_HOLDER_TTL_SEC", "3600")
holderOK := atomic.Bool{}
holderOK.Store(true)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.URL.Path, "/announced") {
_, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`))
return
}
if holderOK.Load() {
_, _ = w.Write([]byte(`{"status":"ok","data":{"holder":"Good AS"}}`))
return
}
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced")
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview")
ctx := context.Background()
hc := srv.Client()
if _, h, err := resolveASNForEntry(ctx, m, hc, 64512); err != nil || h != "Good AS" {
t.Fatalf("first resolve: holder=%q err=%v", h, err)
}
// Prefix cache expired; holder endpoint now broken — previous holder must survive.
holderOK.Store(false)
time.Sleep(1100 * time.Millisecond)
pfxs, h, err := resolveASNForEntry(ctx, m, hc, 64512)
if err != nil {
t.Fatal(err)
}
if len(pfxs) != 1 {
t.Fatalf("expected 1 prefix, got %+v", pfxs)
}
if h != "Good AS" {
t.Fatalf("expected previous holder preserved, got %q", h)
}
}
func TestModuleDueForScheduler_BucketRollover(t *testing.T) { func TestModuleDueForScheduler_BucketRollover(t *testing.T) {
mod := &store.Module{Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300} mod := &store.Module{ID: "mod-jitter", Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300}
boundary := time.Unix(300, 0) win := int64(300)
off := moduleSchedulerOffset(mod.ID, win)
boundary := time.Unix(win+off, 0)
if !ModuleDueForScheduler(mod, boundary) { if !ModuleDueForScheduler(mod, boundary) {
t.Fatal("expected due when refresh bucket rolls") t.Fatal("expected due when refresh bucket rolls")
} }
mid := time.Unix(330, 0) mid := time.Unix(win+off+SchedulerTickSec, 0)
if ModuleDueForScheduler(mod, mid) { if ModuleDueForScheduler(mod, mid) {
t.Fatal("expected not due within same bucket") t.Fatal("expected not due within same bucket")
} }
internal/pipeline/baseline_bench_test.go
+56
View File
@@ -0,0 +1,56 @@
package pipeline
import (
"fmt"
"math/rand"
"net/netip"
"testing"
"evobgp/internal/store"
)
// benchPrefixRows generates seed-stable pseudo-random prefixes for aggregation benchmarks.
// v4 blocks are carved from TEST-NET-style ranges; v6 from 2001:db8::/32.
func benchPrefixRows(n int, v6Share float64) []store.PrefixRow {
r := rand.New(rand.NewSource(42))
rows := make([]store.PrefixRow, 0, n)
seen := make(map[string]struct{}, n)
for len(rows) < n {
var pfx netip.Prefix
if r.Float64() < v6Share {
var a [16]byte
a[0], a[1] = 0x20, 0x01
a[2], a[3] = 0x0d, 0xb8
for i := 4; i < 10; i++ {
a[i] = byte(r.Intn(256))
}
addr := netip.AddrFrom16(a)
pfx = netip.PrefixFrom(addr, 48+r.Intn(17))
} else {
b := []byte{203, 0, 113, 0}
b[1] = byte(r.Intn(256))
b[2] = byte(r.Intn(256))
b[3] = byte(r.Intn(256))
addr := netip.AddrFrom4([4]byte{b[0], b[1], b[2], b[3]})
pfx = netip.PrefixFrom(addr, 16+r.Intn(9))
}
s := pfx.Masked().String()
if _, ok := seen[s]; ok {
continue
}
seen[s] = struct{}{}
rows = append(rows, store.PrefixRow{Prefix: s, Source: "ip_range"})
}
return rows
}
func BenchmarkSmartAggregatePrefixRows(b *testing.B) {
for _, n := range []int{1_000, 10_000, 50_000} {
rows := benchPrefixRows(n, 0.3)
b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) {
for i := 0; i < b.N; i++ {
_ = smartAggregatePrefixRows(rows)
}
})
}
}
internal/pipeline/cdn_prefetch_test.go
+62
View File
@@ -5,7 +5,9 @@ import (
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings" "strings"
"sync/atomic"
"testing" "testing"
"time"
"evobgp/internal/store" "evobgp/internal/store"
) )
@@ -194,3 +196,63 @@ func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) {
t.Fatalf("unexpected collected rows: %+v", collected) t.Fatalf("unexpected collected rows: %+v", collected)
} }
} }
func TestPrefetchCDNSourceETags_SkipsFreshSources(t *testing.T) {
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
mod, err := m.CreateModule(tenant, &store.Module{
Type: "CDN_CIDRS",
Name: "cdn-prefetch-due",
Enabled: true,
})
if err != nil {
t.Fatal(err)
}
var hits atomic.Int32
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
w.Header().Set("ETag", `"v1"`)
_, _ = w.Write([]byte("198.51.100.0/24\n"))
}))
defer srv.Close()
// Fresh source: refreshed 30s ago with a 3600s interval — prefetch must skip it.
freshAt := time.Now().UTC().Add(-30 * time.Second)
interval := 3600
if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
SourceKind: "txt",
URL: srv.URL,
RefreshIntervalSec: &interval,
LastRefreshedAt: &freshAt,
}); err != nil {
t.Fatal(err)
}
if err := PrefetchCDNSourceETags(context.Background(), m, srv.Client()); err != nil {
t.Fatal(err)
}
if hits.Load() != 0 {
t.Fatalf("fresh source must be skipped by prefetch, got %d HTTP hits", hits.Load())
}
// Stale source: last refresh older than its interval — prefetch must probe it.
staleAt := time.Now().UTC().Add(-7200 * time.Second)
if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
SourceKind: "txt",
URL: srv.URL + "?stale",
RefreshIntervalSec: &interval,
LastRefreshedAt: &staleAt,
}); err != nil {
t.Fatal(err)
}
if err := PrefetchCDNSourceETags(context.Background(), m, srv.Client()); err != nil {
t.Fatal(err)
}
if hits.Load() == 0 {
t.Fatal("stale source must be probed by prefetch")
}
}
internal/pipeline/cdn_snapshot.go
+30 -86
View File
@@ -42,32 +42,49 @@ func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.P
return out return out
} }
// mergeSnapshotDropCDNSources removes all cdn:* rows (used before batch CDN merge). // mergeSnapshotKeepSkippedCDN keeps non-CDN rows and cdn:* rows whose source is still skipped
func mergeSnapshotDropCDNSources(rows []store.PrefixRow) []store.PrefixRow { // (not in fetchedSourceIDs). Deleted sources (absent from allSourceIDs) are dropped.
if len(rows) == 0 { func mergeSnapshotKeepSkippedCDN(rows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) []store.PrefixRow {
return nil fetched := make(map[string]struct{}, len(fetchedSourceIDs))
for _, id := range fetchedSourceIDs {
fetched[cdnSourceKey(id)] = struct{}{}
}
keepCDN := make(map[string]struct{})
for _, id := range allSourceIDs {
k := cdnSourceKey(id)
if _, ok := fetched[k]; !ok {
keepCDN[k] = struct{}{}
}
} }
out := make([]store.PrefixRow, 0, len(rows)) out := make([]store.PrefixRow, 0, len(rows))
for _, row := range rows { for _, row := range rows {
if !strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") { src := strings.TrimSpace(row.Source)
out = append(out, row) if strings.HasPrefix(src, "cdn:") {
if _, ok := keepCDN[src]; !ok {
continue
}
} }
out = append(out, row)
} }
return out return out
} }
// mergeAllCDNSourcesIntoModuleSnapshot replaces all CDN rows in one write (avoids parallel read-modify-write races). // mergeAllCDNSourcesIntoModuleSnapshot replaces fetched CDN source rows in one write.
func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow) error { // skipped sources (errors with EVOBGP_CDN_PARTIAL_OK) keep their prior rows.
func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) error {
if st == nil || mod == nil { if st == nil || mod == nil {
return nil return nil
} }
unlock := st.LockModuleSnapshot(tenantID, mod.ID)
defer unlock()
var base []store.PrefixRow var base []store.PrefixRow
if len(priorSnapshot) > 0 { if len(priorSnapshot) > 0 {
base = mergeSnapshotDropCDNSources(priorSnapshot) base = priorSnapshot
} else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil { } else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
base = mergeSnapshotDropCDNSources(snap.Prefixes) base = snap.Prefixes
} }
merged := append(base, cdnRows...) kept := mergeSnapshotKeepSkippedCDN(base, fetchedSourceIDs, allSourceIDs)
merged := append(kept, cdnRows...)
return persistModuleSnapshot(st, tenantID, mod, merged) return persistModuleSnapshot(st, tenantID, mod, merged)
} }
@@ -93,6 +110,8 @@ func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *st
if st == nil || mod == nil { if st == nil || mod == nil {
return nil return nil
} }
unlock := st.LockModuleSnapshot(tenantID, mod.ID)
defer unlock()
sourceKey := cdnSourceKey(sourceID) sourceKey := cdnSourceKey(sourceID)
var base []store.PrefixRow var base []store.PrefixRow
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil { if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
@@ -114,81 +133,6 @@ func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
return out, nil return out, nil
} }
func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
u := strings.TrimSpace(src.URL)
if u == "" {
return nil, nil
}
if _, err := ValidateCDNURL(u); err != nil {
return nil, err
}
if err := ResolveCDNURLHost(ctx, u); err != nil {
return nil, err
}
sourceKey := cdnSourceKey(src.ID)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
if etag := strings.TrimSpace(src.Etag); etag != "" {
req.Header.Set("If-None-Match", etag)
}
resp, err := upstreamHTTPDo(ctx, hc, req)
if err != nil {
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
}
if resp.StatusCode == http.StatusNotModified {
if cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey); len(cached) > 0 {
_ = resp.Body.Close()
return cached, nil
}
// ETag is known but local snapshot is empty — force a full download.
_ = resp.Body.Close()
req2, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
resp, err = upstreamHTTPDo(ctx, hc, req2)
if err != nil {
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
}
if resp.StatusCode == http.StatusNotModified {
_ = resp.Body.Close()
return nil, fmt.Errorf("cdn url %s: 304 without cached prefixes", u)
}
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body)
return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
if err != nil {
return nil, err
}
prefixStrs, err := parseCDNBody(string(body), src)
if err != nil {
return nil, fmt.Errorf("cdn parse %s: %w", u, err)
}
etag := strings.TrimSpace(resp.Header.Get("ETag"))
patch := &store.CDNSourcePatch{}
if etag != "" && etag != strings.TrimSpace(src.Etag) {
e := etag
patch.Etag = &e
}
refreshedAt := now
patch.LastRefreshedAt = &refreshedAt
_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
rows := cdnRowsFromParsed(mod, src, prefixStrs)
if err := mergeCDNSourceIntoModuleSnapshot(st, tenantID, mod, src.ID, rows); err != nil {
return nil, err
}
return rows, nil
}
// fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once). // fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once).
func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) { func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
u := strings.TrimSpace(src.URL) u := strings.TrimSpace(src.URL)
internal/pipeline/cdn_url.go
+64
View File
@@ -7,7 +7,9 @@ import (
"net/netip" "net/netip"
"net/url" "net/url"
"os" "os"
"strconv"
"strings" "strings"
"sync"
"time" "time"
) )
@@ -91,6 +93,9 @@ func ResolveCDNURLHost(ctx context.Context, raw string) error {
if isBlockedCDNHostname(host) { if isBlockedCDNHostname(host) {
return fmt.Errorf("pipeline: cdn url blocked host") return fmt.Errorf("pipeline: cdn url blocked host")
} }
if ok := cdnDNSVerifyCache.hit(host); ok {
return nil
}
if ctx == nil { if ctx == nil {
ctx = context.Background() ctx = context.Background()
} }
@@ -112,5 +117,64 @@ func ResolveCDNURLHost(ctx context.Context, raw string) error {
return fmt.Errorf("pipeline: cdn url resolves to blocked address") return fmt.Errorf("pipeline: cdn url resolves to blocked address")
} }
} }
cdnDNSVerifyCache.store(host)
return nil return nil
} }
// cdnDNSVerifyTTL bounds how long a successful SSRF check is trusted for one hostname.
// Failures are never cached: a transient DNS outage must not open an unsafe window,
// and a blocked host is rejected before this cache anyway.
func cdnDNSVerifyTTL() time.Duration {
sec := 300
if s := strings.TrimSpace(os.Getenv("EVOBGP_CDN_DNS_CACHE_TTL_SEC")); s != "" {
if v, err := strconv.Atoi(s); err == nil && v > 0 {
sec = v
}
}
return time.Duration(sec) * time.Second
}
type dnsVerifyCache struct {
mu sync.Mutex
seen map[string]time.Time
}
var cdnDNSVerifyCache = &dnsVerifyCache{seen: make(map[string]time.Time)}
func (c *dnsVerifyCache) hit(host string) bool {
c.mu.Lock()
defer c.mu.Unlock()
at, ok := c.seen[host]
return ok && time.Since(at) < cdnDNSVerifyTTL()
}
func (c *dnsVerifyCache) store(host string) {
c.mu.Lock()
defer c.mu.Unlock()
if c.seen == nil {
c.seen = make(map[string]time.Time)
}
c.seen[host] = time.Now()
if len(c.seen) > 4096 {
// Size cap for long-running workers: drop expired entries, then the oldest if needed.
now := time.Now()
for h, at := range c.seen {
if now.Sub(at) >= cdnDNSVerifyTTL() {
delete(c.seen, h)
}
}
if len(c.seen) > 4096 {
var oldestK string
var oldestT time.Time
first := true
for h, at := range c.seen {
if first || at.Before(oldestT) {
oldestK, oldestT, first = h, at, false
}
}
if oldestK != "" {
delete(c.seen, oldestK)
}
}
}
}
internal/pipeline/collect_parallel.go
+36 -8
View File
@@ -26,6 +26,14 @@ func prefixRowsForSource(rows []store.PrefixRow, sourceKey string) []store.Prefi
return out return out
} }
func prefixCommunityKey(row store.PrefixRow) string {
comm := ""
if row.CommunityID != nil {
comm = strings.TrimSpace(*row.CommunityID)
}
return row.Prefix + "\x00" + comm
}
func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, list []*store.ASEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) { func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, list []*store.ASEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
moduleID := mod.ID moduleID := mod.ID
legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0" legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0"
@@ -147,7 +155,7 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client,
}) })
} }
for _, row := range r.rows { for _, row := range r.rows {
k := row.Prefix k := prefixCommunityKey(row)
if _, ok := seenPfx[k]; ok { if _, ok := seenPfx[k]; ok {
continue continue
} }
@@ -218,9 +226,14 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
} }
wg.Wait() wg.Wait()
var out []store.PrefixRow var fetchedRows []store.PrefixRow
var skipped int var skipped int
for _, r := range results { var fetchedIDs []string
allIDs := make([]string, 0, len(valid))
for _, src := range valid {
allIDs = append(allIDs, src.ID)
}
for i, r := range results {
if r.err != nil { if r.err != nil {
if cdnPartialOK() { if cdnPartialOK() {
logging.Default().Info(fmt.Sprintf("pipeline: CDN partial skip source error: %v", r.err)) logging.Default().Info(fmt.Sprintf("pipeline: CDN partial skip source error: %v", r.err))
@@ -229,20 +242,35 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
} }
return nil, r.err return nil, r.err
} }
out = append(out, r.rows...) fetchedIDs = append(fetchedIDs, valid[i].ID)
fetchedRows = append(fetchedRows, r.rows...)
} }
if skipped > 0 && len(out) == 0 && len(valid) > 0 { if skipped > 0 && len(fetchedRows) == 0 && len(valid) > 0 {
return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid)) return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid))
} }
out := fetchedRows
if skipped > 0 {
base := priorSnapshot
if len(base) == 0 {
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
base = snap.Prefixes
}
}
for _, row := range mergeSnapshotKeepSkippedCDN(base, fetchedIDs, allIDs) {
if strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") {
out = append(out, row)
}
}
}
if len(valid) > 0 { if len(valid) > 0 {
if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, out); err != nil { if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, fetchedRows, fetchedIDs, allIDs); err != nil {
return nil, err return nil, err
} }
} }
return out, nil return out, nil
} }
func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) { func collectDomainPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
var validDom []*store.DomainEntry var validDom []*store.DomainEntry
for _, e := range entries { for _, e := range entries {
if e != nil { if e != nil {
@@ -269,7 +297,7 @@ func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Mo
c := *mod.DefaultCommunityID c := *mod.DefaultCommunityID
comm = &c comm = &c
} }
addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, entry.FQDN) addrs, err := resolveDomainIPsCached(ctx, st, hc, profiles, policy, entry.FQDN)
if err != nil { if err != nil {
if staleOnUpstreamError() { if staleOnUpstreamError() {
if cached, ok := staleDomainPrefixes(priorSnapshot, entry.FQDN); ok { if cached, ok := staleDomainPrefixes(priorSnapshot, entry.FQDN); ok {
internal/pipeline/doh_resolve.go
+19 -2
View File
@@ -6,6 +6,7 @@ import (
"net/http" "net/http"
"net/netip" "net/netip"
"strings" "strings"
"sync"
"time" "time"
"evobgp/internal/store" "evobgp/internal/store"
@@ -108,8 +109,7 @@ func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, prof
defer cancel() defer cancel()
baseURL := strings.TrimSpace(profile.URL) baseURL := strings.TrimSpace(profile.URL)
v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA) v4, v6, err4, err6 := resolveDOHMessagePair(dctx, hc, baseURL, host)
v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA)
if err4 != nil { if err4 != nil {
v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A") v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
} }
@@ -122,6 +122,23 @@ func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, prof
return uniqAddrs(append(v4, v6...)), nil return uniqAddrs(append(v4, v6...)), nil
} }
// resolveDOHMessagePair issues RFC8484 dns-message A and AAAA queries concurrently
// and waits for both (fallbacks are handled by the caller).
func resolveDOHMessagePair(ctx context.Context, hc *http.Client, baseURL, host string) (v4, v6 []netip.Addr, err4, err6 error) {
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
v4, err4 = resolveDomainWithDOHMessage(ctx, hc, baseURL, host, dns.TypeA)
}()
go func() {
defer wg.Done()
v6, err6 = resolveDomainWithDOHMessage(ctx, hc, baseURL, host, dns.TypeAAAA)
}()
wg.Wait()
return v4, v6, err4, err6
}
func dohProfileTimeout(profile *store.DohProfile) time.Duration { func dohProfileTimeout(profile *store.DohProfile) time.Duration {
timeout := 10 * time.Second timeout := 10 * time.Second
if profile != nil && profile.TimeoutMs != nil && *profile.TimeoutMs > 0 { if profile != nil && profile.TimeoutMs != nil && *profile.TimeoutMs > 0 {
internal/pipeline/doh_resolve_test.go
+79 -129
View File
@@ -2,147 +2,97 @@ package pipeline
import ( import (
"context" "context"
"encoding/base64"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"sync/atomic"
"testing" "testing"
"time"
"evobgp/internal/store" "evobgp/internal/store"
"github.com/miekg/dns"
) )
func TestResolveDomainIPsWithPolicy_Union(t *testing.T) { // TestResolveDomainIPs_ParallelAAndAAAA verifies that A and AAAA queries are issued
srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // concurrently: with a 250ms upstream latency the combined resolve must stay near
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.1"}]}`)) // one round-trip instead of two.
})) func TestResolveDomainIPs_ParallelAAndAAAA(t *testing.T) {
defer srvRU.Close() const delay = 250 * time.Millisecond
srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var inflight, maxInflight atomic.Int32
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.1"}]}`)) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
})) cur := inflight.Add(1)
defer srvEU.Close() for {
old := maxInflight.Load()
if cur <= old || maxInflight.CompareAndSwap(old, cur) {
break
}
}
defer inflight.Add(-1)
time.Sleep(delay)
profiles := []*store.DohProfile{ if wire := r.URL.Query().Get("dns"); wire != "" {
{URL: srvRU.URL}, // RFC8484 dns-message: decode the query and answer on the wire.
{URL: srvEU.URL}, raw, err := base64.RawURLEncoding.DecodeString(wire)
} if err != nil {
ips, err := resolveDomainIPsWithPolicy(context.Background(), srvRU.Client(), profiles, store.DohPolicyUnion, "example.com") w.WriteHeader(http.StatusBadRequest)
return
}
msg := new(dns.Msg)
if err := msg.Unpack(raw); err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
resp := new(dns.Msg)
resp.SetReply(msg)
switch msg.Question[0].Qtype {
case dns.TypeA:
resp.Answer = append(resp.Answer, &dns.A{
Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
A: []byte{203, 0, 113, 10},
})
case dns.TypeAAAA:
resp.Answer = append(resp.Answer, &dns.AAAA{
Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60},
AAAA: []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x10},
})
}
out, err := resp.Pack()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/dns-message")
_, _ = w.Write(out)
return
}
w.Header().Set("Content-Type", "application/dns-json")
switch r.URL.Query().Get("type") {
case "A":
_, _ = w.Write([]byte(`{"Status":0,"Answer":[{"type":1,"data":"203.0.113.10"}]}`))
default:
_, _ = w.Write([]byte(`{"Status":0,"Answer":[{"type":28,"data":"2001:db8::10"}]}`))
}
}))
defer srv.Close()
prof := &store.DohProfile{URL: srv.URL, TimeoutMs: ptrInt(5000)}
ctx := context.Background()
start := time.Now()
ips, err := resolveDomainIPsNoSystemFallback(ctx, srv.Client(), prof, "example.test")
elapsed := time.Since(start)
if err != nil { if err != nil {
t.Fatal(err) t.Fatalf("resolve failed: %v", err)
} }
if len(ips) != 2 { if len(ips) != 2 {
t.Fatalf("want 2 ips, got %v", ips) t.Fatalf("expected 2 addrs, got %+v", ips)
} }
seen := map[string]bool{ips[0].String(): true, ips[1].String(): true} if maxInflight.Load() < 2 {
if !seen["198.51.100.1"] || !seen["203.0.113.1"] { t.Fatalf("expected concurrent A/AAAA queries, max inflight=%d", maxInflight.Load())
t.Fatalf("unexpected ips: %v", ips) }
if elapsed >= 2*delay {
t.Fatalf("resolve took %v; expected one round-trip (<2*%v)", elapsed, delay)
} }
} }
func TestResolveDomainIPsWithPolicy_Failover(t *testing.T) { func ptrInt(v int) *int { return &v }
var calls int
srvBad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
http.Error(w, "fail", http.StatusBadGateway)
}))
defer srvBad.Close()
srvOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.5"}]}`))
}))
defer srvOK.Close()
profiles := []*store.DohProfile{
{URL: srvBad.URL},
{URL: srvOK.URL},
}
ips, err := resolveDomainIPsWithPolicy(context.Background(), srvBad.Client(), profiles, store.DohPolicyFailover, "example.com")
if err != nil {
t.Fatal(err)
}
if len(ips) != 1 || ips[0].String() != "198.51.100.5" {
t.Fatalf("unexpected ips: %v", ips)
}
if calls < 2 {
t.Fatalf("want at least 2 resolver calls, got %d", calls)
}
}
func TestResolveDomainIPsWithPolicy_PrimaryOnly(t *testing.T) {
var secondCalled bool
srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.9"}]}`))
}))
defer srv1.Close()
srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
secondCalled = true
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.9"}]}`))
}))
defer srv2.Close()
profiles := []*store.DohProfile{
{URL: srv1.URL},
{URL: srv2.URL},
}
ips, err := resolveDomainIPsWithPolicy(context.Background(), srv1.Client(), profiles, store.DohPolicyPrimaryOnly, "example.com")
if err != nil {
t.Fatal(err)
}
if len(ips) != 1 || ips[0].String() != "198.51.100.9" {
t.Fatalf("unexpected ips: %v", ips)
}
if secondCalled {
t.Fatal("secondary resolver must not be queried in primary_only mode")
}
}
func TestCollectModulePrefixRows_DohUnion(t *testing.T) {
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
mod, err := m.CreateModule(tenant, &store.Module{
Type: "DOMAINS",
Name: "domains-union",
Enabled: true,
DohResolverPolicy: store.DohPolicyUnion,
})
if err != nil {
t.Fatal(err)
}
srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.2"}]}`))
}))
defer srvRU.Close()
srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.2"}]}`))
}))
defer srvEU.Close()
ru, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "ru", URL: srvRU.URL})
if err != nil {
t.Fatal(err)
}
eu, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "eu", URL: srvEU.URL})
if err != nil {
t.Fatal(err)
}
if _, err := m.UpdateModule(tenant, mod.ID, &store.ModulePatch{
DohProfileIDs: &[]string{ru.ID, eu.ID},
}); err != nil {
t.Fatal(err)
}
mod, err = m.GetModule(tenant, mod.ID)
if err != nil {
t.Fatal(err)
}
if _, err := m.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "svc.example.com"}); err != nil {
t.Fatal(err)
}
rows, err := collectModulePrefixRows(context.Background(), m, srvRU.Client(), tenant, mod, nil)
if err != nil {
t.Fatal(err)
}
if len(rows) != 2 {
t.Fatalf("want 2 prefix rows, got %+v", rows)
}
}
internal/pipeline/domain_cache.go
+67
View File
@@ -0,0 +1,67 @@
package pipeline
import (
"context"
"net/http"
"net/netip"
"os"
"strconv"
"strings"
"time"
"evobgp/internal/store"
)
func domainCacheTTL() time.Duration {
sec := 300
if s := strings.TrimSpace(os.Getenv("EVOBGP_DOMAIN_CACHE_TTL_SEC")); s != "" {
if v, err := strconv.Atoi(s); err == nil && v > 0 {
sec = v
}
}
return time.Duration(sec) * time.Second
}
func resolveDomainIPsCached(ctx context.Context, st store.Backend, hc *http.Client, profiles []*store.DohProfile, policy, fqdn string) ([]netip.Addr, error) {
key := strings.TrimSpace(fqdn)
ttl := domainCacheTTL()
if st != nil && ttl > 0 {
if ent, ok, err := st.GetDomainResolveCache(key); err == nil && ok && ent != nil {
if time.Since(ent.ResolvedAt) < ttl {
if addrs := parseCachedDomainAddrs(ent.Addrs); len(addrs) > 0 {
return addrs, nil
}
}
}
}
addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, key)
if err != nil {
return nil, err
}
if st != nil {
_ = st.SetDomainResolveCache(key, domainAddrsToStrings(addrs))
}
return addrs, nil
}
func parseCachedDomainAddrs(raw []string) []netip.Addr {
out := make([]netip.Addr, 0, len(raw))
for _, s := range raw {
a, err := netip.ParseAddr(strings.TrimSpace(s))
if err != nil {
continue
}
out = append(out, a)
}
return out
}
func domainAddrsToStrings(addrs []netip.Addr) []string {
out := make([]string, 0, len(addrs))
for _, a := range addrs {
if a.IsValid() {
out = append(out, a.String())
}
}
return out
}
internal/pipeline/module_hash.go
+21 -93
View File
@@ -1,10 +1,6 @@
package pipeline package pipeline
import ( import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings" "strings"
"evobgp/internal/store" "evobgp/internal/store"
@@ -12,89 +8,7 @@ import (
// moduleIngestInputHash fingerprints module config and child entries so snapshots invalidate on CRUD. // moduleIngestInputHash fingerprints module config and child entries so snapshots invalidate on CRUD.
func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module) (string, error) { func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module) (string, error) {
if st == nil || mod == nil { return store.ComputeModuleInputHash(st, tenantID, mod)
return "", fmt.Errorf("pipeline: module hash: missing store or module")
}
h := sha256.New()
_, _ = fmt.Fprintf(h, "type=%s\n", strings.TrimSpace(mod.Type))
_, _ = fmt.Fprintf(h, "enabled=%t\n", mod.Enabled)
if mod.DefaultCommunityID != nil {
_, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID))
}
_, _ = fmt.Fprintf(h, "doh_policy=%s\n", store.NormalizeDohResolverPolicy(mod.DohResolverPolicy))
for _, pid := range mod.EffectiveDohProfileIDs() {
_, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid)
if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil {
_, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL))
if prof.TimeoutMs != nil {
_, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs)
}
}
}
switch mod.Type {
case "IP_RANGES":
list, err := st.ListIPRangeEntries(tenantID, mod.ID)
if err != nil {
return "", err
}
sort.Slice(list, func(i, j int) bool { return list[i].Prefix < list[j].Prefix })
for _, e := range list {
comm := ""
if e.CommunityID != nil {
comm = *e.CommunityID
}
_, _ = fmt.Fprintf(h, "ip=%s|c=%s\n", e.Prefix, comm)
}
case "AS_PREFIXES":
list, err := st.ListASEntries(tenantID, mod.ID)
if err != nil {
return "", err
}
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
for _, e := range list {
comm := ""
if e.CommunityID != nil {
comm = *e.CommunityID
}
_, _ = fmt.Fprintf(h, "as=%d|c=%s\n", e.ASN, comm)
}
case "CDN_CIDRS":
list, err := st.ListCDNSources(tenantID, mod.ID)
if err != nil {
return "", err
}
sort.Slice(list, func(i, j int) bool { return list[i].ID < list[j].ID })
for _, s := range list {
comm := ""
if s.CommunityID != nil {
comm = *s.CommunityID
}
interval := 0
if s.RefreshIntervalSec != nil {
interval = *s.RefreshIntervalSec
}
_, _ = fmt.Fprintf(h, "cdn=%s|url=%s|kind=%s|path=%s|c=%s|etag=%s|interval=%d\n",
s.ID, strings.TrimSpace(s.URL), s.SourceKind, strings.TrimSpace(s.PrefixPath), comm,
strings.TrimSpace(s.Etag), interval)
}
case "DOMAINS":
list, err := st.ListDomainEntries(tenantID, mod.ID)
if err != nil {
return "", err
}
sort.Slice(list, func(i, j int) bool { return list[i].FQDN < list[j].FQDN })
for _, e := range list {
comm := ""
if e.CommunityID != nil {
comm = *e.CommunityID
}
_, _ = fmt.Fprintf(h, "dom=%s|c=%s\n", strings.TrimSpace(e.FQDN), comm)
}
default:
_, _ = fmt.Fprintf(h, "unknown_type=%s\n", mod.Type)
}
return hex.EncodeToString(h.Sum(nil)), nil
} }
func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error { func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error {
@@ -105,18 +19,32 @@ func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module,
if err != nil { if err != nil {
return err return err
} }
return st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows) if err := st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows); err != nil {
return err
}
_ = st.SetModuleInputHash(tenantID, mod.ID, hash)
return nil
} }
func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module) ([]store.PrefixRow, bool, error) { func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module) ([]store.PrefixRow, bool, error) {
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
if err != nil || !ok || snap == nil {
return nil, false, err
}
if h := strings.TrimSpace(mod.InputHash); h != "" {
if snap.InputHash != h {
return nil, false, nil
}
return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil
}
// Fallback for rows written before module.input_hash existed: recompute and backfill.
hash, err := moduleIngestInputHash(st, tenantID, mod) hash, err := moduleIngestInputHash(st, tenantID, mod)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID) if snap.InputHash != hash {
if err != nil || !ok || snap == nil || snap.InputHash != hash { return nil, false, nil
return nil, false, err
} }
cp := append([]store.PrefixRow(nil), snap.Prefixes...) _ = st.SetModuleInputHash(tenantID, mod.ID, hash)
return cp, true, nil return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil
} }
internal/pipeline/opt_collect_test.go
+189
View File
@@ -0,0 +1,189 @@
package pipeline
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"evobgp/internal/store"
)
func TestModuleDueForScheduler_JitterSpreads(t *testing.T) {
dueTicks := map[int64]int{}
for i := 0; i < 100; i++ {
mod := &store.Module{
ID: fmt.Sprintf("module-%d", i),
Enabled: true,
Type: "AS_PREFIXES",
RefreshIntervalSec: 3600,
}
for tick := int64(0); tick < 7200; tick += SchedulerTickSec {
if ModuleDueForScheduler(mod, time.Unix(tick, 0)) {
dueTicks[tick]++
}
}
}
if len(dueTicks) < 10 {
t.Fatalf("expected due events across many ticks, got %d buckets", len(dueTicks))
}
for tick, n := range dueTicks {
if n == 100 {
t.Fatalf("all 100 modules due on tick %d", tick)
}
}
}
func TestCollectASPrefixRows_KeepsSamePrefixDifferentCommunity(t *testing.T) {
st := store.NewMemory()
st.SeedDemo()
tenant, _, _, _, _ := st.DemoIDs()
mod, err := st.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "as-dedup", Enabled: true})
if err != nil {
t.Fatal(err)
}
c1, c2 := "comm-a", "comm-b"
e1, err := st.CreateASEntry(tenant, mod.ID, &store.ASEntry{ASN: 64500, CommunityID: &c1})
if err != nil {
t.Fatal(err)
}
e2, err := st.CreateASEntry(tenant, mod.ID, &store.ASEntry{ASN: 64501, CommunityID: &c2})
if err != nil {
t.Fatal(err)
}
if err := st.SetASNPrefixCache(64500, "a", []string{"192.0.2.0/24"}); err != nil {
t.Fatal(err)
}
if err := st.SetASNPrefixCache(64501, "b", []string{"192.0.2.0/24"}); err != nil {
t.Fatal(err)
}
rows, err := collectASPrefixRows(context.Background(), st, http.DefaultClient, tenant, mod, []*store.ASEntry{e1, e2}, nil)
if err != nil {
t.Fatal(err)
}
if len(rows) != 2 {
t.Fatalf("expected 2 rows (same prefix, different community), got %d: %+v", len(rows), rows)
}
}
func TestCollectCDNPrefixRows_PartialSkipKeepsPrior(t *testing.T) {
t.Setenv("EVOBGP_CDN_PARTIAL_OK", "1")
t.Setenv("EVOBGP_STALE_ON_UPSTREAM_ERROR", "0")
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
good := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("198.51.100.0/24\n"))
}))
defer good.Close()
bad := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "down", http.StatusServiceUnavailable)
}))
defer bad.Close()
st := store.NewMemory()
st.SeedDemo()
tenant, _, _, _, _ := st.DemoIDs()
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-partial", Enabled: true})
if err != nil {
t.Fatal(err)
}
prior := []store.PrefixRow{
{Prefix: "203.0.113.0/24", Source: "cdn:bad"},
{Prefix: "1.2.3.0/24", Source: "cdn:good"},
}
sources := []*store.CDNSource{
{ID: "good", URL: good.URL, SourceKind: "plain"},
{ID: "bad", URL: bad.URL, SourceKind: "plain"},
}
rows, err := collectCDNPrefixRows(context.Background(), st, good.Client(), tenant, mod, sources, prior)
if err != nil {
t.Fatalf("partial skip should succeed: %v", err)
}
got := map[string]string{}
for _, r := range rows {
got[r.Source] = r.Prefix
}
if got["cdn:bad"] != "203.0.113.0/24" {
t.Fatalf("skipped source lost prior row: %+v", rows)
}
if got["cdn:good"] != "198.51.100.0/24" {
t.Fatalf("fetched source missing new row: %+v", rows)
}
}
func TestMergeCDNSource_ParallelNoLostUpdate(t *testing.T) {
st := store.NewMemory()
st.SeedDemo()
tenant, _, _, _, _ := st.DemoIDs()
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-lock", Enabled: true})
if err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
_ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s1", []store.PrefixRow{
{Prefix: "1.0.0.0/24", Source: "cdn:s1"},
})
}()
go func() {
defer wg.Done()
_ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s2", []store.PrefixRow{
{Prefix: "2.0.0.0/24", Source: "cdn:s2"},
})
}()
wg.Wait()
snap, ok, err := st.GetModulePrefixSnapshot(tenant, mod.ID)
if err != nil || !ok || snap == nil {
t.Fatalf("snapshot missing: ok=%v err=%v", ok, err)
}
got := map[string]bool{}
for _, r := range snap.Prefixes {
got[r.Prefix] = true
}
if !got["1.0.0.0/24"] || !got["2.0.0.0/24"] {
t.Fatalf("lost parallel merge update: %+v", snap.Prefixes)
}
}
func TestResolveDomainIPsCached_UsesTTL(t *testing.T) {
t.Setenv("EVOBGP_DOMAIN_CACHE_TTL_SEC", "300")
st := store.NewMemory()
if err := st.SetDomainResolveCache("cached.test", []string{"192.0.2.9"}); err != nil {
t.Fatal(err)
}
addrs, err := resolveDomainIPsCached(context.Background(), st, nil, nil, "", "cached.test")
if err != nil {
t.Fatal(err)
}
if len(addrs) != 1 || addrs[0].String() != "192.0.2.9" {
t.Fatalf("expected cached addr, got %v", addrs)
}
}
func TestInputHashInvalidatedOnEntryCRUD(t *testing.T) {
st := store.NewMemory()
st.SeedDemo()
tenant, _, _, _, _ := st.DemoIDs()
mod, err := st.CreateModule(tenant, &store.Module{Type: "IP_RANGES", Name: "ip-hash", Enabled: true})
if err != nil {
t.Fatal(err)
}
if err := st.SetModuleInputHash(tenant, mod.ID, "pre"); err != nil {
t.Fatal(err)
}
if _, err := st.CreateIPRangeEntry(tenant, mod.ID, &store.IPRangeEntry{Prefix: "10.0.0.0/8"}); err != nil {
t.Fatal(err)
}
got, err := st.GetModule(tenant, mod.ID)
if err != nil {
t.Fatal(err)
}
if got.InputHash != "" {
t.Fatalf("expected hash cleared after CRUD, got %q", got.InputHash)
}
}
internal/pipeline/prefetch.go
+13 -47
View File
@@ -2,7 +2,6 @@ package pipeline
import ( import (
"context" "context"
"io"
"net/http" "net/http"
"strings" "strings"
"sync" "sync"
@@ -31,6 +30,7 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
return err return err
} }
var tasks []prefetchTask var tasks []prefetchTask
now := time.Now().UTC()
for _, tid := range tenants { for _, tid := range tenants {
for _, mod := range st.ListModules(tid) { for _, mod := range st.ListModules(tid) {
if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" { if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
@@ -41,9 +41,15 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
continue continue
} }
for _, src := range sources { for _, src := range sources {
if src != nil && strings.TrimSpace(src.URL) != "" { if src == nil || strings.TrimSpace(src.URL) == "" {
tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src}) continue
} }
// Respect per-source refresh intervals: conditional GET only for due sources.
// The ETag probe still lets 304s skip body downloads for the rest.
if shouldSkipCDNSourceFetch(src, now) {
continue
}
tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src})
} }
} }
} }
@@ -68,53 +74,13 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) { func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) {
now := time.Now().UTC() now := time.Now().UTC()
tid, mod, src := t.tenantID, t.mod, t.src tid, mod, src := t.tenantID, t.mod, t.src
u := strings.TrimSpace(src.URL)
if _, err := ValidateCDNURL(u); err != nil {
return
}
if err := ResolveCDNURLHost(ctx, u); err != nil {
return
}
omod, err := st.GetModule(tid, mod.ID) omod, err := st.GetModule(tid, mod.ID)
if err != nil || omod == nil {
return
}
rows, err := fetchCDNSourceRows(ctx, st, hc, tid, mod.ID, omod, src, nil, now)
if err != nil { if err != nil {
return return
} }
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return
}
if etag := strings.TrimSpace(src.Etag); etag != "" {
req.Header.Set("If-None-Match", etag)
}
resp, err := upstreamHTTPDo(ctx, hc, req)
if err != nil {
return
}
if resp.StatusCode == http.StatusNotModified {
_ = resp.Body.Close()
return
}
if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
return
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
_ = resp.Body.Close()
if err != nil {
return
}
prefixStrs, err := parseCDNBody(string(body), src)
if err != nil {
return
}
newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
e := newEtag
patch.Etag = &e
}
_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
rows := cdnRowsFromParsed(omod, src, prefixStrs)
_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows) _ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
} }
internal/pipeline/refresh.go
+5 -164
View File
@@ -8,7 +8,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"math/big"
"net" "net"
"net/http" "net/http"
"net/netip" "net/netip"
@@ -190,7 +189,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
if err != nil { if err != nil {
return nil, err return nil, err
} }
return collectDomainPrefixRows(ctx, hc, mod, profiles, policy, entries, priorSnapshot) return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, priorSnapshot)
default: default:
return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type) return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type)
} }
@@ -230,9 +229,8 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr
defer cancel() defer cancel()
baseURL := strings.TrimSpace(profile.URL) baseURL := strings.TrimSpace(profile.URL)
// Prefer RFC8484 dns-message transport. Some providers don't support dns-json. // A and AAAA queries run concurrently: per-domain latency drops from ~2×RTT to ~1×RTT.
v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA) v4, v6, err4, err6 := resolveDOHMessagePair(dctx, hc, baseURL, host)
v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA)
if err4 != nil { if err4 != nil {
// Fallback to JSON mode for providers that only expose dns-json. // Fallback to JSON mode for providers that only expose dns-json.
v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A") v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
@@ -496,168 +494,11 @@ func prefixRowCommunity(r store.PrefixRow) string {
} }
func aggregateIPv4Group(rows []store.PrefixRow) []store.PrefixRow { func aggregateIPv4Group(rows []store.PrefixRow) []store.PrefixRow {
return aggregateCIDRGroup(rows, mergeSiblingPrefixesIPv4) return collapsePrefixGroup(rows, true)
} }
func aggregateIPv6Group(rows []store.PrefixRow) []store.PrefixRow { func aggregateIPv6Group(rows []store.PrefixRow) []store.PrefixRow {
return aggregateCIDRGroup(rows, mergeSiblingPrefixesIPv6) return collapsePrefixGroup(rows, false)
}
func aggregateCIDRGroup(rows []store.PrefixRow, mergeFn func(map[string]store.PrefixRow) bool) []store.PrefixRow {
if len(rows) <= 1 {
return rows
}
set := make(map[string]store.PrefixRow, len(rows))
for _, row := range rows {
set[row.Prefix] = row
}
pruneCoveredPrefixes(set)
for {
if !mergeFn(set) {
break
}
pruneCoveredPrefixes(set)
}
out := make([]store.PrefixRow, 0, len(set))
for _, row := range set {
out = append(out, row)
}
sortPrefixRows(out)
return out
}
func pruneCoveredPrefixes(set map[string]store.PrefixRow) {
type item struct {
key string
pfx netip.Prefix
bits int
}
items := make([]item, 0, len(set))
for k := range set {
p, err := netip.ParsePrefix(k)
if err != nil {
continue
}
items = append(items, item{key: k, pfx: p, bits: p.Bits()})
}
sort.Slice(items, func(i, j int) bool {
if items[i].bits != items[j].bits {
return items[i].bits < items[j].bits
}
return items[i].key < items[j].key
})
for i := 0; i < len(items); i++ {
for j := i + 1; j < len(items); j++ {
if items[j].bits <= items[i].bits {
continue
}
if items[i].pfx.Contains(items[j].pfx.Addr()) {
delete(set, items[j].key)
}
}
}
}
func mergeSiblingPrefixesIPv4(set map[string]store.PrefixRow) bool {
merged := false
seen := make(map[string]struct{}, len(set))
for key, row := range set {
if _, done := seen[key]; done {
continue
}
pfx, err := netip.ParsePrefix(key)
if err != nil || !pfx.Addr().Is4() {
continue
}
bits := pfx.Bits()
if bits <= 8 {
continue
}
netNum := ipv4PrefixNetwork(pfx)
blockSize := uint32(1) << (32 - bits)
siblingNet := netNum ^ blockSize
siblingPfx := netip.PrefixFrom(u32ToIPv4(siblingNet), bits).Masked().String()
_, ok := set[siblingPfx]
if !ok {
continue
}
parentBits := bits - 1
parentBlock := uint32(1) << (32 - parentBits)
parentNet := netNum & ^(parentBlock - 1)
parentPfx := netip.PrefixFrom(u32ToIPv4(parentNet), parentBits).Masked().String()
delete(set, key)
delete(set, siblingPfx)
parentRow := row
parentRow.Prefix = parentPfx
set[parentPfx] = parentRow
seen[key] = struct{}{}
seen[siblingPfx] = struct{}{}
merged = true
}
return merged
}
func ipv4PrefixNetwork(p netip.Prefix) uint32 {
a := p.Masked().Addr().As4()
return uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3])
}
func u32ToIPv4(v uint32) netip.Addr {
return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
}
func mergeSiblingPrefixesIPv6(set map[string]store.PrefixRow) bool {
merged := false
seen := make(map[string]struct{}, len(set))
for key, row := range set {
if _, done := seen[key]; done {
continue
}
pfx, err := netip.ParsePrefix(key)
if err != nil || !pfx.Addr().Is6() {
continue
}
bits := pfx.Bits()
if bits <= 16 {
continue
}
netNum := ipv6PrefixNetwork(pfx)
blockSize := new(big.Int).Lsh(big.NewInt(1), uint(128-bits))
siblingNet := new(big.Int).Xor(netNum, blockSize)
siblingPfx := ipv6PrefixFromBigInt(siblingNet, bits).String()
if _, ok := set[siblingPfx]; !ok {
continue
}
parentBits := bits - 1
parentBlock := new(big.Int).Lsh(big.NewInt(1), uint(128-parentBits))
mask := new(big.Int).Sub(parentBlock, big.NewInt(1))
mask.Not(mask)
parentNet := new(big.Int).And(netNum, mask)
parentPfx := ipv6PrefixFromBigInt(parentNet, parentBits).String()
delete(set, key)
delete(set, siblingPfx)
parentRow := row
parentRow.Prefix = parentPfx
set[parentPfx] = parentRow
seen[key] = struct{}{}
seen[siblingPfx] = struct{}{}
merged = true
}
return merged
}
func ipv6PrefixNetwork(p netip.Prefix) *big.Int {
a := p.Masked().Addr().As16()
n := new(big.Int)
n.SetBytes(a[:])
return n
}
func ipv6PrefixFromBigInt(n *big.Int, bits int) netip.Prefix {
b := n.Bytes()
var a [16]byte
copy(a[16-len(b):], b)
return netip.PrefixFrom(netip.AddrFrom16(a), bits).Masked()
} }
func parentRevision(st store.Backend, tenantID, moduleID string) *string { func parentRevision(st store.Backend, tenantID, moduleID string) *string {
internal/pipeline/scheduler_due.go
+19 -2
View File
@@ -1,6 +1,7 @@
package pipeline package pipeline
import ( import (
"hash/fnv"
"time" "time"
"evobgp/internal/store" "evobgp/internal/store"
@@ -10,6 +11,8 @@ import (
const SchedulerTickSec = 30 const SchedulerTickSec = 30
// ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick. // ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick.
// A stable per-module offset (fnv32 of ID) spreads bucket boundaries so modules with the same interval
// do not all become due on the same tick (thundering herd).
func ModuleDueForScheduler(mod *store.Module, now time.Time) bool { func ModuleDueForScheduler(mod *store.Module, now time.Time) bool {
if mod == nil || !mod.Enabled || mod.Type == "IP_RANGES" || mod.RefreshIntervalSec <= 0 { if mod == nil || !mod.Enabled || mod.Type == "IP_RANGES" || mod.RefreshIntervalSec <= 0 {
return false return false
@@ -18,7 +21,21 @@ func ModuleDueForScheduler(mod *store.Module, now time.Time) bool {
if win < 60 { if win < 60 {
win = 60 win = 60
} }
cur := now.Unix() / win offset := moduleSchedulerOffset(mod.ID, win)
prev := (now.Unix() - SchedulerTickSec) / win cur := (now.Unix() - offset) / win
prev := (now.Unix() - offset - SchedulerTickSec) / win
return cur != prev return cur != prev
} }
func moduleSchedulerOffset(moduleID string, win int64) int64 {
if win <= 0 {
return 0
}
return int64(fnv32a(moduleID) % uint32(win))
}
func fnv32a(s string) uint32 {
h := fnv.New32a()
_, _ = h.Write([]byte(s))
return h.Sum32()
}
internal/repository/asn_cache.go
+20 -6
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"sync"
"time" "time"
"evobgp/internal/store" "evobgp/internal/store"
@@ -11,13 +12,26 @@ import (
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
) )
// asnCacheRowExistsOnce caches the schema check for the process lifetime (see moduleSnapshotRowTableExists).
var (
asnCacheRowExistsOnce sync.Once
asnCacheRowExistsCached bool
)
func asnCacheRowTableExists(ctx context.Context, q queryRower) bool { func asnCacheRowTableExists(ctx context.Context, q queryRower) bool {
var n int asnCacheRowExistsOnce.Do(func() {
err := q.QueryRow(ctx, ` var n int
SELECT 1 FROM information_schema.tables err := q.QueryRow(ctx, `
WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row' SELECT 1 FROM information_schema.tables
LIMIT 1`).Scan(&n) WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row'
return err == nil LIMIT 1`).Scan(&n)
if err != nil {
asnCacheRowExistsOnce = sync.Once{}
return
}
asnCacheRowExistsCached = true
})
return asnCacheRowExistsCached
} }
func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, bool, error) { func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, bool, error) {
internal/repository/domain_cache.go
+64
View File
@@ -0,0 +1,64 @@
package repository
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"evobgp/internal/store"
"github.com/jackc/pgx/v5"
)
func (p *Postgres) GetDomainResolveCache(fqdn string) (*store.DomainResolveCacheEntry, bool, error) {
key := strings.ToLower(strings.TrimSpace(fqdn))
if key == "" {
return nil, false, nil
}
ctx := context.Background()
var raw []byte
var resolvedAt time.Time
err := p.pool.QueryRow(ctx, `
SELECT addrs_json, resolved_at FROM domain_resolve_cache WHERE fqdn = $1`, key).
Scan(&raw, &resolvedAt)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, false, nil
}
return nil, false, err
}
var addrs []string
if len(raw) > 0 {
_ = json.Unmarshal(raw, &addrs)
}
return &store.DomainResolveCacheEntry{
FQDN: key,
Addrs: addrs,
ResolvedAt: resolvedAt.UTC(),
}, true, nil
}
func (p *Postgres) SetDomainResolveCache(fqdn string, addrs []string) error {
key := strings.ToLower(strings.TrimSpace(fqdn))
if key == "" {
return store.ErrInvalidInput
}
if addrs == nil {
addrs = []string{}
}
raw, err := json.Marshal(addrs)
if err != nil {
return err
}
ctx := context.Background()
_, err = p.pool.Exec(ctx, `
INSERT INTO domain_resolve_cache (fqdn, addrs_json, resolved_at)
VALUES ($1, $2::jsonb, now())
ON CONFLICT (fqdn) DO UPDATE SET
addrs_json = EXCLUDED.addrs_json,
resolved_at = EXCLUDED.resolved_at`,
key, string(raw))
return err
}
internal/repository/module_snapshot.go
+74 -19
View File
@@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"strings" "strings"
"sync"
"time" "time"
"evobgp/internal/store" "evobgp/internal/store"
@@ -12,13 +13,29 @@ import (
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
) )
// moduleSnapshotRowExistsOnce caches the schema check for the process lifetime.
// The table is created by migrations and never disappears at runtime; DB errors are
// not cached so a transient outage falls back to the JSON path only once.
var moduleSnapshotRowExistsOnce sync.Once
var moduleSnapshotRowExistsCached bool
func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool { func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool {
var n int moduleSnapshotRowExistsOnce.Do(func() {
err := q.QueryRow(ctx, ` var n int
SELECT 1 FROM information_schema.tables err := q.QueryRow(ctx, `
WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row' SELECT 1 FROM information_schema.tables
LIMIT 1`).Scan(&n) WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row'
return err == nil LIMIT 1`).Scan(&n)
// A query error means the backend is unreachable or information_schema is hidden;
// treat as "absent" so callers fall back to prefixes_json, but retry next call.
if err != nil {
moduleSnapshotRowExistsOnce = sync.Once{}
return
}
moduleSnapshotRowExistsCached = true
})
return moduleSnapshotRowExistsCached
} }
func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) { func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
@@ -97,19 +114,25 @@ func (p *Postgres) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string,
WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil { WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil {
return err return err
} }
for i, pr := range prefixes { if len(prefixes) > 0 {
var comm any _, err = tx.CopyFrom(
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" { ctx,
comm = strings.TrimSpace(*pr.CommunityID) pgx.Identifier{"module_prefix_snapshot_row"},
} []string{"tenant_id", "module_id", "ord", "prefix", "community_id", "source"},
src := pr.Source pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) {
if strings.TrimSpace(src) == "" { pr := prefixes[i]
src = "render" var comm any
} if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
if _, err := tx.Exec(ctx, ` comm = strings.TrimSpace(*pr.CommunityID)
INSERT INTO module_prefix_snapshot_row (tenant_id, module_id, ord, prefix, community_id, source) }
VALUES ($1::uuid, $2::uuid, $3, $4, $5::uuid, $6)`, src := pr.Source
tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src); err != nil { if strings.TrimSpace(src) == "" {
src = "render"
}
return []any{tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil
}),
)
if err != nil {
return err return err
} }
} }
@@ -135,3 +158,35 @@ func (p *Postgres) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
tenantID, moduleID) tenantID, moduleID)
return err return err
} }
func (p *Postgres) SetModuleInputHash(tenantID, moduleID, hash string) error {
ctx := context.Background()
tag, err := p.pool.Exec(ctx, `
UPDATE module SET input_hash = $3
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`,
moduleID, tenantID, hash)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return store.ErrNotFound
}
return nil
}
func (p *Postgres) LockModuleSnapshot(tenantID, moduleID string) func() {
ctx := context.Background()
conn, err := p.pool.Acquire(ctx)
if err != nil {
return func() {}
}
key := strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID)
if _, err := conn.Exec(ctx, `SELECT pg_advisory_lock(hashtext($1))`, key); err != nil {
conn.Release()
return func() {}
}
return func() {
_, _ = conn.Exec(ctx, `SELECT pg_advisory_unlock(hashtext($1))`, key)
conn.Release()
}
}
internal/repository/postgres.go
+21 -6
View File
@@ -119,7 +119,8 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module {
ctx := context.Background() ctx := context.Background()
rows, err := p.pool.Query(ctx, ` rows, err := p.pool.Query(ctx, `
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy, SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
COALESCE(input_hash, '')
FROM module WHERE tenant_id = $1 AND deleted_at IS NULL ORDER BY priority, name`, tenantID) FROM module WHERE tenant_id = $1 AND deleted_at IS NULL ORDER BY priority, name`, tenantID)
if err != nil { if err != nil {
return nil return nil
@@ -134,7 +135,7 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module {
var refresh *int32 var refresh *int32
var last *time.Time var last *time.Time
var createdBy *string var createdBy *string
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy); err != nil { if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash); err != nil {
continue continue
} }
m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy) m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
@@ -179,7 +180,8 @@ func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store
ctx := context.Background() ctx := context.Background()
rows, err := p.pool.Query(ctx, ` rows, err := p.pool.Query(ctx, `
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy, SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
COALESCE(input_hash, '')
FROM module WHERE tenant_id = $1 AND deleted_at IS NULL FROM module WHERE tenant_id = $1 AND deleted_at IS NULL
ORDER BY priority, name ORDER BY priority, name
LIMIT $2 OFFSET $3`, tenantID, limit+1, off) LIMIT $2 OFFSET $3`, tenantID, limit+1, off)
@@ -196,7 +198,7 @@ func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store
var refresh *int32 var refresh *int32
var last *time.Time var last *time.Time
var createdBy *string var createdBy *string
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy); err != nil { if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash); err != nil {
continue continue
} }
m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy) m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
@@ -249,9 +251,10 @@ func (p *Postgres) GetModule(tenantID, moduleID string) (*store.Module, error) {
var createdBy *string var createdBy *string
err := p.pool.QueryRow(ctx, ` err := p.pool.QueryRow(ctx, `
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy, SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
COALESCE(input_hash, '')
FROM module WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, moduleID, tenantID).Scan( FROM module WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, moduleID, tenantID).Scan(
&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy) &m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash)
if err != nil { if err != nil {
if errors.Is(err, pgx.ErrNoRows) { if errors.Is(err, pgx.ErrNoRows) {
return nil, store.ErrNotFound return nil, store.ErrNotFound
@@ -325,9 +328,19 @@ func (p *Postgres) CreateModule(tenantID string, in *store.Module) (*store.Modul
if err := p.setModuleDohProfiles(ctx, id, in.DohProfileIDs); err != nil { if err := p.setModuleDohProfiles(ctx, id, in.DohProfileIDs); err != nil {
return nil, err return nil, err
} }
store.TouchModuleInputHash(p, tenantID, id)
return p.GetModule(tenantID, id) return p.GetModule(tenantID, id)
} }
func (p *Postgres) invalidateModuleInputHash(tenantID, moduleID string) {
_ = p.SetModuleInputHash(tenantID, moduleID, "")
}
func (p *Postgres) invalidateTenantModuleHashes(tenantID string) {
ctx := context.Background()
_, _ = p.pool.Exec(ctx, `UPDATE module SET input_hash = '' WHERE tenant_id = $1 AND deleted_at IS NULL`, tenantID)
}
func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePatch) (*store.Module, error) { func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePatch) (*store.Module, error) {
if patch == nil { if patch == nil {
return nil, store.ErrInvalidInput return nil, store.ErrInvalidInput
@@ -399,6 +412,7 @@ func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePa
return nil, err return nil, err
} }
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return p.GetModule(tenantID, moduleID) return p.GetModule(tenantID, moduleID)
} }
@@ -1191,6 +1205,7 @@ func (p *Postgres) UpdateDohProfile(tenantID, id string, patch *store.DohProfile
if err != nil { if err != nil {
return nil, err return nil, err
} }
p.invalidateTenantModuleHashes(tenantID)
return p.GetDohProfile(tenantID, id) return p.GetDohProfile(tenantID, id)
} }
internal/repository/postgres_entities.go
+14
View File
@@ -73,6 +73,7 @@ func (p *Postgres) CreateCDNSource(tenantID, moduleID string, in *store.CDNSourc
if err != nil { if err != nil {
return nil, err return nil, err
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return p.getCDNSource(ctx, moduleID, id) return p.getCDNSource(ctx, moduleID, id)
} }
@@ -154,6 +155,9 @@ func (p *Postgres) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *s
if err != nil { if err != nil {
return nil, err return nil, err
} }
if store.CDNPatchAffectsInputHash(patch) {
p.invalidateModuleInputHash(tenantID, moduleID)
}
return p.getCDNSource(ctx, moduleID, sourceID) return p.getCDNSource(ctx, moduleID, sourceID)
} }
@@ -169,6 +173,7 @@ func (p *Postgres) DeleteCDNSource(tenantID, moduleID, sourceID string) error {
if tag.RowsAffected() == 0 { if tag.RowsAffected() == 0 {
return store.ErrNotFound return store.ErrNotFound
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return nil return nil
} }
@@ -229,6 +234,7 @@ func (p *Postgres) CreateASEntry(tenantID, moduleID string, in *store.ASEntry) (
if err != nil { if err != nil {
return nil, err return nil, err
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return p.getASEntry(ctx, moduleID, id) return p.getASEntry(ctx, moduleID, id)
} }
@@ -297,6 +303,7 @@ func (p *Postgres) UpdateASEntry(tenantID, moduleID, entryID string, patch *stor
if err != nil { if err != nil {
return nil, err return nil, err
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return p.getASEntry(ctx, moduleID, entryID) return p.getASEntry(ctx, moduleID, entryID)
} }
@@ -369,6 +376,7 @@ func (p *Postgres) DeleteASEntry(tenantID, moduleID, entryID string) error {
if tag.RowsAffected() == 0 { if tag.RowsAffected() == 0 {
return store.ErrNotFound return store.ErrNotFound
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return nil return nil
} }
@@ -417,6 +425,7 @@ func (p *Postgres) CreateDomainEntry(tenantID, moduleID string, in *store.Domain
if err != nil { if err != nil {
return nil, err return nil, err
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return p.getDomainEntry(ctx, moduleID, id) return p.getDomainEntry(ctx, moduleID, id)
} }
@@ -462,6 +471,7 @@ func (p *Postgres) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *
if err != nil { if err != nil {
return nil, err return nil, err
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return p.getDomainEntry(ctx, moduleID, entryID) return p.getDomainEntry(ctx, moduleID, entryID)
} }
@@ -477,6 +487,7 @@ func (p *Postgres) DeleteDomainEntry(tenantID, moduleID, entryID string) error {
if tag.RowsAffected() == 0 { if tag.RowsAffected() == 0 {
return store.ErrNotFound return store.ErrNotFound
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return nil return nil
} }
@@ -525,6 +536,7 @@ func (p *Postgres) CreateIPRangeEntry(tenantID, moduleID string, in *store.IPRan
if err != nil { if err != nil {
return nil, err return nil, err
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return p.getIPRangeEntry(ctx, moduleID, id) return p.getIPRangeEntry(ctx, moduleID, id)
} }
@@ -570,6 +582,7 @@ func (p *Postgres) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch
if err != nil { if err != nil {
return nil, err return nil, err
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return p.getIPRangeEntry(ctx, moduleID, entryID) return p.getIPRangeEntry(ctx, moduleID, entryID)
} }
@@ -585,6 +598,7 @@ func (p *Postgres) DeleteIPRangeEntry(tenantID, moduleID, entryID string) error
if tag.RowsAffected() == 0 { if tag.RowsAffected() == 0 {
return store.ErrNotFound return store.ErrNotFound
} }
p.invalidateModuleInputHash(tenantID, moduleID)
return nil return nil
} }
internal/store/backend.go
+15
View File
@@ -116,11 +116,19 @@ type Backend interface {
GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error) GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
DeleteModulePrefixSnapshot(tenantID, moduleID string) error DeleteModulePrefixSnapshot(tenantID, moduleID string) error
// SetModuleInputHash stores the ingest fingerprint on the module row (O(1) snapshot check).
SetModuleInputHash(tenantID, moduleID, hash string) error
// LockModuleSnapshot serializes read-modify-write of one module snapshot; unlock must be called.
LockModuleSnapshot(tenantID, moduleID string) (unlock func())
// ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache). // ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache).
GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error) GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error)
SetASNPrefixCache(asn int64, holder string, prefixes []string) error SetASNPrefixCache(asn int64, holder string, prefixes []string) error
// DomainResolveCache stores DoH results per FQDN (global TTL cache).
GetDomainResolveCache(fqdn string) (*DomainResolveCacheEntry, bool, error)
SetDomainResolveCache(fqdn string, addrs []string) error
// Ping verifies backend connectivity (no-op for in-memory). // Ping verifies backend connectivity (no-op for in-memory).
Ping(ctx context.Context) error Ping(ctx context.Context) error
@@ -178,6 +186,13 @@ type ASNPrefixCacheEntry struct {
FetchedAt time.Time FetchedAt time.Time
} }
// DomainResolveCacheEntry is a cached DoH A/AAAA result for one FQDN.
type DomainResolveCacheEntry struct {
FQDN string
Addrs []string
ResolvedAt time.Time
}
// ModulePrefixSnapshot is the cached materialization for one module between refreshes. // ModulePrefixSnapshot is the cached materialization for one module between refreshes.
type ModulePrefixSnapshot struct { type ModulePrefixSnapshot struct {
InputHash string InputHash string
internal/store/memory.go
+18
View File
@@ -44,7 +44,9 @@ type Memory struct {
settings map[string]map[string]any // tenantID -> key -> JSON-compatible value settings map[string]map[string]any // tenantID -> key -> JSON-compatible value
revPrefixes map[string][]PrefixRow revPrefixes map[string][]PrefixRow
moduleSnapshots map[string]*moduleSnapshotRec moduleSnapshots map[string]*moduleSnapshotRec
snapshotLocks sync.Map // key -> *sync.Mutex (per-module snapshot RMW)
asnPrefixCache map[int64]*ASNPrefixCacheEntry asnPrefixCache map[int64]*ASNPrefixCacheEntry
domainResolveCache map[string]*DomainResolveCacheEntry
apiKeys map[string]*apiKeyRec apiKeys map[string]*apiKeyRec
firewallClients map[string]*firewallClientRec firewallClients map[string]*firewallClientRec
firewallRules map[string]*FirewallRule firewallRules map[string]*FirewallRule
@@ -99,6 +101,7 @@ type Module struct {
LastRefreshedAt *time.Time LastRefreshedAt *time.Time
DeletedAt *time.Time DeletedAt *time.Time
CreatedByUserID string // portal JWT sub; empty = system / API key CreatedByUserID string // portal JWT sub; empty = system / API key
InputHash string // ingest fingerprint; empty = not yet computed
} }
type Revision struct { type Revision struct {
@@ -156,6 +159,7 @@ func NewMemory() *Memory {
revPrefixes: make(map[string][]PrefixRow), revPrefixes: make(map[string][]PrefixRow),
moduleSnapshots: make(map[string]*moduleSnapshotRec), moduleSnapshots: make(map[string]*moduleSnapshotRec),
asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry), asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry),
domainResolveCache: make(map[string]*DomainResolveCacheEntry),
apiKeys: make(map[string]*apiKeyRec), apiKeys: make(map[string]*apiKeyRec),
firewallClients: make(map[string]*firewallClientRec), firewallClients: make(map[string]*firewallClientRec),
firewallRules: make(map[string]*FirewallRule), firewallRules: make(map[string]*FirewallRule),
@@ -696,6 +700,20 @@ func cloneStringPtr(s *string) *string {
return &v return &v
} }
func (m *Memory) clearModuleInputHashLocked(moduleID string) {
if mod, ok := m.modules[moduleID]; ok && mod != nil {
mod.InputHash = ""
}
}
func (m *Memory) clearTenantModuleHashesLocked(tenantID string) {
for _, mod := range m.modules {
if mod != nil && mod.TenantID == tenantID && mod.DeletedAt == nil {
mod.InputHash = ""
}
}
}
func cloneModule(m *Module) *Module { func cloneModule(m *Module) *Module {
if m == nil { if m == nil {
return nil return nil
internal/store/memory_crud.go
+24 -4
View File
@@ -81,6 +81,7 @@ func (m *Memory) UpdateModule(tenantID, moduleID string, patch *ModulePatch) (*M
t := patch.LastRefreshedAt.UTC() t := patch.LastRefreshedAt.UTC()
mod.LastRefreshedAt = &t mod.LastRefreshedAt = &t
} }
mod.InputHash = ""
return cloneModule(mod), nil return cloneModule(mod), nil
} }
@@ -152,6 +153,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN
LastRefreshedAt: in.LastRefreshedAt, LastRefreshedAt: in.LastRefreshedAt,
} }
m.cdnSources[id] = s m.cdnSources[id] = s
mod.InputHash = ""
return s, nil return s, nil
} }
@@ -161,7 +163,8 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
} }
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil { mod, err := m.moduleWriteOK(tenantID, moduleID)
if err != nil {
return nil, err return nil, err
} }
s, ok := m.cdnSources[sourceID] s, ok := m.cdnSources[sourceID]
@@ -195,6 +198,9 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
t := patch.LastRefreshedAt.UTC() t := patch.LastRefreshedAt.UTC()
s.LastRefreshedAt = &t s.LastRefreshedAt = &t
} }
if CDNPatchAffectsInputHash(patch) {
mod.InputHash = ""
}
return s, nil return s, nil
} }
@@ -209,6 +215,7 @@ func (m *Memory) DeleteCDNSource(tenantID, moduleID, sourceID string) error {
return ErrNotFound return ErrNotFound
} }
delete(m.cdnSources, sourceID) delete(m.cdnSources, sourceID)
m.clearModuleInputHashLocked(moduleID)
return nil return nil
} }
@@ -247,6 +254,7 @@ func (m *Memory) CreateASEntry(tenantID, moduleID string, in *ASEntry) (*ASEntry
id := uuid.NewString() id := uuid.NewString()
e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID} e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID}
m.asEntries[id] = e m.asEntries[id] = e
mod.InputHash = ""
return e, nil return e, nil
} }
@@ -256,7 +264,8 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr
} }
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil { mod, err := m.moduleWriteOK(tenantID, moduleID)
if err != nil {
return nil, err return nil, err
} }
e, ok := m.asEntries[entryID] e, ok := m.asEntries[entryID]
@@ -283,6 +292,7 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr
e.PrefixCount = nil e.PrefixCount = nil
e.ASNResolvedAt = nil e.ASNResolvedAt = nil
} }
mod.InputHash = ""
return e, nil return e, nil
} }
@@ -324,6 +334,7 @@ func (m *Memory) DeleteASEntry(tenantID, moduleID, entryID string) error {
return ErrNotFound return ErrNotFound
} }
delete(m.asEntries, entryID) delete(m.asEntries, entryID)
m.clearModuleInputHashLocked(moduleID)
return nil return nil
} }
@@ -362,6 +373,7 @@ func (m *Memory) CreateDomainEntry(tenantID, moduleID string, in *DomainEntry) (
id := uuid.NewString() id := uuid.NewString()
e := &DomainEntry{ID: id, ModuleID: moduleID, FQDN: strings.TrimSpace(in.FQDN), CommunityID: in.CommunityID} e := &DomainEntry{ID: id, ModuleID: moduleID, FQDN: strings.TrimSpace(in.FQDN), CommunityID: in.CommunityID}
m.domainEnt[id] = e m.domainEnt[id] = e
mod.InputHash = ""
return e, nil return e, nil
} }
@@ -371,7 +383,8 @@ func (m *Memory) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *Do
} }
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil { mod, err := m.moduleWriteOK(tenantID, moduleID)
if err != nil {
return nil, err return nil, err
} }
e, ok := m.domainEnt[entryID] e, ok := m.domainEnt[entryID]
@@ -389,6 +402,7 @@ func (m *Memory) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *Do
e.CommunityID = &v e.CommunityID = &v
} }
} }
mod.InputHash = ""
return e, nil return e, nil
} }
@@ -403,6 +417,7 @@ func (m *Memory) DeleteDomainEntry(tenantID, moduleID, entryID string) error {
return ErrNotFound return ErrNotFound
} }
delete(m.domainEnt, entryID) delete(m.domainEnt, entryID)
m.clearModuleInputHashLocked(moduleID)
return nil return nil
} }
@@ -441,6 +456,7 @@ func (m *Memory) CreateIPRangeEntry(tenantID, moduleID string, in *IPRangeEntry)
id := uuid.NewString() id := uuid.NewString()
e := &IPRangeEntry{ID: id, ModuleID: moduleID, Prefix: strings.TrimSpace(in.Prefix), CommunityID: in.CommunityID} e := &IPRangeEntry{ID: id, ModuleID: moduleID, Prefix: strings.TrimSpace(in.Prefix), CommunityID: in.CommunityID}
m.ipRanges[id] = e m.ipRanges[id] = e
mod.InputHash = ""
return e, nil return e, nil
} }
@@ -450,7 +466,8 @@ func (m *Memory) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch *I
} }
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil { mod, err := m.moduleWriteOK(tenantID, moduleID)
if err != nil {
return nil, err return nil, err
} }
e, ok := m.ipRanges[entryID] e, ok := m.ipRanges[entryID]
@@ -468,6 +485,7 @@ func (m *Memory) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch *I
e.CommunityID = &v e.CommunityID = &v
} }
} }
mod.InputHash = ""
return e, nil return e, nil
} }
@@ -482,6 +500,7 @@ func (m *Memory) DeleteIPRangeEntry(tenantID, moduleID, entryID string) error {
return ErrNotFound return ErrNotFound
} }
delete(m.ipRanges, entryID) delete(m.ipRanges, entryID)
m.clearModuleInputHashLocked(moduleID)
return nil return nil
} }
@@ -551,6 +570,7 @@ func (m *Memory) UpdateDohProfile(tenantID, id string, patch *DohProfilePatch) (
if patch.SecretRef != nil { if patch.SecretRef != nil {
p.SecretRef = patch.SecretRef p.SecretRef = patch.SecretRef
} }
m.clearTenantModuleHashesLocked(tenantID)
return p, nil return p, nil
} }
internal/store/memory_domain_cache.go
+41
View File
@@ -0,0 +1,41 @@
package store
import (
"strings"
"time"
)
func (m *Memory) GetDomainResolveCache(fqdn string) (*DomainResolveCacheEntry, bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
if m.domainResolveCache == nil {
return nil, false, nil
}
e, ok := m.domainResolveCache[strings.ToLower(strings.TrimSpace(fqdn))]
if !ok || e == nil {
return nil, false, nil
}
return &DomainResolveCacheEntry{
FQDN: e.FQDN,
Addrs: append([]string(nil), e.Addrs...),
ResolvedAt: e.ResolvedAt,
}, true, nil
}
func (m *Memory) SetDomainResolveCache(fqdn string, addrs []string) error {
key := strings.ToLower(strings.TrimSpace(fqdn))
if key == "" {
return ErrInvalidInput
}
m.mu.Lock()
defer m.mu.Unlock()
if m.domainResolveCache == nil {
m.domainResolveCache = make(map[string]*DomainResolveCacheEntry)
}
m.domainResolveCache[key] = &DomainResolveCacheEntry{
FQDN: key,
Addrs: append([]string(nil), addrs...),
ResolvedAt: time.Now().UTC(),
}
return nil
}
internal/store/memory_snapshot.go
+23
View File
@@ -2,6 +2,7 @@ package store
import ( import (
"strings" "strings"
"sync"
"time" "time"
) )
@@ -51,6 +52,28 @@ func (m *Memory) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
return nil return nil
} }
func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error {
m.mu.Lock()
defer m.mu.Unlock()
mod, ok := m.modules[moduleID]
if !ok || mod.DeletedAt != nil {
return ErrNotFound
}
if mod.TenantID != tenantID {
return ErrTenantScope
}
mod.InputHash = strings.TrimSpace(hash)
return nil
}
func (m *Memory) LockModuleSnapshot(tenantID, moduleID string) func() {
key := moduleSnapshotKey(tenantID, moduleID)
v, _ := m.snapshotLocks.LoadOrStore(key, &sync.Mutex{})
mu := v.(*sync.Mutex)
mu.Lock()
return func() { mu.Unlock() }
}
type moduleSnapshotRec struct { type moduleSnapshotRec struct {
InputHash string InputHash string
CollectedAt time.Time CollectedAt time.Time
internal/store/module_input_hash.go
+123
View File
@@ -0,0 +1,123 @@
package store
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
)
// ComputeModuleInputHash fingerprints module config and child entries so snapshots
// invalidate on CRUD without re-reading every child at render time.
func ComputeModuleInputHash(st Backend, tenantID string, mod *Module) (string, error) {
if st == nil || mod == nil {
return "", fmt.Errorf("store: module hash: missing store or module")
}
h := sha256.New()
_, _ = fmt.Fprintf(h, "type=%s\n", strings.TrimSpace(mod.Type))
_, _ = fmt.Fprintf(h, "enabled=%t\n", mod.Enabled)
if mod.DefaultCommunityID != nil {
_, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID))
}
_, _ = fmt.Fprintf(h, "doh_policy=%s\n", NormalizeDohResolverPolicy(mod.DohResolverPolicy))
for _, pid := range mod.EffectiveDohProfileIDs() {
_, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid)
if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil {
_, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL))
if prof.TimeoutMs != nil {
_, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs)
}
}
}
switch mod.Type {
case "IP_RANGES":
list, err := st.ListIPRangeEntries(tenantID, mod.ID)
if err != nil {
return "", err
}
sort.Slice(list, func(i, j int) bool { return list[i].Prefix < list[j].Prefix })
for _, e := range list {
comm := ""
if e.CommunityID != nil {
comm = *e.CommunityID
}
_, _ = fmt.Fprintf(h, "ip=%s|c=%s\n", e.Prefix, comm)
}
case "AS_PREFIXES":
list, err := st.ListASEntries(tenantID, mod.ID)
if err != nil {
return "", err
}
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
for _, e := range list {
comm := ""
if e.CommunityID != nil {
comm = *e.CommunityID
}
_, _ = fmt.Fprintf(h, "as=%d|c=%s\n", e.ASN, comm)
}
case "CDN_CIDRS":
list, err := st.ListCDNSources(tenantID, mod.ID)
if err != nil {
return "", err
}
sort.Slice(list, func(i, j int) bool { return list[i].ID < list[j].ID })
for _, s := range list {
comm := ""
if s.CommunityID != nil {
comm = *s.CommunityID
}
interval := 0
if s.RefreshIntervalSec != nil {
interval = *s.RefreshIntervalSec
}
_, _ = fmt.Fprintf(h, "cdn=%s|url=%s|kind=%s|path=%s|c=%s|etag=%s|interval=%d\n",
s.ID, strings.TrimSpace(s.URL), s.SourceKind, strings.TrimSpace(s.PrefixPath), comm,
strings.TrimSpace(s.Etag), interval)
}
case "DOMAINS":
list, err := st.ListDomainEntries(tenantID, mod.ID)
if err != nil {
return "", err
}
sort.Slice(list, func(i, j int) bool { return list[i].FQDN < list[j].FQDN })
for _, e := range list {
comm := ""
if e.CommunityID != nil {
comm = *e.CommunityID
}
_, _ = fmt.Fprintf(h, "dom=%s|c=%s\n", strings.TrimSpace(e.FQDN), comm)
}
default:
_, _ = fmt.Fprintf(h, "unknown_type=%s\n", mod.Type)
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// TouchModuleInputHash recomputes and stores module.input_hash (ARCH-01: hash lives in store).
func TouchModuleInputHash(st Backend, tenantID, moduleID string) {
if st == nil {
return
}
mod, err := st.GetModule(tenantID, moduleID)
if err != nil || mod == nil {
return
}
h, err := ComputeModuleInputHash(st, tenantID, mod)
if err != nil {
return
}
_ = st.SetModuleInputHash(tenantID, moduleID, h)
}
// CDNPatchAffectsInputHash reports whether a CDN source patch changes ingest fingerprint
// fields (URL/kind/path/community/interval). ETag and last_refreshed_at do not.
func CDNPatchAffectsInputHash(patch *CDNSourcePatch) bool {
if patch == nil {
return false
}
return patch.SourceKind != nil || patch.URL != nil || patch.PrefixPath != nil ||
patch.CommunityID != nil || patch.RefreshIntervalSec != nil
}
migrations/postgres/000032_module_input_hash.down.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE module
DROP COLUMN IF EXISTS input_hash;
migrations/postgres/000032_module_input_hash.up.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE module
ADD COLUMN input_hash TEXT NOT NULL DEFAULT '';
migrations/postgres/000033_domain_resolve_cache.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS domain_resolve_cache;
migrations/postgres/000033_domain_resolve_cache.up.sql
+8
View File
@@ -0,0 +1,8 @@
-- TTL cache for DoH/A/AAAA resolutions (pipeline domain ingest).
CREATE TABLE domain_resolve_cache (
fqdn TEXT PRIMARY KEY,
addrs_json JSONB NOT NULL DEFAULT '[]',
resolved_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX idx_domain_resolve_cache_resolved ON domain_resolve_cache (resolved_at);
migrations/sqlite/000032_module_input_hash.down.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE module
DROP COLUMN input_hash;
migrations/sqlite/000032_module_input_hash.up.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE module
ADD COLUMN input_hash TEXT NOT NULL DEFAULT '';
migrations/sqlite/000033_domain_resolve_cache.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS domain_resolve_cache;
migrations/sqlite/000033_domain_resolve_cache.up.sql
+7
View File
@@ -0,0 +1,7 @@
CREATE TABLE domain_resolve_cache (
fqdn TEXT PRIMARY KEY,
addrs_json TEXT NOT NULL DEFAULT '[]',
resolved_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
);
CREATE INDEX idx_domain_resolve_cache_resolved ON domain_resolve_cache (resolved_at);