refactor(web): remove deprecated dashboard components and enhance KPI grid
quality / commitlint (push) Skipped
quality / changes (push) Successful in 9s
quality / docker-check (push) Skipped
quality / openapi (push) Successful in 46s
quality / web (push) Successful in 1m16s
quality / go (push) Successful in 2m42s
quality / bird2 (push) Successful in 16s
CD / quality (push) Successful in 5m19s
CD / publish (push) Successful in 7m19s
quality / commitlint (push) Skipped
quality / changes (push) Successful in 9s
quality / docker-check (push) Skipped
quality / openapi (push) Successful in 46s
quality / web (push) Successful in 1m16s
quality / go (push) Successful in 2m42s
quality / bird2 (push) Successful in 16s
CD / quality (push) Successful in 5m19s
CD / publish (push) Successful in 7m19s
- Deleted unused components: `DashboardActivityTimeline`, `DashboardFramePanel`, `DashboardModulesGrid`, `DashboardRecentJobsGrid`, and `DashboardRecentRevisionsGrid` to streamline the dashboard. - Updated `DashboardKpiGrid` to improve KPI display logic, including progress indicators and enhanced badge functionality. - Refactored `DashboardNetworkHealth` to provide better status representation based on loading states and network conditions. - Introduced new properties for KPI cards to support progress tracking and improved visual feedback. This cleanup aims to enhance performance and maintainability of the dashboard while providing a better user experience.
This commit is contained in:
@@ -1,102 +0,0 @@
|
||||
import { AlertTriangle, CheckCircle, Info } from 'lucide-react'
|
||||
|
||||
import { DashboardFramePanel } from '@/components/dashboard/dashboard-frame-panel'
|
||||
import { Badge } from '@/components/reui/badge'
|
||||
import {
|
||||
Timeline,
|
||||
TimelineContent,
|
||||
TimelineHeader,
|
||||
TimelineIndicator,
|
||||
TimelineItem,
|
||||
TimelineSeparator,
|
||||
TimelineTitle,
|
||||
} from '@/components/reui/timeline'
|
||||
import { cn } from '@evobgp/ui/lib/utils'
|
||||
|
||||
import { recentPlatformActivity } from '@/lib/metrics'
|
||||
import type { JobRow, PeerRow, RevisionRow, SpeakerRow } from '@/types/api'
|
||||
|
||||
const KIND_META = {
|
||||
job: { icon: Info, className: 'text-info' },
|
||||
revision: { icon: CheckCircle, className: 'text-success' },
|
||||
network: { icon: AlertTriangle, className: 'text-warning' },
|
||||
} as const
|
||||
|
||||
function statusBadgeVariant(status: string) {
|
||||
const s = status.toLowerCase()
|
||||
if (['ok', 'success', 'completed', 'done'].includes(s)) return 'success-light' as const
|
||||
if (['running', 'queued', 'pending'].includes(s)) return 'info-light' as const
|
||||
if (['warning', 'mismatch'].includes(s)) return 'warning-light' as const
|
||||
if (['failed', 'error', 'cancelled'].includes(s)) return 'destructive-light' as const
|
||||
return 'outline' as const
|
||||
}
|
||||
|
||||
export function DashboardActivityTimeline({
|
||||
jobs,
|
||||
revisions,
|
||||
peers,
|
||||
speakers,
|
||||
loading,
|
||||
}: {
|
||||
jobs: JobRow[]
|
||||
revisions: RevisionRow[]
|
||||
peers: PeerRow[]
|
||||
speakers: SpeakerRow[]
|
||||
loading?: boolean
|
||||
}) {
|
||||
const items = recentPlatformActivity(jobs, revisions, peers, speakers, 6)
|
||||
|
||||
return (
|
||||
<DashboardFramePanel
|
||||
title="Недавняя активность"
|
||||
description="Задачи, ревизии и сетевые события"
|
||||
className="h-full min-w-0"
|
||||
>
|
||||
{loading ? (
|
||||
<p className="text-muted-foreground px-4 py-6 text-sm">Загрузка…</p>
|
||||
) : items.length === 0 ? (
|
||||
<p className="text-muted-foreground px-4 py-6 text-sm">Нет недавних событий</p>
|
||||
) : (
|
||||
<div className="px-4 py-4">
|
||||
<Timeline defaultValue={items.length}>
|
||||
{items.map((item, index) => {
|
||||
const meta = KIND_META[item.kind]
|
||||
const Icon = meta.icon
|
||||
return (
|
||||
<TimelineItem
|
||||
key={item.id}
|
||||
step={index + 1}
|
||||
className="group-data-[orientation=vertical]/timeline:ms-8 group-data-[orientation=vertical]/timeline:not-last:pb-4"
|
||||
>
|
||||
<TimelineHeader>
|
||||
<TimelineSeparator className="bg-border! group-data-[orientation=vertical]/timeline:-left-6 group-data-[orientation=vertical]/timeline:top-2 group-data-[orientation=vertical]/timeline:h-[calc(100%-1.5rem)] group-data-[orientation=vertical]/timeline:translate-y-5" />
|
||||
<TimelineIndicator className="border-none bg-transparent group-data-[orientation=vertical]/timeline:-left-6">
|
||||
<span
|
||||
className={cn(
|
||||
'bg-muted/70 flex size-7 items-center justify-center rounded-full',
|
||||
meta.className,
|
||||
)}
|
||||
>
|
||||
<Icon className="size-3.5" aria-hidden />
|
||||
</span>
|
||||
</TimelineIndicator>
|
||||
</TimelineHeader>
|
||||
<TimelineContent className="min-w-0 pb-1 text-foreground">
|
||||
<TimelineTitle className="text-sm leading-snug font-normal break-words">
|
||||
{item.message}
|
||||
</TimelineTitle>
|
||||
<div className="mt-2">
|
||||
<Badge variant={statusBadgeVariant(item.status)} size="sm">
|
||||
{item.statusLabel ?? item.status}
|
||||
</Badge>
|
||||
</div>
|
||||
</TimelineContent>
|
||||
</TimelineItem>
|
||||
)
|
||||
})}
|
||||
</Timeline>
|
||||
</div>
|
||||
)}
|
||||
</DashboardFramePanel>
|
||||
)
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import type { ReactNode } from 'react'
|
||||
|
||||
import {
|
||||
FrameSection,
|
||||
panelCardContentFlushClassName,
|
||||
} from '@/components/reui-kit'
|
||||
import { cn } from '@evobgp/ui/lib/utils'
|
||||
|
||||
/** Frame panel for dashboard sections. Preview: https://reui.io/docs/components/base/frame */
|
||||
export function DashboardFramePanel({
|
||||
title,
|
||||
description,
|
||||
actions,
|
||||
children,
|
||||
className,
|
||||
contentClassName,
|
||||
}: {
|
||||
title?: string
|
||||
description?: string
|
||||
actions?: ReactNode
|
||||
children: ReactNode
|
||||
className?: string
|
||||
contentClassName?: string
|
||||
}) {
|
||||
return (
|
||||
<FrameSection
|
||||
title={title}
|
||||
description={description}
|
||||
actions={actions}
|
||||
className={cn('h-full', className)}
|
||||
contentClassName={cn(panelCardContentFlushClassName, contentClassName)}
|
||||
>
|
||||
{children}
|
||||
</FrameSection>
|
||||
)
|
||||
}
|
||||
@@ -3,8 +3,6 @@ import {
|
||||
Boxes,
|
||||
ListChecks,
|
||||
Network,
|
||||
ServerCog,
|
||||
Share2,
|
||||
} from 'lucide-react'
|
||||
import type { ReactNode } from 'react'
|
||||
|
||||
@@ -16,6 +14,11 @@ import type { JobRow, ModuleRow, PeerRow, SpeakerRow } from '@/types/api'
|
||||
|
||||
type KpiCard = KpiStatItem & { icon: ReactNode }
|
||||
|
||||
function ratioPercent(part: number, total: number): number | undefined {
|
||||
if (total <= 0) return undefined
|
||||
return Math.round((part / total) * 100)
|
||||
}
|
||||
|
||||
function buildKpis({
|
||||
modules,
|
||||
peers,
|
||||
@@ -40,17 +43,34 @@ function buildKpis({
|
||||
).length
|
||||
const offlineSpeakers = Math.max(0, speakers.length - network.speakersOnline)
|
||||
const riskCount = network.peersMismatch + failedJobs + offlineSpeakers
|
||||
const disabledModules = Math.max(0, modules.length - enabledModules)
|
||||
|
||||
return [
|
||||
{
|
||||
id: 'modules',
|
||||
icon: <Boxes aria-hidden />,
|
||||
iconClassName: 'text-primary',
|
||||
value: loading ? '—' : `${enabledModules}/${modules.length || 0}`,
|
||||
value: loading ? '—' : enabledModules,
|
||||
label: 'Модули активны',
|
||||
progress: loading ? undefined : ratioPercent(enabledModules, modules.length),
|
||||
footer: (
|
||||
<Badge variant="primary-light" size="sm">
|
||||
{loading ? '…' : `${modules.length} всего`}
|
||||
<Badge
|
||||
variant={
|
||||
loading || modules.length === 0
|
||||
? 'outline'
|
||||
: disabledModules === 0
|
||||
? 'success-light'
|
||||
: 'warning-light'
|
||||
}
|
||||
size="sm"
|
||||
>
|
||||
{loading
|
||||
? '…'
|
||||
: modules.length === 0
|
||||
? 'нет модулей'
|
||||
: disabledModules === 0
|
||||
? 'все активны'
|
||||
: `${disabledModules} выкл`}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
@@ -60,6 +80,7 @@ function buildKpis({
|
||||
iconClassName: 'text-info',
|
||||
value: loading || bgpPct === null ? '—' : `${bgpPct}%`,
|
||||
label: 'BGP готовность',
|
||||
progress: loading || bgpPct === null ? undefined : bgpPct,
|
||||
footer: (
|
||||
<Badge
|
||||
variant={
|
||||
@@ -73,34 +94,9 @@ function buildKpis({
|
||||
>
|
||||
{loading || bgpPct === null
|
||||
? 'нет включённых пиров'
|
||||
: `${network.peersEstablished} установлено`}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: 'peers',
|
||||
icon: <Share2 aria-hidden />,
|
||||
iconClassName: 'text-success',
|
||||
value: loading ? '—' : `${network.peersEstablished}/${peersEnabled}`,
|
||||
label: 'Пиры установлены',
|
||||
footer: (
|
||||
<Badge variant="success-light" size="sm">
|
||||
{loading ? '…' : `${network.peersTotal} в каталоге`}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: 'speakers',
|
||||
icon: <ServerCog aria-hidden />,
|
||||
iconClassName: 'text-warning',
|
||||
value: loading ? '—' : `${network.speakersOnline}/${network.speakersTotal}`,
|
||||
label: 'Спикеры в сети',
|
||||
footer: (
|
||||
<Badge
|
||||
variant={network.speakersOnline === network.speakersTotal ? 'success-light' : 'warning-light'}
|
||||
size="sm"
|
||||
>
|
||||
{loading ? '…' : 'в сети'}
|
||||
: bgpPct >= 90
|
||||
? 'сессии в норме'
|
||||
: `${network.peersEstablished} установлено`}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
@@ -112,7 +108,7 @@ function buildKpis({
|
||||
label: 'Активные задачи',
|
||||
footer: (
|
||||
<Badge variant={running > 0 ? 'info-light' : 'outline'} size="sm">
|
||||
{loading ? '…' : `${jobs.length} в выборке`}
|
||||
{loading ? '…' : running > 0 ? 'выполняются' : 'очередь пуста'}
|
||||
</Badge>
|
||||
),
|
||||
},
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
import { useMemo, useState } from 'react'
|
||||
import { Link, useNavigate } from '@tanstack/react-router'
|
||||
import { BoxesIcon, PlusIcon, SearchIcon } from 'lucide-react'
|
||||
|
||||
import { CategoryBadge } from '@/components/category-badge'
|
||||
import { DataGridNameCell } from '@/components/data-grid-cell'
|
||||
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
|
||||
import type { FilterField, FilterQuery } from '@/components/reui/filters/filters-types'
|
||||
import {
|
||||
ResourcePage,
|
||||
createTextFilterQuery,
|
||||
type DataGridColumnDef,
|
||||
} from '@/components/reui-kit'
|
||||
import { Button } from '@evobgp/ui/components/button'
|
||||
import { moduleTypeRu } from '@/lib/ui-labels'
|
||||
import type { ModuleRow } from '@/types/api'
|
||||
|
||||
const MODULE_TABS = [
|
||||
{ id: 'all', label: 'Все' },
|
||||
{ id: 'enabled', label: 'Вкл' },
|
||||
{ id: 'disabled', label: 'Выкл' },
|
||||
]
|
||||
|
||||
const filterFields: FilterField[] = [
|
||||
{
|
||||
id: 'search',
|
||||
label: 'Поиск',
|
||||
icon: <SearchIcon className="size-3.5" aria-hidden />,
|
||||
type: 'text',
|
||||
placeholder: 'Поиск модулей…',
|
||||
},
|
||||
]
|
||||
|
||||
function getFilterFieldValue(item: ModuleRow, field: string): unknown {
|
||||
if (field === 'search') {
|
||||
return `${item.name} ${item.type} ${moduleTypeRu(item.type)}`
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function tabFilter(item: ModuleRow, tabId: string): boolean {
|
||||
if (tabId === 'enabled') return item.enabled !== false
|
||||
if (tabId === 'disabled') return item.enabled === false
|
||||
return true
|
||||
}
|
||||
|
||||
export function DashboardModulesGrid({
|
||||
modules,
|
||||
isLoading = false,
|
||||
}: {
|
||||
modules: ModuleRow[]
|
||||
isLoading?: boolean
|
||||
}) {
|
||||
const navigate = useNavigate()
|
||||
const [filterQuery, setFilterQuery] = useState<FilterQuery>(() =>
|
||||
createTextFilterQuery('search'),
|
||||
)
|
||||
|
||||
const columns = useMemo<DataGridColumnDef<ModuleRow>[]>(
|
||||
() => [
|
||||
{
|
||||
accessorKey: 'name',
|
||||
id: 'name',
|
||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Модуль" />,
|
||||
cell: ({ row }) => <DataGridNameCell icon={BoxesIcon} title={row.original.name} />,
|
||||
minSize: 180,
|
||||
meta: { headerTitle: 'Модуль' },
|
||||
},
|
||||
{
|
||||
accessorKey: 'type',
|
||||
id: 'type',
|
||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Тип" />,
|
||||
cell: ({ row }) => <CategoryBadge>{moduleTypeRu(row.original.type)}</CategoryBadge>,
|
||||
meta: { headerTitle: 'Тип' },
|
||||
},
|
||||
{
|
||||
accessorKey: 'priority',
|
||||
id: 'priority',
|
||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Приоритет" />,
|
||||
cell: ({ row }) => (
|
||||
<span className="font-mono text-sm tabular-nums">{row.original.priority}</span>
|
||||
),
|
||||
size: 88,
|
||||
meta: { headerTitle: 'Приоритет' },
|
||||
},
|
||||
],
|
||||
[],
|
||||
)
|
||||
|
||||
return (
|
||||
<ResourcePage
|
||||
title="Модули"
|
||||
description="Поиск и быстрый переход к настройке"
|
||||
tabs={MODULE_TABS}
|
||||
tabFilter={tabFilter}
|
||||
filterFields={filterFields}
|
||||
filterQuery={filterQuery}
|
||||
onFilterQueryChange={setFilterQuery}
|
||||
onClearFilters={() => setFilterQuery(createTextFilterQuery('search'))}
|
||||
getFilterFieldValue={getFilterFieldValue}
|
||||
columns={columns}
|
||||
data={modules}
|
||||
getRowId={(row) => row.id}
|
||||
isLoading={isLoading}
|
||||
primaryAction={
|
||||
<Button variant="outline" size="sm" render={<Link to="/modules" search={{ create: true }} />}>
|
||||
<PlusIcon />
|
||||
Создать
|
||||
</Button>
|
||||
}
|
||||
onRowClick={(row) =>
|
||||
void navigate({ to: '/modules/$moduleId', params: { moduleId: row.id } })
|
||||
}
|
||||
emptyState={{ title: 'Нет модулей по выбранным фильтрам.' }}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -60,8 +60,28 @@ export function DashboardNetworkHealth({
|
||||
label: mode === 'peers' ? 'Утилизация пиров' : 'Спикеры в сети',
|
||||
percent: loading ? 0 : utilization,
|
||||
badge: (
|
||||
<Badge variant="outline" radius="full" className="h-6 px-2 text-[10px]">
|
||||
{loading ? '…' : `${established}/${total}`}
|
||||
<Badge
|
||||
variant={
|
||||
loading || total === 0
|
||||
? 'outline'
|
||||
: offline === 0
|
||||
? 'success-light'
|
||||
: 'warning-light'
|
||||
}
|
||||
size="sm"
|
||||
radius="full"
|
||||
>
|
||||
{loading
|
||||
? '…'
|
||||
: total === 0
|
||||
? 'нет данных'
|
||||
: offline === 0
|
||||
? mode === 'peers'
|
||||
? 'все установлены'
|
||||
: 'все в сети'
|
||||
: mode === 'peers'
|
||||
? `${offline} не установлены`
|
||||
: `${offline} офлайн`}
|
||||
</Badge>
|
||||
),
|
||||
}}
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
import { useMemo } from 'react'
|
||||
import { ListTodo } from 'lucide-react'
|
||||
|
||||
import { DataGridNameCell } from '@/components/data-grid-cell'
|
||||
import { StatusBadge } from '@/components/status-badge'
|
||||
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
|
||||
import { FrameDataGrid, type DataGridColumnDef } from '@/components/reui-kit'
|
||||
import { jobKindRu } from '@/lib/ui-labels'
|
||||
import type { JobRow } from '@/types/api'
|
||||
|
||||
export function DashboardRecentJobsGrid({
|
||||
jobs,
|
||||
nameById,
|
||||
isLoading = false,
|
||||
}: {
|
||||
jobs: JobRow[]
|
||||
nameById: Map<string, string>
|
||||
isLoading?: boolean
|
||||
}) {
|
||||
const data = useMemo(() => jobs.slice(0, 8), [jobs])
|
||||
|
||||
const columns = useMemo<DataGridColumnDef<JobRow>[]>(
|
||||
() => [
|
||||
{
|
||||
accessorKey: 'kind',
|
||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Вид" />,
|
||||
cell: ({ row }) => (
|
||||
<DataGridNameCell
|
||||
icon={ListTodo}
|
||||
title={jobKindRu(row.original.kind)}
|
||||
subtitle={
|
||||
row.original.meta?.module_id
|
||||
? (nameById.get(String(row.original.meta.module_id)) ?? undefined)
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
),
|
||||
meta: { headerTitle: 'Вид' },
|
||||
},
|
||||
{
|
||||
accessorKey: 'status',
|
||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Статус" />,
|
||||
cell: ({ row }) => <StatusBadge status={row.original.status} />,
|
||||
meta: { headerTitle: 'Статус' },
|
||||
},
|
||||
],
|
||||
[nameById],
|
||||
)
|
||||
|
||||
return (
|
||||
<FrameDataGrid
|
||||
title="Недавние задачи"
|
||||
description="Последние фоновые операции"
|
||||
columns={columns}
|
||||
data={data}
|
||||
rowId={(row) => row.job_id}
|
||||
emptyTitle="Нет задач"
|
||||
pagination={false}
|
||||
isLoading={isLoading}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
import { useMemo } from 'react'
|
||||
import { GitCommitHorizontal } from 'lucide-react'
|
||||
|
||||
import { DataGridMutedCell, DataGridNameCell } from '@/components/data-grid-cell'
|
||||
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
|
||||
import { FrameDataGrid, type DataGridColumnDef } from '@/components/reui-kit'
|
||||
import type { RevisionRow } from '@/types/api'
|
||||
|
||||
export function DashboardRecentRevisionsGrid({
|
||||
revisions,
|
||||
isLoading = false,
|
||||
}: {
|
||||
revisions: RevisionRow[]
|
||||
isLoading?: boolean
|
||||
}) {
|
||||
const data = useMemo(() => revisions.slice(0, 8), [revisions])
|
||||
|
||||
const columns = useMemo<DataGridColumnDef<RevisionRow>[]>(
|
||||
() => [
|
||||
{
|
||||
id: 'id',
|
||||
accessorFn: (row) => row.id,
|
||||
header: ({ column }) => <DataGridColumnHeader column={column} title="ID" />,
|
||||
cell: ({ row }) => (
|
||||
<DataGridNameCell icon={GitCommitHorizontal} title={`${row.original.id.slice(0, 10)}…`} />
|
||||
),
|
||||
meta: { headerTitle: 'ID' },
|
||||
},
|
||||
{
|
||||
accessorKey: 'created_at',
|
||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Создана" />,
|
||||
cell: ({ row }) => (
|
||||
<DataGridMutedCell>
|
||||
{new Date(row.original.created_at).toLocaleString('ru-RU')}
|
||||
</DataGridMutedCell>
|
||||
),
|
||||
meta: { headerTitle: 'Создана' },
|
||||
},
|
||||
],
|
||||
[],
|
||||
)
|
||||
|
||||
return (
|
||||
<FrameDataGrid
|
||||
title="Последние ревизии"
|
||||
description="История конфигураций"
|
||||
columns={columns}
|
||||
data={data}
|
||||
rowId={(row) => row.id}
|
||||
emptyTitle="Нет ревизий"
|
||||
pagination={false}
|
||||
isLoading={isLoading}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { Link } from '@tanstack/react-router'
|
||||
|
||||
import { Frame, FramePanel } from '@/components/reui/frame'
|
||||
import { Badge } from '@/components/reui/badge'
|
||||
import { Progress } from '@evobgp/ui/components/progress'
|
||||
import { cn } from '@evobgp/ui/lib/utils'
|
||||
import { kpiCols } from './kpi-cols'
|
||||
import { IconTile } from '@/components/reui/icon-tile'
|
||||
@@ -29,6 +30,8 @@ export type KpiStatItem = {
|
||||
iconClassName?: string
|
||||
variant?: KpiStatVariant
|
||||
footer?: ReactNode
|
||||
/** 0–100 completion bar under the value (stats-4). Omit to hide. */
|
||||
progress?: number
|
||||
}
|
||||
|
||||
/** CFDM-compatible card shape (id required). */
|
||||
@@ -48,6 +51,17 @@ const VALUE_VARIANT_CLASS: Record<KpiStatVariant, string> = {
|
||||
destructive: 'text-destructive',
|
||||
}
|
||||
|
||||
const PROGRESS_TONE_CLASS: Record<KpiStatVariant, string> = {
|
||||
default: '',
|
||||
warning: '[&_[data-slot=progress-indicator]]:bg-warning',
|
||||
destructive: '[&_[data-slot=progress-indicator]]:bg-destructive',
|
||||
}
|
||||
|
||||
function clampProgress(value: number): number {
|
||||
if (Number.isNaN(value)) return 0
|
||||
return Math.min(100, Math.max(0, value))
|
||||
}
|
||||
|
||||
function handleCardKeyDown(onActivate: () => void, event: KeyboardEvent<HTMLDivElement>) {
|
||||
if (event.key === 'Enter' || event.key === ' ') {
|
||||
event.preventDefault()
|
||||
@@ -111,6 +125,20 @@ function KpiStatCardBody({ item }: { item: KpiStatItem }) {
|
||||
>
|
||||
{item.value}
|
||||
</div>
|
||||
{item.progress !== undefined ? (
|
||||
<Progress
|
||||
value={clampProgress(item.progress)}
|
||||
className={cn(
|
||||
'mt-1.5 w-full gap-0 **:data-[slot=progress-track]:h-1.5',
|
||||
PROGRESS_TONE_CLASS[valueVariant],
|
||||
)}
|
||||
aria-label={
|
||||
typeof item.label === 'string'
|
||||
? `${item.label}: ${clampProgress(item.progress)}%`
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
) : null}
|
||||
{footer ? (
|
||||
<div className="min-w-0 max-w-full @[20rem]:hidden">{footer}</div>
|
||||
) : null}
|
||||
|
||||
@@ -12,7 +12,7 @@ interface OpsDashboardProps {
|
||||
/** Slot after KPI (QuickActionGrid). Preview: stats-12 · card-12 */
|
||||
afterKpi?: ReactNode
|
||||
charts: ReactNode
|
||||
queue: ReactNode
|
||||
queue?: ReactNode
|
||||
queueTitle?: string
|
||||
queueDescription?: string
|
||||
headerActions?: ReactNode
|
||||
@@ -31,10 +31,12 @@ function OpsDashboardSkeleton() {
|
||||
</header>
|
||||
<KpiStatGrid cards={[]} isLoading skeletonCount={4} />
|
||||
<div className="flex min-w-0 flex-col gap-4">
|
||||
<Skeleton className="h-64 w-full rounded-xl" />
|
||||
<Skeleton className="h-64 w-full rounded-xl" />
|
||||
<Skeleton className="h-40 w-full rounded-xl" />
|
||||
<div className="grid min-w-0 grid-cols-1 gap-4 @5xl:grid-cols-2">
|
||||
<Skeleton className="h-56 w-full rounded-xl" />
|
||||
<Skeleton className="h-56 w-full rounded-xl" />
|
||||
</div>
|
||||
</div>
|
||||
<Skeleton className="h-48 w-full rounded-xl" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -76,15 +78,17 @@ export function OpsDashboard({
|
||||
{charts}
|
||||
</section>
|
||||
|
||||
<section aria-label={queueTitle} className="flex min-w-0 flex-col gap-4">
|
||||
<div className="flex min-w-0 flex-col gap-1">
|
||||
<h2 className="text-sm font-semibold tracking-tight">{queueTitle}</h2>
|
||||
{queueDescription ? (
|
||||
<p className="text-muted-foreground max-w-prose text-sm">{queueDescription}</p>
|
||||
) : null}
|
||||
</div>
|
||||
{queue}
|
||||
</section>
|
||||
{queue ? (
|
||||
<section aria-label={queueTitle} className="flex min-w-0 flex-col gap-4">
|
||||
<div className="flex min-w-0 flex-col gap-1">
|
||||
<h2 className="text-sm font-semibold tracking-tight">{queueTitle}</h2>
|
||||
{queueDescription ? (
|
||||
<p className="text-muted-foreground max-w-prose text-sm">{queueDescription}</p>
|
||||
) : null}
|
||||
</div>
|
||||
{queue}
|
||||
</section>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -4,24 +4,17 @@ import { RefreshCw } from 'lucide-react'
|
||||
import { useState } from 'react'
|
||||
|
||||
import { Button } from '@evobgp/ui/components/button'
|
||||
import { Skeleton } from '@evobgp/ui/components/skeleton'
|
||||
|
||||
import { DashboardActivityTimeline } from '@/components/dashboard/dashboard-activity-timeline'
|
||||
import { buildDashboardKpiCards } from '@/components/dashboard/dashboard-kpi-grid'
|
||||
import { DashboardModulesGrid } from '@/components/dashboard/dashboard-modules-grid'
|
||||
import { DashboardNetworkHealth } from '@/components/dashboard/dashboard-network-health'
|
||||
import { DashboardOperationsBreakdown } from '@/components/dashboard/dashboard-operations-breakdown'
|
||||
import { DashboardQuickLinks } from '@/components/dashboard/dashboard-quick-links'
|
||||
import { DashboardRecentJobsGrid } from '@/components/dashboard/dashboard-recent-jobs-grid'
|
||||
import { DashboardRecentRevisionsGrid } from '@/components/dashboard/dashboard-recent-revisions-grid'
|
||||
import { OpsDashboard } from '@/components/reui-kit'
|
||||
import { chartPanelGridClassName, dashboardMainSidebarClassName } from '@/lib/ui-surface'
|
||||
import { chartPanelGridClassName } from '@/lib/ui-surface'
|
||||
import {
|
||||
moduleNameById,
|
||||
overviewJobsQueryOptions,
|
||||
overviewModulesQueryOptions,
|
||||
overviewPeersQueryOptions,
|
||||
overviewRevisionsQueryOptions,
|
||||
overviewSpeakersQueryOptions,
|
||||
} from '@/queries/overview'
|
||||
import { settingsQueryOptions } from '@/queries/settings'
|
||||
@@ -36,11 +29,12 @@ function parseShowQuickActions(value: unknown): boolean {
|
||||
}
|
||||
|
||||
/**
|
||||
* Dashboard — OpsDashboard kit (KPI → QuickActions → modules → activity/health → queue).
|
||||
* Charts slot is a vertical stack: modules stay full-width; activity shares a row
|
||||
* with BGP widgets only at @5xl (container), never nested 8+4 inside a 2-col parent.
|
||||
* Dashboard — KPI infographic + Quick Actions + charts (no list duplicates).
|
||||
* @see https://reui.io/preview/base/dashboard-1
|
||||
* @see https://reui.io/preview/base/stats-12
|
||||
* @see https://reui.io/preview/base/stats-4
|
||||
* @see https://reui.io/preview/base/card-12
|
||||
* @see https://reui.io/preview/base/chart-27
|
||||
*/
|
||||
function DashboardComponent() {
|
||||
const [lastUpdated, setLastUpdated] = useState<Date | null>(null)
|
||||
@@ -52,14 +46,13 @@ function DashboardComponent() {
|
||||
overviewModulesQueryOptions(),
|
||||
overviewPeersQueryOptions(),
|
||||
overviewSpeakersQueryOptions(),
|
||||
overviewRevisionsQueryOptions(),
|
||||
overviewJobsQueryOptions(),
|
||||
],
|
||||
})
|
||||
|
||||
const [modulesQ, peersQ, speakersQ, revisionsQ, jobsQ] = results
|
||||
const [modulesQ, peersQ, speakersQ, jobsQ] = results
|
||||
const initialLoading =
|
||||
modulesQ.isLoading || peersQ.isLoading || speakersQ.isLoading || revisionsQ.isLoading || jobsQ.isLoading
|
||||
modulesQ.isLoading || peersQ.isLoading || speakersQ.isLoading || jobsQ.isLoading
|
||||
const refreshing = results.some((r) => r.isFetching && !r.isLoading)
|
||||
|
||||
if (!lastUpdated && !initialLoading && results.every((r) => r.isSuccess || r.isError)) {
|
||||
@@ -74,11 +67,7 @@ function DashboardComponent() {
|
||||
const modules = modulesQ.data?.items ?? []
|
||||
const peers = peersQ.data?.items ?? []
|
||||
const speakers = speakersQ.data?.items ?? []
|
||||
const revisions = revisionsQ.data?.items ?? []
|
||||
const jobs = jobsQ.data?.items ?? []
|
||||
const nameById = moduleNameById(modules)
|
||||
|
||||
const activityLoading = refreshing && jobs.length === 0 && revisions.length === 0
|
||||
const kpiCards = buildDashboardKpiCards({ modules, peers, speakers, jobs })
|
||||
|
||||
return (
|
||||
@@ -99,38 +88,18 @@ function DashboardComponent() {
|
||||
isLoading={initialLoading}
|
||||
afterKpi={showQuickActions ? <DashboardQuickLinks /> : null}
|
||||
charts={
|
||||
<>
|
||||
<div className="min-w-0">
|
||||
<DashboardModulesGrid modules={modules} isLoading={refreshing} />
|
||||
</div>
|
||||
<div className={dashboardMainSidebarClassName}>
|
||||
<DashboardActivityTimeline
|
||||
jobs={jobs}
|
||||
revisions={revisions}
|
||||
peers={peers}
|
||||
speakers={speakers}
|
||||
/>
|
||||
<div className="grid min-w-0 items-start gap-4">
|
||||
<DashboardNetworkHealth peers={peers} speakers={speakers} jobs={jobs} />
|
||||
<DashboardOperationsBreakdown jobs={jobs} modules={modules} />
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
}
|
||||
queueTitle="Задачи и ревизии"
|
||||
queueDescription="Последние фоновые операции и история конфигураций"
|
||||
queue={
|
||||
<div className={chartPanelGridClassName}>
|
||||
{activityLoading ? (
|
||||
<Skeleton className="m-4 h-24 w-auto" />
|
||||
) : (
|
||||
<DashboardRecentJobsGrid jobs={jobs.slice(0, 8)} nameById={nameById} isLoading={refreshing} />
|
||||
)}
|
||||
{activityLoading ? (
|
||||
<Skeleton className="m-4 h-24 w-auto" />
|
||||
) : (
|
||||
<DashboardRecentRevisionsGrid revisions={revisions} isLoading={refreshing} />
|
||||
)}
|
||||
<DashboardNetworkHealth
|
||||
peers={peers}
|
||||
speakers={speakers}
|
||||
jobs={jobs}
|
||||
loading={refreshing && peers.length === 0 && speakers.length === 0}
|
||||
/>
|
||||
<DashboardOperationsBreakdown
|
||||
jobs={jobs}
|
||||
modules={modules}
|
||||
loading={refreshing && jobs.length === 0 && modules.length === 0}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -168,6 +168,16 @@ http://localhost:5173,http://127.0.0.1:5173,https://ui.example.com
|
||||
|
||||
Разрешённые заголовки включают `Authorization`, `Content-Type`, `Idempotency-Key`, `Accept`, `X-Tenant-Id` (см. `internal/httpapi/cors.go`).
|
||||
|
||||
## Pipeline: TTL и внешние вызовы
|
||||
|
||||
| Переменная | Назначение |
|
||||
|------------|------------|
|
||||
| `EVOBGP_ASN_CACHE_TTL_SEC` | TTL кэша объявленных префиксов RIPEstat (default 1800) |
|
||||
| `EVOBGP_ASN_HOLDER_TTL_SEC` | TTL имени holder AS (default 7 суток) |
|
||||
| `EVOBGP_CDN_DNS_CACHE_TTL_SEC` | TTL DNS при проверке CDN URL (default 300) |
|
||||
| `EVOBGP_DOMAIN_CACHE_TTL_SEC` | TTL кэша DoH A/AAAA (default 300) |
|
||||
| `EVOBGP_CDN_PARTIAL_OK` | Сохранять prior-строки skipped CDN-источников при частичном сбое |
|
||||
|
||||
## Заголовок `X-Tenant-Id` (решение: не реализован)
|
||||
|
||||
**Решение (done):** заголовок **`X-Tenant-Id` не переключает tenant** в handlers и **не планируется** без отдельного ADR на супер-роли.
|
||||
|
||||
+12
-1
@@ -39,7 +39,7 @@
|
||||
| `config` | Переменные окружения `EVOBGP_*`. |
|
||||
| `observability` | Метрики Prometheus, HTTP middleware. |
|
||||
| `broker` | Опциональный `EVOBGP_BROKER_URL` для будущей шины; сейчас задачи только in-process (`jobs.Registry`), пакет лишь логирует факт настройки URL. |
|
||||
| `pipeline` | Ingest+render в одном шаге для `module_refresh`: выборка префиксов (CDN/AS/IP/пустые DOMAINS), `CreateRenderRevision`, превью BIRD через `birdfmt`. |
|
||||
| `pipeline` | Ingest+render для `module_refresh`: CDN/AS/IP/DOMAINS → `module_prefix_snapshot` (batch `COPY`, per-module lock) → агрегация CIDR O(n log n) → `CreateRenderRevision`. Fast-path снапшота — `module.input_hash`; DoH — TTL-кэш `domain_resolve_cache`; scheduler — jitter границ интервала. |
|
||||
| `nodedispatch` | Panel→Node HTTP wake-up (`POST /v1/agent/sync`) после `deploy_apply`. |
|
||||
| `agentserver` | HTTP API на реплике (`serve`): sync + health для Traefik; опционально firewall failover (`/v1/firewall/*`). |
|
||||
| `firewall` | Вычисление policy block/accept → плоский CIDR blocklist. |
|
||||
@@ -98,6 +98,17 @@ flowchart LR
|
||||
|
||||
Профиль Compose **`microvps-full`** добавляет к этому стеку **Web UI** (nginx → `evobgp-all`), **NATS** и **Prometheus** без отдельных контейнеров воркеров (функционально то же, что отдельные `scheduler`/`ingest`/… в reference). Запуск и лимиты под ~1 ГиБ RAM — в [quickstart.md](quickstart.md).
|
||||
|
||||
## Поток pipeline (ingest → render)
|
||||
|
||||
1. **Scheduler** (`evobgp-scheduler` / in-process в `evobgp-all`) ставит `tenant_refresh`, если `ModuleDueForScheduler`: граница окна `refresh_interval_sec` со **сдвигом `fnv32(module.ID) % interval`**, чтобы модули с одним интервалом не били внешние API одновременно.
|
||||
2. **Ingest** (`RefreshModuleIngest`):
|
||||
- `AS_PREFIXES` — RIPEstat prefixes + holder параллельно, кэш `asn_prefix_cache`; дедуп строк по `prefix + community`.
|
||||
- `CDN_CIDRS` — единый `fetchCDNSourceRows` (conditional GET); prefetch уважает `RefreshIntervalSec`; merge снапшота под `LockModuleSnapshot` (пропущенные по ошибке источники сохраняют prior-строки при `EVOBGP_CDN_PARTIAL_OK`).
|
||||
- `DOMAINS` — DoH A+AAAA параллельно; попадания в `domain_resolve_cache` с TTL `EVOBGP_DOMAIN_CACHE_TTL_SEC` (default 300).
|
||||
- `IP_RANGES` — напрямую из записей модуля.
|
||||
3. Снапшот пишется **batch** (`pgx.CopyFrom` в PostgreSQL). Совпадение `module.input_hash` со снапшотом — O(1) пропуск повторного ingest при render; CRUD entries обнуляет hash.
|
||||
4. **Render** (`RenderTenantRevision`): `smartAggregatePrefixRows` (стек-схлопывание O(n log n), IPv6 без `math/big`) → ревизия, если набор префиксов изменился.
|
||||
|
||||
## Поток: ревизия и бандл для ноды
|
||||
|
||||
1. Оператор (роль `operator` или выше по политике) изменяет модули и запускает цепочку, приводящую к новой **ревизии** (часть шагов может быть асинхронной через jobs — см. OpenAPI).
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
- `EVOBGP_CORS_ORIGINS` — явный whitelist origin веб-панели.
|
||||
- `EVOBGP_STALE_ON_UPSTREAM_ERROR=1` (по умолчанию) — stale snapshot при сбоях CDN/ASN/DoH.
|
||||
- Опционально `EVOBGP_CDN_PARTIAL_OK=1` — при сбое одного CDN source без stale cache продолжать refresh остальных (иначе fail модуля).
|
||||
- `EVOBGP_ASN_HOLDER_TTL_SEC` — TTL имени holder AS (default 7 суток); prefixes кэшируются отдельно (`EVOBGP_ASN_CACHE_TTL_SEC`, default 1800).
|
||||
- `EVOBGP_CDN_DNS_CACHE_TTL_SEC` — TTL кэша DNS при SSRF-проверке CDN URL (default 300).
|
||||
- `EVOBGP_DOMAIN_CACHE_TTL_SEC` — TTL кэша DoH A/AAAA (`domain_resolve_cache`, default 300).
|
||||
|
||||
## Рекомендуется
|
||||
|
||||
|
||||
@@ -10,9 +10,7 @@ import (
|
||||
"net/netip"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/httpclient"
|
||||
)
|
||||
@@ -137,22 +135,3 @@ func truncateForErr(b []byte, n int) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// PolitePause is a short delay between upstream ASN lookups (same refresh).
|
||||
func PolitePause() {
|
||||
d := 150 * time.Millisecond
|
||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE_PAUSE_MS")); s != "" {
|
||||
if ms, err := parsePositiveInt(s); err == nil && ms > 0 {
|
||||
d = time.Duration(ms) * time.Millisecond
|
||||
}
|
||||
}
|
||||
time.Sleep(d)
|
||||
}
|
||||
|
||||
func parsePositiveInt(s string) (int, error) {
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil || n <= 0 {
|
||||
return 0, fmt.Errorf("invalid")
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"sort"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// Prefix collapse replaces the former prune+merge fixed-point loop (O(n²) per pass,
|
||||
// multiple passes) with a single sort + linear stack pass: O(n log n) overall.
|
||||
//
|
||||
// Classic trie-collapse without building a trie:
|
||||
// 1. Sort prefixes by (address, mask len ascending).
|
||||
// 2. Walk left to right keeping a stack of "open" prefixes:
|
||||
// - while the stack top contains the new prefix -> the top covers it, drop the new one (prune);
|
||||
// - else, while the stack top is a sibling of the new prefix (same mask, XOR of the
|
||||
// network bit equals the parent block) and merging is allowed for that mask length
|
||||
// -> pop the sibling, replace the new prefix with the parent and re-check;
|
||||
// - otherwise push the new prefix.
|
||||
//
|
||||
// A parent absorbs its sibling children only when both halves are present, which matches
|
||||
// the previous merge-to-fixed-point semantics, including the guard that forbids merging
|
||||
// above a floor mask (IPv4 /8, IPv6 /16).
|
||||
|
||||
type collapseItem struct {
|
||||
row store.PrefixRow
|
||||
pfx netip.Prefix
|
||||
}
|
||||
|
||||
// collapsePrefixGroup collapses one (community, source) group of same-family prefixes.
|
||||
// All rows must be masked; family and floor are enforced by minBits.
|
||||
func collapsePrefixGroup(rows []store.PrefixRow, is4 bool) []store.PrefixRow {
|
||||
if len(rows) <= 1 {
|
||||
return rows
|
||||
}
|
||||
items := make([]collapseItem, 0, len(rows))
|
||||
seen := make(map[string]struct{}, len(rows))
|
||||
for _, row := range rows {
|
||||
if _, dup := seen[row.Prefix]; dup {
|
||||
continue
|
||||
}
|
||||
seen[row.Prefix] = struct{}{}
|
||||
pfx, err := netip.ParsePrefix(row.Prefix)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
pfx = pfx.Masked()
|
||||
if is4 != pfx.Addr().Is4() {
|
||||
continue
|
||||
}
|
||||
items = append(items, collapseItem{row: row, pfx: pfx})
|
||||
}
|
||||
if len(items) <= 1 {
|
||||
out := make([]store.PrefixRow, 0, len(items))
|
||||
for _, it := range items {
|
||||
out = append(out, it.row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Sort by (address, mask len): a parent always sorts before its children, and a
|
||||
// shorter sibling sorts before a longer one within the same parent block.
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
a, b := items[i].pfx, items[j].pfx
|
||||
if a.Addr() != b.Addr() {
|
||||
return lessAddr(a.Addr(), b.Addr())
|
||||
}
|
||||
return a.Bits() < b.Bits()
|
||||
})
|
||||
|
||||
minBits := 8
|
||||
if !is4 {
|
||||
minBits = 16
|
||||
}
|
||||
|
||||
stack := make([]collapseItem, 0, len(items))
|
||||
for _, it := range items {
|
||||
cur := it
|
||||
dropped := false
|
||||
for len(stack) > 0 {
|
||||
top := stack[len(stack)-1]
|
||||
if top.pfx.Contains(cur.pfx.Addr()) && top.pfx.Bits() <= cur.pfx.Bits() {
|
||||
// Covered by an existing prefix: drop (prune).
|
||||
dropped = true
|
||||
break
|
||||
}
|
||||
if cur.pfx.Bits() == top.pfx.Bits() && cur.pfx.Bits() > minBits && areSiblings(top.pfx, cur.pfx) {
|
||||
// Merge siblings into the parent (parent keeps the lower sibling's attributes),
|
||||
// then re-check the parent against the new stack top.
|
||||
stack = stack[:len(stack)-1]
|
||||
parentBits := cur.pfx.Bits() - 1
|
||||
parent := netip.PrefixFrom(maskAddr(cur.pfx.Addr(), parentBits, is4), parentBits).Masked()
|
||||
cur = collapseItem{row: top.row, pfx: parent}
|
||||
cur.row.Prefix = parent.String()
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if !dropped {
|
||||
stack = append(stack, cur)
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]store.PrefixRow, 0, len(stack))
|
||||
for _, it := range stack {
|
||||
out = append(out, it.row)
|
||||
}
|
||||
sortPrefixRows(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// areSiblings reports whether two same-length prefixes combine into their common parent.
|
||||
func areSiblings(a, b netip.Prefix) bool {
|
||||
if a.Bits() != b.Bits() || a.Bits() == 0 {
|
||||
return false
|
||||
}
|
||||
parentBits := a.Bits() - 1
|
||||
pa := maskAddr(a.Addr(), parentBits, a.Addr().Is4())
|
||||
pb := maskAddr(b.Addr(), parentBits, b.Addr().Is4())
|
||||
return pa == pb
|
||||
}
|
||||
|
||||
// maskAddr clears the host bits below prefixLen (IPv4: 32-bit space; IPv6: 128-bit).
|
||||
func maskAddr(a netip.Addr, prefixLen int, is4 bool) netip.Addr {
|
||||
if is4 {
|
||||
v := uint32FromIPv4(a)
|
||||
if prefixLen <= 0 {
|
||||
v = 0
|
||||
} else if prefixLen < 32 {
|
||||
v &= ^(uint32(1)<<(32-prefixLen) - 1)
|
||||
}
|
||||
return u32ToIPv4(v)
|
||||
}
|
||||
b := a.As16()
|
||||
hostBits := 128 - prefixLen
|
||||
fullBytes := hostBits / 8
|
||||
for i := 15; i > 15-fullBytes; i-- {
|
||||
b[i] = 0
|
||||
}
|
||||
if rem := hostBits % 8; rem > 0 {
|
||||
idx := 15 - fullBytes
|
||||
if idx >= 0 && idx < 16 {
|
||||
b[idx] &= byte(0xFF << rem)
|
||||
}
|
||||
}
|
||||
return netip.AddrFrom16(b)
|
||||
}
|
||||
|
||||
func uint32FromIPv4(a netip.Addr) uint32 {
|
||||
o := a.As4()
|
||||
return uint32(o[0])<<24 | uint32(o[1])<<16 | uint32(o[2])<<8 | uint32(o[3])
|
||||
}
|
||||
|
||||
func u32ToIPv4(v uint32) netip.Addr {
|
||||
return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
|
||||
}
|
||||
|
||||
func lessAddr(a, b netip.Addr) bool {
|
||||
if a.Is4() != b.Is4() {
|
||||
return a.Is4()
|
||||
}
|
||||
if a.Is4() {
|
||||
return uint32FromIPv4(a) < uint32FromIPv4(b)
|
||||
}
|
||||
a16, b16 := a.As16(), b.As16()
|
||||
for i := 0; i < 16; i++ {
|
||||
if a16[i] != b16[i] {
|
||||
return a16[i] < b16[i]
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"net/netip"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// The functions below are a verbatim copy of the pre-2.1 prune+merge loop
|
||||
// (from git HEAD internal/pipeline/refresh.go). They exist only as an
|
||||
// equivalence oracle for collapsePrefixGroup.
|
||||
|
||||
func legacyAggregateCIDRGroup(rows []store.PrefixRow, mergeFn func(map[string]store.PrefixRow) bool) []store.PrefixRow {
|
||||
if len(rows) <= 1 {
|
||||
return rows
|
||||
}
|
||||
set := make(map[string]store.PrefixRow, len(rows))
|
||||
for _, row := range rows {
|
||||
set[row.Prefix] = row
|
||||
}
|
||||
legacyPruneCoveredPrefixes(set)
|
||||
for {
|
||||
if !mergeFn(set) {
|
||||
break
|
||||
}
|
||||
legacyPruneCoveredPrefixes(set)
|
||||
}
|
||||
out := make([]store.PrefixRow, 0, len(set))
|
||||
for _, row := range set {
|
||||
out = append(out, row)
|
||||
}
|
||||
sortPrefixRows(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func legacyPruneCoveredPrefixes(set map[string]store.PrefixRow) {
|
||||
type item struct {
|
||||
key string
|
||||
pfx netip.Prefix
|
||||
bits int
|
||||
}
|
||||
items := make([]item, 0, len(set))
|
||||
for k := range set {
|
||||
p, err := netip.ParsePrefix(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
items = append(items, item{key: k, pfx: p, bits: p.Bits()})
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i].bits != items[j].bits {
|
||||
return items[i].bits < items[j].bits
|
||||
}
|
||||
return items[i].key < items[j].key
|
||||
})
|
||||
for i := 0; i < len(items); i++ {
|
||||
for j := i + 1; j < len(items); j++ {
|
||||
if items[j].bits <= items[i].bits {
|
||||
continue
|
||||
}
|
||||
if items[i].pfx.Contains(items[j].pfx.Addr()) {
|
||||
delete(set, items[j].key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func legacyMergeSiblingPrefixesIPv4(set map[string]store.PrefixRow) bool {
|
||||
merged := false
|
||||
seen := make(map[string]struct{}, len(set))
|
||||
for key, row := range set {
|
||||
if _, done := seen[key]; done {
|
||||
continue
|
||||
}
|
||||
pfx, err := netip.ParsePrefix(key)
|
||||
if err != nil || !pfx.Addr().Is4() {
|
||||
continue
|
||||
}
|
||||
bits := pfx.Bits()
|
||||
if bits <= 8 {
|
||||
continue
|
||||
}
|
||||
netNum := ipv4PrefixNetworkLegacy(pfx)
|
||||
blockSize := uint32(1) << (32 - bits)
|
||||
siblingNet := netNum ^ blockSize
|
||||
siblingPfx := netip.PrefixFrom(u32ToIPv4(siblingNet), bits).Masked().String()
|
||||
if _, ok := set[siblingPfx]; !ok {
|
||||
continue
|
||||
}
|
||||
parentBits := bits - 1
|
||||
parentBlock := uint32(1) << (32 - parentBits)
|
||||
parentNet := netNum & ^(parentBlock - 1)
|
||||
parentPfx := netip.PrefixFrom(u32ToIPv4(parentNet), parentBits).Masked().String()
|
||||
delete(set, key)
|
||||
delete(set, siblingPfx)
|
||||
parentRow := row
|
||||
parentRow.Prefix = parentPfx
|
||||
set[parentPfx] = parentRow
|
||||
seen[key] = struct{}{}
|
||||
seen[siblingPfx] = struct{}{}
|
||||
merged = true
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func ipv4PrefixNetworkLegacy(p netip.Prefix) uint32 {
|
||||
a := p.Masked().Addr().As4()
|
||||
return uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3])
|
||||
}
|
||||
|
||||
func legacyMergeSiblingPrefixesIPv6(set map[string]store.PrefixRow) bool {
|
||||
merged := false
|
||||
seen := make(map[string]struct{}, len(set))
|
||||
for key, row := range set {
|
||||
if _, done := seen[key]; done {
|
||||
continue
|
||||
}
|
||||
pfx, err := netip.ParsePrefix(key)
|
||||
if err != nil || !pfx.Addr().Is6() {
|
||||
continue
|
||||
}
|
||||
bits := pfx.Bits()
|
||||
if bits <= 16 {
|
||||
continue
|
||||
}
|
||||
netNum := ipv6PrefixNetworkLegacy(pfx)
|
||||
blockSize := new(big.Int).Lsh(big.NewInt(1), uint(128-bits))
|
||||
siblingNet := new(big.Int).Xor(netNum, blockSize)
|
||||
siblingPfx := ipv6PrefixFromBigIntLegacy(siblingNet, bits).String()
|
||||
if _, ok := set[siblingPfx]; !ok {
|
||||
continue
|
||||
}
|
||||
parentBits := bits - 1
|
||||
parentBlock := new(big.Int).Lsh(big.NewInt(1), uint(128-parentBits))
|
||||
mask := new(big.Int).Sub(parentBlock, big.NewInt(1))
|
||||
mask.Not(mask)
|
||||
parentNet := new(big.Int).And(netNum, mask)
|
||||
parentPfx := ipv6PrefixFromBigIntLegacy(parentNet, parentBits).String()
|
||||
delete(set, key)
|
||||
delete(set, siblingPfx)
|
||||
parentRow := row
|
||||
parentRow.Prefix = parentPfx
|
||||
set[parentPfx] = parentRow
|
||||
seen[key] = struct{}{}
|
||||
seen[siblingPfx] = struct{}{}
|
||||
merged = true
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func ipv6PrefixNetworkLegacy(p netip.Prefix) *big.Int {
|
||||
a := p.Masked().Addr().As16()
|
||||
n := new(big.Int)
|
||||
n.SetBytes(a[:])
|
||||
return n
|
||||
}
|
||||
|
||||
func ipv6PrefixFromBigIntLegacy(n *big.Int, bits int) netip.Prefix {
|
||||
b := n.Bytes()
|
||||
var a [16]byte
|
||||
copy(a[16-len(b):], b)
|
||||
return netip.PrefixFrom(netip.AddrFrom16(a), bits).Masked()
|
||||
}
|
||||
|
||||
func normalizeForCompare(rows []store.PrefixRow) []string {
|
||||
type line struct{ p, c, s string }
|
||||
lines := make([]line, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
p := strings.TrimSpace(r.Prefix)
|
||||
if p != "" {
|
||||
if pfx, err := netip.ParsePrefix(p); err == nil {
|
||||
p = pfx.Masked().String()
|
||||
}
|
||||
}
|
||||
lines = append(lines, line{p, prefixRowCommunity(r), r.Source})
|
||||
}
|
||||
sort.Slice(lines, func(i, j int) bool {
|
||||
if lines[i].p != lines[j].p {
|
||||
return lines[i].p < lines[j].p
|
||||
}
|
||||
if lines[i].c != lines[j].c {
|
||||
return lines[i].c < lines[j].c
|
||||
}
|
||||
return lines[i].s < lines[j].s
|
||||
})
|
||||
out := make([]string, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
out = append(out, fmt.Sprintf("%s|%s|%s", l.p, l.c, l.s))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestCollapsePrefixGroup_EquivalenceWithLegacy(t *testing.T) {
|
||||
r := rand.New(rand.NewSource(7))
|
||||
for iter := 0; iter < 200; iter++ {
|
||||
n := 1 + r.Intn(60)
|
||||
v4rows := make([]store.PrefixRow, 0, n)
|
||||
for i := 0; i < n; i++ {
|
||||
addr := netip.AddrFrom4([4]byte{203, byte(r.Intn(4)), byte(r.Intn(256)), byte(r.Intn(256))})
|
||||
bits := 16 + r.Intn(9)
|
||||
pfx := netip.PrefixFrom(addr, bits).Masked()
|
||||
v4rows = append(v4rows, store.PrefixRow{Prefix: pfx.String(), Source: "ip_range"})
|
||||
}
|
||||
legacy := legacyAggregateCIDRGroup(append([]store.PrefixRow(nil), v4rows...), legacyMergeSiblingPrefixesIPv4)
|
||||
got := collapsePrefixGroup(append([]store.PrefixRow(nil), v4rows...), true)
|
||||
if fmt.Sprint(normalizeForCompare(legacy)) != fmt.Sprint(normalizeForCompare(got)) {
|
||||
t.Fatalf("iter %d mismatch:\nlegacy=%v\ngot =%v", iter, normalizeForCompare(legacy), normalizeForCompare(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapsePrefixGroup_EquivalenceWithLegacyIPv6(t *testing.T) {
|
||||
r := rand.New(rand.NewSource(11))
|
||||
for iter := 0; iter < 200; iter++ {
|
||||
n := 1 + r.Intn(60)
|
||||
rows := make([]store.PrefixRow, 0, n)
|
||||
for i := 0; i < n; i++ {
|
||||
var a [16]byte
|
||||
a[0], a[1] = 0x20, 0x01
|
||||
a[2], a[3] = 0x0d, 0xb8
|
||||
a[4] = byte(r.Intn(2))
|
||||
a[5] = byte(r.Intn(256))
|
||||
a[6] = byte(r.Intn(256))
|
||||
addr := netip.AddrFrom16(a)
|
||||
bits := 32 + r.Intn(17)
|
||||
pfx := netip.PrefixFrom(addr, bits).Masked()
|
||||
rows = append(rows, store.PrefixRow{Prefix: pfx.String(), Source: "ip_range"})
|
||||
}
|
||||
legacy := legacyAggregateCIDRGroup(append([]store.PrefixRow(nil), rows...), legacyMergeSiblingPrefixesIPv6)
|
||||
got := collapsePrefixGroup(append([]store.PrefixRow(nil), rows...), false)
|
||||
if fmt.Sprint(normalizeForCompare(legacy)) != fmt.Sprint(normalizeForCompare(got)) {
|
||||
t.Fatalf("iter %d mismatch:\nlegacy=%v\ngot =%v", iter, normalizeForCompare(legacy), normalizeForCompare(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapsePrefixGroup_CoversAndSiblingChain(t *testing.T) {
|
||||
rows := []store.PrefixRow{
|
||||
{Prefix: "10.0.0.0/16", Source: "ip_range"},
|
||||
{Prefix: "10.0.0.0/24", Source: "ip_range"},
|
||||
{Prefix: "10.0.1.0/24", Source: "ip_range"},
|
||||
{Prefix: "10.0.2.0/24", Source: "ip_range"},
|
||||
{Prefix: "10.0.3.0/24", Source: "ip_range"},
|
||||
{Prefix: "10.1.0.0/24", Source: "ip_range"},
|
||||
}
|
||||
got := collapsePrefixGroup(rows, true)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 rows (/16 + /24), got %d: %+v", len(got), got)
|
||||
}
|
||||
set := map[string]bool{}
|
||||
for _, r := range got {
|
||||
set[r.Prefix] = true
|
||||
}
|
||||
if !set["10.0.0.0/16"] || !set["10.1.0.0/24"] {
|
||||
t.Fatalf("unexpected rows: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapsePrefixGroup_RespectsMinBitsFloor(t *testing.T) {
|
||||
rows := []store.PrefixRow{
|
||||
{Prefix: "10.0.0.0/8", Source: "ip_range"},
|
||||
{Prefix: "11.0.0.0/8", Source: "ip_range"},
|
||||
}
|
||||
got := collapsePrefixGroup(rows, true)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("floor must prevent /8+/8 -> /7, got %+v", got)
|
||||
}
|
||||
|
||||
rows9 := []store.PrefixRow{
|
||||
{Prefix: "10.0.0.0/9", Source: "ip_range"},
|
||||
{Prefix: "10.128.0.0/9", Source: "ip_range"},
|
||||
}
|
||||
got9 := collapsePrefixGroup(rows9, true)
|
||||
if len(got9) != 1 || got9[0].Prefix != "10.0.0.0/8" {
|
||||
t.Fatalf("expected /9+/9 -> /8, got %+v", got9)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollapsePrefixGroup_PrunesCovered(t *testing.T) {
|
||||
rows := []store.PrefixRow{
|
||||
{Prefix: "10.0.0.0/16", Source: "ip_range"},
|
||||
{Prefix: "10.0.0.0/24", Source: "ip_range"},
|
||||
}
|
||||
got := collapsePrefixGroup(rows, true)
|
||||
if len(got) != 1 || got[0].Prefix != "10.0.0.0/16" {
|
||||
t.Fatalf("want covered prune to /16, got %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -24,28 +24,43 @@ func asnCacheTTL() time.Duration {
|
||||
return time.Duration(sec) * time.Second
|
||||
}
|
||||
|
||||
// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache).
|
||||
func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) {
|
||||
ttl := asnCacheTTL()
|
||||
if st != nil {
|
||||
if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil && time.Since(ent.FetchedAt) < ttl {
|
||||
out := make([]netip.Prefix, 0, len(ent.Prefixes))
|
||||
for _, p := range ent.Prefixes {
|
||||
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
|
||||
if perr != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, pfx.Masked())
|
||||
}
|
||||
return out, ent.Holder, nil
|
||||
// asnHolderTTL is how long a holder name stays authoritative between refreshes;
|
||||
// holder text changes rarely, so it survives short prefix-cache TTLs.
|
||||
func asnHolderTTL() time.Duration {
|
||||
sec := 7 * 24 * 3600
|
||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_HOLDER_TTL_SEC")); s != "" {
|
||||
if v, err := strconv.Atoi(s); err == nil && v > 0 {
|
||||
sec = v
|
||||
}
|
||||
}
|
||||
return time.Duration(sec) * time.Second
|
||||
}
|
||||
|
||||
// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache).
|
||||
// The holder lookup starts concurrently with the prefix fetch (one RTT instead of two);
|
||||
// a holder failure is non-fatal — the previously cached holder name is kept.
|
||||
func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) {
|
||||
ttl := asnCacheTTL()
|
||||
var prevHolder string
|
||||
if st != nil {
|
||||
if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil {
|
||||
prevHolder = ent.Holder
|
||||
if time.Since(ent.FetchedAt) < ttl {
|
||||
return parseASNCachePrefixes(ent.Prefixes), ent.Holder, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
holderCh := startASNHolderFetch(ctx, hc, st, asn, prevHolder)
|
||||
|
||||
pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, asn)
|
||||
if err != nil {
|
||||
holder := <-holderCh // drain to avoid leaking the goroutine's channel send
|
||||
_ = holder
|
||||
return nil, "", err
|
||||
}
|
||||
asnresolve.PolitePause()
|
||||
holder, _ := asnresolve.ASHolderName(ctx, hc, asn)
|
||||
|
||||
holder := <-holderCh
|
||||
if st != nil {
|
||||
strs := make([]string, len(pfxs))
|
||||
for i, p := range pfxs {
|
||||
@@ -57,3 +72,55 @@ func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client,
|
||||
}
|
||||
return pfxs, holder, nil
|
||||
}
|
||||
|
||||
// startASNHolderFetch launches the holder lookup concurrently. The returned buffered
|
||||
// channel always yields exactly one value, so callers may abandon it without leaking.
|
||||
func startASNHolderFetch(ctx context.Context, hc *http.Client, st store.Backend, asn int64, prevHolder string) <-chan string {
|
||||
ch := make(chan string, 1)
|
||||
go func() {
|
||||
if prevHolder != "" && holderStillFresh(st, asn, prevHolder) {
|
||||
ch <- prevHolder
|
||||
return
|
||||
}
|
||||
holder, err := asnresolve.ASHolderName(ctx, hc, asn)
|
||||
if err != nil || strings.TrimSpace(holder) == "" {
|
||||
ch <- prevHolder
|
||||
return
|
||||
}
|
||||
ch <- strings.TrimSpace(holder)
|
||||
}()
|
||||
return ch
|
||||
}
|
||||
|
||||
// holderStillFresh reports whether the cached holder name is within its own (long) TTL.
|
||||
func holderStillFresh(st store.Backend, asn int64, prevHolder string) bool {
|
||||
if st == nil {
|
||||
return false
|
||||
}
|
||||
ent, ok, err := st.GetASNPrefixCache(asn)
|
||||
return err == nil && ok && ent != nil && ent.Holder == prevHolder && time.Since(ent.FetchedAt) < asnHolderTTL()
|
||||
}
|
||||
|
||||
func parseASNCachePrefixes(raw []string) []netip.Prefix {
|
||||
out := make([]netip.Prefix, 0, len(raw))
|
||||
for _, p := range raw {
|
||||
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
|
||||
if perr != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, pfx.Masked())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseCachedASNCachedPrefixes(raw []string) []netip.Prefix {
|
||||
out := make([]netip.Prefix, 0, len(raw))
|
||||
for _, p := range raw {
|
||||
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
|
||||
if perr != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, pfx.Masked())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -51,13 +51,92 @@ func TestResolveASNForEntry_UsesTTLCache(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveASNForEntry_HolderFailureIsNonFatal(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "1")
|
||||
|
||||
var holderCalls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.Contains(r.URL.Path, "/announced") {
|
||||
_, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`))
|
||||
return
|
||||
}
|
||||
holderCalls.Add(1)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced")
|
||||
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview")
|
||||
|
||||
ctx := context.Background()
|
||||
hc := srv.Client()
|
||||
pfxs, holder, err := resolveASNForEntry(ctx, m, hc, 64512)
|
||||
if err != nil {
|
||||
t.Fatalf("holder failure must not fail ingest: %v", err)
|
||||
}
|
||||
if len(pfxs) != 1 {
|
||||
t.Fatalf("expected 1 prefix, got %+v", pfxs)
|
||||
}
|
||||
if holder != "" {
|
||||
t.Fatalf("expected empty holder on upstream failure, got %q", holder)
|
||||
}
|
||||
if holderCalls.Load() == 0 {
|
||||
t.Fatal("expected holder endpoint to be attempted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveASNForEntry_KeepsPreviousHolderOnFailure(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "1")
|
||||
t.Setenv("EVOBGP_ASN_HOLDER_TTL_SEC", "3600")
|
||||
|
||||
holderOK := atomic.Bool{}
|
||||
holderOK.Store(true)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.Contains(r.URL.Path, "/announced") {
|
||||
_, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`))
|
||||
return
|
||||
}
|
||||
if holderOK.Load() {
|
||||
_, _ = w.Write([]byte(`{"status":"ok","data":{"holder":"Good AS"}}`))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced")
|
||||
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview")
|
||||
|
||||
ctx := context.Background()
|
||||
hc := srv.Client()
|
||||
if _, h, err := resolveASNForEntry(ctx, m, hc, 64512); err != nil || h != "Good AS" {
|
||||
t.Fatalf("first resolve: holder=%q err=%v", h, err)
|
||||
}
|
||||
|
||||
// Prefix cache expired; holder endpoint now broken — previous holder must survive.
|
||||
holderOK.Store(false)
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
pfxs, h, err := resolveASNForEntry(ctx, m, hc, 64512)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(pfxs) != 1 {
|
||||
t.Fatalf("expected 1 prefix, got %+v", pfxs)
|
||||
}
|
||||
if h != "Good AS" {
|
||||
t.Fatalf("expected previous holder preserved, got %q", h)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleDueForScheduler_BucketRollover(t *testing.T) {
|
||||
mod := &store.Module{Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300}
|
||||
boundary := time.Unix(300, 0)
|
||||
mod := &store.Module{ID: "mod-jitter", Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300}
|
||||
win := int64(300)
|
||||
off := moduleSchedulerOffset(mod.ID, win)
|
||||
boundary := time.Unix(win+off, 0)
|
||||
if !ModuleDueForScheduler(mod, boundary) {
|
||||
t.Fatal("expected due when refresh bucket rolls")
|
||||
}
|
||||
mid := time.Unix(330, 0)
|
||||
mid := time.Unix(win+off+SchedulerTickSec, 0)
|
||||
if ModuleDueForScheduler(mod, mid) {
|
||||
t.Fatal("expected not due within same bucket")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// benchPrefixRows generates seed-stable pseudo-random prefixes for aggregation benchmarks.
|
||||
// v4 blocks are carved from TEST-NET-style ranges; v6 from 2001:db8::/32.
|
||||
func benchPrefixRows(n int, v6Share float64) []store.PrefixRow {
|
||||
r := rand.New(rand.NewSource(42))
|
||||
rows := make([]store.PrefixRow, 0, n)
|
||||
seen := make(map[string]struct{}, n)
|
||||
for len(rows) < n {
|
||||
var pfx netip.Prefix
|
||||
if r.Float64() < v6Share {
|
||||
var a [16]byte
|
||||
a[0], a[1] = 0x20, 0x01
|
||||
a[2], a[3] = 0x0d, 0xb8
|
||||
for i := 4; i < 10; i++ {
|
||||
a[i] = byte(r.Intn(256))
|
||||
}
|
||||
addr := netip.AddrFrom16(a)
|
||||
pfx = netip.PrefixFrom(addr, 48+r.Intn(17))
|
||||
} else {
|
||||
b := []byte{203, 0, 113, 0}
|
||||
b[1] = byte(r.Intn(256))
|
||||
b[2] = byte(r.Intn(256))
|
||||
b[3] = byte(r.Intn(256))
|
||||
addr := netip.AddrFrom4([4]byte{b[0], b[1], b[2], b[3]})
|
||||
pfx = netip.PrefixFrom(addr, 16+r.Intn(9))
|
||||
}
|
||||
s := pfx.Masked().String()
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
rows = append(rows, store.PrefixRow{Prefix: s, Source: "ip_range"})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func BenchmarkSmartAggregatePrefixRows(b *testing.B) {
|
||||
for _, n := range []int{1_000, 10_000, 50_000} {
|
||||
rows := benchPrefixRows(n, 0.3)
|
||||
b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = smartAggregatePrefixRows(rows)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
@@ -194,3 +196,63 @@ func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) {
|
||||
t.Fatalf("unexpected collected rows: %+v", collected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefetchCDNSourceETags_SkipsFreshSources(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
|
||||
mod, err := m.CreateModule(tenant, &store.Module{
|
||||
Type: "CDN_CIDRS",
|
||||
Name: "cdn-prefetch-due",
|
||||
Enabled: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var hits atomic.Int32
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
hits.Add(1)
|
||||
w.Header().Set("ETag", `"v1"`)
|
||||
_, _ = w.Write([]byte("198.51.100.0/24\n"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Fresh source: refreshed 30s ago with a 3600s interval — prefetch must skip it.
|
||||
freshAt := time.Now().UTC().Add(-30 * time.Second)
|
||||
interval := 3600
|
||||
if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
|
||||
SourceKind: "txt",
|
||||
URL: srv.URL,
|
||||
RefreshIntervalSec: &interval,
|
||||
LastRefreshedAt: &freshAt,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := PrefetchCDNSourceETags(context.Background(), m, srv.Client()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hits.Load() != 0 {
|
||||
t.Fatalf("fresh source must be skipped by prefetch, got %d HTTP hits", hits.Load())
|
||||
}
|
||||
|
||||
// Stale source: last refresh older than its interval — prefetch must probe it.
|
||||
staleAt := time.Now().UTC().Add(-7200 * time.Second)
|
||||
if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
|
||||
SourceKind: "txt",
|
||||
URL: srv.URL + "?stale",
|
||||
RefreshIntervalSec: &interval,
|
||||
LastRefreshedAt: &staleAt,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := PrefetchCDNSourceETags(context.Background(), m, srv.Client()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hits.Load() == 0 {
|
||||
t.Fatal("stale source must be probed by prefetch")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,32 +42,49 @@ func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.P
|
||||
return out
|
||||
}
|
||||
|
||||
// mergeSnapshotDropCDNSources removes all cdn:* rows (used before batch CDN merge).
|
||||
func mergeSnapshotDropCDNSources(rows []store.PrefixRow) []store.PrefixRow {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
// mergeSnapshotKeepSkippedCDN keeps non-CDN rows and cdn:* rows whose source is still skipped
|
||||
// (not in fetchedSourceIDs). Deleted sources (absent from allSourceIDs) are dropped.
|
||||
func mergeSnapshotKeepSkippedCDN(rows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) []store.PrefixRow {
|
||||
fetched := make(map[string]struct{}, len(fetchedSourceIDs))
|
||||
for _, id := range fetchedSourceIDs {
|
||||
fetched[cdnSourceKey(id)] = struct{}{}
|
||||
}
|
||||
keepCDN := make(map[string]struct{})
|
||||
for _, id := range allSourceIDs {
|
||||
k := cdnSourceKey(id)
|
||||
if _, ok := fetched[k]; !ok {
|
||||
keepCDN[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make([]store.PrefixRow, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if !strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") {
|
||||
out = append(out, row)
|
||||
src := strings.TrimSpace(row.Source)
|
||||
if strings.HasPrefix(src, "cdn:") {
|
||||
if _, ok := keepCDN[src]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, row)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mergeAllCDNSourcesIntoModuleSnapshot replaces all CDN rows in one write (avoids parallel read-modify-write races).
|
||||
func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow) error {
|
||||
// mergeAllCDNSourcesIntoModuleSnapshot replaces fetched CDN source rows in one write.
|
||||
// skipped sources (errors with EVOBGP_CDN_PARTIAL_OK) keep their prior rows.
|
||||
func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) error {
|
||||
if st == nil || mod == nil {
|
||||
return nil
|
||||
}
|
||||
unlock := st.LockModuleSnapshot(tenantID, mod.ID)
|
||||
defer unlock()
|
||||
var base []store.PrefixRow
|
||||
if len(priorSnapshot) > 0 {
|
||||
base = mergeSnapshotDropCDNSources(priorSnapshot)
|
||||
base = priorSnapshot
|
||||
} else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
||||
base = mergeSnapshotDropCDNSources(snap.Prefixes)
|
||||
base = snap.Prefixes
|
||||
}
|
||||
merged := append(base, cdnRows...)
|
||||
kept := mergeSnapshotKeepSkippedCDN(base, fetchedSourceIDs, allSourceIDs)
|
||||
merged := append(kept, cdnRows...)
|
||||
return persistModuleSnapshot(st, tenantID, mod, merged)
|
||||
}
|
||||
|
||||
@@ -93,6 +110,8 @@ func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *st
|
||||
if st == nil || mod == nil {
|
||||
return nil
|
||||
}
|
||||
unlock := st.LockModuleSnapshot(tenantID, mod.ID)
|
||||
defer unlock()
|
||||
sourceKey := cdnSourceKey(sourceID)
|
||||
var base []store.PrefixRow
|
||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
||||
@@ -114,81 +133,6 @@ func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
|
||||
u := strings.TrimSpace(src.URL)
|
||||
if u == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if _, err := ValidateCDNURL(u); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ResolveCDNURLHost(ctx, u); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sourceKey := cdnSourceKey(src.ID)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
resp, err := upstreamHTTPDo(ctx, hc, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
if cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey); len(cached) > 0 {
|
||||
_ = resp.Body.Close()
|
||||
return cached, nil
|
||||
}
|
||||
// ETag is known but local snapshot is empty — force a full download.
|
||||
_ = resp.Body.Close()
|
||||
req2, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err = upstreamHTTPDo(ctx, hc, req2)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
_ = resp.Body.Close()
|
||||
return nil, fmt.Errorf("cdn url %s: 304 without cached prefixes", u)
|
||||
}
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prefixStrs, err := parseCDNBody(string(body), src)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cdn parse %s: %w", u, err)
|
||||
}
|
||||
etag := strings.TrimSpace(resp.Header.Get("ETag"))
|
||||
patch := &store.CDNSourcePatch{}
|
||||
if etag != "" && etag != strings.TrimSpace(src.Etag) {
|
||||
e := etag
|
||||
patch.Etag = &e
|
||||
}
|
||||
refreshedAt := now
|
||||
patch.LastRefreshedAt = &refreshedAt
|
||||
_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
|
||||
|
||||
rows := cdnRowsFromParsed(mod, src, prefixStrs)
|
||||
if err := mergeCDNSourceIntoModuleSnapshot(st, tenantID, mod, src.ID, rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once).
|
||||
func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
|
||||
u := strings.TrimSpace(src.URL)
|
||||
|
||||
@@ -7,7 +7,9 @@ import (
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -91,6 +93,9 @@ func ResolveCDNURLHost(ctx context.Context, raw string) error {
|
||||
if isBlockedCDNHostname(host) {
|
||||
return fmt.Errorf("pipeline: cdn url blocked host")
|
||||
}
|
||||
if ok := cdnDNSVerifyCache.hit(host); ok {
|
||||
return nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
@@ -112,5 +117,64 @@ func ResolveCDNURLHost(ctx context.Context, raw string) error {
|
||||
return fmt.Errorf("pipeline: cdn url resolves to blocked address")
|
||||
}
|
||||
}
|
||||
cdnDNSVerifyCache.store(host)
|
||||
return nil
|
||||
}
|
||||
|
||||
// cdnDNSVerifyTTL bounds how long a successful SSRF check is trusted for one hostname.
|
||||
// Failures are never cached: a transient DNS outage must not open an unsafe window,
|
||||
// and a blocked host is rejected before this cache anyway.
|
||||
func cdnDNSVerifyTTL() time.Duration {
|
||||
sec := 300
|
||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_CDN_DNS_CACHE_TTL_SEC")); s != "" {
|
||||
if v, err := strconv.Atoi(s); err == nil && v > 0 {
|
||||
sec = v
|
||||
}
|
||||
}
|
||||
return time.Duration(sec) * time.Second
|
||||
}
|
||||
|
||||
type dnsVerifyCache struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]time.Time
|
||||
}
|
||||
|
||||
var cdnDNSVerifyCache = &dnsVerifyCache{seen: make(map[string]time.Time)}
|
||||
|
||||
func (c *dnsVerifyCache) hit(host string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
at, ok := c.seen[host]
|
||||
return ok && time.Since(at) < cdnDNSVerifyTTL()
|
||||
}
|
||||
|
||||
func (c *dnsVerifyCache) store(host string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.seen == nil {
|
||||
c.seen = make(map[string]time.Time)
|
||||
}
|
||||
c.seen[host] = time.Now()
|
||||
if len(c.seen) > 4096 {
|
||||
// Size cap for long-running workers: drop expired entries, then the oldest if needed.
|
||||
now := time.Now()
|
||||
for h, at := range c.seen {
|
||||
if now.Sub(at) >= cdnDNSVerifyTTL() {
|
||||
delete(c.seen, h)
|
||||
}
|
||||
}
|
||||
if len(c.seen) > 4096 {
|
||||
var oldestK string
|
||||
var oldestT time.Time
|
||||
first := true
|
||||
for h, at := range c.seen {
|
||||
if first || at.Before(oldestT) {
|
||||
oldestK, oldestT, first = h, at, false
|
||||
}
|
||||
}
|
||||
if oldestK != "" {
|
||||
delete(c.seen, oldestK)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,14 @@ func prefixRowsForSource(rows []store.PrefixRow, sourceKey string) []store.Prefi
|
||||
return out
|
||||
}
|
||||
|
||||
func prefixCommunityKey(row store.PrefixRow) string {
|
||||
comm := ""
|
||||
if row.CommunityID != nil {
|
||||
comm = strings.TrimSpace(*row.CommunityID)
|
||||
}
|
||||
return row.Prefix + "\x00" + comm
|
||||
}
|
||||
|
||||
func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, list []*store.ASEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
||||
moduleID := mod.ID
|
||||
legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0"
|
||||
@@ -147,7 +155,7 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client,
|
||||
})
|
||||
}
|
||||
for _, row := range r.rows {
|
||||
k := row.Prefix
|
||||
k := prefixCommunityKey(row)
|
||||
if _, ok := seenPfx[k]; ok {
|
||||
continue
|
||||
}
|
||||
@@ -218,9 +226,14 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var out []store.PrefixRow
|
||||
var fetchedRows []store.PrefixRow
|
||||
var skipped int
|
||||
for _, r := range results {
|
||||
var fetchedIDs []string
|
||||
allIDs := make([]string, 0, len(valid))
|
||||
for _, src := range valid {
|
||||
allIDs = append(allIDs, src.ID)
|
||||
}
|
||||
for i, r := range results {
|
||||
if r.err != nil {
|
||||
if cdnPartialOK() {
|
||||
logging.Default().Info(fmt.Sprintf("pipeline: CDN partial skip source error: %v", r.err))
|
||||
@@ -229,20 +242,35 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
|
||||
}
|
||||
return nil, r.err
|
||||
}
|
||||
out = append(out, r.rows...)
|
||||
fetchedIDs = append(fetchedIDs, valid[i].ID)
|
||||
fetchedRows = append(fetchedRows, r.rows...)
|
||||
}
|
||||
if skipped > 0 && len(out) == 0 && len(valid) > 0 {
|
||||
if skipped > 0 && len(fetchedRows) == 0 && len(valid) > 0 {
|
||||
return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid))
|
||||
}
|
||||
out := fetchedRows
|
||||
if skipped > 0 {
|
||||
base := priorSnapshot
|
||||
if len(base) == 0 {
|
||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
|
||||
base = snap.Prefixes
|
||||
}
|
||||
}
|
||||
for _, row := range mergeSnapshotKeepSkippedCDN(base, fetchedIDs, allIDs) {
|
||||
if strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") {
|
||||
out = append(out, row)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(valid) > 0 {
|
||||
if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, out); err != nil {
|
||||
if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, fetchedRows, fetchedIDs, allIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
||||
func collectDomainPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
||||
var validDom []*store.DomainEntry
|
||||
for _, e := range entries {
|
||||
if e != nil {
|
||||
@@ -269,7 +297,7 @@ func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Mo
|
||||
c := *mod.DefaultCommunityID
|
||||
comm = &c
|
||||
}
|
||||
addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, entry.FQDN)
|
||||
addrs, err := resolveDomainIPsCached(ctx, st, hc, profiles, policy, entry.FQDN)
|
||||
if err != nil {
|
||||
if staleOnUpstreamError() {
|
||||
if cached, ok := staleDomainPrefixes(priorSnapshot, entry.FQDN); ok {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
@@ -108,8 +109,7 @@ func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, prof
|
||||
defer cancel()
|
||||
|
||||
baseURL := strings.TrimSpace(profile.URL)
|
||||
v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA)
|
||||
v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA)
|
||||
v4, v6, err4, err6 := resolveDOHMessagePair(dctx, hc, baseURL, host)
|
||||
if err4 != nil {
|
||||
v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
|
||||
}
|
||||
@@ -122,6 +122,23 @@ func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, prof
|
||||
return uniqAddrs(append(v4, v6...)), nil
|
||||
}
|
||||
|
||||
// resolveDOHMessagePair issues RFC8484 dns-message A and AAAA queries concurrently
|
||||
// and waits for both (fallbacks are handled by the caller).
|
||||
func resolveDOHMessagePair(ctx context.Context, hc *http.Client, baseURL, host string) (v4, v6 []netip.Addr, err4, err6 error) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
v4, err4 = resolveDomainWithDOHMessage(ctx, hc, baseURL, host, dns.TypeA)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
v6, err6 = resolveDomainWithDOHMessage(ctx, hc, baseURL, host, dns.TypeAAAA)
|
||||
}()
|
||||
wg.Wait()
|
||||
return v4, v6, err4, err6
|
||||
}
|
||||
|
||||
func dohProfileTimeout(profile *store.DohProfile) time.Duration {
|
||||
timeout := 10 * time.Second
|
||||
if profile != nil && profile.TimeoutMs != nil && *profile.TimeoutMs > 0 {
|
||||
|
||||
@@ -2,147 +2,97 @@ package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
func TestResolveDomainIPsWithPolicy_Union(t *testing.T) {
|
||||
srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.1"}]}`))
|
||||
}))
|
||||
defer srvRU.Close()
|
||||
srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.1"}]}`))
|
||||
}))
|
||||
defer srvEU.Close()
|
||||
// TestResolveDomainIPs_ParallelAAndAAAA verifies that A and AAAA queries are issued
|
||||
// concurrently: with a 250ms upstream latency the combined resolve must stay near
|
||||
// one round-trip instead of two.
|
||||
func TestResolveDomainIPs_ParallelAAndAAAA(t *testing.T) {
|
||||
const delay = 250 * time.Millisecond
|
||||
var inflight, maxInflight atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
cur := inflight.Add(1)
|
||||
for {
|
||||
old := maxInflight.Load()
|
||||
if cur <= old || maxInflight.CompareAndSwap(old, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
defer inflight.Add(-1)
|
||||
time.Sleep(delay)
|
||||
|
||||
profiles := []*store.DohProfile{
|
||||
{URL: srvRU.URL},
|
||||
{URL: srvEU.URL},
|
||||
}
|
||||
ips, err := resolveDomainIPsWithPolicy(context.Background(), srvRU.Client(), profiles, store.DohPolicyUnion, "example.com")
|
||||
if wire := r.URL.Query().Get("dns"); wire != "" {
|
||||
// RFC8484 dns-message: decode the query and answer on the wire.
|
||||
raw, err := base64.RawURLEncoding.DecodeString(wire)
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
msg := new(dns.Msg)
|
||||
if err := msg.Unpack(raw); err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
resp := new(dns.Msg)
|
||||
resp.SetReply(msg)
|
||||
switch msg.Question[0].Qtype {
|
||||
case dns.TypeA:
|
||||
resp.Answer = append(resp.Answer, &dns.A{
|
||||
Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
|
||||
A: []byte{203, 0, 113, 10},
|
||||
})
|
||||
case dns.TypeAAAA:
|
||||
resp.Answer = append(resp.Answer, &dns.AAAA{
|
||||
Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60},
|
||||
AAAA: []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x10},
|
||||
})
|
||||
}
|
||||
out, err := resp.Pack()
|
||||
if err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/dns-message")
|
||||
_, _ = w.Write(out)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/dns-json")
|
||||
switch r.URL.Query().Get("type") {
|
||||
case "A":
|
||||
_, _ = w.Write([]byte(`{"Status":0,"Answer":[{"type":1,"data":"203.0.113.10"}]}`))
|
||||
default:
|
||||
_, _ = w.Write([]byte(`{"Status":0,"Answer":[{"type":28,"data":"2001:db8::10"}]}`))
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
prof := &store.DohProfile{URL: srv.URL, TimeoutMs: ptrInt(5000)}
|
||||
ctx := context.Background()
|
||||
start := time.Now()
|
||||
ips, err := resolveDomainIPsNoSystemFallback(ctx, srv.Client(), prof, "example.test")
|
||||
elapsed := time.Since(start)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
t.Fatalf("resolve failed: %v", err)
|
||||
}
|
||||
if len(ips) != 2 {
|
||||
t.Fatalf("want 2 ips, got %v", ips)
|
||||
t.Fatalf("expected 2 addrs, got %+v", ips)
|
||||
}
|
||||
seen := map[string]bool{ips[0].String(): true, ips[1].String(): true}
|
||||
if !seen["198.51.100.1"] || !seen["203.0.113.1"] {
|
||||
t.Fatalf("unexpected ips: %v", ips)
|
||||
if maxInflight.Load() < 2 {
|
||||
t.Fatalf("expected concurrent A/AAAA queries, max inflight=%d", maxInflight.Load())
|
||||
}
|
||||
if elapsed >= 2*delay {
|
||||
t.Fatalf("resolve took %v; expected one round-trip (<2*%v)", elapsed, delay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDomainIPsWithPolicy_Failover(t *testing.T) {
|
||||
var calls int
|
||||
srvBad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
http.Error(w, "fail", http.StatusBadGateway)
|
||||
}))
|
||||
defer srvBad.Close()
|
||||
srvOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.5"}]}`))
|
||||
}))
|
||||
defer srvOK.Close()
|
||||
|
||||
profiles := []*store.DohProfile{
|
||||
{URL: srvBad.URL},
|
||||
{URL: srvOK.URL},
|
||||
}
|
||||
ips, err := resolveDomainIPsWithPolicy(context.Background(), srvBad.Client(), profiles, store.DohPolicyFailover, "example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ips) != 1 || ips[0].String() != "198.51.100.5" {
|
||||
t.Fatalf("unexpected ips: %v", ips)
|
||||
}
|
||||
if calls < 2 {
|
||||
t.Fatalf("want at least 2 resolver calls, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDomainIPsWithPolicy_PrimaryOnly(t *testing.T) {
|
||||
var secondCalled bool
|
||||
srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.9"}]}`))
|
||||
}))
|
||||
defer srv1.Close()
|
||||
srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
secondCalled = true
|
||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.9"}]}`))
|
||||
}))
|
||||
defer srv2.Close()
|
||||
|
||||
profiles := []*store.DohProfile{
|
||||
{URL: srv1.URL},
|
||||
{URL: srv2.URL},
|
||||
}
|
||||
ips, err := resolveDomainIPsWithPolicy(context.Background(), srv1.Client(), profiles, store.DohPolicyPrimaryOnly, "example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ips) != 1 || ips[0].String() != "198.51.100.9" {
|
||||
t.Fatalf("unexpected ips: %v", ips)
|
||||
}
|
||||
if secondCalled {
|
||||
t.Fatal("secondary resolver must not be queried in primary_only mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectModulePrefixRows_DohUnion(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
|
||||
mod, err := m.CreateModule(tenant, &store.Module{
|
||||
Type: "DOMAINS",
|
||||
Name: "domains-union",
|
||||
Enabled: true,
|
||||
DohResolverPolicy: store.DohPolicyUnion,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.2"}]}`))
|
||||
}))
|
||||
defer srvRU.Close()
|
||||
srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.2"}]}`))
|
||||
}))
|
||||
defer srvEU.Close()
|
||||
|
||||
ru, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "ru", URL: srvRU.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eu, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "eu", URL: srvEU.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.UpdateModule(tenant, mod.ID, &store.ModulePatch{
|
||||
DohProfileIDs: &[]string{ru.ID, eu.ID},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mod, err = m.GetModule(tenant, mod.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "svc.example.com"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rows, err := collectModulePrefixRows(context.Background(), m, srvRU.Client(), tenant, mod, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("want 2 prefix rows, got %+v", rows)
|
||||
}
|
||||
}
|
||||
func ptrInt(v int) *int { return &v }
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func domainCacheTTL() time.Duration {
|
||||
sec := 300
|
||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_DOMAIN_CACHE_TTL_SEC")); s != "" {
|
||||
if v, err := strconv.Atoi(s); err == nil && v > 0 {
|
||||
sec = v
|
||||
}
|
||||
}
|
||||
return time.Duration(sec) * time.Second
|
||||
}
|
||||
|
||||
func resolveDomainIPsCached(ctx context.Context, st store.Backend, hc *http.Client, profiles []*store.DohProfile, policy, fqdn string) ([]netip.Addr, error) {
|
||||
key := strings.TrimSpace(fqdn)
|
||||
ttl := domainCacheTTL()
|
||||
if st != nil && ttl > 0 {
|
||||
if ent, ok, err := st.GetDomainResolveCache(key); err == nil && ok && ent != nil {
|
||||
if time.Since(ent.ResolvedAt) < ttl {
|
||||
if addrs := parseCachedDomainAddrs(ent.Addrs); len(addrs) > 0 {
|
||||
return addrs, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if st != nil {
|
||||
_ = st.SetDomainResolveCache(key, domainAddrsToStrings(addrs))
|
||||
}
|
||||
return addrs, nil
|
||||
}
|
||||
|
||||
func parseCachedDomainAddrs(raw []string) []netip.Addr {
|
||||
out := make([]netip.Addr, 0, len(raw))
|
||||
for _, s := range raw {
|
||||
a, err := netip.ParseAddr(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func domainAddrsToStrings(addrs []netip.Addr) []string {
|
||||
out := make([]string, 0, len(addrs))
|
||||
for _, a := range addrs {
|
||||
if a.IsValid() {
|
||||
out = append(out, a.String())
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,10 +1,6 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"evobgp/internal/store"
|
||||
@@ -12,89 +8,7 @@ import (
|
||||
|
||||
// moduleIngestInputHash fingerprints module config and child entries so snapshots invalidate on CRUD.
|
||||
func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module) (string, error) {
|
||||
if st == nil || mod == nil {
|
||||
return "", fmt.Errorf("pipeline: module hash: missing store or module")
|
||||
}
|
||||
h := sha256.New()
|
||||
_, _ = fmt.Fprintf(h, "type=%s\n", strings.TrimSpace(mod.Type))
|
||||
_, _ = fmt.Fprintf(h, "enabled=%t\n", mod.Enabled)
|
||||
if mod.DefaultCommunityID != nil {
|
||||
_, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID))
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "doh_policy=%s\n", store.NormalizeDohResolverPolicy(mod.DohResolverPolicy))
|
||||
for _, pid := range mod.EffectiveDohProfileIDs() {
|
||||
_, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid)
|
||||
if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil {
|
||||
_, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL))
|
||||
if prof.TimeoutMs != nil {
|
||||
_, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch mod.Type {
|
||||
case "IP_RANGES":
|
||||
list, err := st.ListIPRangeEntries(tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].Prefix < list[j].Prefix })
|
||||
for _, e := range list {
|
||||
comm := ""
|
||||
if e.CommunityID != nil {
|
||||
comm = *e.CommunityID
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "ip=%s|c=%s\n", e.Prefix, comm)
|
||||
}
|
||||
case "AS_PREFIXES":
|
||||
list, err := st.ListASEntries(tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
|
||||
for _, e := range list {
|
||||
comm := ""
|
||||
if e.CommunityID != nil {
|
||||
comm = *e.CommunityID
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "as=%d|c=%s\n", e.ASN, comm)
|
||||
}
|
||||
case "CDN_CIDRS":
|
||||
list, err := st.ListCDNSources(tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].ID < list[j].ID })
|
||||
for _, s := range list {
|
||||
comm := ""
|
||||
if s.CommunityID != nil {
|
||||
comm = *s.CommunityID
|
||||
}
|
||||
interval := 0
|
||||
if s.RefreshIntervalSec != nil {
|
||||
interval = *s.RefreshIntervalSec
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "cdn=%s|url=%s|kind=%s|path=%s|c=%s|etag=%s|interval=%d\n",
|
||||
s.ID, strings.TrimSpace(s.URL), s.SourceKind, strings.TrimSpace(s.PrefixPath), comm,
|
||||
strings.TrimSpace(s.Etag), interval)
|
||||
}
|
||||
case "DOMAINS":
|
||||
list, err := st.ListDomainEntries(tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].FQDN < list[j].FQDN })
|
||||
for _, e := range list {
|
||||
comm := ""
|
||||
if e.CommunityID != nil {
|
||||
comm = *e.CommunityID
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "dom=%s|c=%s\n", strings.TrimSpace(e.FQDN), comm)
|
||||
}
|
||||
default:
|
||||
_, _ = fmt.Fprintf(h, "unknown_type=%s\n", mod.Type)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
return store.ComputeModuleInputHash(st, tenantID, mod)
|
||||
}
|
||||
|
||||
func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error {
|
||||
@@ -105,18 +19,32 @@ func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows)
|
||||
if err := st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = st.SetModuleInputHash(tenantID, mod.ID, hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module) ([]store.PrefixRow, bool, error) {
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
|
||||
if err != nil || !ok || snap == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if h := strings.TrimSpace(mod.InputHash); h != "" {
|
||||
if snap.InputHash != h {
|
||||
return nil, false, nil
|
||||
}
|
||||
return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil
|
||||
}
|
||||
// Fallback for rows written before module.input_hash existed: recompute and backfill.
|
||||
hash, err := moduleIngestInputHash(st, tenantID, mod)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
|
||||
if err != nil || !ok || snap == nil || snap.InputHash != hash {
|
||||
return nil, false, err
|
||||
if snap.InputHash != hash {
|
||||
return nil, false, nil
|
||||
}
|
||||
cp := append([]store.PrefixRow(nil), snap.Prefixes...)
|
||||
return cp, true, nil
|
||||
_ = st.SetModuleInputHash(tenantID, mod.ID, hash)
|
||||
return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestModuleDueForScheduler_JitterSpreads(t *testing.T) {
|
||||
dueTicks := map[int64]int{}
|
||||
for i := 0; i < 100; i++ {
|
||||
mod := &store.Module{
|
||||
ID: fmt.Sprintf("module-%d", i),
|
||||
Enabled: true,
|
||||
Type: "AS_PREFIXES",
|
||||
RefreshIntervalSec: 3600,
|
||||
}
|
||||
for tick := int64(0); tick < 7200; tick += SchedulerTickSec {
|
||||
if ModuleDueForScheduler(mod, time.Unix(tick, 0)) {
|
||||
dueTicks[tick]++
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(dueTicks) < 10 {
|
||||
t.Fatalf("expected due events across many ticks, got %d buckets", len(dueTicks))
|
||||
}
|
||||
for tick, n := range dueTicks {
|
||||
if n == 100 {
|
||||
t.Fatalf("all 100 modules due on tick %d", tick)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectASPrefixRows_KeepsSamePrefixDifferentCommunity(t *testing.T) {
|
||||
st := store.NewMemory()
|
||||
st.SeedDemo()
|
||||
tenant, _, _, _, _ := st.DemoIDs()
|
||||
mod, err := st.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "as-dedup", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c1, c2 := "comm-a", "comm-b"
|
||||
e1, err := st.CreateASEntry(tenant, mod.ID, &store.ASEntry{ASN: 64500, CommunityID: &c1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e2, err := st.CreateASEntry(tenant, mod.ID, &store.ASEntry{ASN: 64501, CommunityID: &c2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SetASNPrefixCache(64500, "a", []string{"192.0.2.0/24"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SetASNPrefixCache(64501, "b", []string{"192.0.2.0/24"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows, err := collectASPrefixRows(context.Background(), st, http.DefaultClient, tenant, mod, []*store.ASEntry{e1, e2}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("expected 2 rows (same prefix, different community), got %d: %+v", len(rows), rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectCDNPrefixRows_PartialSkipKeepsPrior(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_PARTIAL_OK", "1")
|
||||
t.Setenv("EVOBGP_STALE_ON_UPSTREAM_ERROR", "0")
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
|
||||
good := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("198.51.100.0/24\n"))
|
||||
}))
|
||||
defer good.Close()
|
||||
bad := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "down", http.StatusServiceUnavailable)
|
||||
}))
|
||||
defer bad.Close()
|
||||
|
||||
st := store.NewMemory()
|
||||
st.SeedDemo()
|
||||
tenant, _, _, _, _ := st.DemoIDs()
|
||||
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-partial", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prior := []store.PrefixRow{
|
||||
{Prefix: "203.0.113.0/24", Source: "cdn:bad"},
|
||||
{Prefix: "1.2.3.0/24", Source: "cdn:good"},
|
||||
}
|
||||
sources := []*store.CDNSource{
|
||||
{ID: "good", URL: good.URL, SourceKind: "plain"},
|
||||
{ID: "bad", URL: bad.URL, SourceKind: "plain"},
|
||||
}
|
||||
rows, err := collectCDNPrefixRows(context.Background(), st, good.Client(), tenant, mod, sources, prior)
|
||||
if err != nil {
|
||||
t.Fatalf("partial skip should succeed: %v", err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, r := range rows {
|
||||
got[r.Source] = r.Prefix
|
||||
}
|
||||
if got["cdn:bad"] != "203.0.113.0/24" {
|
||||
t.Fatalf("skipped source lost prior row: %+v", rows)
|
||||
}
|
||||
if got["cdn:good"] != "198.51.100.0/24" {
|
||||
t.Fatalf("fetched source missing new row: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeCDNSource_ParallelNoLostUpdate(t *testing.T) {
|
||||
st := store.NewMemory()
|
||||
st.SeedDemo()
|
||||
tenant, _, _, _, _ := st.DemoIDs()
|
||||
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-lock", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s1", []store.PrefixRow{
|
||||
{Prefix: "1.0.0.0/24", Source: "cdn:s1"},
|
||||
})
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s2", []store.PrefixRow{
|
||||
{Prefix: "2.0.0.0/24", Source: "cdn:s2"},
|
||||
})
|
||||
}()
|
||||
wg.Wait()
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(tenant, mod.ID)
|
||||
if err != nil || !ok || snap == nil {
|
||||
t.Fatalf("snapshot missing: ok=%v err=%v", ok, err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, r := range snap.Prefixes {
|
||||
got[r.Prefix] = true
|
||||
}
|
||||
if !got["1.0.0.0/24"] || !got["2.0.0.0/24"] {
|
||||
t.Fatalf("lost parallel merge update: %+v", snap.Prefixes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDomainIPsCached_UsesTTL(t *testing.T) {
|
||||
t.Setenv("EVOBGP_DOMAIN_CACHE_TTL_SEC", "300")
|
||||
st := store.NewMemory()
|
||||
if err := st.SetDomainResolveCache("cached.test", []string{"192.0.2.9"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addrs, err := resolveDomainIPsCached(context.Background(), st, nil, nil, "", "cached.test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(addrs) != 1 || addrs[0].String() != "192.0.2.9" {
|
||||
t.Fatalf("expected cached addr, got %v", addrs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInputHashInvalidatedOnEntryCRUD(t *testing.T) {
|
||||
st := store.NewMemory()
|
||||
st.SeedDemo()
|
||||
tenant, _, _, _, _ := st.DemoIDs()
|
||||
mod, err := st.CreateModule(tenant, &store.Module{Type: "IP_RANGES", Name: "ip-hash", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := st.SetModuleInputHash(tenant, mod.ID, "pre"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.CreateIPRangeEntry(tenant, mod.ID, &store.IPRangeEntry{Prefix: "10.0.0.0/8"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := st.GetModule(tenant, mod.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.InputHash != "" {
|
||||
t.Fatalf("expected hash cleared after CRUD, got %q", got.InputHash)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -31,6 +30,7 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
|
||||
return err
|
||||
}
|
||||
var tasks []prefetchTask
|
||||
now := time.Now().UTC()
|
||||
for _, tid := range tenants {
|
||||
for _, mod := range st.ListModules(tid) {
|
||||
if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
|
||||
@@ -41,9 +41,15 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
|
||||
continue
|
||||
}
|
||||
for _, src := range sources {
|
||||
if src != nil && strings.TrimSpace(src.URL) != "" {
|
||||
tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src})
|
||||
if src == nil || strings.TrimSpace(src.URL) == "" {
|
||||
continue
|
||||
}
|
||||
// Respect per-source refresh intervals: conditional GET only for due sources.
|
||||
// The ETag probe still lets 304s skip body downloads for the rest.
|
||||
if shouldSkipCDNSourceFetch(src, now) {
|
||||
continue
|
||||
}
|
||||
tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,53 +74,13 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
|
||||
func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) {
|
||||
now := time.Now().UTC()
|
||||
tid, mod, src := t.tenantID, t.mod, t.src
|
||||
u := strings.TrimSpace(src.URL)
|
||||
if _, err := ValidateCDNURL(u); err != nil {
|
||||
return
|
||||
}
|
||||
if err := ResolveCDNURLHost(ctx, u); err != nil {
|
||||
return
|
||||
}
|
||||
omod, err := st.GetModule(tid, mod.ID)
|
||||
if err != nil || omod == nil {
|
||||
return
|
||||
}
|
||||
rows, err := fetchCDNSourceRows(ctx, st, hc, tid, mod.ID, omod, src, nil, now)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
resp, err := upstreamHTTPDo(ctx, hc, req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
_ = resp.Body.Close()
|
||||
return
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
prefixStrs, err := parseCDNBody(string(body), src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
|
||||
patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
|
||||
if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
|
||||
e := newEtag
|
||||
patch.Etag = &e
|
||||
}
|
||||
_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
|
||||
rows := cdnRowsFromParsed(omod, src, prefixStrs)
|
||||
_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
@@ -190,7 +189,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return collectDomainPrefixRows(ctx, hc, mod, profiles, policy, entries, priorSnapshot)
|
||||
return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, priorSnapshot)
|
||||
default:
|
||||
return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type)
|
||||
}
|
||||
@@ -230,9 +229,8 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr
|
||||
defer cancel()
|
||||
|
||||
baseURL := strings.TrimSpace(profile.URL)
|
||||
// Prefer RFC8484 dns-message transport. Some providers don't support dns-json.
|
||||
v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA)
|
||||
v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA)
|
||||
// A and AAAA queries run concurrently: per-domain latency drops from ~2×RTT to ~1×RTT.
|
||||
v4, v6, err4, err6 := resolveDOHMessagePair(dctx, hc, baseURL, host)
|
||||
if err4 != nil {
|
||||
// Fallback to JSON mode for providers that only expose dns-json.
|
||||
v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
|
||||
@@ -496,168 +494,11 @@ func prefixRowCommunity(r store.PrefixRow) string {
|
||||
}
|
||||
|
||||
func aggregateIPv4Group(rows []store.PrefixRow) []store.PrefixRow {
|
||||
return aggregateCIDRGroup(rows, mergeSiblingPrefixesIPv4)
|
||||
return collapsePrefixGroup(rows, true)
|
||||
}
|
||||
|
||||
func aggregateIPv6Group(rows []store.PrefixRow) []store.PrefixRow {
|
||||
return aggregateCIDRGroup(rows, mergeSiblingPrefixesIPv6)
|
||||
}
|
||||
|
||||
func aggregateCIDRGroup(rows []store.PrefixRow, mergeFn func(map[string]store.PrefixRow) bool) []store.PrefixRow {
|
||||
if len(rows) <= 1 {
|
||||
return rows
|
||||
}
|
||||
set := make(map[string]store.PrefixRow, len(rows))
|
||||
for _, row := range rows {
|
||||
set[row.Prefix] = row
|
||||
}
|
||||
pruneCoveredPrefixes(set)
|
||||
for {
|
||||
if !mergeFn(set) {
|
||||
break
|
||||
}
|
||||
pruneCoveredPrefixes(set)
|
||||
}
|
||||
out := make([]store.PrefixRow, 0, len(set))
|
||||
for _, row := range set {
|
||||
out = append(out, row)
|
||||
}
|
||||
sortPrefixRows(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func pruneCoveredPrefixes(set map[string]store.PrefixRow) {
|
||||
type item struct {
|
||||
key string
|
||||
pfx netip.Prefix
|
||||
bits int
|
||||
}
|
||||
items := make([]item, 0, len(set))
|
||||
for k := range set {
|
||||
p, err := netip.ParsePrefix(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
items = append(items, item{key: k, pfx: p, bits: p.Bits()})
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if items[i].bits != items[j].bits {
|
||||
return items[i].bits < items[j].bits
|
||||
}
|
||||
return items[i].key < items[j].key
|
||||
})
|
||||
for i := 0; i < len(items); i++ {
|
||||
for j := i + 1; j < len(items); j++ {
|
||||
if items[j].bits <= items[i].bits {
|
||||
continue
|
||||
}
|
||||
if items[i].pfx.Contains(items[j].pfx.Addr()) {
|
||||
delete(set, items[j].key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mergeSiblingPrefixesIPv4(set map[string]store.PrefixRow) bool {
|
||||
merged := false
|
||||
seen := make(map[string]struct{}, len(set))
|
||||
for key, row := range set {
|
||||
if _, done := seen[key]; done {
|
||||
continue
|
||||
}
|
||||
pfx, err := netip.ParsePrefix(key)
|
||||
if err != nil || !pfx.Addr().Is4() {
|
||||
continue
|
||||
}
|
||||
bits := pfx.Bits()
|
||||
if bits <= 8 {
|
||||
continue
|
||||
}
|
||||
netNum := ipv4PrefixNetwork(pfx)
|
||||
blockSize := uint32(1) << (32 - bits)
|
||||
siblingNet := netNum ^ blockSize
|
||||
siblingPfx := netip.PrefixFrom(u32ToIPv4(siblingNet), bits).Masked().String()
|
||||
_, ok := set[siblingPfx]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
parentBits := bits - 1
|
||||
parentBlock := uint32(1) << (32 - parentBits)
|
||||
parentNet := netNum & ^(parentBlock - 1)
|
||||
parentPfx := netip.PrefixFrom(u32ToIPv4(parentNet), parentBits).Masked().String()
|
||||
delete(set, key)
|
||||
delete(set, siblingPfx)
|
||||
parentRow := row
|
||||
parentRow.Prefix = parentPfx
|
||||
set[parentPfx] = parentRow
|
||||
seen[key] = struct{}{}
|
||||
seen[siblingPfx] = struct{}{}
|
||||
merged = true
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func ipv4PrefixNetwork(p netip.Prefix) uint32 {
|
||||
a := p.Masked().Addr().As4()
|
||||
return uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3])
|
||||
}
|
||||
|
||||
func u32ToIPv4(v uint32) netip.Addr {
|
||||
return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
|
||||
}
|
||||
|
||||
func mergeSiblingPrefixesIPv6(set map[string]store.PrefixRow) bool {
|
||||
merged := false
|
||||
seen := make(map[string]struct{}, len(set))
|
||||
for key, row := range set {
|
||||
if _, done := seen[key]; done {
|
||||
continue
|
||||
}
|
||||
pfx, err := netip.ParsePrefix(key)
|
||||
if err != nil || !pfx.Addr().Is6() {
|
||||
continue
|
||||
}
|
||||
bits := pfx.Bits()
|
||||
if bits <= 16 {
|
||||
continue
|
||||
}
|
||||
netNum := ipv6PrefixNetwork(pfx)
|
||||
blockSize := new(big.Int).Lsh(big.NewInt(1), uint(128-bits))
|
||||
siblingNet := new(big.Int).Xor(netNum, blockSize)
|
||||
siblingPfx := ipv6PrefixFromBigInt(siblingNet, bits).String()
|
||||
if _, ok := set[siblingPfx]; !ok {
|
||||
continue
|
||||
}
|
||||
parentBits := bits - 1
|
||||
parentBlock := new(big.Int).Lsh(big.NewInt(1), uint(128-parentBits))
|
||||
mask := new(big.Int).Sub(parentBlock, big.NewInt(1))
|
||||
mask.Not(mask)
|
||||
parentNet := new(big.Int).And(netNum, mask)
|
||||
parentPfx := ipv6PrefixFromBigInt(parentNet, parentBits).String()
|
||||
delete(set, key)
|
||||
delete(set, siblingPfx)
|
||||
parentRow := row
|
||||
parentRow.Prefix = parentPfx
|
||||
set[parentPfx] = parentRow
|
||||
seen[key] = struct{}{}
|
||||
seen[siblingPfx] = struct{}{}
|
||||
merged = true
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
func ipv6PrefixNetwork(p netip.Prefix) *big.Int {
|
||||
a := p.Masked().Addr().As16()
|
||||
n := new(big.Int)
|
||||
n.SetBytes(a[:])
|
||||
return n
|
||||
}
|
||||
|
||||
func ipv6PrefixFromBigInt(n *big.Int, bits int) netip.Prefix {
|
||||
b := n.Bytes()
|
||||
var a [16]byte
|
||||
copy(a[16-len(b):], b)
|
||||
return netip.PrefixFrom(netip.AddrFrom16(a), bits).Masked()
|
||||
return collapsePrefixGroup(rows, false)
|
||||
}
|
||||
|
||||
func parentRevision(st store.Backend, tenantID, moduleID string) *string {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"hash/fnv"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
@@ -10,6 +11,8 @@ import (
|
||||
const SchedulerTickSec = 30
|
||||
|
||||
// ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick.
|
||||
// A stable per-module offset (fnv32 of ID) spreads bucket boundaries so modules with the same interval
|
||||
// do not all become due on the same tick (thundering herd).
|
||||
func ModuleDueForScheduler(mod *store.Module, now time.Time) bool {
|
||||
if mod == nil || !mod.Enabled || mod.Type == "IP_RANGES" || mod.RefreshIntervalSec <= 0 {
|
||||
return false
|
||||
@@ -18,7 +21,21 @@ func ModuleDueForScheduler(mod *store.Module, now time.Time) bool {
|
||||
if win < 60 {
|
||||
win = 60
|
||||
}
|
||||
cur := now.Unix() / win
|
||||
prev := (now.Unix() - SchedulerTickSec) / win
|
||||
offset := moduleSchedulerOffset(mod.ID, win)
|
||||
cur := (now.Unix() - offset) / win
|
||||
prev := (now.Unix() - offset - SchedulerTickSec) / win
|
||||
return cur != prev
|
||||
}
|
||||
|
||||
func moduleSchedulerOffset(moduleID string, win int64) int64 {
|
||||
if win <= 0 {
|
||||
return 0
|
||||
}
|
||||
return int64(fnv32a(moduleID) % uint32(win))
|
||||
}
|
||||
|
||||
func fnv32a(s string) uint32 {
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(s))
|
||||
return h.Sum32()
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
@@ -11,13 +12,26 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// asnCacheRowExistsOnce caches the schema check for the process lifetime (see moduleSnapshotRowTableExists).
|
||||
var (
|
||||
asnCacheRowExistsOnce sync.Once
|
||||
asnCacheRowExistsCached bool
|
||||
)
|
||||
|
||||
func asnCacheRowTableExists(ctx context.Context, q queryRower) bool {
|
||||
var n int
|
||||
err := q.QueryRow(ctx, `
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row'
|
||||
LIMIT 1`).Scan(&n)
|
||||
return err == nil
|
||||
asnCacheRowExistsOnce.Do(func() {
|
||||
var n int
|
||||
err := q.QueryRow(ctx, `
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row'
|
||||
LIMIT 1`).Scan(&n)
|
||||
if err != nil {
|
||||
asnCacheRowExistsOnce = sync.Once{}
|
||||
return
|
||||
}
|
||||
asnCacheRowExistsCached = true
|
||||
})
|
||||
return asnCacheRowExistsCached
|
||||
}
|
||||
|
||||
func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, bool, error) {
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func (p *Postgres) GetDomainResolveCache(fqdn string) (*store.DomainResolveCacheEntry, bool, error) {
|
||||
key := strings.ToLower(strings.TrimSpace(fqdn))
|
||||
if key == "" {
|
||||
return nil, false, nil
|
||||
}
|
||||
ctx := context.Background()
|
||||
var raw []byte
|
||||
var resolvedAt time.Time
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
SELECT addrs_json, resolved_at FROM domain_resolve_cache WHERE fqdn = $1`, key).
|
||||
Scan(&raw, &resolvedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
var addrs []string
|
||||
if len(raw) > 0 {
|
||||
_ = json.Unmarshal(raw, &addrs)
|
||||
}
|
||||
return &store.DomainResolveCacheEntry{
|
||||
FQDN: key,
|
||||
Addrs: addrs,
|
||||
ResolvedAt: resolvedAt.UTC(),
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) SetDomainResolveCache(fqdn string, addrs []string) error {
|
||||
key := strings.ToLower(strings.TrimSpace(fqdn))
|
||||
if key == "" {
|
||||
return store.ErrInvalidInput
|
||||
}
|
||||
if addrs == nil {
|
||||
addrs = []string{}
|
||||
}
|
||||
raw, err := json.Marshal(addrs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := context.Background()
|
||||
_, err = p.pool.Exec(ctx, `
|
||||
INSERT INTO domain_resolve_cache (fqdn, addrs_json, resolved_at)
|
||||
VALUES ($1, $2::jsonb, now())
|
||||
ON CONFLICT (fqdn) DO UPDATE SET
|
||||
addrs_json = EXCLUDED.addrs_json,
|
||||
resolved_at = EXCLUDED.resolved_at`,
|
||||
key, string(raw))
|
||||
return err
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
@@ -12,13 +13,29 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// moduleSnapshotRowExistsOnce caches the schema check for the process lifetime.
|
||||
// The table is created by migrations and never disappears at runtime; DB errors are
|
||||
// not cached so a transient outage falls back to the JSON path only once.
|
||||
var moduleSnapshotRowExistsOnce sync.Once
|
||||
|
||||
var moduleSnapshotRowExistsCached bool
|
||||
|
||||
func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool {
|
||||
var n int
|
||||
err := q.QueryRow(ctx, `
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row'
|
||||
LIMIT 1`).Scan(&n)
|
||||
return err == nil
|
||||
moduleSnapshotRowExistsOnce.Do(func() {
|
||||
var n int
|
||||
err := q.QueryRow(ctx, `
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row'
|
||||
LIMIT 1`).Scan(&n)
|
||||
// A query error means the backend is unreachable or information_schema is hidden;
|
||||
// treat as "absent" so callers fall back to prefixes_json, but retry next call.
|
||||
if err != nil {
|
||||
moduleSnapshotRowExistsOnce = sync.Once{}
|
||||
return
|
||||
}
|
||||
moduleSnapshotRowExistsCached = true
|
||||
})
|
||||
return moduleSnapshotRowExistsCached
|
||||
}
|
||||
|
||||
func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
||||
@@ -97,19 +114,25 @@ func (p *Postgres) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string,
|
||||
WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, pr := range prefixes {
|
||||
var comm any
|
||||
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||
comm = strings.TrimSpace(*pr.CommunityID)
|
||||
}
|
||||
src := pr.Source
|
||||
if strings.TrimSpace(src) == "" {
|
||||
src = "render"
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO module_prefix_snapshot_row (tenant_id, module_id, ord, prefix, community_id, source)
|
||||
VALUES ($1::uuid, $2::uuid, $3, $4, $5::uuid, $6)`,
|
||||
tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src); err != nil {
|
||||
if len(prefixes) > 0 {
|
||||
_, err = tx.CopyFrom(
|
||||
ctx,
|
||||
pgx.Identifier{"module_prefix_snapshot_row"},
|
||||
[]string{"tenant_id", "module_id", "ord", "prefix", "community_id", "source"},
|
||||
pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) {
|
||||
pr := prefixes[i]
|
||||
var comm any
|
||||
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||
comm = strings.TrimSpace(*pr.CommunityID)
|
||||
}
|
||||
src := pr.Source
|
||||
if strings.TrimSpace(src) == "" {
|
||||
src = "render"
|
||||
}
|
||||
return []any{tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -135,3 +158,35 @@ func (p *Postgres) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
|
||||
tenantID, moduleID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *Postgres) SetModuleInputHash(tenantID, moduleID, hash string) error {
|
||||
ctx := context.Background()
|
||||
tag, err := p.pool.Exec(ctx, `
|
||||
UPDATE module SET input_hash = $3
|
||||
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`,
|
||||
moduleID, tenantID, hash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return store.ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) LockModuleSnapshot(tenantID, moduleID string) func() {
|
||||
ctx := context.Background()
|
||||
conn, err := p.pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return func() {}
|
||||
}
|
||||
key := strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID)
|
||||
if _, err := conn.Exec(ctx, `SELECT pg_advisory_lock(hashtext($1))`, key); err != nil {
|
||||
conn.Release()
|
||||
return func() {}
|
||||
}
|
||||
return func() {
|
||||
_, _ = conn.Exec(ctx, `SELECT pg_advisory_unlock(hashtext($1))`, key)
|
||||
conn.Release()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +119,8 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module {
|
||||
ctx := context.Background()
|
||||
rows, err := p.pool.Query(ctx, `
|
||||
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id
|
||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
|
||||
COALESCE(input_hash, '')
|
||||
FROM module WHERE tenant_id = $1 AND deleted_at IS NULL ORDER BY priority, name`, tenantID)
|
||||
if err != nil {
|
||||
return nil
|
||||
@@ -134,7 +135,7 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module {
|
||||
var refresh *int32
|
||||
var last *time.Time
|
||||
var createdBy *string
|
||||
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy); err != nil {
|
||||
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash); err != nil {
|
||||
continue
|
||||
}
|
||||
m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
|
||||
@@ -179,7 +180,8 @@ func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store
|
||||
ctx := context.Background()
|
||||
rows, err := p.pool.Query(ctx, `
|
||||
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id
|
||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
|
||||
COALESCE(input_hash, '')
|
||||
FROM module WHERE tenant_id = $1 AND deleted_at IS NULL
|
||||
ORDER BY priority, name
|
||||
LIMIT $2 OFFSET $3`, tenantID, limit+1, off)
|
||||
@@ -196,7 +198,7 @@ func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store
|
||||
var refresh *int32
|
||||
var last *time.Time
|
||||
var createdBy *string
|
||||
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy); err != nil {
|
||||
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash); err != nil {
|
||||
continue
|
||||
}
|
||||
m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
|
||||
@@ -249,9 +251,10 @@ func (p *Postgres) GetModule(tenantID, moduleID string) (*store.Module, error) {
|
||||
var createdBy *string
|
||||
err := p.pool.QueryRow(ctx, `
|
||||
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id
|
||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
|
||||
COALESCE(input_hash, '')
|
||||
FROM module WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, moduleID, tenantID).Scan(
|
||||
&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy)
|
||||
&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, store.ErrNotFound
|
||||
@@ -325,9 +328,19 @@ func (p *Postgres) CreateModule(tenantID string, in *store.Module) (*store.Modul
|
||||
if err := p.setModuleDohProfiles(ctx, id, in.DohProfileIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store.TouchModuleInputHash(p, tenantID, id)
|
||||
return p.GetModule(tenantID, id)
|
||||
}
|
||||
|
||||
func (p *Postgres) invalidateModuleInputHash(tenantID, moduleID string) {
|
||||
_ = p.SetModuleInputHash(tenantID, moduleID, "")
|
||||
}
|
||||
|
||||
func (p *Postgres) invalidateTenantModuleHashes(tenantID string) {
|
||||
ctx := context.Background()
|
||||
_, _ = p.pool.Exec(ctx, `UPDATE module SET input_hash = '' WHERE tenant_id = $1 AND deleted_at IS NULL`, tenantID)
|
||||
}
|
||||
|
||||
func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePatch) (*store.Module, error) {
|
||||
if patch == nil {
|
||||
return nil, store.ErrInvalidInput
|
||||
@@ -399,6 +412,7 @@ func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePa
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return p.GetModule(tenantID, moduleID)
|
||||
}
|
||||
|
||||
@@ -1191,6 +1205,7 @@ func (p *Postgres) UpdateDohProfile(tenantID, id string, patch *store.DohProfile
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.invalidateTenantModuleHashes(tenantID)
|
||||
return p.GetDohProfile(tenantID, id)
|
||||
}
|
||||
|
||||
|
||||
@@ -73,6 +73,7 @@ func (p *Postgres) CreateCDNSource(tenantID, moduleID string, in *store.CDNSourc
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return p.getCDNSource(ctx, moduleID, id)
|
||||
}
|
||||
|
||||
@@ -154,6 +155,9 @@ func (p *Postgres) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *s
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if store.CDNPatchAffectsInputHash(patch) {
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
}
|
||||
return p.getCDNSource(ctx, moduleID, sourceID)
|
||||
}
|
||||
|
||||
@@ -169,6 +173,7 @@ func (p *Postgres) DeleteCDNSource(tenantID, moduleID, sourceID string) error {
|
||||
if tag.RowsAffected() == 0 {
|
||||
return store.ErrNotFound
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -229,6 +234,7 @@ func (p *Postgres) CreateASEntry(tenantID, moduleID string, in *store.ASEntry) (
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return p.getASEntry(ctx, moduleID, id)
|
||||
}
|
||||
|
||||
@@ -297,6 +303,7 @@ func (p *Postgres) UpdateASEntry(tenantID, moduleID, entryID string, patch *stor
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return p.getASEntry(ctx, moduleID, entryID)
|
||||
}
|
||||
|
||||
@@ -369,6 +376,7 @@ func (p *Postgres) DeleteASEntry(tenantID, moduleID, entryID string) error {
|
||||
if tag.RowsAffected() == 0 {
|
||||
return store.ErrNotFound
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -417,6 +425,7 @@ func (p *Postgres) CreateDomainEntry(tenantID, moduleID string, in *store.Domain
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return p.getDomainEntry(ctx, moduleID, id)
|
||||
}
|
||||
|
||||
@@ -462,6 +471,7 @@ func (p *Postgres) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return p.getDomainEntry(ctx, moduleID, entryID)
|
||||
}
|
||||
|
||||
@@ -477,6 +487,7 @@ func (p *Postgres) DeleteDomainEntry(tenantID, moduleID, entryID string) error {
|
||||
if tag.RowsAffected() == 0 {
|
||||
return store.ErrNotFound
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -525,6 +536,7 @@ func (p *Postgres) CreateIPRangeEntry(tenantID, moduleID string, in *store.IPRan
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return p.getIPRangeEntry(ctx, moduleID, id)
|
||||
}
|
||||
|
||||
@@ -570,6 +582,7 @@ func (p *Postgres) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return p.getIPRangeEntry(ctx, moduleID, entryID)
|
||||
}
|
||||
|
||||
@@ -585,6 +598,7 @@ func (p *Postgres) DeleteIPRangeEntry(tenantID, moduleID, entryID string) error
|
||||
if tag.RowsAffected() == 0 {
|
||||
return store.ErrNotFound
|
||||
}
|
||||
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -116,11 +116,19 @@ type Backend interface {
|
||||
GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
|
||||
SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
|
||||
DeleteModulePrefixSnapshot(tenantID, moduleID string) error
|
||||
// SetModuleInputHash stores the ingest fingerprint on the module row (O(1) snapshot check).
|
||||
SetModuleInputHash(tenantID, moduleID, hash string) error
|
||||
// LockModuleSnapshot serializes read-modify-write of one module snapshot; unlock must be called.
|
||||
LockModuleSnapshot(tenantID, moduleID string) (unlock func())
|
||||
|
||||
// ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache).
|
||||
GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error)
|
||||
SetASNPrefixCache(asn int64, holder string, prefixes []string) error
|
||||
|
||||
// DomainResolveCache stores DoH results per FQDN (global TTL cache).
|
||||
GetDomainResolveCache(fqdn string) (*DomainResolveCacheEntry, bool, error)
|
||||
SetDomainResolveCache(fqdn string, addrs []string) error
|
||||
|
||||
// Ping verifies backend connectivity (no-op for in-memory).
|
||||
Ping(ctx context.Context) error
|
||||
|
||||
@@ -178,6 +186,13 @@ type ASNPrefixCacheEntry struct {
|
||||
FetchedAt time.Time
|
||||
}
|
||||
|
||||
// DomainResolveCacheEntry is a cached DoH A/AAAA result for one FQDN.
|
||||
type DomainResolveCacheEntry struct {
|
||||
FQDN string
|
||||
Addrs []string
|
||||
ResolvedAt time.Time
|
||||
}
|
||||
|
||||
// ModulePrefixSnapshot is the cached materialization for one module between refreshes.
|
||||
type ModulePrefixSnapshot struct {
|
||||
InputHash string
|
||||
|
||||
@@ -44,7 +44,9 @@ type Memory struct {
|
||||
settings map[string]map[string]any // tenantID -> key -> JSON-compatible value
|
||||
revPrefixes map[string][]PrefixRow
|
||||
moduleSnapshots map[string]*moduleSnapshotRec
|
||||
snapshotLocks sync.Map // key -> *sync.Mutex (per-module snapshot RMW)
|
||||
asnPrefixCache map[int64]*ASNPrefixCacheEntry
|
||||
domainResolveCache map[string]*DomainResolveCacheEntry
|
||||
apiKeys map[string]*apiKeyRec
|
||||
firewallClients map[string]*firewallClientRec
|
||||
firewallRules map[string]*FirewallRule
|
||||
@@ -99,6 +101,7 @@ type Module struct {
|
||||
LastRefreshedAt *time.Time
|
||||
DeletedAt *time.Time
|
||||
CreatedByUserID string // portal JWT sub; empty = system / API key
|
||||
InputHash string // ingest fingerprint; empty = not yet computed
|
||||
}
|
||||
|
||||
type Revision struct {
|
||||
@@ -156,6 +159,7 @@ func NewMemory() *Memory {
|
||||
revPrefixes: make(map[string][]PrefixRow),
|
||||
moduleSnapshots: make(map[string]*moduleSnapshotRec),
|
||||
asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry),
|
||||
domainResolveCache: make(map[string]*DomainResolveCacheEntry),
|
||||
apiKeys: make(map[string]*apiKeyRec),
|
||||
firewallClients: make(map[string]*firewallClientRec),
|
||||
firewallRules: make(map[string]*FirewallRule),
|
||||
@@ -696,6 +700,20 @@ func cloneStringPtr(s *string) *string {
|
||||
return &v
|
||||
}
|
||||
|
||||
func (m *Memory) clearModuleInputHashLocked(moduleID string) {
|
||||
if mod, ok := m.modules[moduleID]; ok && mod != nil {
|
||||
mod.InputHash = ""
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Memory) clearTenantModuleHashesLocked(tenantID string) {
|
||||
for _, mod := range m.modules {
|
||||
if mod != nil && mod.TenantID == tenantID && mod.DeletedAt == nil {
|
||||
mod.InputHash = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cloneModule(m *Module) *Module {
|
||||
if m == nil {
|
||||
return nil
|
||||
|
||||
@@ -81,6 +81,7 @@ func (m *Memory) UpdateModule(tenantID, moduleID string, patch *ModulePatch) (*M
|
||||
t := patch.LastRefreshedAt.UTC()
|
||||
mod.LastRefreshedAt = &t
|
||||
}
|
||||
mod.InputHash = ""
|
||||
return cloneModule(mod), nil
|
||||
}
|
||||
|
||||
@@ -152,6 +153,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN
|
||||
LastRefreshedAt: in.LastRefreshedAt,
|
||||
}
|
||||
m.cdnSources[id] = s
|
||||
mod.InputHash = ""
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -161,7 +163,8 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil {
|
||||
mod, err := m.moduleWriteOK(tenantID, moduleID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s, ok := m.cdnSources[sourceID]
|
||||
@@ -195,6 +198,9 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
|
||||
t := patch.LastRefreshedAt.UTC()
|
||||
s.LastRefreshedAt = &t
|
||||
}
|
||||
if CDNPatchAffectsInputHash(patch) {
|
||||
mod.InputHash = ""
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -209,6 +215,7 @@ func (m *Memory) DeleteCDNSource(tenantID, moduleID, sourceID string) error {
|
||||
return ErrNotFound
|
||||
}
|
||||
delete(m.cdnSources, sourceID)
|
||||
m.clearModuleInputHashLocked(moduleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -247,6 +254,7 @@ func (m *Memory) CreateASEntry(tenantID, moduleID string, in *ASEntry) (*ASEntry
|
||||
id := uuid.NewString()
|
||||
e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID}
|
||||
m.asEntries[id] = e
|
||||
mod.InputHash = ""
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -256,7 +264,8 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil {
|
||||
mod, err := m.moduleWriteOK(tenantID, moduleID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, ok := m.asEntries[entryID]
|
||||
@@ -283,6 +292,7 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr
|
||||
e.PrefixCount = nil
|
||||
e.ASNResolvedAt = nil
|
||||
}
|
||||
mod.InputHash = ""
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -324,6 +334,7 @@ func (m *Memory) DeleteASEntry(tenantID, moduleID, entryID string) error {
|
||||
return ErrNotFound
|
||||
}
|
||||
delete(m.asEntries, entryID)
|
||||
m.clearModuleInputHashLocked(moduleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -362,6 +373,7 @@ func (m *Memory) CreateDomainEntry(tenantID, moduleID string, in *DomainEntry) (
|
||||
id := uuid.NewString()
|
||||
e := &DomainEntry{ID: id, ModuleID: moduleID, FQDN: strings.TrimSpace(in.FQDN), CommunityID: in.CommunityID}
|
||||
m.domainEnt[id] = e
|
||||
mod.InputHash = ""
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -371,7 +383,8 @@ func (m *Memory) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *Do
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil {
|
||||
mod, err := m.moduleWriteOK(tenantID, moduleID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, ok := m.domainEnt[entryID]
|
||||
@@ -389,6 +402,7 @@ func (m *Memory) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *Do
|
||||
e.CommunityID = &v
|
||||
}
|
||||
}
|
||||
mod.InputHash = ""
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -403,6 +417,7 @@ func (m *Memory) DeleteDomainEntry(tenantID, moduleID, entryID string) error {
|
||||
return ErrNotFound
|
||||
}
|
||||
delete(m.domainEnt, entryID)
|
||||
m.clearModuleInputHashLocked(moduleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -441,6 +456,7 @@ func (m *Memory) CreateIPRangeEntry(tenantID, moduleID string, in *IPRangeEntry)
|
||||
id := uuid.NewString()
|
||||
e := &IPRangeEntry{ID: id, ModuleID: moduleID, Prefix: strings.TrimSpace(in.Prefix), CommunityID: in.CommunityID}
|
||||
m.ipRanges[id] = e
|
||||
mod.InputHash = ""
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -450,7 +466,8 @@ func (m *Memory) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch *I
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil {
|
||||
mod, err := m.moduleWriteOK(tenantID, moduleID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e, ok := m.ipRanges[entryID]
|
||||
@@ -468,6 +485,7 @@ func (m *Memory) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch *I
|
||||
e.CommunityID = &v
|
||||
}
|
||||
}
|
||||
mod.InputHash = ""
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -482,6 +500,7 @@ func (m *Memory) DeleteIPRangeEntry(tenantID, moduleID, entryID string) error {
|
||||
return ErrNotFound
|
||||
}
|
||||
delete(m.ipRanges, entryID)
|
||||
m.clearModuleInputHashLocked(moduleID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -551,6 +570,7 @@ func (m *Memory) UpdateDohProfile(tenantID, id string, patch *DohProfilePatch) (
|
||||
if patch.SecretRef != nil {
|
||||
p.SecretRef = patch.SecretRef
|
||||
}
|
||||
m.clearTenantModuleHashesLocked(tenantID)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (m *Memory) GetDomainResolveCache(fqdn string) (*DomainResolveCacheEntry, bool, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
if m.domainResolveCache == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
e, ok := m.domainResolveCache[strings.ToLower(strings.TrimSpace(fqdn))]
|
||||
if !ok || e == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
return &DomainResolveCacheEntry{
|
||||
FQDN: e.FQDN,
|
||||
Addrs: append([]string(nil), e.Addrs...),
|
||||
ResolvedAt: e.ResolvedAt,
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func (m *Memory) SetDomainResolveCache(fqdn string, addrs []string) error {
|
||||
key := strings.ToLower(strings.TrimSpace(fqdn))
|
||||
if key == "" {
|
||||
return ErrInvalidInput
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.domainResolveCache == nil {
|
||||
m.domainResolveCache = make(map[string]*DomainResolveCacheEntry)
|
||||
}
|
||||
m.domainResolveCache[key] = &DomainResolveCacheEntry{
|
||||
FQDN: key,
|
||||
Addrs: append([]string(nil), addrs...),
|
||||
ResolvedAt: time.Now().UTC(),
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package store
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -51,6 +52,28 @@ func (m *Memory) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
mod, ok := m.modules[moduleID]
|
||||
if !ok || mod.DeletedAt != nil {
|
||||
return ErrNotFound
|
||||
}
|
||||
if mod.TenantID != tenantID {
|
||||
return ErrTenantScope
|
||||
}
|
||||
mod.InputHash = strings.TrimSpace(hash)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Memory) LockModuleSnapshot(tenantID, moduleID string) func() {
|
||||
key := moduleSnapshotKey(tenantID, moduleID)
|
||||
v, _ := m.snapshotLocks.LoadOrStore(key, &sync.Mutex{})
|
||||
mu := v.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return func() { mu.Unlock() }
|
||||
}
|
||||
|
||||
type moduleSnapshotRec struct {
|
||||
InputHash string
|
||||
CollectedAt time.Time
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ComputeModuleInputHash fingerprints module config and child entries so snapshots
|
||||
// invalidate on CRUD without re-reading every child at render time.
|
||||
func ComputeModuleInputHash(st Backend, tenantID string, mod *Module) (string, error) {
|
||||
if st == nil || mod == nil {
|
||||
return "", fmt.Errorf("store: module hash: missing store or module")
|
||||
}
|
||||
h := sha256.New()
|
||||
_, _ = fmt.Fprintf(h, "type=%s\n", strings.TrimSpace(mod.Type))
|
||||
_, _ = fmt.Fprintf(h, "enabled=%t\n", mod.Enabled)
|
||||
if mod.DefaultCommunityID != nil {
|
||||
_, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID))
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "doh_policy=%s\n", NormalizeDohResolverPolicy(mod.DohResolverPolicy))
|
||||
for _, pid := range mod.EffectiveDohProfileIDs() {
|
||||
_, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid)
|
||||
if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil {
|
||||
_, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL))
|
||||
if prof.TimeoutMs != nil {
|
||||
_, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch mod.Type {
|
||||
case "IP_RANGES":
|
||||
list, err := st.ListIPRangeEntries(tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].Prefix < list[j].Prefix })
|
||||
for _, e := range list {
|
||||
comm := ""
|
||||
if e.CommunityID != nil {
|
||||
comm = *e.CommunityID
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "ip=%s|c=%s\n", e.Prefix, comm)
|
||||
}
|
||||
case "AS_PREFIXES":
|
||||
list, err := st.ListASEntries(tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
|
||||
for _, e := range list {
|
||||
comm := ""
|
||||
if e.CommunityID != nil {
|
||||
comm = *e.CommunityID
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "as=%d|c=%s\n", e.ASN, comm)
|
||||
}
|
||||
case "CDN_CIDRS":
|
||||
list, err := st.ListCDNSources(tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].ID < list[j].ID })
|
||||
for _, s := range list {
|
||||
comm := ""
|
||||
if s.CommunityID != nil {
|
||||
comm = *s.CommunityID
|
||||
}
|
||||
interval := 0
|
||||
if s.RefreshIntervalSec != nil {
|
||||
interval = *s.RefreshIntervalSec
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "cdn=%s|url=%s|kind=%s|path=%s|c=%s|etag=%s|interval=%d\n",
|
||||
s.ID, strings.TrimSpace(s.URL), s.SourceKind, strings.TrimSpace(s.PrefixPath), comm,
|
||||
strings.TrimSpace(s.Etag), interval)
|
||||
}
|
||||
case "DOMAINS":
|
||||
list, err := st.ListDomainEntries(tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].FQDN < list[j].FQDN })
|
||||
for _, e := range list {
|
||||
comm := ""
|
||||
if e.CommunityID != nil {
|
||||
comm = *e.CommunityID
|
||||
}
|
||||
_, _ = fmt.Fprintf(h, "dom=%s|c=%s\n", strings.TrimSpace(e.FQDN), comm)
|
||||
}
|
||||
default:
|
||||
_, _ = fmt.Fprintf(h, "unknown_type=%s\n", mod.Type)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// TouchModuleInputHash recomputes and stores module.input_hash (ARCH-01: hash lives in store).
|
||||
func TouchModuleInputHash(st Backend, tenantID, moduleID string) {
|
||||
if st == nil {
|
||||
return
|
||||
}
|
||||
mod, err := st.GetModule(tenantID, moduleID)
|
||||
if err != nil || mod == nil {
|
||||
return
|
||||
}
|
||||
h, err := ComputeModuleInputHash(st, tenantID, mod)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = st.SetModuleInputHash(tenantID, moduleID, h)
|
||||
}
|
||||
|
||||
// CDNPatchAffectsInputHash reports whether a CDN source patch changes ingest fingerprint
|
||||
// fields (URL/kind/path/community/interval). ETag and last_refreshed_at do not.
|
||||
func CDNPatchAffectsInputHash(patch *CDNSourcePatch) bool {
|
||||
if patch == nil {
|
||||
return false
|
||||
}
|
||||
return patch.SourceKind != nil || patch.URL != nil || patch.PrefixPath != nil ||
|
||||
patch.CommunityID != nil || patch.RefreshIntervalSec != nil
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE module
|
||||
DROP COLUMN IF EXISTS input_hash;
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE module
|
||||
ADD COLUMN input_hash TEXT NOT NULL DEFAULT '';
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS domain_resolve_cache;
|
||||
@@ -0,0 +1,8 @@
|
||||
-- TTL cache for DoH/A/AAAA resolutions (pipeline domain ingest).
|
||||
CREATE TABLE domain_resolve_cache (
|
||||
fqdn TEXT PRIMARY KEY,
|
||||
addrs_json JSONB NOT NULL DEFAULT '[]',
|
||||
resolved_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_domain_resolve_cache_resolved ON domain_resolve_cache (resolved_at);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE module
|
||||
DROP COLUMN input_hash;
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE module
|
||||
ADD COLUMN input_hash TEXT NOT NULL DEFAULT '';
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS domain_resolve_cache;
|
||||
@@ -0,0 +1,7 @@
|
||||
CREATE TABLE domain_resolve_cache (
|
||||
fqdn TEXT PRIMARY KEY,
|
||||
addrs_json TEXT NOT NULL DEFAULT '[]',
|
||||
resolved_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||
);
|
||||
|
||||
CREATE INDEX idx_domain_resolve_cache_resolved ON domain_resolve_cache (resolved_at);
|
||||
Reference in New Issue
Block a user