Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dc803bcb34 |
@@ -1,102 +0,0 @@
|
|||||||
import { AlertTriangle, CheckCircle, Info } from 'lucide-react'
|
|
||||||
|
|
||||||
import { DashboardFramePanel } from '@/components/dashboard/dashboard-frame-panel'
|
|
||||||
import { Badge } from '@/components/reui/badge'
|
|
||||||
import {
|
|
||||||
Timeline,
|
|
||||||
TimelineContent,
|
|
||||||
TimelineHeader,
|
|
||||||
TimelineIndicator,
|
|
||||||
TimelineItem,
|
|
||||||
TimelineSeparator,
|
|
||||||
TimelineTitle,
|
|
||||||
} from '@/components/reui/timeline'
|
|
||||||
import { cn } from '@evobgp/ui/lib/utils'
|
|
||||||
|
|
||||||
import { recentPlatformActivity } from '@/lib/metrics'
|
|
||||||
import type { JobRow, PeerRow, RevisionRow, SpeakerRow } from '@/types/api'
|
|
||||||
|
|
||||||
const KIND_META = {
|
|
||||||
job: { icon: Info, className: 'text-info' },
|
|
||||||
revision: { icon: CheckCircle, className: 'text-success' },
|
|
||||||
network: { icon: AlertTriangle, className: 'text-warning' },
|
|
||||||
} as const
|
|
||||||
|
|
||||||
function statusBadgeVariant(status: string) {
|
|
||||||
const s = status.toLowerCase()
|
|
||||||
if (['ok', 'success', 'completed', 'done'].includes(s)) return 'success-light' as const
|
|
||||||
if (['running', 'queued', 'pending'].includes(s)) return 'info-light' as const
|
|
||||||
if (['warning', 'mismatch'].includes(s)) return 'warning-light' as const
|
|
||||||
if (['failed', 'error', 'cancelled'].includes(s)) return 'destructive-light' as const
|
|
||||||
return 'outline' as const
|
|
||||||
}
|
|
||||||
|
|
||||||
export function DashboardActivityTimeline({
|
|
||||||
jobs,
|
|
||||||
revisions,
|
|
||||||
peers,
|
|
||||||
speakers,
|
|
||||||
loading,
|
|
||||||
}: {
|
|
||||||
jobs: JobRow[]
|
|
||||||
revisions: RevisionRow[]
|
|
||||||
peers: PeerRow[]
|
|
||||||
speakers: SpeakerRow[]
|
|
||||||
loading?: boolean
|
|
||||||
}) {
|
|
||||||
const items = recentPlatformActivity(jobs, revisions, peers, speakers, 6)
|
|
||||||
|
|
||||||
return (
|
|
||||||
<DashboardFramePanel
|
|
||||||
title="Недавняя активность"
|
|
||||||
description="Задачи, ревизии и сетевые события"
|
|
||||||
className="h-full min-w-0"
|
|
||||||
>
|
|
||||||
{loading ? (
|
|
||||||
<p className="text-muted-foreground px-4 py-6 text-sm">Загрузка…</p>
|
|
||||||
) : items.length === 0 ? (
|
|
||||||
<p className="text-muted-foreground px-4 py-6 text-sm">Нет недавних событий</p>
|
|
||||||
) : (
|
|
||||||
<div className="px-4 py-4">
|
|
||||||
<Timeline defaultValue={items.length}>
|
|
||||||
{items.map((item, index) => {
|
|
||||||
const meta = KIND_META[item.kind]
|
|
||||||
const Icon = meta.icon
|
|
||||||
return (
|
|
||||||
<TimelineItem
|
|
||||||
key={item.id}
|
|
||||||
step={index + 1}
|
|
||||||
className="group-data-[orientation=vertical]/timeline:ms-8 group-data-[orientation=vertical]/timeline:not-last:pb-4"
|
|
||||||
>
|
|
||||||
<TimelineHeader>
|
|
||||||
<TimelineSeparator className="bg-border! group-data-[orientation=vertical]/timeline:-left-6 group-data-[orientation=vertical]/timeline:top-2 group-data-[orientation=vertical]/timeline:h-[calc(100%-1.5rem)] group-data-[orientation=vertical]/timeline:translate-y-5" />
|
|
||||||
<TimelineIndicator className="border-none bg-transparent group-data-[orientation=vertical]/timeline:-left-6">
|
|
||||||
<span
|
|
||||||
className={cn(
|
|
||||||
'bg-muted/70 flex size-7 items-center justify-center rounded-full',
|
|
||||||
meta.className,
|
|
||||||
)}
|
|
||||||
>
|
|
||||||
<Icon className="size-3.5" aria-hidden />
|
|
||||||
</span>
|
|
||||||
</TimelineIndicator>
|
|
||||||
</TimelineHeader>
|
|
||||||
<TimelineContent className="min-w-0 pb-1 text-foreground">
|
|
||||||
<TimelineTitle className="text-sm leading-snug font-normal break-words">
|
|
||||||
{item.message}
|
|
||||||
</TimelineTitle>
|
|
||||||
<div className="mt-2">
|
|
||||||
<Badge variant={statusBadgeVariant(item.status)} size="sm">
|
|
||||||
{item.statusLabel ?? item.status}
|
|
||||||
</Badge>
|
|
||||||
</div>
|
|
||||||
</TimelineContent>
|
|
||||||
</TimelineItem>
|
|
||||||
)
|
|
||||||
})}
|
|
||||||
</Timeline>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</DashboardFramePanel>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
import type { ReactNode } from 'react'
|
|
||||||
|
|
||||||
import {
|
|
||||||
FrameSection,
|
|
||||||
panelCardContentFlushClassName,
|
|
||||||
} from '@/components/reui-kit'
|
|
||||||
import { cn } from '@evobgp/ui/lib/utils'
|
|
||||||
|
|
||||||
/** Frame panel for dashboard sections. Preview: https://reui.io/docs/components/base/frame */
|
|
||||||
export function DashboardFramePanel({
|
|
||||||
title,
|
|
||||||
description,
|
|
||||||
actions,
|
|
||||||
children,
|
|
||||||
className,
|
|
||||||
contentClassName,
|
|
||||||
}: {
|
|
||||||
title?: string
|
|
||||||
description?: string
|
|
||||||
actions?: ReactNode
|
|
||||||
children: ReactNode
|
|
||||||
className?: string
|
|
||||||
contentClassName?: string
|
|
||||||
}) {
|
|
||||||
return (
|
|
||||||
<FrameSection
|
|
||||||
title={title}
|
|
||||||
description={description}
|
|
||||||
actions={actions}
|
|
||||||
className={cn('h-full', className)}
|
|
||||||
contentClassName={cn(panelCardContentFlushClassName, contentClassName)}
|
|
||||||
>
|
|
||||||
{children}
|
|
||||||
</FrameSection>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -3,8 +3,6 @@ import {
|
|||||||
Boxes,
|
Boxes,
|
||||||
ListChecks,
|
ListChecks,
|
||||||
Network,
|
Network,
|
||||||
ServerCog,
|
|
||||||
Share2,
|
|
||||||
} from 'lucide-react'
|
} from 'lucide-react'
|
||||||
import type { ReactNode } from 'react'
|
import type { ReactNode } from 'react'
|
||||||
|
|
||||||
@@ -16,6 +14,11 @@ import type { JobRow, ModuleRow, PeerRow, SpeakerRow } from '@/types/api'
|
|||||||
|
|
||||||
type KpiCard = KpiStatItem & { icon: ReactNode }
|
type KpiCard = KpiStatItem & { icon: ReactNode }
|
||||||
|
|
||||||
|
function ratioPercent(part: number, total: number): number | undefined {
|
||||||
|
if (total <= 0) return undefined
|
||||||
|
return Math.round((part / total) * 100)
|
||||||
|
}
|
||||||
|
|
||||||
function buildKpis({
|
function buildKpis({
|
||||||
modules,
|
modules,
|
||||||
peers,
|
peers,
|
||||||
@@ -40,17 +43,34 @@ function buildKpis({
|
|||||||
).length
|
).length
|
||||||
const offlineSpeakers = Math.max(0, speakers.length - network.speakersOnline)
|
const offlineSpeakers = Math.max(0, speakers.length - network.speakersOnline)
|
||||||
const riskCount = network.peersMismatch + failedJobs + offlineSpeakers
|
const riskCount = network.peersMismatch + failedJobs + offlineSpeakers
|
||||||
|
const disabledModules = Math.max(0, modules.length - enabledModules)
|
||||||
|
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
id: 'modules',
|
id: 'modules',
|
||||||
icon: <Boxes aria-hidden />,
|
icon: <Boxes aria-hidden />,
|
||||||
iconClassName: 'text-primary',
|
iconClassName: 'text-primary',
|
||||||
value: loading ? '—' : `${enabledModules}/${modules.length || 0}`,
|
value: loading ? '—' : enabledModules,
|
||||||
label: 'Модули активны',
|
label: 'Модули активны',
|
||||||
|
progress: loading ? undefined : ratioPercent(enabledModules, modules.length),
|
||||||
footer: (
|
footer: (
|
||||||
<Badge variant="primary-light" size="sm">
|
<Badge
|
||||||
{loading ? '…' : `${modules.length} всего`}
|
variant={
|
||||||
|
loading || modules.length === 0
|
||||||
|
? 'outline'
|
||||||
|
: disabledModules === 0
|
||||||
|
? 'success-light'
|
||||||
|
: 'warning-light'
|
||||||
|
}
|
||||||
|
size="sm"
|
||||||
|
>
|
||||||
|
{loading
|
||||||
|
? '…'
|
||||||
|
: modules.length === 0
|
||||||
|
? 'нет модулей'
|
||||||
|
: disabledModules === 0
|
||||||
|
? 'все активны'
|
||||||
|
: `${disabledModules} выкл`}
|
||||||
</Badge>
|
</Badge>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -60,6 +80,7 @@ function buildKpis({
|
|||||||
iconClassName: 'text-info',
|
iconClassName: 'text-info',
|
||||||
value: loading || bgpPct === null ? '—' : `${bgpPct}%`,
|
value: loading || bgpPct === null ? '—' : `${bgpPct}%`,
|
||||||
label: 'BGP готовность',
|
label: 'BGP готовность',
|
||||||
|
progress: loading || bgpPct === null ? undefined : bgpPct,
|
||||||
footer: (
|
footer: (
|
||||||
<Badge
|
<Badge
|
||||||
variant={
|
variant={
|
||||||
@@ -73,34 +94,9 @@ function buildKpis({
|
|||||||
>
|
>
|
||||||
{loading || bgpPct === null
|
{loading || bgpPct === null
|
||||||
? 'нет включённых пиров'
|
? 'нет включённых пиров'
|
||||||
: `${network.peersEstablished} установлено`}
|
: bgpPct >= 90
|
||||||
</Badge>
|
? 'сессии в норме'
|
||||||
),
|
: `${network.peersEstablished} установлено`}
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'peers',
|
|
||||||
icon: <Share2 aria-hidden />,
|
|
||||||
iconClassName: 'text-success',
|
|
||||||
value: loading ? '—' : `${network.peersEstablished}/${peersEnabled}`,
|
|
||||||
label: 'Пиры установлены',
|
|
||||||
footer: (
|
|
||||||
<Badge variant="success-light" size="sm">
|
|
||||||
{loading ? '…' : `${network.peersTotal} в каталоге`}
|
|
||||||
</Badge>
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'speakers',
|
|
||||||
icon: <ServerCog aria-hidden />,
|
|
||||||
iconClassName: 'text-warning',
|
|
||||||
value: loading ? '—' : `${network.speakersOnline}/${network.speakersTotal}`,
|
|
||||||
label: 'Спикеры в сети',
|
|
||||||
footer: (
|
|
||||||
<Badge
|
|
||||||
variant={network.speakersOnline === network.speakersTotal ? 'success-light' : 'warning-light'}
|
|
||||||
size="sm"
|
|
||||||
>
|
|
||||||
{loading ? '…' : 'в сети'}
|
|
||||||
</Badge>
|
</Badge>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -112,7 +108,7 @@ function buildKpis({
|
|||||||
label: 'Активные задачи',
|
label: 'Активные задачи',
|
||||||
footer: (
|
footer: (
|
||||||
<Badge variant={running > 0 ? 'info-light' : 'outline'} size="sm">
|
<Badge variant={running > 0 ? 'info-light' : 'outline'} size="sm">
|
||||||
{loading ? '…' : `${jobs.length} в выборке`}
|
{loading ? '…' : running > 0 ? 'выполняются' : 'очередь пуста'}
|
||||||
</Badge>
|
</Badge>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,117 +0,0 @@
|
|||||||
import { useMemo, useState } from 'react'
|
|
||||||
import { Link, useNavigate } from '@tanstack/react-router'
|
|
||||||
import { BoxesIcon, PlusIcon, SearchIcon } from 'lucide-react'
|
|
||||||
|
|
||||||
import { CategoryBadge } from '@/components/category-badge'
|
|
||||||
import { DataGridNameCell } from '@/components/data-grid-cell'
|
|
||||||
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
|
|
||||||
import type { FilterField, FilterQuery } from '@/components/reui/filters/filters-types'
|
|
||||||
import {
|
|
||||||
ResourcePage,
|
|
||||||
createTextFilterQuery,
|
|
||||||
type DataGridColumnDef,
|
|
||||||
} from '@/components/reui-kit'
|
|
||||||
import { Button } from '@evobgp/ui/components/button'
|
|
||||||
import { moduleTypeRu } from '@/lib/ui-labels'
|
|
||||||
import type { ModuleRow } from '@/types/api'
|
|
||||||
|
|
||||||
const MODULE_TABS = [
|
|
||||||
{ id: 'all', label: 'Все' },
|
|
||||||
{ id: 'enabled', label: 'Вкл' },
|
|
||||||
{ id: 'disabled', label: 'Выкл' },
|
|
||||||
]
|
|
||||||
|
|
||||||
const filterFields: FilterField[] = [
|
|
||||||
{
|
|
||||||
id: 'search',
|
|
||||||
label: 'Поиск',
|
|
||||||
icon: <SearchIcon className="size-3.5" aria-hidden />,
|
|
||||||
type: 'text',
|
|
||||||
placeholder: 'Поиск модулей…',
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
function getFilterFieldValue(item: ModuleRow, field: string): unknown {
|
|
||||||
if (field === 'search') {
|
|
||||||
return `${item.name} ${item.type} ${moduleTypeRu(item.type)}`
|
|
||||||
}
|
|
||||||
return undefined
|
|
||||||
}
|
|
||||||
|
|
||||||
function tabFilter(item: ModuleRow, tabId: string): boolean {
|
|
||||||
if (tabId === 'enabled') return item.enabled !== false
|
|
||||||
if (tabId === 'disabled') return item.enabled === false
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
export function DashboardModulesGrid({
|
|
||||||
modules,
|
|
||||||
isLoading = false,
|
|
||||||
}: {
|
|
||||||
modules: ModuleRow[]
|
|
||||||
isLoading?: boolean
|
|
||||||
}) {
|
|
||||||
const navigate = useNavigate()
|
|
||||||
const [filterQuery, setFilterQuery] = useState<FilterQuery>(() =>
|
|
||||||
createTextFilterQuery('search'),
|
|
||||||
)
|
|
||||||
|
|
||||||
const columns = useMemo<DataGridColumnDef<ModuleRow>[]>(
|
|
||||||
() => [
|
|
||||||
{
|
|
||||||
accessorKey: 'name',
|
|
||||||
id: 'name',
|
|
||||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Модуль" />,
|
|
||||||
cell: ({ row }) => <DataGridNameCell icon={BoxesIcon} title={row.original.name} />,
|
|
||||||
minSize: 180,
|
|
||||||
meta: { headerTitle: 'Модуль' },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
accessorKey: 'type',
|
|
||||||
id: 'type',
|
|
||||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Тип" />,
|
|
||||||
cell: ({ row }) => <CategoryBadge>{moduleTypeRu(row.original.type)}</CategoryBadge>,
|
|
||||||
meta: { headerTitle: 'Тип' },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
accessorKey: 'priority',
|
|
||||||
id: 'priority',
|
|
||||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Приоритет" />,
|
|
||||||
cell: ({ row }) => (
|
|
||||||
<span className="font-mono text-sm tabular-nums">{row.original.priority}</span>
|
|
||||||
),
|
|
||||||
size: 88,
|
|
||||||
meta: { headerTitle: 'Приоритет' },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[],
|
|
||||||
)
|
|
||||||
|
|
||||||
return (
|
|
||||||
<ResourcePage
|
|
||||||
title="Модули"
|
|
||||||
description="Поиск и быстрый переход к настройке"
|
|
||||||
tabs={MODULE_TABS}
|
|
||||||
tabFilter={tabFilter}
|
|
||||||
filterFields={filterFields}
|
|
||||||
filterQuery={filterQuery}
|
|
||||||
onFilterQueryChange={setFilterQuery}
|
|
||||||
onClearFilters={() => setFilterQuery(createTextFilterQuery('search'))}
|
|
||||||
getFilterFieldValue={getFilterFieldValue}
|
|
||||||
columns={columns}
|
|
||||||
data={modules}
|
|
||||||
getRowId={(row) => row.id}
|
|
||||||
isLoading={isLoading}
|
|
||||||
primaryAction={
|
|
||||||
<Button variant="outline" size="sm" render={<Link to="/modules" search={{ create: true }} />}>
|
|
||||||
<PlusIcon />
|
|
||||||
Создать
|
|
||||||
</Button>
|
|
||||||
}
|
|
||||||
onRowClick={(row) =>
|
|
||||||
void navigate({ to: '/modules/$moduleId', params: { moduleId: row.id } })
|
|
||||||
}
|
|
||||||
emptyState={{ title: 'Нет модулей по выбранным фильтрам.' }}
|
|
||||||
/>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -60,8 +60,28 @@ export function DashboardNetworkHealth({
|
|||||||
label: mode === 'peers' ? 'Утилизация пиров' : 'Спикеры в сети',
|
label: mode === 'peers' ? 'Утилизация пиров' : 'Спикеры в сети',
|
||||||
percent: loading ? 0 : utilization,
|
percent: loading ? 0 : utilization,
|
||||||
badge: (
|
badge: (
|
||||||
<Badge variant="outline" radius="full" className="h-6 px-2 text-[10px]">
|
<Badge
|
||||||
{loading ? '…' : `${established}/${total}`}
|
variant={
|
||||||
|
loading || total === 0
|
||||||
|
? 'outline'
|
||||||
|
: offline === 0
|
||||||
|
? 'success-light'
|
||||||
|
: 'warning-light'
|
||||||
|
}
|
||||||
|
size="sm"
|
||||||
|
radius="full"
|
||||||
|
>
|
||||||
|
{loading
|
||||||
|
? '…'
|
||||||
|
: total === 0
|
||||||
|
? 'нет данных'
|
||||||
|
: offline === 0
|
||||||
|
? mode === 'peers'
|
||||||
|
? 'все установлены'
|
||||||
|
: 'все в сети'
|
||||||
|
: mode === 'peers'
|
||||||
|
? `${offline} не установлены`
|
||||||
|
: `${offline} офлайн`}
|
||||||
</Badge>
|
</Badge>
|
||||||
),
|
),
|
||||||
}}
|
}}
|
||||||
|
|||||||
@@ -1,62 +0,0 @@
|
|||||||
import { useMemo } from 'react'
|
|
||||||
import { ListTodo } from 'lucide-react'
|
|
||||||
|
|
||||||
import { DataGridNameCell } from '@/components/data-grid-cell'
|
|
||||||
import { StatusBadge } from '@/components/status-badge'
|
|
||||||
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
|
|
||||||
import { FrameDataGrid, type DataGridColumnDef } from '@/components/reui-kit'
|
|
||||||
import { jobKindRu } from '@/lib/ui-labels'
|
|
||||||
import type { JobRow } from '@/types/api'
|
|
||||||
|
|
||||||
export function DashboardRecentJobsGrid({
|
|
||||||
jobs,
|
|
||||||
nameById,
|
|
||||||
isLoading = false,
|
|
||||||
}: {
|
|
||||||
jobs: JobRow[]
|
|
||||||
nameById: Map<string, string>
|
|
||||||
isLoading?: boolean
|
|
||||||
}) {
|
|
||||||
const data = useMemo(() => jobs.slice(0, 8), [jobs])
|
|
||||||
|
|
||||||
const columns = useMemo<DataGridColumnDef<JobRow>[]>(
|
|
||||||
() => [
|
|
||||||
{
|
|
||||||
accessorKey: 'kind',
|
|
||||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Вид" />,
|
|
||||||
cell: ({ row }) => (
|
|
||||||
<DataGridNameCell
|
|
||||||
icon={ListTodo}
|
|
||||||
title={jobKindRu(row.original.kind)}
|
|
||||||
subtitle={
|
|
||||||
row.original.meta?.module_id
|
|
||||||
? (nameById.get(String(row.original.meta.module_id)) ?? undefined)
|
|
||||||
: undefined
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
),
|
|
||||||
meta: { headerTitle: 'Вид' },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
accessorKey: 'status',
|
|
||||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Статус" />,
|
|
||||||
cell: ({ row }) => <StatusBadge status={row.original.status} />,
|
|
||||||
meta: { headerTitle: 'Статус' },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[nameById],
|
|
||||||
)
|
|
||||||
|
|
||||||
return (
|
|
||||||
<FrameDataGrid
|
|
||||||
title="Недавние задачи"
|
|
||||||
description="Последние фоновые операции"
|
|
||||||
columns={columns}
|
|
||||||
data={data}
|
|
||||||
rowId={(row) => row.job_id}
|
|
||||||
emptyTitle="Нет задач"
|
|
||||||
pagination={false}
|
|
||||||
isLoading={isLoading}
|
|
||||||
/>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
import { useMemo } from 'react'
|
|
||||||
import { GitCommitHorizontal } from 'lucide-react'
|
|
||||||
|
|
||||||
import { DataGridMutedCell, DataGridNameCell } from '@/components/data-grid-cell'
|
|
||||||
import { DataGridColumnHeader } from '@/components/reui/data-grid/data-grid-column-header'
|
|
||||||
import { FrameDataGrid, type DataGridColumnDef } from '@/components/reui-kit'
|
|
||||||
import type { RevisionRow } from '@/types/api'
|
|
||||||
|
|
||||||
export function DashboardRecentRevisionsGrid({
|
|
||||||
revisions,
|
|
||||||
isLoading = false,
|
|
||||||
}: {
|
|
||||||
revisions: RevisionRow[]
|
|
||||||
isLoading?: boolean
|
|
||||||
}) {
|
|
||||||
const data = useMemo(() => revisions.slice(0, 8), [revisions])
|
|
||||||
|
|
||||||
const columns = useMemo<DataGridColumnDef<RevisionRow>[]>(
|
|
||||||
() => [
|
|
||||||
{
|
|
||||||
id: 'id',
|
|
||||||
accessorFn: (row) => row.id,
|
|
||||||
header: ({ column }) => <DataGridColumnHeader column={column} title="ID" />,
|
|
||||||
cell: ({ row }) => (
|
|
||||||
<DataGridNameCell icon={GitCommitHorizontal} title={`${row.original.id.slice(0, 10)}…`} />
|
|
||||||
),
|
|
||||||
meta: { headerTitle: 'ID' },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
accessorKey: 'created_at',
|
|
||||||
header: ({ column }) => <DataGridColumnHeader column={column} title="Создана" />,
|
|
||||||
cell: ({ row }) => (
|
|
||||||
<DataGridMutedCell>
|
|
||||||
{new Date(row.original.created_at).toLocaleString('ru-RU')}
|
|
||||||
</DataGridMutedCell>
|
|
||||||
),
|
|
||||||
meta: { headerTitle: 'Создана' },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
[],
|
|
||||||
)
|
|
||||||
|
|
||||||
return (
|
|
||||||
<FrameDataGrid
|
|
||||||
title="Последние ревизии"
|
|
||||||
description="История конфигураций"
|
|
||||||
columns={columns}
|
|
||||||
data={data}
|
|
||||||
rowId={(row) => row.id}
|
|
||||||
emptyTitle="Нет ревизий"
|
|
||||||
pagination={false}
|
|
||||||
isLoading={isLoading}
|
|
||||||
/>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -3,6 +3,7 @@ import { Link } from '@tanstack/react-router'
|
|||||||
|
|
||||||
import { Frame, FramePanel } from '@/components/reui/frame'
|
import { Frame, FramePanel } from '@/components/reui/frame'
|
||||||
import { Badge } from '@/components/reui/badge'
|
import { Badge } from '@/components/reui/badge'
|
||||||
|
import { Progress } from '@evobgp/ui/components/progress'
|
||||||
import { cn } from '@evobgp/ui/lib/utils'
|
import { cn } from '@evobgp/ui/lib/utils'
|
||||||
import { kpiCols } from './kpi-cols'
|
import { kpiCols } from './kpi-cols'
|
||||||
import { IconTile } from '@/components/reui/icon-tile'
|
import { IconTile } from '@/components/reui/icon-tile'
|
||||||
@@ -29,6 +30,8 @@ export type KpiStatItem = {
|
|||||||
iconClassName?: string
|
iconClassName?: string
|
||||||
variant?: KpiStatVariant
|
variant?: KpiStatVariant
|
||||||
footer?: ReactNode
|
footer?: ReactNode
|
||||||
|
/** 0–100 completion bar under the value (stats-4). Omit to hide. */
|
||||||
|
progress?: number
|
||||||
}
|
}
|
||||||
|
|
||||||
/** CFDM-compatible card shape (id required). */
|
/** CFDM-compatible card shape (id required). */
|
||||||
@@ -48,6 +51,17 @@ const VALUE_VARIANT_CLASS: Record<KpiStatVariant, string> = {
|
|||||||
destructive: 'text-destructive',
|
destructive: 'text-destructive',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const PROGRESS_TONE_CLASS: Record<KpiStatVariant, string> = {
|
||||||
|
default: '',
|
||||||
|
warning: '[&_[data-slot=progress-indicator]]:bg-warning',
|
||||||
|
destructive: '[&_[data-slot=progress-indicator]]:bg-destructive',
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampProgress(value: number): number {
|
||||||
|
if (Number.isNaN(value)) return 0
|
||||||
|
return Math.min(100, Math.max(0, value))
|
||||||
|
}
|
||||||
|
|
||||||
function handleCardKeyDown(onActivate: () => void, event: KeyboardEvent<HTMLDivElement>) {
|
function handleCardKeyDown(onActivate: () => void, event: KeyboardEvent<HTMLDivElement>) {
|
||||||
if (event.key === 'Enter' || event.key === ' ') {
|
if (event.key === 'Enter' || event.key === ' ') {
|
||||||
event.preventDefault()
|
event.preventDefault()
|
||||||
@@ -111,6 +125,20 @@ function KpiStatCardBody({ item }: { item: KpiStatItem }) {
|
|||||||
>
|
>
|
||||||
{item.value}
|
{item.value}
|
||||||
</div>
|
</div>
|
||||||
|
{item.progress !== undefined ? (
|
||||||
|
<Progress
|
||||||
|
value={clampProgress(item.progress)}
|
||||||
|
className={cn(
|
||||||
|
'mt-1.5 w-full gap-0 **:data-[slot=progress-track]:h-1.5',
|
||||||
|
PROGRESS_TONE_CLASS[valueVariant],
|
||||||
|
)}
|
||||||
|
aria-label={
|
||||||
|
typeof item.label === 'string'
|
||||||
|
? `${item.label}: ${clampProgress(item.progress)}%`
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
{footer ? (
|
{footer ? (
|
||||||
<div className="min-w-0 max-w-full @[20rem]:hidden">{footer}</div>
|
<div className="min-w-0 max-w-full @[20rem]:hidden">{footer}</div>
|
||||||
) : null}
|
) : null}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ interface OpsDashboardProps {
|
|||||||
/** Slot after KPI (QuickActionGrid). Preview: stats-12 · card-12 */
|
/** Slot after KPI (QuickActionGrid). Preview: stats-12 · card-12 */
|
||||||
afterKpi?: ReactNode
|
afterKpi?: ReactNode
|
||||||
charts: ReactNode
|
charts: ReactNode
|
||||||
queue: ReactNode
|
queue?: ReactNode
|
||||||
queueTitle?: string
|
queueTitle?: string
|
||||||
queueDescription?: string
|
queueDescription?: string
|
||||||
headerActions?: ReactNode
|
headerActions?: ReactNode
|
||||||
@@ -31,10 +31,12 @@ function OpsDashboardSkeleton() {
|
|||||||
</header>
|
</header>
|
||||||
<KpiStatGrid cards={[]} isLoading skeletonCount={4} />
|
<KpiStatGrid cards={[]} isLoading skeletonCount={4} />
|
||||||
<div className="flex min-w-0 flex-col gap-4">
|
<div className="flex min-w-0 flex-col gap-4">
|
||||||
<Skeleton className="h-64 w-full rounded-xl" />
|
<Skeleton className="h-40 w-full rounded-xl" />
|
||||||
<Skeleton className="h-64 w-full rounded-xl" />
|
<div className="grid min-w-0 grid-cols-1 gap-4 @5xl:grid-cols-2">
|
||||||
|
<Skeleton className="h-56 w-full rounded-xl" />
|
||||||
|
<Skeleton className="h-56 w-full rounded-xl" />
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<Skeleton className="h-48 w-full rounded-xl" />
|
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -76,15 +78,17 @@ export function OpsDashboard({
|
|||||||
{charts}
|
{charts}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section aria-label={queueTitle} className="flex min-w-0 flex-col gap-4">
|
{queue ? (
|
||||||
<div className="flex min-w-0 flex-col gap-1">
|
<section aria-label={queueTitle} className="flex min-w-0 flex-col gap-4">
|
||||||
<h2 className="text-sm font-semibold tracking-tight">{queueTitle}</h2>
|
<div className="flex min-w-0 flex-col gap-1">
|
||||||
{queueDescription ? (
|
<h2 className="text-sm font-semibold tracking-tight">{queueTitle}</h2>
|
||||||
<p className="text-muted-foreground max-w-prose text-sm">{queueDescription}</p>
|
{queueDescription ? (
|
||||||
) : null}
|
<p className="text-muted-foreground max-w-prose text-sm">{queueDescription}</p>
|
||||||
</div>
|
) : null}
|
||||||
{queue}
|
</div>
|
||||||
</section>
|
{queue}
|
||||||
|
</section>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,24 +4,17 @@ import { RefreshCw } from 'lucide-react'
|
|||||||
import { useState } from 'react'
|
import { useState } from 'react'
|
||||||
|
|
||||||
import { Button } from '@evobgp/ui/components/button'
|
import { Button } from '@evobgp/ui/components/button'
|
||||||
import { Skeleton } from '@evobgp/ui/components/skeleton'
|
|
||||||
|
|
||||||
import { DashboardActivityTimeline } from '@/components/dashboard/dashboard-activity-timeline'
|
|
||||||
import { buildDashboardKpiCards } from '@/components/dashboard/dashboard-kpi-grid'
|
import { buildDashboardKpiCards } from '@/components/dashboard/dashboard-kpi-grid'
|
||||||
import { DashboardModulesGrid } from '@/components/dashboard/dashboard-modules-grid'
|
|
||||||
import { DashboardNetworkHealth } from '@/components/dashboard/dashboard-network-health'
|
import { DashboardNetworkHealth } from '@/components/dashboard/dashboard-network-health'
|
||||||
import { DashboardOperationsBreakdown } from '@/components/dashboard/dashboard-operations-breakdown'
|
import { DashboardOperationsBreakdown } from '@/components/dashboard/dashboard-operations-breakdown'
|
||||||
import { DashboardQuickLinks } from '@/components/dashboard/dashboard-quick-links'
|
import { DashboardQuickLinks } from '@/components/dashboard/dashboard-quick-links'
|
||||||
import { DashboardRecentJobsGrid } from '@/components/dashboard/dashboard-recent-jobs-grid'
|
|
||||||
import { DashboardRecentRevisionsGrid } from '@/components/dashboard/dashboard-recent-revisions-grid'
|
|
||||||
import { OpsDashboard } from '@/components/reui-kit'
|
import { OpsDashboard } from '@/components/reui-kit'
|
||||||
import { chartPanelGridClassName, dashboardMainSidebarClassName } from '@/lib/ui-surface'
|
import { chartPanelGridClassName } from '@/lib/ui-surface'
|
||||||
import {
|
import {
|
||||||
moduleNameById,
|
|
||||||
overviewJobsQueryOptions,
|
overviewJobsQueryOptions,
|
||||||
overviewModulesQueryOptions,
|
overviewModulesQueryOptions,
|
||||||
overviewPeersQueryOptions,
|
overviewPeersQueryOptions,
|
||||||
overviewRevisionsQueryOptions,
|
|
||||||
overviewSpeakersQueryOptions,
|
overviewSpeakersQueryOptions,
|
||||||
} from '@/queries/overview'
|
} from '@/queries/overview'
|
||||||
import { settingsQueryOptions } from '@/queries/settings'
|
import { settingsQueryOptions } from '@/queries/settings'
|
||||||
@@ -36,11 +29,12 @@ function parseShowQuickActions(value: unknown): boolean {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Dashboard — OpsDashboard kit (KPI → QuickActions → modules → activity/health → queue).
|
* Dashboard — KPI infographic + Quick Actions + charts (no list duplicates).
|
||||||
* Charts slot is a vertical stack: modules stay full-width; activity shares a row
|
|
||||||
* with BGP widgets only at @5xl (container), never nested 8+4 inside a 2-col parent.
|
|
||||||
* @see https://reui.io/preview/base/dashboard-1
|
* @see https://reui.io/preview/base/dashboard-1
|
||||||
* @see https://reui.io/preview/base/stats-12
|
* @see https://reui.io/preview/base/stats-12
|
||||||
|
* @see https://reui.io/preview/base/stats-4
|
||||||
|
* @see https://reui.io/preview/base/card-12
|
||||||
|
* @see https://reui.io/preview/base/chart-27
|
||||||
*/
|
*/
|
||||||
function DashboardComponent() {
|
function DashboardComponent() {
|
||||||
const [lastUpdated, setLastUpdated] = useState<Date | null>(null)
|
const [lastUpdated, setLastUpdated] = useState<Date | null>(null)
|
||||||
@@ -52,14 +46,13 @@ function DashboardComponent() {
|
|||||||
overviewModulesQueryOptions(),
|
overviewModulesQueryOptions(),
|
||||||
overviewPeersQueryOptions(),
|
overviewPeersQueryOptions(),
|
||||||
overviewSpeakersQueryOptions(),
|
overviewSpeakersQueryOptions(),
|
||||||
overviewRevisionsQueryOptions(),
|
|
||||||
overviewJobsQueryOptions(),
|
overviewJobsQueryOptions(),
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
|
||||||
const [modulesQ, peersQ, speakersQ, revisionsQ, jobsQ] = results
|
const [modulesQ, peersQ, speakersQ, jobsQ] = results
|
||||||
const initialLoading =
|
const initialLoading =
|
||||||
modulesQ.isLoading || peersQ.isLoading || speakersQ.isLoading || revisionsQ.isLoading || jobsQ.isLoading
|
modulesQ.isLoading || peersQ.isLoading || speakersQ.isLoading || jobsQ.isLoading
|
||||||
const refreshing = results.some((r) => r.isFetching && !r.isLoading)
|
const refreshing = results.some((r) => r.isFetching && !r.isLoading)
|
||||||
|
|
||||||
if (!lastUpdated && !initialLoading && results.every((r) => r.isSuccess || r.isError)) {
|
if (!lastUpdated && !initialLoading && results.every((r) => r.isSuccess || r.isError)) {
|
||||||
@@ -74,11 +67,7 @@ function DashboardComponent() {
|
|||||||
const modules = modulesQ.data?.items ?? []
|
const modules = modulesQ.data?.items ?? []
|
||||||
const peers = peersQ.data?.items ?? []
|
const peers = peersQ.data?.items ?? []
|
||||||
const speakers = speakersQ.data?.items ?? []
|
const speakers = speakersQ.data?.items ?? []
|
||||||
const revisions = revisionsQ.data?.items ?? []
|
|
||||||
const jobs = jobsQ.data?.items ?? []
|
const jobs = jobsQ.data?.items ?? []
|
||||||
const nameById = moduleNameById(modules)
|
|
||||||
|
|
||||||
const activityLoading = refreshing && jobs.length === 0 && revisions.length === 0
|
|
||||||
const kpiCards = buildDashboardKpiCards({ modules, peers, speakers, jobs })
|
const kpiCards = buildDashboardKpiCards({ modules, peers, speakers, jobs })
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -99,38 +88,18 @@ function DashboardComponent() {
|
|||||||
isLoading={initialLoading}
|
isLoading={initialLoading}
|
||||||
afterKpi={showQuickActions ? <DashboardQuickLinks /> : null}
|
afterKpi={showQuickActions ? <DashboardQuickLinks /> : null}
|
||||||
charts={
|
charts={
|
||||||
<>
|
|
||||||
<div className="min-w-0">
|
|
||||||
<DashboardModulesGrid modules={modules} isLoading={refreshing} />
|
|
||||||
</div>
|
|
||||||
<div className={dashboardMainSidebarClassName}>
|
|
||||||
<DashboardActivityTimeline
|
|
||||||
jobs={jobs}
|
|
||||||
revisions={revisions}
|
|
||||||
peers={peers}
|
|
||||||
speakers={speakers}
|
|
||||||
/>
|
|
||||||
<div className="grid min-w-0 items-start gap-4">
|
|
||||||
<DashboardNetworkHealth peers={peers} speakers={speakers} jobs={jobs} />
|
|
||||||
<DashboardOperationsBreakdown jobs={jobs} modules={modules} />
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</>
|
|
||||||
}
|
|
||||||
queueTitle="Задачи и ревизии"
|
|
||||||
queueDescription="Последние фоновые операции и история конфигураций"
|
|
||||||
queue={
|
|
||||||
<div className={chartPanelGridClassName}>
|
<div className={chartPanelGridClassName}>
|
||||||
{activityLoading ? (
|
<DashboardNetworkHealth
|
||||||
<Skeleton className="m-4 h-24 w-auto" />
|
peers={peers}
|
||||||
) : (
|
speakers={speakers}
|
||||||
<DashboardRecentJobsGrid jobs={jobs.slice(0, 8)} nameById={nameById} isLoading={refreshing} />
|
jobs={jobs}
|
||||||
)}
|
loading={refreshing && peers.length === 0 && speakers.length === 0}
|
||||||
{activityLoading ? (
|
/>
|
||||||
<Skeleton className="m-4 h-24 w-auto" />
|
<DashboardOperationsBreakdown
|
||||||
) : (
|
jobs={jobs}
|
||||||
<DashboardRecentRevisionsGrid revisions={revisions} isLoading={refreshing} />
|
modules={modules}
|
||||||
)}
|
loading={refreshing && jobs.length === 0 && modules.length === 0}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
/>
|
/>
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -168,6 +168,16 @@ http://localhost:5173,http://127.0.0.1:5173,https://ui.example.com
|
|||||||
|
|
||||||
Разрешённые заголовки включают `Authorization`, `Content-Type`, `Idempotency-Key`, `Accept`, `X-Tenant-Id` (см. `internal/httpapi/cors.go`).
|
Разрешённые заголовки включают `Authorization`, `Content-Type`, `Idempotency-Key`, `Accept`, `X-Tenant-Id` (см. `internal/httpapi/cors.go`).
|
||||||
|
|
||||||
|
## Pipeline: TTL и внешние вызовы
|
||||||
|
|
||||||
|
| Переменная | Назначение |
|
||||||
|
|------------|------------|
|
||||||
|
| `EVOBGP_ASN_CACHE_TTL_SEC` | TTL кэша объявленных префиксов RIPEstat (default 1800) |
|
||||||
|
| `EVOBGP_ASN_HOLDER_TTL_SEC` | TTL имени holder AS (default 7 суток) |
|
||||||
|
| `EVOBGP_CDN_DNS_CACHE_TTL_SEC` | TTL DNS при проверке CDN URL (default 300) |
|
||||||
|
| `EVOBGP_DOMAIN_CACHE_TTL_SEC` | TTL кэша DoH A/AAAA (default 300) |
|
||||||
|
| `EVOBGP_CDN_PARTIAL_OK` | Сохранять prior-строки skipped CDN-источников при частичном сбое |
|
||||||
|
|
||||||
## Заголовок `X-Tenant-Id` (решение: не реализован)
|
## Заголовок `X-Tenant-Id` (решение: не реализован)
|
||||||
|
|
||||||
**Решение (done):** заголовок **`X-Tenant-Id` не переключает tenant** в handlers и **не планируется** без отдельного ADR на супер-роли.
|
**Решение (done):** заголовок **`X-Tenant-Id` не переключает tenant** в handlers и **не планируется** без отдельного ADR на супер-роли.
|
||||||
|
|||||||
+12
-1
@@ -39,7 +39,7 @@
|
|||||||
| `config` | Переменные окружения `EVOBGP_*`. |
|
| `config` | Переменные окружения `EVOBGP_*`. |
|
||||||
| `observability` | Метрики Prometheus, HTTP middleware. |
|
| `observability` | Метрики Prometheus, HTTP middleware. |
|
||||||
| `broker` | Опциональный `EVOBGP_BROKER_URL` для будущей шины; сейчас задачи только in-process (`jobs.Registry`), пакет лишь логирует факт настройки URL. |
|
| `broker` | Опциональный `EVOBGP_BROKER_URL` для будущей шины; сейчас задачи только in-process (`jobs.Registry`), пакет лишь логирует факт настройки URL. |
|
||||||
| `pipeline` | Ingest+render в одном шаге для `module_refresh`: выборка префиксов (CDN/AS/IP/пустые DOMAINS), `CreateRenderRevision`, превью BIRD через `birdfmt`. |
|
| `pipeline` | Ingest+render для `module_refresh`: CDN/AS/IP/DOMAINS → `module_prefix_snapshot` (batch `COPY`, per-module lock) → агрегация CIDR O(n log n) → `CreateRenderRevision`. Fast-path снапшота — `module.input_hash`; DoH — TTL-кэш `domain_resolve_cache`; scheduler — jitter границ интервала. |
|
||||||
| `nodedispatch` | Panel→Node HTTP wake-up (`POST /v1/agent/sync`) после `deploy_apply`. |
|
| `nodedispatch` | Panel→Node HTTP wake-up (`POST /v1/agent/sync`) после `deploy_apply`. |
|
||||||
| `agentserver` | HTTP API на реплике (`serve`): sync + health для Traefik; опционально firewall failover (`/v1/firewall/*`). |
|
| `agentserver` | HTTP API на реплике (`serve`): sync + health для Traefik; опционально firewall failover (`/v1/firewall/*`). |
|
||||||
| `firewall` | Вычисление policy block/accept → плоский CIDR blocklist. |
|
| `firewall` | Вычисление policy block/accept → плоский CIDR blocklist. |
|
||||||
@@ -98,6 +98,17 @@ flowchart LR
|
|||||||
|
|
||||||
Профиль Compose **`microvps-full`** добавляет к этому стеку **Web UI** (nginx → `evobgp-all`), **NATS** и **Prometheus** без отдельных контейнеров воркеров (функционально то же, что отдельные `scheduler`/`ingest`/… в reference). Запуск и лимиты под ~1 ГиБ RAM — в [quickstart.md](quickstart.md).
|
Профиль Compose **`microvps-full`** добавляет к этому стеку **Web UI** (nginx → `evobgp-all`), **NATS** и **Prometheus** без отдельных контейнеров воркеров (функционально то же, что отдельные `scheduler`/`ingest`/… в reference). Запуск и лимиты под ~1 ГиБ RAM — в [quickstart.md](quickstart.md).
|
||||||
|
|
||||||
|
## Поток pipeline (ingest → render)
|
||||||
|
|
||||||
|
1. **Scheduler** (`evobgp-scheduler` / in-process в `evobgp-all`) ставит `tenant_refresh`, если `ModuleDueForScheduler`: граница окна `refresh_interval_sec` со **сдвигом `fnv32(module.ID) % interval`**, чтобы модули с одним интервалом не били внешние API одновременно.
|
||||||
|
2. **Ingest** (`RefreshModuleIngest`):
|
||||||
|
- `AS_PREFIXES` — RIPEstat prefixes + holder параллельно, кэш `asn_prefix_cache`; дедуп строк по `prefix + community`.
|
||||||
|
- `CDN_CIDRS` — единый `fetchCDNSourceRows` (conditional GET); prefetch уважает `RefreshIntervalSec`; merge снапшота под `LockModuleSnapshot` (пропущенные по ошибке источники сохраняют prior-строки при `EVOBGP_CDN_PARTIAL_OK`).
|
||||||
|
- `DOMAINS` — DoH A+AAAA параллельно; попадания в `domain_resolve_cache` с TTL `EVOBGP_DOMAIN_CACHE_TTL_SEC` (default 300).
|
||||||
|
- `IP_RANGES` — напрямую из записей модуля.
|
||||||
|
3. Снапшот пишется **batch** (`pgx.CopyFrom` в PostgreSQL). Совпадение `module.input_hash` со снапшотом — O(1) пропуск повторного ingest при render; CRUD entries обнуляет hash.
|
||||||
|
4. **Render** (`RenderTenantRevision`): `smartAggregatePrefixRows` (стек-схлопывание O(n log n), IPv6 без `math/big`) → ревизия, если набор префиксов изменился.
|
||||||
|
|
||||||
## Поток: ревизия и бандл для ноды
|
## Поток: ревизия и бандл для ноды
|
||||||
|
|
||||||
1. Оператор (роль `operator` или выше по политике) изменяет модули и запускает цепочку, приводящую к новой **ревизии** (часть шагов может быть асинхронной через jobs — см. OpenAPI).
|
1. Оператор (роль `operator` или выше по политике) изменяет модули и запускает цепочку, приводящую к новой **ревизии** (часть шагов может быть асинхронной через jobs — см. OpenAPI).
|
||||||
|
|||||||
@@ -12,6 +12,9 @@
|
|||||||
- `EVOBGP_CORS_ORIGINS` — явный whitelist origin веб-панели.
|
- `EVOBGP_CORS_ORIGINS` — явный whitelist origin веб-панели.
|
||||||
- `EVOBGP_STALE_ON_UPSTREAM_ERROR=1` (по умолчанию) — stale snapshot при сбоях CDN/ASN/DoH.
|
- `EVOBGP_STALE_ON_UPSTREAM_ERROR=1` (по умолчанию) — stale snapshot при сбоях CDN/ASN/DoH.
|
||||||
- Опционально `EVOBGP_CDN_PARTIAL_OK=1` — при сбое одного CDN source без stale cache продолжать refresh остальных (иначе fail модуля).
|
- Опционально `EVOBGP_CDN_PARTIAL_OK=1` — при сбое одного CDN source без stale cache продолжать refresh остальных (иначе fail модуля).
|
||||||
|
- `EVOBGP_ASN_HOLDER_TTL_SEC` — TTL имени holder AS (default 7 суток); prefixes кэшируются отдельно (`EVOBGP_ASN_CACHE_TTL_SEC`, default 1800).
|
||||||
|
- `EVOBGP_CDN_DNS_CACHE_TTL_SEC` — TTL кэша DNS при SSRF-проверке CDN URL (default 300).
|
||||||
|
- `EVOBGP_DOMAIN_CACHE_TTL_SEC` — TTL кэша DoH A/AAAA (`domain_resolve_cache`, default 300).
|
||||||
|
|
||||||
## Рекомендуется
|
## Рекомендуется
|
||||||
|
|
||||||
|
|||||||
@@ -10,9 +10,7 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"evobgp/internal/httpclient"
|
"evobgp/internal/httpclient"
|
||||||
)
|
)
|
||||||
@@ -137,22 +135,3 @@ func truncateForErr(b []byte, n int) string {
|
|||||||
}
|
}
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
// PolitePause is a short delay between upstream ASN lookups (same refresh).
|
|
||||||
func PolitePause() {
|
|
||||||
d := 150 * time.Millisecond
|
|
||||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE_PAUSE_MS")); s != "" {
|
|
||||||
if ms, err := parsePositiveInt(s); err == nil && ms > 0 {
|
|
||||||
d = time.Duration(ms) * time.Millisecond
|
|
||||||
}
|
|
||||||
}
|
|
||||||
time.Sleep(d)
|
|
||||||
}
|
|
||||||
|
|
||||||
func parsePositiveInt(s string) (int, error) {
|
|
||||||
n, err := strconv.Atoi(s)
|
|
||||||
if err != nil || n <= 0 {
|
|
||||||
return 0, fmt.Errorf("invalid")
|
|
||||||
}
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,173 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/netip"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Prefix collapse replaces the former prune+merge fixed-point loop (O(n²) per pass,
|
||||||
|
// multiple passes) with a single sort + linear stack pass: O(n log n) overall.
|
||||||
|
//
|
||||||
|
// Classic trie-collapse without building a trie:
|
||||||
|
// 1. Sort prefixes by (address, mask len ascending).
|
||||||
|
// 2. Walk left to right keeping a stack of "open" prefixes:
|
||||||
|
// - while the stack top contains the new prefix -> the top covers it, drop the new one (prune);
|
||||||
|
// - else, while the stack top is a sibling of the new prefix (same mask, XOR of the
|
||||||
|
// network bit equals the parent block) and merging is allowed for that mask length
|
||||||
|
// -> pop the sibling, replace the new prefix with the parent and re-check;
|
||||||
|
// - otherwise push the new prefix.
|
||||||
|
//
|
||||||
|
// A parent absorbs its sibling children only when both halves are present, which matches
|
||||||
|
// the previous merge-to-fixed-point semantics, including the guard that forbids merging
|
||||||
|
// above a floor mask (IPv4 /8, IPv6 /16).
|
||||||
|
|
||||||
|
type collapseItem struct {
|
||||||
|
row store.PrefixRow
|
||||||
|
pfx netip.Prefix
|
||||||
|
}
|
||||||
|
|
||||||
|
// collapsePrefixGroup collapses one (community, source) group of same-family prefixes.
|
||||||
|
// All rows must be masked; family and floor are enforced by minBits.
|
||||||
|
func collapsePrefixGroup(rows []store.PrefixRow, is4 bool) []store.PrefixRow {
|
||||||
|
if len(rows) <= 1 {
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
items := make([]collapseItem, 0, len(rows))
|
||||||
|
seen := make(map[string]struct{}, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
if _, dup := seen[row.Prefix]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[row.Prefix] = struct{}{}
|
||||||
|
pfx, err := netip.ParsePrefix(row.Prefix)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pfx = pfx.Masked()
|
||||||
|
if is4 != pfx.Addr().Is4() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
items = append(items, collapseItem{row: row, pfx: pfx})
|
||||||
|
}
|
||||||
|
if len(items) <= 1 {
|
||||||
|
out := make([]store.PrefixRow, 0, len(items))
|
||||||
|
for _, it := range items {
|
||||||
|
out = append(out, it.row)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by (address, mask len): a parent always sorts before its children, and a
|
||||||
|
// shorter sibling sorts before a longer one within the same parent block.
|
||||||
|
sort.Slice(items, func(i, j int) bool {
|
||||||
|
a, b := items[i].pfx, items[j].pfx
|
||||||
|
if a.Addr() != b.Addr() {
|
||||||
|
return lessAddr(a.Addr(), b.Addr())
|
||||||
|
}
|
||||||
|
return a.Bits() < b.Bits()
|
||||||
|
})
|
||||||
|
|
||||||
|
minBits := 8
|
||||||
|
if !is4 {
|
||||||
|
minBits = 16
|
||||||
|
}
|
||||||
|
|
||||||
|
stack := make([]collapseItem, 0, len(items))
|
||||||
|
for _, it := range items {
|
||||||
|
cur := it
|
||||||
|
dropped := false
|
||||||
|
for len(stack) > 0 {
|
||||||
|
top := stack[len(stack)-1]
|
||||||
|
if top.pfx.Contains(cur.pfx.Addr()) && top.pfx.Bits() <= cur.pfx.Bits() {
|
||||||
|
// Covered by an existing prefix: drop (prune).
|
||||||
|
dropped = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if cur.pfx.Bits() == top.pfx.Bits() && cur.pfx.Bits() > minBits && areSiblings(top.pfx, cur.pfx) {
|
||||||
|
// Merge siblings into the parent (parent keeps the lower sibling's attributes),
|
||||||
|
// then re-check the parent against the new stack top.
|
||||||
|
stack = stack[:len(stack)-1]
|
||||||
|
parentBits := cur.pfx.Bits() - 1
|
||||||
|
parent := netip.PrefixFrom(maskAddr(cur.pfx.Addr(), parentBits, is4), parentBits).Masked()
|
||||||
|
cur = collapseItem{row: top.row, pfx: parent}
|
||||||
|
cur.row.Prefix = parent.String()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !dropped {
|
||||||
|
stack = append(stack, cur)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]store.PrefixRow, 0, len(stack))
|
||||||
|
for _, it := range stack {
|
||||||
|
out = append(out, it.row)
|
||||||
|
}
|
||||||
|
sortPrefixRows(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// areSiblings reports whether two same-length prefixes combine into their common parent.
|
||||||
|
func areSiblings(a, b netip.Prefix) bool {
|
||||||
|
if a.Bits() != b.Bits() || a.Bits() == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
parentBits := a.Bits() - 1
|
||||||
|
pa := maskAddr(a.Addr(), parentBits, a.Addr().Is4())
|
||||||
|
pb := maskAddr(b.Addr(), parentBits, b.Addr().Is4())
|
||||||
|
return pa == pb
|
||||||
|
}
|
||||||
|
|
||||||
|
// maskAddr clears the host bits below prefixLen (IPv4: 32-bit space; IPv6: 128-bit).
|
||||||
|
func maskAddr(a netip.Addr, prefixLen int, is4 bool) netip.Addr {
|
||||||
|
if is4 {
|
||||||
|
v := uint32FromIPv4(a)
|
||||||
|
if prefixLen <= 0 {
|
||||||
|
v = 0
|
||||||
|
} else if prefixLen < 32 {
|
||||||
|
v &= ^(uint32(1)<<(32-prefixLen) - 1)
|
||||||
|
}
|
||||||
|
return u32ToIPv4(v)
|
||||||
|
}
|
||||||
|
b := a.As16()
|
||||||
|
hostBits := 128 - prefixLen
|
||||||
|
fullBytes := hostBits / 8
|
||||||
|
for i := 15; i > 15-fullBytes; i-- {
|
||||||
|
b[i] = 0
|
||||||
|
}
|
||||||
|
if rem := hostBits % 8; rem > 0 {
|
||||||
|
idx := 15 - fullBytes
|
||||||
|
if idx >= 0 && idx < 16 {
|
||||||
|
b[idx] &= byte(0xFF << rem)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return netip.AddrFrom16(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
func uint32FromIPv4(a netip.Addr) uint32 {
|
||||||
|
o := a.As4()
|
||||||
|
return uint32(o[0])<<24 | uint32(o[1])<<16 | uint32(o[2])<<8 | uint32(o[3])
|
||||||
|
}
|
||||||
|
|
||||||
|
func u32ToIPv4(v uint32) netip.Addr {
|
||||||
|
return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
|
||||||
|
}
|
||||||
|
|
||||||
|
func lessAddr(a, b netip.Addr) bool {
|
||||||
|
if a.Is4() != b.Is4() {
|
||||||
|
return a.Is4()
|
||||||
|
}
|
||||||
|
if a.Is4() {
|
||||||
|
return uint32FromIPv4(a) < uint32FromIPv4(b)
|
||||||
|
}
|
||||||
|
a16, b16 := a.As16(), b.As16()
|
||||||
|
for i := 0; i < 16; i++ {
|
||||||
|
if a16[i] != b16[i] {
|
||||||
|
return a16[i] < b16[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math/big"
|
||||||
|
"math/rand"
|
||||||
|
"net/netip"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The functions below are a verbatim copy of the pre-2.1 prune+merge loop
|
||||||
|
// (from git HEAD internal/pipeline/refresh.go). They exist only as an
|
||||||
|
// equivalence oracle for collapsePrefixGroup.
|
||||||
|
|
||||||
|
func legacyAggregateCIDRGroup(rows []store.PrefixRow, mergeFn func(map[string]store.PrefixRow) bool) []store.PrefixRow {
|
||||||
|
if len(rows) <= 1 {
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
set := make(map[string]store.PrefixRow, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
set[row.Prefix] = row
|
||||||
|
}
|
||||||
|
legacyPruneCoveredPrefixes(set)
|
||||||
|
for {
|
||||||
|
if !mergeFn(set) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
legacyPruneCoveredPrefixes(set)
|
||||||
|
}
|
||||||
|
out := make([]store.PrefixRow, 0, len(set))
|
||||||
|
for _, row := range set {
|
||||||
|
out = append(out, row)
|
||||||
|
}
|
||||||
|
sortPrefixRows(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func legacyPruneCoveredPrefixes(set map[string]store.PrefixRow) {
|
||||||
|
type item struct {
|
||||||
|
key string
|
||||||
|
pfx netip.Prefix
|
||||||
|
bits int
|
||||||
|
}
|
||||||
|
items := make([]item, 0, len(set))
|
||||||
|
for k := range set {
|
||||||
|
p, err := netip.ParsePrefix(k)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
items = append(items, item{key: k, pfx: p, bits: p.Bits()})
|
||||||
|
}
|
||||||
|
sort.Slice(items, func(i, j int) bool {
|
||||||
|
if items[i].bits != items[j].bits {
|
||||||
|
return items[i].bits < items[j].bits
|
||||||
|
}
|
||||||
|
return items[i].key < items[j].key
|
||||||
|
})
|
||||||
|
for i := 0; i < len(items); i++ {
|
||||||
|
for j := i + 1; j < len(items); j++ {
|
||||||
|
if items[j].bits <= items[i].bits {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if items[i].pfx.Contains(items[j].pfx.Addr()) {
|
||||||
|
delete(set, items[j].key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func legacyMergeSiblingPrefixesIPv4(set map[string]store.PrefixRow) bool {
|
||||||
|
merged := false
|
||||||
|
seen := make(map[string]struct{}, len(set))
|
||||||
|
for key, row := range set {
|
||||||
|
if _, done := seen[key]; done {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pfx, err := netip.ParsePrefix(key)
|
||||||
|
if err != nil || !pfx.Addr().Is4() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bits := pfx.Bits()
|
||||||
|
if bits <= 8 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
netNum := ipv4PrefixNetworkLegacy(pfx)
|
||||||
|
blockSize := uint32(1) << (32 - bits)
|
||||||
|
siblingNet := netNum ^ blockSize
|
||||||
|
siblingPfx := netip.PrefixFrom(u32ToIPv4(siblingNet), bits).Masked().String()
|
||||||
|
if _, ok := set[siblingPfx]; !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
parentBits := bits - 1
|
||||||
|
parentBlock := uint32(1) << (32 - parentBits)
|
||||||
|
parentNet := netNum & ^(parentBlock - 1)
|
||||||
|
parentPfx := netip.PrefixFrom(u32ToIPv4(parentNet), parentBits).Masked().String()
|
||||||
|
delete(set, key)
|
||||||
|
delete(set, siblingPfx)
|
||||||
|
parentRow := row
|
||||||
|
parentRow.Prefix = parentPfx
|
||||||
|
set[parentPfx] = parentRow
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
seen[siblingPfx] = struct{}{}
|
||||||
|
merged = true
|
||||||
|
}
|
||||||
|
return merged
|
||||||
|
}
|
||||||
|
|
||||||
|
func ipv4PrefixNetworkLegacy(p netip.Prefix) uint32 {
|
||||||
|
a := p.Masked().Addr().As4()
|
||||||
|
return uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3])
|
||||||
|
}
|
||||||
|
|
||||||
|
func legacyMergeSiblingPrefixesIPv6(set map[string]store.PrefixRow) bool {
|
||||||
|
merged := false
|
||||||
|
seen := make(map[string]struct{}, len(set))
|
||||||
|
for key, row := range set {
|
||||||
|
if _, done := seen[key]; done {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pfx, err := netip.ParsePrefix(key)
|
||||||
|
if err != nil || !pfx.Addr().Is6() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bits := pfx.Bits()
|
||||||
|
if bits <= 16 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
netNum := ipv6PrefixNetworkLegacy(pfx)
|
||||||
|
blockSize := new(big.Int).Lsh(big.NewInt(1), uint(128-bits))
|
||||||
|
siblingNet := new(big.Int).Xor(netNum, blockSize)
|
||||||
|
siblingPfx := ipv6PrefixFromBigIntLegacy(siblingNet, bits).String()
|
||||||
|
if _, ok := set[siblingPfx]; !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
parentBits := bits - 1
|
||||||
|
parentBlock := new(big.Int).Lsh(big.NewInt(1), uint(128-parentBits))
|
||||||
|
mask := new(big.Int).Sub(parentBlock, big.NewInt(1))
|
||||||
|
mask.Not(mask)
|
||||||
|
parentNet := new(big.Int).And(netNum, mask)
|
||||||
|
parentPfx := ipv6PrefixFromBigIntLegacy(parentNet, parentBits).String()
|
||||||
|
delete(set, key)
|
||||||
|
delete(set, siblingPfx)
|
||||||
|
parentRow := row
|
||||||
|
parentRow.Prefix = parentPfx
|
||||||
|
set[parentPfx] = parentRow
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
seen[siblingPfx] = struct{}{}
|
||||||
|
merged = true
|
||||||
|
}
|
||||||
|
return merged
|
||||||
|
}
|
||||||
|
|
||||||
|
func ipv6PrefixNetworkLegacy(p netip.Prefix) *big.Int {
|
||||||
|
a := p.Masked().Addr().As16()
|
||||||
|
n := new(big.Int)
|
||||||
|
n.SetBytes(a[:])
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func ipv6PrefixFromBigIntLegacy(n *big.Int, bits int) netip.Prefix {
|
||||||
|
b := n.Bytes()
|
||||||
|
var a [16]byte
|
||||||
|
copy(a[16-len(b):], b)
|
||||||
|
return netip.PrefixFrom(netip.AddrFrom16(a), bits).Masked()
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeForCompare(rows []store.PrefixRow) []string {
|
||||||
|
type line struct{ p, c, s string }
|
||||||
|
lines := make([]line, 0, len(rows))
|
||||||
|
for _, r := range rows {
|
||||||
|
p := strings.TrimSpace(r.Prefix)
|
||||||
|
if p != "" {
|
||||||
|
if pfx, err := netip.ParsePrefix(p); err == nil {
|
||||||
|
p = pfx.Masked().String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines = append(lines, line{p, prefixRowCommunity(r), r.Source})
|
||||||
|
}
|
||||||
|
sort.Slice(lines, func(i, j int) bool {
|
||||||
|
if lines[i].p != lines[j].p {
|
||||||
|
return lines[i].p < lines[j].p
|
||||||
|
}
|
||||||
|
if lines[i].c != lines[j].c {
|
||||||
|
return lines[i].c < lines[j].c
|
||||||
|
}
|
||||||
|
return lines[i].s < lines[j].s
|
||||||
|
})
|
||||||
|
out := make([]string, 0, len(lines))
|
||||||
|
for _, l := range lines {
|
||||||
|
out = append(out, fmt.Sprintf("%s|%s|%s", l.p, l.c, l.s))
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollapsePrefixGroup_EquivalenceWithLegacy(t *testing.T) {
|
||||||
|
r := rand.New(rand.NewSource(7))
|
||||||
|
for iter := 0; iter < 200; iter++ {
|
||||||
|
n := 1 + r.Intn(60)
|
||||||
|
v4rows := make([]store.PrefixRow, 0, n)
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
addr := netip.AddrFrom4([4]byte{203, byte(r.Intn(4)), byte(r.Intn(256)), byte(r.Intn(256))})
|
||||||
|
bits := 16 + r.Intn(9)
|
||||||
|
pfx := netip.PrefixFrom(addr, bits).Masked()
|
||||||
|
v4rows = append(v4rows, store.PrefixRow{Prefix: pfx.String(), Source: "ip_range"})
|
||||||
|
}
|
||||||
|
legacy := legacyAggregateCIDRGroup(append([]store.PrefixRow(nil), v4rows...), legacyMergeSiblingPrefixesIPv4)
|
||||||
|
got := collapsePrefixGroup(append([]store.PrefixRow(nil), v4rows...), true)
|
||||||
|
if fmt.Sprint(normalizeForCompare(legacy)) != fmt.Sprint(normalizeForCompare(got)) {
|
||||||
|
t.Fatalf("iter %d mismatch:\nlegacy=%v\ngot =%v", iter, normalizeForCompare(legacy), normalizeForCompare(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollapsePrefixGroup_EquivalenceWithLegacyIPv6(t *testing.T) {
|
||||||
|
r := rand.New(rand.NewSource(11))
|
||||||
|
for iter := 0; iter < 200; iter++ {
|
||||||
|
n := 1 + r.Intn(60)
|
||||||
|
rows := make([]store.PrefixRow, 0, n)
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
var a [16]byte
|
||||||
|
a[0], a[1] = 0x20, 0x01
|
||||||
|
a[2], a[3] = 0x0d, 0xb8
|
||||||
|
a[4] = byte(r.Intn(2))
|
||||||
|
a[5] = byte(r.Intn(256))
|
||||||
|
a[6] = byte(r.Intn(256))
|
||||||
|
addr := netip.AddrFrom16(a)
|
||||||
|
bits := 32 + r.Intn(17)
|
||||||
|
pfx := netip.PrefixFrom(addr, bits).Masked()
|
||||||
|
rows = append(rows, store.PrefixRow{Prefix: pfx.String(), Source: "ip_range"})
|
||||||
|
}
|
||||||
|
legacy := legacyAggregateCIDRGroup(append([]store.PrefixRow(nil), rows...), legacyMergeSiblingPrefixesIPv6)
|
||||||
|
got := collapsePrefixGroup(append([]store.PrefixRow(nil), rows...), false)
|
||||||
|
if fmt.Sprint(normalizeForCompare(legacy)) != fmt.Sprint(normalizeForCompare(got)) {
|
||||||
|
t.Fatalf("iter %d mismatch:\nlegacy=%v\ngot =%v", iter, normalizeForCompare(legacy), normalizeForCompare(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollapsePrefixGroup_CoversAndSiblingChain(t *testing.T) {
|
||||||
|
rows := []store.PrefixRow{
|
||||||
|
{Prefix: "10.0.0.0/16", Source: "ip_range"},
|
||||||
|
{Prefix: "10.0.0.0/24", Source: "ip_range"},
|
||||||
|
{Prefix: "10.0.1.0/24", Source: "ip_range"},
|
||||||
|
{Prefix: "10.0.2.0/24", Source: "ip_range"},
|
||||||
|
{Prefix: "10.0.3.0/24", Source: "ip_range"},
|
||||||
|
{Prefix: "10.1.0.0/24", Source: "ip_range"},
|
||||||
|
}
|
||||||
|
got := collapsePrefixGroup(rows, true)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("want 2 rows (/16 + /24), got %d: %+v", len(got), got)
|
||||||
|
}
|
||||||
|
set := map[string]bool{}
|
||||||
|
for _, r := range got {
|
||||||
|
set[r.Prefix] = true
|
||||||
|
}
|
||||||
|
if !set["10.0.0.0/16"] || !set["10.1.0.0/24"] {
|
||||||
|
t.Fatalf("unexpected rows: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollapsePrefixGroup_RespectsMinBitsFloor(t *testing.T) {
|
||||||
|
rows := []store.PrefixRow{
|
||||||
|
{Prefix: "10.0.0.0/8", Source: "ip_range"},
|
||||||
|
{Prefix: "11.0.0.0/8", Source: "ip_range"},
|
||||||
|
}
|
||||||
|
got := collapsePrefixGroup(rows, true)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("floor must prevent /8+/8 -> /7, got %+v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows9 := []store.PrefixRow{
|
||||||
|
{Prefix: "10.0.0.0/9", Source: "ip_range"},
|
||||||
|
{Prefix: "10.128.0.0/9", Source: "ip_range"},
|
||||||
|
}
|
||||||
|
got9 := collapsePrefixGroup(rows9, true)
|
||||||
|
if len(got9) != 1 || got9[0].Prefix != "10.0.0.0/8" {
|
||||||
|
t.Fatalf("expected /9+/9 -> /8, got %+v", got9)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollapsePrefixGroup_PrunesCovered(t *testing.T) {
|
||||||
|
rows := []store.PrefixRow{
|
||||||
|
{Prefix: "10.0.0.0/16", Source: "ip_range"},
|
||||||
|
{Prefix: "10.0.0.0/24", Source: "ip_range"},
|
||||||
|
}
|
||||||
|
got := collapsePrefixGroup(rows, true)
|
||||||
|
if len(got) != 1 || got[0].Prefix != "10.0.0.0/16" {
|
||||||
|
t.Fatalf("want covered prune to /16, got %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,28 +24,43 @@ func asnCacheTTL() time.Duration {
|
|||||||
return time.Duration(sec) * time.Second
|
return time.Duration(sec) * time.Second
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache).
|
// asnHolderTTL is how long a holder name stays authoritative between refreshes;
|
||||||
func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) {
|
// holder text changes rarely, so it survives short prefix-cache TTLs.
|
||||||
ttl := asnCacheTTL()
|
func asnHolderTTL() time.Duration {
|
||||||
if st != nil {
|
sec := 7 * 24 * 3600
|
||||||
if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil && time.Since(ent.FetchedAt) < ttl {
|
if s := strings.TrimSpace(os.Getenv("EVOBGP_ASN_HOLDER_TTL_SEC")); s != "" {
|
||||||
out := make([]netip.Prefix, 0, len(ent.Prefixes))
|
if v, err := strconv.Atoi(s); err == nil && v > 0 {
|
||||||
for _, p := range ent.Prefixes {
|
sec = v
|
||||||
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
|
|
||||||
if perr != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
out = append(out, pfx.Masked())
|
|
||||||
}
|
|
||||||
return out, ent.Holder, nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return time.Duration(sec) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveASNForEntry fetches prefixes and holder with shared TTL cache (asn_prefix_cache).
|
||||||
|
// The holder lookup starts concurrently with the prefix fetch (one RTT instead of two);
|
||||||
|
// a holder failure is non-fatal — the previously cached holder name is kept.
|
||||||
|
func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client, asn int64) ([]netip.Prefix, string, error) {
|
||||||
|
ttl := asnCacheTTL()
|
||||||
|
var prevHolder string
|
||||||
|
if st != nil {
|
||||||
|
if ent, ok, err := st.GetASNPrefixCache(asn); err == nil && ok && ent != nil {
|
||||||
|
prevHolder = ent.Holder
|
||||||
|
if time.Since(ent.FetchedAt) < ttl {
|
||||||
|
return parseASNCachePrefixes(ent.Prefixes), ent.Holder, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
holderCh := startASNHolderFetch(ctx, hc, st, asn, prevHolder)
|
||||||
|
|
||||||
pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, asn)
|
pfxs, err := asnresolve.AnnouncedPrefixes(ctx, hc, asn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
holder := <-holderCh // drain to avoid leaking the goroutine's channel send
|
||||||
|
_ = holder
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
asnresolve.PolitePause()
|
|
||||||
holder, _ := asnresolve.ASHolderName(ctx, hc, asn)
|
holder := <-holderCh
|
||||||
if st != nil {
|
if st != nil {
|
||||||
strs := make([]string, len(pfxs))
|
strs := make([]string, len(pfxs))
|
||||||
for i, p := range pfxs {
|
for i, p := range pfxs {
|
||||||
@@ -57,3 +72,55 @@ func resolveASNForEntry(ctx context.Context, st store.Backend, hc *http.Client,
|
|||||||
}
|
}
|
||||||
return pfxs, holder, nil
|
return pfxs, holder, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// startASNHolderFetch launches the holder lookup concurrently. The returned buffered
|
||||||
|
// channel always yields exactly one value, so callers may abandon it without leaking.
|
||||||
|
func startASNHolderFetch(ctx context.Context, hc *http.Client, st store.Backend, asn int64, prevHolder string) <-chan string {
|
||||||
|
ch := make(chan string, 1)
|
||||||
|
go func() {
|
||||||
|
if prevHolder != "" && holderStillFresh(st, asn, prevHolder) {
|
||||||
|
ch <- prevHolder
|
||||||
|
return
|
||||||
|
}
|
||||||
|
holder, err := asnresolve.ASHolderName(ctx, hc, asn)
|
||||||
|
if err != nil || strings.TrimSpace(holder) == "" {
|
||||||
|
ch <- prevHolder
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ch <- strings.TrimSpace(holder)
|
||||||
|
}()
|
||||||
|
return ch
|
||||||
|
}
|
||||||
|
|
||||||
|
// holderStillFresh reports whether the cached holder name is within its own (long) TTL.
|
||||||
|
func holderStillFresh(st store.Backend, asn int64, prevHolder string) bool {
|
||||||
|
if st == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
ent, ok, err := st.GetASNPrefixCache(asn)
|
||||||
|
return err == nil && ok && ent != nil && ent.Holder == prevHolder && time.Since(ent.FetchedAt) < asnHolderTTL()
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseASNCachePrefixes(raw []string) []netip.Prefix {
|
||||||
|
out := make([]netip.Prefix, 0, len(raw))
|
||||||
|
for _, p := range raw {
|
||||||
|
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
|
||||||
|
if perr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, pfx.Masked())
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseCachedASNCachedPrefixes(raw []string) []netip.Prefix {
|
||||||
|
out := make([]netip.Prefix, 0, len(raw))
|
||||||
|
for _, p := range raw {
|
||||||
|
pfx, perr := netip.ParsePrefix(strings.TrimSpace(p))
|
||||||
|
if perr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, pfx.Masked())
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@@ -51,13 +51,92 @@ func TestResolveASNForEntry_UsesTTLCache(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveASNForEntry_HolderFailureIsNonFatal(t *testing.T) {
|
||||||
|
m := store.NewMemory()
|
||||||
|
t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "1")
|
||||||
|
|
||||||
|
var holderCalls atomic.Int32
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if strings.Contains(r.URL.Path, "/announced") {
|
||||||
|
_, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
holderCalls.Add(1)
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced")
|
||||||
|
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview")
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
hc := srv.Client()
|
||||||
|
pfxs, holder, err := resolveASNForEntry(ctx, m, hc, 64512)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("holder failure must not fail ingest: %v", err)
|
||||||
|
}
|
||||||
|
if len(pfxs) != 1 {
|
||||||
|
t.Fatalf("expected 1 prefix, got %+v", pfxs)
|
||||||
|
}
|
||||||
|
if holder != "" {
|
||||||
|
t.Fatalf("expected empty holder on upstream failure, got %q", holder)
|
||||||
|
}
|
||||||
|
if holderCalls.Load() == 0 {
|
||||||
|
t.Fatal("expected holder endpoint to be attempted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveASNForEntry_KeepsPreviousHolderOnFailure(t *testing.T) {
|
||||||
|
m := store.NewMemory()
|
||||||
|
t.Setenv("EVOBGP_ASN_CACHE_TTL_SEC", "1")
|
||||||
|
t.Setenv("EVOBGP_ASN_HOLDER_TTL_SEC", "3600")
|
||||||
|
|
||||||
|
holderOK := atomic.Bool{}
|
||||||
|
holderOK.Store(true)
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if strings.Contains(r.URL.Path, "/announced") {
|
||||||
|
_, _ = w.Write([]byte(`{"status":"ok","data":{"prefixes":[{"prefix":"203.0.113.0/24"}]}}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if holderOK.Load() {
|
||||||
|
_, _ = w.Write([]byte(`{"status":"ok","data":{"holder":"Good AS"}}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
t.Setenv("EVOBGP_RIPESTAT_ANNOUNCED_PREFIXES_URL", srv.URL+"/announced")
|
||||||
|
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", srv.URL+"/overview")
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
hc := srv.Client()
|
||||||
|
if _, h, err := resolveASNForEntry(ctx, m, hc, 64512); err != nil || h != "Good AS" {
|
||||||
|
t.Fatalf("first resolve: holder=%q err=%v", h, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prefix cache expired; holder endpoint now broken — previous holder must survive.
|
||||||
|
holderOK.Store(false)
|
||||||
|
time.Sleep(1100 * time.Millisecond)
|
||||||
|
pfxs, h, err := resolveASNForEntry(ctx, m, hc, 64512)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(pfxs) != 1 {
|
||||||
|
t.Fatalf("expected 1 prefix, got %+v", pfxs)
|
||||||
|
}
|
||||||
|
if h != "Good AS" {
|
||||||
|
t.Fatalf("expected previous holder preserved, got %q", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestModuleDueForScheduler_BucketRollover(t *testing.T) {
|
func TestModuleDueForScheduler_BucketRollover(t *testing.T) {
|
||||||
mod := &store.Module{Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300}
|
mod := &store.Module{ID: "mod-jitter", Enabled: true, Type: "CDN_CIDRS", RefreshIntervalSec: 300}
|
||||||
boundary := time.Unix(300, 0)
|
win := int64(300)
|
||||||
|
off := moduleSchedulerOffset(mod.ID, win)
|
||||||
|
boundary := time.Unix(win+off, 0)
|
||||||
if !ModuleDueForScheduler(mod, boundary) {
|
if !ModuleDueForScheduler(mod, boundary) {
|
||||||
t.Fatal("expected due when refresh bucket rolls")
|
t.Fatal("expected due when refresh bucket rolls")
|
||||||
}
|
}
|
||||||
mid := time.Unix(330, 0)
|
mid := time.Unix(win+off+SchedulerTickSec, 0)
|
||||||
if ModuleDueForScheduler(mod, mid) {
|
if ModuleDueForScheduler(mod, mid) {
|
||||||
t.Fatal("expected not due within same bucket")
|
t.Fatal("expected not due within same bucket")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
|
"net/netip"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// benchPrefixRows generates seed-stable pseudo-random prefixes for aggregation benchmarks.
|
||||||
|
// v4 blocks are carved from TEST-NET-style ranges; v6 from 2001:db8::/32.
|
||||||
|
func benchPrefixRows(n int, v6Share float64) []store.PrefixRow {
|
||||||
|
r := rand.New(rand.NewSource(42))
|
||||||
|
rows := make([]store.PrefixRow, 0, n)
|
||||||
|
seen := make(map[string]struct{}, n)
|
||||||
|
for len(rows) < n {
|
||||||
|
var pfx netip.Prefix
|
||||||
|
if r.Float64() < v6Share {
|
||||||
|
var a [16]byte
|
||||||
|
a[0], a[1] = 0x20, 0x01
|
||||||
|
a[2], a[3] = 0x0d, 0xb8
|
||||||
|
for i := 4; i < 10; i++ {
|
||||||
|
a[i] = byte(r.Intn(256))
|
||||||
|
}
|
||||||
|
addr := netip.AddrFrom16(a)
|
||||||
|
pfx = netip.PrefixFrom(addr, 48+r.Intn(17))
|
||||||
|
} else {
|
||||||
|
b := []byte{203, 0, 113, 0}
|
||||||
|
b[1] = byte(r.Intn(256))
|
||||||
|
b[2] = byte(r.Intn(256))
|
||||||
|
b[3] = byte(r.Intn(256))
|
||||||
|
addr := netip.AddrFrom4([4]byte{b[0], b[1], b[2], b[3]})
|
||||||
|
pfx = netip.PrefixFrom(addr, 16+r.Intn(9))
|
||||||
|
}
|
||||||
|
s := pfx.Masked().String()
|
||||||
|
if _, ok := seen[s]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[s] = struct{}{}
|
||||||
|
rows = append(rows, store.PrefixRow{Prefix: s, Source: "ip_range"})
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkSmartAggregatePrefixRows(b *testing.B) {
|
||||||
|
for _, n := range []int{1_000, 10_000, 50_000} {
|
||||||
|
rows := benchPrefixRows(n, 0.3)
|
||||||
|
b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) {
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_ = smartAggregatePrefixRows(rows)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,9 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
@@ -194,3 +196,63 @@ func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) {
|
|||||||
t.Fatalf("unexpected collected rows: %+v", collected)
|
t.Fatalf("unexpected collected rows: %+v", collected)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPrefetchCDNSourceETags_SkipsFreshSources(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := m.DemoIDs()
|
||||||
|
|
||||||
|
mod, err := m.CreateModule(tenant, &store.Module{
|
||||||
|
Type: "CDN_CIDRS",
|
||||||
|
Name: "cdn-prefetch-due",
|
||||||
|
Enabled: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var hits atomic.Int32
|
||||||
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
hits.Add(1)
|
||||||
|
w.Header().Set("ETag", `"v1"`)
|
||||||
|
_, _ = w.Write([]byte("198.51.100.0/24\n"))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
// Fresh source: refreshed 30s ago with a 3600s interval — prefetch must skip it.
|
||||||
|
freshAt := time.Now().UTC().Add(-30 * time.Second)
|
||||||
|
interval := 3600
|
||||||
|
if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
|
||||||
|
SourceKind: "txt",
|
||||||
|
URL: srv.URL,
|
||||||
|
RefreshIntervalSec: &interval,
|
||||||
|
LastRefreshedAt: &freshAt,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := PrefetchCDNSourceETags(context.Background(), m, srv.Client()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if hits.Load() != 0 {
|
||||||
|
t.Fatalf("fresh source must be skipped by prefetch, got %d HTTP hits", hits.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stale source: last refresh older than its interval — prefetch must probe it.
|
||||||
|
staleAt := time.Now().UTC().Add(-7200 * time.Second)
|
||||||
|
if _, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
|
||||||
|
SourceKind: "txt",
|
||||||
|
URL: srv.URL + "?stale",
|
||||||
|
RefreshIntervalSec: &interval,
|
||||||
|
LastRefreshedAt: &staleAt,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := PrefetchCDNSourceETags(context.Background(), m, srv.Client()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if hits.Load() == 0 {
|
||||||
|
t.Fatal("stale source must be probed by prefetch")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -42,32 +42,49 @@ func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.P
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// mergeSnapshotDropCDNSources removes all cdn:* rows (used before batch CDN merge).
|
// mergeSnapshotKeepSkippedCDN keeps non-CDN rows and cdn:* rows whose source is still skipped
|
||||||
func mergeSnapshotDropCDNSources(rows []store.PrefixRow) []store.PrefixRow {
|
// (not in fetchedSourceIDs). Deleted sources (absent from allSourceIDs) are dropped.
|
||||||
if len(rows) == 0 {
|
func mergeSnapshotKeepSkippedCDN(rows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) []store.PrefixRow {
|
||||||
return nil
|
fetched := make(map[string]struct{}, len(fetchedSourceIDs))
|
||||||
|
for _, id := range fetchedSourceIDs {
|
||||||
|
fetched[cdnSourceKey(id)] = struct{}{}
|
||||||
|
}
|
||||||
|
keepCDN := make(map[string]struct{})
|
||||||
|
for _, id := range allSourceIDs {
|
||||||
|
k := cdnSourceKey(id)
|
||||||
|
if _, ok := fetched[k]; !ok {
|
||||||
|
keepCDN[k] = struct{}{}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
out := make([]store.PrefixRow, 0, len(rows))
|
out := make([]store.PrefixRow, 0, len(rows))
|
||||||
for _, row := range rows {
|
for _, row := range rows {
|
||||||
if !strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") {
|
src := strings.TrimSpace(row.Source)
|
||||||
out = append(out, row)
|
if strings.HasPrefix(src, "cdn:") {
|
||||||
|
if _, ok := keepCDN[src]; !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
out = append(out, row)
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// mergeAllCDNSourcesIntoModuleSnapshot replaces all CDN rows in one write (avoids parallel read-modify-write races).
|
// mergeAllCDNSourcesIntoModuleSnapshot replaces fetched CDN source rows in one write.
|
||||||
func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow) error {
|
// skipped sources (errors with EVOBGP_CDN_PARTIAL_OK) keep their prior rows.
|
||||||
|
func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) error {
|
||||||
if st == nil || mod == nil {
|
if st == nil || mod == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
unlock := st.LockModuleSnapshot(tenantID, mod.ID)
|
||||||
|
defer unlock()
|
||||||
var base []store.PrefixRow
|
var base []store.PrefixRow
|
||||||
if len(priorSnapshot) > 0 {
|
if len(priorSnapshot) > 0 {
|
||||||
base = mergeSnapshotDropCDNSources(priorSnapshot)
|
base = priorSnapshot
|
||||||
} else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
} else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
||||||
base = mergeSnapshotDropCDNSources(snap.Prefixes)
|
base = snap.Prefixes
|
||||||
}
|
}
|
||||||
merged := append(base, cdnRows...)
|
kept := mergeSnapshotKeepSkippedCDN(base, fetchedSourceIDs, allSourceIDs)
|
||||||
|
merged := append(kept, cdnRows...)
|
||||||
return persistModuleSnapshot(st, tenantID, mod, merged)
|
return persistModuleSnapshot(st, tenantID, mod, merged)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,6 +110,8 @@ func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *st
|
|||||||
if st == nil || mod == nil {
|
if st == nil || mod == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
unlock := st.LockModuleSnapshot(tenantID, mod.ID)
|
||||||
|
defer unlock()
|
||||||
sourceKey := cdnSourceKey(sourceID)
|
sourceKey := cdnSourceKey(sourceID)
|
||||||
var base []store.PrefixRow
|
var base []store.PrefixRow
|
||||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
||||||
@@ -114,81 +133,6 @@ func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyCDNSourceHTTPResult(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
|
|
||||||
u := strings.TrimSpace(src.URL)
|
|
||||||
if u == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if _, err := ValidateCDNURL(u); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := ResolveCDNURLHost(ctx, u); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
sourceKey := cdnSourceKey(src.ID)
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
|
||||||
req.Header.Set("If-None-Match", etag)
|
|
||||||
}
|
|
||||||
resp, err := upstreamHTTPDo(ctx, hc, req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if resp.StatusCode == http.StatusNotModified {
|
|
||||||
if cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey); len(cached) > 0 {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
return cached, nil
|
|
||||||
}
|
|
||||||
// ETag is known but local snapshot is empty — force a full download.
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
req2, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
resp, err = upstreamHTTPDo(ctx, hc, req2)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("cdn fetch %s: %w", u, err)
|
|
||||||
}
|
|
||||||
if resp.StatusCode == http.StatusNotModified {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
return nil, fmt.Errorf("cdn url %s: 304 without cached prefixes", u)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
_, _ = io.Copy(io.Discard, resp.Body)
|
|
||||||
return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
prefixStrs, err := parseCDNBody(string(body), src)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("cdn parse %s: %w", u, err)
|
|
||||||
}
|
|
||||||
etag := strings.TrimSpace(resp.Header.Get("ETag"))
|
|
||||||
patch := &store.CDNSourcePatch{}
|
|
||||||
if etag != "" && etag != strings.TrimSpace(src.Etag) {
|
|
||||||
e := etag
|
|
||||||
patch.Etag = &e
|
|
||||||
}
|
|
||||||
refreshedAt := now
|
|
||||||
patch.LastRefreshedAt = &refreshedAt
|
|
||||||
_, _ = st.UpdateCDNSource(tenantID, moduleID, src.ID, patch)
|
|
||||||
|
|
||||||
rows := cdnRowsFromParsed(mod, src, prefixStrs)
|
|
||||||
if err := mergeCDNSourceIntoModuleSnapshot(st, tenantID, mod, src.ID, rows); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return rows, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once).
|
// fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once).
|
||||||
func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
|
func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
|
||||||
u := strings.TrimSpace(src.URL)
|
u := strings.TrimSpace(src.URL)
|
||||||
|
|||||||
@@ -7,7 +7,9 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -91,6 +93,9 @@ func ResolveCDNURLHost(ctx context.Context, raw string) error {
|
|||||||
if isBlockedCDNHostname(host) {
|
if isBlockedCDNHostname(host) {
|
||||||
return fmt.Errorf("pipeline: cdn url blocked host")
|
return fmt.Errorf("pipeline: cdn url blocked host")
|
||||||
}
|
}
|
||||||
|
if ok := cdnDNSVerifyCache.hit(host); ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
if ctx == nil {
|
if ctx == nil {
|
||||||
ctx = context.Background()
|
ctx = context.Background()
|
||||||
}
|
}
|
||||||
@@ -112,5 +117,64 @@ func ResolveCDNURLHost(ctx context.Context, raw string) error {
|
|||||||
return fmt.Errorf("pipeline: cdn url resolves to blocked address")
|
return fmt.Errorf("pipeline: cdn url resolves to blocked address")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
cdnDNSVerifyCache.store(host)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cdnDNSVerifyTTL bounds how long a successful SSRF check is trusted for one hostname.
|
||||||
|
// Failures are never cached: a transient DNS outage must not open an unsafe window,
|
||||||
|
// and a blocked host is rejected before this cache anyway.
|
||||||
|
func cdnDNSVerifyTTL() time.Duration {
|
||||||
|
sec := 300
|
||||||
|
if s := strings.TrimSpace(os.Getenv("EVOBGP_CDN_DNS_CACHE_TTL_SEC")); s != "" {
|
||||||
|
if v, err := strconv.Atoi(s); err == nil && v > 0 {
|
||||||
|
sec = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return time.Duration(sec) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
type dnsVerifyCache struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
seen map[string]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
var cdnDNSVerifyCache = &dnsVerifyCache{seen: make(map[string]time.Time)}
|
||||||
|
|
||||||
|
func (c *dnsVerifyCache) hit(host string) bool {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
at, ok := c.seen[host]
|
||||||
|
return ok && time.Since(at) < cdnDNSVerifyTTL()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *dnsVerifyCache) store(host string) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if c.seen == nil {
|
||||||
|
c.seen = make(map[string]time.Time)
|
||||||
|
}
|
||||||
|
c.seen[host] = time.Now()
|
||||||
|
if len(c.seen) > 4096 {
|
||||||
|
// Size cap for long-running workers: drop expired entries, then the oldest if needed.
|
||||||
|
now := time.Now()
|
||||||
|
for h, at := range c.seen {
|
||||||
|
if now.Sub(at) >= cdnDNSVerifyTTL() {
|
||||||
|
delete(c.seen, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(c.seen) > 4096 {
|
||||||
|
var oldestK string
|
||||||
|
var oldestT time.Time
|
||||||
|
first := true
|
||||||
|
for h, at := range c.seen {
|
||||||
|
if first || at.Before(oldestT) {
|
||||||
|
oldestK, oldestT, first = h, at, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if oldestK != "" {
|
||||||
|
delete(c.seen, oldestK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -26,6 +26,14 @@ func prefixRowsForSource(rows []store.PrefixRow, sourceKey string) []store.Prefi
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func prefixCommunityKey(row store.PrefixRow) string {
|
||||||
|
comm := ""
|
||||||
|
if row.CommunityID != nil {
|
||||||
|
comm = strings.TrimSpace(*row.CommunityID)
|
||||||
|
}
|
||||||
|
return row.Prefix + "\x00" + comm
|
||||||
|
}
|
||||||
|
|
||||||
func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, list []*store.ASEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module, list []*store.ASEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
||||||
moduleID := mod.ID
|
moduleID := mod.ID
|
||||||
legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0"
|
legacy := strings.TrimSpace(os.Getenv("EVOBGP_ASN_RESOLVE")) == "0"
|
||||||
@@ -147,7 +155,7 @@ func collectASPrefixRows(ctx context.Context, st store.Backend, hc *http.Client,
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
for _, row := range r.rows {
|
for _, row := range r.rows {
|
||||||
k := row.Prefix
|
k := prefixCommunityKey(row)
|
||||||
if _, ok := seenPfx[k]; ok {
|
if _, ok := seenPfx[k]; ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -218,9 +226,14 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
|
|||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
var out []store.PrefixRow
|
var fetchedRows []store.PrefixRow
|
||||||
var skipped int
|
var skipped int
|
||||||
for _, r := range results {
|
var fetchedIDs []string
|
||||||
|
allIDs := make([]string, 0, len(valid))
|
||||||
|
for _, src := range valid {
|
||||||
|
allIDs = append(allIDs, src.ID)
|
||||||
|
}
|
||||||
|
for i, r := range results {
|
||||||
if r.err != nil {
|
if r.err != nil {
|
||||||
if cdnPartialOK() {
|
if cdnPartialOK() {
|
||||||
logging.Default().Info(fmt.Sprintf("pipeline: CDN partial skip source error: %v", r.err))
|
logging.Default().Info(fmt.Sprintf("pipeline: CDN partial skip source error: %v", r.err))
|
||||||
@@ -229,20 +242,35 @@ func collectCDNPrefixRows(ctx context.Context, st store.Backend, hc *http.Client
|
|||||||
}
|
}
|
||||||
return nil, r.err
|
return nil, r.err
|
||||||
}
|
}
|
||||||
out = append(out, r.rows...)
|
fetchedIDs = append(fetchedIDs, valid[i].ID)
|
||||||
|
fetchedRows = append(fetchedRows, r.rows...)
|
||||||
}
|
}
|
||||||
if skipped > 0 && len(out) == 0 && len(valid) > 0 {
|
if skipped > 0 && len(fetchedRows) == 0 && len(valid) > 0 {
|
||||||
return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid))
|
return nil, fmt.Errorf("cdn: all %d source(s) failed (partial ok)", len(valid))
|
||||||
}
|
}
|
||||||
|
out := fetchedRows
|
||||||
|
if skipped > 0 {
|
||||||
|
base := priorSnapshot
|
||||||
|
if len(base) == 0 {
|
||||||
|
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
|
||||||
|
base = snap.Prefixes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, row := range mergeSnapshotKeepSkippedCDN(base, fetchedIDs, allIDs) {
|
||||||
|
if strings.HasPrefix(strings.TrimSpace(row.Source), "cdn:") {
|
||||||
|
out = append(out, row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(valid) > 0 {
|
if len(valid) > 0 {
|
||||||
if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, out); err != nil {
|
if err := mergeAllCDNSourcesIntoModuleSnapshot(st, tenantID, mod, priorSnapshot, fetchedRows, fetchedIDs, allIDs); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
func collectDomainPrefixRows(ctx context.Context, st store.Backend, hc *http.Client, mod *store.Module, profiles []*store.DohProfile, policy string, entries []*store.DomainEntry, priorSnapshot []store.PrefixRow) ([]store.PrefixRow, error) {
|
||||||
var validDom []*store.DomainEntry
|
var validDom []*store.DomainEntry
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
if e != nil {
|
if e != nil {
|
||||||
@@ -269,7 +297,7 @@ func collectDomainPrefixRows(ctx context.Context, hc *http.Client, mod *store.Mo
|
|||||||
c := *mod.DefaultCommunityID
|
c := *mod.DefaultCommunityID
|
||||||
comm = &c
|
comm = &c
|
||||||
}
|
}
|
||||||
addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, entry.FQDN)
|
addrs, err := resolveDomainIPsCached(ctx, st, hc, profiles, policy, entry.FQDN)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if staleOnUpstreamError() {
|
if staleOnUpstreamError() {
|
||||||
if cached, ok := staleDomainPrefixes(priorSnapshot, entry.FQDN); ok {
|
if cached, ok := staleDomainPrefixes(priorSnapshot, entry.FQDN); ok {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
@@ -108,8 +109,7 @@ func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, prof
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
baseURL := strings.TrimSpace(profile.URL)
|
baseURL := strings.TrimSpace(profile.URL)
|
||||||
v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA)
|
v4, v6, err4, err6 := resolveDOHMessagePair(dctx, hc, baseURL, host)
|
||||||
v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA)
|
|
||||||
if err4 != nil {
|
if err4 != nil {
|
||||||
v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
|
v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
|
||||||
}
|
}
|
||||||
@@ -122,6 +122,23 @@ func resolveDomainIPsNoSystemFallback(ctx context.Context, hc *http.Client, prof
|
|||||||
return uniqAddrs(append(v4, v6...)), nil
|
return uniqAddrs(append(v4, v6...)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolveDOHMessagePair issues RFC8484 dns-message A and AAAA queries concurrently
|
||||||
|
// and waits for both (fallbacks are handled by the caller).
|
||||||
|
func resolveDOHMessagePair(ctx context.Context, hc *http.Client, baseURL, host string) (v4, v6 []netip.Addr, err4, err6 error) {
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
v4, err4 = resolveDomainWithDOHMessage(ctx, hc, baseURL, host, dns.TypeA)
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
v6, err6 = resolveDomainWithDOHMessage(ctx, hc, baseURL, host, dns.TypeAAAA)
|
||||||
|
}()
|
||||||
|
wg.Wait()
|
||||||
|
return v4, v6, err4, err6
|
||||||
|
}
|
||||||
|
|
||||||
func dohProfileTimeout(profile *store.DohProfile) time.Duration {
|
func dohProfileTimeout(profile *store.DohProfile) time.Duration {
|
||||||
timeout := 10 * time.Second
|
timeout := 10 * time.Second
|
||||||
if profile != nil && profile.TimeoutMs != nil && *profile.TimeoutMs > 0 {
|
if profile != nil && profile.TimeoutMs != nil && *profile.TimeoutMs > 0 {
|
||||||
|
|||||||
@@ -2,147 +2,97 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
|
|
||||||
|
"github.com/miekg/dns"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestResolveDomainIPsWithPolicy_Union(t *testing.T) {
|
// TestResolveDomainIPs_ParallelAAndAAAA verifies that A and AAAA queries are issued
|
||||||
srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
// concurrently: with a 250ms upstream latency the combined resolve must stay near
|
||||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.1"}]}`))
|
// one round-trip instead of two.
|
||||||
}))
|
func TestResolveDomainIPs_ParallelAAndAAAA(t *testing.T) {
|
||||||
defer srvRU.Close()
|
const delay = 250 * time.Millisecond
|
||||||
srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
var inflight, maxInflight atomic.Int32
|
||||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.1"}]}`))
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
}))
|
cur := inflight.Add(1)
|
||||||
defer srvEU.Close()
|
for {
|
||||||
|
old := maxInflight.Load()
|
||||||
|
if cur <= old || maxInflight.CompareAndSwap(old, cur) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
defer inflight.Add(-1)
|
||||||
|
time.Sleep(delay)
|
||||||
|
|
||||||
profiles := []*store.DohProfile{
|
if wire := r.URL.Query().Get("dns"); wire != "" {
|
||||||
{URL: srvRU.URL},
|
// RFC8484 dns-message: decode the query and answer on the wire.
|
||||||
{URL: srvEU.URL},
|
raw, err := base64.RawURLEncoding.DecodeString(wire)
|
||||||
}
|
if err != nil {
|
||||||
ips, err := resolveDomainIPsWithPolicy(context.Background(), srvRU.Client(), profiles, store.DohPolicyUnion, "example.com")
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
msg := new(dns.Msg)
|
||||||
|
if err := msg.Unpack(raw); err != nil {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resp := new(dns.Msg)
|
||||||
|
resp.SetReply(msg)
|
||||||
|
switch msg.Question[0].Qtype {
|
||||||
|
case dns.TypeA:
|
||||||
|
resp.Answer = append(resp.Answer, &dns.A{
|
||||||
|
Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
|
||||||
|
A: []byte{203, 0, 113, 10},
|
||||||
|
})
|
||||||
|
case dns.TypeAAAA:
|
||||||
|
resp.Answer = append(resp.Answer, &dns.AAAA{
|
||||||
|
Hdr: dns.RR_Header{Name: msg.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60},
|
||||||
|
AAAA: []byte{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x10},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
out, err := resp.Pack()
|
||||||
|
if err != nil {
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/dns-message")
|
||||||
|
_, _ = w.Write(out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/dns-json")
|
||||||
|
switch r.URL.Query().Get("type") {
|
||||||
|
case "A":
|
||||||
|
_, _ = w.Write([]byte(`{"Status":0,"Answer":[{"type":1,"data":"203.0.113.10"}]}`))
|
||||||
|
default:
|
||||||
|
_, _ = w.Write([]byte(`{"Status":0,"Answer":[{"type":28,"data":"2001:db8::10"}]}`))
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
prof := &store.DohProfile{URL: srv.URL, TimeoutMs: ptrInt(5000)}
|
||||||
|
ctx := context.Background()
|
||||||
|
start := time.Now()
|
||||||
|
ips, err := resolveDomainIPsNoSystemFallback(ctx, srv.Client(), prof, "example.test")
|
||||||
|
elapsed := time.Since(start)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatalf("resolve failed: %v", err)
|
||||||
}
|
}
|
||||||
if len(ips) != 2 {
|
if len(ips) != 2 {
|
||||||
t.Fatalf("want 2 ips, got %v", ips)
|
t.Fatalf("expected 2 addrs, got %+v", ips)
|
||||||
}
|
}
|
||||||
seen := map[string]bool{ips[0].String(): true, ips[1].String(): true}
|
if maxInflight.Load() < 2 {
|
||||||
if !seen["198.51.100.1"] || !seen["203.0.113.1"] {
|
t.Fatalf("expected concurrent A/AAAA queries, max inflight=%d", maxInflight.Load())
|
||||||
t.Fatalf("unexpected ips: %v", ips)
|
}
|
||||||
|
if elapsed >= 2*delay {
|
||||||
|
t.Fatalf("resolve took %v; expected one round-trip (<2*%v)", elapsed, delay)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResolveDomainIPsWithPolicy_Failover(t *testing.T) {
|
func ptrInt(v int) *int { return &v }
|
||||||
var calls int
|
|
||||||
srvBad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
calls++
|
|
||||||
http.Error(w, "fail", http.StatusBadGateway)
|
|
||||||
}))
|
|
||||||
defer srvBad.Close()
|
|
||||||
srvOK := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
calls++
|
|
||||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.5"}]}`))
|
|
||||||
}))
|
|
||||||
defer srvOK.Close()
|
|
||||||
|
|
||||||
profiles := []*store.DohProfile{
|
|
||||||
{URL: srvBad.URL},
|
|
||||||
{URL: srvOK.URL},
|
|
||||||
}
|
|
||||||
ips, err := resolveDomainIPsWithPolicy(context.Background(), srvBad.Client(), profiles, store.DohPolicyFailover, "example.com")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(ips) != 1 || ips[0].String() != "198.51.100.5" {
|
|
||||||
t.Fatalf("unexpected ips: %v", ips)
|
|
||||||
}
|
|
||||||
if calls < 2 {
|
|
||||||
t.Fatalf("want at least 2 resolver calls, got %d", calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveDomainIPsWithPolicy_PrimaryOnly(t *testing.T) {
|
|
||||||
var secondCalled bool
|
|
||||||
srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.9"}]}`))
|
|
||||||
}))
|
|
||||||
defer srv1.Close()
|
|
||||||
srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
secondCalled = true
|
|
||||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.9"}]}`))
|
|
||||||
}))
|
|
||||||
defer srv2.Close()
|
|
||||||
|
|
||||||
profiles := []*store.DohProfile{
|
|
||||||
{URL: srv1.URL},
|
|
||||||
{URL: srv2.URL},
|
|
||||||
}
|
|
||||||
ips, err := resolveDomainIPsWithPolicy(context.Background(), srv1.Client(), profiles, store.DohPolicyPrimaryOnly, "example.com")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(ips) != 1 || ips[0].String() != "198.51.100.9" {
|
|
||||||
t.Fatalf("unexpected ips: %v", ips)
|
|
||||||
}
|
|
||||||
if secondCalled {
|
|
||||||
t.Fatal("secondary resolver must not be queried in primary_only mode")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCollectModulePrefixRows_DohUnion(t *testing.T) {
|
|
||||||
m := store.NewMemory()
|
|
||||||
m.SeedDemo()
|
|
||||||
tenant, _, _, _, _ := m.DemoIDs()
|
|
||||||
|
|
||||||
mod, err := m.CreateModule(tenant, &store.Module{
|
|
||||||
Type: "DOMAINS",
|
|
||||||
Name: "domains-union",
|
|
||||||
Enabled: true,
|
|
||||||
DohResolverPolicy: store.DohPolicyUnion,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
srvRU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"198.51.100.2"}]}`))
|
|
||||||
}))
|
|
||||||
defer srvRU.Close()
|
|
||||||
srvEU := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = w.Write([]byte(`{"Answer":[{"type":1,"data":"203.0.113.2"}]}`))
|
|
||||||
}))
|
|
||||||
defer srvEU.Close()
|
|
||||||
|
|
||||||
ru, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "ru", URL: srvRU.URL})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
eu, err := m.CreateDohProfile(tenant, &store.DohProfile{Name: "eu", URL: srvEU.URL})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := m.UpdateModule(tenant, mod.ID, &store.ModulePatch{
|
|
||||||
DohProfileIDs: &[]string{ru.ID, eu.ID},
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mod, err = m.GetModule(tenant, mod.ID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := m.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "svc.example.com"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rows, err := collectModulePrefixRows(context.Background(), m, srvRU.Client(), tenant, mod, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(rows) != 2 {
|
|
||||||
t.Fatalf("want 2 prefix rows, got %+v", rows)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func domainCacheTTL() time.Duration {
|
||||||
|
sec := 300
|
||||||
|
if s := strings.TrimSpace(os.Getenv("EVOBGP_DOMAIN_CACHE_TTL_SEC")); s != "" {
|
||||||
|
if v, err := strconv.Atoi(s); err == nil && v > 0 {
|
||||||
|
sec = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return time.Duration(sec) * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveDomainIPsCached(ctx context.Context, st store.Backend, hc *http.Client, profiles []*store.DohProfile, policy, fqdn string) ([]netip.Addr, error) {
|
||||||
|
key := strings.TrimSpace(fqdn)
|
||||||
|
ttl := domainCacheTTL()
|
||||||
|
if st != nil && ttl > 0 {
|
||||||
|
if ent, ok, err := st.GetDomainResolveCache(key); err == nil && ok && ent != nil {
|
||||||
|
if time.Since(ent.ResolvedAt) < ttl {
|
||||||
|
if addrs := parseCachedDomainAddrs(ent.Addrs); len(addrs) > 0 {
|
||||||
|
return addrs, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
addrs, err := resolveDomainIPsWithPolicy(ctx, hc, profiles, policy, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if st != nil {
|
||||||
|
_ = st.SetDomainResolveCache(key, domainAddrsToStrings(addrs))
|
||||||
|
}
|
||||||
|
return addrs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseCachedDomainAddrs(raw []string) []netip.Addr {
|
||||||
|
out := make([]netip.Addr, 0, len(raw))
|
||||||
|
for _, s := range raw {
|
||||||
|
a, err := netip.ParseAddr(strings.TrimSpace(s))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func domainAddrsToStrings(addrs []netip.Addr) []string {
|
||||||
|
out := make([]string, 0, len(addrs))
|
||||||
|
for _, a := range addrs {
|
||||||
|
if a.IsValid() {
|
||||||
|
out = append(out, a.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -1,10 +1,6 @@
|
|||||||
package pipeline
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
@@ -12,89 +8,7 @@ import (
|
|||||||
|
|
||||||
// moduleIngestInputHash fingerprints module config and child entries so snapshots invalidate on CRUD.
|
// moduleIngestInputHash fingerprints module config and child entries so snapshots invalidate on CRUD.
|
||||||
func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module) (string, error) {
|
func moduleIngestInputHash(st store.Backend, tenantID string, mod *store.Module) (string, error) {
|
||||||
if st == nil || mod == nil {
|
return store.ComputeModuleInputHash(st, tenantID, mod)
|
||||||
return "", fmt.Errorf("pipeline: module hash: missing store or module")
|
|
||||||
}
|
|
||||||
h := sha256.New()
|
|
||||||
_, _ = fmt.Fprintf(h, "type=%s\n", strings.TrimSpace(mod.Type))
|
|
||||||
_, _ = fmt.Fprintf(h, "enabled=%t\n", mod.Enabled)
|
|
||||||
if mod.DefaultCommunityID != nil {
|
|
||||||
_, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID))
|
|
||||||
}
|
|
||||||
_, _ = fmt.Fprintf(h, "doh_policy=%s\n", store.NormalizeDohResolverPolicy(mod.DohResolverPolicy))
|
|
||||||
for _, pid := range mod.EffectiveDohProfileIDs() {
|
|
||||||
_, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid)
|
|
||||||
if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil {
|
|
||||||
_, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL))
|
|
||||||
if prof.TimeoutMs != nil {
|
|
||||||
_, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
switch mod.Type {
|
|
||||||
case "IP_RANGES":
|
|
||||||
list, err := st.ListIPRangeEntries(tenantID, mod.ID)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
sort.Slice(list, func(i, j int) bool { return list[i].Prefix < list[j].Prefix })
|
|
||||||
for _, e := range list {
|
|
||||||
comm := ""
|
|
||||||
if e.CommunityID != nil {
|
|
||||||
comm = *e.CommunityID
|
|
||||||
}
|
|
||||||
_, _ = fmt.Fprintf(h, "ip=%s|c=%s\n", e.Prefix, comm)
|
|
||||||
}
|
|
||||||
case "AS_PREFIXES":
|
|
||||||
list, err := st.ListASEntries(tenantID, mod.ID)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
|
|
||||||
for _, e := range list {
|
|
||||||
comm := ""
|
|
||||||
if e.CommunityID != nil {
|
|
||||||
comm = *e.CommunityID
|
|
||||||
}
|
|
||||||
_, _ = fmt.Fprintf(h, "as=%d|c=%s\n", e.ASN, comm)
|
|
||||||
}
|
|
||||||
case "CDN_CIDRS":
|
|
||||||
list, err := st.ListCDNSources(tenantID, mod.ID)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
sort.Slice(list, func(i, j int) bool { return list[i].ID < list[j].ID })
|
|
||||||
for _, s := range list {
|
|
||||||
comm := ""
|
|
||||||
if s.CommunityID != nil {
|
|
||||||
comm = *s.CommunityID
|
|
||||||
}
|
|
||||||
interval := 0
|
|
||||||
if s.RefreshIntervalSec != nil {
|
|
||||||
interval = *s.RefreshIntervalSec
|
|
||||||
}
|
|
||||||
_, _ = fmt.Fprintf(h, "cdn=%s|url=%s|kind=%s|path=%s|c=%s|etag=%s|interval=%d\n",
|
|
||||||
s.ID, strings.TrimSpace(s.URL), s.SourceKind, strings.TrimSpace(s.PrefixPath), comm,
|
|
||||||
strings.TrimSpace(s.Etag), interval)
|
|
||||||
}
|
|
||||||
case "DOMAINS":
|
|
||||||
list, err := st.ListDomainEntries(tenantID, mod.ID)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
sort.Slice(list, func(i, j int) bool { return list[i].FQDN < list[j].FQDN })
|
|
||||||
for _, e := range list {
|
|
||||||
comm := ""
|
|
||||||
if e.CommunityID != nil {
|
|
||||||
comm = *e.CommunityID
|
|
||||||
}
|
|
||||||
_, _ = fmt.Fprintf(h, "dom=%s|c=%s\n", strings.TrimSpace(e.FQDN), comm)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
_, _ = fmt.Fprintf(h, "unknown_type=%s\n", mod.Type)
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(h.Sum(nil)), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error {
|
func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, rows []store.PrefixRow) error {
|
||||||
@@ -105,18 +19,32 @@ func persistModuleSnapshot(st store.Backend, tenantID string, mod *store.Module,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows)
|
if err := st.SetModulePrefixSnapshot(tenantID, mod.ID, hash, rows); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_ = st.SetModuleInputHash(tenantID, mod.ID, hash)
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module) ([]store.PrefixRow, bool, error) {
|
func moduleRowsFromSnapshot(st store.Backend, tenantID string, mod *store.Module) ([]store.PrefixRow, bool, error) {
|
||||||
|
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
|
||||||
|
if err != nil || !ok || snap == nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if h := strings.TrimSpace(mod.InputHash); h != "" {
|
||||||
|
if snap.InputHash != h {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil
|
||||||
|
}
|
||||||
|
// Fallback for rows written before module.input_hash existed: recompute and backfill.
|
||||||
hash, err := moduleIngestInputHash(st, tenantID, mod)
|
hash, err := moduleIngestInputHash(st, tenantID, mod)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
|
if snap.InputHash != hash {
|
||||||
if err != nil || !ok || snap == nil || snap.InputHash != hash {
|
return nil, false, nil
|
||||||
return nil, false, err
|
|
||||||
}
|
}
|
||||||
cp := append([]store.PrefixRow(nil), snap.Prefixes...)
|
_ = st.SetModuleInputHash(tenantID, mod.ID, hash)
|
||||||
return cp, true, nil
|
return append([]store.PrefixRow(nil), snap.Prefixes...), true, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,189 @@
|
|||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestModuleDueForScheduler_JitterSpreads(t *testing.T) {
|
||||||
|
dueTicks := map[int64]int{}
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
mod := &store.Module{
|
||||||
|
ID: fmt.Sprintf("module-%d", i),
|
||||||
|
Enabled: true,
|
||||||
|
Type: "AS_PREFIXES",
|
||||||
|
RefreshIntervalSec: 3600,
|
||||||
|
}
|
||||||
|
for tick := int64(0); tick < 7200; tick += SchedulerTickSec {
|
||||||
|
if ModuleDueForScheduler(mod, time.Unix(tick, 0)) {
|
||||||
|
dueTicks[tick]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(dueTicks) < 10 {
|
||||||
|
t.Fatalf("expected due events across many ticks, got %d buckets", len(dueTicks))
|
||||||
|
}
|
||||||
|
for tick, n := range dueTicks {
|
||||||
|
if n == 100 {
|
||||||
|
t.Fatalf("all 100 modules due on tick %d", tick)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectASPrefixRows_KeepsSamePrefixDifferentCommunity(t *testing.T) {
|
||||||
|
st := store.NewMemory()
|
||||||
|
st.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := st.DemoIDs()
|
||||||
|
mod, err := st.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "as-dedup", Enabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
c1, c2 := "comm-a", "comm-b"
|
||||||
|
e1, err := st.CreateASEntry(tenant, mod.ID, &store.ASEntry{ASN: 64500, CommunityID: &c1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
e2, err := st.CreateASEntry(tenant, mod.ID, &store.ASEntry{ASN: 64501, CommunityID: &c2})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := st.SetASNPrefixCache(64500, "a", []string{"192.0.2.0/24"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := st.SetASNPrefixCache(64501, "b", []string{"192.0.2.0/24"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rows, err := collectASPrefixRows(context.Background(), st, http.DefaultClient, tenant, mod, []*store.ASEntry{e1, e2}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(rows) != 2 {
|
||||||
|
t.Fatalf("expected 2 rows (same prefix, different community), got %d: %+v", len(rows), rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectCDNPrefixRows_PartialSkipKeepsPrior(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_CDN_PARTIAL_OK", "1")
|
||||||
|
t.Setenv("EVOBGP_STALE_ON_UPSTREAM_ERROR", "0")
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
|
|
||||||
|
good := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte("198.51.100.0/24\n"))
|
||||||
|
}))
|
||||||
|
defer good.Close()
|
||||||
|
bad := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, "down", http.StatusServiceUnavailable)
|
||||||
|
}))
|
||||||
|
defer bad.Close()
|
||||||
|
|
||||||
|
st := store.NewMemory()
|
||||||
|
st.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := st.DemoIDs()
|
||||||
|
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-partial", Enabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prior := []store.PrefixRow{
|
||||||
|
{Prefix: "203.0.113.0/24", Source: "cdn:bad"},
|
||||||
|
{Prefix: "1.2.3.0/24", Source: "cdn:good"},
|
||||||
|
}
|
||||||
|
sources := []*store.CDNSource{
|
||||||
|
{ID: "good", URL: good.URL, SourceKind: "plain"},
|
||||||
|
{ID: "bad", URL: bad.URL, SourceKind: "plain"},
|
||||||
|
}
|
||||||
|
rows, err := collectCDNPrefixRows(context.Background(), st, good.Client(), tenant, mod, sources, prior)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("partial skip should succeed: %v", err)
|
||||||
|
}
|
||||||
|
got := map[string]string{}
|
||||||
|
for _, r := range rows {
|
||||||
|
got[r.Source] = r.Prefix
|
||||||
|
}
|
||||||
|
if got["cdn:bad"] != "203.0.113.0/24" {
|
||||||
|
t.Fatalf("skipped source lost prior row: %+v", rows)
|
||||||
|
}
|
||||||
|
if got["cdn:good"] != "198.51.100.0/24" {
|
||||||
|
t.Fatalf("fetched source missing new row: %+v", rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeCDNSource_ParallelNoLostUpdate(t *testing.T) {
|
||||||
|
st := store.NewMemory()
|
||||||
|
st.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := st.DemoIDs()
|
||||||
|
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-lock", Enabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(2)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
_ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s1", []store.PrefixRow{
|
||||||
|
{Prefix: "1.0.0.0/24", Source: "cdn:s1"},
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
_ = mergeCDNSourceIntoModuleSnapshot(st, tenant, mod, "s2", []store.PrefixRow{
|
||||||
|
{Prefix: "2.0.0.0/24", Source: "cdn:s2"},
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
wg.Wait()
|
||||||
|
snap, ok, err := st.GetModulePrefixSnapshot(tenant, mod.ID)
|
||||||
|
if err != nil || !ok || snap == nil {
|
||||||
|
t.Fatalf("snapshot missing: ok=%v err=%v", ok, err)
|
||||||
|
}
|
||||||
|
got := map[string]bool{}
|
||||||
|
for _, r := range snap.Prefixes {
|
||||||
|
got[r.Prefix] = true
|
||||||
|
}
|
||||||
|
if !got["1.0.0.0/24"] || !got["2.0.0.0/24"] {
|
||||||
|
t.Fatalf("lost parallel merge update: %+v", snap.Prefixes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveDomainIPsCached_UsesTTL(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_DOMAIN_CACHE_TTL_SEC", "300")
|
||||||
|
st := store.NewMemory()
|
||||||
|
if err := st.SetDomainResolveCache("cached.test", []string{"192.0.2.9"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
addrs, err := resolveDomainIPsCached(context.Background(), st, nil, nil, "", "cached.test")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(addrs) != 1 || addrs[0].String() != "192.0.2.9" {
|
||||||
|
t.Fatalf("expected cached addr, got %v", addrs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInputHashInvalidatedOnEntryCRUD(t *testing.T) {
|
||||||
|
st := store.NewMemory()
|
||||||
|
st.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := st.DemoIDs()
|
||||||
|
mod, err := st.CreateModule(tenant, &store.Module{Type: "IP_RANGES", Name: "ip-hash", Enabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := st.SetModuleInputHash(tenant, mod.ID, "pre"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := st.CreateIPRangeEntry(tenant, mod.ID, &store.IPRangeEntry{Prefix: "10.0.0.0/8"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := st.GetModule(tenant, mod.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.InputHash != "" {
|
||||||
|
t.Fatalf("expected hash cleared after CRUD, got %q", got.InputHash)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,7 +2,6 @@ package pipeline
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -31,6 +30,7 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
var tasks []prefetchTask
|
var tasks []prefetchTask
|
||||||
|
now := time.Now().UTC()
|
||||||
for _, tid := range tenants {
|
for _, tid := range tenants {
|
||||||
for _, mod := range st.ListModules(tid) {
|
for _, mod := range st.ListModules(tid) {
|
||||||
if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
|
if mod == nil || !mod.Enabled || mod.Type != "CDN_CIDRS" {
|
||||||
@@ -41,9 +41,15 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
for _, src := range sources {
|
for _, src := range sources {
|
||||||
if src != nil && strings.TrimSpace(src.URL) != "" {
|
if src == nil || strings.TrimSpace(src.URL) == "" {
|
||||||
tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src})
|
continue
|
||||||
}
|
}
|
||||||
|
// Respect per-source refresh intervals: conditional GET only for due sources.
|
||||||
|
// The ETag probe still lets 304s skip body downloads for the rest.
|
||||||
|
if shouldSkipCDNSourceFetch(src, now) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tasks = append(tasks, prefetchTask{tenantID: tid, mod: mod, src: src})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -68,53 +74,13 @@ func PrefetchCDNSourceETags(ctx context.Context, st store.Backend, hc *http.Clie
|
|||||||
func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) {
|
func prefetchOneCDNSource(ctx context.Context, st store.Backend, hc *http.Client, t prefetchTask) {
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
tid, mod, src := t.tenantID, t.mod, t.src
|
tid, mod, src := t.tenantID, t.mod, t.src
|
||||||
u := strings.TrimSpace(src.URL)
|
|
||||||
if _, err := ValidateCDNURL(u); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := ResolveCDNURLHost(ctx, u); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
omod, err := st.GetModule(tid, mod.ID)
|
omod, err := st.GetModule(tid, mod.ID)
|
||||||
|
if err != nil || omod == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rows, err := fetchCDNSourceRows(ctx, st, hc, tid, mod.ID, omod, src, nil, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if etag := strings.TrimSpace(src.Etag); etag != "" {
|
|
||||||
req.Header.Set("If-None-Match", etag)
|
|
||||||
}
|
|
||||||
resp, err := upstreamHTTPDo(ctx, hc, req)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if resp.StatusCode == http.StatusNotModified {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
_, _ = io.Copy(io.Discard, resp.Body)
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
prefixStrs, err := parseCDNBody(string(body), src)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
newEtag := strings.TrimSpace(resp.Header.Get("ETag"))
|
|
||||||
patch := &store.CDNSourcePatch{LastRefreshedAt: &now}
|
|
||||||
if newEtag != "" && newEtag != strings.TrimSpace(src.Etag) {
|
|
||||||
e := newEtag
|
|
||||||
patch.Etag = &e
|
|
||||||
}
|
|
||||||
_, _ = st.UpdateCDNSource(tid, mod.ID, src.ID, patch)
|
|
||||||
rows := cdnRowsFromParsed(omod, src, prefixStrs)
|
|
||||||
_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
|
_ = mergeCDNSourceIntoModuleSnapshot(st, tid, omod, src.ID, rows)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"math/big"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -190,7 +189,7 @@ func collectModulePrefixRows(ctx context.Context, st store.Backend, hc *http.Cli
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return collectDomainPrefixRows(ctx, hc, mod, profiles, policy, entries, priorSnapshot)
|
return collectDomainPrefixRows(ctx, st, hc, mod, profiles, policy, entries, priorSnapshot)
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type)
|
return nil, fmt.Errorf("pipeline: unknown module type %q", mod.Type)
|
||||||
}
|
}
|
||||||
@@ -230,9 +229,8 @@ func resolveDomainIPs(ctx context.Context, hc *http.Client, profile *store.DohPr
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
baseURL := strings.TrimSpace(profile.URL)
|
baseURL := strings.TrimSpace(profile.URL)
|
||||||
// Prefer RFC8484 dns-message transport. Some providers don't support dns-json.
|
// A and AAAA queries run concurrently: per-domain latency drops from ~2×RTT to ~1×RTT.
|
||||||
v4, err4 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeA)
|
v4, v6, err4, err6 := resolveDOHMessagePair(dctx, hc, baseURL, host)
|
||||||
v6, err6 := resolveDomainWithDOHMessage(dctx, hc, baseURL, host, dns.TypeAAAA)
|
|
||||||
if err4 != nil {
|
if err4 != nil {
|
||||||
// Fallback to JSON mode for providers that only expose dns-json.
|
// Fallback to JSON mode for providers that only expose dns-json.
|
||||||
v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
|
v4, err4 = resolveDomainWithDOHJSON(dctx, hc, baseURL, host, "A")
|
||||||
@@ -496,168 +494,11 @@ func prefixRowCommunity(r store.PrefixRow) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func aggregateIPv4Group(rows []store.PrefixRow) []store.PrefixRow {
|
func aggregateIPv4Group(rows []store.PrefixRow) []store.PrefixRow {
|
||||||
return aggregateCIDRGroup(rows, mergeSiblingPrefixesIPv4)
|
return collapsePrefixGroup(rows, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
func aggregateIPv6Group(rows []store.PrefixRow) []store.PrefixRow {
|
func aggregateIPv6Group(rows []store.PrefixRow) []store.PrefixRow {
|
||||||
return aggregateCIDRGroup(rows, mergeSiblingPrefixesIPv6)
|
return collapsePrefixGroup(rows, false)
|
||||||
}
|
|
||||||
|
|
||||||
func aggregateCIDRGroup(rows []store.PrefixRow, mergeFn func(map[string]store.PrefixRow) bool) []store.PrefixRow {
|
|
||||||
if len(rows) <= 1 {
|
|
||||||
return rows
|
|
||||||
}
|
|
||||||
set := make(map[string]store.PrefixRow, len(rows))
|
|
||||||
for _, row := range rows {
|
|
||||||
set[row.Prefix] = row
|
|
||||||
}
|
|
||||||
pruneCoveredPrefixes(set)
|
|
||||||
for {
|
|
||||||
if !mergeFn(set) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
pruneCoveredPrefixes(set)
|
|
||||||
}
|
|
||||||
out := make([]store.PrefixRow, 0, len(set))
|
|
||||||
for _, row := range set {
|
|
||||||
out = append(out, row)
|
|
||||||
}
|
|
||||||
sortPrefixRows(out)
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func pruneCoveredPrefixes(set map[string]store.PrefixRow) {
|
|
||||||
type item struct {
|
|
||||||
key string
|
|
||||||
pfx netip.Prefix
|
|
||||||
bits int
|
|
||||||
}
|
|
||||||
items := make([]item, 0, len(set))
|
|
||||||
for k := range set {
|
|
||||||
p, err := netip.ParsePrefix(k)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
items = append(items, item{key: k, pfx: p, bits: p.Bits()})
|
|
||||||
}
|
|
||||||
sort.Slice(items, func(i, j int) bool {
|
|
||||||
if items[i].bits != items[j].bits {
|
|
||||||
return items[i].bits < items[j].bits
|
|
||||||
}
|
|
||||||
return items[i].key < items[j].key
|
|
||||||
})
|
|
||||||
for i := 0; i < len(items); i++ {
|
|
||||||
for j := i + 1; j < len(items); j++ {
|
|
||||||
if items[j].bits <= items[i].bits {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if items[i].pfx.Contains(items[j].pfx.Addr()) {
|
|
||||||
delete(set, items[j].key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeSiblingPrefixesIPv4(set map[string]store.PrefixRow) bool {
|
|
||||||
merged := false
|
|
||||||
seen := make(map[string]struct{}, len(set))
|
|
||||||
for key, row := range set {
|
|
||||||
if _, done := seen[key]; done {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
pfx, err := netip.ParsePrefix(key)
|
|
||||||
if err != nil || !pfx.Addr().Is4() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
bits := pfx.Bits()
|
|
||||||
if bits <= 8 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
netNum := ipv4PrefixNetwork(pfx)
|
|
||||||
blockSize := uint32(1) << (32 - bits)
|
|
||||||
siblingNet := netNum ^ blockSize
|
|
||||||
siblingPfx := netip.PrefixFrom(u32ToIPv4(siblingNet), bits).Masked().String()
|
|
||||||
_, ok := set[siblingPfx]
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
parentBits := bits - 1
|
|
||||||
parentBlock := uint32(1) << (32 - parentBits)
|
|
||||||
parentNet := netNum & ^(parentBlock - 1)
|
|
||||||
parentPfx := netip.PrefixFrom(u32ToIPv4(parentNet), parentBits).Masked().String()
|
|
||||||
delete(set, key)
|
|
||||||
delete(set, siblingPfx)
|
|
||||||
parentRow := row
|
|
||||||
parentRow.Prefix = parentPfx
|
|
||||||
set[parentPfx] = parentRow
|
|
||||||
seen[key] = struct{}{}
|
|
||||||
seen[siblingPfx] = struct{}{}
|
|
||||||
merged = true
|
|
||||||
}
|
|
||||||
return merged
|
|
||||||
}
|
|
||||||
|
|
||||||
func ipv4PrefixNetwork(p netip.Prefix) uint32 {
|
|
||||||
a := p.Masked().Addr().As4()
|
|
||||||
return uint32(a[0])<<24 | uint32(a[1])<<16 | uint32(a[2])<<8 | uint32(a[3])
|
|
||||||
}
|
|
||||||
|
|
||||||
func u32ToIPv4(v uint32) netip.Addr {
|
|
||||||
return netip.AddrFrom4([4]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeSiblingPrefixesIPv6(set map[string]store.PrefixRow) bool {
|
|
||||||
merged := false
|
|
||||||
seen := make(map[string]struct{}, len(set))
|
|
||||||
for key, row := range set {
|
|
||||||
if _, done := seen[key]; done {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
pfx, err := netip.ParsePrefix(key)
|
|
||||||
if err != nil || !pfx.Addr().Is6() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
bits := pfx.Bits()
|
|
||||||
if bits <= 16 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
netNum := ipv6PrefixNetwork(pfx)
|
|
||||||
blockSize := new(big.Int).Lsh(big.NewInt(1), uint(128-bits))
|
|
||||||
siblingNet := new(big.Int).Xor(netNum, blockSize)
|
|
||||||
siblingPfx := ipv6PrefixFromBigInt(siblingNet, bits).String()
|
|
||||||
if _, ok := set[siblingPfx]; !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
parentBits := bits - 1
|
|
||||||
parentBlock := new(big.Int).Lsh(big.NewInt(1), uint(128-parentBits))
|
|
||||||
mask := new(big.Int).Sub(parentBlock, big.NewInt(1))
|
|
||||||
mask.Not(mask)
|
|
||||||
parentNet := new(big.Int).And(netNum, mask)
|
|
||||||
parentPfx := ipv6PrefixFromBigInt(parentNet, parentBits).String()
|
|
||||||
delete(set, key)
|
|
||||||
delete(set, siblingPfx)
|
|
||||||
parentRow := row
|
|
||||||
parentRow.Prefix = parentPfx
|
|
||||||
set[parentPfx] = parentRow
|
|
||||||
seen[key] = struct{}{}
|
|
||||||
seen[siblingPfx] = struct{}{}
|
|
||||||
merged = true
|
|
||||||
}
|
|
||||||
return merged
|
|
||||||
}
|
|
||||||
|
|
||||||
func ipv6PrefixNetwork(p netip.Prefix) *big.Int {
|
|
||||||
a := p.Masked().Addr().As16()
|
|
||||||
n := new(big.Int)
|
|
||||||
n.SetBytes(a[:])
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
func ipv6PrefixFromBigInt(n *big.Int, bits int) netip.Prefix {
|
|
||||||
b := n.Bytes()
|
|
||||||
var a [16]byte
|
|
||||||
copy(a[16-len(b):], b)
|
|
||||||
return netip.PrefixFrom(netip.AddrFrom16(a), bits).Masked()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func parentRevision(st store.Backend, tenantID, moduleID string) *string {
|
func parentRevision(st store.Backend, tenantID, moduleID string) *string {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package pipeline
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"hash/fnv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
@@ -10,6 +11,8 @@ import (
|
|||||||
const SchedulerTickSec = 30
|
const SchedulerTickSec = 30
|
||||||
|
|
||||||
// ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick.
|
// ModuleDueForScheduler reports whether a module's refresh interval bucket rolled since the last scheduler tick.
|
||||||
|
// A stable per-module offset (fnv32 of ID) spreads bucket boundaries so modules with the same interval
|
||||||
|
// do not all become due on the same tick (thundering herd).
|
||||||
func ModuleDueForScheduler(mod *store.Module, now time.Time) bool {
|
func ModuleDueForScheduler(mod *store.Module, now time.Time) bool {
|
||||||
if mod == nil || !mod.Enabled || mod.Type == "IP_RANGES" || mod.RefreshIntervalSec <= 0 {
|
if mod == nil || !mod.Enabled || mod.Type == "IP_RANGES" || mod.RefreshIntervalSec <= 0 {
|
||||||
return false
|
return false
|
||||||
@@ -18,7 +21,21 @@ func ModuleDueForScheduler(mod *store.Module, now time.Time) bool {
|
|||||||
if win < 60 {
|
if win < 60 {
|
||||||
win = 60
|
win = 60
|
||||||
}
|
}
|
||||||
cur := now.Unix() / win
|
offset := moduleSchedulerOffset(mod.ID, win)
|
||||||
prev := (now.Unix() - SchedulerTickSec) / win
|
cur := (now.Unix() - offset) / win
|
||||||
|
prev := (now.Unix() - offset - SchedulerTickSec) / win
|
||||||
return cur != prev
|
return cur != prev
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func moduleSchedulerOffset(moduleID string, win int64) int64 {
|
||||||
|
if win <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return int64(fnv32a(moduleID) % uint32(win))
|
||||||
|
}
|
||||||
|
|
||||||
|
func fnv32a(s string) uint32 {
|
||||||
|
h := fnv.New32a()
|
||||||
|
_, _ = h.Write([]byte(s))
|
||||||
|
return h.Sum32()
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
@@ -11,13 +12,26 @@ import (
|
|||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// asnCacheRowExistsOnce caches the schema check for the process lifetime (see moduleSnapshotRowTableExists).
|
||||||
|
var (
|
||||||
|
asnCacheRowExistsOnce sync.Once
|
||||||
|
asnCacheRowExistsCached bool
|
||||||
|
)
|
||||||
|
|
||||||
func asnCacheRowTableExists(ctx context.Context, q queryRower) bool {
|
func asnCacheRowTableExists(ctx context.Context, q queryRower) bool {
|
||||||
var n int
|
asnCacheRowExistsOnce.Do(func() {
|
||||||
err := q.QueryRow(ctx, `
|
var n int
|
||||||
SELECT 1 FROM information_schema.tables
|
err := q.QueryRow(ctx, `
|
||||||
WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row'
|
SELECT 1 FROM information_schema.tables
|
||||||
LIMIT 1`).Scan(&n)
|
WHERE table_schema = 'public' AND table_name = 'asn_prefix_cache_row'
|
||||||
return err == nil
|
LIMIT 1`).Scan(&n)
|
||||||
|
if err != nil {
|
||||||
|
asnCacheRowExistsOnce = sync.Once{}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
asnCacheRowExistsCached = true
|
||||||
|
})
|
||||||
|
return asnCacheRowExistsCached
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, bool, error) {
|
func (p *Postgres) GetASNPrefixCache(asn int64) (*store.ASNPrefixCacheEntry, bool, error) {
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
package repository
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (p *Postgres) GetDomainResolveCache(fqdn string) (*store.DomainResolveCacheEntry, bool, error) {
|
||||||
|
key := strings.ToLower(strings.TrimSpace(fqdn))
|
||||||
|
if key == "" {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
var raw []byte
|
||||||
|
var resolvedAt time.Time
|
||||||
|
err := p.pool.QueryRow(ctx, `
|
||||||
|
SELECT addrs_json, resolved_at FROM domain_resolve_cache WHERE fqdn = $1`, key).
|
||||||
|
Scan(&raw, &resolvedAt)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
var addrs []string
|
||||||
|
if len(raw) > 0 {
|
||||||
|
_ = json.Unmarshal(raw, &addrs)
|
||||||
|
}
|
||||||
|
return &store.DomainResolveCacheEntry{
|
||||||
|
FQDN: key,
|
||||||
|
Addrs: addrs,
|
||||||
|
ResolvedAt: resolvedAt.UTC(),
|
||||||
|
}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) SetDomainResolveCache(fqdn string, addrs []string) error {
|
||||||
|
key := strings.ToLower(strings.TrimSpace(fqdn))
|
||||||
|
if key == "" {
|
||||||
|
return store.ErrInvalidInput
|
||||||
|
}
|
||||||
|
if addrs == nil {
|
||||||
|
addrs = []string{}
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(addrs)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
_, err = p.pool.Exec(ctx, `
|
||||||
|
INSERT INTO domain_resolve_cache (fqdn, addrs_json, resolved_at)
|
||||||
|
VALUES ($1, $2::jsonb, now())
|
||||||
|
ON CONFLICT (fqdn) DO UPDATE SET
|
||||||
|
addrs_json = EXCLUDED.addrs_json,
|
||||||
|
resolved_at = EXCLUDED.resolved_at`,
|
||||||
|
key, string(raw))
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
@@ -12,13 +13,29 @@ import (
|
|||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// moduleSnapshotRowExistsOnce caches the schema check for the process lifetime.
|
||||||
|
// The table is created by migrations and never disappears at runtime; DB errors are
|
||||||
|
// not cached so a transient outage falls back to the JSON path only once.
|
||||||
|
var moduleSnapshotRowExistsOnce sync.Once
|
||||||
|
|
||||||
|
var moduleSnapshotRowExistsCached bool
|
||||||
|
|
||||||
func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool {
|
func moduleSnapshotRowTableExists(ctx context.Context, q queryRower) bool {
|
||||||
var n int
|
moduleSnapshotRowExistsOnce.Do(func() {
|
||||||
err := q.QueryRow(ctx, `
|
var n int
|
||||||
SELECT 1 FROM information_schema.tables
|
err := q.QueryRow(ctx, `
|
||||||
WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row'
|
SELECT 1 FROM information_schema.tables
|
||||||
LIMIT 1`).Scan(&n)
|
WHERE table_schema = 'public' AND table_name = 'module_prefix_snapshot_row'
|
||||||
return err == nil
|
LIMIT 1`).Scan(&n)
|
||||||
|
// A query error means the backend is unreachable or information_schema is hidden;
|
||||||
|
// treat as "absent" so callers fall back to prefixes_json, but retry next call.
|
||||||
|
if err != nil {
|
||||||
|
moduleSnapshotRowExistsOnce = sync.Once{}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
moduleSnapshotRowExistsCached = true
|
||||||
|
})
|
||||||
|
return moduleSnapshotRowExistsCached
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
func (p *Postgres) GetModulePrefixSnapshot(tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
||||||
@@ -97,19 +114,25 @@ func (p *Postgres) SetModulePrefixSnapshot(tenantID, moduleID, inputHash string,
|
|||||||
WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil {
|
WHERE tenant_id = $1::uuid AND module_id = $2::uuid`, tenantID, moduleID); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for i, pr := range prefixes {
|
if len(prefixes) > 0 {
|
||||||
var comm any
|
_, err = tx.CopyFrom(
|
||||||
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
ctx,
|
||||||
comm = strings.TrimSpace(*pr.CommunityID)
|
pgx.Identifier{"module_prefix_snapshot_row"},
|
||||||
}
|
[]string{"tenant_id", "module_id", "ord", "prefix", "community_id", "source"},
|
||||||
src := pr.Source
|
pgx.CopyFromSlice(len(prefixes), func(i int) ([]any, error) {
|
||||||
if strings.TrimSpace(src) == "" {
|
pr := prefixes[i]
|
||||||
src = "render"
|
var comm any
|
||||||
}
|
if pr.CommunityID != nil && strings.TrimSpace(*pr.CommunityID) != "" {
|
||||||
if _, err := tx.Exec(ctx, `
|
comm = strings.TrimSpace(*pr.CommunityID)
|
||||||
INSERT INTO module_prefix_snapshot_row (tenant_id, module_id, ord, prefix, community_id, source)
|
}
|
||||||
VALUES ($1::uuid, $2::uuid, $3, $4, $5::uuid, $6)`,
|
src := pr.Source
|
||||||
tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src); err != nil {
|
if strings.TrimSpace(src) == "" {
|
||||||
|
src = "render"
|
||||||
|
}
|
||||||
|
return []any{tenantID, moduleID, i, strings.TrimSpace(pr.Prefix), comm, src}, nil
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -135,3 +158,35 @@ func (p *Postgres) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
|
|||||||
tenantID, moduleID)
|
tenantID, moduleID)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) SetModuleInputHash(tenantID, moduleID, hash string) error {
|
||||||
|
ctx := context.Background()
|
||||||
|
tag, err := p.pool.Exec(ctx, `
|
||||||
|
UPDATE module SET input_hash = $3
|
||||||
|
WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`,
|
||||||
|
moduleID, tenantID, hash)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if tag.RowsAffected() == 0 {
|
||||||
|
return store.ErrNotFound
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) LockModuleSnapshot(tenantID, moduleID string) func() {
|
||||||
|
ctx := context.Background()
|
||||||
|
conn, err := p.pool.Acquire(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
key := strings.TrimSpace(tenantID) + "\x00" + strings.TrimSpace(moduleID)
|
||||||
|
if _, err := conn.Exec(ctx, `SELECT pg_advisory_lock(hashtext($1))`, key); err != nil {
|
||||||
|
conn.Release()
|
||||||
|
return func() {}
|
||||||
|
}
|
||||||
|
return func() {
|
||||||
|
_, _ = conn.Exec(ctx, `SELECT pg_advisory_unlock(hashtext($1))`, key)
|
||||||
|
conn.Release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -119,7 +119,8 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module {
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
rows, err := p.pool.Query(ctx, `
|
rows, err := p.pool.Query(ctx, `
|
||||||
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
||||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id
|
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
|
||||||
|
COALESCE(input_hash, '')
|
||||||
FROM module WHERE tenant_id = $1 AND deleted_at IS NULL ORDER BY priority, name`, tenantID)
|
FROM module WHERE tenant_id = $1 AND deleted_at IS NULL ORDER BY priority, name`, tenantID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -134,7 +135,7 @@ func (p *Postgres) ListModules(tenantID string) []*store.Module {
|
|||||||
var refresh *int32
|
var refresh *int32
|
||||||
var last *time.Time
|
var last *time.Time
|
||||||
var createdBy *string
|
var createdBy *string
|
||||||
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy); err != nil {
|
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash); err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
|
m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
|
||||||
@@ -179,7 +180,8 @@ func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
rows, err := p.pool.Query(ctx, `
|
rows, err := p.pool.Query(ctx, `
|
||||||
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
||||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id
|
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
|
||||||
|
COALESCE(input_hash, '')
|
||||||
FROM module WHERE tenant_id = $1 AND deleted_at IS NULL
|
FROM module WHERE tenant_id = $1 AND deleted_at IS NULL
|
||||||
ORDER BY priority, name
|
ORDER BY priority, name
|
||||||
LIMIT $2 OFFSET $3`, tenantID, limit+1, off)
|
LIMIT $2 OFFSET $3`, tenantID, limit+1, off)
|
||||||
@@ -196,7 +198,7 @@ func (p *Postgres) ListModulesPage(tenantID, cursor string, limit int) ([]*store
|
|||||||
var refresh *int32
|
var refresh *int32
|
||||||
var last *time.Time
|
var last *time.Time
|
||||||
var createdBy *string
|
var createdBy *string
|
||||||
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy); err != nil {
|
if err := rows.Scan(&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash); err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
|
m.DohResolverPolicy = store.NormalizeDohResolverPolicy(m.DohResolverPolicy)
|
||||||
@@ -249,9 +251,10 @@ func (p *Postgres) GetModule(tenantID, moduleID string) (*store.Module, error) {
|
|||||||
var createdBy *string
|
var createdBy *string
|
||||||
err := p.pool.QueryRow(ctx, `
|
err := p.pool.QueryRow(ctx, `
|
||||||
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
SELECT id, type, name, enabled, priority, doh_profile_id::text, doh_resolver_policy,
|
||||||
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id
|
refresh_interval_sec, cron_expr, default_community_id::text, last_refreshed_at, created_by_user_id,
|
||||||
|
COALESCE(input_hash, '')
|
||||||
FROM module WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, moduleID, tenantID).Scan(
|
FROM module WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL`, moduleID, tenantID).Scan(
|
||||||
&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy)
|
&m.ID, &m.Type, &m.Name, &m.Enabled, &m.Priority, &doh, &m.DohResolverPolicy, &refresh, &cron, &dc, &last, &createdBy, &m.InputHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, pgx.ErrNoRows) {
|
if errors.Is(err, pgx.ErrNoRows) {
|
||||||
return nil, store.ErrNotFound
|
return nil, store.ErrNotFound
|
||||||
@@ -325,9 +328,19 @@ func (p *Postgres) CreateModule(tenantID string, in *store.Module) (*store.Modul
|
|||||||
if err := p.setModuleDohProfiles(ctx, id, in.DohProfileIDs); err != nil {
|
if err := p.setModuleDohProfiles(ctx, id, in.DohProfileIDs); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
store.TouchModuleInputHash(p, tenantID, id)
|
||||||
return p.GetModule(tenantID, id)
|
return p.GetModule(tenantID, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) invalidateModuleInputHash(tenantID, moduleID string) {
|
||||||
|
_ = p.SetModuleInputHash(tenantID, moduleID, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *Postgres) invalidateTenantModuleHashes(tenantID string) {
|
||||||
|
ctx := context.Background()
|
||||||
|
_, _ = p.pool.Exec(ctx, `UPDATE module SET input_hash = '' WHERE tenant_id = $1 AND deleted_at IS NULL`, tenantID)
|
||||||
|
}
|
||||||
|
|
||||||
func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePatch) (*store.Module, error) {
|
func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePatch) (*store.Module, error) {
|
||||||
if patch == nil {
|
if patch == nil {
|
||||||
return nil, store.ErrInvalidInput
|
return nil, store.ErrInvalidInput
|
||||||
@@ -399,6 +412,7 @@ func (p *Postgres) UpdateModule(tenantID, moduleID string, patch *store.ModulePa
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return p.GetModule(tenantID, moduleID)
|
return p.GetModule(tenantID, moduleID)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1191,6 +1205,7 @@ func (p *Postgres) UpdateDohProfile(tenantID, id string, patch *store.DohProfile
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
p.invalidateTenantModuleHashes(tenantID)
|
||||||
return p.GetDohProfile(tenantID, id)
|
return p.GetDohProfile(tenantID, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ func (p *Postgres) CreateCDNSource(tenantID, moduleID string, in *store.CDNSourc
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return p.getCDNSource(ctx, moduleID, id)
|
return p.getCDNSource(ctx, moduleID, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -154,6 +155,9 @@ func (p *Postgres) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *s
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if store.CDNPatchAffectsInputHash(patch) {
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
|
}
|
||||||
return p.getCDNSource(ctx, moduleID, sourceID)
|
return p.getCDNSource(ctx, moduleID, sourceID)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,6 +173,7 @@ func (p *Postgres) DeleteCDNSource(tenantID, moduleID, sourceID string) error {
|
|||||||
if tag.RowsAffected() == 0 {
|
if tag.RowsAffected() == 0 {
|
||||||
return store.ErrNotFound
|
return store.ErrNotFound
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,6 +234,7 @@ func (p *Postgres) CreateASEntry(tenantID, moduleID string, in *store.ASEntry) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return p.getASEntry(ctx, moduleID, id)
|
return p.getASEntry(ctx, moduleID, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -297,6 +303,7 @@ func (p *Postgres) UpdateASEntry(tenantID, moduleID, entryID string, patch *stor
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return p.getASEntry(ctx, moduleID, entryID)
|
return p.getASEntry(ctx, moduleID, entryID)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -369,6 +376,7 @@ func (p *Postgres) DeleteASEntry(tenantID, moduleID, entryID string) error {
|
|||||||
if tag.RowsAffected() == 0 {
|
if tag.RowsAffected() == 0 {
|
||||||
return store.ErrNotFound
|
return store.ErrNotFound
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -417,6 +425,7 @@ func (p *Postgres) CreateDomainEntry(tenantID, moduleID string, in *store.Domain
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return p.getDomainEntry(ctx, moduleID, id)
|
return p.getDomainEntry(ctx, moduleID, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -462,6 +471,7 @@ func (p *Postgres) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return p.getDomainEntry(ctx, moduleID, entryID)
|
return p.getDomainEntry(ctx, moduleID, entryID)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -477,6 +487,7 @@ func (p *Postgres) DeleteDomainEntry(tenantID, moduleID, entryID string) error {
|
|||||||
if tag.RowsAffected() == 0 {
|
if tag.RowsAffected() == 0 {
|
||||||
return store.ErrNotFound
|
return store.ErrNotFound
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -525,6 +536,7 @@ func (p *Postgres) CreateIPRangeEntry(tenantID, moduleID string, in *store.IPRan
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return p.getIPRangeEntry(ctx, moduleID, id)
|
return p.getIPRangeEntry(ctx, moduleID, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -570,6 +582,7 @@ func (p *Postgres) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return p.getIPRangeEntry(ctx, moduleID, entryID)
|
return p.getIPRangeEntry(ctx, moduleID, entryID)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -585,6 +598,7 @@ func (p *Postgres) DeleteIPRangeEntry(tenantID, moduleID, entryID string) error
|
|||||||
if tag.RowsAffected() == 0 {
|
if tag.RowsAffected() == 0 {
|
||||||
return store.ErrNotFound
|
return store.ErrNotFound
|
||||||
}
|
}
|
||||||
|
p.invalidateModuleInputHash(tenantID, moduleID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -116,11 +116,19 @@ type Backend interface {
|
|||||||
GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
|
GetModulePrefixSnapshot(tenantID, moduleID string) (*ModulePrefixSnapshot, bool, error)
|
||||||
SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
|
SetModulePrefixSnapshot(tenantID, moduleID, inputHash string, prefixes []PrefixRow) error
|
||||||
DeleteModulePrefixSnapshot(tenantID, moduleID string) error
|
DeleteModulePrefixSnapshot(tenantID, moduleID string) error
|
||||||
|
// SetModuleInputHash stores the ingest fingerprint on the module row (O(1) snapshot check).
|
||||||
|
SetModuleInputHash(tenantID, moduleID, hash string) error
|
||||||
|
// LockModuleSnapshot serializes read-modify-write of one module snapshot; unlock must be called.
|
||||||
|
LockModuleSnapshot(tenantID, moduleID string) (unlock func())
|
||||||
|
|
||||||
// ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache).
|
// ASNPrefixCache stores RIPEstat announced-prefixes per ASN (global TTL cache).
|
||||||
GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error)
|
GetASNPrefixCache(asn int64) (*ASNPrefixCacheEntry, bool, error)
|
||||||
SetASNPrefixCache(asn int64, holder string, prefixes []string) error
|
SetASNPrefixCache(asn int64, holder string, prefixes []string) error
|
||||||
|
|
||||||
|
// DomainResolveCache stores DoH results per FQDN (global TTL cache).
|
||||||
|
GetDomainResolveCache(fqdn string) (*DomainResolveCacheEntry, bool, error)
|
||||||
|
SetDomainResolveCache(fqdn string, addrs []string) error
|
||||||
|
|
||||||
// Ping verifies backend connectivity (no-op for in-memory).
|
// Ping verifies backend connectivity (no-op for in-memory).
|
||||||
Ping(ctx context.Context) error
|
Ping(ctx context.Context) error
|
||||||
|
|
||||||
@@ -178,6 +186,13 @@ type ASNPrefixCacheEntry struct {
|
|||||||
FetchedAt time.Time
|
FetchedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DomainResolveCacheEntry is a cached DoH A/AAAA result for one FQDN.
|
||||||
|
type DomainResolveCacheEntry struct {
|
||||||
|
FQDN string
|
||||||
|
Addrs []string
|
||||||
|
ResolvedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
// ModulePrefixSnapshot is the cached materialization for one module between refreshes.
|
// ModulePrefixSnapshot is the cached materialization for one module between refreshes.
|
||||||
type ModulePrefixSnapshot struct {
|
type ModulePrefixSnapshot struct {
|
||||||
InputHash string
|
InputHash string
|
||||||
|
|||||||
@@ -44,7 +44,9 @@ type Memory struct {
|
|||||||
settings map[string]map[string]any // tenantID -> key -> JSON-compatible value
|
settings map[string]map[string]any // tenantID -> key -> JSON-compatible value
|
||||||
revPrefixes map[string][]PrefixRow
|
revPrefixes map[string][]PrefixRow
|
||||||
moduleSnapshots map[string]*moduleSnapshotRec
|
moduleSnapshots map[string]*moduleSnapshotRec
|
||||||
|
snapshotLocks sync.Map // key -> *sync.Mutex (per-module snapshot RMW)
|
||||||
asnPrefixCache map[int64]*ASNPrefixCacheEntry
|
asnPrefixCache map[int64]*ASNPrefixCacheEntry
|
||||||
|
domainResolveCache map[string]*DomainResolveCacheEntry
|
||||||
apiKeys map[string]*apiKeyRec
|
apiKeys map[string]*apiKeyRec
|
||||||
firewallClients map[string]*firewallClientRec
|
firewallClients map[string]*firewallClientRec
|
||||||
firewallRules map[string]*FirewallRule
|
firewallRules map[string]*FirewallRule
|
||||||
@@ -99,6 +101,7 @@ type Module struct {
|
|||||||
LastRefreshedAt *time.Time
|
LastRefreshedAt *time.Time
|
||||||
DeletedAt *time.Time
|
DeletedAt *time.Time
|
||||||
CreatedByUserID string // portal JWT sub; empty = system / API key
|
CreatedByUserID string // portal JWT sub; empty = system / API key
|
||||||
|
InputHash string // ingest fingerprint; empty = not yet computed
|
||||||
}
|
}
|
||||||
|
|
||||||
type Revision struct {
|
type Revision struct {
|
||||||
@@ -156,6 +159,7 @@ func NewMemory() *Memory {
|
|||||||
revPrefixes: make(map[string][]PrefixRow),
|
revPrefixes: make(map[string][]PrefixRow),
|
||||||
moduleSnapshots: make(map[string]*moduleSnapshotRec),
|
moduleSnapshots: make(map[string]*moduleSnapshotRec),
|
||||||
asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry),
|
asnPrefixCache: make(map[int64]*ASNPrefixCacheEntry),
|
||||||
|
domainResolveCache: make(map[string]*DomainResolveCacheEntry),
|
||||||
apiKeys: make(map[string]*apiKeyRec),
|
apiKeys: make(map[string]*apiKeyRec),
|
||||||
firewallClients: make(map[string]*firewallClientRec),
|
firewallClients: make(map[string]*firewallClientRec),
|
||||||
firewallRules: make(map[string]*FirewallRule),
|
firewallRules: make(map[string]*FirewallRule),
|
||||||
@@ -696,6 +700,20 @@ func cloneStringPtr(s *string) *string {
|
|||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Memory) clearModuleInputHashLocked(moduleID string) {
|
||||||
|
if mod, ok := m.modules[moduleID]; ok && mod != nil {
|
||||||
|
mod.InputHash = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Memory) clearTenantModuleHashesLocked(tenantID string) {
|
||||||
|
for _, mod := range m.modules {
|
||||||
|
if mod != nil && mod.TenantID == tenantID && mod.DeletedAt == nil {
|
||||||
|
mod.InputHash = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func cloneModule(m *Module) *Module {
|
func cloneModule(m *Module) *Module {
|
||||||
if m == nil {
|
if m == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ func (m *Memory) UpdateModule(tenantID, moduleID string, patch *ModulePatch) (*M
|
|||||||
t := patch.LastRefreshedAt.UTC()
|
t := patch.LastRefreshedAt.UTC()
|
||||||
mod.LastRefreshedAt = &t
|
mod.LastRefreshedAt = &t
|
||||||
}
|
}
|
||||||
|
mod.InputHash = ""
|
||||||
return cloneModule(mod), nil
|
return cloneModule(mod), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,6 +153,7 @@ func (m *Memory) CreateCDNSource(tenantID, moduleID string, in *CDNSource) (*CDN
|
|||||||
LastRefreshedAt: in.LastRefreshedAt,
|
LastRefreshedAt: in.LastRefreshedAt,
|
||||||
}
|
}
|
||||||
m.cdnSources[id] = s
|
m.cdnSources[id] = s
|
||||||
|
mod.InputHash = ""
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,7 +163,8 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
|
|||||||
}
|
}
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil {
|
mod, err := m.moduleWriteOK(tenantID, moduleID)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
s, ok := m.cdnSources[sourceID]
|
s, ok := m.cdnSources[sourceID]
|
||||||
@@ -195,6 +198,9 @@ func (m *Memory) UpdateCDNSource(tenantID, moduleID, sourceID string, patch *CDN
|
|||||||
t := patch.LastRefreshedAt.UTC()
|
t := patch.LastRefreshedAt.UTC()
|
||||||
s.LastRefreshedAt = &t
|
s.LastRefreshedAt = &t
|
||||||
}
|
}
|
||||||
|
if CDNPatchAffectsInputHash(patch) {
|
||||||
|
mod.InputHash = ""
|
||||||
|
}
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,6 +215,7 @@ func (m *Memory) DeleteCDNSource(tenantID, moduleID, sourceID string) error {
|
|||||||
return ErrNotFound
|
return ErrNotFound
|
||||||
}
|
}
|
||||||
delete(m.cdnSources, sourceID)
|
delete(m.cdnSources, sourceID)
|
||||||
|
m.clearModuleInputHashLocked(moduleID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,6 +254,7 @@ func (m *Memory) CreateASEntry(tenantID, moduleID string, in *ASEntry) (*ASEntry
|
|||||||
id := uuid.NewString()
|
id := uuid.NewString()
|
||||||
e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID}
|
e := &ASEntry{ID: id, ModuleID: moduleID, ASN: in.ASN, CommunityID: in.CommunityID}
|
||||||
m.asEntries[id] = e
|
m.asEntries[id] = e
|
||||||
|
mod.InputHash = ""
|
||||||
return e, nil
|
return e, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -256,7 +264,8 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr
|
|||||||
}
|
}
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil {
|
mod, err := m.moduleWriteOK(tenantID, moduleID)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
e, ok := m.asEntries[entryID]
|
e, ok := m.asEntries[entryID]
|
||||||
@@ -283,6 +292,7 @@ func (m *Memory) UpdateASEntry(tenantID, moduleID, entryID string, patch *ASEntr
|
|||||||
e.PrefixCount = nil
|
e.PrefixCount = nil
|
||||||
e.ASNResolvedAt = nil
|
e.ASNResolvedAt = nil
|
||||||
}
|
}
|
||||||
|
mod.InputHash = ""
|
||||||
return e, nil
|
return e, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -324,6 +334,7 @@ func (m *Memory) DeleteASEntry(tenantID, moduleID, entryID string) error {
|
|||||||
return ErrNotFound
|
return ErrNotFound
|
||||||
}
|
}
|
||||||
delete(m.asEntries, entryID)
|
delete(m.asEntries, entryID)
|
||||||
|
m.clearModuleInputHashLocked(moduleID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,6 +373,7 @@ func (m *Memory) CreateDomainEntry(tenantID, moduleID string, in *DomainEntry) (
|
|||||||
id := uuid.NewString()
|
id := uuid.NewString()
|
||||||
e := &DomainEntry{ID: id, ModuleID: moduleID, FQDN: strings.TrimSpace(in.FQDN), CommunityID: in.CommunityID}
|
e := &DomainEntry{ID: id, ModuleID: moduleID, FQDN: strings.TrimSpace(in.FQDN), CommunityID: in.CommunityID}
|
||||||
m.domainEnt[id] = e
|
m.domainEnt[id] = e
|
||||||
|
mod.InputHash = ""
|
||||||
return e, nil
|
return e, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -371,7 +383,8 @@ func (m *Memory) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *Do
|
|||||||
}
|
}
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil {
|
mod, err := m.moduleWriteOK(tenantID, moduleID)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
e, ok := m.domainEnt[entryID]
|
e, ok := m.domainEnt[entryID]
|
||||||
@@ -389,6 +402,7 @@ func (m *Memory) UpdateDomainEntry(tenantID, moduleID, entryID string, patch *Do
|
|||||||
e.CommunityID = &v
|
e.CommunityID = &v
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
mod.InputHash = ""
|
||||||
return e, nil
|
return e, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -403,6 +417,7 @@ func (m *Memory) DeleteDomainEntry(tenantID, moduleID, entryID string) error {
|
|||||||
return ErrNotFound
|
return ErrNotFound
|
||||||
}
|
}
|
||||||
delete(m.domainEnt, entryID)
|
delete(m.domainEnt, entryID)
|
||||||
|
m.clearModuleInputHashLocked(moduleID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -441,6 +456,7 @@ func (m *Memory) CreateIPRangeEntry(tenantID, moduleID string, in *IPRangeEntry)
|
|||||||
id := uuid.NewString()
|
id := uuid.NewString()
|
||||||
e := &IPRangeEntry{ID: id, ModuleID: moduleID, Prefix: strings.TrimSpace(in.Prefix), CommunityID: in.CommunityID}
|
e := &IPRangeEntry{ID: id, ModuleID: moduleID, Prefix: strings.TrimSpace(in.Prefix), CommunityID: in.CommunityID}
|
||||||
m.ipRanges[id] = e
|
m.ipRanges[id] = e
|
||||||
|
mod.InputHash = ""
|
||||||
return e, nil
|
return e, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -450,7 +466,8 @@ func (m *Memory) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch *I
|
|||||||
}
|
}
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
if _, err := m.moduleWriteOK(tenantID, moduleID); err != nil {
|
mod, err := m.moduleWriteOK(tenantID, moduleID)
|
||||||
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
e, ok := m.ipRanges[entryID]
|
e, ok := m.ipRanges[entryID]
|
||||||
@@ -468,6 +485,7 @@ func (m *Memory) UpdateIPRangeEntry(tenantID, moduleID, entryID string, patch *I
|
|||||||
e.CommunityID = &v
|
e.CommunityID = &v
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
mod.InputHash = ""
|
||||||
return e, nil
|
return e, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -482,6 +500,7 @@ func (m *Memory) DeleteIPRangeEntry(tenantID, moduleID, entryID string) error {
|
|||||||
return ErrNotFound
|
return ErrNotFound
|
||||||
}
|
}
|
||||||
delete(m.ipRanges, entryID)
|
delete(m.ipRanges, entryID)
|
||||||
|
m.clearModuleInputHashLocked(moduleID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -551,6 +570,7 @@ func (m *Memory) UpdateDohProfile(tenantID, id string, patch *DohProfilePatch) (
|
|||||||
if patch.SecretRef != nil {
|
if patch.SecretRef != nil {
|
||||||
p.SecretRef = patch.SecretRef
|
p.SecretRef = patch.SecretRef
|
||||||
}
|
}
|
||||||
|
m.clearTenantModuleHashesLocked(tenantID)
|
||||||
return p, nil
|
return p, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (m *Memory) GetDomainResolveCache(fqdn string) (*DomainResolveCacheEntry, bool, error) {
|
||||||
|
m.mu.RLock()
|
||||||
|
defer m.mu.RUnlock()
|
||||||
|
if m.domainResolveCache == nil {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
e, ok := m.domainResolveCache[strings.ToLower(strings.TrimSpace(fqdn))]
|
||||||
|
if !ok || e == nil {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
return &DomainResolveCacheEntry{
|
||||||
|
FQDN: e.FQDN,
|
||||||
|
Addrs: append([]string(nil), e.Addrs...),
|
||||||
|
ResolvedAt: e.ResolvedAt,
|
||||||
|
}, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Memory) SetDomainResolveCache(fqdn string, addrs []string) error {
|
||||||
|
key := strings.ToLower(strings.TrimSpace(fqdn))
|
||||||
|
if key == "" {
|
||||||
|
return ErrInvalidInput
|
||||||
|
}
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
if m.domainResolveCache == nil {
|
||||||
|
m.domainResolveCache = make(map[string]*DomainResolveCacheEntry)
|
||||||
|
}
|
||||||
|
m.domainResolveCache[key] = &DomainResolveCacheEntry{
|
||||||
|
FQDN: key,
|
||||||
|
Addrs: append([]string(nil), addrs...),
|
||||||
|
ResolvedAt: time.Now().UTC(),
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ package store
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -51,6 +52,28 @@ func (m *Memory) DeleteModulePrefixSnapshot(tenantID, moduleID string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Memory) SetModuleInputHash(tenantID, moduleID, hash string) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
mod, ok := m.modules[moduleID]
|
||||||
|
if !ok || mod.DeletedAt != nil {
|
||||||
|
return ErrNotFound
|
||||||
|
}
|
||||||
|
if mod.TenantID != tenantID {
|
||||||
|
return ErrTenantScope
|
||||||
|
}
|
||||||
|
mod.InputHash = strings.TrimSpace(hash)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Memory) LockModuleSnapshot(tenantID, moduleID string) func() {
|
||||||
|
key := moduleSnapshotKey(tenantID, moduleID)
|
||||||
|
v, _ := m.snapshotLocks.LoadOrStore(key, &sync.Mutex{})
|
||||||
|
mu := v.(*sync.Mutex)
|
||||||
|
mu.Lock()
|
||||||
|
return func() { mu.Unlock() }
|
||||||
|
}
|
||||||
|
|
||||||
type moduleSnapshotRec struct {
|
type moduleSnapshotRec struct {
|
||||||
InputHash string
|
InputHash string
|
||||||
CollectedAt time.Time
|
CollectedAt time.Time
|
||||||
|
|||||||
@@ -0,0 +1,123 @@
|
|||||||
|
package store
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ComputeModuleInputHash fingerprints module config and child entries so snapshots
|
||||||
|
// invalidate on CRUD without re-reading every child at render time.
|
||||||
|
func ComputeModuleInputHash(st Backend, tenantID string, mod *Module) (string, error) {
|
||||||
|
if st == nil || mod == nil {
|
||||||
|
return "", fmt.Errorf("store: module hash: missing store or module")
|
||||||
|
}
|
||||||
|
h := sha256.New()
|
||||||
|
_, _ = fmt.Fprintf(h, "type=%s\n", strings.TrimSpace(mod.Type))
|
||||||
|
_, _ = fmt.Fprintf(h, "enabled=%t\n", mod.Enabled)
|
||||||
|
if mod.DefaultCommunityID != nil {
|
||||||
|
_, _ = fmt.Fprintf(h, "default_community=%s\n", strings.TrimSpace(*mod.DefaultCommunityID))
|
||||||
|
}
|
||||||
|
_, _ = fmt.Fprintf(h, "doh_policy=%s\n", NormalizeDohResolverPolicy(mod.DohResolverPolicy))
|
||||||
|
for _, pid := range mod.EffectiveDohProfileIDs() {
|
||||||
|
_, _ = fmt.Fprintf(h, "doh_profile=%s\n", pid)
|
||||||
|
if prof, err := st.GetDohProfile(tenantID, pid); err == nil && prof != nil {
|
||||||
|
_, _ = fmt.Fprintf(h, "doh_url=%s\n", strings.TrimSpace(prof.URL))
|
||||||
|
if prof.TimeoutMs != nil {
|
||||||
|
_, _ = fmt.Fprintf(h, "doh_timeout=%d\n", *prof.TimeoutMs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch mod.Type {
|
||||||
|
case "IP_RANGES":
|
||||||
|
list, err := st.ListIPRangeEntries(tenantID, mod.ID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
sort.Slice(list, func(i, j int) bool { return list[i].Prefix < list[j].Prefix })
|
||||||
|
for _, e := range list {
|
||||||
|
comm := ""
|
||||||
|
if e.CommunityID != nil {
|
||||||
|
comm = *e.CommunityID
|
||||||
|
}
|
||||||
|
_, _ = fmt.Fprintf(h, "ip=%s|c=%s\n", e.Prefix, comm)
|
||||||
|
}
|
||||||
|
case "AS_PREFIXES":
|
||||||
|
list, err := st.ListASEntries(tenantID, mod.ID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
sort.Slice(list, func(i, j int) bool { return list[i].ASN < list[j].ASN })
|
||||||
|
for _, e := range list {
|
||||||
|
comm := ""
|
||||||
|
if e.CommunityID != nil {
|
||||||
|
comm = *e.CommunityID
|
||||||
|
}
|
||||||
|
_, _ = fmt.Fprintf(h, "as=%d|c=%s\n", e.ASN, comm)
|
||||||
|
}
|
||||||
|
case "CDN_CIDRS":
|
||||||
|
list, err := st.ListCDNSources(tenantID, mod.ID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
sort.Slice(list, func(i, j int) bool { return list[i].ID < list[j].ID })
|
||||||
|
for _, s := range list {
|
||||||
|
comm := ""
|
||||||
|
if s.CommunityID != nil {
|
||||||
|
comm = *s.CommunityID
|
||||||
|
}
|
||||||
|
interval := 0
|
||||||
|
if s.RefreshIntervalSec != nil {
|
||||||
|
interval = *s.RefreshIntervalSec
|
||||||
|
}
|
||||||
|
_, _ = fmt.Fprintf(h, "cdn=%s|url=%s|kind=%s|path=%s|c=%s|etag=%s|interval=%d\n",
|
||||||
|
s.ID, strings.TrimSpace(s.URL), s.SourceKind, strings.TrimSpace(s.PrefixPath), comm,
|
||||||
|
strings.TrimSpace(s.Etag), interval)
|
||||||
|
}
|
||||||
|
case "DOMAINS":
|
||||||
|
list, err := st.ListDomainEntries(tenantID, mod.ID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
sort.Slice(list, func(i, j int) bool { return list[i].FQDN < list[j].FQDN })
|
||||||
|
for _, e := range list {
|
||||||
|
comm := ""
|
||||||
|
if e.CommunityID != nil {
|
||||||
|
comm = *e.CommunityID
|
||||||
|
}
|
||||||
|
_, _ = fmt.Fprintf(h, "dom=%s|c=%s\n", strings.TrimSpace(e.FQDN), comm)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
_, _ = fmt.Fprintf(h, "unknown_type=%s\n", mod.Type)
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(h.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TouchModuleInputHash recomputes and stores module.input_hash (ARCH-01: hash lives in store).
|
||||||
|
func TouchModuleInputHash(st Backend, tenantID, moduleID string) {
|
||||||
|
if st == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
mod, err := st.GetModule(tenantID, moduleID)
|
||||||
|
if err != nil || mod == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h, err := ComputeModuleInputHash(st, tenantID, mod)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = st.SetModuleInputHash(tenantID, moduleID, h)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CDNPatchAffectsInputHash reports whether a CDN source patch changes ingest fingerprint
|
||||||
|
// fields (URL/kind/path/community/interval). ETag and last_refreshed_at do not.
|
||||||
|
func CDNPatchAffectsInputHash(patch *CDNSourcePatch) bool {
|
||||||
|
if patch == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return patch.SourceKind != nil || patch.URL != nil || patch.PrefixPath != nil ||
|
||||||
|
patch.CommunityID != nil || patch.RefreshIntervalSec != nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE module
|
||||||
|
DROP COLUMN IF EXISTS input_hash;
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE module
|
||||||
|
ADD COLUMN input_hash TEXT NOT NULL DEFAULT '';
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
DROP TABLE IF EXISTS domain_resolve_cache;
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
-- TTL cache for DoH/A/AAAA resolutions (pipeline domain ingest).
|
||||||
|
CREATE TABLE domain_resolve_cache (
|
||||||
|
fqdn TEXT PRIMARY KEY,
|
||||||
|
addrs_json JSONB NOT NULL DEFAULT '[]',
|
||||||
|
resolved_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_domain_resolve_cache_resolved ON domain_resolve_cache (resolved_at);
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE module
|
||||||
|
DROP COLUMN input_hash;
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
ALTER TABLE module
|
||||||
|
ADD COLUMN input_hash TEXT NOT NULL DEFAULT '';
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
DROP TABLE IF EXISTS domain_resolve_cache;
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
CREATE TABLE domain_resolve_cache (
|
||||||
|
fqdn TEXT PRIMARY KEY,
|
||||||
|
addrs_json TEXT NOT NULL DEFAULT '[]',
|
||||||
|
resolved_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_domain_resolve_cache_resolved ON domain_resolve_cache (resolved_at);
|
||||||
Reference in New Issue
Block a user