fix(api): сохранять ошибки наблюдения BGP для оповещений
This commit is contained in:
1 parent
5ac5a25dfc
commit
4f084222ad
18 files changed
+478
-25
No files matched your search
@@ -8,7 +8,7 @@
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"build": "npm run build -w @mmapp/contracts && tsc",
|
||||
"test:alerts-telegram": "tsx src/services/alerts-outbox-retry.test.ts && tsx src/services/alerts-outbox-overview.test.ts && tsx src/services/alert-engine/outbox-execution.test.ts && tsx src/services/alerts-telegram-delivery.test.ts && tsx src/services/alerts-revision.test.ts && tsx src/services/alerts-overview.test.ts && tsx ../packages/contracts/src/alerts-normalization.test.ts && tsx src/services/alerts-config-execution.test.ts && tsx src/services/alerts-telegram-settings.test.ts && tsx src/services/alerts-telegram-send.test.ts && tsx src/routes/alerts.test.ts && tsx src/plugins/auth.alerts.test.ts",
|
||||
"test:alert-engine": "tsx src/services/alert-engine/server-signal-identity.test.ts && tsx src/services/alert-engine/collector-source-signals.test.ts && tsx src/services/alert-engine/monitoring-history.test.ts && tsx src/services/alert-engine/groups.test.ts && tsx src/services/alert-engine/evaluate-rule.test.ts && tsx src/services/gre-bgp-snapshot-collector.test.ts && tsx src/services/gre-bgp-snapshot-history.test.ts && tsx src/services/gre-bgp-snapshot-sql.test.ts && tsx src/services/alert-engine/collector-source-sql.test.ts && tsx src/services/alert-engine/confirm-stability.test.ts",
|
||||
"test:alert-engine": "tsx src/services/alert-engine/server-signal-identity.test.ts && tsx src/services/alert-engine/collector-source-signals.test.ts && tsx src/services/alert-engine/monitoring-history.test.ts && tsx src/services/alert-engine/groups.test.ts && tsx src/services/alert-engine/evaluate-rule.test.ts && tsx src/services/bgp-peers-live.test.ts && tsx src/services/gre-bgp-snapshot-collector.test.ts && tsx src/services/gre-bgp-snapshot-history.test.ts && tsx src/services/gre-bgp-snapshot-sql.test.ts && tsx src/services/alert-engine/collector-source-sql.test.ts && tsx src/services/alert-engine/confirm-stability.test.ts",
|
||||
"start": "node dist/index.js",
|
||||
"db:generate": "drizzle-kit generate",
|
||||
"db:migrate": "drizzle-kit migrate",
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import fs from 'node:fs'
|
||||
import {createServer} from 'node:http'
|
||||
import {randomBytes} from 'node:crypto'
|
||||
const database=process.env.MMAPP_OWN_SQL_DB,url=new URL(process.env.DATABASE_URL)
|
||||
assert(/^mm_accept_166_\d+$/.test(database??''));assert.equal(url.pathname,'/'+database);assert.equal(url.username,database)
|
||||
process.env.AUTH_REQUIRED='true';process.env.AUTH_JWT_SECRET=randomBytes(32).toString('hex');process.env.AUTH_ISSUER='https://own216.invalid';process.env.AUTH_PORTAL_URL='https://own216.invalid'
|
||||
const {pool,closePool}=await import('/app/backend/dist/db/index.js'),{initDatabase}=await import('/app/backend/dist/db/bootstrap.js'),{buildApp}=await import('/app/backend/dist/index.js'),{fetchBgpSessionsForAlerts}=await import('/app/backend/dist/services/bgp-peers-live.js'),{collectGreBgpSnapshotOnce}=await import('/app/backend/dist/services/gre-bgp-snapshot-collector.js')
|
||||
assert.equal((await pool.query('SELECT current_database() AS name')).rows[0].name,database);assert(Object.values((await pool.query('SELECT rolsuper,rolcreatedb,rolcreaterole,rolbypassrls FROM pg_roles WHERE rolname=current_user')).rows[0]).every(v=>v===false))
|
||||
let phase='populated',app,nativeCalls=0,nativeWrites=0,disabledCalls=0;const cases=[]
|
||||
const fixture=label=>createServer((req,res)=>{nativeCalls++;if(label==='disabled')disabledCalls++;if(req.method!=='GET')nativeWrites++;res.setHeader('content-type','application/json');if(phase==='failed'||(phase==='partial'&&label==='second')){res.statusCode=401;res.end(JSON.stringify({error:401,message:'own216 fixture denied'}));return}const rows=phase==='populated'?[{'.id':'*1',name:'own216-peer',state:'established'}]:[];res.end(JSON.stringify(rows))})
|
||||
const natives=['first','second','disabled'].map(fixture);for(const n of natives)await new Promise(r=>n.listen(0,'127.0.0.1',r))
|
||||
const baseline=process.env.MMAPP_BASELINE==='true';const report={date:'2026-10-14',block:'12.5.216',baseline,database,cases,scope:'Compiled production routes/auth, isolated PostgreSQL and RouterOS HTTP fixture; real CHR unchanged',passed:false}
|
||||
try{
|
||||
await initDatabase();const ids=[];for(let i=0;i<natives.length;i++)ids.push(Number((await pool.query("INSERT INTO servers(name,host,port,use_ssl,enabled,username,password,type) VALUES($1,'127.0.0.1',$2,false,$3,'fixture','fixture',$4) RETURNING id",['own216-'+i,natives[i].address().port,i<2,i===2?'home-router':'exit-node'])).rows[0].id))
|
||||
app=await buildApp({logger:false,startScheduler:false});await app.ready();const token=app.jwt.sign({sub:'own216-reader',apps:['mm'],permissions:['mm:bgp:read'],iss:process.env.AUTH_ISSUER},{expiresIn:'5m'}),headers={authorization:'Bearer '+token}
|
||||
const wrongToken=app.jwt.sign({sub:'own216-other-reader',apps:['mm'],permissions:['mm:network:read'],iss:process.env.AUTH_ISSUER},{expiresIn:'5m'});for(const [name,h,status] of [['anonymous',{},401],['wrong permission',{authorization:'Bearer '+wrongToken},403]]){const before=nativeCalls,r=await app.inject({method:'GET',url:'/api/bgp/sessions',headers:h});assert.equal(r.statusCode,status);assert.equal(nativeCalls,before);cases.push({phase:name,endpoint:'bgp/sessions',status,nativeCalls:0})}
|
||||
for(phase of ['populated','empty','partial','failed','disabled']){
|
||||
if(phase==='disabled')await pool.query('UPDATE servers SET enabled=false')
|
||||
const hasFailure=['partial','failed'].includes(phase),expected=phase==='partial'?[ids[1]]:phase==='failed'?ids.slice(0,2):[]
|
||||
for(const endpoint of ['bgp/sessions','bgp/catalog']){
|
||||
const before=nativeCalls,r=await app.inject({method:'GET',url:'/api/'+endpoint,headers}),body=r.json(),isCatalog=endpoint==='bgp/catalog'
|
||||
assert.equal(r.statusCode,!baseline&&hasFailure&&!isCatalog?503:200)
|
||||
if(isCatalog||(!baseline&&hasFailure)){assert.deepEqual(body.failures.map(f=>f.serverId),expected);if(isCatalog)assert.equal(body.sessions.length,phase==='populated'?2:0)}else{assert(Array.isArray(body));assert.equal(body.length,phase==='populated'?2:0)}
|
||||
if(phase==='disabled')assert.equal(nativeCalls,before)
|
||||
cases.push({phase,endpoint,status:r.statusCode,failures:body.failures?.length??0,nativeCalls:nativeCalls-before})
|
||||
}
|
||||
const snapshot=await collectGreBgpSnapshotOnce({fetchGre:async()=>[{targetLabel:'own216-GRE-control',status:'up'}],fetchBgp:fetchBgpSessionsForAlerts})
|
||||
assert.equal(snapshot.greWritten,1);assert.equal(snapshot.bgpWritten,phase==='populated'?2:0);assert.equal(snapshot.errors.length,!baseline&&hasFailure?1:0)
|
||||
if(snapshot.errors.length)assert.match(snapshot.errors[0],/BGP: Не опрошены устройства BGP/)
|
||||
cases.push({phase,endpoint:'compiled snapshot collector',bgpWritten:snapshot.bgpWritten,greWritten:1,sourceErrors:snapshot.errors.length})
|
||||
}
|
||||
assert.equal(nativeWrites,0);assert.equal(disabledCalls,0);report.passed=true
|
||||
}catch(e){report.errorName=e.name;fs.writeFileSync('/app/backend/acceptance/bgp216-private.log',String(e.stack??e),{mode:0o600})}finally{if(app)await app.close();for(const n of natives){n.closeAllConnections();await new Promise(r=>n.close(r))}await closePool();report.nativeWrites=nativeWrites;report.disabledHomeRouterCalls=disabledCalls;fs.writeFileSync('/app/backend/acceptance/bgp216-'+(baseline?'baseline':'fixed')+'.json',JSON.stringify(report,null,2));console.log(JSON.stringify(report))}if(!report.passed)process.exitCode=1
|
||||
@@ -39,7 +39,8 @@ const catalog = await app.inject({ method: "GET", url: "/api/bgp/catalog" })
|
||||
assert.equal(catalog.statusCode, 200)
|
||||
assert.deepEqual(catalog.json(), mixed)
|
||||
const legacy = await app.inject({ method: "GET", url: "/api/bgp/sessions" })
|
||||
assert.deepEqual(legacy.json(), [row])
|
||||
assert.equal(legacy.statusCode, 503)
|
||||
assert.deepEqual(legacy.json().failures, mixed.failures)
|
||||
const selected = await app.inject({ method: "GET", url: "/api/servers/2/bgp/sessions" })
|
||||
assert.equal(selected.statusCode, 503)
|
||||
assert.equal(selected.json().error, "Таймаут BGP")
|
||||
|
||||
@@ -51,6 +51,9 @@ export function createBgpRoutes(dependencies: BgpDependencies = defaults): Fasti
|
||||
app.get("/bgp/sessions", async (_req, reply) => {
|
||||
const devices = await dependencies.listEnabledServers()
|
||||
const catalog = await pollBgpCatalog(devices, dependencies.readSessions)
|
||||
if (catalog.failures.length) return reply.status(503).send({
|
||||
error: "Не удалось опросить все устройства BGP", failures: catalog.failures,
|
||||
})
|
||||
return reply.send(catalog.sessions)
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import {fetchBgpSessionsForAlerts} from './bgp-peers-live.js'
|
||||
import {collectGreBgpSnapshotOnce} from './gre-bgp-snapshot-collector.js'
|
||||
import {servers} from '../db/schema.js'
|
||||
import type {BgpSessionRead} from '../types/server.js'
|
||||
const devices=[{id:1,name:'Первый'},{id:2,name:'Второй'}] as typeof servers.$inferSelect[]
|
||||
let calls=0
|
||||
for(const phase of ['populated','empty','partial','failed','disabled','malformed'] as const){
|
||||
const deps={listEnabledServers:async()=>phase==='disabled'?[]:devices,readSessions:async(server:typeof servers.$inferSelect)=>{calls++;if(phase==='failed'||(phase==='partial'&&server.id===2))throw Error('native401');if(phase==='malformed')return [{id:'*1',serverId:99}] as BgpSessionRead[];return phase==='populated'?[{id:'*1',serverId:server.id,serverName:server.name,name:'own-peer',state:'Established'} as BgpSessionRead]:[]}}
|
||||
const before=calls,result=await collectGreBgpSnapshotOnce({fetchGre:async()=>[{targetLabel:'GRE-control',status:'up'}],fetchBgp:()=>fetchBgpSessionsForAlerts(deps)})
|
||||
assert.equal(result.greWritten,1);assert.equal(result.bgpWritten,phase==='populated'?2:0);assert.equal(result.errors.length,['partial','failed','malformed'].includes(phase)?1:0)
|
||||
if(result.errors.length)assert.match(result.errors[0],/BGP: Не опрошены устройства BGP/)
|
||||
if(phase==='disabled')assert.equal(calls,before)
|
||||
}
|
||||
console.log('BGP alert observations: populated/empty/partial/failed/disabled/malformed; source errors recorded, GRE preserved')
|
||||
@@ -14,21 +14,21 @@ export function bgpPeerAlertKey(s: Pick<BgpSessionRead, "serverName" | "name">):
|
||||
return `${sn}: ${nm}`.trim()
|
||||
}
|
||||
|
||||
/** Агрегат BGP-сессий с включённых серверов — как `/api/bgp/sessions`. */
|
||||
export async function fetchBgpSessionsForAlerts(): Promise<BgpSessionRead[]> {
|
||||
const allServers = await db.select().from(servers).where(eq(servers.enabled, true))
|
||||
const results = await Promise.all(
|
||||
allServers.map(async (server) => {
|
||||
try {
|
||||
const client = MikrotikClient.fromServer(server)
|
||||
const raw = await client.getBgpSessions()
|
||||
return parseBgpSessions(server, raw)
|
||||
} catch {
|
||||
return [] as BgpSessionRead[]
|
||||
}
|
||||
}),
|
||||
)
|
||||
return results.flat()
|
||||
interface BgpAlertDependencies {
|
||||
listEnabledServers: () => Promise<Array<typeof servers.$inferSelect>>
|
||||
readSessions: (server: typeof servers.$inferSelect) => Promise<BgpSessionRead[]>
|
||||
}
|
||||
const alertDefaults: BgpAlertDependencies = {
|
||||
listEnabledServers: () => db.select().from(servers).where(eq(servers.enabled, true)),
|
||||
readSessions: async server => parseBgpSessions(server, await MikrotikClient.fromServer(server).getBgpSessions()),
|
||||
}
|
||||
|
||||
/** An incomplete poll is an observation error, never a confirmed empty snapshot. */
|
||||
export async function fetchBgpSessionsForAlerts(dependencies: BgpAlertDependencies = alertDefaults): Promise<BgpSessionRead[]> {
|
||||
const catalog = await pollBgpCatalog(await dependencies.listEnabledServers(), dependencies.readSessions)
|
||||
if (catalog.failures.length) throw new Error("Не опрошены устройства BGP: " +
|
||||
catalog.failures.map(failure => `${failure.serverName}: ${failure.error}`).join("; "))
|
||||
return catalog.sessions
|
||||
}
|
||||
|
||||
export async function countBgpSessions():Promise<number|undefined>{
|
||||
|
||||
Reference in new issue
Block a user