import { type Express } from "express"; import { readFileSync, writeFileSync, existsSync } from "fs"; import { resolve } from "path"; import { query } from "./db-client"; import { getTableConfig, getIncomeCategories, getExpenseCategories, getIncomeAutoDetectConfig, getExpenseAutoDetectConfig, type IncomeCategoryFile, type AutoDetectConfig, } from "./table-config"; import { authenticateToken, requirePermission, type AuthenticatedRequest } from "../middleware/auth.middleware"; // ===================== // Аудит целостности финансовых данных (перенесено из Data-Insight2 audit-agent.ts) // ===================== interface CategoryRule { column: string; operator: string; value: string; } interface Category { id: string; name: string; rules: CategoryRule[]; ruleLogic?: "and" | "or"; children: Category[]; excluded?: boolean; autoDetected?: boolean; treeId?: string; } interface AuditResult { timestamp: string; income: DirectionAudit; expense: DirectionAudit; status: "ok" | "warning" | "error"; summary: string; } interface DirectionAudit { dbTotal: number; dbCount: number; treeTotal: number; treeCount: number; excludedTotal: number; excludedCount: number; uncategorizedTotal: number; uncategorizedCount: number; gapTotal: number; gapCount: number; gapRows: { columns: Record; sum: number; count: number }[]; treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[]; } const HISTORY_PATH = resolve(process.cwd(), "data", "finance-audit-history.json"); const MAX_HISTORY = 50; let auditHistory: AuditResult[] = []; function loadHistory(): void { if (!existsSync(HISTORY_PATH)) return; try { const parsed = JSON.parse(readFileSync(HISTORY_PATH, "utf-8")); if (Array.isArray(parsed)) auditHistory = parsed; } catch { // повреждённый файл — начинаем с чистой истории } } function saveHistory(): void { try { writeFileSync(HISTORY_PATH, JSON.stringify(auditHistory, null, 2), "utf-8"); } catch (err) { console.error("[AUDIT-AGENT] Ошибка сохранения истории:", err); } } loadHistory(); function flattenCategories(cats: Category[]): Category[] { const result: Category[] = []; for (const cat of cats) { result.push(cat); if (cat.children?.length) result.push(...flattenCategories(cat.children)); } return result; } function matchTokens(haystack: string, needle: string): boolean { const tokens = needle.split(/\s+/).filter(Boolean); const lower = haystack.toLowerCase(); return tokens.every(t => lower.includes(t.toLowerCase())); } function rowMatchesRule(row: Record, rule: CategoryRule): boolean { const val = row[rule.column]; if (val === undefined) return false; switch (rule.operator) { case "equals": return val === rule.value; case "contains": return typeof val === "string" && matchTokens(val, rule.value); case "not_equals": return val !== rule.value; case "greater_than": return Number(val) > Number(rule.value); case "less_than": return Number(val) < Number(rule.value); case "equal_number": return Number(val) === Number(rule.value); default: return val === rule.value; } } function rowMatchesRules(row: Record, rules: CategoryRule[], logic: "and" | "or" = "and"): boolean { if (!rules || rules.length === 0) return false; if (logic === "or") return rules.some(r => rowMatchesRule(row, r)); return rules.every(r => rowMatchesRule(row, r)); } function collectRuleColumns(cats: Category[]): Set { const cols = new Set(); for (const cat of cats) { for (const rule of (cat.rules || [])) cols.add(rule.column); if (cat.children?.length) { for (const col of collectRuleColumns(cat.children)) cols.add(col); } } return cols; } function rowMatchesAutoPath(row: Record, autoPath: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }): boolean { for (let i = 0; i < autoPath.ruleSets.length - 1; i++) { if (!rowMatchesRules(row, autoPath.ruleSets[i])) return false; } return rowMatchesRules(row, autoPath.ruleSets[autoPath.ruleSets.length - 1], autoPath.logic); } function computeNodeFromAgg( cat: Category, siblings: Category[], ancestorRuleSets: CategoryRule[][], ancestorIds: Set, allFlat: Category[], autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[], aggRows: Record[], exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [] ): { total: number; count: number; excluded: boolean; children: any[] } { const catLogic = cat.ruleLogic || "and"; const idx = siblings.indexOf(cat); const laterSibs = siblings.slice(idx + 1); const allRuleSets = [...ancestorRuleSets, cat.rules]; let total = 0; let count = 0; for (const row of aggRows) { const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue; if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue; if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) continue; if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue; total += row.total; count += row.cnt; } const children: any[] = []; if (cat.children?.length) { const childAncestorIds = new Set(ancestorIds); childAncestorIds.add(cat.id); for (const child of cat.children) { children.push(computeNodeFromAgg(child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, aggRows, exclPaths)); } } return { total, count, excluded: !!cat.excluded, children }; } function nodeSubtreeTotal(n: { total: number; excluded: boolean; children: any[] }): number { if (n.excluded) { let t = 0; for (const ch of n.children) t += nodeSubtreeTotal(ch); return t; } let t = n.total; for (const ch of n.children) t += nodeSubtreeTotal(ch); return t; } function nodeSubtreeCount(n: { count: number; excluded: boolean; children: any[] }): number { if (n.excluded) { let c = 0; for (const ch of n.children) c += nodeSubtreeCount(ch); return c; } let c = n.count; for (const ch of n.children) c += nodeSubtreeCount(ch); return c; } function nodeExcludedTotal(n: { total: number; excluded: boolean; children: any[] }): number { if (n.excluded) { let t = n.total; for (const ch of n.children) t += nodeExcludedTotal(ch); return t; } let t = 0; for (const ch of n.children) t += nodeExcludedTotal(ch); return t; } function nodeExcludedCount(n: { count: number; excluded: boolean; children: any[] }): number { if (n.excluded) { let c = n.count; for (const ch of n.children) c += nodeExcludedCount(ch); return c; } let c = 0; for (const ch of n.children) c += nodeExcludedCount(ch); return c; } async function auditDirection( table: string, sumColumn: string, cats: Category[], autoConfig: AutoDetectConfig, allowedColumns: string[] ): Promise { const allFlat = flattenCategories(cats); const ruleColumns = collectRuleColumns(cats); for (const tree of autoConfig.trees) { for (const col of tree.columns) ruleColumns.add(col); } const groupCols = ruleColumns.size > 0 ? [...ruleColumns].filter(c => allowedColumns.includes(c)) : allowedColumns.slice(0, 4); const selectCols = groupCols.map(c => `"${c}"`).join(", "); const aggResult = await query( `SELECT ${selectCols}, SUM("${sumColumn}"::numeric) as total, COUNT(*) as cnt FROM "${table}" GROUP BY ${selectCols}`, [] ); const aggRows = aggResult.rows.map((r: any) => { const row: Record = { total: Number(r.total), cnt: Number(r.cnt) }; for (const col of groupCols) row[col] = r[col] ?? ""; return row; }); let dbTotal = 0, dbCount = 0; for (const row of aggRows) { dbTotal += row.total; dbCount += row.cnt; } const primaryTreeIds = new Set(autoConfig.trees.filter(t => t.isPrimary).map(t => t.id)); const groupedByTree = new Map(); for (const cat of cats) { const tid = cat.treeId || "__manual__"; if (!groupedByTree.has(tid)) groupedByTree.set(tid, []); groupedByTree.get(tid)!.push(cat); } const autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []; const excludedPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []; function collectAllPaths(categories: Category[], ancestors: CategoryRule[][]) { for (const cat of categories) { const path = [...ancestors, cat.rules || []]; if (cat.excluded) { excludedPaths.push({ id: cat.id, ruleSets: path, logic: (cat.ruleLogic || "and") }); } else if (cat.autoDetected && cat.rules?.length) { autoPaths.push({ ruleSets: path, logic: (cat.ruleLogic || "and") }); } if (cat.children?.length) { collectAllPaths(cat.children, path); } } } collectAllPaths(cats, []); const treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[] = []; let allTreeTotal = 0, allTreeCount = 0; let allExclTotal = 0, allExclCount = 0; for (const [tid, treeCats] of groupedByTree) { const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); const treeName = tid === "__manual__" ? "Ручные категории" : autoConfig.trees.find(t => t.id === tid)?.name || tid; const nodes: any[] = []; for (const cat of treeCats) { nodes.push(computeNodeFromAgg(cat, treeCats, [], new Set(), allFlat, autoPaths, aggRows, excludedPaths)); } const tTotal = nodes.reduce((s, n) => s + nodeSubtreeTotal(n), 0); const tCount = nodes.reduce((s, n) => s + nodeSubtreeCount(n), 0); const eTotal = nodes.reduce((s, n) => s + nodeExcludedTotal(n), 0); const eCount = nodes.reduce((s, n) => s + nodeExcludedCount(n), 0); treeBreakdown.push({ name: treeName, total: tTotal, count: tCount, isPrimary }); if (isPrimary) { allTreeTotal += tTotal; allTreeCount += tCount; allExclTotal += eTotal; allExclCount += eCount; } } function isRowMatchedByTreeAudit(row: Record, treeCats: Category[]): boolean { for (const cat of treeCats) { const catLogic = cat.ruleLogic || "and"; if (!cat.rules?.length || !rowMatchesRules(row, cat.rules, catLogic)) continue; return true; } return false; } let uncatTotal = 0, uncatCount = 0; const gapRows: { columns: Record; sum: number; count: number }[] = []; for (const row of aggRows) { let matched = false; for (const [tid, treeCats] of groupedByTree) { const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); if (!isPrimary) continue; if (isRowMatchedByTreeAudit(row, treeCats)) { matched = true; break; } } if (!matched) { uncatTotal += row.total; uncatCount += row.cnt; } } const accountedTotal = allTreeTotal + allExclTotal + uncatTotal; const accountedCount = allTreeCount + allExclCount + uncatCount; const gapTotal = dbTotal - accountedTotal; const gapCount = dbCount - accountedCount; if (Math.abs(gapCount) > 0) { for (const row of aggRows) { let inTree = false; let inUncat = true; for (const [tid, treeCats] of groupedByTree) { const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); if (!isPrimary) continue; if (isRowMatchedByTreeAudit(row, treeCats)) { inUncat = false; break; } } if (!inUncat) { for (const [tid, treeCats] of groupedByTree) { const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); if (!isPrimary) continue; for (const cat of treeCats) { if (isRowCapturedByNode(row, cat, treeCats, [], new Set(), allFlat, autoPaths, excludedPaths)) { inTree = true; break; } } if (inTree) break; } } if (!inUncat && !inTree) { const cols: Record = {}; for (const col of groupCols) cols[col] = String(row[col] || ""); gapRows.push({ columns: cols, sum: row.total, count: row.cnt }); } } } return { dbTotal, dbCount, treeTotal: allTreeTotal, treeCount: allTreeCount, excludedTotal: allExclTotal, excludedCount: allExclCount, uncategorizedTotal: uncatTotal, uncategorizedCount: uncatCount, gapTotal: Math.round(gapTotal * 100) / 100, gapCount, gapRows: gapRows.slice(0, 50), treeBreakdown, }; } function isRowCapturedByNode( row: Record, cat: Category, siblings: Category[], ancestorRuleSets: CategoryRule[][], ancestorIds: Set, allFlat: Category[], autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[], exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [] ): boolean { const catLogic = cat.ruleLogic || "and"; const idx = siblings.indexOf(cat); const laterSibs = siblings.slice(idx + 1); const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) return false; if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) return false; if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) return false; if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) return false; if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) { const allRuleSets = [...ancestorRuleSets, cat.rules]; const childAncestorIds = new Set(ancestorIds); childAncestorIds.add(cat.id); for (const child of cat.children) { if (isRowCapturedByNode(row, child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, exclPaths)) return true; } return false; } return true; } const INCOME_COLUMNS = [ "statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands", "naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta", "dogovorzakazchika", "statyadop", "schetnaoplatu", "nomervhodyaschegodokumenta", ]; const EXPENSE_COLUMNS = [ "statya", "dochka", "vnuchka", "zakazchik", "naznachenieplatezha", "statyadop", "kommentarii", "otvetstvenniipokontraktu", "dogovorzakazchika", "dogovorkontragenta", ]; let auditRunning = false; export async function runIntegrityAudit(): Promise { if (auditRunning) throw new Error("Аудит уже выполняется"); auditRunning = true; try { console.log("[AUDIT-AGENT] Запуск аудита целостности данных..."); const config = getTableConfig(); const [income, expense] = await Promise.all([ auditDirection( config.incomeTable, config.incomeSumColumn, getIncomeCategories() as Category[], getIncomeAutoDetectConfig(), INCOME_COLUMNS ), auditDirection( config.expenseTable, config.expenseSumColumn, getExpenseCategories() as Category[], getExpenseAutoDetectConfig(), EXPENSE_COLUMNS ), ]); const hasGap = income.gapCount !== 0 || expense.gapCount !== 0; const status = hasGap ? "warning" : "ok"; const summaryParts: string[] = []; if (income.gapCount === 0 && expense.gapCount === 0) { summaryParts.push("Все строки распределены корректно"); } else { if (income.gapCount !== 0) summaryParts.push(`Доходы: ${income.gapCount} строк (${income.gapTotal.toLocaleString("ru")}₽) не распределены`); if (expense.gapCount !== 0) summaryParts.push(`Расходы: ${expense.gapCount} строк (${expense.gapTotal.toLocaleString("ru")}₽) не распределены`); } const result: AuditResult = { timestamp: new Date().toISOString(), income, expense, status, summary: summaryParts.join(". "), }; auditHistory.unshift(result); if (auditHistory.length > MAX_HISTORY) auditHistory.pop(); saveHistory(); console.log(`[AUDIT-AGENT] Аудит завершён: ${result.summary}`); console.log(`[AUDIT-AGENT] Доходы: БД=${income.dbCount} строк, деревья=${income.treeCount}, excluded=${income.excludedCount}, нераспр=${income.uncategorizedCount}, gap=${income.gapCount}`); console.log(`[AUDIT-AGENT] Расходы: БД=${expense.dbCount} строк, деревья=${expense.treeCount}, excluded=${expense.excludedCount}, нераспр=${expense.uncategorizedCount}, gap=${expense.gapCount}`); return result; } finally { auditRunning = false; } } export function getAuditHistory(): AuditResult[] { return auditHistory; } // ===================== // Планировщик: ежедневно в 03:00 (МСК), без node-cron // ===================== const TICK_MS = 60_000; const MSK_OFFSET_MS = 3 * 60 * 60 * 1000; // МСК = UTC+3 const AUDIT_HOUR_MSK = 3; let lastRunDay: string | null = auditHistory[0]?.timestamp ? new Date(new Date(auditHistory[0].timestamp).getTime() + MSK_OFFSET_MS).toISOString().slice(0, 10) : null; let schedulerStarted = false; function mskToday(): string { return new Date(Date.now() + MSK_OFFSET_MS).toISOString().slice(0, 10); } function mskHourNow(): number { return new Date(Date.now() + MSK_OFFSET_MS).getUTCHours(); } async function tick(): Promise { const today = mskToday(); if (lastRunDay === today) return; // сегодня уже запускался if (mskHourNow() < AUDIT_HOUR_MSK) return; // ещё не наступило 03:00 по МСК lastRunDay = today; try { await runIntegrityAudit(); } catch (err) { console.error("[AUDIT-AGENT] Ошибка при выполнении аудита:", err); } } export function startAuditScheduler(): void { if (schedulerStarted) return; schedulerStarted = true; setInterval(() => { tick().catch((err) => console.error("[AUDIT-AGENT] Tick error:", err)); }, TICK_MS); console.log("[AUDIT-AGENT] Планировщик запущен (ежедневно в 03:00 по МСК)"); // Первый запуск через 10 секунд после старта сервера setTimeout(async () => { try { const result = await runIntegrityAudit(); lastRunDay = new Date(new Date(result.timestamp).getTime() + MSK_OFFSET_MS).toISOString().slice(0, 10); } catch (err) { console.error("[AUDIT-AGENT] Ошибка при начальном аудите:", err); } }, 10000); } // ===================== // HTTP endpoints // ===================== export function registerAuditRoutes(app: Express) { app.get("/api/audit/history", authenticateToken, requirePermission('finance.manage'), (_req: AuthenticatedRequest, res) => { res.json({ success: true, history: getAuditHistory() }); }); app.post("/api/audit/run", authenticateToken, requirePermission('finance.manage'), async (_req: AuthenticatedRequest, res) => { try { const result = await runIntegrityAudit(); res.json({ success: true, result }); } catch (err: any) { console.error("[AUDIT-AGENT] Ошибка ручного запуска:", err); res.status(500).json({ success: false, error: err.message }); } }); app.get("/api/audit/status", authenticateToken, requirePermission('finance.manage'), (_req: AuthenticatedRequest, res) => { res.json({ success: true, last: auditHistory[0] || null, running: auditRunning }); }); }