Files
iistwin/server/finance/audit.ts

567 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { type Express } from "express";
import { readFileSync, writeFileSync, existsSync } from "fs";
import { resolve } from "path";
import { query } from "./db-client";
import {
getTableConfig,
getIncomeCategories,
getExpenseCategories,
getIncomeAutoDetectConfig,
getExpenseAutoDetectConfig,
type IncomeCategoryFile,
type AutoDetectConfig,
} from "./table-config";
import { authenticateToken, requirePermission, type AuthenticatedRequest } from "../middleware/auth.middleware";
// =====================
// Аудит целостности финансовых данных (перенесено из Data-Insight2 audit-agent.ts)
// =====================
interface CategoryRule {
column: string;
operator: string;
value: string;
}
interface Category {
id: string;
name: string;
rules: CategoryRule[];
ruleLogic?: "and" | "or";
children: Category[];
excluded?: boolean;
autoDetected?: boolean;
treeId?: string;
}
interface AuditResult {
timestamp: string;
income: DirectionAudit;
expense: DirectionAudit;
status: "ok" | "warning" | "error";
summary: string;
}
interface DirectionAudit {
dbTotal: number;
dbCount: number;
treeTotal: number;
treeCount: number;
excludedTotal: number;
excludedCount: number;
uncategorizedTotal: number;
uncategorizedCount: number;
gapTotal: number;
gapCount: number;
gapRows: { columns: Record<string, string>; sum: number; count: number }[];
treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[];
}
const HISTORY_PATH = resolve(process.cwd(), "data", "finance-audit-history.json");
const MAX_HISTORY = 50;
let auditHistory: AuditResult[] = [];
function loadHistory(): void {
if (!existsSync(HISTORY_PATH)) return;
try {
const parsed = JSON.parse(readFileSync(HISTORY_PATH, "utf-8"));
if (Array.isArray(parsed)) auditHistory = parsed;
} catch {
// повреждённый файл — начинаем с чистой истории
}
}
function saveHistory(): void {
try {
writeFileSync(HISTORY_PATH, JSON.stringify(auditHistory, null, 2), "utf-8");
} catch (err) {
console.error("[AUDIT-AGENT] Ошибка сохранения истории:", err);
}
}
loadHistory();
function flattenCategories(cats: Category[]): Category[] {
const result: Category[] = [];
for (const cat of cats) {
result.push(cat);
if (cat.children?.length) result.push(...flattenCategories(cat.children));
}
return result;
}
function matchTokens(haystack: string, needle: string): boolean {
const tokens = needle.split(/\s+/).filter(Boolean);
const lower = haystack.toLowerCase();
return tokens.every(t => lower.includes(t.toLowerCase()));
}
function rowMatchesRule(row: Record<string, any>, rule: CategoryRule): boolean {
const val = row[rule.column];
if (val === undefined) return false;
switch (rule.operator) {
case "equals": return val === rule.value;
case "contains": return typeof val === "string" && matchTokens(val, rule.value);
case "not_equals": return val !== rule.value;
case "greater_than": return Number(val) > Number(rule.value);
case "less_than": return Number(val) < Number(rule.value);
case "equal_number": return Number(val) === Number(rule.value);
default: return val === rule.value;
}
}
function rowMatchesRules(row: Record<string, any>, rules: CategoryRule[], logic: "and" | "or" = "and"): boolean {
if (!rules || rules.length === 0) return false;
if (logic === "or") return rules.some(r => rowMatchesRule(row, r));
return rules.every(r => rowMatchesRule(row, r));
}
function collectRuleColumns(cats: Category[]): Set<string> {
const cols = new Set<string>();
for (const cat of cats) {
for (const rule of (cat.rules || [])) cols.add(rule.column);
if (cat.children?.length) {
for (const col of collectRuleColumns(cat.children)) cols.add(col);
}
}
return cols;
}
function rowMatchesAutoPath(row: Record<string, any>, autoPath: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }): boolean {
for (let i = 0; i < autoPath.ruleSets.length - 1; i++) {
if (!rowMatchesRules(row, autoPath.ruleSets[i])) return false;
}
return rowMatchesRules(row, autoPath.ruleSets[autoPath.ruleSets.length - 1], autoPath.logic);
}
function computeNodeFromAgg(
cat: Category,
siblings: Category[],
ancestorRuleSets: CategoryRule[][],
ancestorIds: Set<string>,
allFlat: Category[],
autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[],
aggRows: Record<string, any>[],
exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []
): { total: number; count: number; excluded: boolean; children: any[] } {
const catLogic = cat.ruleLogic || "and";
const idx = siblings.indexOf(cat);
const laterSibs = siblings.slice(idx + 1);
const allRuleSets = [...ancestorRuleSets, cat.rules];
let total = 0;
let count = 0;
for (const row of aggRows) {
const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs));
if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue;
if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue;
if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue;
if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) continue;
if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue;
total += row.total;
count += row.cnt;
}
const children: any[] = [];
if (cat.children?.length) {
const childAncestorIds = new Set(ancestorIds);
childAncestorIds.add(cat.id);
for (const child of cat.children) {
children.push(computeNodeFromAgg(child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, aggRows, exclPaths));
}
}
return { total, count, excluded: !!cat.excluded, children };
}
function nodeSubtreeTotal(n: { total: number; excluded: boolean; children: any[] }): number {
if (n.excluded) {
let t = 0;
for (const ch of n.children) t += nodeSubtreeTotal(ch);
return t;
}
let t = n.total;
for (const ch of n.children) t += nodeSubtreeTotal(ch);
return t;
}
function nodeSubtreeCount(n: { count: number; excluded: boolean; children: any[] }): number {
if (n.excluded) {
let c = 0;
for (const ch of n.children) c += nodeSubtreeCount(ch);
return c;
}
let c = n.count;
for (const ch of n.children) c += nodeSubtreeCount(ch);
return c;
}
function nodeExcludedTotal(n: { total: number; excluded: boolean; children: any[] }): number {
if (n.excluded) {
let t = n.total;
for (const ch of n.children) t += nodeExcludedTotal(ch);
return t;
}
let t = 0;
for (const ch of n.children) t += nodeExcludedTotal(ch);
return t;
}
function nodeExcludedCount(n: { count: number; excluded: boolean; children: any[] }): number {
if (n.excluded) {
let c = n.count;
for (const ch of n.children) c += nodeExcludedCount(ch);
return c;
}
let c = 0;
for (const ch of n.children) c += nodeExcludedCount(ch);
return c;
}
async function auditDirection(
table: string,
sumColumn: string,
cats: Category[],
autoConfig: AutoDetectConfig,
allowedColumns: string[]
): Promise<DirectionAudit> {
const allFlat = flattenCategories(cats);
const ruleColumns = collectRuleColumns(cats);
for (const tree of autoConfig.trees) {
for (const col of tree.columns) ruleColumns.add(col);
}
const groupCols = ruleColumns.size > 0
? [...ruleColumns].filter(c => allowedColumns.includes(c))
: allowedColumns.slice(0, 4);
const selectCols = groupCols.map(c => `"${c}"`).join(", ");
const aggResult = await query(
`SELECT ${selectCols}, SUM("${sumColumn}"::numeric) as total, COUNT(*) as cnt FROM "${table}" GROUP BY ${selectCols}`,
[]
);
const aggRows = aggResult.rows.map((r: any) => {
const row: Record<string, any> = { total: Number(r.total), cnt: Number(r.cnt) };
for (const col of groupCols) row[col] = r[col] ?? "";
return row;
});
let dbTotal = 0, dbCount = 0;
for (const row of aggRows) { dbTotal += row.total; dbCount += row.cnt; }
const primaryTreeIds = new Set(autoConfig.trees.filter(t => t.isPrimary).map(t => t.id));
const groupedByTree = new Map<string, Category[]>();
for (const cat of cats) {
const tid = cat.treeId || "__manual__";
if (!groupedByTree.has(tid)) groupedByTree.set(tid, []);
groupedByTree.get(tid)!.push(cat);
}
const autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [];
const excludedPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [];
function collectAllPaths(categories: Category[], ancestors: CategoryRule[][]) {
for (const cat of categories) {
const path = [...ancestors, cat.rules || []];
if (cat.excluded) {
excludedPaths.push({ id: cat.id, ruleSets: path, logic: (cat.ruleLogic || "and") });
} else if (cat.autoDetected && cat.rules?.length) {
autoPaths.push({ ruleSets: path, logic: (cat.ruleLogic || "and") });
}
if (cat.children?.length) {
collectAllPaths(cat.children, path);
}
}
}
collectAllPaths(cats, []);
const treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[] = [];
let allTreeTotal = 0, allTreeCount = 0;
let allExclTotal = 0, allExclCount = 0;
for (const [tid, treeCats] of groupedByTree) {
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
const treeName = tid === "__manual__"
? "Ручные категории"
: autoConfig.trees.find(t => t.id === tid)?.name || tid;
const nodes: any[] = [];
for (const cat of treeCats) {
nodes.push(computeNodeFromAgg(cat, treeCats, [], new Set(), allFlat, autoPaths, aggRows, excludedPaths));
}
const tTotal = nodes.reduce((s, n) => s + nodeSubtreeTotal(n), 0);
const tCount = nodes.reduce((s, n) => s + nodeSubtreeCount(n), 0);
const eTotal = nodes.reduce((s, n) => s + nodeExcludedTotal(n), 0);
const eCount = nodes.reduce((s, n) => s + nodeExcludedCount(n), 0);
treeBreakdown.push({ name: treeName, total: tTotal, count: tCount, isPrimary });
if (isPrimary) {
allTreeTotal += tTotal;
allTreeCount += tCount;
allExclTotal += eTotal;
allExclCount += eCount;
}
}
function isRowMatchedByTreeAudit(row: Record<string, any>, treeCats: Category[]): boolean {
for (const cat of treeCats) {
const catLogic = cat.ruleLogic || "and";
if (!cat.rules?.length || !rowMatchesRules(row, cat.rules, catLogic)) continue;
return true;
}
return false;
}
let uncatTotal = 0, uncatCount = 0;
const gapRows: { columns: Record<string, string>; sum: number; count: number }[] = [];
for (const row of aggRows) {
let matched = false;
for (const [tid, treeCats] of groupedByTree) {
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
if (!isPrimary) continue;
if (isRowMatchedByTreeAudit(row, treeCats)) { matched = true; break; }
}
if (!matched) {
uncatTotal += row.total;
uncatCount += row.cnt;
}
}
const accountedTotal = allTreeTotal + allExclTotal + uncatTotal;
const accountedCount = allTreeCount + allExclCount + uncatCount;
const gapTotal = dbTotal - accountedTotal;
const gapCount = dbCount - accountedCount;
if (Math.abs(gapCount) > 0) {
for (const row of aggRows) {
let inTree = false;
let inUncat = true;
for (const [tid, treeCats] of groupedByTree) {
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
if (!isPrimary) continue;
if (isRowMatchedByTreeAudit(row, treeCats)) { inUncat = false; break; }
}
if (!inUncat) {
for (const [tid, treeCats] of groupedByTree) {
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
if (!isPrimary) continue;
for (const cat of treeCats) {
if (isRowCapturedByNode(row, cat, treeCats, [], new Set(), allFlat, autoPaths, excludedPaths)) {
inTree = true; break;
}
}
if (inTree) break;
}
}
if (!inUncat && !inTree) {
const cols: Record<string, string> = {};
for (const col of groupCols) cols[col] = String(row[col] || "");
gapRows.push({ columns: cols, sum: row.total, count: row.cnt });
}
}
}
return {
dbTotal, dbCount,
treeTotal: allTreeTotal, treeCount: allTreeCount,
excludedTotal: allExclTotal, excludedCount: allExclCount,
uncategorizedTotal: uncatTotal, uncategorizedCount: uncatCount,
gapTotal: Math.round(gapTotal * 100) / 100,
gapCount,
gapRows: gapRows.slice(0, 50),
treeBreakdown,
};
}
function isRowCapturedByNode(
row: Record<string, any>,
cat: Category,
siblings: Category[],
ancestorRuleSets: CategoryRule[][],
ancestorIds: Set<string>,
allFlat: Category[],
autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[],
exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []
): boolean {
const catLogic = cat.ruleLogic || "and";
const idx = siblings.indexOf(cat);
const laterSibs = siblings.slice(idx + 1);
const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs));
if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) return false;
if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) return false;
if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) return false;
if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) return false;
if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) {
const allRuleSets = [...ancestorRuleSets, cat.rules];
const childAncestorIds = new Set(ancestorIds);
childAncestorIds.add(cat.id);
for (const child of cat.children) {
if (isRowCapturedByNode(row, child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, exclPaths)) return true;
}
return false;
}
return true;
}
const INCOME_COLUMNS = [
"statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands",
"naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta",
"dogovorzakazchika", "statyadop", "schetnaoplatu", "nomervhodyaschegodokumenta",
];
const EXPENSE_COLUMNS = [
"statya", "dochka", "vnuchka", "zakazchik", "naznachenieplatezha",
"statyadop", "kommentarii", "otvetstvenniipokontraktu", "dogovorzakazchika",
"dogovorkontragenta",
];
let auditRunning = false;
export async function runIntegrityAudit(): Promise<AuditResult> {
if (auditRunning) throw new Error("Аудит уже выполняется");
auditRunning = true;
try {
console.log("[AUDIT-AGENT] Запуск аудита целостности данных...");
const config = getTableConfig();
const [income, expense] = await Promise.all([
auditDirection(
config.incomeTable,
config.incomeSumColumn,
getIncomeCategories() as Category[],
getIncomeAutoDetectConfig(),
INCOME_COLUMNS
),
auditDirection(
config.expenseTable,
config.expenseSumColumn,
getExpenseCategories() as Category[],
getExpenseAutoDetectConfig(),
EXPENSE_COLUMNS
),
]);
const hasGap = income.gapCount !== 0 || expense.gapCount !== 0;
const status = hasGap ? "warning" : "ok";
const summaryParts: string[] = [];
if (income.gapCount === 0 && expense.gapCount === 0) {
summaryParts.push("Все строки распределены корректно");
} else {
if (income.gapCount !== 0) summaryParts.push(`Доходы: ${income.gapCount} строк (${income.gapTotal.toLocaleString("ru")}₽) не распределены`);
if (expense.gapCount !== 0) summaryParts.push(`Расходы: ${expense.gapCount} строк (${expense.gapTotal.toLocaleString("ru")}₽) не распределены`);
}
const result: AuditResult = {
timestamp: new Date().toISOString(),
income,
expense,
status,
summary: summaryParts.join(". "),
};
auditHistory.unshift(result);
if (auditHistory.length > MAX_HISTORY) auditHistory.pop();
saveHistory();
console.log(`[AUDIT-AGENT] Аудит завершён: ${result.summary}`);
console.log(`[AUDIT-AGENT] Доходы: БД=${income.dbCount} строк, деревья=${income.treeCount}, excluded=${income.excludedCount}, нераспр=${income.uncategorizedCount}, gap=${income.gapCount}`);
console.log(`[AUDIT-AGENT] Расходы: БД=${expense.dbCount} строк, деревья=${expense.treeCount}, excluded=${expense.excludedCount}, нераспр=${expense.uncategorizedCount}, gap=${expense.gapCount}`);
return result;
} finally {
auditRunning = false;
}
}
export function getAuditHistory(): AuditResult[] {
return auditHistory;
}
// =====================
// Планировщик: ежедневно в 03:00 (МСК), без node-cron
// =====================
const TICK_MS = 60_000;
const MSK_OFFSET_MS = 3 * 60 * 60 * 1000; // МСК = UTC+3
const AUDIT_HOUR_MSK = 3;
let lastRunDay: string | null = auditHistory[0]?.timestamp
? new Date(new Date(auditHistory[0].timestamp).getTime() + MSK_OFFSET_MS).toISOString().slice(0, 10)
: null;
let schedulerStarted = false;
function mskToday(): string {
return new Date(Date.now() + MSK_OFFSET_MS).toISOString().slice(0, 10);
}
function mskHourNow(): number {
return new Date(Date.now() + MSK_OFFSET_MS).getUTCHours();
}
async function tick(): Promise<void> {
const today = mskToday();
if (lastRunDay === today) return; // сегодня уже запускался
if (mskHourNow() < AUDIT_HOUR_MSK) return; // ещё не наступило 03:00 по МСК
lastRunDay = today;
try {
await runIntegrityAudit();
} catch (err) {
console.error("[AUDIT-AGENT] Ошибка при выполнении аудита:", err);
}
}
export function startAuditScheduler(): void {
if (schedulerStarted) return;
schedulerStarted = true;
setInterval(() => {
tick().catch((err) => console.error("[AUDIT-AGENT] Tick error:", err));
}, TICK_MS);
console.log("[AUDIT-AGENT] Планировщик запущен (ежедневно в 03:00 по МСК)");
// Первый запуск через 10 секунд после старта сервера
setTimeout(async () => {
try {
const result = await runIntegrityAudit();
lastRunDay = new Date(new Date(result.timestamp).getTime() + MSK_OFFSET_MS).toISOString().slice(0, 10);
} catch (err) {
console.error("[AUDIT-AGENT] Ошибка при начальном аудите:", err);
}
}, 10000);
}
// =====================
// HTTP endpoints
// =====================
export function registerAuditRoutes(app: Express) {
app.get("/api/audit/history", authenticateToken, requirePermission('finance.manage'), (_req: AuthenticatedRequest, res) => {
res.json({ success: true, history: getAuditHistory() });
});
app.post("/api/audit/run", authenticateToken, requirePermission('finance.manage'), async (_req: AuthenticatedRequest, res) => {
try {
const result = await runIntegrityAudit();
res.json({ success: true, result });
} catch (err: any) {
console.error("[AUDIT-AGENT] Ошибка ручного запуска:", err);
res.status(500).json({ success: false, error: err.message });
}
});
app.get("/api/audit/status", authenticateToken, requirePermission('finance.manage'), (_req: AuthenticatedRequest, res) => {
res.json({ success: true, last: auditHistory[0] || null, running: auditRunning });
});
}