DI2 внедрение, таск 1: серверная интеграция Data-Insight2 (server/finance-di2)
- Копия DI2-сервера в server/finance-di2/ с правками: schema/db-client/cache/table-config/override-tables/google-auth/audit-agent/routes - db-client — обёртка над существующим пулом server/finance/db-client (второй пул не создаётся, добавлен экспорт isConnectionError) - ai-config.ts — чистое IO конфигов из ai-agent.ts без Telegram-поллинга; роуты /api/ai/toggle и /api/ai/status удалены, без compression/startAuditScheduler/autoStartIfEnabled - Обёртка registerDi2Routes с auth-gate (authenticateToken + finance.manage), регистрация строго перед registerFinanceRoutes - Статика /di2 из dist/public-di2 перед веткой vite/static (dev и prod) - Фикс предсуществующего бага DI2: buildGlobalExclusionConditions без cats в /api/profitability
This commit is contained in:
450
server/finance-di2/audit-agent.ts
Normal file
450
server/finance-di2/audit-agent.ts
Normal file
@@ -0,0 +1,450 @@
|
||||
import { query } from "./db-client";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
const CATEGORIES_FILE = path.join(process.cwd(), "data", "income-categories.json");
|
||||
const EXPENSE_CATEGORIES_FILE = path.join(process.cwd(), "data", "expense-categories.json");
|
||||
const INCOME_AUTO_DETECT_CONFIG_FILE = path.join(process.cwd(), "data", "income-auto-detect-config.json");
|
||||
const EXPENSE_AUTO_DETECT_CONFIG_FILE = path.join(process.cwd(), "data", "auto-detect-config.json");
|
||||
|
||||
interface CategoryRule {
|
||||
column: string;
|
||||
operator: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface Category {
|
||||
id: string;
|
||||
name: string;
|
||||
rules: CategoryRule[];
|
||||
ruleLogic?: "and" | "or";
|
||||
children: Category[];
|
||||
excluded?: boolean;
|
||||
autoDetected?: boolean;
|
||||
treeId?: string;
|
||||
}
|
||||
|
||||
interface AuditResult {
|
||||
timestamp: string;
|
||||
income: DirectionAudit;
|
||||
expense: DirectionAudit;
|
||||
status: "ok" | "warning" | "error";
|
||||
summary: string;
|
||||
}
|
||||
|
||||
interface DirectionAudit {
|
||||
dbTotal: number;
|
||||
dbCount: number;
|
||||
treeTotal: number;
|
||||
treeCount: number;
|
||||
excludedTotal: number;
|
||||
excludedCount: number;
|
||||
uncategorizedTotal: number;
|
||||
uncategorizedCount: number;
|
||||
gapTotal: number;
|
||||
gapCount: number;
|
||||
gapRows: { columns: Record<string, string>; sum: number; count: number }[];
|
||||
treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[];
|
||||
}
|
||||
|
||||
const auditHistory: AuditResult[] = [];
|
||||
const MAX_HISTORY = 50;
|
||||
|
||||
function loadJSON(filePath: string): any {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf-8"));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function flattenCategories(cats: Category[]): Category[] {
|
||||
const result: Category[] = [];
|
||||
for (const cat of cats) {
|
||||
result.push(cat);
|
||||
if (cat.children?.length) result.push(...flattenCategories(cat.children));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function matchTokens(haystack: string, needle: string): boolean {
|
||||
const tokens = needle.split(/\s+/).filter(Boolean);
|
||||
const lower = haystack.toLowerCase();
|
||||
return tokens.every(t => lower.includes(t.toLowerCase()));
|
||||
}
|
||||
|
||||
function rowMatchesRule(row: Record<string, any>, rule: CategoryRule): boolean {
|
||||
const val = row[rule.column];
|
||||
if (val === undefined) return false;
|
||||
switch (rule.operator) {
|
||||
case "equals": return val === rule.value;
|
||||
case "contains": return typeof val === "string" && matchTokens(val, rule.value);
|
||||
case "not_equals": return val !== rule.value;
|
||||
case "greater_than": return Number(val) > Number(rule.value);
|
||||
case "less_than": return Number(val) < Number(rule.value);
|
||||
case "equal_number": return Number(val) === Number(rule.value);
|
||||
default: return val === rule.value;
|
||||
}
|
||||
}
|
||||
|
||||
function rowMatchesRules(row: Record<string, any>, rules: CategoryRule[], logic: "and" | "or" = "and"): boolean {
|
||||
if (!rules || rules.length === 0) return false;
|
||||
if (logic === "or") return rules.some(r => rowMatchesRule(row, r));
|
||||
return rules.every(r => rowMatchesRule(row, r));
|
||||
}
|
||||
|
||||
function collectRuleColumns(cats: Category[]): Set<string> {
|
||||
const cols = new Set<string>();
|
||||
for (const cat of cats) {
|
||||
for (const rule of (cat.rules || [])) cols.add(rule.column);
|
||||
if (cat.children?.length) {
|
||||
for (const col of collectRuleColumns(cat.children)) cols.add(col);
|
||||
}
|
||||
}
|
||||
return cols;
|
||||
}
|
||||
|
||||
interface AutoDetectConfig {
|
||||
trees: { id: string; name: string; columns: string[]; isPrimary?: boolean; filters?: any[]; filterLogic?: string }[];
|
||||
}
|
||||
|
||||
function rowMatchesAutoPath(row: Record<string, any>, autoPath: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }): boolean {
|
||||
for (let i = 0; i < autoPath.ruleSets.length - 1; i++) {
|
||||
if (!rowMatchesRules(row, autoPath.ruleSets[i])) return false;
|
||||
}
|
||||
return rowMatchesRules(row, autoPath.ruleSets[autoPath.ruleSets.length - 1], autoPath.logic);
|
||||
}
|
||||
|
||||
function computeNodeFromAgg(
|
||||
cat: Category,
|
||||
siblings: Category[],
|
||||
ancestorRuleSets: CategoryRule[][],
|
||||
ancestorIds: Set<string>,
|
||||
allFlat: Category[],
|
||||
autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[],
|
||||
aggRows: Record<string, any>[],
|
||||
exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []
|
||||
): { total: number; count: number; excluded: boolean; children: any[] } {
|
||||
const catLogic = cat.ruleLogic || "and";
|
||||
const idx = siblings.indexOf(cat);
|
||||
const laterSibs = siblings.slice(idx + 1);
|
||||
|
||||
const allRuleSets = [...ancestorRuleSets, cat.rules];
|
||||
|
||||
let total = 0;
|
||||
let count = 0;
|
||||
for (const row of aggRows) {
|
||||
const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs));
|
||||
if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue;
|
||||
if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue;
|
||||
if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue;
|
||||
if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) continue;
|
||||
if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue;
|
||||
total += row.total;
|
||||
count += row.cnt;
|
||||
}
|
||||
|
||||
const children: any[] = [];
|
||||
if (cat.children?.length) {
|
||||
const childAncestorIds = new Set(ancestorIds);
|
||||
childAncestorIds.add(cat.id);
|
||||
for (const child of cat.children) {
|
||||
children.push(computeNodeFromAgg(child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, aggRows, exclPaths));
|
||||
}
|
||||
}
|
||||
|
||||
return { total, count, excluded: !!cat.excluded, children };
|
||||
}
|
||||
|
||||
function nodeSubtreeTotal(n: { total: number; excluded: boolean; children: any[] }): number {
|
||||
if (n.excluded) {
|
||||
let t = 0;
|
||||
for (const ch of n.children) t += nodeSubtreeTotal(ch);
|
||||
return t;
|
||||
}
|
||||
let t = n.total;
|
||||
for (const ch of n.children) t += nodeSubtreeTotal(ch);
|
||||
return t;
|
||||
}
|
||||
|
||||
function nodeSubtreeCount(n: { count: number; excluded: boolean; children: any[] }): number {
|
||||
if (n.excluded) {
|
||||
let c = 0;
|
||||
for (const ch of n.children) c += nodeSubtreeCount(ch);
|
||||
return c;
|
||||
}
|
||||
let c = n.count;
|
||||
for (const ch of n.children) c += nodeSubtreeCount(ch);
|
||||
return c;
|
||||
}
|
||||
|
||||
function nodeExcludedTotal(n: { total: number; excluded: boolean; children: any[] }): number {
|
||||
if (n.excluded) {
|
||||
let t = n.total;
|
||||
for (const ch of n.children) t += nodeExcludedTotal(ch);
|
||||
return t;
|
||||
}
|
||||
let t = 0;
|
||||
for (const ch of n.children) t += nodeExcludedTotal(ch);
|
||||
return t;
|
||||
}
|
||||
|
||||
function nodeExcludedCount(n: { count: number; excluded: boolean; children: any[] }): number {
|
||||
if (n.excluded) {
|
||||
let c = n.count;
|
||||
for (const ch of n.children) c += nodeExcludedCount(ch);
|
||||
return c;
|
||||
}
|
||||
let c = 0;
|
||||
for (const ch of n.children) c += nodeExcludedCount(ch);
|
||||
return c;
|
||||
}
|
||||
|
||||
async function auditDirection(
|
||||
table: string,
|
||||
sumColumn: string,
|
||||
categoriesFile: string,
|
||||
autoDetectConfigFile: string,
|
||||
allowedColumns: string[]
|
||||
): Promise<DirectionAudit> {
|
||||
const cats: Category[] = loadJSON(categoriesFile);
|
||||
const autoConfigRaw = loadJSON(autoDetectConfigFile);
|
||||
const autoConfig: AutoDetectConfig = { trees: autoConfigRaw?.trees || [] };
|
||||
const allFlat = flattenCategories(cats);
|
||||
|
||||
const ruleColumns = collectRuleColumns(cats);
|
||||
for (const tree of autoConfig.trees) {
|
||||
for (const col of tree.columns) ruleColumns.add(col);
|
||||
}
|
||||
const groupCols = ruleColumns.size > 0
|
||||
? [...ruleColumns].filter(c => allowedColumns.includes(c))
|
||||
: allowedColumns.slice(0, 4);
|
||||
|
||||
const selectCols = groupCols.join(", ");
|
||||
const aggResult = await query(
|
||||
`SELECT ${selectCols}, SUM(${sumColumn}::numeric) as total, COUNT(*) as cnt FROM ${table} GROUP BY ${selectCols}`,
|
||||
[]
|
||||
);
|
||||
|
||||
const aggRows = aggResult.rows.map((r: any) => {
|
||||
const row: Record<string, any> = { total: Number(r.total), cnt: Number(r.cnt) };
|
||||
for (const col of groupCols) row[col] = r[col] ?? "";
|
||||
return row;
|
||||
});
|
||||
|
||||
let dbTotal = 0, dbCount = 0;
|
||||
for (const row of aggRows) { dbTotal += row.total; dbCount += row.cnt; }
|
||||
|
||||
const primaryTreeIds = new Set(autoConfig.trees.filter(t => t.isPrimary).map(t => t.id));
|
||||
const configTreeIds = new Set(autoConfig.trees.map(t => t.id));
|
||||
|
||||
const groupedByTree = new Map<string, Category[]>();
|
||||
for (const cat of cats) {
|
||||
const tid = cat.treeId || "__manual__";
|
||||
if (!groupedByTree.has(tid)) groupedByTree.set(tid, []);
|
||||
groupedByTree.get(tid)!.push(cat);
|
||||
}
|
||||
|
||||
const autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [];
|
||||
const excludedPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [];
|
||||
function collectAllPaths(categories: Category[], ancestors: CategoryRule[][]) {
|
||||
for (const cat of categories) {
|
||||
const path = [...ancestors, cat.rules || []];
|
||||
if (cat.excluded) {
|
||||
excludedPaths.push({ id: cat.id, ruleSets: path, logic: (cat.ruleLogic || "and") });
|
||||
} else if (cat.autoDetected && cat.rules?.length) {
|
||||
autoPaths.push({ ruleSets: path, logic: (cat.ruleLogic || "and") });
|
||||
}
|
||||
if (cat.children?.length) {
|
||||
collectAllPaths(cat.children, path);
|
||||
}
|
||||
}
|
||||
}
|
||||
collectAllPaths(cats, []);
|
||||
|
||||
const treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[] = [];
|
||||
let allTreeTotal = 0, allTreeCount = 0;
|
||||
let allExclTotal = 0, allExclCount = 0;
|
||||
|
||||
for (const [tid, treeCats] of groupedByTree) {
|
||||
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
|
||||
const treeName = tid === "__manual__"
|
||||
? "Ручные категории"
|
||||
: autoConfig.trees.find(t => t.id === tid)?.name || tid;
|
||||
|
||||
const nodes: any[] = [];
|
||||
for (const cat of treeCats) {
|
||||
nodes.push(computeNodeFromAgg(cat, treeCats, [], new Set(), allFlat, autoPaths, aggRows, excludedPaths));
|
||||
}
|
||||
|
||||
const tTotal = nodes.reduce((s, n) => s + nodeSubtreeTotal(n), 0);
|
||||
const tCount = nodes.reduce((s, n) => s + nodeSubtreeCount(n), 0);
|
||||
const eTotal = nodes.reduce((s, n) => s + nodeExcludedTotal(n), 0);
|
||||
const eCount = nodes.reduce((s, n) => s + nodeExcludedCount(n), 0);
|
||||
|
||||
treeBreakdown.push({ name: treeName, total: tTotal, count: tCount, isPrimary });
|
||||
|
||||
if (isPrimary) {
|
||||
allTreeTotal += tTotal;
|
||||
allTreeCount += tCount;
|
||||
allExclTotal += eTotal;
|
||||
allExclCount += eCount;
|
||||
}
|
||||
}
|
||||
|
||||
function isRowMatchedByTreeAudit(row: Record<string, any>, treeCats: Category[]): boolean {
|
||||
for (const cat of treeCats) {
|
||||
const catLogic = cat.ruleLogic || "and";
|
||||
if (!cat.rules?.length || !rowMatchesRules(row, cat.rules, catLogic)) continue;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
let uncatTotal = 0, uncatCount = 0;
|
||||
const gapRows: { columns: Record<string, string>; sum: number; count: number }[] = [];
|
||||
|
||||
for (const row of aggRows) {
|
||||
let matched = false;
|
||||
for (const [tid, treeCats] of groupedByTree) {
|
||||
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
|
||||
if (!isPrimary) continue;
|
||||
if (isRowMatchedByTreeAudit(row, treeCats)) { matched = true; break; }
|
||||
}
|
||||
if (!matched) {
|
||||
uncatTotal += row.total;
|
||||
uncatCount += row.cnt;
|
||||
}
|
||||
}
|
||||
|
||||
const accountedTotal = allTreeTotal + allExclTotal + uncatTotal;
|
||||
const accountedCount = allTreeCount + allExclCount + uncatCount;
|
||||
const gapTotal = dbTotal - accountedTotal;
|
||||
const gapCount = dbCount - accountedCount;
|
||||
|
||||
if (Math.abs(gapCount) > 0) {
|
||||
for (const row of aggRows) {
|
||||
let inTree = false;
|
||||
let inUncat = true;
|
||||
for (const [tid, treeCats] of groupedByTree) {
|
||||
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
|
||||
if (!isPrimary) continue;
|
||||
if (isRowMatchedByTreeAudit(row, treeCats)) { inUncat = false; break; }
|
||||
}
|
||||
if (!inUncat) {
|
||||
for (const [tid, treeCats] of groupedByTree) {
|
||||
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
|
||||
if (!isPrimary) continue;
|
||||
for (const cat of treeCats) {
|
||||
if (isRowCapturedByNode(row, cat, treeCats, [], new Set(), allFlat, autoPaths, excludedPaths)) {
|
||||
inTree = true; break;
|
||||
}
|
||||
}
|
||||
if (inTree) break;
|
||||
}
|
||||
}
|
||||
if (!inUncat && !inTree) {
|
||||
const cols: Record<string, string> = {};
|
||||
for (const col of groupCols) cols[col] = String(row[col] || "");
|
||||
gapRows.push({ columns: cols, sum: row.total, count: row.cnt });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dbTotal, dbCount,
|
||||
treeTotal: allTreeTotal, treeCount: allTreeCount,
|
||||
excludedTotal: allExclTotal, excludedCount: allExclCount,
|
||||
uncategorizedTotal: uncatTotal, uncategorizedCount: uncatCount,
|
||||
gapTotal: Math.round(gapTotal * 100) / 100,
|
||||
gapCount,
|
||||
gapRows: gapRows.slice(0, 50),
|
||||
treeBreakdown,
|
||||
};
|
||||
}
|
||||
|
||||
function isRowCapturedByNode(
|
||||
row: Record<string, any>,
|
||||
cat: Category,
|
||||
siblings: Category[],
|
||||
ancestorRuleSets: CategoryRule[][],
|
||||
ancestorIds: Set<string>,
|
||||
allFlat: Category[],
|
||||
autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[],
|
||||
exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []
|
||||
): boolean {
|
||||
const catLogic = cat.ruleLogic || "and";
|
||||
const idx = siblings.indexOf(cat);
|
||||
const laterSibs = siblings.slice(idx + 1);
|
||||
|
||||
const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs));
|
||||
if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) return false;
|
||||
if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) return false;
|
||||
if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) return false;
|
||||
if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) return false;
|
||||
|
||||
if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) {
|
||||
const allRuleSets = [...ancestorRuleSets, cat.rules];
|
||||
const childAncestorIds = new Set(ancestorIds);
|
||||
childAncestorIds.add(cat.id);
|
||||
for (const child of cat.children) {
|
||||
if (isRowCapturedByNode(row, child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, exclPaths)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
const INCOME_COLUMNS = [
|
||||
"statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands",
|
||||
"naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta",
|
||||
"dogovorzakazchika", "statyadop", "schetnaoplatu", "nomervhodyaschegodokumenta",
|
||||
];
|
||||
|
||||
const EXPENSE_COLUMNS = [
|
||||
"statya", "dochka", "vnuchka", "zakazchik", "naznachenieplatezha",
|
||||
"statyadop", "kommentarii", "otvetstvenniipokontraktu", "dogovorzakazchika",
|
||||
"dogovorkontragenta",
|
||||
];
|
||||
|
||||
export async function runIntegrityAudit(): Promise<AuditResult> {
|
||||
console.log("[AUDIT-AGENT] Запуск аудита целостности данных...");
|
||||
|
||||
const [income, expense] = await Promise.all([
|
||||
auditDirection("opdohodi", "summa", CATEGORIES_FILE, INCOME_AUTO_DETECT_CONFIG_FILE, INCOME_COLUMNS),
|
||||
auditDirection("oprashodi", "summadokumenta", EXPENSE_CATEGORIES_FILE, EXPENSE_AUTO_DETECT_CONFIG_FILE, EXPENSE_COLUMNS),
|
||||
]);
|
||||
|
||||
const hasGap = income.gapCount !== 0 || expense.gapCount !== 0;
|
||||
const status = hasGap ? "warning" : "ok";
|
||||
|
||||
const summaryParts: string[] = [];
|
||||
if (income.gapCount === 0 && expense.gapCount === 0) {
|
||||
summaryParts.push("Все строки распределены корректно");
|
||||
} else {
|
||||
if (income.gapCount !== 0) summaryParts.push(`Доходы: ${income.gapCount} строк (${income.gapTotal.toLocaleString("ru")}₽) не распределены`);
|
||||
if (expense.gapCount !== 0) summaryParts.push(`Расходы: ${expense.gapCount} строк (${expense.gapTotal.toLocaleString("ru")}₽) не распределены`);
|
||||
}
|
||||
|
||||
const result: AuditResult = {
|
||||
timestamp: new Date().toISOString(),
|
||||
income,
|
||||
expense,
|
||||
status,
|
||||
summary: summaryParts.join(". "),
|
||||
};
|
||||
|
||||
auditHistory.unshift(result);
|
||||
if (auditHistory.length > MAX_HISTORY) auditHistory.pop();
|
||||
|
||||
console.log(`[AUDIT-AGENT] Аудит завершён: ${result.summary}`);
|
||||
console.log(`[AUDIT-AGENT] Доходы: БД=${income.dbCount} строк, деревья=${income.treeCount}, excluded=${income.excludedCount}, нераспр=${income.uncategorizedCount}, gap=${income.gapCount}`);
|
||||
console.log(`[AUDIT-AGENT] Расходы: БД=${expense.dbCount} строк, деревья=${expense.treeCount}, excluded=${expense.excludedCount}, нераспр=${expense.uncategorizedCount}, gap=${expense.gapCount}`);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export function getAuditHistory(): AuditResult[] {
|
||||
return auditHistory;
|
||||
}
|
||||
Reference in New Issue
Block a user