Финансы: дебиторка (vw_debt_active), расчёты НДС/налога на прибыль, аудит целостности с автопланировщиком, синхронизация зарплат из Google Sheets, миграция настроек Data-Insight2

This commit is contained in:
2026-09-05 15:07:36 +03:00
parent d2c993b613
commit 7a8db03163
17 changed files with 4809 additions and 34 deletions

566
server/finance/audit.ts Normal file
View File

@@ -0,0 +1,566 @@
import { type Express } from "express";
import { readFileSync, writeFileSync, existsSync } from "fs";
import { resolve } from "path";
import { query } from "./db-client";
import {
getTableConfig,
getIncomeCategories,
getExpenseCategories,
getIncomeAutoDetectConfig,
getExpenseAutoDetectConfig,
type IncomeCategoryFile,
type AutoDetectConfig,
} from "./table-config";
import { authenticateToken, requirePermission, type AuthenticatedRequest } from "../middleware/auth.middleware";
// =====================
// Аудит целостности финансовых данных (перенесено из Data-Insight2 audit-agent.ts)
// =====================
interface CategoryRule {
column: string;
operator: string;
value: string;
}
interface Category {
id: string;
name: string;
rules: CategoryRule[];
ruleLogic?: "and" | "or";
children: Category[];
excluded?: boolean;
autoDetected?: boolean;
treeId?: string;
}
interface AuditResult {
timestamp: string;
income: DirectionAudit;
expense: DirectionAudit;
status: "ok" | "warning" | "error";
summary: string;
}
interface DirectionAudit {
dbTotal: number;
dbCount: number;
treeTotal: number;
treeCount: number;
excludedTotal: number;
excludedCount: number;
uncategorizedTotal: number;
uncategorizedCount: number;
gapTotal: number;
gapCount: number;
gapRows: { columns: Record<string, string>; sum: number; count: number }[];
treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[];
}
const HISTORY_PATH = resolve(process.cwd(), "data", "finance-audit-history.json");
const MAX_HISTORY = 50;
let auditHistory: AuditResult[] = [];
function loadHistory(): void {
if (!existsSync(HISTORY_PATH)) return;
try {
const parsed = JSON.parse(readFileSync(HISTORY_PATH, "utf-8"));
if (Array.isArray(parsed)) auditHistory = parsed;
} catch {
// повреждённый файл — начинаем с чистой истории
}
}
function saveHistory(): void {
try {
writeFileSync(HISTORY_PATH, JSON.stringify(auditHistory, null, 2), "utf-8");
} catch (err) {
console.error("[AUDIT-AGENT] Ошибка сохранения истории:", err);
}
}
loadHistory();
function flattenCategories(cats: Category[]): Category[] {
const result: Category[] = [];
for (const cat of cats) {
result.push(cat);
if (cat.children?.length) result.push(...flattenCategories(cat.children));
}
return result;
}
function matchTokens(haystack: string, needle: string): boolean {
const tokens = needle.split(/\s+/).filter(Boolean);
const lower = haystack.toLowerCase();
return tokens.every(t => lower.includes(t.toLowerCase()));
}
function rowMatchesRule(row: Record<string, any>, rule: CategoryRule): boolean {
const val = row[rule.column];
if (val === undefined) return false;
switch (rule.operator) {
case "equals": return val === rule.value;
case "contains": return typeof val === "string" && matchTokens(val, rule.value);
case "not_equals": return val !== rule.value;
case "greater_than": return Number(val) > Number(rule.value);
case "less_than": return Number(val) < Number(rule.value);
case "equal_number": return Number(val) === Number(rule.value);
default: return val === rule.value;
}
}
function rowMatchesRules(row: Record<string, any>, rules: CategoryRule[], logic: "and" | "or" = "and"): boolean {
if (!rules || rules.length === 0) return false;
if (logic === "or") return rules.some(r => rowMatchesRule(row, r));
return rules.every(r => rowMatchesRule(row, r));
}
function collectRuleColumns(cats: Category[]): Set<string> {
const cols = new Set<string>();
for (const cat of cats) {
for (const rule of (cat.rules || [])) cols.add(rule.column);
if (cat.children?.length) {
for (const col of collectRuleColumns(cat.children)) cols.add(col);
}
}
return cols;
}
function rowMatchesAutoPath(row: Record<string, any>, autoPath: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }): boolean {
for (let i = 0; i < autoPath.ruleSets.length - 1; i++) {
if (!rowMatchesRules(row, autoPath.ruleSets[i])) return false;
}
return rowMatchesRules(row, autoPath.ruleSets[autoPath.ruleSets.length - 1], autoPath.logic);
}
function computeNodeFromAgg(
cat: Category,
siblings: Category[],
ancestorRuleSets: CategoryRule[][],
ancestorIds: Set<string>,
allFlat: Category[],
autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[],
aggRows: Record<string, any>[],
exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []
): { total: number; count: number; excluded: boolean; children: any[] } {
const catLogic = cat.ruleLogic || "and";
const idx = siblings.indexOf(cat);
const laterSibs = siblings.slice(idx + 1);
const allRuleSets = [...ancestorRuleSets, cat.rules];
let total = 0;
let count = 0;
for (const row of aggRows) {
const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs));
if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue;
if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue;
if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue;
if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) continue;
if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue;
total += row.total;
count += row.cnt;
}
const children: any[] = [];
if (cat.children?.length) {
const childAncestorIds = new Set(ancestorIds);
childAncestorIds.add(cat.id);
for (const child of cat.children) {
children.push(computeNodeFromAgg(child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, aggRows, exclPaths));
}
}
return { total, count, excluded: !!cat.excluded, children };
}
function nodeSubtreeTotal(n: { total: number; excluded: boolean; children: any[] }): number {
if (n.excluded) {
let t = 0;
for (const ch of n.children) t += nodeSubtreeTotal(ch);
return t;
}
let t = n.total;
for (const ch of n.children) t += nodeSubtreeTotal(ch);
return t;
}
function nodeSubtreeCount(n: { count: number; excluded: boolean; children: any[] }): number {
if (n.excluded) {
let c = 0;
for (const ch of n.children) c += nodeSubtreeCount(ch);
return c;
}
let c = n.count;
for (const ch of n.children) c += nodeSubtreeCount(ch);
return c;
}
function nodeExcludedTotal(n: { total: number; excluded: boolean; children: any[] }): number {
if (n.excluded) {
let t = n.total;
for (const ch of n.children) t += nodeExcludedTotal(ch);
return t;
}
let t = 0;
for (const ch of n.children) t += nodeExcludedTotal(ch);
return t;
}
function nodeExcludedCount(n: { count: number; excluded: boolean; children: any[] }): number {
if (n.excluded) {
let c = n.count;
for (const ch of n.children) c += nodeExcludedCount(ch);
return c;
}
let c = 0;
for (const ch of n.children) c += nodeExcludedCount(ch);
return c;
}
async function auditDirection(
table: string,
sumColumn: string,
cats: Category[],
autoConfig: AutoDetectConfig,
allowedColumns: string[]
): Promise<DirectionAudit> {
const allFlat = flattenCategories(cats);
const ruleColumns = collectRuleColumns(cats);
for (const tree of autoConfig.trees) {
for (const col of tree.columns) ruleColumns.add(col);
}
const groupCols = ruleColumns.size > 0
? [...ruleColumns].filter(c => allowedColumns.includes(c))
: allowedColumns.slice(0, 4);
const selectCols = groupCols.map(c => `"${c}"`).join(", ");
const aggResult = await query(
`SELECT ${selectCols}, SUM("${sumColumn}"::numeric) as total, COUNT(*) as cnt FROM "${table}" GROUP BY ${selectCols}`,
[]
);
const aggRows = aggResult.rows.map((r: any) => {
const row: Record<string, any> = { total: Number(r.total), cnt: Number(r.cnt) };
for (const col of groupCols) row[col] = r[col] ?? "";
return row;
});
let dbTotal = 0, dbCount = 0;
for (const row of aggRows) { dbTotal += row.total; dbCount += row.cnt; }
const primaryTreeIds = new Set(autoConfig.trees.filter(t => t.isPrimary).map(t => t.id));
const groupedByTree = new Map<string, Category[]>();
for (const cat of cats) {
const tid = cat.treeId || "__manual__";
if (!groupedByTree.has(tid)) groupedByTree.set(tid, []);
groupedByTree.get(tid)!.push(cat);
}
const autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [];
const excludedPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [];
function collectAllPaths(categories: Category[], ancestors: CategoryRule[][]) {
for (const cat of categories) {
const path = [...ancestors, cat.rules || []];
if (cat.excluded) {
excludedPaths.push({ id: cat.id, ruleSets: path, logic: (cat.ruleLogic || "and") });
} else if (cat.autoDetected && cat.rules?.length) {
autoPaths.push({ ruleSets: path, logic: (cat.ruleLogic || "and") });
}
if (cat.children?.length) {
collectAllPaths(cat.children, path);
}
}
}
collectAllPaths(cats, []);
const treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[] = [];
let allTreeTotal = 0, allTreeCount = 0;
let allExclTotal = 0, allExclCount = 0;
for (const [tid, treeCats] of groupedByTree) {
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
const treeName = tid === "__manual__"
? "Ручные категории"
: autoConfig.trees.find(t => t.id === tid)?.name || tid;
const nodes: any[] = [];
for (const cat of treeCats) {
nodes.push(computeNodeFromAgg(cat, treeCats, [], new Set(), allFlat, autoPaths, aggRows, excludedPaths));
}
const tTotal = nodes.reduce((s, n) => s + nodeSubtreeTotal(n), 0);
const tCount = nodes.reduce((s, n) => s + nodeSubtreeCount(n), 0);
const eTotal = nodes.reduce((s, n) => s + nodeExcludedTotal(n), 0);
const eCount = nodes.reduce((s, n) => s + nodeExcludedCount(n), 0);
treeBreakdown.push({ name: treeName, total: tTotal, count: tCount, isPrimary });
if (isPrimary) {
allTreeTotal += tTotal;
allTreeCount += tCount;
allExclTotal += eTotal;
allExclCount += eCount;
}
}
function isRowMatchedByTreeAudit(row: Record<string, any>, treeCats: Category[]): boolean {
for (const cat of treeCats) {
const catLogic = cat.ruleLogic || "and";
if (!cat.rules?.length || !rowMatchesRules(row, cat.rules, catLogic)) continue;
return true;
}
return false;
}
let uncatTotal = 0, uncatCount = 0;
const gapRows: { columns: Record<string, string>; sum: number; count: number }[] = [];
for (const row of aggRows) {
let matched = false;
for (const [tid, treeCats] of groupedByTree) {
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
if (!isPrimary) continue;
if (isRowMatchedByTreeAudit(row, treeCats)) { matched = true; break; }
}
if (!matched) {
uncatTotal += row.total;
uncatCount += row.cnt;
}
}
const accountedTotal = allTreeTotal + allExclTotal + uncatTotal;
const accountedCount = allTreeCount + allExclCount + uncatCount;
const gapTotal = dbTotal - accountedTotal;
const gapCount = dbCount - accountedCount;
if (Math.abs(gapCount) > 0) {
for (const row of aggRows) {
let inTree = false;
let inUncat = true;
for (const [tid, treeCats] of groupedByTree) {
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
if (!isPrimary) continue;
if (isRowMatchedByTreeAudit(row, treeCats)) { inUncat = false; break; }
}
if (!inUncat) {
for (const [tid, treeCats] of groupedByTree) {
const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
if (!isPrimary) continue;
for (const cat of treeCats) {
if (isRowCapturedByNode(row, cat, treeCats, [], new Set(), allFlat, autoPaths, excludedPaths)) {
inTree = true; break;
}
}
if (inTree) break;
}
}
if (!inUncat && !inTree) {
const cols: Record<string, string> = {};
for (const col of groupCols) cols[col] = String(row[col] || "");
gapRows.push({ columns: cols, sum: row.total, count: row.cnt });
}
}
}
return {
dbTotal, dbCount,
treeTotal: allTreeTotal, treeCount: allTreeCount,
excludedTotal: allExclTotal, excludedCount: allExclCount,
uncategorizedTotal: uncatTotal, uncategorizedCount: uncatCount,
gapTotal: Math.round(gapTotal * 100) / 100,
gapCount,
gapRows: gapRows.slice(0, 50),
treeBreakdown,
};
}
function isRowCapturedByNode(
row: Record<string, any>,
cat: Category,
siblings: Category[],
ancestorRuleSets: CategoryRule[][],
ancestorIds: Set<string>,
allFlat: Category[],
autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[],
exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []
): boolean {
const catLogic = cat.ruleLogic || "and";
const idx = siblings.indexOf(cat);
const laterSibs = siblings.slice(idx + 1);
const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs));
if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) return false;
if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) return false;
if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) return false;
if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) return false;
if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) {
const allRuleSets = [...ancestorRuleSets, cat.rules];
const childAncestorIds = new Set(ancestorIds);
childAncestorIds.add(cat.id);
for (const child of cat.children) {
if (isRowCapturedByNode(row, child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, exclPaths)) return true;
}
return false;
}
return true;
}
const INCOME_COLUMNS = [
"statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands",
"naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta",
"dogovorzakazchika", "statyadop", "schetnaoplatu", "nomervhodyaschegodokumenta",
];
const EXPENSE_COLUMNS = [
"statya", "dochka", "vnuchka", "zakazchik", "naznachenieplatezha",
"statyadop", "kommentarii", "otvetstvenniipokontraktu", "dogovorzakazchika",
"dogovorkontragenta",
];
let auditRunning = false;
export async function runIntegrityAudit(): Promise<AuditResult> {
if (auditRunning) throw new Error("Аудит уже выполняется");
auditRunning = true;
try {
console.log("[AUDIT-AGENT] Запуск аудита целостности данных...");
const config = getTableConfig();
const [income, expense] = await Promise.all([
auditDirection(
config.incomeTable,
config.incomeSumColumn,
getIncomeCategories() as Category[],
getIncomeAutoDetectConfig(),
INCOME_COLUMNS
),
auditDirection(
config.expenseTable,
config.expenseSumColumn,
getExpenseCategories() as Category[],
getExpenseAutoDetectConfig(),
EXPENSE_COLUMNS
),
]);
const hasGap = income.gapCount !== 0 || expense.gapCount !== 0;
const status = hasGap ? "warning" : "ok";
const summaryParts: string[] = [];
if (income.gapCount === 0 && expense.gapCount === 0) {
summaryParts.push("Все строки распределены корректно");
} else {
if (income.gapCount !== 0) summaryParts.push(`Доходы: ${income.gapCount} строк (${income.gapTotal.toLocaleString("ru")}₽) не распределены`);
if (expense.gapCount !== 0) summaryParts.push(`Расходы: ${expense.gapCount} строк (${expense.gapTotal.toLocaleString("ru")}₽) не распределены`);
}
const result: AuditResult = {
timestamp: new Date().toISOString(),
income,
expense,
status,
summary: summaryParts.join(". "),
};
auditHistory.unshift(result);
if (auditHistory.length > MAX_HISTORY) auditHistory.pop();
saveHistory();
console.log(`[AUDIT-AGENT] Аудит завершён: ${result.summary}`);
console.log(`[AUDIT-AGENT] Доходы: БД=${income.dbCount} строк, деревья=${income.treeCount}, excluded=${income.excludedCount}, нераспр=${income.uncategorizedCount}, gap=${income.gapCount}`);
console.log(`[AUDIT-AGENT] Расходы: БД=${expense.dbCount} строк, деревья=${expense.treeCount}, excluded=${expense.excludedCount}, нераспр=${expense.uncategorizedCount}, gap=${expense.gapCount}`);
return result;
} finally {
auditRunning = false;
}
}
export function getAuditHistory(): AuditResult[] {
return auditHistory;
}
// =====================
// Планировщик: ежедневно в 03:00 (МСК), без node-cron
// =====================
const TICK_MS = 60_000;
const MSK_OFFSET_MS = 3 * 60 * 60 * 1000; // МСК = UTC+3
const AUDIT_HOUR_MSK = 3;
let lastRunDay: string | null = auditHistory[0]?.timestamp
? new Date(new Date(auditHistory[0].timestamp).getTime() + MSK_OFFSET_MS).toISOString().slice(0, 10)
: null;
let schedulerStarted = false;
function mskToday(): string {
return new Date(Date.now() + MSK_OFFSET_MS).toISOString().slice(0, 10);
}
function mskHourNow(): number {
return new Date(Date.now() + MSK_OFFSET_MS).getUTCHours();
}
async function tick(): Promise<void> {
const today = mskToday();
if (lastRunDay === today) return; // сегодня уже запускался
if (mskHourNow() < AUDIT_HOUR_MSK) return; // ещё не наступило 03:00 по МСК
lastRunDay = today;
try {
await runIntegrityAudit();
} catch (err) {
console.error("[AUDIT-AGENT] Ошибка при выполнении аудита:", err);
}
}
export function startAuditScheduler(): void {
if (schedulerStarted) return;
schedulerStarted = true;
setInterval(() => {
tick().catch((err) => console.error("[AUDIT-AGENT] Tick error:", err));
}, TICK_MS);
console.log("[AUDIT-AGENT] Планировщик запущен (ежедневно в 03:00 по МСК)");
// Первый запуск через 10 секунд после старта сервера
setTimeout(async () => {
try {
const result = await runIntegrityAudit();
lastRunDay = new Date(new Date(result.timestamp).getTime() + MSK_OFFSET_MS).toISOString().slice(0, 10);
} catch (err) {
console.error("[AUDIT-AGENT] Ошибка при начальном аудите:", err);
}
}, 10000);
}
// =====================
// HTTP endpoints
// =====================
export function registerAuditRoutes(app: Express) {
app.get("/api/audit/history", authenticateToken, requirePermission('finance.manage'), (_req: AuthenticatedRequest, res) => {
res.json({ success: true, history: getAuditHistory() });
});
app.post("/api/audit/run", authenticateToken, requirePermission('finance.manage'), async (_req: AuthenticatedRequest, res) => {
try {
const result = await runIntegrityAudit();
res.json({ success: true, result });
} catch (err: any) {
console.error("[AUDIT-AGENT] Ошибка ручного запуска:", err);
res.status(500).json({ success: false, error: err.message });
}
});
app.get("/api/audit/status", authenticateToken, requirePermission('finance.manage'), (_req: AuthenticatedRequest, res) => {
res.json({ success: true, last: auditHistory[0] || null, running: auditRunning });
});
}