iistwin/server/finance-di2/audit-agent.ts

import { query } from "./db-client";
import fs from "fs";
import path from "path";

const CATEGORIES_FILE = path.join(process.cwd(), "data", "income-categories.json");
const EXPENSE_CATEGORIES_FILE = path.join(process.cwd(), "data", "expense-categories.json");
const INCOME_AUTO_DETECT_CONFIG_FILE = path.join(process.cwd(), "data", "income-auto-detect-config.json");
const EXPENSE_AUTO_DETECT_CONFIG_FILE = path.join(process.cwd(), "data", "auto-detect-config.json");

interface CategoryRule {
  column: string;
  operator: string;
  value: string;
}

interface Category {
  id: string;
  name: string;
  rules: CategoryRule[];
  ruleLogic?: "and" | "or";
  children: Category[];
  excluded?: boolean;
  autoDetected?: boolean;
  treeId?: string;
}

interface AuditResult {
  timestamp: string;
  income: DirectionAudit;
  expense: DirectionAudit;
  status: "ok" | "warning" | "error";
  summary: string;
}

interface DirectionAudit {
  dbTotal: number;
  dbCount: number;
  treeTotal: number;
  treeCount: number;
  excludedTotal: number;
  excludedCount: number;
  uncategorizedTotal: number;
  uncategorizedCount: number;
  gapTotal: number;
  gapCount: number;
  gapRows: { columns: Record<string, string>; sum: number; count: number }[];
  treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[];
}

const HISTORY_PATH = path.join(process.cwd(), "data", "finance-audit-history.json");
const MAX_HISTORY = 50;

let auditHistory: AuditResult[] = [];

// Персистентность истории аудита (порт из удалённого server/finance/audit.ts):
// загрузка при старте модуля, запись после каждого runIntegrityAudit.
function loadHistory(): void {
  if (!fs.existsSync(HISTORY_PATH)) return;
  try {
    const parsed = JSON.parse(fs.readFileSync(HISTORY_PATH, "utf-8"));
    if (Array.isArray(parsed)) auditHistory = parsed;
  } catch {
    // повреждённый файл — начинаем с чистой истории
  }
}

function saveHistory(): void {
  try {
    fs.writeFileSync(HISTORY_PATH, JSON.stringify(auditHistory, null, 2), "utf-8");
  } catch (err) {
    console.error("[AUDIT-AGENT] Ошибка сохранения истории:", err);
  }
}

loadHistory();

function loadJSON(filePath: string): any {
  try {
    return JSON.parse(fs.readFileSync(filePath, "utf-8"));
  } catch {
    return [];
  }
}

function flattenCategories(cats: Category[]): Category[] {
  const result: Category[] = [];
  for (const cat of cats) {
    result.push(cat);
    if (cat.children?.length) result.push(...flattenCategories(cat.children));
  }
  return result;
}

function matchTokens(haystack: string, needle: string): boolean {
  const tokens = needle.split(/\s+/).filter(Boolean);
  const lower = haystack.toLowerCase();
  return tokens.every(t => lower.includes(t.toLowerCase()));
}

function rowMatchesRule(row: Record<string, any>, rule: CategoryRule): boolean {
  const val = row[rule.column];
  if (val === undefined) return false;
  switch (rule.operator) {
    case "equals": return val === rule.value;
    case "contains": return typeof val === "string" && matchTokens(val, rule.value);
    case "not_equals": return val !== rule.value;
    case "greater_than": return Number(val) > Number(rule.value);
    case "less_than": return Number(val) < Number(rule.value);
    case "equal_number": return Number(val) === Number(rule.value);
    default: return val === rule.value;
  }
}

function rowMatchesRules(row: Record<string, any>, rules: CategoryRule[], logic: "and" | "or" = "and"): boolean {
  if (!rules || rules.length === 0) return false;
  if (logic === "or") return rules.some(r => rowMatchesRule(row, r));
  return rules.every(r => rowMatchesRule(row, r));
}

function collectRuleColumns(cats: Category[]): Set<string> {
  const cols = new Set<string>();
  for (const cat of cats) {
    for (const rule of (cat.rules || [])) cols.add(rule.column);
    if (cat.children?.length) {
      for (const col of collectRuleColumns(cat.children)) cols.add(col);
    }
  }
  return cols;
}

interface AutoDetectConfig {
  trees: { id: string; name: string; columns: string[]; isPrimary?: boolean; filters?: any[]; filterLogic?: string }[];
}

function rowMatchesAutoPath(row: Record<string, any>, autoPath: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }): boolean {
  for (let i = 0; i < autoPath.ruleSets.length - 1; i++) {
    if (!rowMatchesRules(row, autoPath.ruleSets[i])) return false;
  }
  return rowMatchesRules(row, autoPath.ruleSets[autoPath.ruleSets.length - 1], autoPath.logic);
}

function computeNodeFromAgg(
  cat: Category,
  siblings: Category[],
  ancestorRuleSets: CategoryRule[][],
  ancestorIds: Set<string>,
  allFlat: Category[],
  autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[],
  aggRows: Record<string, any>[],
  exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []
): { total: number; count: number; excluded: boolean; children: any[] } {
  const catLogic = cat.ruleLogic || "and";
  const idx = siblings.indexOf(cat);
  const laterSibs = siblings.slice(idx + 1);

  const allRuleSets = [...ancestorRuleSets, cat.rules];

  let total = 0;
  let count = 0;
  for (const row of aggRows) {
    const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs));
    if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue;
    if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue;
    if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue;
    if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) continue;
    if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue;
    total += row.total;
    count += row.cnt;
  }

  const children: any[] = [];
  if (cat.children?.length) {
    const childAncestorIds = new Set(ancestorIds);
    childAncestorIds.add(cat.id);
    for (const child of cat.children) {
      children.push(computeNodeFromAgg(child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, aggRows, exclPaths));
    }
  }

  return { total, count, excluded: !!cat.excluded, children };
}

function nodeSubtreeTotal(n: { total: number; excluded: boolean; children: any[] }): number {
  if (n.excluded) {
    let t = 0;
    for (const ch of n.children) t += nodeSubtreeTotal(ch);
    return t;
  }
  let t = n.total;
  for (const ch of n.children) t += nodeSubtreeTotal(ch);
  return t;
}

function nodeSubtreeCount(n: { count: number; excluded: boolean; children: any[] }): number {
  if (n.excluded) {
    let c = 0;
    for (const ch of n.children) c += nodeSubtreeCount(ch);
    return c;
  }
  let c = n.count;
  for (const ch of n.children) c += nodeSubtreeCount(ch);
  return c;
}

function nodeExcludedTotal(n: { total: number; excluded: boolean; children: any[] }): number {
  if (n.excluded) {
    let t = n.total;
    for (const ch of n.children) t += nodeExcludedTotal(ch);
    return t;
  }
  let t = 0;
  for (const ch of n.children) t += nodeExcludedTotal(ch);
  return t;
}

function nodeExcludedCount(n: { count: number; excluded: boolean; children: any[] }): number {
  if (n.excluded) {
    let c = n.count;
    for (const ch of n.children) c += nodeExcludedCount(ch);
    return c;
  }
  let c = 0;
  for (const ch of n.children) c += nodeExcludedCount(ch);
  return c;
}

async function auditDirection(
  table: string,
  sumColumn: string,
  categoriesFile: string,
  autoDetectConfigFile: string,
  allowedColumns: string[]
): Promise<DirectionAudit> {
  const cats: Category[] = loadJSON(categoriesFile);
  const autoConfigRaw = loadJSON(autoDetectConfigFile);
  const autoConfig: AutoDetectConfig = { trees: autoConfigRaw?.trees || [] };
  const allFlat = flattenCategories(cats);

  const ruleColumns = collectRuleColumns(cats);
  for (const tree of autoConfig.trees) {
    for (const col of tree.columns) ruleColumns.add(col);
  }
  const groupCols = ruleColumns.size > 0
    ? [...ruleColumns].filter(c => allowedColumns.includes(c))
    : allowedColumns.slice(0, 4);

  const selectCols = groupCols.join(", ");
  const aggResult = await query(
    `SELECT ${selectCols}, SUM(${sumColumn}::numeric) as total, COUNT(*) as cnt FROM ${table} GROUP BY ${selectCols}`,
    []
  );

  const aggRows = aggResult.rows.map((r: any) => {
    const row: Record<string, any> = { total: Number(r.total), cnt: Number(r.cnt) };
    for (const col of groupCols) row[col] = r[col] ?? "";
    return row;
  });

  let dbTotal = 0, dbCount = 0;
  for (const row of aggRows) { dbTotal += row.total; dbCount += row.cnt; }

  const primaryTreeIds = new Set(autoConfig.trees.filter(t => t.isPrimary).map(t => t.id));
  const configTreeIds = new Set(autoConfig.trees.map(t => t.id));

  const groupedByTree = new Map<string, Category[]>();
  for (const cat of cats) {
    const tid = cat.treeId || "__manual__";
    if (!groupedByTree.has(tid)) groupedByTree.set(tid, []);
    groupedByTree.get(tid)!.push(cat);
  }

  const autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [];
  const excludedPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [];
  function collectAllPaths(categories: Category[], ancestors: CategoryRule[][]) {
    for (const cat of categories) {
      const path = [...ancestors, cat.rules || []];
      if (cat.excluded) {
        excludedPaths.push({ id: cat.id, ruleSets: path, logic: (cat.ruleLogic || "and") });
      } else if (cat.autoDetected && cat.rules?.length) {
        autoPaths.push({ ruleSets: path, logic: (cat.ruleLogic || "and") });
      }
      if (cat.children?.length) {
        collectAllPaths(cat.children, path);
      }
    }
  }
  collectAllPaths(cats, []);

  const treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[] = [];
  let allTreeTotal = 0, allTreeCount = 0;
  let allExclTotal = 0, allExclCount = 0;

  for (const [tid, treeCats] of groupedByTree) {
    const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
    const treeName = tid === "__manual__"
      ? "Ручные категории"
      : autoConfig.trees.find(t => t.id === tid)?.name || tid;

    const nodes: any[] = [];
    for (const cat of treeCats) {
      nodes.push(computeNodeFromAgg(cat, treeCats, [], new Set(), allFlat, autoPaths, aggRows, excludedPaths));
    }

    const tTotal = nodes.reduce((s, n) => s + nodeSubtreeTotal(n), 0);
    const tCount = nodes.reduce((s, n) => s + nodeSubtreeCount(n), 0);
    const eTotal = nodes.reduce((s, n) => s + nodeExcludedTotal(n), 0);
    const eCount = nodes.reduce((s, n) => s + nodeExcludedCount(n), 0);

    treeBreakdown.push({ name: treeName, total: tTotal, count: tCount, isPrimary });

    if (isPrimary) {
      allTreeTotal += tTotal;
      allTreeCount += tCount;
      allExclTotal += eTotal;
      allExclCount += eCount;
    }
  }

  function isRowMatchedByTreeAudit(row: Record<string, any>, treeCats: Category[]): boolean {
    for (const cat of treeCats) {
      const catLogic = cat.ruleLogic || "and";
      if (!cat.rules?.length || !rowMatchesRules(row, cat.rules, catLogic)) continue;
      return true;
    }
    return false;
  }

  let uncatTotal = 0, uncatCount = 0;
  const gapRows: { columns: Record<string, string>; sum: number; count: number }[] = [];

  for (const row of aggRows) {
    let matched = false;
    for (const [tid, treeCats] of groupedByTree) {
      const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
      if (!isPrimary) continue;
      if (isRowMatchedByTreeAudit(row, treeCats)) { matched = true; break; }
    }
    if (!matched) {
      uncatTotal += row.total;
      uncatCount += row.cnt;
    }
  }

  const accountedTotal = allTreeTotal + allExclTotal + uncatTotal;
  const accountedCount = allTreeCount + allExclCount + uncatCount;
  const gapTotal = dbTotal - accountedTotal;
  const gapCount = dbCount - accountedCount;

  if (Math.abs(gapCount) > 0) {
    for (const row of aggRows) {
      let inTree = false;
      let inUncat = true;
      for (const [tid, treeCats] of groupedByTree) {
        const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
        if (!isPrimary) continue;
        if (isRowMatchedByTreeAudit(row, treeCats)) { inUncat = false; break; }
      }
      if (!inUncat) {
        for (const [tid, treeCats] of groupedByTree) {
          const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid);
          if (!isPrimary) continue;
          for (const cat of treeCats) {
            if (isRowCapturedByNode(row, cat, treeCats, [], new Set(), allFlat, autoPaths, excludedPaths)) {
              inTree = true; break;
            }
          }
          if (inTree) break;
        }
      }
      if (!inUncat && !inTree) {
        const cols: Record<string, string> = {};
        for (const col of groupCols) cols[col] = String(row[col] || "");
        gapRows.push({ columns: cols, sum: row.total, count: row.cnt });
      }
    }
  }

  return {
    dbTotal, dbCount,
    treeTotal: allTreeTotal, treeCount: allTreeCount,
    excludedTotal: allExclTotal, excludedCount: allExclCount,
    uncategorizedTotal: uncatTotal, uncategorizedCount: uncatCount,
    gapTotal: Math.round(gapTotal * 100) / 100,
    gapCount,
    gapRows: gapRows.slice(0, 50),
    treeBreakdown,
  };
}

function isRowCapturedByNode(
  row: Record<string, any>,
  cat: Category,
  siblings: Category[],
  ancestorRuleSets: CategoryRule[][],
  ancestorIds: Set<string>,
  allFlat: Category[],
  autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[],
  exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []
): boolean {
  const catLogic = cat.ruleLogic || "and";
  const idx = siblings.indexOf(cat);
  const laterSibs = siblings.slice(idx + 1);

  const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs));
  if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) return false;
  if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) return false;
  if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) return false;
  if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) return false;

  if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) {
    const allRuleSets = [...ancestorRuleSets, cat.rules];
    const childAncestorIds = new Set(ancestorIds);
    childAncestorIds.add(cat.id);
    for (const child of cat.children) {
      if (isRowCapturedByNode(row, child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, exclPaths)) return true;
    }
    return false;
  }

  return true;
}

const INCOME_COLUMNS = [
  "statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands",
  "naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta",
  "dogovorzakazchika", "statyadop", "schetnaoplatu", "nomervhodyaschegodokumenta",
];

const EXPENSE_COLUMNS = [
  "statya", "dochka", "vnuchka", "zakazchik", "naznachenieplatezha",
  "statyadop", "kommentarii", "otvetstvenniipokontraktu", "dogovorzakazchika",
  "dogovorkontragenta",
];

export async function runIntegrityAudit(): Promise<AuditResult> {
  console.log("[AUDIT-AGENT] Запуск аудита целостности данных...");

  const [income, expense] = await Promise.all([
    auditDirection("opdohodi", "summa", CATEGORIES_FILE, INCOME_AUTO_DETECT_CONFIG_FILE, INCOME_COLUMNS),
    auditDirection("oprashodi", "summadokumenta", EXPENSE_CATEGORIES_FILE, EXPENSE_AUTO_DETECT_CONFIG_FILE, EXPENSE_COLUMNS),
  ]);

  const hasGap = income.gapCount !== 0 || expense.gapCount !== 0;
  const status = hasGap ? "warning" : "ok";

  const summaryParts: string[] = [];
  if (income.gapCount === 0 && expense.gapCount === 0) {
    summaryParts.push("Все строки распределены корректно");
  } else {
    if (income.gapCount !== 0) summaryParts.push(`Доходы: ${income.gapCount} строк (${income.gapTotal.toLocaleString("ru")}₽) не распределены`);
    if (expense.gapCount !== 0) summaryParts.push(`Расходы: ${expense.gapCount} строк (${expense.gapTotal.toLocaleString("ru")}₽) не распределены`);
  }

  const result: AuditResult = {
    timestamp: new Date().toISOString(),
    income,
    expense,
    status,
    summary: summaryParts.join(". "),
  };

  auditHistory.unshift(result);
  if (auditHistory.length > MAX_HISTORY) auditHistory.pop();
  saveHistory();

  console.log(`[AUDIT-AGENT] Аудит завершён: ${result.summary}`);
  console.log(`[AUDIT-AGENT]   Доходы: БД=${income.dbCount} строк, деревья=${income.treeCount}, excluded=${income.excludedCount}, нераспр=${income.uncategorizedCount}, gap=${income.gapCount}`);
  console.log(`[AUDIT-AGENT]   Расходы: БД=${expense.dbCount} строк, деревья=${expense.treeCount}, excluded=${expense.excludedCount}, нераспр=${expense.uncategorizedCount}, gap=${expense.gapCount}`);

  return result;
}

export function getAuditHistory(): AuditResult[] {
  return auditHistory;
}