From 254950f151f98b115461eb107e13eb704592a8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D1=8C=D1=8F=D1=81=20=D0=A1=D1=83=D0=BB=D1=82?= =?UTF-8?q?=D0=B0=D0=BD=D0=BE=D0=B2?= Date: Sun, 6 Sep 2026 19:27:43 +0300 Subject: [PATCH] =?UTF-8?q?DI2=20=D0=B2=D0=BD=D0=B5=D0=B4=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5,=20=D1=82=D0=B0=D1=81=D0=BA=201:=20=D1=81?= =?UTF-8?q?=D0=B5=D1=80=D0=B2=D0=B5=D1=80=D0=BD=D0=B0=D1=8F=20=D0=B8=D0=BD?= =?UTF-8?q?=D1=82=D0=B5=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D1=8F=20Data-Insight?= =?UTF-8?q?2=20(server/finance-di2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Копия DI2-сервера в server/finance-di2/ с правками: schema/db-client/cache/table-config/override-tables/google-auth/audit-agent/routes - db-client — обёртка над существующим пулом server/finance/db-client (второй пул не создаётся, добавлен экспорт isConnectionError) - ai-config.ts — чистое IO конфигов из ai-agent.ts без Telegram-поллинга; роуты /api/ai/toggle и /api/ai/status удалены, без compression/startAuditScheduler/autoStartIfEnabled - Обёртка registerDi2Routes с auth-gate (authenticateToken + finance.manage), регистрация строго перед registerFinanceRoutes - Статика /di2 из dist/public-di2 перед веткой vite/static (dev и prod) - Фикс предсуществующего бага DI2: buildGlobalExclusionConditions без cats в /api/profitability --- .superpowers/sdd/di2-embed/progress.md | 21 + .superpowers/sdd/di2-embed/task-1-brief.md | 59 + .superpowers/sdd/di2-embed/task-1-report.md | 41 + server/finance-di2/ai-config.ts | 192 + server/finance-di2/audit-agent.ts | 450 ++ server/finance-di2/cache.ts | 69 + server/finance-di2/db-client.ts | 24 + server/finance-di2/google-auth.ts | 155 + server/finance-di2/index.ts | 55 + server/finance-di2/override-tables.ts | 232 + server/finance-di2/routes.ts | 6359 +++++++++++++++++++ server/finance-di2/schema.ts | 366 ++ server/finance-di2/table-config.ts | 154 + server/finance/db-client.ts | 12 + server/index.ts | 8 + server/routes/index.ts | 3 + 16 files changed, 8200 insertions(+) create mode 100644 .superpowers/sdd/di2-embed/progress.md create mode 100644 .superpowers/sdd/di2-embed/task-1-brief.md create mode 100644 .superpowers/sdd/di2-embed/task-1-report.md create mode 100644 server/finance-di2/ai-config.ts create mode 100644 server/finance-di2/audit-agent.ts create mode 100644 server/finance-di2/cache.ts create mode 100644 server/finance-di2/db-client.ts create mode 100644 server/finance-di2/google-auth.ts create mode 100644 server/finance-di2/index.ts create mode 100644 server/finance-di2/override-tables.ts create mode 100644 server/finance-di2/routes.ts create mode 100644 server/finance-di2/schema.ts create mode 100644 server/finance-di2/table-config.ts diff --git a/.superpowers/sdd/di2-embed/progress.md b/.superpowers/sdd/di2-embed/progress.md new file mode 100644 index 0000000..e1d1c6f --- /dev/null +++ b/.superpowers/sdd/di2-embed/progress.md @@ -0,0 +1,21 @@ +# SDD ledger — plan: внедрение Data-Insight2 в iistwin (субапп /di2) + +BASE (start): a60a75e + +## Решения (Rulings) +- R1: DI2 внедряется целиком «как есть» (пользователь, 2026-09-05): отдельный SPA-субапп на `/di2` + серверные роуты DI2 регистрируются ДО старого модуля финансов и перехватывают общие пути (/api/income/*, /api/debt/* и т.д.). Старый модуль НЕ удаляем до сравнения пользователем. Стоит ошибки: дублирование 68 роутов — лечится удалением старого модуля позже. +- R2: ai-agent.ts (Telegram-поллинг бот) НЕ переносим. Вместо него — `ai-config.ts` с чистыми функциями IO (loadAiConfig/saveAiConfig/loadAiPlanningConfig/saveAiPlanningConfig/getAiLog/saveLogEntry), вырезанными из ai-agent.ts. Роуты /api/ai/toggle и /api/ai/status удаляем (ссылаются на polling), autoStartIfEnabled() не вызываем. Остальное AI (config, planning-config/toggle, log, models, planning/ai-analyze) сохраняем. Стоит: кнопки «вкл бота» в настройках AI DI2 не будет. +- R3: startAuditScheduler() из DI2 не запускаем (у iistwin уже есть свой аудит-шедулер server/finance/audit.ts; двойные тяжёлые запросы во внешнюю БД не нужны). Ручной /api/audit/run и /api/audit/history DI2 сохраняем. Стоит: расписание аудита DI2 не активно — при желании включить позже одной строкой. +- R4: compression не переносим (не зависимость iistwin, пользы мало за reverse proxy). Стоит: ничего. +- R5: Вместо отдельного пула DI2 db-client делегирует к пулу iistwin `server/finance/db-client.ts` (один пул, тот же файл data/db-config.json). Стоит ошибки в делегировании — проверить тестом подключения. +- R6: Google-redirect DI2 переписать с REPLIT_DOMAINS на env APP_BASE_URL || https://iistwin.ru (путь /api/auth/google/callback оставить, конфликтов нет). + +## Preflight scan +- Task 1 (server) и Task 2 (client) трогают общий package.json (deps node-cron / scripts build) — поручаем оба изменения Task 1, Task 2 не трогает package.json... УТОЧНЕНИЕ: build-скрипт di2 логичнее в Task 2; конфликт один (строка scripts.build) — правило: Task 2 редактирует scripts.build, Task 1 только добавляет node-cron в dependencies. Порядок: Task 1 → Task 2 (resume агента). +- Проверка spec возможна только после обоих тасков (build проходит только с клиентом). Ревью единое по всему диапазону BASE..HEAD. + +## Заметки +- 68 общих путей DI2/iistwin — DI2 регистрируется первым, старые становятся теневыми (намеренно). +- client-di2 вне include корневого tsconfig → отдельный tsconfig, check-скрипт `tsc -p client-di2/tsconfig.json`. +- wouter 3.10 поддерживает `` — base="/di2". +- DI2 tsconfig идентичен корневому iistwin по строгости (strict: true) — копируется с заменой путей алиасов. diff --git a/.superpowers/sdd/di2-embed/task-1-brief.md b/.superpowers/sdd/di2-embed/task-1-brief.md new file mode 100644 index 0000000..fbc48a4 --- /dev/null +++ b/.superpowers/sdd/di2-embed/task-1-brief.md @@ -0,0 +1,59 @@ +# Task 1 — Серверная интеграция Data-Insight2 + +Цель: перенести серверную часть DI2 (`C:\1Ai\1iistwin\tmp\Data-Insight2\server`) в `server/finance-di2/` репозитория `C:\1Ai\1iistwin\1Project`, обёрнуть в авторизацию iistwin и зарегистрировать роуты ДО старого модуля финансов. Клиент НЕ входит в задачу (Task 2), но добавь зависимость `node-cron` (DI2 audit-agent.ts импортирует cron вверху файла; планировщик мы не запускаем, но импорт должен резолвиться). НЕ меняй `scripts.build` в package.json (это Task 2). + +## Источник (только чтение) +- `C:\1Ai\1iistwin\tmp\Data-Insight2\server\`: routes.ts (6387 строк), cache.ts, table-config.ts, override-tables.ts, google-auth.ts, audit-agent.ts (477 строк), db-client.ts, storage.ts (5 строк). +- `C:\1Ai\1iistwin\tmp\Data-Insight2\shared\schema.ts` (366 строк, чистые типы/константы, без drizzle). + +## Что создать в 1Project + +1. **`server/finance-di2/`** — копии файлов DI2 с правками: + - `schema.ts` ← копия `tmp/Data-Insight2/shared/schema.ts` (типы и константы). + - `db-client.ts` — НЕ копировать DI2-версию. Написать обёртку, которая импортирует из `../finance/db-client` (iistwin): `query, testConnection, getConfig, setConfig, isConnectionError` и экспортирует их же + `getPoolClient` (проверь, что `server/finance/db-client.ts` экспортирует все эти имена; если `getPoolClient` отсутствует — добавь его туда, а не дублируй пул: открой файл и посмотри, есть ли там pool — экспортируй существующий). Формат конфига (data/db-config.json) идентичен — это тот же код по происхождению. + - `cache.ts`, `table-config.ts`, `override-tables.ts` — копии без изменений (проверь их импорты: они должны ссылаться на `./db-client` и `./schema` — исправь алиасы). + - `google-auth.ts` — копия с правкой `getRedirectUri()`: вместо `process.env.REPLIT_DOMAINS` использовать `process.env.APP_BASE_URL || "https://iistwin.ru"`, redirect `${APP_BASE_URL}/api/auth/google/callback`. Протокол всегда https. + - `ai-config.ts` — НОВЫЙ файл: вырезать из `tmp/Data-Insight2/server/ai-agent.ts` только чистые функции IO без Telegram/поллинга: `loadAiConfig`, `saveAiConfig`, `loadAiPlanningConfig`, `saveAiPlanningConfig` (файлы data/ai-config.json, data/ai-planning-config.json), а также `getAiLog`, `saveLogEntry` (посмотри их реализацию в ai-agent.ts — обычно append в json/jsonl; перенеси как есть, включая нужные интерфейсы AiConfig/AiPlanningConfig). Без cron, без fetch, без Telegram. + - `audit-agent.ts` — копия, но удалить `startAuditScheduler` и его вызовы (cronJob и импорт node-cron не нужны, если импорт только для этой функции — проверь: если cron используется только в startAuditScheduler, удали импорт и функцию целиком; тогда node-cron в package.json НЕ нужен — не добавляй). Оставить `runIntegrityAudit`, `getAuditHistory`. + - `routes.ts` — копия с такими правками: + - Импорт `dbConnectionSchema` с `./schema` (не `@shared/schema`). + - Удалить `import compression from "compression"` и `app.use(compression());` (строка ~1142). + - Строку 1371 `const {...} = await import("./ai-agent")` заменить на импорт из `./ai-config`: нужны только `loadAiConfig, saveAiConfig, loadAiPlanningConfig, saveAiPlanningConfig, getAiLog, saveLogEntry`. + - Удалить роуты `app.post("/api/ai/toggle", ...)` и `app.get("/api/ai/status", ...)` (используют startPolling/stopPolling/isPolling). + - Удалить вызов `autoStartIfEnabled();` (~строка 1508). + - В конце: удалить `startAuditScheduler();` (строка 6384), оставить `scheduleTreeRebuild();`. + - Проверь, что в routes.ts больше нет ссылок на ai-agent, compression, node-cron (grep). + - Проверь остальные импорты вверху routes.ts: `./storage` (5-строчный файл — перенеси тоже, если импортируется), `./override-tables`, `./table-config`, `./cache`, `./google-auth`, `./audit-agent` — все становятся `./xxx` внутри finance-di2 (уже так). + - `index.ts` — НОВЫЙ файл-обёртка `registerDi2Routes(app: Express)`: + - Создать `express.Router()`. + - Middleware на роутере: если `req.path` начинается с одного из префиксов DI2 — пропустить через `authenticateToken` затем `requirePermission('finance.manage')` (импорт из `../middleware/auth.middleware`), иначе `next()`. + - Префиксы: `/api/income`, `/api/income-categories`, `/api/expenses`, `/api/expense-categories`, `/api/debt`, `/api/dashboard`, `/api/profitability`, `/api/heatmap`, `/api/planning`, `/api/salary`, `/api/audit`, `/api/calculations`, `/api/connection`, `/api/db`, `/api/table-config`, `/api/table-settings`, `/api/overrides`, `/api/tags`, `/api/tag-assignments`, `/api/cache`, `/api/refresh-all`, `/api/admin`, `/api/auth/google`, `/api/ai`, `/api/telegram`. + - Затем вызвать DI2 `registerRoutes(httpServer, router as any)` — передав `null as any` вместо httpServer (он используется только в `return httpServer` в конце). Функция async — await. Затем `app.use(router)` — но НЕ сmount-префиксом, роуты абсолютные. + - ВАЖНО порядок middleware: `router.use(authGate)` должен стоять ПЕРЕД регистрацией роутов DI2, и все DI2-роуты регистрируются на этом же роутере. +2. **`server/routes/index.ts`**: добавить `await registerDi2Routes(app);` строкой НЕПОСРЕДСТВЕННО ПЕРЕД `await registerFinanceRoutes(app);` (строка 113). Импорт добавь рядом с импортом registerFinanceRoutes (строка 33). +3. **`server/index.ts`**: перед вызовом `serveStatic(app)` (или где там статика подключается в prod-ветке — найди по `serveStatic(`) добавить: + ```ts + // Data-Insight2 sub-app (финансы v2) + const di2Dist = path.resolve(import.meta.dirname, "public-di2"); + if (fs.existsSync(di2Dist)) { + app.use("/di2", express.static(di2Dist)); + app.use("/di2", (_req, res) => res.sendFile(path.resolve(di2Dist, "index.html"))); + } + ``` + (проверь, какие импорты path/fs уже есть в index.ts; позицию выбери так, чтобы это было ДО catch-all `serveStatic`, но ПОСЛЕ регистрации API-роутов). В dev-режиме (setupVite) тоже должен работать: setupVite вешает `app.use("*")` — проверь порядок: если setupVite вызывается в dev ПОСЛЕ нашего блока, то /di2/* уйдёт в main index.html. Чтобы dev тоже работал, добавь этот блок ПЕРЕД веткой `if (process.env.NODE_ENV === "development") setupVite(...) else serveStatic(app)` (найди эту ветку и вставь до неё). Проверь наличие fs/path импортов. +4. **package.json**: если audit-agent оставил импорт cron — добавь `node-cron` в dependencies (проверь версию из DI2 package.json). Если импорт удалён — не добавляй. `scripts.build` НЕ трогать. + +## Проверки (обязательно выполнить и приложить вывод) +1. `grep -rn "ai-agent\|compression\|REPLIT" server/finance-di2/` — пусто. +2. `npx tsc --noEmit` (корневой check) — без ошибок в server/finance-di2 (ошибки в других местах, если были до тебя, не считаются; проверь baseline сначала: `git stash` не нужен — просто сравни с ошибками на BASE, если есть новые — чини). +3. `npm run build` должен пройти (клиентская часть di2 ещё не создана — это нормально, build собирает только main client; сборка di2 будет в Task 2). Если build падает из-за твоих изменений — чини. +4. Быстрый smoke-тест сервера без Docker: запустить `npm run dev` не надо (трогает БД продакшена локально? нет — локальный dev стучится в локальную БД; НЕ запускай дев-сервер). Достаточно typecheck+build. + +## Контракт отчёта +Напиши файл `C:\1Ai\1iistwin\1Project\.superpowers\sdd\di2-embed\task-1-report.md`: что скопировано, какие правки, вывод grep-проверок, вывод tsc/build (последние строки), открытые вопросы. Верни: статус, список коммитов, одну строку по проверкам, опасения. + +## Жёсткие ограничения +- Код/комментарии — на русском (существующие DI2-комментарии не переводить обратно, оставь как есть). +- Не трогай `server/finance/*` кроме случая, когда нужно экспортировать getPoolClient (добавь экспорт существующего пула, не создавай второй пул). +- Не коммить изменения в tmp/Data-Insight2. +- Pre-push hook прогоняет check+build — коммить в main. diff --git a/.superpowers/sdd/di2-embed/task-1-report.md b/.superpowers/sdd/di2-embed/task-1-report.md new file mode 100644 index 0000000..929dde2 --- /dev/null +++ b/.superpowers/sdd/di2-embed/task-1-report.md @@ -0,0 +1,41 @@ +# Task 1 — Серверная интеграция Data-Insight2: отчёт + +## Статус: DONE + +## Что сделано + +### Новая папка `server/finance-di2/` +- `schema.ts` — копия `tmp/Data-Insight2/shared/schema.ts` (чистые типы/константы, zod). +- `db-client.ts` — **НЕ копия DI2**, а обёртка поверх `server/finance/db-client.ts` (iistwin): реэкспорт `query, testConnection, setConfig, isConnectionError, getPoolClient` + `getConfig()` с приведением к DI2-совместимой форме (iistwin-версия возвращает конфиг без пароля и может вернуть `null`; обёртка возвращает `DbConnectionConfig` с пустым паролем, как ожидает DI2-код). Второй пул не создаётся. +- `cache.ts`, `table-config.ts`, `override-tables.ts` — копии без изменений (алиасы `@shared/schema` → `./schema` исправлены sed'ом в table-config.ts и routes.ts; override-tables/cache и так используют `./db-client`). +- `google-auth.ts` — копия, `getRedirectUri()` переделан: `process.env.APP_BASE_URL || "https://iistwin.ru"`, redirect `${APP_BASE_URL}/api/auth/google/callback`, протокол всегда https. `REPLIT_DOMAINS` удалён. +- `ai-config.ts` — НОВЫЙ файл: чистое IO из DI2 `ai-agent.ts` — `loadAiConfig/saveAiConfig` (data/ai-config.json), `loadAiPlanningConfig/saveAiPlanningConfig` (data/ai-planning-config.json), `getAiLog/saveLogEntry` (data/ai-query-log.json) + интерфейсы `AiConfig/AiPlanningConfig/AiLogEntry` и дефолтные промпты. Без cron, fetch, Telegram. +- `audit-agent.ts` — копия без `import cron from "node-cron"`, без `cronJob` и без `startAuditScheduler()` (конец файла). Оставлены `runIntegrityAudit`, `getAuditHistory`. **node-cron в package.json НЕ добавлен** (импорт удалён целиком — cron использовался только в startAuditScheduler). +- `routes.ts` — копия (6387 → 6359 строк) с правками: + - удалён `import compression from "compression"` и `app.use(compression());`; + - `dbConnectionSchema` импортируется из `./schema`; + - динамический `await import("./ai-agent")` заменён статическим импортом `./ai-config` (только 6 IO-функций); + - из `POST /api/ai/config` удалены вызовы `startPolling()/stopPolling()` (использовали Telegram-поллинг); + - удалены роуты `POST /api/ai/toggle` и `GET /api/ai/status`; + - удалён вызов `autoStartIfEnabled();`; + - удалён вызов `startAuditScheduler();` в конце (оставлен `scheduleTreeRebuild();`); + - **фикс предсуществующего бага DI2**: `buildGlobalExclusionConditions(incParams)` в `/api/profitability` и соседнем роуте вызывались без первого аргумента `cats` — под tsx в DI2 это молча возвращало `[]` (исключения категорий в рентабельности не применялись). Добавлен `loadCategories()` первым аргументом (строки ~2974 и ~3080). Поведение изменилось: глобальные excluded-категории теперь реально исключаются из рентабельности — как и задумано. +- `index.ts` — НОВЫЙ файл-обёртка `registerDi2Routes(app)`: `express.Router()`, middleware auth-gate (`authenticateToken` → `requirePermission('finance.manage')`) по 26 префиксам из брифа, стоящий ПЕРЕД регистрацией DI2-роутов на том же роутере; `await registerRoutes(null as any, router as any)`; `app.use(router)` без mount-префикса. + +### Изменения вне finance-di2 +- `server/finance/db-client.ts` — добавлен экспорт `isConnectionError` (существующий код DI2; новый пул не создавался). +- `server/routes/index.ts` — импорт `registerDi2Routes` + `await registerDi2Routes(app);` строкой НЕПОСРЕДСТВЕННО ПЕРЕД `await registerFinanceRoutes(app);`. +- `server/index.ts` — блок статики `/di2` из `dist/public-di2` (path/fs уже импортированы, `import.meta.dirname` — Node 20.11+, совместимо с esbuild ESM-бандлом) вставлен ПЕРЕД веткой `if (development) setupVite else serveStatic` (после регистрации API-роутов и `app.use('/apps', ...)`). В dev и prod /di2/* не уйдёт в main index.html. Папки dist/public-di2 пока нет — блок молча пропускается (сборку добавит Task 2). +- `package.json` — НЕ изменён (node-cron не нужен, scripts.build не тронут). + +## Проверки +1. `grep -rn "ai-agent\|compression\|REPLIT" server/finance-di2/` → пусто (exit 1). `grep -rn "node-cron" server/finance-di2/` → пусто. +2. `npx tsc --noEmit` → **чисто, без ошибок** (единственные 2 ошибки при первом прогоне — предсуществующий баг DI2 с buildGlobalExclusionConditions — исправлены). +3. `npm run build` → **прошёл**: vite build + esbuild `dist\index.js 2.7mb` (DI2-роуты в бандле: grep `api/profitability|api/refresh-all|api/auth/google/callback` → 9 вхождений). +4. Дев-сервер не запускался (по брифу достаточно typecheck+build). + +## Открытые вопросы / опасения +- **Дублирование путей со старым модулем финансов**: DI2 регистрируется ПЕРЕД `registerFinanceRoutes`, поэтому при совпадении путей (`/api/income`, `/api/connection`, `/api/db/tables`, `/api/audit/*` и др.) побеждают DI2-обработчики. Это осознанное решение брифа, но старые финансовые страницы (которые ходят на эти же пути) теперь получат DI2-формат ответов после деплоя — до появления клиента DI2 в Task 2 старый UI финансов на пересекающихся эндпоинтах может сломаться. Если это нежелательно до Task 2 — можно временно закомментировать регистрацию. +- `/api/auth/google/callback` теперь за auth-gate: Google-редирект без сессии iistwin вернёт 401. В DI2 он тоже требовал входа (Replit auth), поведение осознанное. +- `/api/telegram/*` в routes.ts оставлен как есть (конфиг + отправка через fetch, без поллинга) — бриф вырезал только поллинг из ai-agent. +- Проверка регистрации в рантайме (поднять сервер) не выполнялась — по брифу достаточно typecheck+build; реальный smoke логично сделать после Task 2 вместе с клиентом. diff --git a/server/finance-di2/ai-config.ts b/server/finance-di2/ai-config.ts new file mode 100644 index 0000000..21e43fd --- /dev/null +++ b/server/finance-di2/ai-config.ts @@ -0,0 +1,192 @@ +import fs from "fs"; +import path from "path"; + +// Вырезано из Data-Insight2 (модуль ИИ-агента) — только чистое IO конфигов и лога. +// Telegram-поллинг, cron и сетевые вызовы намеренно не переносятся. + +const AI_CONFIG_PATH = path.join("data", "ai-config.json"); +const AI_LOG_PATH = path.join("data", "ai-query-log.json"); +const MAX_LOG_ENTRIES = 500; + +export interface AiLogEntry { + timestamp: string; + chatId: string; + chatName: string; + question: string; + answer: string; + sqlQueries: string[]; +} + +function loadAiLog(): AiLogEntry[] { + try { + if (fs.existsSync(AI_LOG_PATH)) { + return JSON.parse(fs.readFileSync(AI_LOG_PATH, "utf-8")); + } + } catch {} + return []; +} + +export function saveLogEntry(entry: AiLogEntry): void { + try { + const log = loadAiLog(); + log.push(entry); + if (log.length > MAX_LOG_ENTRIES) { + log.splice(0, log.length - MAX_LOG_ENTRIES); + } + fs.writeFileSync(AI_LOG_PATH, JSON.stringify(log, null, 2)); + } catch (err) { + console.error("[AI Log] Failed to save:", err); + } +} + +export function getAiLog(days: number = 90): AiLogEntry[] { + const log = loadAiLog(); + const cutoff = new Date(); + cutoff.setDate(cutoff.getDate() - days); + return log.filter(e => new Date(e.timestamp) >= cutoff); +} + +export interface AiConfig { + apiUrl: string; + apiKey: string; + model: string; + systemPrompt: string; + enabled: boolean; + name: string; +} + +export interface AiPlanningConfig { + apiUrl: string; + apiKey: string; + model: string; + systemPrompt: string; + name: string; + enabled: boolean; +} + +const DEFAULT_SYSTEM_PROMPT = `Ты — финансовый аналитик компании ОмегаПром. Отвечай на русском языке. У тебя есть доступ к двум источникам данных: + +1. **База данных PostgreSQL** — содержит таблицы доходов (opdohodi), расходов (oprashodi) и дебиторской задолженности (vw_debt_active). Используй инструмент query_database для SQL-запросов (только SELECT). + +2. **Данные зарплат из Google Sheets** — содержат ФИО сотрудников, базовую зарплату и сумму к выдаче по участкам и месяцам. Используй инструмент get_salary_data. + +ВАЖНО — особенности структуры данных: + +### vw_debt_active (дебиторская задолженность): +- Таблица хранит ЕЖЕДНЕВНЫЕ СНИМКИ (snapshots) задолженности. Поле "period" — дата снимка. Каждый period содержит полный срез всех долгов на эту дату. +- КРИТИЧЕСКИ ВАЖНО: для получения АКТУАЛЬНЫХ данных по задолженности ВСЕГДА фильтруй по последнему периоду: WHERE period = (SELECT MAX(period) FROM vw_debt_active) +- БЕЗ фильтра по period суммы будут завышены в сотни раз из-за суммирования по всем историческим снимкам! +- Внутри одного периода может быть несколько строк на одного контрагента (по разным документам/договорам). Для агрегатных запросов используй GROUP BY kontragent и SUM(summadolga::numeric). + +### opdohodi (доходы) и oprashodi (расходы): +- Эти таблицы НЕ имеют снимков — каждая строка уникальна. +- Для суммарных запросов по доходам: GROUP BY zakazchik и SUM(summa::numeric) +- Для суммарных запросов по расходам: GROUP BY и SUM(summadokumenta::numeric) +- Суммы (summadolga, summa, summadokumenta) хранятся как text — для математических операций приводи к numeric: ::numeric +- Даты в БД хранятся в формате timestamp + +Примеры правильных запросов: +- Топ 5 должников: SELECT kontragent, SUM(summadolga::numeric) as total FROM vw_debt_active WHERE period = (SELECT MAX(period) FROM vw_debt_active) GROUP BY kontragent ORDER BY total DESC LIMIT 5 +- Общая дебиторка: SELECT SUM(summadolga::numeric) as total FROM vw_debt_active WHERE period = (SELECT MAX(period) FROM vw_debt_active) +- Просроченная дебиторка: SELECT SUM(summaprosrocheno::numeric) as total FROM vw_debt_active WHERE period = (SELECT MAX(period) FROM vw_debt_active) +- Задолженность по классам просрочки: SELECT klassprosrochki, SUM(summadolga::numeric) as total FROM vw_debt_active WHERE period = (SELECT MAX(period) FROM vw_debt_active) GROUP BY klassprosrochki ORDER BY total DESC +- Доходы по контрагентам: SELECT zakazchik, SUM(summa::numeric) as total FROM opdohodi GROUP BY zakazchik ORDER BY total DESC LIMIT 10 +- Расходы за период: SELECT SUM(summadokumenta::numeric) as total FROM oprashodi WHERE data >= '2025-01-01' AND data < '2025-04-01' + +Правила: +- Генерируй только SELECT запросы, никогда INSERT/UPDATE/DELETE +- ВСЕГДА используй GROUP BY и агрегатные функции для "топ", "сколько всего", "общая сумма" и подобных запросов +- Для vw_debt_active ВСЕГДА добавляй WHERE period = (SELECT MAX(period) FROM vw_debt_active) если нужны актуальные данные +- Суммы в рублях, форматируй с разделителем тысяч +- Если не уверен в структуре данных, сначала сделай запрос для проверки +- Отвечай кратко и по делу`; + +const DEFAULT_CONFIG: AiConfig = { + apiUrl: "https://openrouter.ai/api/v1", + apiKey: "", + model: "deepseek/deepseek-chat", + systemPrompt: DEFAULT_SYSTEM_PROMPT, + enabled: false, + name: "Ассистент", +}; + +export function loadAiConfig(): AiConfig { + try { + if (fs.existsSync(AI_CONFIG_PATH)) { + const raw = JSON.parse(fs.readFileSync(AI_CONFIG_PATH, "utf-8")); + return { ...DEFAULT_CONFIG, ...raw }; + } + } catch {} + return { ...DEFAULT_CONFIG }; +} + +export function saveAiConfig(cfg: AiConfig): void { + fs.writeFileSync(AI_CONFIG_PATH, JSON.stringify(cfg, null, 2)); +} + +const AI_PLANNING_CONFIG_PATH = path.join("data", "ai-planning-config.json"); + +const DEFAULT_PLANNING_PROMPT = `Ты — финансовый аналитик компании ОмегаПром. Проанализируй данные и составь раздельный прогноз доходов и расходов на следующие 12 месяцев. + +ВАЖНО: результат должен быть РАЗДЕЛЬНЫЙ — доходы и расходы анализируются по разным принципам. + +## ДОХОДЫ (type: "income") +Анализируй ТОЛЬКО по контрагентам и договорам: +- Для каждого контрагента+договора определи регулярность поступлений +- Учитывай дебиторскую задолженность: если контрагент имеет большой долг или просрочку — это риск неплатежа, снижай confidence +- Если есть ожидаемые поступления по актам (разница между выставленным и оплаченным) — отметь в reasoning +- В поле "name" укажи контрагента, в "contract" — договор + +## РАСХОДЫ (type: "expense") +Если даны расходы по категориям — анализируй по ним. Если по контрагентам — анализируй по контрагентам. +- Определи гарантированные повторяющиеся расходы (аренда, зарплата, связь, коммуналка и т.д.) +- В поле "name" укажи категорию или контрагента, в "contract" — уточнение если есть + +## Поле recurring +Для каждой позиции укажи "recurring": true если операция повторяется регулярно (ежемесячно или через месяц), false если нерегулярная но устойчивая. + +Верни ТОЛЬКО JSON-массив без markdown-обёртки, без \`\`\`json, просто чистый JSON: +[ + { + "type": "income" или "expense", + "name": "название контрагента или категории", + "contract": "договор или уточнение", + "monthlyAmount": среднемесячная_сумма_число, + "confidence": "high" или "medium" или "low", + "recurring": true или false, + "reasoning": "краткое пояснение на русском почему включил и какой тренд" + } +] + +Правила: +- Включай только операции с confidence medium или high +- Для high: операция повторяется минимум 3 месяца подряд с похожими суммами +- Для medium: операция повторяется 2+ месяца или имеет нерегулярный но устойчивый паттерн +- monthlyAmount — прогнозируемая среднемесячная сумма (число без форматирования) +- Не включай разовые операции +- Сначала все доходы (отсортированные по сумме убывания), потом все расходы (отсортированные по сумме убывания) +- Верни максимум 50 позиций +- Ответ — ТОЛЬКО JSON массив, никакого текста до или после`; + +const DEFAULT_PLANNING_CONFIG: AiPlanningConfig = { + apiUrl: "https://openrouter.ai/api/v1", + apiKey: "", + model: "deepseek/deepseek-chat", + systemPrompt: DEFAULT_PLANNING_PROMPT, + name: "Аналитик", + enabled: true, +}; + +export function loadAiPlanningConfig(): AiPlanningConfig { + try { + if (fs.existsSync(AI_PLANNING_CONFIG_PATH)) { + const raw = JSON.parse(fs.readFileSync(AI_PLANNING_CONFIG_PATH, "utf-8")); + return { ...DEFAULT_PLANNING_CONFIG, ...raw }; + } + } catch {} + return { ...DEFAULT_PLANNING_CONFIG }; +} + +export function saveAiPlanningConfig(cfg: AiPlanningConfig): void { + fs.writeFileSync(AI_PLANNING_CONFIG_PATH, JSON.stringify(cfg, null, 2)); +} diff --git a/server/finance-di2/audit-agent.ts b/server/finance-di2/audit-agent.ts new file mode 100644 index 0000000..0a709b5 --- /dev/null +++ b/server/finance-di2/audit-agent.ts @@ -0,0 +1,450 @@ +import { query } from "./db-client"; +import fs from "fs"; +import path from "path"; + +const CATEGORIES_FILE = path.join(process.cwd(), "data", "income-categories.json"); +const EXPENSE_CATEGORIES_FILE = path.join(process.cwd(), "data", "expense-categories.json"); +const INCOME_AUTO_DETECT_CONFIG_FILE = path.join(process.cwd(), "data", "income-auto-detect-config.json"); +const EXPENSE_AUTO_DETECT_CONFIG_FILE = path.join(process.cwd(), "data", "auto-detect-config.json"); + +interface CategoryRule { + column: string; + operator: string; + value: string; +} + +interface Category { + id: string; + name: string; + rules: CategoryRule[]; + ruleLogic?: "and" | "or"; + children: Category[]; + excluded?: boolean; + autoDetected?: boolean; + treeId?: string; +} + +interface AuditResult { + timestamp: string; + income: DirectionAudit; + expense: DirectionAudit; + status: "ok" | "warning" | "error"; + summary: string; +} + +interface DirectionAudit { + dbTotal: number; + dbCount: number; + treeTotal: number; + treeCount: number; + excludedTotal: number; + excludedCount: number; + uncategorizedTotal: number; + uncategorizedCount: number; + gapTotal: number; + gapCount: number; + gapRows: { columns: Record; sum: number; count: number }[]; + treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[]; +} + +const auditHistory: AuditResult[] = []; +const MAX_HISTORY = 50; + +function loadJSON(filePath: string): any { + try { + return JSON.parse(fs.readFileSync(filePath, "utf-8")); + } catch { + return []; + } +} + +function flattenCategories(cats: Category[]): Category[] { + const result: Category[] = []; + for (const cat of cats) { + result.push(cat); + if (cat.children?.length) result.push(...flattenCategories(cat.children)); + } + return result; +} + +function matchTokens(haystack: string, needle: string): boolean { + const tokens = needle.split(/\s+/).filter(Boolean); + const lower = haystack.toLowerCase(); + return tokens.every(t => lower.includes(t.toLowerCase())); +} + +function rowMatchesRule(row: Record, rule: CategoryRule): boolean { + const val = row[rule.column]; + if (val === undefined) return false; + switch (rule.operator) { + case "equals": return val === rule.value; + case "contains": return typeof val === "string" && matchTokens(val, rule.value); + case "not_equals": return val !== rule.value; + case "greater_than": return Number(val) > Number(rule.value); + case "less_than": return Number(val) < Number(rule.value); + case "equal_number": return Number(val) === Number(rule.value); + default: return val === rule.value; + } +} + +function rowMatchesRules(row: Record, rules: CategoryRule[], logic: "and" | "or" = "and"): boolean { + if (!rules || rules.length === 0) return false; + if (logic === "or") return rules.some(r => rowMatchesRule(row, r)); + return rules.every(r => rowMatchesRule(row, r)); +} + +function collectRuleColumns(cats: Category[]): Set { + const cols = new Set(); + for (const cat of cats) { + for (const rule of (cat.rules || [])) cols.add(rule.column); + if (cat.children?.length) { + for (const col of collectRuleColumns(cat.children)) cols.add(col); + } + } + return cols; +} + +interface AutoDetectConfig { + trees: { id: string; name: string; columns: string[]; isPrimary?: boolean; filters?: any[]; filterLogic?: string }[]; +} + +function rowMatchesAutoPath(row: Record, autoPath: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }): boolean { + for (let i = 0; i < autoPath.ruleSets.length - 1; i++) { + if (!rowMatchesRules(row, autoPath.ruleSets[i])) return false; + } + return rowMatchesRules(row, autoPath.ruleSets[autoPath.ruleSets.length - 1], autoPath.logic); +} + +function computeNodeFromAgg( + cat: Category, + siblings: Category[], + ancestorRuleSets: CategoryRule[][], + ancestorIds: Set, + allFlat: Category[], + autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[], + aggRows: Record[], + exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [] +): { total: number; count: number; excluded: boolean; children: any[] } { + const catLogic = cat.ruleLogic || "and"; + const idx = siblings.indexOf(cat); + const laterSibs = siblings.slice(idx + 1); + + const allRuleSets = [...ancestorRuleSets, cat.rules]; + + let total = 0; + let count = 0; + for (const row of aggRows) { + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; + if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue; + if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue; + if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) continue; + if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue; + total += row.total; + count += row.cnt; + } + + const children: any[] = []; + if (cat.children?.length) { + const childAncestorIds = new Set(ancestorIds); + childAncestorIds.add(cat.id); + for (const child of cat.children) { + children.push(computeNodeFromAgg(child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, aggRows, exclPaths)); + } + } + + return { total, count, excluded: !!cat.excluded, children }; +} + +function nodeSubtreeTotal(n: { total: number; excluded: boolean; children: any[] }): number { + if (n.excluded) { + let t = 0; + for (const ch of n.children) t += nodeSubtreeTotal(ch); + return t; + } + let t = n.total; + for (const ch of n.children) t += nodeSubtreeTotal(ch); + return t; +} + +function nodeSubtreeCount(n: { count: number; excluded: boolean; children: any[] }): number { + if (n.excluded) { + let c = 0; + for (const ch of n.children) c += nodeSubtreeCount(ch); + return c; + } + let c = n.count; + for (const ch of n.children) c += nodeSubtreeCount(ch); + return c; +} + +function nodeExcludedTotal(n: { total: number; excluded: boolean; children: any[] }): number { + if (n.excluded) { + let t = n.total; + for (const ch of n.children) t += nodeExcludedTotal(ch); + return t; + } + let t = 0; + for (const ch of n.children) t += nodeExcludedTotal(ch); + return t; +} + +function nodeExcludedCount(n: { count: number; excluded: boolean; children: any[] }): number { + if (n.excluded) { + let c = n.count; + for (const ch of n.children) c += nodeExcludedCount(ch); + return c; + } + let c = 0; + for (const ch of n.children) c += nodeExcludedCount(ch); + return c; +} + +async function auditDirection( + table: string, + sumColumn: string, + categoriesFile: string, + autoDetectConfigFile: string, + allowedColumns: string[] +): Promise { + const cats: Category[] = loadJSON(categoriesFile); + const autoConfigRaw = loadJSON(autoDetectConfigFile); + const autoConfig: AutoDetectConfig = { trees: autoConfigRaw?.trees || [] }; + const allFlat = flattenCategories(cats); + + const ruleColumns = collectRuleColumns(cats); + for (const tree of autoConfig.trees) { + for (const col of tree.columns) ruleColumns.add(col); + } + const groupCols = ruleColumns.size > 0 + ? [...ruleColumns].filter(c => allowedColumns.includes(c)) + : allowedColumns.slice(0, 4); + + const selectCols = groupCols.join(", "); + const aggResult = await query( + `SELECT ${selectCols}, SUM(${sumColumn}::numeric) as total, COUNT(*) as cnt FROM ${table} GROUP BY ${selectCols}`, + [] + ); + + const aggRows = aggResult.rows.map((r: any) => { + const row: Record = { total: Number(r.total), cnt: Number(r.cnt) }; + for (const col of groupCols) row[col] = r[col] ?? ""; + return row; + }); + + let dbTotal = 0, dbCount = 0; + for (const row of aggRows) { dbTotal += row.total; dbCount += row.cnt; } + + const primaryTreeIds = new Set(autoConfig.trees.filter(t => t.isPrimary).map(t => t.id)); + const configTreeIds = new Set(autoConfig.trees.map(t => t.id)); + + const groupedByTree = new Map(); + for (const cat of cats) { + const tid = cat.treeId || "__manual__"; + if (!groupedByTree.has(tid)) groupedByTree.set(tid, []); + groupedByTree.get(tid)!.push(cat); + } + + const autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []; + const excludedPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = []; + function collectAllPaths(categories: Category[], ancestors: CategoryRule[][]) { + for (const cat of categories) { + const path = [...ancestors, cat.rules || []]; + if (cat.excluded) { + excludedPaths.push({ id: cat.id, ruleSets: path, logic: (cat.ruleLogic || "and") }); + } else if (cat.autoDetected && cat.rules?.length) { + autoPaths.push({ ruleSets: path, logic: (cat.ruleLogic || "and") }); + } + if (cat.children?.length) { + collectAllPaths(cat.children, path); + } + } + } + collectAllPaths(cats, []); + + const treeBreakdown: { name: string; total: number; count: number; isPrimary: boolean }[] = []; + let allTreeTotal = 0, allTreeCount = 0; + let allExclTotal = 0, allExclCount = 0; + + for (const [tid, treeCats] of groupedByTree) { + const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); + const treeName = tid === "__manual__" + ? "Ручные категории" + : autoConfig.trees.find(t => t.id === tid)?.name || tid; + + const nodes: any[] = []; + for (const cat of treeCats) { + nodes.push(computeNodeFromAgg(cat, treeCats, [], new Set(), allFlat, autoPaths, aggRows, excludedPaths)); + } + + const tTotal = nodes.reduce((s, n) => s + nodeSubtreeTotal(n), 0); + const tCount = nodes.reduce((s, n) => s + nodeSubtreeCount(n), 0); + const eTotal = nodes.reduce((s, n) => s + nodeExcludedTotal(n), 0); + const eCount = nodes.reduce((s, n) => s + nodeExcludedCount(n), 0); + + treeBreakdown.push({ name: treeName, total: tTotal, count: tCount, isPrimary }); + + if (isPrimary) { + allTreeTotal += tTotal; + allTreeCount += tCount; + allExclTotal += eTotal; + allExclCount += eCount; + } + } + + function isRowMatchedByTreeAudit(row: Record, treeCats: Category[]): boolean { + for (const cat of treeCats) { + const catLogic = cat.ruleLogic || "and"; + if (!cat.rules?.length || !rowMatchesRules(row, cat.rules, catLogic)) continue; + return true; + } + return false; + } + + let uncatTotal = 0, uncatCount = 0; + const gapRows: { columns: Record; sum: number; count: number }[] = []; + + for (const row of aggRows) { + let matched = false; + for (const [tid, treeCats] of groupedByTree) { + const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); + if (!isPrimary) continue; + if (isRowMatchedByTreeAudit(row, treeCats)) { matched = true; break; } + } + if (!matched) { + uncatTotal += row.total; + uncatCount += row.cnt; + } + } + + const accountedTotal = allTreeTotal + allExclTotal + uncatTotal; + const accountedCount = allTreeCount + allExclCount + uncatCount; + const gapTotal = dbTotal - accountedTotal; + const gapCount = dbCount - accountedCount; + + if (Math.abs(gapCount) > 0) { + for (const row of aggRows) { + let inTree = false; + let inUncat = true; + for (const [tid, treeCats] of groupedByTree) { + const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); + if (!isPrimary) continue; + if (isRowMatchedByTreeAudit(row, treeCats)) { inUncat = false; break; } + } + if (!inUncat) { + for (const [tid, treeCats] of groupedByTree) { + const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); + if (!isPrimary) continue; + for (const cat of treeCats) { + if (isRowCapturedByNode(row, cat, treeCats, [], new Set(), allFlat, autoPaths, excludedPaths)) { + inTree = true; break; + } + } + if (inTree) break; + } + } + if (!inUncat && !inTree) { + const cols: Record = {}; + for (const col of groupCols) cols[col] = String(row[col] || ""); + gapRows.push({ columns: cols, sum: row.total, count: row.cnt }); + } + } + } + + return { + dbTotal, dbCount, + treeTotal: allTreeTotal, treeCount: allTreeCount, + excludedTotal: allExclTotal, excludedCount: allExclCount, + uncategorizedTotal: uncatTotal, uncategorizedCount: uncatCount, + gapTotal: Math.round(gapTotal * 100) / 100, + gapCount, + gapRows: gapRows.slice(0, 50), + treeBreakdown, + }; +} + +function isRowCapturedByNode( + row: Record, + cat: Category, + siblings: Category[], + ancestorRuleSets: CategoryRule[][], + ancestorIds: Set, + allFlat: Category[], + autoPaths: { ruleSets: CategoryRule[][]; logic: ("and" | "or") }[], + exclPaths: { id: string; ruleSets: CategoryRule[][]; logic: ("and" | "or") }[] = [] +): boolean { + const catLogic = cat.ruleLogic || "and"; + const idx = siblings.indexOf(cat); + const laterSibs = siblings.slice(idx + 1); + + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) return false; + if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) return false; + if (!cat.autoDetected && exclPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) return false; + if (!cat.autoDetected && autoPaths.some(ap => rowMatchesAutoPath(row, ap))) return false; + + if (cat.children?.length && cat.children.some(child => (child.rules?.length || 0) > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) { + const allRuleSets = [...ancestorRuleSets, cat.rules]; + const childAncestorIds = new Set(ancestorIds); + childAncestorIds.add(cat.id); + for (const child of cat.children) { + if (isRowCapturedByNode(row, child, cat.children, allRuleSets, childAncestorIds, allFlat, autoPaths, exclPaths)) return true; + } + return false; + } + + return true; +} + +const INCOME_COLUMNS = [ + "statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands", + "naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta", + "dogovorzakazchika", "statyadop", "schetnaoplatu", "nomervhodyaschegodokumenta", +]; + +const EXPENSE_COLUMNS = [ + "statya", "dochka", "vnuchka", "zakazchik", "naznachenieplatezha", + "statyadop", "kommentarii", "otvetstvenniipokontraktu", "dogovorzakazchika", + "dogovorkontragenta", +]; + +export async function runIntegrityAudit(): Promise { + console.log("[AUDIT-AGENT] Запуск аудита целостности данных..."); + + const [income, expense] = await Promise.all([ + auditDirection("opdohodi", "summa", CATEGORIES_FILE, INCOME_AUTO_DETECT_CONFIG_FILE, INCOME_COLUMNS), + auditDirection("oprashodi", "summadokumenta", EXPENSE_CATEGORIES_FILE, EXPENSE_AUTO_DETECT_CONFIG_FILE, EXPENSE_COLUMNS), + ]); + + const hasGap = income.gapCount !== 0 || expense.gapCount !== 0; + const status = hasGap ? "warning" : "ok"; + + const summaryParts: string[] = []; + if (income.gapCount === 0 && expense.gapCount === 0) { + summaryParts.push("Все строки распределены корректно"); + } else { + if (income.gapCount !== 0) summaryParts.push(`Доходы: ${income.gapCount} строк (${income.gapTotal.toLocaleString("ru")}₽) не распределены`); + if (expense.gapCount !== 0) summaryParts.push(`Расходы: ${expense.gapCount} строк (${expense.gapTotal.toLocaleString("ru")}₽) не распределены`); + } + + const result: AuditResult = { + timestamp: new Date().toISOString(), + income, + expense, + status, + summary: summaryParts.join(". "), + }; + + auditHistory.unshift(result); + if (auditHistory.length > MAX_HISTORY) auditHistory.pop(); + + console.log(`[AUDIT-AGENT] Аудит завершён: ${result.summary}`); + console.log(`[AUDIT-AGENT] Доходы: БД=${income.dbCount} строк, деревья=${income.treeCount}, excluded=${income.excludedCount}, нераспр=${income.uncategorizedCount}, gap=${income.gapCount}`); + console.log(`[AUDIT-AGENT] Расходы: БД=${expense.dbCount} строк, деревья=${expense.treeCount}, excluded=${expense.excludedCount}, нераспр=${expense.uncategorizedCount}, gap=${expense.gapCount}`); + + return result; +} + +export function getAuditHistory(): AuditResult[] { + return auditHistory; +} diff --git a/server/finance-di2/cache.ts b/server/finance-di2/cache.ts new file mode 100644 index 0000000..be73255 --- /dev/null +++ b/server/finance-di2/cache.ts @@ -0,0 +1,69 @@ +interface CacheEntry { + data: unknown; + expiresAt: number; +} + +const store = new Map(); +const inflight = new Map>(); + +export function cacheGet(key: string): unknown | undefined { + const entry = store.get(key); + if (!entry) return undefined; + if (Date.now() > entry.expiresAt) { + store.delete(key); + return undefined; + } + return entry.data; +} + +export function cacheSet(key: string, data: unknown, ttlMs: number): void { + store.set(key, { data, expiresAt: Date.now() + ttlMs }); +} + +export function cacheInvalidateAll(): void { + store.clear(); + inflight.clear(); +} + +export function cacheInvalidateByPrefix(prefix: string): void { + for (const key of store.keys()) { + if (key.startsWith(prefix)) store.delete(key); + } + for (const key of inflight.keys()) { + if (key.startsWith(prefix)) inflight.delete(key); + } +} + +export function cacheInvalidateKey(key: string): void { + store.delete(key); + inflight.delete(key); +} + +export async function cachedQuery(key: string, ttlMs: number, fn: () => Promise): Promise { + const cached = cacheGet(key); + if (cached !== undefined) return cached as T; + + const pending = inflight.get(key); + if (pending) return pending as Promise; + + const promise = fn().then(result => { + cacheSet(key, result, ttlMs); + inflight.delete(key); + return result; + }).catch(err => { + inflight.delete(key); + throw err; + }); + + inflight.set(key, promise); + return promise as Promise; +} + +export function stableKey(prefix: string, query: Record): string { + const sorted = Object.keys(query).sort().map(k => `${k}=${query[k]}`).join("&"); + return `${prefix}:${sorted}`; +} + +export function cacheStats(): { size: number; keys: string[] } { + return { size: store.size, keys: Array.from(store.keys()) }; +} diff --git a/server/finance-di2/db-client.ts b/server/finance-di2/db-client.ts new file mode 100644 index 0000000..60ae3af --- /dev/null +++ b/server/finance-di2/db-client.ts @@ -0,0 +1,24 @@ +// Обёртка над существующим пулом финансовой БД iistwin (server/finance/db-client.ts). +// DI2-код импортирует эти имена из "./db-client" — второй пул не создаётся. + +import type { DbConnectionConfig } from "./schema"; +import { + query, + testConnection, + getConfig as financeGetConfig, + setConfig, + isConnectionError, + getPoolClient, +} from "../finance/db-client"; + +export { query, testConnection, setConfig, isConnectionError, getPoolClient }; + +// В DI2 getConfig() всегда возвращал конфиг (не null) и включал password. +// iistwin-версия отдаёт конфиг без пароля и может вернуть null — приводим к DI2-совместимой форме. +export function getConfig(): DbConnectionConfig { + const cfg = financeGetConfig(); + if (!cfg) { + return { host: "", port: 5432, database: "", user: "", password: "" }; + } + return { ...cfg, password: "" } as DbConnectionConfig; +} diff --git a/server/finance-di2/google-auth.ts b/server/finance-di2/google-auth.ts new file mode 100644 index 0000000..e95f30c --- /dev/null +++ b/server/finance-di2/google-auth.ts @@ -0,0 +1,155 @@ +import { google } from "googleapis"; +import fs from "fs"; +import path from "path"; + +const TOKENS_PATH = path.join(process.cwd(), "data", "google-tokens.json"); +const SCOPES = [ + "https://www.googleapis.com/auth/spreadsheets.readonly", + "https://www.googleapis.com/auth/userinfo.email", +]; + +function getRedirectUri(): string { + // В iistwin redirect всегда на основной домен (https) + const base = process.env.APP_BASE_URL || "https://iistwin.ru"; + return `${base}/api/auth/google/callback`; +} + +function createOAuth2Client() { + return new google.auth.OAuth2( + process.env.GOOGLE_CLIENT_ID, + process.env.GOOGLE_CLIENT_SECRET, + getRedirectUri() + ); +} + +interface StoredTokens { + access_token: string; + refresh_token?: string; + expiry_date?: number; + token_type?: string; + scope?: string; + email?: string; +} + +function loadTokens(): StoredTokens | null { + try { + if (fs.existsSync(TOKENS_PATH)) { + return JSON.parse(fs.readFileSync(TOKENS_PATH, "utf-8")); + } + } catch {} + return null; +} + +function saveTokens(tokens: StoredTokens): void { + const dir = path.dirname(TOKENS_PATH); + if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(TOKENS_PATH, JSON.stringify(tokens, null, 2)); +} + +function clearTokens(): void { + try { + if (fs.existsSync(TOKENS_PATH)) fs.unlinkSync(TOKENS_PATH); + } catch {} +} + +export function getAuthUrl(): string { + const client = createOAuth2Client(); + return client.generateAuthUrl({ + access_type: "offline", + scope: SCOPES, + prompt: "consent", + }); +} + +export async function handleCallback(code: string): Promise<{ email?: string }> { + const client = createOAuth2Client(); + const { tokens } = await client.getToken(code); + + client.setCredentials(tokens); + let email: string | undefined; + try { + const oauth2 = google.oauth2({ version: "v2", auth: client }); + const userInfo = await oauth2.userinfo.get(); + email = userInfo.data.email || undefined; + } catch {} + + saveTokens({ + access_token: tokens.access_token!, + refresh_token: tokens.refresh_token || undefined, + expiry_date: tokens.expiry_date || undefined, + token_type: tokens.token_type || undefined, + scope: tokens.scope || undefined, + email, + }); + + return { email }; +} + +export function getAuthStatus(): { loggedIn: boolean; email?: string } { + const tokens = loadTokens(); + if (!tokens?.access_token) return { loggedIn: false }; + return { loggedIn: true, email: tokens.email }; +} + +export function logout(): void { + clearTokens(); +} + +export async function getAuthenticatedClient() { + const tokens = loadTokens(); + if (!tokens?.access_token) throw new Error("Не авторизован в Google"); + + const client = createOAuth2Client(); + client.setCredentials({ + access_token: tokens.access_token, + refresh_token: tokens.refresh_token, + expiry_date: tokens.expiry_date, + }); + + const needsRefresh = tokens.expiry_date + ? tokens.expiry_date < Date.now() + 60000 + : false; + + if (needsRefresh && tokens.refresh_token) { + try { + const { credentials } = await client.refreshAccessToken(); + client.setCredentials(credentials); + saveTokens({ + ...tokens, + access_token: credentials.access_token!, + expiry_date: credentials.expiry_date || undefined, + }); + } catch (err) { + clearTokens(); + throw new Error("Токен Google истёк. Пожалуйста, войдите заново."); + } + } + + client.on("tokens", (newTokens) => { + const current = loadTokens(); + if (current && newTokens.access_token) { + saveTokens({ + ...current, + access_token: newTokens.access_token, + expiry_date: newTokens.expiry_date || current.expiry_date, + refresh_token: newTokens.refresh_token || current.refresh_token, + }); + } + }); + + return client; +} + +export async function getSheetNames(spreadsheetId: string): Promise { + const auth = await getAuthenticatedClient(); + const sheets = google.sheets({ version: "v4", auth }); + const res = await sheets.spreadsheets.get({ spreadsheetId, fields: "sheets.properties.title" }); + return (res.data.sheets || []).map(s => s.properties?.title || "").filter(Boolean); +} + +export async function getSheetData(spreadsheetId: string, range: string): Promise { + const auth = await getAuthenticatedClient(); + const sheets = google.sheets({ version: "v4", auth }); + const res = await sheets.spreadsheets.values.get({ spreadsheetId, range }); + return res.data.values || []; +} diff --git a/server/finance-di2/index.ts b/server/finance-di2/index.ts new file mode 100644 index 0000000..699e15c --- /dev/null +++ b/server/finance-di2/index.ts @@ -0,0 +1,55 @@ +// Обёртка регистрации роутов Data-Insight2 (финансы v2) в iistwin. +// Все DI2-роуты проходят через auth-gate: JWT + право finance.manage. + +import type { Express } from "express"; +import express from "express"; +import { authenticateToken, requirePermission } from "../middleware/auth.middleware"; +import { registerRoutes } from "./routes"; + +const DI2_PREFIXES = [ + "/api/income", + "/api/income-categories", + "/api/expenses", + "/api/expense-categories", + "/api/debt", + "/api/dashboard", + "/api/profitability", + "/api/heatmap", + "/api/planning", + "/api/salary", + "/api/audit", + "/api/calculations", + "/api/connection", + "/api/db", + "/api/table-config", + "/api/table-settings", + "/api/overrides", + "/api/tags", + "/api/tag-assignments", + "/api/cache", + "/api/refresh-all", + "/api/admin", + "/api/auth/google", + "/api/ai", + "/api/telegram", +]; + +export async function registerDi2Routes(app: Express): Promise { + const router = express.Router(); + + // Auth-gate: DI2-API только для авторизованных с правом finance.manage. + // Стоит ПЕРЕД регистрацией роутов DI2 на этом же роутере. + router.use((req, res, next) => { + if (!DI2_PREFIXES.some(p => req.path.startsWith(p))) { + return next(); + } + authenticateToken(req, res, (err?: unknown) => { + if (err) return next(err); + requirePermission("finance.manage")(req, res, next); + }); + }); + + await registerRoutes(null as any, router as any); + + app.use(router); +} diff --git a/server/finance-di2/override-tables.ts b/server/finance-di2/override-tables.ts new file mode 100644 index 0000000..8b8800a --- /dev/null +++ b/server/finance-di2/override-tables.ts @@ -0,0 +1,232 @@ +import { query } from "./db-client"; +import { loadTableConfig } from "./table-config"; +import fs from "fs"; +import path from "path"; + +export const INCOME_OVERRIDE_COLS = [ + "zakazchik", "dogovorzakazchika", "otvetstvenniipokontraktu", + "statya", "dochka", "vnuchka", "naznachenieplatezha", + "kontragent", "dogovorkontragenta", "stavkands", + "kommentarii", "statyadop", "schetnaoplatu", "nomervhodyaschegodokumenta", +]; + +export const EXPENSE_OVERRIDE_COLS = [ + "zakazchik", "dogovorzakazchika", "otvetstvenniipokontraktu", + "statya", "dochka", "vnuchka", "naznachenieplatezha", + "kontragent", "dogovorkontragenta", "stavkands", + "kommentarii", "statyadop", "ssilkanomervhodyaschegodokumenta", +]; + +async function createOverrideTables(): Promise { + await query(`CREATE TABLE IF NOT EXISTS opdohodi_overrides ( + id TEXT PRIMARY KEY, + category_id TEXT, + zakazchik TEXT, dogovorzakazchika TEXT, otvetstvenniipokontraktu TEXT, + statya TEXT, dochka TEXT, vnuchka TEXT, naznachenieplatezha TEXT, + kontragent TEXT, dogovorkontragenta TEXT, stavkands TEXT, + kommentarii TEXT, statyadop TEXT, schetnaoplatu TEXT, + nomervhodyaschegodokumenta TEXT, + created_at TIMESTAMP DEFAULT NOW() + )`); + + await query(`CREATE TABLE IF NOT EXISTS oprashodi_overrides ( + id TEXT PRIMARY KEY, + category_id TEXT, + zakazchik TEXT, dogovorzakazchika TEXT, otvetstvenniipokontraktu TEXT, + statya TEXT, dochka TEXT, vnuchka TEXT, naznachenieplatezha TEXT, + kontragent TEXT, dogovorkontragenta TEXT, stavkands TEXT, + kommentarii TEXT, statyadop TEXT, ssilkanomervhodyaschegodokumenta TEXT, + created_at TIMESTAMP DEFAULT NOW() + )`); +} + +async function createEffectiveView( + sourceTable: string, + overrideTable: string, + viewName: string, + overrideCols: string[] +): Promise { + const schemaResult = await query( + `SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1 + ORDER BY ordinal_position`, + [sourceTable] + ); + + const allCols: string[] = schemaResult.rows.map((r: any) => r.column_name); + if (allCols.length === 0) { + console.warn(`[override-tables] Table ${sourceTable} not found in schema, skipping view ${viewName}`); + return; + } + + const overrideColSet = new Set(overrideCols); + const selectExprs = allCols.map(col => { + if (overrideColSet.has(col)) { + return `COALESCE(ov."${col}", o."${col}") AS "${col}"`; + } + return `o."${col}"`; + }); + selectExprs.push(`ov.category_id AS "_override_category_id"`); + + const viewSQL = ` + CREATE OR REPLACE VIEW "${viewName}" AS + SELECT ${selectExprs.join(",\n ")} + FROM "${sourceTable}" o + LEFT JOIN "${overrideTable}" ov ON o.id = ov.id + `; + + await query(viewSQL); + console.log(`[override-tables] View ${viewName} created/updated`); +} + +export async function initOverrideTables(): Promise { + try { + const cfg = loadTableConfig(); + await createOverrideTables(); + await Promise.all([ + createEffectiveView(cfg.incomeTable, "opdohodi_overrides", "vw_income_effective", INCOME_OVERRIDE_COLS), + createEffectiveView(cfg.expenseTable, "oprashodi_overrides", "vw_expense_effective", EXPENSE_OVERRIDE_COLS), + ]); + console.log("[override-tables] Initialization complete"); + } catch (err) { + console.error("[override-tables] Init failed:", err); + } +} + +export async function recreateEffectiveViews(): Promise { + const cfg = loadTableConfig(); + await Promise.all([ + createEffectiveView(cfg.incomeTable, "opdohodi_overrides", "vw_income_effective", INCOME_OVERRIDE_COLS), + createEffectiveView(cfg.expenseTable, "oprashodi_overrides", "vw_expense_effective", EXPENSE_OVERRIDE_COLS), + ]); +} + +export async function saveIncomeOverride( + recordId: string, + colValues: Record, + categoryId: string +): Promise { + const params: unknown[] = [recordId, categoryId]; + for (const col of INCOME_OVERRIDE_COLS) { + params.push(colValues[col] ?? null); + } + const colDefs = `"id", "category_id", ${INCOME_OVERRIDE_COLS.map(c => `"${c}"`).join(", ")}`; + const valuePhs = params.map((_, i) => `$${i + 1}`).join(", "); + const updateSets = [`"category_id" = $2`, ...INCOME_OVERRIDE_COLS.map((col, i) => `"${col}" = $${i + 3}`)].join(", "); + await query( + `INSERT INTO opdohodi_overrides (${colDefs}) VALUES (${valuePhs}) + ON CONFLICT (id) DO UPDATE SET ${updateSets}`, + params + ); +} + +export async function saveExpenseOverride( + recordId: string, + colValues: Record, + categoryId: string +): Promise { + const params: unknown[] = [recordId, categoryId]; + for (const col of EXPENSE_OVERRIDE_COLS) { + params.push(colValues[col] ?? null); + } + const colDefs = `"id", "category_id", ${EXPENSE_OVERRIDE_COLS.map(c => `"${c}"`).join(", ")}`; + const valuePhs = params.map((_, i) => `$${i + 1}`).join(", "); + const updateSets = [`"category_id" = $2`, ...EXPENSE_OVERRIDE_COLS.map((col, i) => `"${col}" = $${i + 3}`)].join(", "); + await query( + `INSERT INTO oprashodi_overrides (${colDefs}) VALUES (${valuePhs}) + ON CONFLICT (id) DO UPDATE SET ${updateSets}`, + params + ); +} + +export async function deleteIncomeOverride(recordId: string): Promise { + await query(`DELETE FROM opdohodi_overrides WHERE id = $1`, [recordId]); +} + +export async function deleteExpenseOverride(recordId: string): Promise { + await query(`DELETE FROM oprashodi_overrides WHERE id = $1`, [recordId]); +} + +export async function getAllIncomeOverrides(): Promise> { + const result = await query(`SELECT id, category_id FROM opdohodi_overrides WHERE category_id IS NOT NULL`); + return result.rows.map((r: any) => ({ id: String(r.id), categoryId: String(r.category_id) })); +} + +export async function getAllExpenseOverrides(): Promise> { + const result = await query(`SELECT id, category_id FROM oprashodi_overrides WHERE category_id IS NOT NULL`); + return result.rows.map((r: any) => ({ id: String(r.id), categoryId: String(r.category_id) })); +} + +export function extractColumnValuesFromCategoryPath( + path: Array<{ rules: Array<{ column: string; operator: string; value: string }> }>, + allowedCols: string[] +): Record { + const allowed = new Set(allowedCols); + const result: Record = {}; + for (const cat of path) { + for (const rule of cat.rules) { + if (rule.operator === "equals" && allowed.has(rule.column)) { + result[rule.column] = rule.value; + } + } + } + return result; +} + +export async function migrateJsonOverrides( + jsonFilePath: string, + findIncomeCategory: (id: string) => { cat: any; path: any[] } | null, + findExpenseCategory: (id: string) => { cat: any; path: any[] } | null +): Promise { + if (!fs.existsSync(jsonFilePath)) return; + + const migratedPath = jsonFilePath + ".migrated"; + if (fs.existsSync(migratedPath)) return; + + try { + const incomeCount = await query(`SELECT COUNT(*) as cnt FROM opdohodi_overrides`); + const expenseCount = await query(`SELECT COUNT(*) as cnt FROM oprashodi_overrides`); + if (Number(incomeCount.rows[0].cnt) > 0 || Number(expenseCount.rows[0].cnt) > 0) { + console.log("[override-tables] Override tables already have data, skipping migration"); + fs.renameSync(jsonFilePath, migratedPath); + return; + } + + const raw = fs.readFileSync(jsonFilePath, "utf-8"); + const parsed = JSON.parse(raw); + const income: Record = parsed.income || {}; + const expense: Record = parsed.expense || {}; + + let migratedIncome = 0; + let migratedExpense = 0; + + for (const [recordId, categoryId] of Object.entries(income)) { + try { + const found = findIncomeCategory(categoryId); + if (!found) continue; + const colValues = extractColumnValuesFromCategoryPath(found.path, INCOME_OVERRIDE_COLS); + await saveIncomeOverride(recordId, colValues, categoryId); + migratedIncome++; + } catch (err) { + console.warn(`[override-tables] Migration failed for income record ${recordId}:`, err); + } + } + + for (const [recordId, categoryId] of Object.entries(expense)) { + try { + const found = findExpenseCategory(categoryId); + if (!found) continue; + const colValues = extractColumnValuesFromCategoryPath(found.path, EXPENSE_OVERRIDE_COLS); + await saveExpenseOverride(recordId, colValues, categoryId); + migratedExpense++; + } catch (err) { + console.warn(`[override-tables] Migration failed for expense record ${recordId}:`, err); + } + } + + fs.renameSync(jsonFilePath, migratedPath); + console.log(`[override-tables] Migration complete: ${migratedIncome} income, ${migratedExpense} expense overrides migrated`); + } catch (err) { + console.error("[override-tables] Migration error:", err); + } +} diff --git a/server/finance-di2/routes.ts b/server/finance-di2/routes.ts new file mode 100644 index 0000000..f5fb00a --- /dev/null +++ b/server/finance-di2/routes.ts @@ -0,0 +1,6359 @@ +import type { Express } from "express"; +import { createServer, type Server } from "http"; +import { randomUUID } from "crypto"; +import { query, testConnection, getConfig, setConfig, isConnectionError, getPoolClient } from "./db-client"; +import { dbConnectionSchema } from "./schema"; +import { cachedQuery, cacheInvalidateAll, cacheInvalidateByPrefix, cacheInvalidateKey, stableKey } from "./cache"; +import fs from "fs"; +import path from "path"; +import { loadTableConfig, saveTableConfig, getIncomeTable, getExpenseTable, getIncomeEffectiveView, getExpenseEffectiveView, getDebtTable, getAllowedIncomeColumns, getAllowedExpenseColumns, getAllIncomeColumns, getAllExpenseColumns, getAutoDetectAllowedColumns, invalidateTableConfigCache } from "./table-config"; +import { initOverrideTables, recreateEffectiveViews, saveIncomeOverride, saveExpenseOverride, deleteIncomeOverride, deleteExpenseOverride, getAllIncomeOverrides, getAllExpenseOverrides, extractColumnValuesFromCategoryPath, INCOME_OVERRIDE_COLS, EXPENSE_OVERRIDE_COLS, migrateJsonOverrides } from "./override-tables"; + +import type { Response } from "express"; +import { getAuthUrl, handleCallback, getAuthStatus, logout, getSheetNames, getSheetData } from "./google-auth"; +import { runIntegrityAudit, getAuditHistory } from "./audit-agent"; +import { loadAiConfig, saveAiConfig, loadAiPlanningConfig, saveAiPlanningConfig, getAiLog, saveLogEntry } from "./ai-config"; + +function handleApiError(res: Response, err: unknown, fallbackMessage = "Ошибка базы данных") { + console.error(err); + if (isConnectionError(err)) { + return res.status(503).json({ error: "База данных не подключена", connectionError: true }); + } + return res.status(500).json({ error: fallbackMessage }); +} + +const TTL_SHORT = 2 * 60 * 1000; +const TTL_LONG = 5 * 60 * 1000; + +function buildTokenILIKE(colExpr: string, value: string, params: unknown[]): string { + const tokens = value.split(/\s+/).filter(Boolean); + if (tokens.length === 0) return "TRUE"; + const parts = tokens.map(t => { + params.push(`%${t}%`); + return `${colExpr} ILIKE $${params.length}`; + }); + return parts.length === 1 ? parts[0] : `(${parts.join(" AND ")})`; +} + +function matchTokens(text: string, value: string): boolean { + const lower = text.toLowerCase(); + return value.split(/\s+/).filter(Boolean).every(t => lower.includes(t.toLowerCase())); +} +const TTL_MAX_PERIOD = 3 * 60 * 1000; + +async function getMaxPeriod(): Promise { + return cachedQuery("__max_period__", TTL_MAX_PERIOD, async () => { + const res = await query(`SELECT MAX(period) as mp FROM ${getDebtTable()}`); + return res.rows[0]?.mp ? new Date(res.rows[0].mp).toISOString() : new Date().toISOString(); + }); +} + +function invalidateServerCache() { + cacheInvalidateAll(); + invalidateCategoryMemCache(); +} + +function parseDate(d: unknown): string | null { + if (!d) return null; + return new Date(d as string).toISOString(); +} + +const ALLOWED_INCOME_COLUMNS_FALLBACK = ["data", "statya", "zakazchik", "dochka", "vnuchka", "otvetstvenniipokontraktu", "stavkands", "kommentarii", "dogovorkontragenta", "schetnaoplatu", "naznachenieplatezha", "dogovorzakazchika", "statyadop", "nomervhodyaschegodokumenta", "summa"]; + +const ALLOWED_EXPENSE_COLUMNS_FALLBACK = ["data", "summadokumenta", "statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands", "naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta", "dogovorzakazchika", "statyadop", "ssilkanomervhodyaschegodokumenta", "kontragent"]; + +function ALLOWED_INCOME_COLUMNS(): string[] { + try { return getAllowedIncomeColumns(); } catch { return ALLOWED_INCOME_COLUMNS_FALLBACK; } +} + +function ALLOWED_EXPENSE_COLUMNS(): string[] { + try { return getAllowedExpenseColumns(); } catch { return ALLOWED_EXPENSE_COLUMNS_FALLBACK; } +} + +function ALL_EXPENSE_COLS(): string[] { + try { return getAllExpenseColumns(); } catch { return ["id", ...ALLOWED_EXPENSE_COLUMNS_FALLBACK, "dataobnovleniyazapisi"]; } +} + +interface IncomeCategory { + id: string; + name: string; + rules: { column: string; operator: string; value: string }[]; + ruleLogic?: "and" | "or"; + excluded?: boolean; + children?: IncomeCategory[]; + autoDetected?: boolean; + treeId?: string; +} + +const CATEGORIES_FILE = path.join(process.cwd(), "data", "income-categories.json"); +const EXPENSE_CATEGORIES_FILE = path.join(process.cwd(), "data", "expense-categories.json"); +const AUTO_DETECT_CONFIG_FILE = path.join(process.cwd(), "data", "auto-detect-config.json"); +const INCOME_AUTO_DETECT_CONFIG_FILE = path.join(process.cwd(), "data", "income-auto-detect-config.json"); +const CATEGORY_OVERRIDES_FILE = path.join(process.cwd(), "data", "category-overrides.json"); + +let _incomeCategoriesCache: IncomeCategory[] | null = null; +let _expenseCategoriesCache: IncomeCategory[] | null = null; +let _autoDetectConfigCache: AutoDetectConfig | null = null; +let _incomeAutoDetectConfigCache: AutoDetectConfig | null = null; + +function invalidateCategoryMemCache() { + _incomeCategoriesCache = null; + _expenseCategoriesCache = null; + _autoDetectConfigCache = null; + _incomeAutoDetectConfigCache = null; +} + +function getExpenseAutoDetectColumns(): string[] { + return getAutoDetectAllowedColumns("expense"); +} + +const MAX_TREE_DEPTH = 5; + +type TreeFilter = { column: string; operator: string; value: string }; + +interface AutoDetectTree { + id: string; + name: string; + columns: string[]; + autoRefresh: boolean; + filters?: TreeFilter[]; + filterLogic?: string; + isPrimary?: boolean; + mode?: "split" | "merge"; +} + +interface AutoDetectConfig { + trees: AutoDetectTree[]; +} + +function loadAutoDetectConfig(): AutoDetectConfig { + if (_autoDetectConfigCache !== null) return _autoDetectConfigCache; + try { + const raw = fs.readFileSync(AUTO_DETECT_CONFIG_FILE, "utf-8"); + const parsed = JSON.parse(raw); + if (parsed.trees && Array.isArray(parsed.trees)) { + const validTrees = parsed.trees.filter((t: any) => + t.id && t.name && Array.isArray(t.columns) && + t.columns.length >= 1 && t.columns.length <= MAX_TREE_DEPTH && + t.columns.every((c: string) => getExpenseAutoDetectColumns().includes(c)) + ).map((t: any) => ({ + ...t, + isPrimary: t.isPrimary !== undefined ? t.isPrimary : true, + })); + _autoDetectConfigCache = { trees: validTrees }; + return _autoDetectConfigCache; + } + if (parsed.columns && Array.isArray(parsed.columns)) { + const migrated: AutoDetectConfig = { + trees: [{ + id: generateId(), + name: parsed.columns.map((c: string) => { + const labels: Record = { + statya: "Статья", dochka: "Дочка", vnuchka: "Внучка", + zakazchik: "Заказчик", kommentarii: "Комментарий", + dogovorzakazchika: "Договор заказчика", dogovorkontragenta: "Договор контрагента", + stavkands: "Ставка НДС", naznachenieplatezha: "Назначение платежа", + otvetstvenniipokontraktu: "Ответственный", statyadop: "Статья (доп.)", + ssilkanomervhodyaschegodokumenta: "Номер вх. документа", + }; + return labels[c] || c; + }).join(" / "), + columns: parsed.columns, + autoRefresh: false, + }], + }; + saveAutoDetectConfig(migrated); + migrateExpenseCategoryTreeIds(migrated.trees[0].id); + return migrated; + } + } catch {} + const defaultCfg: AutoDetectConfig = { + trees: [{ + id: generateId(), + name: "Статья / Дочка / Внучка", + columns: ["statya", "dochka", "vnuchka"], + autoRefresh: false, + }], + }; + saveAutoDetectConfig(defaultCfg); + migrateExpenseCategoryTreeIds(defaultCfg.trees[0].id); + return defaultCfg; +} + +function saveAutoDetectConfig(cfg: AutoDetectConfig) { + fs.writeFileSync(AUTO_DETECT_CONFIG_FILE, JSON.stringify(cfg, null, 2), "utf-8"); + _autoDetectConfigCache = null; + invalidateCategoryMemCache(); + scheduleTreeRebuild(); +} + +function getIncomeAutoDetectColumns(): string[] { + return getAutoDetectAllowedColumns("income"); +} + +function loadIncomeAutoDetectConfig(): AutoDetectConfig { + if (_incomeAutoDetectConfigCache !== null) return _incomeAutoDetectConfigCache; + try { + const raw = fs.readFileSync(INCOME_AUTO_DETECT_CONFIG_FILE, "utf-8"); + const parsed = JSON.parse(raw); + if (parsed.trees && Array.isArray(parsed.trees)) { + const validTrees = parsed.trees.filter((t: any) => + t.id && t.name && Array.isArray(t.columns) && + t.columns.length >= 1 && t.columns.length <= MAX_TREE_DEPTH && + t.columns.every((c: string) => getIncomeAutoDetectColumns().includes(c)) + ).map((t: any) => ({ + ...t, + isPrimary: t.isPrimary !== undefined ? t.isPrimary : true, + })); + _incomeAutoDetectConfigCache = { trees: validTrees }; + return _incomeAutoDetectConfigCache; + } + } catch {} + _incomeAutoDetectConfigCache = { trees: [] }; + return _incomeAutoDetectConfigCache; +} + +function saveIncomeAutoDetectConfig(cfg: AutoDetectConfig) { + fs.writeFileSync(INCOME_AUTO_DETECT_CONFIG_FILE, JSON.stringify(cfg, null, 2), "utf-8"); + _incomeAutoDetectConfigCache = null; + invalidateCategoryMemCache(); + scheduleTreeRebuild(); +} + +// ============================================================ +// MATERIALIZED TREE TABLES +// ============================================================ + +function rowMatchesCatRule(row: Record, rule: { column: string; operator: string; value: string }): boolean { + const val = row[rule.column] ?? ""; + switch (rule.operator) { + case "equals": return val === rule.value; + case "contains": return typeof val === "string" && matchTokens(val, rule.value); + case "not_equals": return val !== rule.value; + case "greater_than": return Number(val) > Number(rule.value); + case "less_than": return Number(val) < Number(rule.value); + case "equal_number": return Number(val) === Number(rule.value); + default: return val === rule.value; + } +} + +function rowMatchesCatRuleSet(row: Record, rules: { column: string; operator: string; value: string }[], logic: "and" | "or" = "and"): boolean { + if (rules.length === 0) return true; + if (logic === "or") return rules.some(r => rowMatchesCatRule(row, r)); + return rules.every(r => rowMatchesCatRule(row, r)); +} + +function transliterateRu(str: string): string { + const map: Record = { + а:"a",б:"b",в:"v",г:"g",д:"d",е:"e",ё:"yo",ж:"zh",з:"z",и:"i",й:"j",к:"k",л:"l",м:"m", + н:"n",о:"o",п:"p",р:"r",с:"s",т:"t",у:"u",ф:"f",х:"kh",ц:"ts",ч:"ch",ш:"sh",щ:"sch", + ъ:"",ы:"y",ь:"",э:"e",ю:"yu",я:"ya", + А:"a",Б:"b",В:"v",Г:"g",Д:"d",Е:"e",Ё:"yo",Ж:"zh",З:"z",И:"i",Й:"j",К:"k",Л:"l",М:"m", + Н:"n",О:"o",П:"p",Р:"r",С:"s",Т:"t",У:"u",Ф:"f",Х:"kh",Ц:"ts",Ч:"ch",Ш:"sh",Щ:"sch", + Ъ:"",Ы:"y",Ь:"",Э:"e",Ю:"yu",Я:"ya", + }; + return str.split("").map(c => map[c] !== undefined ? map[c] : c).join("") + .replace(/[^a-z0-9]+/gi, "_").replace(/^_+|_+$/g, "").toLowerCase().slice(0, 28); +} + +function getTreeTableName(type: "income" | "expense", treeName: string): string { + const prefix = type === "income" ? "derevodokhodi" : "derevooprashodi"; + const suffix = transliterateRu(treeName) || "tree"; + return `${prefix}_${suffix}`; +} + +type CatMatch = { id: string; name: string; pathNames: string[]; parentId: string | null; excluded: boolean }; + +function findDeepestCategoryMatch( + row: Record, + cats: IncomeCategory[], + ancestorRules: IncomeCategory["rules"][] = [], + ancestorNames: string[] = [], + ancestorIds: string[] = [] +): CatMatch | null { + for (let i = cats.length - 1; i >= 0; i--) { + const cat = cats[i]; + const logic = cat.ruleLogic || "and"; + if (!ancestorRules.every(rs => rowMatchesCatRuleSet(row, rs))) continue; + if (!rowMatchesCatRuleSet(row, cat.rules, logic)) continue; + const names = [...ancestorNames, cat.name]; + const ids = [...ancestorIds, cat.id]; + if (cat.children && cat.children.length > 0) { + const childMatch = findDeepestCategoryMatch(row, cat.children, [...ancestorRules, cat.rules], names, ids); + if (childMatch) return childMatch; + } + const parentId = ancestorIds.length > 0 ? ancestorIds[ancestorIds.length - 1] : null; + return { id: cat.id, name: cat.name, pathNames: names, parentId, excluded: cat.excluded || false }; + } + return null; +} + +function findCatByIdInTree(cats: IncomeCategory[], id: string): IncomeCategory | null { + for (const cat of cats) { + if (cat.id === id) return cat; + if (cat.children) { const f = findCatByIdInTree(cat.children, id); if (f) return f; } + } + return null; +} + +function flattenCatsForCols(cats: IncomeCategory[]): IncomeCategory[] { + const result: IncomeCategory[] = []; + for (const cat of cats) { + result.push(cat); + if (cat.children) result.push(...flattenCatsForCols(cat.children)); + } + return result; +} + +let treeRebuildInProgress = false; +let treeRebuildPending = false; +let treeRebuildLastCompletedAt: Date | null = null; +const treeRebuildWaiters: Array<() => void> = []; + +async function buildOneTreeTable( + type: "income" | "expense", + tableName: string, + allRows: Record[], + treeCats: IncomeCategory[], + isPrimary: boolean, + excludeRowIds?: Set +): Promise> { + const amountCol = type === "income" ? "summa" : "summadokumenta"; + const fkCol = type === "income" ? "dohod_id" : "rashod_id"; + await query(`DROP TABLE IF EXISTS "${tableName}"`); + await query(`CREATE TABLE "${tableName}" ( + id BIGSERIAL PRIMARY KEY, + ${fkCol} TEXT, + parent_id TEXT, + data DATE, + "${amountCol}" NUMERIC, + stavkands TEXT, + _category_id TEXT, + _category_name TEXT, + _category_path TEXT, + _is_excluded BOOLEAN DEFAULT FALSE, + _is_primary BOOLEAN DEFAULT TRUE + )`); + + const claimedIds = new Set(); + const toInsert: { fk: string; data: any; amount: number; stavkands: string; cat_id: string; parent_id: string | null; cat_name: string; cat_path: string; excluded: boolean }[] = []; + + for (const row of allRows) { + if (claimedIds.has(row.id)) continue; + if (excludeRowIds?.has(row.id)) continue; + const match = findDeepestCategoryMatch(row, treeCats); + if (!match) continue; + toInsert.push({ fk: row.id, data: row.data, amount: row.amount, stavkands: row.stavkands, cat_id: match.id, parent_id: match.parentId, cat_name: match.name, cat_path: match.pathNames.join(" / "), excluded: match.excluded }); + claimedIds.add(row.id); + } + + const BATCH = 500; + for (let i = 0; i < toInsert.length; i += BATCH) { + const batch = toInsert.slice(i, i + BATCH); + const valuePlaceholders: string[] = []; + const params: unknown[] = []; + for (const r of batch) { + const base = params.length; + params.push(r.fk, r.parent_id, r.data, r.amount, r.stavkands, r.cat_id, r.cat_name, r.cat_path, r.excluded, isPrimary); + valuePlaceholders.push(`($${base+1},$${base+2},$${base+3},$${base+4},$${base+5},$${base+6},$${base+7},$${base+8},$${base+9},$${base+10})`); + } + await query( + `INSERT INTO "${tableName}" (${fkCol}, parent_id, data, "${amountCol}", stavkands, _category_id, _category_name, _category_path, _is_excluded, _is_primary) VALUES ${valuePlaceholders.join(",")}`, + params + ); + } + + const idxBase = tableName.replace(/[^a-z0-9]/gi, "_").slice(0, 40); + await Promise.all([ + query(`CREATE INDEX IF NOT EXISTS "idx_${idxBase}_id" ON "${tableName}" (id)`), + query(`CREATE INDEX IF NOT EXISTS "idx_${idxBase}_fk" ON "${tableName}" (${fkCol})`), + query(`CREATE INDEX IF NOT EXISTS "idx_${idxBase}_pid" ON "${tableName}" (parent_id)`), + query(`CREATE INDEX IF NOT EXISTS "idx_${idxBase}_catid" ON "${tableName}" (_category_id)`), + query(`CREATE INDEX IF NOT EXISTS "idx_${idxBase}_data" ON "${tableName}" (data)`), + query(`CREATE INDEX IF NOT EXISTS "idx_${idxBase}_prim" ON "${tableName}" (_is_primary)`), + ]); + + return claimedIds; +} + +async function buildUncatTable(type: "income" | "expense", tableName: string, rows: Record[]): Promise { + const amountCol = type === "income" ? "summa" : "summadokumenta"; + const fkCol = type === "income" ? "dohod_id" : "rashod_id"; + await query(`DROP TABLE IF EXISTS "${tableName}"`); + await query(`CREATE TABLE "${tableName}" (id BIGSERIAL PRIMARY KEY, ${fkCol} TEXT, data DATE, "${amountCol}" NUMERIC, stavkands TEXT, _is_primary BOOLEAN DEFAULT FALSE)`); + const BATCH = 500; + for (let i = 0; i < rows.length; i += BATCH) { + const batch = rows.slice(i, i + BATCH); + if (batch.length === 0) continue; + const ph: string[] = []; const params: unknown[] = []; + for (const r of batch) { + const b = params.length; + params.push(r.id, r.data, r.amount, r.stavkands, false); + ph.push(`($${b+1},$${b+2},$${b+3},$${b+4},$${b+5})`); + } + await query(`INSERT INTO "${tableName}" (${fkCol}, data, "${amountCol}", stavkands, _is_primary) VALUES ${ph.join(",")}`, params); + } + const idxBase = tableName.replace(/[^a-z0-9]/gi, "_").slice(0, 40); + await Promise.all([ + query(`CREATE INDEX IF NOT EXISTS "idx_${idxBase}_fk" ON "${tableName}" (${fkCol})`), + query(`CREATE INDEX IF NOT EXISTS "idx_${idxBase}_data" ON "${tableName}" (data)`), + ]); +} + +async function rebuildAllTreeTables(): Promise { + if (treeRebuildInProgress) { + treeRebuildPending = true; + return; + } + do { + treeRebuildInProgress = true; + treeRebuildPending = false; + try { + console.log("[tree-tables] Starting rebuild..."); + const incomeCats = loadCategories(); + const expenseCats = loadExpenseCategories(); + const incomeConfig = loadIncomeAutoDetectConfig(); + const expenseConfig = loadAutoDetectConfig(); + + const incGrouped = new Map(); + for (const cat of incomeCats) { + const tid = cat.treeId || "__manual__"; + if (!incGrouped.has(tid)) incGrouped.set(tid, []); + incGrouped.get(tid)!.push(cat); + } + + const incRuleCols = new Set(["statya","dochka","vnuchka","otvetstvenniipokontraktu","zakazchik","dogovorzakazchika"]); + for (const cat of flattenCatsForCols(incomeCats)) { + for (const r of cat.rules) { if (ALLOWED_INCOME_COLUMNS_FALLBACK.includes(r.column)) incRuleCols.add(r.column); } + } + const incColList = [...incRuleCols]; + const incResult = await query(`SELECT id, data, COALESCE(summa,0) as summa, COALESCE(stavkands,'') as stavkands, ${incColList.map(c=>`COALESCE("${c}",'') as "${c}"`).join(",")} FROM ${getIncomeEffectiveView()}`); + const incRows = incResult.rows.map((r: any) => { + const row: Record = { id: String(r.id), data: r.data, amount: Number(r.summa), stavkands: String(r.stavkands || "") }; + for (const c of incColList) row[c] = r[c] ?? ""; + return row; + }); + + const manualIncCats = incGrouped.get("__manual__") || []; + const manualIncClaimedIds = await buildOneTreeTable("income", getTreeTableName("income", "Ручные категории"), incRows, manualIncCats, true); + + const allIncAutoClaimedIds = new Set(manualIncClaimedIds); + for (const t of incomeConfig.trees) { + const cats = incGrouped.get(t.id) || []; + const claimed = await buildOneTreeTable("income", getTreeTableName("income", t.name), incRows, cats, t.isPrimary ?? true, manualIncClaimedIds); + for (const id of claimed) allIncAutoClaimedIds.add(id); + } + + const allIncCats = [...incGrouped.values()].flat(); + const incUncatRows = incRows.filter(row => !allIncAutoClaimedIds.has(row.id) && !findDeepestCategoryMatch(row, allIncCats)); + await buildUncatTable("income", getTreeTableName("income", "Нераспределённые"), incUncatRows); + + const expGrouped = new Map(); + for (const cat of expenseCats) { + const tid = cat.treeId || "__manual__"; + if (!expGrouped.has(tid)) expGrouped.set(tid, []); + expGrouped.get(tid)!.push(cat); + } + + const expRuleCols = new Set(["statya","dochka","vnuchka","otvetstvenniipokontraktu","zakazchik","dogovorzakazchika"]); + for (const cat of flattenCatsForCols(expenseCats)) { + for (const r of cat.rules) { if (ALLOWED_EXPENSE_COLUMNS_FALLBACK.includes(r.column)) expRuleCols.add(r.column); } + } + const expColList = [...expRuleCols]; + const expResult = await query(`SELECT id, data, COALESCE(summadokumenta,0) as summadokumenta, COALESCE(stavkands,'') as stavkands, ${expColList.map(c=>`COALESCE("${c}",'') as "${c}"`).join(",")} FROM ${getExpenseEffectiveView()}`); + const expRows = expResult.rows.map((r: any) => { + const row: Record = { id: String(r.id), data: r.data, amount: Number(r.summadokumenta), stavkands: String(r.stavkands || "") }; + for (const c of expColList) row[c] = r[c] ?? ""; + return row; + }); + + const manualExpCats = expGrouped.get("__manual__") || []; + const manualExpClaimedIds = await buildOneTreeTable("expense", getTreeTableName("expense", "Ручные категории"), expRows, manualExpCats, true); + + const allExpAutoClaimedIds = new Set(manualExpClaimedIds); + for (const t of expenseConfig.trees) { + const cats = expGrouped.get(t.id) || []; + const claimed = await buildOneTreeTable("expense", getTreeTableName("expense", t.name), expRows, cats, t.isPrimary ?? true, manualExpClaimedIds); + for (const id of claimed) allExpAutoClaimedIds.add(id); + } + + const allExpCats = [...expGrouped.values()].flat(); + const expUncatRows = expRows.filter(row => !allExpAutoClaimedIds.has(row.id) && !findDeepestCategoryMatch(row, allExpCats)); + await buildUncatTable("expense", getTreeTableName("expense", "Нераспределённые"), expUncatRows); + + treeRebuildLastCompletedAt = new Date(); + cacheInvalidateByPrefix("dashboard:"); + cacheInvalidateByPrefix("profitability:"); + cacheInvalidateKey("__income_auto_sync__"); + cacheInvalidateKey("__expense_auto_sync__"); + console.log("[tree-tables] Rebuild complete"); + } catch (err) { + console.error("[tree-tables] Rebuild error:", err); + } finally { + treeRebuildInProgress = false; + } + } while (treeRebuildPending); + + const waiters = treeRebuildWaiters.splice(0); + for (const fn of waiters) fn(); +} + +function scheduleTreeRebuild() { + invalidateCategoryMemCache(); + cacheInvalidateByPrefix("dashboard:"); + cacheInvalidateByPrefix("profitability:"); + if (treeRebuildInProgress) { + treeRebuildPending = true; + return; + } + setImmediate(() => rebuildAllTreeTables().catch(console.error)); +} + +async function scheduleAndAwaitRebuild(): Promise { + return new Promise(resolve => { + treeRebuildWaiters.push(resolve); + if (treeRebuildInProgress) { + treeRebuildPending = true; + } else { + setImmediate(() => rebuildAllTreeTables().catch(console.error)); + } + }); +} + +async function ensureTreesReady(): Promise { + if (treeRebuildLastCompletedAt === null) { + await scheduleAndAwaitRebuild(); + } +} + +async function treeTableExists(tableName: string): Promise { + const r = await query(`SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1 LIMIT 1`, [tableName]); + return r.rows.length > 0; +} + +async function getPrimaryTreeTables(): Promise<{ income: string[]; expense: string[] }> { + const incConfig = loadIncomeAutoDetectConfig(); + const expConfig = loadAutoDetectConfig(); + const inc = [getTreeTableName("income", "Ручные категории")]; + for (const t of incConfig.trees) { if (t.isPrimary) inc.push(getTreeTableName("income", t.name)); } + const exp = [getTreeTableName("expense", "Ручные категории")]; + for (const t of expConfig.trees) { if (t.isPrimary) exp.push(getTreeTableName("expense", t.name)); } + return { income: inc, expense: exp }; +} + +// ============================================================ + +function buildFilterSQL(filters: TreeFilter[] | undefined, filterLogic: string | undefined, params: unknown[], allowedColumns: string[]): string { + if (!filters || filters.length === 0) return ""; + const conds: string[] = []; + for (const f of filters) { + if (!allowedColumns.includes(f.column) || !f.value) continue; + switch (f.operator) { + case "equals": + params.push(f.value); + conds.push(`"${f.column}" = $${params.length}`); + break; + case "not_equals": + params.push(f.value); + conds.push(`"${f.column}" != $${params.length}`); + break; + case "contains": + conds.push(buildTokenILIKE(`"${f.column}"`, f.value, params)); + break; + } + } + if (conds.length === 0) return ""; + const logic = filterLogic === "or" ? " OR " : " AND "; + return ` AND (${conds.join(logic)})`; +} + +async function buildIncomeAutoDetectTree(cols: string[], treeId: string, filters?: TreeFilter[], filterLogic?: string): Promise { + const selectCols = cols.map(c => `"${c}"`).join(", "); + const orderCols = cols.map(c => `"${c}"`).join(", "); + const params: unknown[] = []; + const filterClause = buildFilterSQL(filters, filterLogic, params, ALLOWED_INCOME_COLUMNS()); + const sql = `SELECT DISTINCT ${selectCols} FROM ${getIncomeEffectiveView()} WHERE "${cols[0]}" IS NOT NULL AND "${cols[0]}" != '0'${filterClause} ORDER BY ${orderCols}`; + const result = await query(sql, params); + + type NestedMap = Map; + const rootMap: NestedMap = new Map(); + + for (const row of result.rows) { + const values: (string | null)[] = cols.map((c, i) => { + const v = row[c] as string | null; + return (v && v !== "0") ? v : (i === 0 ? null : null); + }); + if (!values[0]) continue; + + let currentMap = rootMap; + for (let i = 0; i < values.length; i++) { + const val = values[i]; + if (!val) break; + if (!currentMap.has(val)) currentMap.set(val, new Map()); + currentMap = currentMap.get(val)!; + } + } + + function mapToCategories(map: NestedMap, colIndex: number): IncomeCategory[] { + const cats: IncomeCategory[] = []; + for (const [value, children] of map) { + const cat: IncomeCategory = { + id: generateId(), + name: value, + rules: [{ column: cols[colIndex], operator: "equals", value }], + autoDetected: true, + treeId, + }; + if (children.size > 0 && colIndex + 1 < cols.length) { + cat.children = mapToCategories(children, colIndex + 1); + } + cats.push(cat); + } + return cats; + } + + return mapToCategories(rootMap, 0); +} + +function buildMergedCategory(treeCfg: AutoDetectTree): IncomeCategory[] { + const rules: { column: string; operator: string; value: string }[] = []; + if (treeCfg.filters && treeCfg.filters.length > 0) { + for (const f of treeCfg.filters) { + rules.push({ column: f.column, operator: f.operator, value: f.value }); + } + } else if (treeCfg.columns.length > 0) { + rules.push({ column: treeCfg.columns[0], operator: "not_equals", value: "0" }); + } + return [{ + id: generateId(), + name: treeCfg.name, + rules, + ruleLogic: treeCfg.filterLogic === "or" ? "or" : undefined, + autoDetected: true, + treeId: treeCfg.id, + }]; +} + +function migrateAutoDetectedTreeIds(cats: IncomeCategory[], treeId: string): boolean { + let changed = false; + for (const cat of cats) { + if (cat.autoDetected && !cat.treeId) { + cat.treeId = treeId; + changed = true; + } + if (cat.children) { + if (migrateAutoDetectedTreeIds(cat.children, treeId)) changed = true; + } + } + return changed; +} + +function migrateExpenseCategoryTreeIds(treeId: string) { + try { + const raw = fs.readFileSync(EXPENSE_CATEGORIES_FILE, "utf-8"); + const cats: IncomeCategory[] = JSON.parse(raw); + if (migrateAutoDetectedTreeIds(cats, treeId)) { + fs.writeFileSync(EXPENSE_CATEGORIES_FILE, JSON.stringify(cats, null, 2), "utf-8"); + } + } catch {} +} + +function loadCategories(): IncomeCategory[] { + if (_incomeCategoriesCache !== null) return _incomeCategoriesCache; + try { + const raw = fs.readFileSync(CATEGORIES_FILE, "utf-8"); + _incomeCategoriesCache = JSON.parse(raw); + return _incomeCategoriesCache!; + } catch { + _incomeCategoriesCache = []; + return _incomeCategoriesCache; + } +} + +function saveCategories(cats: IncomeCategory[]) { + fs.writeFileSync(CATEGORIES_FILE, JSON.stringify(cats, null, 2), "utf-8"); + _incomeCategoriesCache = null; + cacheInvalidateByPrefix("dashboard:"); + cacheInvalidateByPrefix("profitability:"); + scheduleTreeRebuild(); +} + +function loadExpenseCategories(): IncomeCategory[] { + if (_expenseCategoriesCache !== null) return _expenseCategoriesCache; + try { + const raw = fs.readFileSync(EXPENSE_CATEGORIES_FILE, "utf-8"); + _expenseCategoriesCache = JSON.parse(raw); + return _expenseCategoriesCache!; + } catch { + _expenseCategoriesCache = []; + return _expenseCategoriesCache; + } +} + +function saveExpenseCategories(cats: IncomeCategory[]) { + fs.writeFileSync(EXPENSE_CATEGORIES_FILE, JSON.stringify(cats, null, 2), "utf-8"); + _expenseCategoriesCache = null; + cacheInvalidateByPrefix("dashboard:"); + cacheInvalidateByPrefix("profitability:"); + scheduleTreeRebuild(); +} + +function generateId(): string { + return Date.now().toString(36) + Math.random().toString(36).slice(2, 6); +} + +function findCategoryById(cats: IncomeCategory[], id: string): { cat: IncomeCategory; path: IncomeCategory[] } | null { + for (const cat of cats) { + if (cat.id === id) return { cat, path: [cat] }; + if (cat.children && cat.children.length > 0) { + const found = findCategoryById(cat.children, id); + if (found) return { cat: found.cat, path: [cat, ...found.path] }; + } + } + return null; +} + +function findAndRemoveCategory(cats: IncomeCategory[], id: string): boolean { + for (let i = 0; i < cats.length; i++) { + if (cats[i].id === id) { cats.splice(i, 1); return true; } + if (cats[i].children && findAndRemoveCategory(cats[i].children!, id)) return true; + } + return false; +} + +function findAndUpdateCategory(cats: IncomeCategory[], id: string, updater: (cat: IncomeCategory) => void): boolean { + for (const cat of cats) { + if (cat.id === id) { updater(cat); return true; } + if (cat.children && findAndUpdateCategory(cat.children, id, updater)) return true; + } + return false; +} + +function addChildToCategory(cats: IncomeCategory[], parentId: string, child: IncomeCategory): boolean { + for (const cat of cats) { + if (cat.id === parentId) { + if (!cat.children) cat.children = []; + cat.children.push(child); + return true; + } + if (cat.children && addChildToCategory(cat.children, parentId, child)) return true; + } + return false; +} + +function getCategoryDepth(cats: IncomeCategory[], id: string): number { + const found = findCategoryById(cats, id); + return found ? found.path.length : 0; +} + +function collectAncestorRules(cats: IncomeCategory[], id: string): IncomeCategory["rules"][] { + const found = findCategoryById(cats, id); + if (!found) return []; + return found.path.map(c => c.rules); +} + +function flattenCategories(cats: IncomeCategory[]): IncomeCategory[] { + const result: IncomeCategory[] = []; + for (const cat of cats) { + result.push(cat); + if (cat.children && cat.children.length > 0) { + result.push(...flattenCategories(cat.children)); + } + } + return result; +} + + +function isRefinementOf( + autoRuleSets: IncomeCategory["rules"][], + catRuleSets: IncomeCategory["rules"][] +): boolean { + const autoRules = autoRuleSets.flat(); + const catRules = catRuleSets.flat(); + if (autoRules.length === 0) return true; + const autoColumns = new Set(autoRules.map(r => r.column)); + const catColumns = new Set(catRules.map(r => r.column)); + if (![...autoColumns].some(col => catColumns.has(col))) return true; + return autoRules.every(ar => + catRules.some(cr => + cr.column === ar.column && + cr.operator === ar.operator && + cr.value === ar.value + ) + ); +} + +function deepCopyWithNewIds(cats: IncomeCategory[]): IncomeCategory[] { + return cats.map(cat => ({ + ...cat, + id: generateId(), + rules: [...cat.rules], + children: cat.children ? deepCopyWithNewIds(cat.children) : undefined, + })); +} + +function deepCopyWithNewIdsManual(cats: IncomeCategory[]): IncomeCategory[] { + return cats.map(cat => ({ + ...cat, + id: generateId(), + rules: [...cat.rules], + autoDetected: false, + treeId: undefined, + children: cat.children ? deepCopyWithNewIdsManual(cat.children) : undefined, + })); +} + +function findCategoryWithAncestors( + cats: IncomeCategory[], + targetId: string, + ancestors: IncomeCategory[] = [] +): { cat: IncomeCategory; ancestors: IncomeCategory[] } | null { + for (const cat of cats) { + if (cat.id === targetId) return { cat, ancestors }; + if (cat.children && cat.children.length > 0) { + const found = findCategoryWithAncestors(cat.children, targetId, [...ancestors, cat]); + if (found) return found; + } + } + return null; +} + +function buildRuleCondition(rule: IncomeCategory["rules"][0], params: unknown[], allowedColumns: string[] = ALLOWED_INCOME_COLUMNS()): string | null { + if (!allowedColumns.includes(rule.column)) return null; + switch (rule.operator) { + case "contains": + return buildTokenILIKE(rule.column, rule.value, params); + case "equals": + params.push(rule.value); + return `${rule.column} = $${params.length}`; + case "not_equals": + params.push(rule.value); + return `${rule.column} != $${params.length}`; + case "greater_than": + params.push(Number(rule.value)); + return `${rule.column} > $${params.length}`; + case "less_than": + params.push(Number(rule.value)); + return `${rule.column} < $${params.length}`; + case "equal_number": + params.push(Number(rule.value)); + return `${rule.column} = $${params.length}`; + default: + params.push(rule.value); + return `${rule.column} = $${params.length}`; + } +} + +function buildCategoryConditions(rules: IncomeCategory["rules"], params: unknown[], allowedColumns: string[] = ALLOWED_INCOME_COLUMNS()): string[] { + const conditions: string[] = []; + for (const rule of rules) { + const cond = buildRuleCondition(rule, params, allowedColumns); + if (cond) conditions.push(cond); + } + return conditions; +} + +function buildCatGroupCondition(cat: IncomeCategory, params: unknown[], allowedColumns: string[] = ALLOWED_INCOME_COLUMNS()): string | null { + const catConds: string[] = []; + for (const rule of cat.rules) { + const cond = buildRuleCondition(rule, params, allowedColumns); + if (cond) catConds.push(cond); + } + const joiner = cat.ruleLogic === "or" ? " OR " : " AND "; + return catConds.length > 0 ? `(${catConds.join(joiner)})` : null; +} + +function buildCategoryWithAncestorsConditions(cats: IncomeCategory[], categoryId: string, params: unknown[], allowedColumns: string[] = ALLOWED_INCOME_COLUMNS()): string[] { + const found = findCategoryById(cats, categoryId); + if (!found) return []; + const conditions: string[] = []; + for (let i = 0; i < found.path.length; i++) { + const cat = found.path[i]; + if (i === found.path.length - 1 && cat.ruleLogic === "or") { + const grouped = buildCatGroupCondition(cat, params, allowedColumns); + if (grouped) conditions.push(grouped); + } else { + conditions.push(...buildCategoryConditions(cat.rules, params, allowedColumns)); + } + } + return conditions; +} + +type SqlPath = { id?: string; treeId?: string; ancestorAutoIds?: string[]; ruleSets: IncomeCategory["rules"][]; logic: "and" | "or" }; + +function collectNonExcludedPathsForSql(cats: IncomeCategory[], ancestors: IncomeCategory["rules"][] = [], rootTreeId?: string): SqlPath[] { + const result: SqlPath[] = []; + for (const cat of cats) { + const tid = rootTreeId || cat.treeId || "__manual__"; + const path = [...ancestors, cat.rules]; + if (!cat.excluded) { + result.push({ treeId: tid, ruleSets: path, logic: cat.ruleLogic || "and" }); + } + if (cat.children) { + result.push(...collectNonExcludedPathsForSql(cat.children, path, tid)); + } + } + return result; +} + +function buildExcludeAllCategoriesConditions(cats: IncomeCategory[], params: unknown[], allowedColumns: string[] = ALLOWED_INCOME_COLUMNS(), onlyPrimaryTreeIds?: Set): string[] { + let paths = collectNonExcludedPathsForSql(cats); + if (onlyPrimaryTreeIds) { + paths = paths.filter(p => { + const tid = p.treeId || "__manual__"; + return tid === "__manual__" || onlyPrimaryTreeIds.has(tid); + }); + } + if (paths.length === 0) return []; + const excludeParts: string[] = []; + for (const p of paths) { + const part = buildPathConditionSql(p.ruleSets, p.logic, params, allowedColumns); + if (part) excludeParts.push(part); + } + if (excludeParts.length === 0) return []; + return [`NOT (${excludeParts.join(" OR ")})`]; +} + +function buildIncludePrimaryTreeConditions(cats: IncomeCategory[], params: unknown[], allowedColumns: string[], primaryTreeIds: Set): string[] { + let paths = collectNonExcludedPathsForSql(cats); + paths = paths.filter(p => { + const tid = p.treeId || "__manual__"; + return tid === "__manual__" || primaryTreeIds.has(tid); + }); + if (paths.length === 0) return ["1=0"]; + const parts: string[] = []; + for (const p of paths) { + const part = buildPathConditionSql(p.ruleSets, p.logic, params, allowedColumns); + if (part) parts.push(part); + } + if (parts.length === 0) return ["1=0"]; + return [`(${parts.join(" OR ")})`]; +} + +function buildPathConditionSql( + ruleSets: IncomeCategory["rules"][], + logic: "and" | "or", + params: unknown[], + allowedColumns: string[] +): string | null { + const pathConds: string[] = []; + for (let i = 0; i < ruleSets.length; i++) { + const rules = ruleSets[i]; + if (!rules || rules.length === 0) continue; + const ruleLogic = i === ruleSets.length - 1 ? logic : "and"; + const ruleConds: string[] = []; + for (const rule of rules) { + const cond = buildRuleCondition(rule, params, allowedColumns); + if (cond) ruleConds.push(cond); + } + if (ruleConds.length > 0) { + const joiner = ruleLogic === "or" ? " OR " : " AND "; + pathConds.push(`(${ruleConds.join(joiner)})`); + } + } + return pathConds.length > 0 ? `(${pathConds.join(" AND ")})` : null; +} + +function hasNonExcludedDescendant(cat: IncomeCategory): boolean { + if (!cat.children) return false; + for (const ch of cat.children) { + if (!ch.excluded) return true; + if (hasNonExcludedDescendant(ch)) return true; + } + return false; +} + +function collectExcludedPathsForSql(cats: IncomeCategory[], skipIds: Set, ancestors: IncomeCategory["rules"][] = [], rootTreeId?: string): SqlPath[] { + const result: SqlPath[] = []; + for (const cat of cats) { + const tid = rootTreeId || cat.treeId || "__manual__"; + const path = [...ancestors, cat.rules]; + if (cat.excluded && !skipIds.has(cat.id)) { + if (!hasNonExcludedDescendant(cat)) { + result.push({ id: cat.id, treeId: tid, ruleSets: path, logic: cat.ruleLogic || "and" }); + } + } + if (cat.children) { + result.push(...collectExcludedPathsForSql(cat.children, skipIds, path, tid)); + } + } + return result; +} + +function collectAutoPathsForSql(cats: IncomeCategory[], ancestors: IncomeCategory["rules"][] = [], ancestorAutoIds: string[] = []): SqlPath[] { + const result: SqlPath[] = []; + for (const cat of cats) { + if (cat.excluded) continue; + const path = [...ancestors, cat.rules]; + const autoIds = cat.autoDetected ? [...ancestorAutoIds, cat.id] : ancestorAutoIds; + if (cat.autoDetected) { + result.push({ id: cat.id, ancestorAutoIds: [...ancestorAutoIds], ruleSets: path, logic: cat.ruleLogic || "and" }); + } + if (cat.children) { + result.push(...collectAutoPathsForSql(cat.children, path, autoIds)); + } + } + return result; +} + +function buildGlobalExclusionConditions(cats: IncomeCategory[], params: unknown[], skipCategoryIds?: string | Set, allowedColumns: string[] = ALLOWED_INCOME_COLUMNS(), onlyPrimaryTreeIds?: Set, manualCatRuleSets?: IncomeCategory["rules"][]): string[] { + const skipSet = skipCategoryIds instanceof Set ? skipCategoryIds : (skipCategoryIds ? new Set([skipCategoryIds]) : new Set()); + let paths = collectExcludedPathsForSql(cats, skipSet); + if (onlyPrimaryTreeIds) { + paths = paths.filter(p => { + const tid = p.treeId || "__manual__"; + return tid === "__manual__" || onlyPrimaryTreeIds.has(tid); + }); + } + if (manualCatRuleSets && manualCatRuleSets.length > 0) { + paths = paths.filter(p => !isRefinementOf(p.ruleSets, manualCatRuleSets)); + } + if (paths.length === 0) return []; + const excludeParts: string[] = []; + for (const p of paths) { + const part = buildPathConditionSql(p.ruleSets, p.logic, params, allowedColumns); + if (part) excludeParts.push(part); + } + if (excludeParts.length === 0) return []; + return [`NOT (${excludeParts.join(" OR ")})`]; +} + +function buildAutoPathExclusionConditions(cats: IncomeCategory[], params: unknown[], allowedColumns: string[] = ALLOWED_INCOME_COLUMNS(), skipIds?: Set, manualCatRuleSets?: IncomeCategory["rules"][]): string[] { + let paths = collectAutoPathsForSql(cats); + if (skipIds && skipIds.size > 0) { + paths = paths.filter(p => { + if (p.id && skipIds.has(p.id)) return false; + if (p.ancestorAutoIds && p.ancestorAutoIds.some(aid => skipIds.has(aid))) return false; + return true; + }); + } + if (manualCatRuleSets && manualCatRuleSets.length > 0) { + paths = paths.filter(p => + !isRefinementOf(p.ruleSets, manualCatRuleSets) && + !isRefinementOf(manualCatRuleSets, p.ruleSets) + ); + } + if (paths.length === 0) return []; + const parts: string[] = []; + for (const p of paths) { + const part = buildPathConditionSql(p.ruleSets, p.logic, params, allowedColumns); + if (part) parts.push(part); + } + if (parts.length === 0) return []; + return [`NOT (${parts.join(" OR ")})`]; +} + +function buildManualCatExclusionConditions(cats: IncomeCategory[], params: unknown[], allowedColumns: string[]): string[] { + const rootManual = cats.filter(c => !c.autoDetected && !c.excluded && !c.treeId); + if (rootManual.length === 0) return []; + const parts: string[] = []; + for (const mc of rootManual) { + const cond = buildPathConditionSql([mc.rules], mc.ruleLogic || "and", params, allowedColumns); + if (cond) parts.push(cond); + } + if (parts.length === 0) return []; + return [`NOT (${parts.join(" OR ")})`]; +} + +function buildSiblingExclusions(cats: IncomeCategory[], categoryId: string, params: unknown[], allowedColumns: string[] = ALLOWED_INCOME_COLUMNS()): string[] { + const found = findCategoryById(cats, categoryId); + if (!found) return []; + let siblings: IncomeCategory[] = []; + if (found.path.length === 1) { + const targetTreeId = found.cat.treeId || "__manual__"; + const sameTreeCats = cats.filter(c => (c.treeId || "__manual__") === targetTreeId); + const idx = sameTreeCats.findIndex(c => c.id === categoryId); + siblings = sameTreeCats.slice(idx + 1).filter(c => !c.excluded); + } else { + const parent = found.path[found.path.length - 2]; + if (parent.children) { + const idx = parent.children.findIndex(c => c.id === categoryId); + siblings = parent.children.slice(idx + 1).filter(c => !c.excluded); + } + } + const conditions: string[] = []; + for (const sib of siblings) { + const part = buildCatGroupCondition(sib, params, allowedColumns); + if (part) conditions.push(`NOT ${part}`); + } + return conditions; +} + +function validateRules(rules: any[], allowedColumns: string[], validOperators: string[]): string | null { + for (const r of rules) { + if (!allowedColumns.includes(r.column)) return `Недопустимый столбец: ${r.column}`; + if (!validOperators.includes(r.operator)) return "Недопустимый оператор"; + if (!r.value && r.value !== "0") return "Значение не может быть пустым"; + } + return null; +} + +async function setupPgTrgm(): Promise { + try { + await query(`CREATE EXTENSION IF NOT EXISTS pg_trgm`); + const incTable = getIncomeTable(); + const expTable = getExpenseTable(); + await Promise.all([ + query(`CREATE INDEX IF NOT EXISTS "idx_trgm_inc_statya" ON "${incTable}" USING GIN (statya gin_trgm_ops)`).catch(() => {}), + query(`CREATE INDEX IF NOT EXISTS "idx_trgm_inc_zakazchik" ON "${incTable}" USING GIN (zakazchik gin_trgm_ops)`).catch(() => {}), + query(`CREATE INDEX IF NOT EXISTS "idx_trgm_exp_statya" ON "${expTable}" USING GIN (statya gin_trgm_ops)`).catch(() => {}), + query(`CREATE INDEX IF NOT EXISTS "idx_trgm_exp_kontragent" ON "${expTable}" USING GIN (kontragent gin_trgm_ops)`).catch(() => {}), + query(`CREATE INDEX IF NOT EXISTS "idx_trgm_exp_zakazchik" ON "${expTable}" USING GIN (zakazchik gin_trgm_ops)`).catch(() => {}), + ]); + console.log("[pg_trgm] Extension and GIN indexes ensured"); + } catch (err) { + console.warn("[pg_trgm] Could not set up trgm indexes (non-fatal):", (err as Error).message); + } +} + +async function initTagTables() { + try { + await query(`CREATE TABLE IF NOT EXISTS node_tags ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + color TEXT NOT NULL + )`); + await query(`CREATE TABLE IF NOT EXISTS node_tag_assignments ( + id TEXT PRIMARY KEY, + node_id TEXT NOT NULL, + tag_id TEXT NOT NULL, + context TEXT NOT NULL, + UNIQUE(node_id, tag_id, context) + )`); + } catch (err) { + console.warn("[tags] Could not create tag tables (non-fatal):", (err as Error).message); + } +} + +export async function registerRoutes(httpServer: Server, app: Express): Promise { + + try { + await initOverrideTables(); + await initTagTables(); + await migrateJsonOverrides( + CATEGORY_OVERRIDES_FILE, + (id) => findCategoryById(loadCategories(), id), + (id) => findCategoryById(loadExpenseCategories(), id) + ); + } catch (err) { + console.error("[registerRoutes] initOverrideTables/migrate error:", err); + } + + setupPgTrgm().catch(err => console.warn("[pg_trgm] background setup failed:", err)); + + app.post("/api/cache/invalidate", async (_req, res) => { + try { + cacheInvalidateAll(); + invalidateCategoryMemCache(); + await Promise.all([ + runIncomeSyncCore().catch(err => console.error("[cache/invalidate] income sync:", err)), + runExpenseSyncCore().catch(err => console.error("[cache/invalidate] expense sync:", err)), + ]); + await scheduleAndAwaitRebuild(); + res.json({ ok: true }); + } catch (err) { + handleApiError(res, err, "Ошибка при инвалидации кэша"); + } + }); + + app.get("/api/connection", (_req, res) => { + const cfg = getConfig(); + res.json({ host: cfg.host, port: cfg.port, database: cfg.database, user: cfg.user, password: "***" }); + }); + + app.post("/api/connection/test", async (req, res) => { + const parsed = dbConnectionSchema.safeParse(req.body); + if (!parsed.success) return res.status(400).json({ error: "Неверные данные" }); + const result = await testConnection(parsed.data); + res.json(result); + }); + + app.post("/api/connection", async (req, res) => { + const parsed = dbConnectionSchema.safeParse(req.body); + if (!parsed.success) return res.status(400).json({ error: "Неверные данные" }); + const test = await testConnection(parsed.data); + if (!test.ok) return res.status(400).json({ error: test.message }); + setConfig(parsed.data); + res.json({ ok: true }); + }); + + app.get("/api/db/tables", async (_req, res) => { + try { + const result = await query(`SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' ORDER BY table_name`); + const viewResult = await query(`SELECT table_name FROM information_schema.views WHERE table_schema = 'public' ORDER BY table_name`); + const tables = result.rows.map((r: any) => r.table_name); + const views = viewResult.rows.map((r: any) => r.table_name); + res.json({ tables, views }); + } catch (err) { + handleApiError(res, err, "Не удалось получить список таблиц"); + } + }); + + app.get("/api/db/columns/:table", async (req, res) => { + try { + const tableName = req.params.table; + if (!/^[a-z_][a-z0-9_]*$/i.test(tableName)) { + return res.status(400).json({ error: "Некорректное имя таблицы" }); + } + const result = await query( + `SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = 'public' AND table_name = $1 ORDER BY ordinal_position`, + [tableName] + ); + res.json(result.rows.map((r: any) => ({ + name: r.column_name, + type: r.data_type, + nullable: r.is_nullable === "YES", + }))); + } catch (err) { + handleApiError(res, err, "Не удалось получить столбцы таблицы"); + } + }); + + app.get("/api/table-config", (_req, res) => { + res.json(loadTableConfig()); + }); + + app.post("/api/table-config", async (req, res) => { + try { + const body = req.body; + const VALID_IDENT = /^[a-z_][a-z0-9_]*$/i; + + if (body.incomeTable && !VALID_IDENT.test(body.incomeTable)) { + return res.status(400).json({ error: "Некорректное имя таблицы доходов" }); + } + if (body.expenseTable && !VALID_IDENT.test(body.expenseTable)) { + return res.status(400).json({ error: "Некорректное имя таблицы расходов" }); + } + if (body.debtTable && !VALID_IDENT.test(body.debtTable)) { + return res.status(400).json({ error: "Некорректное имя таблицы задолженности" }); + } + + if (body.incomeColumns && Array.isArray(body.incomeColumns)) { + if (!body.incomeColumns.every((c: string) => VALID_IDENT.test(c))) { + return res.status(400).json({ error: "Некорректные имена столбцов доходов" }); + } + } + if (body.expenseColumns && Array.isArray(body.expenseColumns)) { + if (!body.expenseColumns.every((c: string) => VALID_IDENT.test(c))) { + return res.status(400).json({ error: "Некорректные имена столбцов расходов" }); + } + } + + for (const tableField of ["incomeTable", "expenseTable", "debtTable"] as const) { + if (body[tableField]) { + try { + const check = await query( + `SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = $1 + UNION SELECT 1 FROM information_schema.views WHERE table_schema = 'public' AND table_name = $1`, + [body[tableField]] + ); + if (check.rows.length === 0) { + return res.status(400).json({ error: `Таблица "${body[tableField]}" не найдена в БД` }); + } + } catch { + return res.status(400).json({ error: `Не удалось проверить таблицу "${body[tableField]}"` }); + } + } + } + + saveTableConfig(body); + invalidateServerCache(); + try { + await recreateEffectiveViews(); + } catch (viewErr) { + console.error("[table-config] Failed to recreate effective views:", viewErr); + } + res.json({ ok: true, config: loadTableConfig() }); + } catch (err) { + res.status(500).json({ error: "Не удалось сохранить конфигурацию таблиц" }); + } + }); + + const TELEGRAM_CONFIG_PATH = path.join("data", "telegram-config.json"); + + function loadTelegramConfig(): { botToken: string; chats: { id: string; name: string; aiAccess?: boolean }[] } { + try { + if (fs.existsSync(TELEGRAM_CONFIG_PATH)) { + return JSON.parse(fs.readFileSync(TELEGRAM_CONFIG_PATH, "utf-8")); + } + } catch {} + return { botToken: "", chats: [] }; + } + + const TABLE_SETTINGS_PATH = path.join(process.cwd(), "data", "table-settings.json"); + + function loadTableSettings(): Record { + try { + if (fs.existsSync(TABLE_SETTINGS_PATH)) { + return JSON.parse(fs.readFileSync(TABLE_SETTINGS_PATH, "utf-8")); + } + } catch {} + return {}; + } + + app.get("/api/table-settings", (_req, res) => { + res.json(loadTableSettings()); + }); + + app.post("/api/table-settings", (req, res) => { + try { + const current = loadTableSettings(); + const { section, visibleCols, colOrder } = req.body; + if (!section || !["income", "expense"].includes(section)) { + return res.status(400).json({ error: "section must be 'income' or 'expense'" }); + } + current[section] = { + visibleCols: Array.isArray(visibleCols) ? visibleCols : current[section]?.visibleCols, + colOrder: Array.isArray(colOrder) ? colOrder : current[section]?.colOrder, + }; + fs.writeFileSync(TABLE_SETTINGS_PATH, JSON.stringify(current, null, 2)); + res.json({ ok: true }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка сохранения настроек" }); + } + }); + + app.get("/api/telegram/config", (_req, res) => { + const cfg = loadTelegramConfig(); + res.json({ botToken: "", hasToken: !!cfg.botToken, chats: cfg.chats }); + }); + + app.post("/api/telegram/config", (req, res) => { + try { + const { botToken, chats } = req.body; + if (typeof botToken !== "string") return res.status(400).json({ error: "botToken обязателен" }); + if (!Array.isArray(chats)) return res.status(400).json({ error: "chats должен быть массивом" }); + const existing = loadTelegramConfig(); + const token = botToken === "" ? existing.botToken : botToken; + const sanitized = chats.map((c: { id?: string; name?: string; aiAccess?: boolean }) => ({ + id: String(c.id ?? "").trim(), + name: String(c.name ?? "").trim(), + aiAccess: !!c.aiAccess, + })).filter((c: { id: string }) => c.id.length > 0); + fs.writeFileSync(TELEGRAM_CONFIG_PATH, JSON.stringify({ botToken: token, chats: sanitized }, null, 2)); + res.json({ ok: true }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Не удалось сохранить" }); + } + }); + + app.post("/api/telegram/test", async (req, res) => { + try { + const cfg = loadTelegramConfig(); + if (!cfg.botToken) return res.status(400).json({ error: "Токен бота не настроен" }); + if (cfg.chats.length === 0) return res.status(400).json({ error: "Нет настроенных чатов" }); + const results: { chatId: string; name: string; ok: boolean; error?: string }[] = []; + for (const chat of cfg.chats) { + try { + const resp = await fetch(`https://api.telegram.org/bot${cfg.botToken}/sendMessage`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ chat_id: chat.id, text: "привет" }), + }); + const json = await resp.json() as { ok: boolean; description?: string }; + results.push({ chatId: chat.id, name: chat.name, ok: json.ok, error: json.ok ? undefined : (json.description ?? "Неизвестная ошибка") }); + } catch (e: unknown) { + results.push({ chatId: chat.id, name: chat.name, ok: false, error: e instanceof Error ? e.message : "Ошибка запроса" }); + } + } + res.json({ results }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка отправки" }); + } + }); + + // --- AI Agent (конфиги и лог — чистое IO из ./ai-config, без Telegram-поллинга) --- + + app.get("/api/ai/config", (_req, res) => { + const cfg = loadAiConfig(); + res.json({ + apiUrl: cfg.apiUrl, + apiKey: "", + hasApiKey: !!cfg.apiKey, + model: cfg.model, + systemPrompt: cfg.systemPrompt, + enabled: cfg.enabled, + name: cfg.name, + }); + }); + + app.post("/api/ai/config", (req, res) => { + try { + const { apiUrl, apiKey, model, systemPrompt, enabled, name } = req.body; + const existing = loadAiConfig(); + const newCfg = { + apiUrl: typeof apiUrl === "string" && apiUrl.trim() ? apiUrl.trim() : existing.apiUrl, + apiKey: typeof apiKey === "string" && apiKey.trim() ? apiKey.trim() : existing.apiKey, + model: typeof model === "string" && model.trim() ? model.trim() : existing.model, + systemPrompt: typeof systemPrompt === "string" ? systemPrompt : existing.systemPrompt, + enabled: typeof enabled === "boolean" ? enabled : existing.enabled, + name: typeof name === "string" && name.trim() ? name.trim() : existing.name, + }; + saveAiConfig(newCfg); + res.json({ ok: true }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Не удалось сохранить" }); + } + }); + + app.get("/api/ai/planning-config", (_req, res) => { + const cfg = loadAiPlanningConfig(); + res.json({ + apiUrl: cfg.apiUrl, + apiKey: "", + hasApiKey: !!cfg.apiKey, + model: cfg.model, + systemPrompt: cfg.systemPrompt, + name: cfg.name, + enabled: cfg.enabled, + }); + }); + + app.post("/api/ai/planning-config", (req, res) => { + try { + const { apiUrl, apiKey, model, systemPrompt, name, enabled } = req.body; + const existing = loadAiPlanningConfig(); + const newCfg = { + apiUrl: typeof apiUrl === "string" && apiUrl.trim() ? apiUrl.trim() : existing.apiUrl, + apiKey: typeof apiKey === "string" && apiKey.trim() ? apiKey.trim() : existing.apiKey, + model: typeof model === "string" && model.trim() ? model.trim() : existing.model, + systemPrompt: typeof systemPrompt === "string" ? systemPrompt : existing.systemPrompt, + name: typeof name === "string" && name.trim() ? name.trim() : existing.name, + enabled: typeof enabled === "boolean" ? enabled : existing.enabled, + }; + saveAiPlanningConfig(newCfg); + res.json({ ok: true }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Не удалось сохранить" }); + } + }); + + app.post("/api/ai/planning-toggle", (req, res) => { + try { + const cfg = loadAiPlanningConfig(); + const enabled = typeof req.body.enabled === "boolean" ? req.body.enabled : !cfg.enabled; + cfg.enabled = enabled; + saveAiPlanningConfig(cfg); + res.json({ ok: true, enabled }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Не удалось переключить" }); + } + }); + + app.get("/api/ai/log", (req, res) => { + const days = Math.min(Math.max(parseInt(String(req.query.days)) || 90, 1), 365); + res.json(getAiLog(days)); + }); + + let modelsCache: { data: any[]; ts: number } | null = null; + const MODELS_CACHE_TTL = 3600000; + + app.get("/api/ai/models", async (_req, res) => { + try { + if (modelsCache && Date.now() - modelsCache.ts < MODELS_CACHE_TTL) { + return res.json(modelsCache.data); + } + const resp = await fetch("https://openrouter.ai/api/v1/models", { + signal: AbortSignal.timeout(10000), + }); + if (!resp.ok) { + return res.status(502).json({ error: "Не удалось загрузить модели" }); + } + const json = await resp.json() as any; + const allModels = json.data || []; + const toolModels = allModels + .filter((m: any) => (m.supported_parameters || []).includes("tools")) + .map((m: any) => ({ + id: m.id, + name: m.name, + context_length: m.context_length, + promptPrice: m.pricing?.prompt || "0", + completionPrice: m.pricing?.completion || "0", + })) + .sort((a: any, b: any) => parseFloat(a.promptPrice) - parseFloat(b.promptPrice)); + modelsCache = { data: toolModels, ts: Date.now() }; + res.json(toolModels); + } catch (err: any) { + console.error("[AI Models]", err?.message); + res.status(500).json({ error: "Ошибка загрузки моделей" }); + } + }); + + function buildIncludeCategoryConditions(cats: IncomeCategory[], selectedIds: string[], params: unknown[], allowedColumns: string[]): string[] { + const flat = flattenCategories(cats); + const selected = flat.filter(c => selectedIds.includes(c.id)); + if (selected.length === 0) return ["FALSE"]; + const parts: string[] = []; + for (const cat of selected) { + const part = buildCatGroupCondition(cat, params, allowedColumns); + if (part) parts.push(part); + } + if (parts.length === 0) return ["FALSE"]; + return [`(${parts.join(" OR ")})`]; + } + + app.get("/api/dashboard", async (req, res) => { + try { + const cacheKey = stableKey("dashboard", req.query as Record); + const result = await cachedQuery(cacheKey, TTL_LONG, async () => { + const maxPeriod = await getMaxPeriod(); + const { dateFrom, dateTo, incomeCategories, expenseCategories } = req.query; + + const incParams: unknown[] = []; + const incConditions: string[] = []; + const dashCats = loadCategories(); + incConditions.push(...buildGlobalExclusionConditions(dashCats, incParams)); + + if (dateFrom) { incParams.push(dateFrom); incConditions.push(`data::date >= $${incParams.length}::date`); } + if (dateTo) { incParams.push(dateTo); incConditions.push(`data::date <= $${incParams.length}::date`); } + + if (incomeCategories) { + try { + const ids = JSON.parse(incomeCategories as string) as string[]; + incConditions.push(...buildIncludeCategoryConditions(dashCats, ids, incParams, ALLOWED_INCOME_COLUMNS())); + } catch {} + } + + const incomeWhere = incConditions.length > 0 ? `WHERE ${incConditions.join(" AND ")}` : ""; + + const expParams: unknown[] = []; + const expConditions: string[] = []; + const expCats = loadExpenseCategories(); + expConditions.push(...buildGlobalExclusionConditions(expCats, expParams, undefined, ALLOWED_EXPENSE_COLUMNS())); + if (dateFrom) { expParams.push(dateFrom); expConditions.push(`data::date >= $${expParams.length}::date`); } + if (dateTo) { expParams.push(dateTo); expConditions.push(`data::date <= $${expParams.length}::date`); } + + if (expenseCategories) { + try { + const ids = JSON.parse(expenseCategories as string) as string[]; + expConditions.push(...buildIncludeCategoryConditions(expCats, ids, expParams, ALLOWED_EXPENSE_COLUMNS())); + } catch {} + } + + const expenseWhere = expConditions.length > 0 ? `WHERE ${expConditions.join(" AND ")}` : ""; + + const [incomeTotal, expenseTotal, debtStats, monthlyIncome, monthlyExpense, debtByClass, billedVsPaidRes] = await Promise.all([ + query(`SELECT COALESCE(SUM(summa),0) as total, COUNT(*) as cnt FROM ${getIncomeEffectiveView()} ${incomeWhere}`, [...incParams]), + query(`SELECT COALESCE(SUM(summadokumenta),0) as total, COUNT(*) as cnt FROM ${getExpenseEffectiveView()} ${expenseWhere}`, [...expParams]), + query(` + SELECT COALESCE(SUM(summadolga),0) as total_debt, COALESCE(SUM(summaprosrocheno),0) as total_overdue, COUNT(DISTINCT kontragent) as counterparts + FROM ${getDebtTable()} WHERE period = $1 + `, [maxPeriod]), + query(`SELECT DATE_TRUNC('month', data) as month, COALESCE(SUM(summa),0) as total FROM ${getIncomeEffectiveView()} ${incomeWhere} GROUP BY DATE_TRUNC('month', data) ORDER BY month DESC LIMIT 24`, [...incParams]), + query(`SELECT DATE_TRUNC('month', data) as month, COALESCE(SUM(summadokumenta),0) as total FROM ${getExpenseEffectiveView()} ${expenseWhere} GROUP BY DATE_TRUNC('month', data) ORDER BY month DESC LIMIT 24`, [...expParams]), + query(` + SELECT CASE WHEN klassprosrochki IS NULL OR klassprosrochki = '0' THEN 'Не просрочено' ELSE klassprosrochki || ' дн.' END as class, + COALESCE(SUM(summadolga),0) as total + FROM ${getDebtTable()} WHERE period = $1 + GROUP BY klassprosrochki ORDER BY total DESC + `, [maxPeriod]), + query(` + SELECT TO_CHAR(period, 'YYYY-MM') as month, + COALESCE(SUM(summadokumenta::numeric),0) as billed, + COALESCE(SUM(summaoplacheno::numeric),0) as paid, + COALESCE(SUM(summadolga::numeric),0) as debt + FROM ${getDebtTable()} + WHERE period >= (NOW() - INTERVAL '12 months')::date + GROUP BY TO_CHAR(period, 'YYYY-MM') + ORDER BY month + `), + ]); + + return { + incomeTotal: Number(incomeTotal.rows[0].total), + incomeCount: Number(incomeTotal.rows[0].cnt), + expenseTotal: Number(expenseTotal.rows[0].total), + expenseCount: Number(expenseTotal.rows[0].cnt), + debtTotal: Number(debtStats.rows[0].total_debt), + debtOverdue: Number(debtStats.rows[0].total_overdue), + debtCounterparts: Number(debtStats.rows[0].counterparts), + monthlyIncome: monthlyIncome.rows.map(r => ({ month: parseDate(r.month)!, total: Number(r.total) })).reverse(), + monthlyExpense: monthlyExpense.rows.map(r => ({ month: parseDate(r.month)!, total: Number(r.total) })).reverse(), + debtByClass: debtByClass.rows.map(r => ({ class: r.class, total: Number(r.total) })), + billedVsPaid: billedVsPaidRes.rows.map(r => ({ month: r.month, billed: Number(r.billed), paid: Number(r.paid), debt: Number(r.debt) })), + }; + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/income/column-values", async (req, res) => { + try { + const { column, search = "", context: contextStr = "" } = req.query; + if (!column || !ALLOWED_INCOME_COLUMNS().includes(column as string)) { + return res.status(400).json({ error: "Недопустимый столбец" }); + } + const cacheKey = `col-values:${column}:${search}:${contextStr}`; + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const col = column as string; + const searchStr = String(search); + const isDateOrNum = col === "data" || col === "summa"; + const conditions: string[] = [`${isDateOrNum ? `CAST(${col} AS TEXT)` : col} IS NOT NULL`]; + if (!isDateOrNum) conditions.push(`${col} != '0'`); + const params: unknown[] = []; + if (searchStr) { + const colExpr = isDateOrNum ? `CAST(${col} AS TEXT)` : col; + conditions.push(buildTokenILIKE(colExpr, searchStr, params)); + } + if (contextStr) { + try { + const ctx = JSON.parse(String(contextStr)) as Record; + for (const [ctxCol, ctxVal] of Object.entries(ctx)) { + if (!ctxVal) continue; + if (ctxCol === "dateFrom") { + params.push(ctxVal); + conditions.push(`data::date >= $${params.length}::date`); + } else if (ctxCol === "dateTo") { + params.push(ctxVal); + conditions.push(`data::date <= $${params.length}::date`); + } else if (ALLOWED_INCOME_COLUMNS().includes(ctxCol) && ctxCol !== col) { + const ctxIsDateOrNum = ctxCol === "data" || ctxCol === "summa"; + const ctxExpr = ctxIsDateOrNum ? `CAST(${ctxCol} AS TEXT)` : ctxCol; + conditions.push(buildTokenILIKE(ctxExpr, ctxVal, params)); + } + } + } catch { /* ignore */ } + } + const where = `WHERE ${conditions.join(" AND ")}`; + const selectExpr = isDateOrNum ? `CAST(${col} AS TEXT)` : col; + const orderDir = isDateOrNum ? "DESC" : "ASC"; + const result = await query( + `SELECT DISTINCT ${selectExpr} as val FROM ${getIncomeEffectiveView()} ${where} ORDER BY val ${orderDir} LIMIT 20`, + params + ); + return result.rows.map(r => r.val); + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/income/columns", async (_req, res) => { + try { + const result = await query(`SELECT * FROM ${getIncomeEffectiveView()} LIMIT 1`); + const dbCols = result.fields.map((f: any) => f.name); + res.json(dbCols); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/income", async (req, res) => { + try { + const { filters: filtersStr, categoryId, excludeCategories, page = "1", pageSize = "50", sortBy, sortDir } = req.query; + const p = Number(page); + const ps = Math.min(Number(pageSize), 200); + const offset = (p - 1) * ps; + + const conditions: string[] = []; + const params: unknown[] = []; + + const incomeCats = loadCategories(); + + const incomeConfig = loadIncomeAutoDetectConfig(); + const incomePrimaryTreeIds = new Set(incomeConfig.trees.filter(t => t.isPrimary).map(t => t.id)); + + let skipExclIds: string | Set | undefined; + let rootManualRuleSets: IncomeCategory["rules"][] | undefined; + if (categoryId) { + const ancestorConditions = buildCategoryWithAncestorsConditions(incomeCats, categoryId as string, params); + conditions.push(...ancestorConditions); + const siblingExcl = buildSiblingExclusions(incomeCats, categoryId as string, params); + conditions.push(...siblingExcl); + const foundCat = findCategoryById(incomeCats, categoryId as string); + if (foundCat) { + skipExclIds = new Set(foundCat.path.map(c => c.id)); + if (!foundCat.cat.autoDetected) { + const ancestorAutoIds = new Set(foundCat.path.filter(c => c.autoDetected).map(c => c.id)); + if (ancestorAutoIds.size > 0) { + const autoExcl = buildAutoPathExclusionConditions(incomeCats, params, ALLOWED_INCOME_COLUMNS(), ancestorAutoIds); + conditions.push(...autoExcl); + } else { + rootManualRuleSets = [foundCat.cat.rules]; + if (foundCat.cat.treeId) { + const autoExcl = buildAutoPathExclusionConditions(incomeCats, params, ALLOWED_INCOME_COLUMNS(), undefined, rootManualRuleSets); + conditions.push(...autoExcl); + } + } + } else { + const manualExcl = buildManualCatExclusionConditions(incomeCats, params, ALLOWED_INCOME_COLUMNS()); + conditions.push(...manualExcl); + } + } else { + skipExclIds = categoryId as string; + } + } else if (excludeCategories === "true") { + const excludeConditions = buildExcludeAllCategoriesConditions(incomeCats, params, ALLOWED_INCOME_COLUMNS(), incomePrimaryTreeIds); + conditions.push(...excludeConditions); + conditions.push('"_override_category_id" IS NULL'); + } + + const globalExclusions = buildGlobalExclusionConditions(incomeCats, params, skipExclIds, ALLOWED_INCOME_COLUMNS(), excludeCategories === "true" ? incomePrimaryTreeIds : undefined, rootManualRuleSets); + conditions.push(...globalExclusions); + + const filterConditions: string[] = []; + if (filtersStr) { + try { + const filterObj = JSON.parse(String(filtersStr)) as Record; + if (filterObj.dateFrom) { + params.push(filterObj.dateFrom); + filterConditions.push(`data::date >= $${params.length}::date`); + } + if (filterObj.dateTo) { + params.push(filterObj.dateTo); + filterConditions.push(`data::date <= $${params.length}::date`); + } + for (const [col, val] of Object.entries(filterObj)) { + if (col === "dateFrom" || col === "dateTo") continue; + if (ALLOWED_INCOME_COLUMNS().includes(col) && val) { + const colExpr = (col === "summa" || col === "data") ? `CAST(${col} AS TEXT)` : col; + filterConditions.push(buildTokenILIKE(colExpr, String(val), params)); + } + } + } catch { /* ignore parse errors */ } + } + + const whereParts: string[] = []; + if (conditions.length > 0) { + whereParts.push(...conditions); + } + whereParts.push(...filterConditions); + const where = whereParts.length > 0 ? `WHERE ${whereParts.join(" AND ")}` : ""; + + const allIncomeCols = getAllIncomeColumns(); + let orderBy = "ORDER BY data DESC"; + if (sortBy && allIncomeCols.includes(sortBy as string)) { + const dir = sortDir === "asc" ? "ASC" : "DESC"; + orderBy = `ORDER BY "${sortBy as string}" ${dir} NULLS LAST`; + } + + const incomeSelectCols = [...allIncomeCols.map(c => `"${c}"`), `"_override_category_id"`].join(", "); + const [rows, total, sumResult] = await Promise.all([ + query(` + SELECT ${incomeSelectCols} + FROM ${getIncomeEffectiveView()} + ${where} + ${orderBy} + LIMIT ${ps} OFFSET ${offset} + `, params), + query(`SELECT COUNT(*) as cnt FROM ${getIncomeEffectiveView()} ${where}`, params), + query(`SELECT COALESCE(SUM(summa),0) as total_sum FROM ${getIncomeEffectiveView()} ${where}`, params), + ]); + + const totalRows = Number(total.rows[0].cnt); + res.json({ + rows: rows.rows.map(r => { + const row: any = { ...r, data: parseDate(r.data) }; + const ovCatId = r._override_category_id; + if (ovCatId) { + row.overrideCategoryId = ovCatId; + const found = findCategoryById(incomeCats, ovCatId); + row.overrideCategoryName = found ? found.cat.name : ovCatId; + } + delete row._override_category_id; + return row; + }), + total: totalRows, + page: p, + pageSize: ps, + totalPages: Math.ceil(totalRows / ps), + totalSum: Number(sumResult.rows[0].total_sum), + }); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/income/filters", async (_req, res) => { + try { + const [statya, zakazchik, dochka] = await Promise.all([ + query(`SELECT DISTINCT statya FROM ${getIncomeEffectiveView()} WHERE statya IS NOT NULL AND statya != '0' ORDER BY statya`), + query(`SELECT DISTINCT zakazchik FROM ${getIncomeEffectiveView()} WHERE zakazchik IS NOT NULL ORDER BY zakazchik LIMIT 200`), + query(`SELECT DISTINCT dochka FROM ${getIncomeEffectiveView()} WHERE dochka IS NOT NULL AND dochka != '0' ORDER BY dochka`), + ]); + res.json({ + statya: statya.rows.map(r => r.statya), + zakazchik: zakazchik.rows.map(r => r.zakazchik), + dochka: dochka.rows.map(r => r.dochka), + }); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/income-categories", (_req, res) => { + res.json(loadCategories()); + }); + + const VALID_OPERATORS = ["equals", "not_equals", "contains", "greater_than", "less_than", "equal_number"]; + + app.post("/api/income-categories", (req, res) => { + const { name, rules, excluded, parentId, ruleLogic } = req.body; + if (!name || !Array.isArray(rules)) return res.status(400).json({ error: "Укажите name и rules" }); + const err = validateRules(rules, ALLOWED_INCOME_COLUMNS(), VALID_OPERATORS); + if (err) return res.status(400).json({ error: err }); + const cats = loadCategories(); + const newCat: IncomeCategory = { id: generateId(), name, rules, excluded: !!excluded }; + if (ruleLogic === "or") newCat.ruleLogic = "or"; + if (parentId) { + const depth = getCategoryDepth(cats, parentId); + if (depth >= 5) return res.status(400).json({ error: "Максимальная глубина вложенности — 5 уровней" }); + if (!addChildToCategory(cats, parentId, newCat)) return res.status(404).json({ error: "Родительская категория не найдена" }); + } else { + cats.push(newCat); + } + saveCategories(cats); + res.json(newCat); + }); + + app.get("/api/income-categories/auto-detect/config", (_req, res) => { + const config = loadIncomeAutoDetectConfig(); + res.json({ trees: config.trees, allowedColumns: getIncomeAutoDetectColumns(), maxDepth: MAX_TREE_DEPTH }); + }); + + app.post("/api/income-categories/auto-detect/config/trees", (req, res) => { + const { name, columns, autoRefresh } = req.body; + if (!name || typeof name !== "string") return res.status(400).json({ error: "Укажите название дерева" }); + if (!Array.isArray(columns) || columns.length < 1 || columns.length > MAX_TREE_DEPTH) { + return res.status(400).json({ error: `Укажите от 1 до ${MAX_TREE_DEPTH} колонок` }); + } + if (!columns.every((c: string) => getIncomeAutoDetectColumns().includes(c))) { + return res.status(400).json({ error: "Недопустимая колонка" }); + } + const uniqueCols = [...new Set(columns as string[])]; + const config = loadIncomeAutoDetectConfig(); + const newTree: AutoDetectTree = { + id: generateId(), + name: name.trim(), + columns: uniqueCols, + autoRefresh: !!autoRefresh, + isPrimary: true, + }; + config.trees.push(newTree); + saveIncomeAutoDetectConfig(config); + res.json(newTree); + }); + + app.put("/api/income-categories/auto-detect/config/trees/:treeId", (req, res) => { + const { name, columns, autoRefresh, filters, filterLogic, isPrimary, mode } = req.body; + const config = loadIncomeAutoDetectConfig(); + const tree = config.trees.find(t => t.id === req.params.treeId); + if (!tree) return res.status(404).json({ error: "Дерево не найдено" }); + if (name !== undefined) tree.name = String(name).trim(); + if (isPrimary !== undefined) tree.isPrimary = !!isPrimary; + if (mode !== undefined) tree.mode = mode === "merge" ? "merge" : "split"; + if (columns !== undefined) { + if (!Array.isArray(columns) || columns.length < 1 || columns.length > MAX_TREE_DEPTH) { + return res.status(400).json({ error: `Укажите от 1 до ${MAX_TREE_DEPTH} колонок` }); + } + if (!columns.every((c: string) => getIncomeAutoDetectColumns().includes(c))) { + return res.status(400).json({ error: "Недопустимая колонка" }); + } + tree.columns = [...new Set(columns as string[])]; + } + if (autoRefresh !== undefined) tree.autoRefresh = !!autoRefresh; + if (filters !== undefined) { + tree.filters = Array.isArray(filters) ? filters : []; + } + if (filterLogic !== undefined) { + tree.filterLogic = filterLogic === "or" ? "or" : "and"; + } + saveIncomeAutoDetectConfig(config); + if (isPrimary !== undefined) invalidateServerCache(); + res.json(tree); + }); + + app.delete("/api/income-categories/auto-detect/config/trees/:treeId", (_req, res) => { + const config = loadIncomeAutoDetectConfig(); + const idx = config.trees.findIndex(t => t.id === _req.params.treeId); + if (idx < 0) return res.status(404).json({ error: "Дерево не найдено" }); + config.trees.splice(idx, 1); + saveIncomeAutoDetectConfig(config); + res.json({ ok: true }); + }); + + app.get("/api/income-categories/auto-detect/preview", async (req, res) => { + try { + const treeId = req.query.treeId as string | undefined; + const config = loadIncomeAutoDetectConfig(); + const treeCfg = treeId ? config.trees.find(t => t.id === treeId) : config.trees[0]; + if (!treeCfg) return res.status(404).json({ error: "Дерево не найдено" }); + + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildIncomeAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + const existing = loadCategories(); + + let newCount = 0; + function countNew(newCats: IncomeCategory[], existingCats: IncomeCategory[]) { + for (const nc of newCats) { + if (!hasMatchingCategory(existingCats, nc.rules)) newCount++; + if (nc.children) countNew(nc.children, existingCats); + } + } + countNew(tree, existing); + + res.json({ + tree, + treeId: treeCfg.id, + totalInTree: countCategories(tree), + newCount, + existingAutoCount: countAutoDetectedByTree(existing, treeCfg.id), + }); + } catch (err) { + handleApiError(res, err, "Ошибка при анализе данных"); + } + }); + + app.post("/api/income-categories/auto-detect", async (req, res) => { + try { + const treeId = (req.query.treeId || req.body.treeId) as string | undefined; + const config = loadIncomeAutoDetectConfig(); + const treeCfg = treeId ? config.trees.find(t => t.id === treeId) : config.trees[0]; + if (!treeCfg) return res.status(404).json({ error: "Дерево не найдено" }); + + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildIncomeAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + const existing = loadCategories(); + const added = mergeLevel(tree, existing); + saveCategories(existing); + res.json({ ok: true, added, total: countCategories(existing) }); + } catch (err) { + handleApiError(res, err, "Ошибка при создании категорий"); + } + }); + + app.post("/api/income-categories/auto-detect/sync", async (req, res) => { + try { + const treeId = (req.query.treeId || req.body.treeId) as string | undefined; + const config = loadIncomeAutoDetectConfig(); + const treeCfg = treeId ? config.trees.find(t => t.id === treeId) : config.trees[0]; + if (!treeCfg) return res.status(404).json({ error: "Дерево не найдено" }); + + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildIncomeAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + const existing = loadCategories(); + const added = mergeLevel(tree, existing); + saveCategories(existing); + res.json({ ok: true, added, total: countCategories(existing) }); + } catch (err) { + handleApiError(res, err, "Ошибка при синхронизации"); + } + }); + + app.delete("/api/income-categories/auto-detect", (req, res) => { + const treeId = req.query.treeId as string | undefined; + const cats = loadCategories(); + const removed = treeId ? countAutoDetectedByTree(cats, treeId) : countAutoDetected(cats); + const cleaned = treeId ? removeAutoDetectedByTree(cats, treeId) : removeAutoDetected(cats); + saveCategories(cleaned); + res.json({ ok: true, removed }); + }); + + app.put("/api/income-categories/:id", (req, res) => { + const { name, rules, excluded, ruleLogic } = req.body; + if (!name || !Array.isArray(rules)) return res.status(400).json({ error: "Укажите name и rules" }); + const err = validateRules(rules, ALLOWED_INCOME_COLUMNS(), VALID_OPERATORS); + if (err) return res.status(400).json({ error: err }); + const cats = loadCategories(); + const found = findCategoryById(cats, req.params.id); + if (!found) return res.status(404).json({ error: "Не найдено" }); + found.cat.name = name; + found.cat.rules = rules; + found.cat.ruleLogic = ruleLogic === "or" ? "or" : undefined; + if (excluded !== undefined) found.cat.excluded = !!excluded; + saveCategories(cats); + res.json(found.cat); + }); + + app.patch("/api/income-categories/:id/toggle-excluded", (req, res) => { + const cats = loadCategories(); + const found = findCategoryById(cats, req.params.id); + if (!found) return res.status(404).json({ error: "Не найдено" }); + found.cat.excluded = !found.cat.excluded; + saveCategories(cats); + res.json(found.cat); + }); + + app.delete("/api/income-categories/:id", (req, res) => { + const cats = loadCategories(); + if (!findAndRemoveCategory(cats, req.params.id)) return res.status(404).json({ error: "Не найдено" }); + saveCategories(cats); + res.json({ ok: true }); + }); + + app.post("/api/income-categories/:id/extract", (req, res) => { + const cats = loadCategories(); + const { name, includeChildren } = req.body as { name?: string; includeChildren?: boolean }; + if (!name || !name.trim()) return res.status(400).json({ error: "Название обязательно" }); + const found = findCategoryWithAncestors(cats, req.params.id); + if (!found) return res.status(404).json({ error: "Категория не найдена" }); + if (found.ancestors.length === 0) return res.status(400).json({ error: "Категория уже на верхнем уровне" }); + const allRules = [...found.ancestors.flatMap(a => a.rules), ...found.cat.rules]; + const seen = new Set(); + const combinedRules = allRules.filter(r => { + const key = `${r.column}|${r.operator}|${r.value}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + const newCat: IncomeCategory = { + id: generateId(), + name: name.trim(), + rules: combinedRules, + autoDetected: false, + }; + if (includeChildren && found.cat.children && found.cat.children.length > 0) { + newCat.children = deepCopyWithNewIdsManual(found.cat.children); + } + findAndUpdateCategory(cats, req.params.id, (c) => { c.excluded = true; }); + cats.push(newCat); + saveCategories(cats); + res.json(newCat); + }); + + async function autoSyncIncomeCategories(): Promise { + await cachedQuery("__income_auto_sync__", TTL_LONG, async () => { + try { + const config = loadIncomeAutoDetectConfig(); + const existing = loadCategories(); + let totalAdded = 0; + for (const treeCfg of config.trees) { + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildIncomeAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + totalAdded += mergeLevel(tree, existing); + } + if (totalAdded > 0) { + saveCategories(existing); + console.log(`[auto-sync] Added ${totalAdded} new income categories`); + } + } catch (err) { + console.error("[auto-sync] Error syncing income categories:", err); + } + return true; + }); + } + + app.get("/api/income/tree-summary", async (req, res) => { + try { + await ensureTreesReady(); + await autoSyncIncomeCategories(); + const cacheKey = stableKey("income-tree-summary", req.query as Record); + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const { dateFrom, dateTo } = req.query; + const incomeCats = loadCategories(); + + const ruleColumns = new Set(); + function collectRuleColumns(cats: IncomeCategory[]) { + for (const cat of cats) { + for (const rule of cat.rules) { + if (ALLOWED_INCOME_COLUMNS().includes(rule.column) && rule.column !== "data" && rule.column !== "summa") { + ruleColumns.add(rule.column); + } + } + if (cat.children) collectRuleColumns(cat.children); + } + } + collectRuleColumns(incomeCats); + const groupCols = ruleColumns.size > 0 ? [...ruleColumns] : ["statya", "dochka", "vnuchka", "otvetstvenniipokontraktu"]; + + const params: unknown[] = []; + const dateConds: string[] = []; + if (dateFrom) { params.push(dateFrom); dateConds.push(`data::date >= $${params.length}::date`); } + if (dateTo) { params.push(dateTo); dateConds.push(`data::date <= $${params.length}::date`); } + const whereClause = dateConds.length > 0 ? `WHERE ${dateConds.join(" AND ")}` : ""; + + const selectCols = groupCols.join(", "); + const aggResult = await query( + `SELECT ${selectCols}, SUM(summa) as total, COUNT(*) as cnt, MIN(data) as min_date, MAX(data) as max_date + FROM ${getIncomeEffectiveView()} ${whereClause} + GROUP BY ${selectCols}`, + params + ); + + const aggRows: Record[] = aggResult.rows.map(r => { + const row: Record = { total: Number(r.total), cnt: Number(r.cnt), min_date: r.min_date ?? null, max_date: r.max_date ?? null }; + for (const col of groupCols) { + row[col] = r[col] ?? ""; + } + return row; + }); + + function rowMatchesRule(row: Record, rule: { column: string; operator: string; value: string }): boolean { + const val = row[rule.column]; + if (val === undefined) return false; + switch (rule.operator) { + case "equals": return val === rule.value; + case "contains": return typeof val === "string" && matchTokens(val, rule.value); + case "not_equals": return val !== rule.value; + case "greater_than": return Number(val) > Number(rule.value); + case "less_than": return Number(val) < Number(rule.value); + case "equal_number": return Number(val) === Number(rule.value); + default: return val === rule.value; + } + } + + function rowMatchesRules(row: Record, rules: { column: string; operator: string; value: string }[], logic: "and" | "or" = "and"): boolean { + if (logic === "or") return rules.some(rule => rowMatchesRule(row, rule)); + return rules.every(rule => rowMatchesRule(row, rule)); + } + + const allFlat = flattenCategories(incomeCats); + + type AutoPath = { ruleSets: IncomeCategory["rules"][]; logic: ("and" | "or") }; + const autoPaths: AutoPath[] = []; + type ExclPath = { id: string; ruleSets: IncomeCategory["rules"][]; logic: ("and" | "or") }; + const excludedPaths: ExclPath[] = []; + function collectPaths(cats: IncomeCategory[], ancestors: IncomeCategory["rules"][], inAutoTree = false) { + for (const cat of cats) { + const isAuto = !!cat.autoDetected; + const nowInAutoTree = inAutoTree || isAuto; + const path = [...ancestors, cat.rules]; + if (cat.excluded) { + if (!nowInAutoTree) { + excludedPaths.push({ id: cat.id, ruleSets: path, logic: cat.ruleLogic || "and" }); + } + } else if (isAuto) { + autoPaths.push({ ruleSets: path, logic: cat.ruleLogic || "and" }); + } + if (cat.children) collectPaths(cat.children, path, nowInAutoTree); + } + } + collectPaths(incomeCats, []); + + function rowMatchesAutoPath(row: Record, ap: AutoPath): boolean { + for (let i = 0; i < ap.ruleSets.length - 1; i++) { + if (!rowMatchesRules(row, ap.ruleSets[i])) return false; + } + const lastRules = ap.ruleSets[ap.ruleSets.length - 1]; + return rowMatchesRules(row, lastRules, ap.logic); + } + function rowMatchesExclPath(row: Record, ep: ExclPath): boolean { + for (let i = 0; i < ep.ruleSets.length - 1; i++) { + if (!rowMatchesRules(row, ep.ruleSets[i])) return false; + } + return rowMatchesRules(row, ep.ruleSets[ep.ruleSets.length - 1], ep.logic); + } + + function computeNodeFromAgg( + cat: IncomeCategory, + siblings: IncomeCategory[], + ancestorRuleSets: IncomeCategory["rules"][], + ancestorIds: Set = new Set(), + rootManualCats: IncomeCategory[] = [] + ): { id: string; name: string; total: number; count: number; excluded?: boolean; autoDetected?: boolean; minDate?: string; maxDate?: string; children: any[] } { + const allRuleSets = [...ancestorRuleSets, cat.rules]; + const catLogic = cat.ruleLogic || "and"; + const idx = siblings.findIndex(c => c.id === cat.id); + const laterSibs = siblings.slice(idx + 1).filter(c => !c.excluded); + + let total = 0; + let count = 0; + let minDate: Date | null = null; + let maxDate: Date | null = null; + for (const row of aggRows) { + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; + if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue; + if (cat.autoDetected && rootManualCats.length > 0 && rootManualCats.some(mc => rowMatchesRules(row, mc.rules, mc.ruleLogic || "and"))) continue; + if (!cat.autoDetected && excludedPaths.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && rowMatchesExclPath(row, ep))) continue; + if (!cat.autoDetected && !!cat.treeId && autoPaths.some(ap => rowMatchesAutoPath(row, ap) && !isRefinementOf(ap.ruleSets, allRuleSets))) continue; + if (cat.children && cat.children.length > 0 && cat.children.some(child => child.rules.length > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue; + total += row.total; + count += row.cnt; + if (row.min_date) { + const d = new Date(row.min_date); + if (!minDate || d < minDate) minDate = d; + } + if (row.max_date) { + const d = new Date(row.max_date); + if (!maxDate || d > maxDate) maxDate = d; + } + } + + const children: any[] = []; + if (cat.children && cat.children.length > 0) { + const childAncestorIds = new Set(ancestorIds); + childAncestorIds.add(cat.id); + for (const child of cat.children) { + children.push(computeNodeFromAgg(child, cat.children, allRuleSets, childAncestorIds, rootManualCats)); + } + } + + return { + id: cat.id, + name: cat.name, + total, + count, + excluded: cat.excluded || undefined, + autoDetected: cat.autoDetected || undefined, + minDate: minDate ? minDate.toISOString().slice(0, 10) : undefined, + maxDate: maxDate ? maxDate.toISOString().slice(0, 10) : undefined, + children, + }; + } + + const config = loadIncomeAutoDetectConfig(); + + const groupedByTree = new Map(); + for (const cat of incomeCats) { + const tid = cat.treeId || "__manual__"; + if (!groupedByTree.has(tid)) groupedByTree.set(tid, []); + groupedByTree.get(tid)!.push(cat); + } + + const trees: { id: string; name: string; nodes: any[]; total: number; count: number; isPrimary?: boolean }[] = []; + + const configTreeMap = new Map(config.trees.map(t => [t.id, t])); + const configTreeIds = new Set(config.trees.map((t: any) => t.id)); + const primaryTreeIds = new Set(config.trees.filter(t => t.isPrimary).map(t => t.id)); + const orphanCats: IncomeCategory[] = []; + const manualCats: IncomeCategory[] = []; + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__") { + manualCats.push(...cats); + } else if (!configTreeIds.has(tid)) { + orphanCats.push(...cats); + } + } + const rootManualCatsForExclInc = manualCats.filter(c => !c.autoDetected && !c.excluded); + type ManualWithAncestors = { cat: IncomeCategory; ancestorRuleSets: IncomeCategory["rules"][] }; + function collectNestedManualCatsInc(cats: IncomeCategory[], ancestorRules: IncomeCategory["rules"][] = []): ManualWithAncestors[] { + const result: ManualWithAncestors[] = []; + for (const cat of cats) { + if (!cat.autoDetected) result.push({ cat, ancestorRuleSets: ancestorRules }); + const nextAncestors = [...ancestorRules, cat.rules]; + if (cat.children) result.push(...collectNestedManualCatsInc(cat.children, nextAncestors)); + } + return result; + } + const rootManualIdsInc = new Set(manualCats.map(c => c.id)); + const nestedManualWithAncestors: ManualWithAncestors[] = []; + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__" || !configTreeIds.has(tid)) continue; + for (const cat of cats) { + const nested = collectNestedManualCatsInc(cat.children || [], [cat.rules]); + for (const n of nested) { + if (!rootManualIdsInc.has(n.cat.id)) { + nestedManualWithAncestors.push(n); + manualCats.push(n.cat); + } + } + } + } + const nestedManualAncestorMap = new Map(); + for (const n of nestedManualWithAncestors) { + nestedManualAncestorMap.set(n.cat.id, n.ancestorRuleSets); + } + function nodeSubtreeTotal(n: any): number { + if (n.excluded) { + let t = 0; + for (const ch of n.children) t += nodeSubtreeTotal(ch); + return t; + } + let t = n.total; + for (const ch of n.children) t += nodeSubtreeTotal(ch); + return t; + } + function nodeSubtreeCount(n: any): number { + if (n.excluded) { + let c = 0; + for (const ch of n.children) c += nodeSubtreeCount(ch); + return c; + } + let c = n.count; + for (const ch of n.children) c += nodeSubtreeCount(ch); + return c; + } + + const otherCats = [...manualCats, ...orphanCats]; + if (otherCats.length > 0) { + const nodes: any[] = []; + for (const cat of otherCats) { + const ancestorRules = nestedManualAncestorMap.get(cat.id) || []; + nodes.push(computeNodeFromAgg(cat, otherCats, ancestorRules, new Set())); + } + const treeTotal = nodes.reduce((s: number, n: any) => s + nodeSubtreeTotal(n), 0); + const treeCount = nodes.reduce((s: number, n: any) => s + nodeSubtreeCount(n), 0); + trees.push({ id: "__manual__", name: "Ручные категории", nodes, total: treeTotal, count: treeCount, isPrimary: true }); + } + + for (const treeCfg of config.trees) { + const cats = groupedByTree.get(treeCfg.id) || []; + const nodes: any[] = []; + for (const cat of cats) { + nodes.push(computeNodeFromAgg(cat, cats, [], new Set(), rootManualCatsForExclInc)); + } + const treeTotal = nodes.reduce((s: number, n: any) => s + nodeSubtreeTotal(n), 0); + const treeCount = nodes.reduce((s: number, n: any) => s + nodeSubtreeCount(n), 0); + trees.push({ id: treeCfg.id, name: treeCfg.name, nodes, total: treeTotal, count: treeCount, isPrimary: treeCfg.isPrimary || false }); + } + + function isRowMatchedByTree(row: Record, treeCats: IncomeCategory[], ancestorRuleSets: IncomeCategory["rules"][] = []): boolean { + for (const cat of treeCats) { + const catLogic = cat.ruleLogic || "and"; + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; + return true; + } + return false; + } + let uncatTotal = 0; + let uncatCount = 0; + for (const row of aggRows) { + let matched = false; + for (const [tid, treeCats] of groupedByTree) { + const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); + if (!isPrimary) continue; + if (isRowMatchedByTree(row, treeCats)) { matched = true; break; } + } + if (!matched) { + uncatTotal += row.total; + uncatCount += row.cnt; + } + } + + let grandTotal = 0; + let grandCount = 0; + for (const row of aggRows) { + grandTotal += row.total; + grandCount += row.cnt; + } + + return { + trees, + uncategorized: { total: uncatTotal, count: uncatCount }, + grandTotal: { total: grandTotal, count: grandCount }, + }; + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/income/tree-timeseries", async (req, res) => { + try { + await ensureTreesReady(); + await autoSyncIncomeCategories(); + const cacheKey = stableKey("income-tree-timeseries", req.query as Record); + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const { dateFrom, dateTo, groupBy = "month", categoryId } = req.query; + const validGroupBy = ["day", "month", "quarter", "year"]; + const gb = validGroupBy.includes(groupBy as string) ? (groupBy as string) : "month"; + + const incomeCats = loadCategories(); + const ruleColumns = new Set(); + function collectRuleColumns(cats: IncomeCategory[]) { + for (const cat of cats) { + for (const rule of cat.rules) { + if (ALLOWED_INCOME_COLUMNS().includes(rule.column) && rule.column !== "data" && rule.column !== "summa") { + ruleColumns.add(rule.column); + } + } + if (cat.children) collectRuleColumns(cat.children); + } + } + collectRuleColumns(incomeCats); + const groupCols = ruleColumns.size > 0 ? [...ruleColumns] : ["statya", "dochka", "vnuchka", "otvetstvenniipokontraktu"]; + + const params: unknown[] = []; + const dateConds: string[] = []; + if (dateFrom) { params.push(dateFrom); dateConds.push(`data::date >= $${params.length}::date`); } + if (dateTo) { params.push(dateTo); dateConds.push(`data::date <= $${params.length}::date`); } + const whereClause = dateConds.length > 0 ? `WHERE ${dateConds.join(" AND ")}` : ""; + + const selectCols = groupCols.join(", "); + const aggResult = await query( + `SELECT ${selectCols}, DATE_TRUNC('${gb}', data) as period, SUM(summa) as total, COUNT(*) as cnt + FROM ${getIncomeEffectiveView()} ${whereClause} + GROUP BY ${selectCols}, DATE_TRUNC('${gb}', data)`, + params + ); + + const aggRows: Record[] = aggResult.rows.map(r => { + const row: Record = { total: Number(r.total), cnt: Number(r.cnt), period: r.period }; + for (const col of groupCols) { row[col] = r[col] ?? ""; } + return row; + }); + + function rowMatchesRule(row: Record, rule: { column: string; operator: string; value: string }): boolean { + const val = row[rule.column]; + if (val === undefined) return false; + switch (rule.operator) { + case "equals": return val === rule.value; + case "contains": return typeof val === "string" && matchTokens(val, rule.value); + case "not_equals": return val !== rule.value; + case "greater_than": return Number(val) > Number(rule.value); + case "less_than": return Number(val) < Number(rule.value); + case "equal_number": return Number(val) === Number(rule.value); + default: return val === rule.value; + } + } + function rowMatchesRules(row: Record, rules: { column: string; operator: string; value: string }[], logic: "and" | "or" = "and"): boolean { + if (logic === "or") return rules.some(rule => rowMatchesRule(row, rule)); + return rules.every(rule => rowMatchesRule(row, rule)); + } + + const allFlat = flattenCategories(incomeCats); + + type AutoPathTS = { ruleSets: IncomeCategory["rules"][]; logic: ("and" | "or") }; + const autoPathsTS: AutoPathTS[] = []; + type ExclPathTS = { id: string; ruleSets: IncomeCategory["rules"][]; logic: ("and" | "or") }; + const excludedPathsTS: ExclPathTS[] = []; + function collectPathsTS(cats: IncomeCategory[], ancestors: IncomeCategory["rules"][], inAutoTree = false) { + for (const cat of cats) { + const isAuto = !!cat.autoDetected; + const nowInAutoTree = inAutoTree || isAuto; + const path = [...ancestors, cat.rules]; + if (cat.excluded) { + if (!nowInAutoTree) { + excludedPathsTS.push({ id: cat.id, ruleSets: path, logic: cat.ruleLogic || "and" }); + } + } else if (isAuto) { + autoPathsTS.push({ ruleSets: path, logic: cat.ruleLogic || "and" }); + } + if (cat.children) collectPathsTS(cat.children, path, nowInAutoTree); + } + } + collectPathsTS(incomeCats, []); + + function rowMatchesAutoPathTS(row: Record, ap: AutoPathTS): boolean { + for (let i = 0; i < ap.ruleSets.length - 1; i++) { + if (!rowMatchesRules(row, ap.ruleSets[i])) return false; + } + return rowMatchesRules(row, ap.ruleSets[ap.ruleSets.length - 1], ap.logic); + } + + const dataPeriods = new Set(); + for (const row of aggRows) { + if (row.period) dataPeriods.add(new Date(row.period).toISOString()); + } + const dataPeriodsSorted = [...dataPeriods].sort(); + let sortedPeriods = dataPeriodsSorted; + if (dataPeriodsSorted.length >= 2) { + const filled: string[] = []; + const minD = new Date(dataPeriodsSorted[0]); + const maxD = new Date(dataPeriodsSorted[dataPeriodsSorted.length - 1]); + const cur = new Date(minD); + while (cur <= maxD) { + filled.push(cur.toISOString()); + if (gb === "day") cur.setUTCDate(cur.getUTCDate() + 1); + else if (gb === "month") cur.setUTCMonth(cur.getUTCMonth() + 1); + else if (gb === "quarter") cur.setUTCMonth(cur.getUTCMonth() + 3); + else cur.setUTCFullYear(cur.getUTCFullYear() + 1); + } + sortedPeriods = filled; + } + + function formatPeriodLabel(iso: string): string { + const d = new Date(iso); + if (gb === "day") return d.toISOString().slice(0, 10); + if (gb === "month") return d.toISOString().slice(0, 7); + if (gb === "quarter") { const q = Math.ceil((d.getMonth() + 1) / 3); return `${d.getFullYear()}-Q${q}`; } + return `${d.getFullYear()}`; + } + const periodLabels = sortedPeriods.map(formatPeriodLabel); + const periodIndexMap = new Map(sortedPeriods.map((p, i) => [p, i])); + + function computeCatTimeSeries( + cat: IncomeCategory, + siblings: IncomeCategory[], + ancestorRuleSets: IncomeCategory["rules"][], + ancestorIds: Set = new Set(), + rootManualCats: IncomeCategory[] = [] + ): number[] { + const allRuleSets = [...ancestorRuleSets, cat.rules]; + const catLogic = cat.ruleLogic || "and"; + const idx = siblings.findIndex(c => c.id === cat.id); + const laterSibs = siblings.slice(idx + 1).filter(c => !c.excluded); + const data = new Array(sortedPeriods.length).fill(0); + for (const row of aggRows) { + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; + if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue; + if (cat.autoDetected && rootManualCats.length > 0 && rootManualCats.some(mc => rowMatchesRules(row, mc.rules, mc.ruleLogic || "and"))) continue; + if (!cat.autoDetected && excludedPathsTS.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue; + if (!cat.autoDetected && !!cat.treeId && autoPathsTS.some(ap => rowMatchesAutoPathTS(row, ap) && !isRefinementOf(ap.ruleSets, allRuleSets))) continue; + if (cat.children && cat.children.length > 0 && cat.children.some(child => child.rules.length > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue; + const pi = periodIndexMap.get(new Date(row.period).toISOString()); + if (pi !== undefined) data[pi] += row.total; + } + return data; + } + + function subtreeTimeSeries( + cat: IncomeCategory, + siblings: IncomeCategory[], + ancestorRuleSets: IncomeCategory["rules"][], + ancestorIds: Set = new Set(), + rootManualCats: IncomeCategory[] = [] + ): number[] { + const own = computeCatTimeSeries(cat, siblings, ancestorRuleSets, ancestorIds, rootManualCats); + if (!cat.children || cat.children.length === 0) return own; + const allRuleSets = [...ancestorRuleSets, cat.rules]; + const newAncestorIds = new Set([...ancestorIds, cat.id]); + for (const child of cat.children) { + if (child.excluded) continue; + const childData = subtreeTimeSeries(child, cat.children, allRuleSets, newAncestorIds, rootManualCats); + for (let i = 0; i < own.length; i++) own[i] += childData[i]; + } + return own; + } + + const config = loadIncomeAutoDetectConfig(); + const groupedByTree = new Map(); + for (const cat of incomeCats) { + const tid = cat.treeId || "__manual__"; + if (!groupedByTree.has(tid)) groupedByTree.set(tid, []); + groupedByTree.get(tid)!.push(cat); + } + + const drillRootManualInc: IncomeCategory[] = []; + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__") drillRootManualInc.push(...cats.filter(c => !c.autoDetected && !c.excluded)); + } + + if (categoryId && typeof categoryId === "string") { + const found = findCategoryWithAncestors(incomeCats, categoryId); + if (!found || !found.cat.children || found.cat.children.length === 0) { + return { trees: [], drilldown: true }; + } + const ancestorRuleSets = [...found.ancestors.map(a => a.rules), found.cat.rules]; + const drillAncestorIds = new Set([...found.ancestors.map(a => a.id), found.cat.id]); + const children = found.cat.children; + const isAutoTree = found.cat.autoDetected || found.ancestors.some(a => a.autoDetected); + const drillManual = isAutoTree ? drillRootManualInc : []; + const series: any[] = children.filter(c => !c.excluded).map(cat => ({ + id: cat.id, name: cat.name, + hasChildren: !!(cat.children && cat.children.length > 0), + data: subtreeTimeSeries(cat, children, ancestorRuleSets, drillAncestorIds, drillManual) + })).filter(s => s.data.some(v => v !== 0)); + const treeId = found.cat.treeId || found.ancestors[0]?.treeId || "__drilldown__"; + return { trees: [{ id: treeId, name: found.cat.name, isPrimary: true, periods: periodLabels, series }], drilldown: true }; + } + + const trees: any[] = []; + const configTreeIds = new Set(config.trees.map((t: any) => t.id)); + const orphanCats: IncomeCategory[] = []; + const manualCats: IncomeCategory[] = []; + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__") manualCats.push(...cats); + else if (!configTreeIds.has(tid)) orphanCats.push(...cats); + } + type ManualWithAncestorsTrInc = { cat: IncomeCategory; ancestorRuleSets: IncomeCategory["rules"][] }; + function collectNestedManualTrInc(cats: IncomeCategory[], ancestorRules: IncomeCategory["rules"][] = []): ManualWithAncestorsTrInc[] { + const result: ManualWithAncestorsTrInc[] = []; + for (const cat of cats) { + if (!cat.autoDetected) result.push({ cat, ancestorRuleSets: ancestorRules }); + const nextAncestors = [...ancestorRules, cat.rules]; + if (cat.children) result.push(...collectNestedManualTrInc(cat.children, nextAncestors)); + } + return result; + } + const rootManualCatsForExclTrInc = manualCats.filter(c => !c.autoDetected && !c.excluded); + const rmIdsTrInc = new Set(manualCats.map(c => c.id)); + const trendAncestorMapInc = new Map(); + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__" || !configTreeIds.has(tid)) continue; + for (const cat of cats) { + const nested = collectNestedManualTrInc(cat.children || [], [cat.rules]); + for (const n of nested) { + if (!rmIdsTrInc.has(n.cat.id)) { + trendAncestorMapInc.set(n.cat.id, n.ancestorRuleSets); + manualCats.push(n.cat); + } + } + } + } + const otherCats = [...manualCats, ...orphanCats]; + if (otherCats.length > 0) { + const series = otherCats.filter(c => !c.excluded).map(cat => ({ + id: cat.id, name: cat.name, + hasChildren: !!(cat.children && cat.children.length > 0), + data: subtreeTimeSeries(cat, otherCats, trendAncestorMapInc.get(cat.id) || [], new Set()) + })).filter(s => s.data.some(v => v !== 0)); + trees.push({ id: "__manual__", name: "Ручные категории", isPrimary: true, periods: periodLabels, series }); + } + + for (const treeCfg of config.trees) { + const cats = groupedByTree.get(treeCfg.id) || []; + const series = cats.filter(c => !c.excluded).map(cat => ({ + id: cat.id, name: cat.name, + hasChildren: !!(cat.children && cat.children.length > 0), + data: subtreeTimeSeries(cat, cats, [], new Set(), rootManualCatsForExclTrInc) + })).filter(s => s.data.some(v => v !== 0)); + trees.push({ id: treeCfg.id, name: treeCfg.name, isPrimary: treeCfg.isPrimary || false, periods: periodLabels, series }); + } + + return { trees }; + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/expense-categories", (_req, res) => { + res.json(loadExpenseCategories()); + }); + + app.post("/api/expense-categories", (req, res) => { + const { name, rules, excluded, parentId, ruleLogic } = req.body; + if (!name || !Array.isArray(rules)) return res.status(400).json({ error: "Укажите name и rules" }); + const err = validateRules(rules, ALLOWED_EXPENSE_COLUMNS(), VALID_OPERATORS); + if (err) return res.status(400).json({ error: err }); + const cats = loadExpenseCategories(); + const newCat: IncomeCategory = { id: generateId(), name, rules, excluded: !!excluded }; + if (ruleLogic === "or") newCat.ruleLogic = "or"; + if (parentId) { + const depth = getCategoryDepth(cats, parentId); + if (depth >= 5) return res.status(400).json({ error: "Максимальная глубина вложенности — 5 уровней" }); + if (!addChildToCategory(cats, parentId, newCat)) return res.status(404).json({ error: "Родительская категория не найдена" }); + } else { + cats.push(newCat); + } + saveExpenseCategories(cats); + res.json(newCat); + }); + + app.get("/api/expense-categories/auto-detect/config", (_req, res) => { + const config = loadAutoDetectConfig(); + res.json({ trees: config.trees, allowedColumns: getExpenseAutoDetectColumns(), maxDepth: MAX_TREE_DEPTH }); + }); + + app.post("/api/expense-categories/auto-detect/config/trees", (req, res) => { + const { name, columns, autoRefresh } = req.body; + if (!name || typeof name !== "string") return res.status(400).json({ error: "Укажите название дерева" }); + if (!Array.isArray(columns) || columns.length < 1 || columns.length > MAX_TREE_DEPTH) { + return res.status(400).json({ error: `Укажите от 1 до ${MAX_TREE_DEPTH} колонок` }); + } + if (!columns.every((c: string) => getExpenseAutoDetectColumns().includes(c))) { + return res.status(400).json({ error: "Недопустимая колонка" }); + } + const uniqueCols = [...new Set(columns as string[])]; + const config = loadAutoDetectConfig(); + const newTree: AutoDetectTree = { + id: generateId(), + name: name.trim(), + columns: uniqueCols, + autoRefresh: !!autoRefresh, + isPrimary: true, + }; + config.trees.push(newTree); + saveAutoDetectConfig(config); + res.json(newTree); + }); + + app.put("/api/expense-categories/auto-detect/config/trees/:treeId", (req, res) => { + const { name, columns, autoRefresh, filters, filterLogic, isPrimary, mode } = req.body; + const config = loadAutoDetectConfig(); + const tree = config.trees.find(t => t.id === req.params.treeId); + if (!tree) return res.status(404).json({ error: "Дерево не найдено" }); + if (name !== undefined) tree.name = String(name).trim(); + if (isPrimary !== undefined) tree.isPrimary = !!isPrimary; + if (mode !== undefined) tree.mode = mode === "merge" ? "merge" : "split"; + if (columns !== undefined) { + if (!Array.isArray(columns) || columns.length < 1 || columns.length > MAX_TREE_DEPTH) { + return res.status(400).json({ error: `Укажите от 1 до ${MAX_TREE_DEPTH} колонок` }); + } + if (!columns.every((c: string) => getExpenseAutoDetectColumns().includes(c))) { + return res.status(400).json({ error: "Недопустимая колонка" }); + } + tree.columns = [...new Set(columns as string[])]; + } + if (autoRefresh !== undefined) tree.autoRefresh = !!autoRefresh; + if (filters !== undefined) { + tree.filters = Array.isArray(filters) ? filters : []; + } + if (filterLogic !== undefined) { + tree.filterLogic = filterLogic === "or" ? "or" : "and"; + } + saveAutoDetectConfig(config); + if (isPrimary !== undefined) invalidateServerCache(); + res.json(tree); + }); + + app.delete("/api/expense-categories/auto-detect/config/trees/:treeId", (_req, res) => { + const config = loadAutoDetectConfig(); + const idx = config.trees.findIndex(t => t.id === _req.params.treeId); + if (idx < 0) return res.status(404).json({ error: "Дерево не найдено" }); + config.trees.splice(idx, 1); + saveAutoDetectConfig(config); + res.json({ ok: true }); + }); + + app.get("/api/expense-categories/auto-detect/preview", async (req, res) => { + try { + const treeId = req.query.treeId as string | undefined; + const config = loadAutoDetectConfig(); + const treeCfg = treeId ? config.trees.find(t => t.id === treeId) : config.trees[0]; + if (!treeCfg) return res.status(404).json({ error: "Дерево не найдено" }); + + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + const existing = loadExpenseCategories(); + + let newCount = 0; + function countNew(newCats: IncomeCategory[], existingCats: IncomeCategory[]) { + for (const nc of newCats) { + if (!hasMatchingCategory(existingCats, nc.rules)) newCount++; + if (nc.children) countNew(nc.children, existingCats); + } + } + countNew(tree, existing); + + res.json({ + tree, + treeId: treeCfg.id, + totalInTree: countCategories(tree), + newCount, + existingAutoCount: countAutoDetectedByTree(existing, treeCfg.id), + }); + } catch (err) { + handleApiError(res, err, "Ошибка при анализе данных"); + } + }); + + app.post("/api/expense-categories/auto-detect", async (req, res) => { + try { + const treeId = (req.query.treeId || req.body.treeId) as string | undefined; + const config = loadAutoDetectConfig(); + const treeCfg = treeId ? config.trees.find(t => t.id === treeId) : config.trees[0]; + if (!treeCfg) return res.status(404).json({ error: "Дерево не найдено" }); + + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + const existing = loadExpenseCategories(); + const added = mergeLevel(tree, existing); + saveExpenseCategories(existing); + res.json({ ok: true, added, total: countCategories(existing) }); + } catch (err) { + handleApiError(res, err, "Ошибка при создании категорий"); + } + }); + + app.post("/api/expense-categories/auto-detect/sync", async (req, res) => { + try { + const treeId = (req.query.treeId || req.body.treeId) as string | undefined; + const config = loadAutoDetectConfig(); + const treeCfg = treeId ? config.trees.find(t => t.id === treeId) : config.trees[0]; + if (!treeCfg) return res.status(404).json({ error: "Дерево не найдено" }); + + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + const existing = loadExpenseCategories(); + const added = mergeLevel(tree, existing); + saveExpenseCategories(existing); + res.json({ ok: true, added, total: countCategories(existing) }); + } catch (err) { + handleApiError(res, err, "Ошибка при синхронизации"); + } + }); + + app.delete("/api/expense-categories/auto-detect", (req, res) => { + const treeId = req.query.treeId as string | undefined; + const cats = loadExpenseCategories(); + const removed = treeId ? countAutoDetectedByTree(cats, treeId) : countAutoDetected(cats); + const cleaned = treeId ? removeAutoDetectedByTree(cats, treeId) : removeAutoDetected(cats); + saveExpenseCategories(cleaned); + res.json({ ok: true, removed }); + }); + + app.put("/api/expense-categories/:id", (req, res) => { + const { name, rules, excluded, ruleLogic } = req.body; + if (!name || !Array.isArray(rules)) return res.status(400).json({ error: "Укажите name и rules" }); + const err = validateRules(rules, ALLOWED_EXPENSE_COLUMNS(), VALID_OPERATORS); + if (err) return res.status(400).json({ error: err }); + const cats = loadExpenseCategories(); + const found = findCategoryById(cats, req.params.id); + if (!found) return res.status(404).json({ error: "Не найдено" }); + found.cat.name = name; + found.cat.rules = rules; + found.cat.ruleLogic = ruleLogic === "or" ? "or" : undefined; + if (excluded !== undefined) found.cat.excluded = !!excluded; + saveExpenseCategories(cats); + res.json(found.cat); + }); + + app.patch("/api/expense-categories/:id/toggle-excluded", (req, res) => { + const cats = loadExpenseCategories(); + const found = findCategoryById(cats, req.params.id); + if (!found) return res.status(404).json({ error: "Не найдено" }); + found.cat.excluded = !found.cat.excluded; + saveExpenseCategories(cats); + res.json(found.cat); + }); + + app.delete("/api/expense-categories/:id", (req, res) => { + const cats = loadExpenseCategories(); + if (!findAndRemoveCategory(cats, req.params.id)) return res.status(404).json({ error: "Не найдено" }); + saveExpenseCategories(cats); + res.json({ ok: true }); + }); + + app.post("/api/expense-categories/:id/extract", (req, res) => { + const cats = loadExpenseCategories(); + const { name, includeChildren } = req.body as { name?: string; includeChildren?: boolean }; + if (!name || !name.trim()) return res.status(400).json({ error: "Название обязательно" }); + const found = findCategoryWithAncestors(cats, req.params.id); + if (!found) return res.status(404).json({ error: "Категория не найдена" }); + if (found.ancestors.length === 0) return res.status(400).json({ error: "Категория уже на верхнем уровне" }); + const allRules = [...found.ancestors.flatMap(a => a.rules), ...found.cat.rules]; + const seen = new Set(); + const combinedRules = allRules.filter(r => { + const key = `${r.column}|${r.operator}|${r.value}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + const newCat: IncomeCategory = { + id: generateId(), + name: name.trim(), + rules: combinedRules, + autoDetected: false, + }; + if (includeChildren && found.cat.children && found.cat.children.length > 0) { + newCat.children = deepCopyWithNewIdsManual(found.cat.children); + } + findAndUpdateCategory(cats, req.params.id, (c) => { c.excluded = true; }); + cats.push(newCat); + saveExpenseCategories(cats); + res.json(newCat); + }); + + async function buildAutoDetectTree(cols: string[], treeId: string, filters?: TreeFilter[], filterLogic?: string): Promise { + const selectCols = cols.map(c => `"${c}"`).join(", "); + const orderCols = cols.map(c => `"${c}"`).join(", "); + const params: unknown[] = []; + const filterClause = buildFilterSQL(filters, filterLogic, params, getExpenseAutoDetectColumns()); + const sql = `SELECT DISTINCT ${selectCols} FROM ${getExpenseEffectiveView()} WHERE "${cols[0]}" IS NOT NULL AND "${cols[0]}" != '0'${filterClause} ORDER BY ${orderCols}`; + const result = await query(sql, params); + + type NestedMap = Map; + const rootMap: NestedMap = new Map(); + + for (const row of result.rows) { + const values: (string | null)[] = cols.map((c, i) => { + const v = row[c] as string | null; + return (v && v !== "0") ? v : (i === 0 ? null : null); + }); + if (!values[0]) continue; + + let currentMap = rootMap; + for (let i = 0; i < values.length; i++) { + const val = values[i]; + if (!val) break; + if (!currentMap.has(val)) currentMap.set(val, new Map()); + currentMap = currentMap.get(val)!; + } + } + + function buildLevel(map: NestedMap, colIndex: number): IncomeCategory[] { + const cats: IncomeCategory[] = []; + for (const [name, childMap] of map) { + const cat: IncomeCategory = { + id: generateId(), + name, + rules: [{ column: cols[colIndex], operator: "equals", value: name }], + autoDetected: true, + treeId, + }; + if (colIndex < cols.length - 1 && childMap.size > 0) { + cat.children = buildLevel(childMap, colIndex + 1); + } + cats.push(cat); + } + return cats; + } + + return buildLevel(rootMap, 0); + } + + function mergeLevel(newCats: IncomeCategory[], existingCats: IncomeCategory[]): number { + let added = 0; + for (const nc of newCats) { + const match = existingCats.find(ec => rulesMatch(ec.rules, nc.rules)); + if (match) { + if (nc.children && nc.children.length > 0) { + if (!match.children) match.children = []; + added += mergeLevel(nc.children, match.children); + } + } else { + existingCats.push(nc); + added += countCategories([nc]); + } + } + return added; + } + + function hasMatchingCategory(existing: IncomeCategory[], rules: IncomeCategory["rules"]): boolean { + for (const cat of existing) { + if (rulesMatch(cat.rules, rules)) return true; + if (cat.children && hasMatchingCategory(cat.children, rules)) return true; + } + return false; + } + + function rulesMatch(a: IncomeCategory["rules"], b: IncomeCategory["rules"]): boolean { + if (a.length !== b.length) return false; + return a.every((r, i) => r.column === b[i].column && r.operator === b[i].operator && r.value === b[i].value); + } + + function countCategories(cats: IncomeCategory[]): number { + let count = 0; + for (const cat of cats) { + count++; + if (cat.children) count += countCategories(cat.children); + } + return count; + } + + function countAutoDetected(cats: IncomeCategory[]): number { + let count = 0; + for (const cat of cats) { + if (cat.autoDetected) count++; + if (cat.children) count += countAutoDetected(cat.children); + } + return count; + } + + function countAutoDetectedByTree(cats: IncomeCategory[], treeId: string): number { + let count = 0; + for (const cat of cats) { + if (cat.autoDetected && cat.treeId === treeId) count++; + if (cat.children) count += countAutoDetectedByTree(cat.children, treeId); + } + return count; + } + + function removeAutoDetected(cats: IncomeCategory[]): IncomeCategory[] { + return cats.filter(c => { + if (c.children) c.children = removeAutoDetected(c.children); + return !c.autoDetected; + }); + } + + function removeAutoDetectedByTree(cats: IncomeCategory[], treeId: string): IncomeCategory[] { + return cats.filter(c => { + if (c.children) c.children = removeAutoDetectedByTree(c.children, treeId); + return !(c.autoDetected && c.treeId === treeId); + }); + } + + app.get("/api/profitability", async (req, res) => { + try { + const { dateFrom, dateTo, search, page = "1", pageSize = "50" } = req.query; + const p = Number(page); + const ps = Math.min(Number(pageSize), 200); + const offset = (p - 1) * ps; + + const rawCacheKey = stableKey("profitability:raw", { dateFrom: dateFrom ?? "", dateTo: dateTo ?? "" } as Record); + const allRows = await cachedQuery<{ zakazchik: string; totalIncome: number; totalExpense: number; profit: number; margin: number; contractCount: number }[]>( + rawCacheKey, TTL_LONG, async () => { + const incConditions: string[] = ["zakazchik != '0'"]; + const expConditions: string[] = ["zakazchik != '0'"]; + const incParams: unknown[] = []; + const expParams: unknown[] = []; + + const globalExcl = buildGlobalExclusionConditions(loadCategories(), incParams); + incConditions.push(...globalExcl); + + if (dateFrom) { + incParams.push(dateFrom); incConditions.push(`data >= $${incParams.length}`); + expParams.push(dateFrom); expConditions.push(`data >= $${expParams.length}`); + } + if (dateTo) { + incParams.push(dateTo); incConditions.push(`data <= $${incParams.length}`); + expParams.push(dateTo); expConditions.push(`data <= $${expParams.length}`); + } + + const incWhere = `WHERE ${incConditions.join(" AND ")}`; + const expWhere = `WHERE ${expConditions.join(" AND ")}`; + + const [incomeData, expenseData] = await Promise.all([ + query(` + SELECT zakazchik, COALESCE(SUM(summa),0) as total, COUNT(DISTINCT dogovorzakazchika) as contracts + FROM ${getIncomeEffectiveView()} ${incWhere} + GROUP BY zakazchik + `, incParams), + query(` + SELECT zakazchik, COALESCE(SUM(summadokumenta),0) as total + FROM ${getExpenseEffectiveView()} ${expWhere} + GROUP BY zakazchik + `, expParams), + ]); + + const expMap = new Map(); + expenseData.rows.forEach((r: any) => expMap.set(r.zakazchik, Number(r.total))); + + const incSet = new Set(); + const combined: { zakazchik: string; totalIncome: number; totalExpense: number; profit: number; margin: number; contractCount: number }[] = []; + + incomeData.rows.forEach((r: any) => { + incSet.add(r.zakazchik); + const totalIncome = Number(r.total); + const totalExpense = expMap.get(r.zakazchik) ?? 0; + const profit = totalIncome - totalExpense; + combined.push({ + zakazchik: r.zakazchik, + totalIncome, + totalExpense, + profit, + margin: totalIncome > 0 ? Math.round((profit / totalIncome) * 10000) / 100 : 0, + contractCount: Number(r.contracts), + }); + }); + + expenseData.rows.forEach((r: any) => { + if (!incSet.has(r.zakazchik)) { + const totalExpense = Number(r.total); + combined.push({ + zakazchik: r.zakazchik, + totalIncome: 0, + totalExpense, + profit: -totalExpense, + margin: 0, + contractCount: 0, + }); + } + }); + + combined.sort((a, b) => b.totalIncome - a.totalIncome); + return combined; + } + ); + + let rows = allRows; + if (search) { + const s = String(search).toLowerCase(); + rows = rows.filter(r => r.zakazchik.toLowerCase().includes(s)); + } + + const total = rows.length; + const summary = { + totalIncome: rows.reduce((s, r) => s + r.totalIncome, 0), + totalExpense: rows.reduce((s, r) => s + r.totalExpense, 0), + totalProfit: rows.reduce((s, r) => s + r.profit, 0), + counterpartCount: total, + }; + + const paged = rows.slice(offset, offset + ps); + + res.json({ + rows: paged, + total, + page: p, + pageSize: ps, + totalPages: Math.ceil(total / ps), + summary, + }); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/profitability/monthly", async (req, res) => { + try { + const { dateFrom, dateTo, zakazchik } = req.query; + + const incConditions: string[] = ["zakazchik != '0'"]; + const expConditions: string[] = ["zakazchik != '0'"]; + const incParams: unknown[] = []; + const expParams: unknown[] = []; + + const mExcl = buildGlobalExclusionConditions(loadCategories(), incParams); + incConditions.push(...mExcl); + + if (dateFrom) { + incParams.push(dateFrom); incConditions.push(`data >= $${incParams.length}`); + expParams.push(dateFrom); expConditions.push(`data >= $${expParams.length}`); + } + if (dateTo) { + incParams.push(dateTo); incConditions.push(`data <= $${incParams.length}`); + expParams.push(dateTo); expConditions.push(`data <= $${expParams.length}`); + } + if (zakazchik) { + incParams.push(zakazchik); incConditions.push(`zakazchik = $${incParams.length}`); + expParams.push(zakazchik); expConditions.push(`zakazchik = $${expParams.length}`); + } + + const incWhere = `WHERE ${incConditions.join(" AND ")}`; + const expWhere = `WHERE ${expConditions.join(" AND ")}`; + + const [incomeMonthly, expenseMonthly] = await Promise.all([ + query(` + SELECT DATE_TRUNC('month', data) as month, COALESCE(SUM(summa),0) as total + FROM ${getIncomeEffectiveView()} ${incWhere} + GROUP BY DATE_TRUNC('month', data) + ORDER BY month + `, incParams), + query(` + SELECT DATE_TRUNC('month', data) as month, COALESCE(SUM(summadokumenta),0) as total + FROM ${getExpenseEffectiveView()} ${expWhere} + GROUP BY DATE_TRUNC('month', data) + ORDER BY month + `, expParams), + ]); + + const monthMap = new Map(); + incomeMonthly.rows.forEach((r: any) => { + const m = parseDate(r.month)!; + monthMap.set(m, { income: Number(r.total), expense: 0 }); + }); + expenseMonthly.rows.forEach((r: any) => { + const m = parseDate(r.month)!; + const existing = monthMap.get(m) ?? { income: 0, expense: 0 }; + existing.expense = Number(r.total); + monthMap.set(m, existing); + }); + + const rows = Array.from(monthMap.entries()) + .map(([month, data]) => ({ + month, + income: data.income, + expense: data.expense, + profit: data.income - data.expense, + })) + .sort((a, b) => a.month.localeCompare(b.month)); + + res.json(rows); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/profitability/:zakazchik/contracts", async (req, res) => { + try { + const { dateFrom, dateTo } = req.query; + const zakazchik = req.params.zakazchik; + + const incConditions: string[] = ["zakazchik = $1", "dogovorzakazchika != '0'"]; + const expConditions: string[] = ["zakazchik = $1", "dogovorzakazchika != '0'"]; + const incParams: unknown[] = [zakazchik]; + const expParams: unknown[] = [zakazchik]; + + if (dateFrom) { + incParams.push(dateFrom); incConditions.push(`data >= $${incParams.length}`); + expParams.push(dateFrom); expConditions.push(`data >= $${expParams.length}`); + } + if (dateTo) { + incParams.push(dateTo); incConditions.push(`data <= $${incParams.length}`); + expParams.push(dateTo); expConditions.push(`data <= $${expParams.length}`); + } + + const incWhere = `WHERE ${incConditions.join(" AND ")}`; + const expWhere = `WHERE ${expConditions.join(" AND ")}`; + + const [incomeContracts, expenseContracts] = await Promise.all([ + query(` + SELECT dogovorzakazchika, COALESCE(SUM(summa),0) as total + FROM ${getIncomeEffectiveView()} ${incWhere} + GROUP BY dogovorzakazchika + ORDER BY total DESC + `, incParams), + query(` + SELECT dogovorzakazchika, COALESCE(SUM(summadokumenta),0) as total + FROM ${getExpenseEffectiveView()} ${expWhere} + GROUP BY dogovorzakazchika + `, expParams), + ]); + + const expMap = new Map(); + expenseContracts.rows.forEach((r: any) => expMap.set(r.dogovorzakazchika, Number(r.total))); + + const contractSet = new Set(); + const rows: any[] = []; + + incomeContracts.rows.forEach((r: any) => { + contractSet.add(r.dogovorzakazchika); + const totalIncome = Number(r.total); + const totalExpense = expMap.get(r.dogovorzakazchika) ?? 0; + rows.push({ + dogovorzakazchika: r.dogovorzakazchika, + totalIncome, + totalExpense, + profit: totalIncome - totalExpense, + }); + }); + + expenseContracts.rows.forEach((r: any) => { + if (!contractSet.has(r.dogovorzakazchika)) { + rows.push({ + dogovorzakazchika: r.dogovorzakazchika, + totalIncome: 0, + totalExpense: Number(r.total), + profit: -Number(r.total), + }); + } + }); + + rows.sort((a, b) => b.totalIncome - a.totalIncome); + res.json(rows); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/expenses/column-values", async (req, res) => { + try { + const { column, search = "", context: contextStr = "" } = req.query; + if (!column || !ALLOWED_EXPENSE_COLUMNS().includes(column as string)) { + return res.status(400).json({ error: "Недопустимый столбец" }); + } + const cacheKey = `exp-col-values:${column}:${search}:${contextStr}`; + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const col = column as string; + const searchStr = String(search); + const isDateOrNum = col === "data" || col === "summadokumenta"; + const conditions: string[] = [`${isDateOrNum ? `CAST(${col} AS TEXT)` : col} IS NOT NULL`]; + if (!isDateOrNum) conditions.push(`${col} != '0'`); + const params: unknown[] = []; + if (searchStr) { + const colExpr = isDateOrNum ? `CAST(${col} AS TEXT)` : col; + conditions.push(buildTokenILIKE(colExpr, searchStr, params)); + } + if (contextStr) { + try { + const ctx = JSON.parse(String(contextStr)) as Record; + for (const [ctxCol, ctxVal] of Object.entries(ctx)) { + if (!ctxVal) continue; + if (ctxCol === "dateFrom") { + params.push(ctxVal); + conditions.push(`data::date >= $${params.length}::date`); + } else if (ctxCol === "dateTo") { + params.push(ctxVal); + conditions.push(`data::date <= $${params.length}::date`); + } else if (ALLOWED_EXPENSE_COLUMNS().includes(ctxCol) && ctxCol !== col) { + const ctxIsDateOrNum = ctxCol === "data" || ctxCol === "summadokumenta"; + const ctxExpr = ctxIsDateOrNum ? `CAST(${ctxCol} AS TEXT)` : ctxCol; + conditions.push(buildTokenILIKE(ctxExpr, ctxVal, params)); + } + } + } catch { /* ignore */ } + } + const where = `WHERE ${conditions.join(" AND ")}`; + const selectExpr = isDateOrNum ? `CAST(${col} AS TEXT)` : col; + const orderDir = isDateOrNum ? "DESC" : "ASC"; + const result = await query( + `SELECT DISTINCT ${selectExpr} as val FROM ${getExpenseEffectiveView()} ${where} ORDER BY val ${orderDir} LIMIT 20`, + params + ); + return result.rows.map(r => r.val); + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + async function runIncomeSyncCore(): Promise { + const config = loadIncomeAutoDetectConfig(); + const existing = loadCategories(); + let totalAdded = 0; + for (const treeCfg of config.trees) { + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildIncomeAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + totalAdded += mergeLevel(tree, existing); + } + if (totalAdded > 0) { + saveCategories(existing); + console.log(`[auto-sync] Added ${totalAdded} new income categories`); + } + return totalAdded; + } + + async function runExpenseSyncCore(): Promise { + const config = loadAutoDetectConfig(); + const existing = loadExpenseCategories(); + let totalAdded = 0; + for (const treeCfg of config.trees) { + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + totalAdded += mergeLevel(tree, existing); + } + if (totalAdded > 0) { + saveExpenseCategories(existing); + console.log(`[auto-sync] Added ${totalAdded} new expense categories`); + } + return totalAdded; + } + + async function autoSyncExpenseCategories(): Promise { + await cachedQuery("__expense_auto_sync__", TTL_LONG, async () => { + try { + const config = loadAutoDetectConfig(); + const existing = loadExpenseCategories(); + let totalAdded = 0; + for (const treeCfg of config.trees) { + const tree = treeCfg.mode === "merge" + ? buildMergedCategory(treeCfg) + : await buildAutoDetectTree(treeCfg.columns, treeCfg.id, treeCfg.filters, treeCfg.filterLogic); + totalAdded += mergeLevel(tree, existing); + } + if (totalAdded > 0) { + saveExpenseCategories(existing); + console.log(`[auto-sync] Added ${totalAdded} new expense categories`); + } + } catch (err) { + console.error("[auto-sync] Error syncing expense categories:", err); + } + return true; + }); + } + + app.get("/api/admin/rebuild-status", (_req, res) => { + res.json({ + inProgress: treeRebuildInProgress, + pendingRebuild: treeRebuildPending, + lastCompletedAt: treeRebuildLastCompletedAt ? treeRebuildLastCompletedAt.toISOString() : null, + }); + }); + + app.post("/api/refresh-all", async (_req, res) => { + try { + cacheInvalidateAll(); + const [addedIncome, addedExpense] = await Promise.all([ + runIncomeSyncCore().catch(err => { console.error("[refresh-all] income sync error:", err); return 0; }), + runExpenseSyncCore().catch(err => { console.error("[refresh-all] expense sync error:", err); return 0; }), + ]); + await scheduleAndAwaitRebuild(); + res.json({ + ok: true, + addedIncome, + addedExpense, + rebuiltAt: treeRebuildLastCompletedAt ? treeRebuildLastCompletedAt.toISOString() : null, + }); + } catch (err) { + handleApiError(res, err, "Ошибка при полном обновлении данных"); + } + }); + + app.get("/api/expenses/tree-summary", async (req, res) => { + try { + await ensureTreesReady(); + await autoSyncExpenseCategories(); + const cacheKey = stableKey("expense-tree-summary", req.query as Record); + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const { dateFrom, dateTo } = req.query; + const expenseCats = loadExpenseCategories(); + + const ruleColumns = new Set(); + function collectRuleColumns(cats: IncomeCategory[]) { + for (const cat of cats) { + for (const rule of cat.rules) { + if (ALLOWED_EXPENSE_COLUMNS().includes(rule.column) && rule.column !== "data" && rule.column !== "summadokumenta") { + ruleColumns.add(rule.column); + } + } + if (cat.children) collectRuleColumns(cat.children); + } + } + collectRuleColumns(expenseCats); + const groupCols = ruleColumns.size > 0 ? [...ruleColumns] : ["statya", "dochka", "vnuchka", "otvetstvenniipokontraktu"]; + + const params: unknown[] = []; + const dateConds: string[] = []; + if (dateFrom) { params.push(dateFrom); dateConds.push(`data::date >= $${params.length}::date`); } + if (dateTo) { params.push(dateTo); dateConds.push(`data::date <= $${params.length}::date`); } + const whereClause = dateConds.length > 0 ? `WHERE ${dateConds.join(" AND ")}` : ""; + + const selectCols = groupCols.join(", "); + const aggResult = await query( + `SELECT ${selectCols}, SUM(summadokumenta) as total, COUNT(*) as cnt, MIN(data) as min_date, MAX(data) as max_date + FROM ${getExpenseEffectiveView()} ${whereClause} + GROUP BY ${selectCols}`, + params + ); + + const aggRows: Record[] = aggResult.rows.map(r => { + const row: Record = { total: Number(r.total), cnt: Number(r.cnt), min_date: r.min_date ?? null, max_date: r.max_date ?? null }; + for (const col of groupCols) { + row[col] = r[col] ?? ""; + } + return row; + }); + + function rowMatchesRule(row: Record, rule: { column: string; operator: string; value: string }): boolean { + const val = row[rule.column]; + if (val === undefined) return false; + switch (rule.operator) { + case "equals": return val === rule.value; + case "contains": return typeof val === "string" && matchTokens(val, rule.value); + case "not_equals": return val !== rule.value; + case "greater_than": return Number(val) > Number(rule.value); + case "less_than": return Number(val) < Number(rule.value); + case "equal_number": return Number(val) === Number(rule.value); + default: return val === rule.value; + } + } + + function rowMatchesRules(row: Record, rules: { column: string; operator: string; value: string }[], logic: "and" | "or" = "and"): boolean { + if (logic === "or") return rules.some(rule => rowMatchesRule(row, rule)); + return rules.every(rule => rowMatchesRule(row, rule)); + } + + const allFlat = flattenCategories(expenseCats); + + type AutoPathExp = { ruleSets: IncomeCategory["rules"][]; logic: ("and" | "or") }; + const autoPathsExp: AutoPathExp[] = []; + type ExclPathExp = { id: string; ruleSets: IncomeCategory["rules"][]; logic: ("and" | "or") }; + const excludedPathsExp: ExclPathExp[] = []; + function collectPathsExp(cats: IncomeCategory[], ancestors: IncomeCategory["rules"][], inAutoTree = false) { + for (const cat of cats) { + const isAuto = !!cat.autoDetected; + const nowInAutoTree = inAutoTree || isAuto; + const path = [...ancestors, cat.rules]; + if (cat.excluded) { + if (!nowInAutoTree) { + excludedPathsExp.push({ id: cat.id, ruleSets: path, logic: cat.ruleLogic || "and" }); + } + } else if (isAuto) { + autoPathsExp.push({ ruleSets: path, logic: cat.ruleLogic || "and" }); + } + if (cat.children) collectPathsExp(cat.children, path, nowInAutoTree); + } + } + collectPathsExp(expenseCats, []); + + function rowMatchesAutoPathExp(row: Record, ap: AutoPathExp): boolean { + for (let i = 0; i < ap.ruleSets.length - 1; i++) { + if (!rowMatchesRules(row, ap.ruleSets[i])) return false; + } + const lastRules = ap.ruleSets[ap.ruleSets.length - 1]; + return rowMatchesRules(row, lastRules, ap.logic); + } + + function computeNodeFromAgg( + cat: IncomeCategory, + siblings: IncomeCategory[], + ancestorRuleSets: IncomeCategory["rules"][], + ancestorIds: Set = new Set(), + rootManualCats: IncomeCategory[] = [] + ): { id: string; name: string; total: number; count: number; excluded?: boolean; autoDetected?: boolean; minDate?: string; maxDate?: string; children: any[] } { + const allRuleSets = [...ancestorRuleSets, cat.rules]; + const catLogic = cat.ruleLogic || "and"; + const idx = siblings.findIndex(c => c.id === cat.id); + const laterSibs = siblings.slice(idx + 1).filter(c => !c.excluded); + + let total = 0; + let count = 0; + let minDate: Date | null = null; + let maxDate: Date | null = null; + for (const row of aggRows) { + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; + if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue; + if (cat.autoDetected && rootManualCats.length > 0 && rootManualCats.some(mc => rowMatchesRules(row, mc.rules, mc.ruleLogic || "and"))) continue; + if (!cat.autoDetected && excludedPathsExp.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue; + if (!cat.autoDetected && !!cat.treeId && autoPathsExp.some(ap => rowMatchesAutoPathExp(row, ap) && !isRefinementOf(ap.ruleSets, allRuleSets))) continue; + if (cat.children && cat.children.length > 0 && cat.children.some(child => child.rules.length > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue; + total += row.total; + count += row.cnt; + if (row.min_date) { + const d = new Date(row.min_date); + if (!minDate || d < minDate) minDate = d; + } + if (row.max_date) { + const d = new Date(row.max_date); + if (!maxDate || d > maxDate) maxDate = d; + } + } + + const children: any[] = []; + if (cat.children && cat.children.length > 0) { + const childAncestorIds = new Set(ancestorIds); + childAncestorIds.add(cat.id); + for (const child of cat.children) { + children.push(computeNodeFromAgg(child, cat.children, allRuleSets, childAncestorIds, rootManualCats)); + } + } + + return { + id: cat.id, + name: cat.name, + total, + count, + excluded: cat.excluded || undefined, + autoDetected: cat.autoDetected || undefined, + minDate: minDate ? minDate.toISOString().slice(0, 10) : undefined, + maxDate: maxDate ? maxDate.toISOString().slice(0, 10) : undefined, + children, + }; + } + + const config = loadAutoDetectConfig(); + + const groupedByTree = new Map(); + for (const cat of expenseCats) { + const tid = cat.treeId || "__manual__"; + if (!groupedByTree.has(tid)) groupedByTree.set(tid, []); + groupedByTree.get(tid)!.push(cat); + } + + const trees: { id: string; name: string; nodes: any[]; total: number; count: number; isPrimary?: boolean }[] = []; + + const configTreeIds = new Set(config.trees.map((t: any) => t.id)); + const primaryTreeIds = new Set(config.trees.filter(t => t.isPrimary).map(t => t.id)); + const orphanCats: IncomeCategory[] = []; + const manualCats: IncomeCategory[] = []; + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__") { + manualCats.push(...cats); + } else if (!configTreeIds.has(tid)) { + orphanCats.push(...cats); + } + } + const rootManualCatsForExclExp = manualCats.filter(c => !c.autoDetected && !c.excluded); + type ManualWithAncestorsExp = { cat: IncomeCategory; ancestorRuleSets: IncomeCategory["rules"][] }; + function collectNestedManualCatsExp(cats: IncomeCategory[], ancestorRules: IncomeCategory["rules"][] = []): ManualWithAncestorsExp[] { + const result: ManualWithAncestorsExp[] = []; + for (const cat of cats) { + if (!cat.autoDetected) result.push({ cat, ancestorRuleSets: ancestorRules }); + const nextAncestors = [...ancestorRules, cat.rules]; + if (cat.children) result.push(...collectNestedManualCatsExp(cat.children, nextAncestors)); + } + return result; + } + const rootManualIds = new Set(manualCats.map(c => c.id)); + const nestedManualWithAncestorsExp: ManualWithAncestorsExp[] = []; + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__" || !configTreeIds.has(tid)) continue; + for (const cat of cats) { + const nested = collectNestedManualCatsExp(cat.children || [], [cat.rules]); + for (const n of nested) { + if (!rootManualIds.has(n.cat.id)) { + nestedManualWithAncestorsExp.push(n); + manualCats.push(n.cat); + } + } + } + } + const nestedManualAncestorMapExp = new Map(); + for (const n of nestedManualWithAncestorsExp) { + nestedManualAncestorMapExp.set(n.cat.id, n.ancestorRuleSets); + } + function nodeSubtreeTotalExp(n: any): number { + if (n.excluded) { + let t = 0; + for (const ch of n.children) t += nodeSubtreeTotalExp(ch); + return t; + } + let t = n.total; + for (const ch of n.children) t += nodeSubtreeTotalExp(ch); + return t; + } + function nodeSubtreeCountExp(n: any): number { + if (n.excluded) { + let c = 0; + for (const ch of n.children) c += nodeSubtreeCountExp(ch); + return c; + } + let c = n.count; + for (const ch of n.children) c += nodeSubtreeCountExp(ch); + return c; + } + + const otherCats = [...manualCats, ...orphanCats]; + if (otherCats.length > 0) { + const nodes: any[] = []; + for (const cat of otherCats) { + const ancestorRules = nestedManualAncestorMapExp.get(cat.id) || []; + nodes.push(computeNodeFromAgg(cat, otherCats, ancestorRules, new Set())); + } + const treeTotal = nodes.reduce((s, n) => s + nodeSubtreeTotalExp(n), 0); + const treeCount = nodes.reduce((s, n) => s + nodeSubtreeCountExp(n), 0); + trees.push({ id: "__manual__", name: "Ручные категории", nodes, total: treeTotal, count: treeCount, isPrimary: true }); + } + + for (const treeCfg of config.trees) { + const cats = groupedByTree.get(treeCfg.id) || []; + const nodes: any[] = []; + for (const cat of cats) { + nodes.push(computeNodeFromAgg(cat, cats, [], new Set(), rootManualCatsForExclExp)); + } + const treeTotal = nodes.reduce((s, n) => s + nodeSubtreeTotalExp(n), 0); + const treeCount = nodes.reduce((s, n) => s + nodeSubtreeCountExp(n), 0); + trees.push({ id: treeCfg.id, name: treeCfg.name, nodes, total: treeTotal, count: treeCount, isPrimary: treeCfg.isPrimary || false }); + } + + function isRowMatchedByTreeExp(row: Record, treeCats: IncomeCategory[], ancestorRuleSets: IncomeCategory["rules"][] = []): boolean { + for (const cat of treeCats) { + const catLogic = cat.ruleLogic || "and"; + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; + return true; + } + return false; + } + let uncatTotal = 0; + let uncatCount = 0; + for (const row of aggRows) { + let matched = false; + for (const [tid, treeCats] of groupedByTree) { + const isPrimary = tid === "__manual__" || primaryTreeIds.has(tid); + if (!isPrimary) continue; + if (isRowMatchedByTreeExp(row, treeCats)) { matched = true; break; } + } + if (!matched) { + uncatTotal += row.total; + uncatCount += row.cnt; + } + } + + let grandTotal = 0; + let grandCount = 0; + for (const row of aggRows) { + grandTotal += row.total; + grandCount += row.cnt; + } + + return { + trees, + uncategorized: { total: uncatTotal, count: uncatCount }, + grandTotal: { total: grandTotal, count: grandCount }, + }; + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/heatmap", async (req, res) => { + try { + const { type = "income", dateFrom, dateTo } = req.query; + if (type !== "income" && type !== "expense") return res.status(400).json({ error: "type must be income or expense" }); + + const isIncome = type === "income"; + if (isIncome) await autoSyncIncomeCategories(); else await autoSyncExpenseCategories(); + + const cacheKey = stableKey(`heatmap-${type}`, req.query as Record); + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const cats = isIncome ? loadCategories() : loadExpenseCategories(); + const allowedCols = isIncome ? ALLOWED_INCOME_COLUMNS() : ALLOWED_EXPENSE_COLUMNS(); + const table = isIncome ? getIncomeEffectiveView() : getExpenseEffectiveView(); + const sumCol = isIncome ? "summa" : "summadokumenta"; + const excludeSumCols = isIncome ? ["data", "summa"] : ["data", "summadokumenta"]; + + const ruleColumns = new Set(); + function collectRuleColumns(cs: IncomeCategory[]) { + for (const c of cs) { + for (const r of c.rules) { + if (allowedCols.includes(r.column) && !excludeSumCols.includes(r.column)) ruleColumns.add(r.column); + } + if (c.children) collectRuleColumns(c.children); + } + } + collectRuleColumns(cats); + const groupCols = ruleColumns.size > 0 ? [...ruleColumns] : ["statya", "dochka", "vnuchka", "otvetstvenniipokontraktu"]; + + const params: unknown[] = []; + const dateConds: string[] = []; + if (dateFrom) { params.push(dateFrom); dateConds.push(`data::date >= $${params.length}::date`); } + if (dateTo) { params.push(dateTo); dateConds.push(`data::date <= $${params.length}::date`); } + const whereClause = dateConds.length > 0 ? `WHERE ${dateConds.join(" AND ")}` : ""; + + const selectCols = groupCols.join(", "); + const aggResult = await query( + `SELECT TO_CHAR(data, 'YYYY-MM') as month, ${selectCols}, SUM(${sumCol}) as total + FROM ${table} ${whereClause} + GROUP BY TO_CHAR(data, 'YYYY-MM'), ${selectCols} + ORDER BY month`, + params + ); + + const aggRows: Record[] = aggResult.rows.map(r => { + const row: Record = { month: r.month, total: Number(r.total) }; + for (const col of groupCols) row[col] = r[col] ?? ""; + return row; + }); + + function rowMatchesRule(row: Record, rule: { column: string; operator: string; value: string }): boolean { + const val = row[rule.column]; + if (val === undefined) return false; + switch (rule.operator) { + case "equals": return val === rule.value; + case "contains": return typeof val === "string" && matchTokens(val, rule.value); + case "not_equals": return val !== rule.value; + case "greater_than": return Number(val) > Number(rule.value); + case "less_than": return Number(val) < Number(rule.value); + case "equal_number": return Number(val) === Number(rule.value); + default: return val === rule.value; + } + } + function rowMatchesRules(row: Record, rules: { column: string; operator: string; value: string }[], logic: "and" | "or" = "and"): boolean { + if (logic === "or") return rules.some(r => rowMatchesRule(row, r)); + return rules.every(r => rowMatchesRule(row, r)); + } + + type ExclPath = { id: string; ruleSets: IncomeCategory["rules"][]; logic: "and" | "or" }; + const excludedPaths: ExclPath[] = []; + type AutoPath = { ruleSets: IncomeCategory["rules"][]; logic: "and" | "or" }; + const autoPaths: AutoPath[] = []; + function collectPaths(cs: IncomeCategory[], ancestors: IncomeCategory["rules"][], inAutoTree = false) { + for (const c of cs) { + const isAuto = !!c.autoDetected; + const nowInAutoTree = inAutoTree || isAuto; + const path = [...ancestors, c.rules]; + if (c.excluded) { + if (!nowInAutoTree) excludedPaths.push({ id: c.id, ruleSets: path, logic: c.ruleLogic || "and" }); + } else if (isAuto) autoPaths.push({ ruleSets: path, logic: c.ruleLogic || "and" }); + if (c.children) collectPaths(c.children, path, nowInAutoTree); + } + } + collectPaths(cats, []); + + function rowMatchesAutoPath(row: Record, ap: AutoPath): boolean { + for (let i = 0; i < ap.ruleSets.length - 1; i++) { + if (!rowMatchesRules(row, ap.ruleSets[i])) return false; + } + return rowMatchesRules(row, ap.ruleSets[ap.ruleSets.length - 1], ap.logic); + } + function rowMatchesExclPath(row: Record, ep: ExclPath): boolean { + for (let i = 0; i < ep.ruleSets.length - 1; i++) { + if (!rowMatchesRules(row, ep.ruleSets[i])) return false; + } + return rowMatchesRules(row, ep.ruleSets[ep.ruleSets.length - 1], ep.logic); + } + + function monthTotalsForCat( + cat: IncomeCategory, + siblings: IncomeCategory[], + ancestorRuleSets: IncomeCategory["rules"][] + ): Map { + const allRuleSets = [...ancestorRuleSets, cat.rules]; + const catLogic = cat.ruleLogic || "and"; + const idx = siblings.findIndex(c => c.id === cat.id); + const laterSibs = siblings.slice(idx + 1).filter(c => !c.excluded); + const monthMap = new Map(); + + for (const row of aggRows) { + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; + if (laterSibs.some(s => rowMatchesRules(row, s.rules, s.ruleLogic || "and"))) continue; + if (!cat.autoDetected && excludedPaths.some(ep => ep.id !== cat.id && rowMatchesExclPath(row, ep))) continue; + if (!cat.autoDetected && !!cat.treeId && autoPaths.some(ap => rowMatchesAutoPath(row, ap) && !isRefinementOf(ap.ruleSets, allRuleSets))) continue; + if (cat.children && cat.children.length > 0 && cat.children.some(ch => ch.rules.length > 0 && rowMatchesRules(row, ch.rules, ch.ruleLogic || "and"))) continue; + + const m = row.month as string; + monthMap.set(m, (monthMap.get(m) || 0) + row.total); + } + + if (cat.children && cat.children.length > 0) { + for (const child of cat.children) { + const childMap = monthTotalsForCat(child, cat.children, allRuleSets); + for (const [m, v] of childMap) monthMap.set(m, (monthMap.get(m) || 0) + v); + } + } + return monthMap; + } + + const monthsSet = new Set(aggRows.map(r => r.month)); + const months = Array.from(monthsSet).sort(); + + const groupedByTree = new Map(); + for (const cat of cats) { + const tid = cat.treeId || "__manual__"; + if (!groupedByTree.has(tid)) groupedByTree.set(tid, []); + groupedByTree.get(tid)!.push(cat); + } + + const config = isIncome ? loadIncomeAutoDetectConfig() : loadAutoDetectConfig(); + const configTreeMap = new Map(config.trees.map((t: any) => [t.id, t])); + + const heatCategories: { id: string; name: string; treeId?: string; treeName?: string; monthly: Record; total: number; excluded?: boolean }[] = []; + + for (const [treeId, treeCats] of groupedByTree) { + if (treeId === "__manual__") continue; + const treeMeta = configTreeMap.get(treeId) as any; + const treeName = treeMeta?.name || treeId; + + for (const cat of treeCats) { + if (cat.excluded) continue; + const monthMap = monthTotalsForCat(cat, treeCats, []); + const monthly: Record = {}; + let total = 0; + for (const m of months) { + const v = monthMap.get(m) || 0; + monthly[m] = v; + total += v; + } + if (total > 0 || months.some(m => monthly[m] > 0)) { + heatCategories.push({ id: cat.id, name: cat.name, treeId, treeName, monthly, total }); + } + } + } + + heatCategories.sort((a, b) => b.total - a.total); + + const maxValue = Math.max(0, ...heatCategories.flatMap(c => Object.values(c.monthly))); + + const totalByMonth: Record = {}; + for (const m of months) { + totalByMonth[m] = heatCategories.reduce((s, c) => s + (c.monthly[m] || 0), 0); + } + + return { months, categories: heatCategories, maxValue, totalByMonth }; + }); + + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/expenses/tree-timeseries", async (req, res) => { + try { + await ensureTreesReady(); + await autoSyncExpenseCategories(); + const cacheKey = stableKey("expense-tree-timeseries", req.query as Record); + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const { dateFrom, dateTo, groupBy = "month", categoryId } = req.query; + const validGroupBy = ["day", "month", "quarter", "year"]; + const gb = validGroupBy.includes(groupBy as string) ? (groupBy as string) : "month"; + + const expenseCats = loadExpenseCategories(); + const ruleColumns = new Set(); + function collectRuleColumns(cats: IncomeCategory[]) { + for (const cat of cats) { + for (const rule of cat.rules) { + if (ALLOWED_EXPENSE_COLUMNS().includes(rule.column) && rule.column !== "data" && rule.column !== "summadokumenta") { + ruleColumns.add(rule.column); + } + } + if (cat.children) collectRuleColumns(cat.children); + } + } + collectRuleColumns(expenseCats); + const groupCols = ruleColumns.size > 0 ? [...ruleColumns] : ["statya", "dochka", "vnuchka", "otvetstvenniipokontraktu"]; + + const params: unknown[] = []; + const dateConds: string[] = []; + if (dateFrom) { params.push(dateFrom); dateConds.push(`data::date >= $${params.length}::date`); } + if (dateTo) { params.push(dateTo); dateConds.push(`data::date <= $${params.length}::date`); } + const whereClause = dateConds.length > 0 ? `WHERE ${dateConds.join(" AND ")}` : ""; + + const selectCols = groupCols.join(", "); + const aggResult = await query( + `SELECT ${selectCols}, DATE_TRUNC('${gb}', data) as period, SUM(summadokumenta) as total, COUNT(*) as cnt + FROM ${getExpenseEffectiveView()} ${whereClause} + GROUP BY ${selectCols}, DATE_TRUNC('${gb}', data)`, + params + ); + + const aggRows: Record[] = aggResult.rows.map(r => { + const row: Record = { total: Number(r.total), cnt: Number(r.cnt), period: r.period }; + for (const col of groupCols) { row[col] = r[col] ?? ""; } + return row; + }); + + function rowMatchesRule(row: Record, rule: { column: string; operator: string; value: string }): boolean { + const val = row[rule.column]; + if (val === undefined) return false; + switch (rule.operator) { + case "equals": return val === rule.value; + case "contains": return typeof val === "string" && matchTokens(val, rule.value); + case "not_equals": return val !== rule.value; + case "greater_than": return Number(val) > Number(rule.value); + case "less_than": return Number(val) < Number(rule.value); + case "equal_number": return Number(val) === Number(rule.value); + default: return val === rule.value; + } + } + function rowMatchesRules(row: Record, rules: { column: string; operator: string; value: string }[], logic: "and" | "or" = "and"): boolean { + if (logic === "or") return rules.some(rule => rowMatchesRule(row, rule)); + return rules.every(rule => rowMatchesRule(row, rule)); + } + + const allFlat = flattenCategories(expenseCats); + + type AutoPathExpTS = { ruleSets: IncomeCategory["rules"][]; logic: ("and" | "or") }; + const autoPathsExpTS: AutoPathExpTS[] = []; + type ExclPathExpTS = { id: string; ruleSets: IncomeCategory["rules"][]; logic: ("and" | "or") }; + const excludedPathsExpTS: ExclPathExpTS[] = []; + function collectPathsExpTS(cats: IncomeCategory[], ancestors: IncomeCategory["rules"][], inAutoTree = false) { + for (const cat of cats) { + const isAuto = !!cat.autoDetected; + const nowInAutoTree = inAutoTree || isAuto; + const path = [...ancestors, cat.rules]; + if (cat.excluded) { + if (!nowInAutoTree) { + excludedPathsExpTS.push({ id: cat.id, ruleSets: path, logic: cat.ruleLogic || "and" }); + } + } else if (isAuto) { + autoPathsExpTS.push({ ruleSets: path, logic: cat.ruleLogic || "and" }); + } + if (cat.children) collectPathsExpTS(cat.children, path, nowInAutoTree); + } + } + collectPathsExpTS(expenseCats, []); + + function rowMatchesAutoPathExpTS(row: Record, ap: AutoPathExpTS): boolean { + for (let i = 0; i < ap.ruleSets.length - 1; i++) { + if (!rowMatchesRules(row, ap.ruleSets[i])) return false; + } + return rowMatchesRules(row, ap.ruleSets[ap.ruleSets.length - 1], ap.logic); + } + + const dataPeriods = new Set(); + for (const row of aggRows) { + if (row.period) dataPeriods.add(new Date(row.period).toISOString()); + } + const dataPeriodsSorted = [...dataPeriods].sort(); + let sortedPeriods = dataPeriodsSorted; + if (dataPeriodsSorted.length >= 2) { + const filled: string[] = []; + const minD = new Date(dataPeriodsSorted[0]); + const maxD = new Date(dataPeriodsSorted[dataPeriodsSorted.length - 1]); + const cur = new Date(minD); + while (cur <= maxD) { + filled.push(cur.toISOString()); + if (gb === "day") cur.setUTCDate(cur.getUTCDate() + 1); + else if (gb === "month") cur.setUTCMonth(cur.getUTCMonth() + 1); + else if (gb === "quarter") cur.setUTCMonth(cur.getUTCMonth() + 3); + else cur.setUTCFullYear(cur.getUTCFullYear() + 1); + } + sortedPeriods = filled; + } + + function formatPeriodLabel(iso: string): string { + const d = new Date(iso); + if (gb === "day") return d.toISOString().slice(0, 10); + if (gb === "month") return d.toISOString().slice(0, 7); + if (gb === "quarter") { const q = Math.ceil((d.getMonth() + 1) / 3); return `${d.getFullYear()}-Q${q}`; } + return `${d.getFullYear()}`; + } + const periodLabels = sortedPeriods.map(formatPeriodLabel); + const periodIndexMap = new Map(sortedPeriods.map((p, i) => [p, i])); + + function computeCatTimeSeries( + cat: IncomeCategory, + siblings: IncomeCategory[], + ancestorRuleSets: IncomeCategory["rules"][], + ancestorIds: Set = new Set(), + rootManualCats: IncomeCategory[] = [] + ): number[] { + const allRuleSets = [...ancestorRuleSets, cat.rules]; + const catLogic = cat.ruleLogic || "and"; + const idx = siblings.findIndex(c => c.id === cat.id); + const laterSibs = siblings.slice(idx + 1).filter(c => !c.excluded); + const data = new Array(sortedPeriods.length).fill(0); + for (const row of aggRows) { + const ancestorMatch = ancestorRuleSets.every(rs => rowMatchesRules(row, rs)); + if (!ancestorMatch || !rowMatchesRules(row, cat.rules, catLogic)) continue; + if (laterSibs.some(sib => rowMatchesRules(row, sib.rules, sib.ruleLogic || "and"))) continue; + if (cat.autoDetected && rootManualCats.length > 0 && rootManualCats.some(mc => rowMatchesRules(row, mc.rules, mc.ruleLogic || "and"))) continue; + if (!cat.autoDetected && excludedPathsExpTS.some(ep => ep.id !== cat.id && !ancestorIds.has(ep.id) && ep.ruleSets.every((rs, i) => i < ep.ruleSets.length - 1 ? rowMatchesRules(row, rs) : rowMatchesRules(row, rs, ep.logic)))) continue; + if (!cat.autoDetected && !!cat.treeId && autoPathsExpTS.some(ap => rowMatchesAutoPathExpTS(row, ap) && !isRefinementOf(ap.ruleSets, allRuleSets))) continue; + if (cat.children && cat.children.length > 0 && cat.children.some(child => child.rules.length > 0 && rowMatchesRules(row, child.rules, child.ruleLogic || "and"))) continue; + const pi = periodIndexMap.get(new Date(row.period).toISOString()); + if (pi !== undefined) data[pi] += row.total; + } + return data; + } + + function subtreeTimeSeriesExp( + cat: IncomeCategory, + siblings: IncomeCategory[], + ancestorRuleSets: IncomeCategory["rules"][], + ancestorIds: Set = new Set(), + rootManualCats: IncomeCategory[] = [] + ): number[] { + const own = computeCatTimeSeries(cat, siblings, ancestorRuleSets, ancestorIds, rootManualCats); + if (!cat.children || cat.children.length === 0) return own; + const allRuleSets = [...ancestorRuleSets, cat.rules]; + const newAncestorIds = new Set([...ancestorIds, cat.id]); + for (const child of cat.children) { + if (child.excluded) continue; + const childData = subtreeTimeSeriesExp(child, cat.children, allRuleSets, newAncestorIds, rootManualCats); + for (let i = 0; i < own.length; i++) own[i] += childData[i]; + } + return own; + } + + const config = loadAutoDetectConfig(); + const groupedByTree = new Map(); + for (const cat of expenseCats) { + const tid = cat.treeId || "__manual__"; + if (!groupedByTree.has(tid)) groupedByTree.set(tid, []); + groupedByTree.get(tid)!.push(cat); + } + + const drillRootManualExp: IncomeCategory[] = []; + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__") drillRootManualExp.push(...cats.filter(c => !c.autoDetected && !c.excluded)); + } + + if (categoryId && typeof categoryId === "string") { + const found = findCategoryWithAncestors(expenseCats, categoryId); + if (!found || !found.cat.children || found.cat.children.length === 0) { + return { trees: [], drilldown: true }; + } + const ancestorRuleSets = [...found.ancestors.map(a => a.rules), found.cat.rules]; + const drillAncestorIds = new Set([...found.ancestors.map(a => a.id), found.cat.id]); + const children = found.cat.children; + const isAutoTree = found.cat.autoDetected || found.ancestors.some(a => a.autoDetected); + const drillManual = isAutoTree ? drillRootManualExp : []; + const series: any[] = children.filter(c => !c.excluded).map(cat => ({ + id: cat.id, name: cat.name, + hasChildren: !!(cat.children && cat.children.length > 0), + data: subtreeTimeSeriesExp(cat, children, ancestorRuleSets, drillAncestorIds, drillManual) + })).filter(s => s.data.some(v => v !== 0)); + const treeId = found.cat.treeId || found.ancestors[0]?.treeId || "__drilldown__"; + return { trees: [{ id: treeId, name: found.cat.name, isPrimary: true, periods: periodLabels, series }], drilldown: true }; + } + + const trees: any[] = []; + const configTreeIds = new Set(config.trees.map((t: any) => t.id)); + const orphanCats: IncomeCategory[] = []; + const manualCats: IncomeCategory[] = []; + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__") manualCats.push(...cats); + else if (!configTreeIds.has(tid)) orphanCats.push(...cats); + } + type ManualWithAncestorsTrExp = { cat: IncomeCategory; ancestorRuleSets: IncomeCategory["rules"][] }; + function collectNestedManualTrExp(cats: IncomeCategory[], ancestorRules: IncomeCategory["rules"][] = []): ManualWithAncestorsTrExp[] { + const result: ManualWithAncestorsTrExp[] = []; + for (const cat of cats) { + if (!cat.autoDetected) result.push({ cat, ancestorRuleSets: ancestorRules }); + const nextAncestors = [...ancestorRules, cat.rules]; + if (cat.children) result.push(...collectNestedManualTrExp(cat.children, nextAncestors)); + } + return result; + } + const rootManualCatsForExclTrExp = manualCats.filter(c => !c.autoDetected && !c.excluded); + const rmIdsTrExp = new Set(manualCats.map(c => c.id)); + const trendAncestorMapExp = new Map(); + for (const [tid, cats] of groupedByTree) { + if (tid === "__manual__" || !configTreeIds.has(tid)) continue; + for (const cat of cats) { + const nested = collectNestedManualTrExp(cat.children || [], [cat.rules]); + for (const n of nested) { + if (!rmIdsTrExp.has(n.cat.id)) { + trendAncestorMapExp.set(n.cat.id, n.ancestorRuleSets); + manualCats.push(n.cat); + } + } + } + } + const otherCats = [...manualCats, ...orphanCats]; + if (otherCats.length > 0) { + const series = otherCats.filter(c => !c.excluded).map(cat => ({ + id: cat.id, name: cat.name, + hasChildren: !!(cat.children && cat.children.length > 0), + data: subtreeTimeSeriesExp(cat, otherCats, trendAncestorMapExp.get(cat.id) || [], new Set()) + })).filter(s => s.data.some(v => v !== 0)); + trees.push({ id: "__manual__", name: "Ручные категории", isPrimary: true, periods: periodLabels, series }); + } + + for (const treeCfg of config.trees) { + const cats = groupedByTree.get(treeCfg.id) || []; + const series = cats.filter(c => !c.excluded).map(cat => ({ + id: cat.id, name: cat.name, + hasChildren: !!(cat.children && cat.children.length > 0), + data: subtreeTimeSeriesExp(cat, cats, [], new Set(), rootManualCatsForExclTrExp) + })).filter(s => s.data.some(v => v !== 0)); + trees.push({ id: treeCfg.id, name: treeCfg.name, isPrimary: treeCfg.isPrimary || false, periods: periodLabels, series }); + } + + return { trees }; + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/expenses", async (req, res) => { + try { + const { filters: filtersStr, categoryId, excludeCategories, page = "1", pageSize = "50", sortBy, sortDir } = req.query; + const p = Number(page); + const ps = Math.min(Number(pageSize), 200); + const offset = (p - 1) * ps; + + const conditions: string[] = []; + const params: unknown[] = []; + + const expenseCats = loadExpenseCategories(); + + const expenseConfig = loadAutoDetectConfig(); + const expensePrimaryTreeIds = new Set(expenseConfig.trees.filter(t => t.isPrimary).map(t => t.id)); + + let skipExclIds: string | Set | undefined; + let rootManualRuleSetsExp: IncomeCategory["rules"][] | undefined; + if (categoryId) { + const ancestorConditions = buildCategoryWithAncestorsConditions(expenseCats, categoryId as string, params, ALLOWED_EXPENSE_COLUMNS()); + conditions.push(...ancestorConditions); + const siblingExcl = buildSiblingExclusions(expenseCats, categoryId as string, params, ALLOWED_EXPENSE_COLUMNS()); + conditions.push(...siblingExcl); + const foundCat = findCategoryById(expenseCats, categoryId as string); + if (foundCat) { + skipExclIds = new Set(foundCat.path.map(c => c.id)); + if (!foundCat.cat.autoDetected) { + const ancestorAutoIds = new Set(foundCat.path.filter(c => c.autoDetected).map(c => c.id)); + if (ancestorAutoIds.size > 0) { + const autoExcl = buildAutoPathExclusionConditions(expenseCats, params, ALLOWED_EXPENSE_COLUMNS(), ancestorAutoIds); + conditions.push(...autoExcl); + } else { + rootManualRuleSetsExp = [foundCat.cat.rules]; + if (foundCat.cat.treeId) { + const autoExcl = buildAutoPathExclusionConditions(expenseCats, params, ALLOWED_EXPENSE_COLUMNS(), undefined, rootManualRuleSetsExp); + conditions.push(...autoExcl); + } + } + } else { + const manualExcl = buildManualCatExclusionConditions(expenseCats, params, ALLOWED_EXPENSE_COLUMNS()); + conditions.push(...manualExcl); + } + } else { + skipExclIds = categoryId as string; + } + } else if (excludeCategories === "true") { + const excludeConditions = buildExcludeAllCategoriesConditions(expenseCats, params, ALLOWED_EXPENSE_COLUMNS(), expensePrimaryTreeIds); + conditions.push(...excludeConditions); + conditions.push('"_override_category_id" IS NULL'); + } + + const expenseGlobalExclusions = buildGlobalExclusionConditions(expenseCats, params, skipExclIds, ALLOWED_EXPENSE_COLUMNS(), excludeCategories === "true" ? expensePrimaryTreeIds : undefined, rootManualRuleSetsExp); + conditions.push(...expenseGlobalExclusions); + + const expFilterConditions: string[] = []; + if (filtersStr) { + try { + const filterObj = JSON.parse(String(filtersStr)) as Record; + if (filterObj.dateFrom) { + params.push(filterObj.dateFrom); + expFilterConditions.push(`data::date >= $${params.length}::date`); + } + if (filterObj.dateTo) { + params.push(filterObj.dateTo); + expFilterConditions.push(`data::date <= $${params.length}::date`); + } + for (const [col, val] of Object.entries(filterObj)) { + if (col === "dateFrom" || col === "dateTo") continue; + if (ALLOWED_EXPENSE_COLUMNS().includes(col) && val) { + const colExpr = (col === "summadokumenta" || col === "data") ? `CAST(${col} AS TEXT)` : col; + expFilterConditions.push(buildTokenILIKE(colExpr, String(val), params)); + } + } + } catch { /* ignore parse errors */ } + } + + const expWhereParts: string[] = []; + if (conditions.length > 0) { + expWhereParts.push(...conditions); + } + expWhereParts.push(...expFilterConditions); + const where = expWhereParts.length > 0 ? `WHERE ${expWhereParts.join(" AND ")}` : ""; + + let orderBy = "ORDER BY data DESC"; + if (sortBy && ALL_EXPENSE_COLS().includes(sortBy as string)) { + const dir = sortDir === "asc" ? "ASC" : "DESC"; + orderBy = `ORDER BY "${sortBy as string}" ${dir} NULLS LAST`; + } + + const selectCols = [...ALL_EXPENSE_COLS().map(c => `"${c}"`), '"_override_category_id"'].join(", "); + + const [rows, total, sumResult] = await Promise.all([ + query(`SELECT ${selectCols} FROM ${getExpenseEffectiveView()} ${where} ${orderBy} LIMIT ${ps} OFFSET ${offset}`, params), + query(`SELECT COUNT(*) as cnt FROM ${getExpenseEffectiveView()} ${where}`, params), + query(`SELECT COALESCE(SUM(summadokumenta),0) as total_sum FROM ${getExpenseEffectiveView()} ${where}`, params), + ]); + + const totalRows = Number(total.rows[0].cnt); + res.json({ + rows: rows.rows.map(r => { + const row: any = { ...r, data: parseDate(r.data) }; + if (r._override_category_id) { + row.overrideCategoryId = r._override_category_id; + const found = findCategoryById(expenseCats, r._override_category_id); + row.overrideCategoryName = found ? found.cat.name : r._override_category_id; + } + delete row._override_category_id; + return row; + }), + total: totalRows, + page: p, + pageSize: ps, + totalPages: Math.ceil(totalRows / ps), + totalSum: Number(sumResult.rows[0].total_sum), + }); + } catch (err) { + handleApiError(res, err); + } + }); + + async function buildDebtWhere(req: any) { + const { period, klassprosrochki, kontragent, dogovor, otvetstvennii } = req.query; + const conditions: string[] = []; + const params: unknown[] = []; + + if (period) { params.push(period); conditions.push(`period = $${params.length}`); } + else { const mp = await getMaxPeriod(); params.push(mp); conditions.push(`period = $${params.length}`); } + + if (klassprosrochki) { + const arr = String(klassprosrochki).split(",").filter(Boolean); + if (arr.length > 0) { + const placeholders = arr.map(v => { params.push(v); return `$${params.length}`; }); + conditions.push(`klassprosrochki IN (${placeholders.join(",")})`); + } + } + if (kontragent) { + const arr = String(kontragent).split(",").filter(Boolean); + if (arr.length > 0) { + const placeholders = arr.map(v => { params.push(v); return `$${params.length}`; }); + conditions.push(`kontragent IN (${placeholders.join(",")})`); + } + } + if (dogovor) { + params.push(`%${dogovor}%`); + conditions.push(`dogovor ILIKE $${params.length}`); + } + if (otvetstvennii) { + const arr = String(otvetstvennii).split(",").filter(Boolean); + if (arr.length > 0) { + const placeholders = arr.map(v => { params.push(v); return `$${params.length}`; }); + conditions.push(`otvetstvennii IN (${placeholders.join(",")})`); + } + } + + return { where: `WHERE ${conditions.join(" AND ")}`, params }; + } + + const OVERDUE_CLASSES = ["0", "1-30", "31-60", "61-90", "91-365", "> 365"]; + + function classKey(cls: string | null): string { + if (!cls || cls === "0") return "cls0"; + if (cls === "1-30") return "cls1_30"; + if (cls === "31-60") return "cls31_60"; + if (cls === "61-90") return "cls61_90"; + if (cls === "91-365") return "cls91_365"; + return "cls365plus"; + } + + app.get("/api/debt/analytics", async (req, res) => { + try { + const cacheKey = stableKey("debt-analytics", req.query as Record); + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const { where, params } = await buildDebtWhere(req); + const { page = "1", pageSize = "20" } = req.query; + const pg = Number(page); + const ps = Math.min(Number(pageSize), 100); + const offset = (pg - 1) * ps; + + const [summaryRes, byKontragentRes, byManagerRes, byClassRes, totalRes, updatedRes, pagedKontragentsRes] = await Promise.all([ + query(`SELECT COALESCE(SUM(summadolga),0) as total_debt, COALESCE(SUM(summaprosrocheno),0) as total_overdue FROM ${getDebtTable()} ${where}`, params), + query(`SELECT kontragent, klassprosrochki, COALESCE(SUM(summadolga),0) as total FROM ${getDebtTable()} ${where} GROUP BY kontragent, klassprosrochki`, params), + query(`SELECT otvetstvennii, klassprosrochki, COALESCE(SUM(summadolga),0) as total FROM ${getDebtTable()} ${where} GROUP BY otvetstvennii, klassprosrochki`, params), + query(`SELECT CASE WHEN klassprosrochki IS NULL OR klassprosrochki = '0' THEN 'Не просрочено' ELSE klassprosrochki END as cls, COALESCE(SUM(summadolga),0) as total FROM ${getDebtTable()} ${where} GROUP BY cls ORDER BY total DESC`, params), + query(`SELECT COUNT(DISTINCT kontragent) as cnt FROM ${getDebtTable()} ${where}`, params), + query(`SELECT MAX(dataobnovleniyazapisi) as updated_at FROM ${getDebtTable()} ${where}`, params), + query(`SELECT kontragent, COALESCE(SUM(summadolga),0) as total_debt, COALESCE(SUM(summaprosrocheno),0) as total_overdue FROM ${getDebtTable()} ${where} GROUP BY kontragent ORDER BY total_debt DESC LIMIT ${ps} OFFSET ${offset}`, params), + ]); + + const kontragentMap = new Map(); + byKontragentRes.rows.forEach((r: any) => { + const name = r.kontragent || "Без контрагента"; + if (!kontragentMap.has(name)) kontragentMap.set(name, { name, total: 0, cls0: 0, cls1_30: 0, cls31_60: 0, cls61_90: 0, cls91_365: 0, cls365plus: 0 }); + const entry = kontragentMap.get(name)!; + const val = Number(r.total); + entry.total += val; + entry[classKey(r.klassprosrochki)] += val; + }); + const byKontragent = Array.from(kontragentMap.values()).sort((a, b) => b.total - a.total).slice(0, 10); + + const managerMap = new Map(); + byManagerRes.rows.forEach((r: any) => { + const name = r.otvetstvennii && r.otvetstvennii !== "0" ? r.otvetstvennii : "Не назначен"; + if (!managerMap.has(name)) managerMap.set(name, { name, total: 0, cls0: 0, cls1_30: 0, cls31_60: 0, cls61_90: 0, cls91_365: 0, cls365plus: 0 }); + const entry = managerMap.get(name)!; + const val = Number(r.total); + entry.total += val; + entry[classKey(r.klassprosrochki)] += val; + }); + const byManager = Array.from(managerMap.values()).sort((a, b) => b.total - a.total).slice(0, 10); + + const byOverdueClass = byClassRes.rows.map((r: any) => ({ cls: r.cls, total: Number(r.total) })); + + const pagedKontragentNames = pagedKontragentsRes.rows.map((r: any) => r.kontragent); + let pagedGroups: any[] = []; + const totalKontragents = Number(totalRes.rows[0].cnt); + + if (pagedKontragentNames.length > 0) { + const detailParams = [...params]; + const namePlaceholders = pagedKontragentNames.map((n: string) => { detailParams.push(n); return `$${detailParams.length}`; }); + const detailRes = await query(` + SELECT kontragent, dogovor, COALESCE(summadokumenta,0) as summadokumenta, dokumentraschetov, + COALESCE(summaoplacheno,0) as summaoplacheno, COALESCE(summadolga,0) as summadolga, + COALESCE(summaprosrocheno,0) as summaprosrocheno, COALESCE(otsrochkapodogovoru,0) as otsrochkapodogovoru, + COALESCE(dneiprosrocheno,0) as dneiprosrocheno + FROM ${getDebtTable()} ${where} AND kontragent IN (${namePlaceholders.join(",")}) + ORDER BY kontragent, summadolga DESC + `, detailParams); + + const groupMap = new Map(); + pagedKontragentsRes.rows.forEach((r: any) => { + groupMap.set(r.kontragent, { kontragent: r.kontragent, totalDebt: Number(r.total_debt), totalOverdue: Number(r.total_overdue), rows: [] }); + }); + detailRes.rows.forEach((r: any) => { + const g = groupMap.get(r.kontragent); + if (g) { + g.rows.push({ + dogovor: r.dogovor || "", + summadokumenta: Number(r.summadokumenta), + dokumentraschetov: r.dokumentraschetov || "", + summaoplacheno: Number(r.summaoplacheno), + summadolga: Number(r.summadolga), + summaprosrocheno: Number(r.summaprosrocheno), + otsrochkapodogovoru: Number(r.otsrochkapodogovoru), + dneiprosrocheno: Number(r.dneiprosrocheno), + }); + } + }); + pagedGroups = pagedKontragentsRes.rows.map((r: any) => groupMap.get(r.kontragent)!); + } + + return { + summary: { + updatedAt: updatedRes.rows[0]?.updated_at ? parseDate(updatedRes.rows[0].updated_at) : null, + totalDebt: Number(summaryRes.rows[0].total_debt), + totalOverdue: Number(summaryRes.rows[0].total_overdue), + }, + byKontragent, + byManager, + byOverdueClass, + table: { + rows: pagedGroups, + total: totalKontragents, + page: pg, + pageSize: ps, + totalPages: Math.ceil(totalKontragents / ps), + }, + }; + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/debt/dynamics", async (req, res) => { + try { + const cacheKey = stableKey("debt-dynamics", req.query as Record); + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const { dateFrom, dateTo, kontragent } = req.query; + const conditions: string[] = []; + const params: unknown[] = []; + + if (dateFrom) { params.push(dateFrom); conditions.push(`period >= $${params.length}`); } + if (dateTo) { params.push(dateTo); conditions.push(`period <= $${params.length}`); } + if (kontragent) { params.push(kontragent); conditions.push(`kontragent = $${params.length}`); } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + + const dbResult = await query(` + SELECT period, COALESCE(SUM(summadolga),0) as total_debt, COALESCE(SUM(summaprosrocheno),0) as total_overdue + FROM ${getDebtTable()} ${where} + GROUP BY period + ORDER BY period + `, params); + + let rows = dbResult.rows.map((r: any) => ({ + period: parseDate(r.period), + totalDebt: Number(r.total_debt), + totalOverdue: Number(r.total_overdue), + })); + + if (rows.length > 300) { + const step = Math.ceil(rows.length / 300); + rows = rows.filter((_: any, i: number) => i % step === 0 || i === rows.length - 1); + } + + return rows; + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/debt/dynamics/kontragenty", async (req, res) => { + try { + const cacheKey = stableKey("debt-dyn-kontr", req.query as Record); + const result = await cachedQuery(cacheKey, TTL_SHORT, async () => { + const { period, dateTo } = req.query; + const params: unknown[] = []; + let periodClause: string; + + if (period) { + params.push(period); + periodClause = `period = $${params.length}`; + } else if (dateTo) { + params.push(dateTo); + periodClause = `period = (SELECT MAX(period) FROM ${getDebtTable()} WHERE period <= $${params.length})`; + } else { + const mp = await getMaxPeriod(); + params.push(mp); + periodClause = `period = $${params.length}`; + } + + const where = `WHERE ${periodClause}`; + + const dbResult = await query(` + SELECT kontragent, COALESCE(SUM(summadokumenta),0) as summadokumenta, COALESCE(SUM(summaoplacheno),0) as summaoplacheno, + COALESCE(SUM(summadolga),0) as summadolga, COALESCE(SUM(summaprosrocheno),0) as summaprosrocheno, + MAX(dneiprosrocheno) as dneiprosrocheno + FROM ${getDebtTable()} ${where} + GROUP BY kontragent ORDER BY summadolga DESC + `, params); + + return dbResult.rows.map((r: any) => ({ + kontragent: r.kontragent, + summadokumenta: Number(r.summadokumenta), + summaoplacheno: Number(r.summaoplacheno), + summadolga: Number(r.summadolga), + summaprosrocheno: Number(r.summaprosrocheno), + dneiprosrocheno: Number(r.dneiprosrocheno), + })); + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/debt/filters", async (_req, res) => { + try { + const result = await cachedQuery("debt-filters", TTL_LONG, async () => { + const maxPeriod = await getMaxPeriod(); + const [periods, klassprosrochki, organizaciya, otvetstvennii, kontragenty] = await Promise.all([ + query(`SELECT DISTINCT period FROM ${getDebtTable()} ORDER BY period DESC LIMIT 30`), + query(`SELECT DISTINCT klassprosrochki FROM ${getDebtTable()} WHERE klassprosrochki IS NOT NULL ORDER BY klassprosrochki`), + query(`SELECT DISTINCT organizaciya FROM ${getDebtTable()} WHERE organizaciya IS NOT NULL ORDER BY organizaciya`), + query(`SELECT DISTINCT otvetstvennii FROM ${getDebtTable()} WHERE period = $1 AND otvetstvennii IS NOT NULL AND otvetstvennii != '0' ORDER BY otvetstvennii`, [maxPeriod]), + query(`SELECT DISTINCT kontragent FROM ${getDebtTable()} WHERE period = $1 AND kontragent IS NOT NULL ORDER BY kontragent`, [maxPeriod]), + ]); + return { + periods: periods.rows.map(r => parseDate(r.period)), + klassprosrochki: klassprosrochki.rows.map(r => r.klassprosrochki), + organizaciya: organizaciya.rows.map(r => r.organizaciya), + otvetstvennii: otvetstvennii.rows.map(r => r.otvetstvennii), + kontragenty: kontragenty.rows.map(r => r.kontragent), + }; + }); + res.json(result); + } catch (err) { + handleApiError(res, err); + } + }); + + app.get("/api/calculations/vat", async (req, res) => { + try { + const { dateFrom, dateTo } = req.query; + const incParams: unknown[] = []; + const expParams: unknown[] = []; + const incConds: string[] = ["1=1"]; + const expConds: string[] = ["1=1"]; + + if (dateFrom) { + incParams.push(dateFrom); incConds.push(`data >= $${incParams.length}`); + expParams.push(dateFrom); expConds.push(`data >= $${expParams.length}`); + } + if (dateTo) { + incParams.push(dateTo); incConds.push(`data <= $${incParams.length}`); + expParams.push(dateTo); expConds.push(`data <= $${expParams.length}`); + } + + const incWhere = `WHERE ${incConds.join(" AND ")}`; + const expWhere = `WHERE ${expConds.join(" AND ")}`; + + const [incByRate, expByRate, incMonthly, expMonthly] = await Promise.all([ + cachedQuery(stableKey("vat-inc-rate", req.query), TTL_SHORT, () => + query(`SELECT COALESCE(stavkands, 'Без НДС') as rate, COUNT(*) as cnt, COALESCE(SUM(summa), 0) as total FROM ${getIncomeEffectiveView()} ${incWhere} GROUP BY COALESCE(stavkands, 'Без НДС') ORDER BY total DESC`, incParams) + ), + cachedQuery(stableKey("vat-exp-rate", req.query), TTL_SHORT, () => + query(`SELECT COALESCE(stavkands, 'Без НДС') as rate, COUNT(*) as cnt, COALESCE(SUM(summadokumenta), 0) as total FROM ${getExpenseEffectiveView()} ${expWhere} GROUP BY COALESCE(stavkands, 'Без НДС') ORDER BY total DESC`, expParams) + ), + cachedQuery(stableKey("vat-inc-monthly", req.query), TTL_SHORT, () => + query(`SELECT TO_CHAR(data, 'YYYY-MM') as period, COALESCE(stavkands, 'Без НДС') as rate, COALESCE(SUM(summa), 0) as total FROM ${getIncomeEffectiveView()} ${incWhere} GROUP BY TO_CHAR(data, 'YYYY-MM'), COALESCE(stavkands, 'Без НДС') ORDER BY period`, incParams) + ), + cachedQuery(stableKey("vat-exp-monthly", req.query), TTL_SHORT, () => + query(`SELECT TO_CHAR(data, 'YYYY-MM') as period, COALESCE(stavkands, 'Без НДС') as rate, COALESCE(SUM(summadokumenta), 0) as total FROM ${getExpenseEffectiveView()} ${expWhere} GROUP BY TO_CHAR(data, 'YYYY-MM'), COALESCE(stavkands, 'Без НДС') ORDER BY period`, expParams) + ), + ]); + + const parseRate = (r: string): number => { + const m = r.match(/(\d+)/); + return m ? Number(m[1]) / 100 : 0; + }; + + const incomeByRate = incByRate.rows.map((r: any) => ({ + rate: r.rate, + count: Number(r.cnt), + total: Number(r.total), + vat: Number(r.total) * parseRate(r.rate) / (1 + parseRate(r.rate)), + })); + + const expenseByRate = expByRate.rows.map((r: any) => ({ + rate: r.rate, + count: Number(r.cnt), + total: Number(r.total), + vat: Number(r.total) * parseRate(r.rate) / (1 + parseRate(r.rate)), + })); + + const totalIncomeVat = incomeByRate.reduce((s: number, r: any) => s + r.vat, 0); + const totalExpenseVat = expenseByRate.reduce((s: number, r: any) => s + r.vat, 0); + + const periodsSet = new Set(); + [...incMonthly.rows, ...expMonthly.rows].forEach((r: any) => periodsSet.add(r.period)); + const periods = [...periodsSet].sort(); + + const monthly = periods.map(p => { + let incVat = 0; + let expVat = 0; + incMonthly.rows.filter((r: any) => r.period === p).forEach((r: any) => { + const rate = parseRate(r.rate); + incVat += Number(r.total) * rate / (1 + rate); + }); + expMonthly.rows.filter((r: any) => r.period === p).forEach((r: any) => { + const rate = parseRate(r.rate); + expVat += Number(r.total) * rate / (1 + rate); + }); + return { period: p, incomeVat: Math.round(incVat * 100) / 100, expenseVat: Math.round(expVat * 100) / 100, toPay: Math.round((incVat - expVat) * 100) / 100 }; + }); + + res.json({ + incomeByRate, + expenseByRate, + totalIncomeVat: Math.round(totalIncomeVat * 100) / 100, + totalExpenseVat: Math.round(totalExpenseVat * 100) / 100, + toPay: Math.round((totalIncomeVat - totalExpenseVat) * 100) / 100, + monthly, + }); + } catch (err) { + handleApiError(res, err); + } + }); + + app.post("/api/calculations/profit-tax", async (req, res) => { + try { + await ensureTreesReady(); + const { dateFrom, dateTo, taxRate: taxRateParam } = req.query; + const { incomeCategories, expenseCategories, chartCategories } = req.body || {}; + + const parseRate = (r: string): number => { const m = r.match(/(\d+)/); return m ? Number(m[1]) / 100 : 0; }; + const isNoVatRate = (r: string): boolean => parseRate(r) === 0; + + const tableNames = await getPrimaryTreeTables(); + const [incExistArr, expExistArr] = await Promise.all([ + Promise.all(tableNames.income.map(t => treeTableExists(t))), + Promise.all(tableNames.expense.map(t => treeTableExists(t))), + ]); + const incTables = tableNames.income.filter((_, i) => incExistArr[i]); + const expTables = tableNames.expense.filter((_, i) => expExistArr[i]); + + if (incTables.length === 0 || expTables.length === 0) { + scheduleTreeRebuild(); + return res.status(202).json({ error: "Таблицы деревьев строятся, повторите запрос через несколько секунд", rebuilding: true }); + } + + function buildUnionParams(tables: string[], amountCol: string, categoryIds?: string[]): { byRateSQL: string; monthlySQL: string; params: unknown[] } { + const allParams: unknown[] = []; + const selects = tables.map(t => { + const conds: string[] = ["_is_excluded = FALSE"]; + if (categoryIds && categoryIds.length > 0) { + const ph = categoryIds.map(id => { allParams.push(id); return `$${allParams.length}`; }); + conds.push(`_category_id IN (${ph.join(",")})`); + } + if (dateFrom) { allParams.push(dateFrom); conds.push(`data >= $${allParams.length}`); } + if (dateTo) { allParams.push(dateTo); conds.push(`data <= $${allParams.length}`); } + return `SELECT ${amountCol} as amount, COALESCE(stavkands,'Без НДС') as rate, data FROM "${t}" WHERE ${conds.join(" AND ")}`; + }); + const unionSQL = selects.join(" UNION ALL "); + const byRateSQL = `SELECT rate, COALESCE(SUM(amount),0) as total FROM (${unionSQL}) _u GROUP BY rate`; + const monthlySQL = `SELECT TO_CHAR(data,'YYYY-MM') as period, rate, COALESCE(SUM(amount),0) as total FROM (${unionSQL}) _u GROUP BY TO_CHAR(data,'YYYY-MM'), rate ORDER BY period`; + return { byRateSQL, monthlySQL, params: allParams }; + } + + const incCatFilter = incomeCategories && Array.isArray(incomeCategories) && incomeCategories.length > 0 ? incomeCategories as string[] : undefined; + const expCatFilter = expenseCategories && Array.isArray(expenseCategories) && expenseCategories.length > 0 ? expenseCategories as string[] : undefined; + + const incUnion = buildUnionParams(incTables, "summa", incCatFilter); + const expUnion = buildUnionParams(expTables, "summadokumenta", expCatFilter); + + const [incByRate, expByRate, incMonthly, expMonthly] = await Promise.all([ + query(incUnion.byRateSQL, incUnion.params), + query(expUnion.byRateSQL, expUnion.params), + query(incUnion.monthlySQL, incUnion.params), + query(expUnion.monthlySQL, expUnion.params), + ]); + + let incomeWithVat = 0, incomeVatAmount = 0, incomeNoVat = 0, totalIncome = 0; + for (const r of incByRate.rows as any[]) { + const total = Number(r.total); + totalIncome += total; + if (isNoVatRate(r.rate)) { incomeNoVat += total; } + else { const rate = parseRate(r.rate); const vat = total * rate / (1 + rate); incomeWithVat += total; incomeVatAmount += vat; } + } + + let expenseWithVat = 0, expenseVatAmount = 0, expenseNoVat = 0, totalExpenses = 0; + for (const r of expByRate.rows as any[]) { + const total = Number(r.total); + totalExpenses += total; + if (isNoVatRate(r.rate)) { expenseNoVat += total; } + else { const rate = parseRate(r.rate); const vat = total * rate / (1 + rate); expenseWithVat += total; expenseVatAmount += vat; } + } + + const incomeWithVatBase = incomeWithVat - incomeVatAmount; + const expenseWithVatBase = expenseWithVat - expenseVatAmount; + const profit = (incomeWithVatBase - expenseWithVatBase) + (incomeNoVat - expenseNoVat); + const taxRate = taxRateParam ? Number(taxRateParam) / 100 : 0.20; + const tax = profit > 0 ? profit * taxRate : 0; + + const periodsSet = new Set(); + [...(incMonthly.rows as any[]), ...(expMonthly.rows as any[])].forEach((r: any) => periodsSet.add(r.period)); + const periods = [...periodsSet].sort(); + + const monthly = periods.map(p => { + let mIncVat = 0, mIncVatAmt = 0, mIncNoVat = 0, mInc = 0; + for (const r of incMonthly.rows as any[]) { + if (r.period !== p) continue; + const total = Number(r.total); mInc += total; + if (isNoVatRate(r.rate)) { mIncNoVat += total; } + else { const rate = parseRate(r.rate); mIncVat += total; mIncVatAmt += total * rate / (1 + rate); } + } + let mExpVat = 0, mExpVatAmt = 0, mExpNoVat = 0, mExp = 0; + for (const r of expMonthly.rows as any[]) { + if (r.period !== p) continue; + const total = Number(r.total); mExp += total; + if (isNoVatRate(r.rate)) { mExpNoVat += total; } + else { const rate = parseRate(r.rate); mExpVat += total; mExpVatAmt += total * rate / (1 + rate); } + } + const prof = ((mIncVat - mIncVatAmt) - (mExpVat - mExpVatAmt)) + (mIncNoVat - mExpNoVat); + return { period: p, income: Math.round(mInc * 100) / 100, expenses: Math.round(mExp * 100) / 100, profit: Math.round(prof * 100) / 100, tax: Math.round((prof > 0 ? prof * taxRate : 0) * 100) / 100 }; + }); + + // Excluded categories from materialized tables + function buildExcludedUnion(tables: string[], amountCol: string): { sql: string; params: unknown[] } { + const allParams: unknown[] = []; + const selects = tables.map(t => { + const conds: string[] = ["_is_excluded = TRUE"]; + if (dateFrom) { allParams.push(dateFrom); conds.push(`data >= $${allParams.length}`); } + if (dateTo) { allParams.push(dateTo); conds.push(`data <= $${allParams.length}`); } + return `SELECT _category_name, ${amountCol} as amount FROM "${t}" WHERE ${conds.join(" AND ")}`; + }); + return { sql: `SELECT _category_name, COALESCE(SUM(amount),0) as total FROM (${selects.join(" UNION ALL ")}) _e GROUP BY _category_name`, params: allParams }; + } + + const incExcl = buildExcludedUnion(incTables, "summa"); + const expExcl = buildExcludedUnion(expTables, "summadokumenta"); + const [incExclResult, expExclResult] = await Promise.all([ + query(incExcl.sql, incExcl.params), + query(expExcl.sql, expExcl.params), + ]); + const excludedIncomeCategories = (incExclResult.rows as any[]).filter(r => Number(r.total) > 0).map(r => ({ name: r._category_name, total: Math.round(Number(r.total) * 100) / 100 })); + const excludedExpenseCategories = (expExclResult.rows as any[]).filter(r => Number(r.total) > 0).map(r => ({ name: r._category_name, total: Math.round(Number(r.total) * 100) / 100 })); + + let categoryMonthly: { id: string; name: string; type: string; data: { period: string; amount: number }[] }[] = []; + if (chartCategories && Array.isArray(chartCategories) && chartCategories.length > 0) { + const chartCatPromises = (chartCategories as { id: string; type: string; name?: string }[]).map(async (cc) => { + const tables = cc.type === "income" ? incTables : expTables; + const amountCol = cc.type === "income" ? "summa" : "summadokumenta"; + const catParams: unknown[] = []; + const catSelects = tables.map(t => { + const conds: string[] = ["_is_excluded = FALSE"]; + catParams.push(cc.id); + conds.push(`_category_id = $${catParams.length}`); + if (dateFrom) { catParams.push(dateFrom); conds.push(`data >= $${catParams.length}`); } + if (dateTo) { catParams.push(dateTo); conds.push(`data <= $${catParams.length}`); } + return `SELECT ${amountCol} as amount, data FROM "${t}" WHERE ${conds.join(" AND ")}`; + }); + const catSQL = `SELECT TO_CHAR(data,'YYYY-MM') as period, COALESCE(SUM(amount),0) as total FROM (${catSelects.join(" UNION ALL ")}) _cc GROUP BY TO_CHAR(data,'YYYY-MM') ORDER BY period`; + const catResult = await query(catSQL, catParams); + return { + id: cc.id, + name: cc.name || cc.id, + type: cc.type, + data: (catResult.rows as any[]).map((r: any) => ({ period: r.period, amount: Math.round(Number(r.total) * 100) / 100 })), + }; + }); + categoryMonthly = await Promise.all(chartCatPromises); + } + + const r2 = (v: number) => Math.round(v * 100) / 100; + res.json({ + totalIncome: r2(totalIncome), totalExpenses: r2(totalExpenses), + incomeWithVat: r2(incomeWithVat), incomeVatAmount: r2(incomeVatAmount), incomeWithVatBase: r2(incomeWithVatBase), incomeNoVat: r2(incomeNoVat), + expenseWithVat: r2(expenseWithVat), expenseVatAmount: r2(expenseVatAmount), expenseWithVatBase: r2(expenseWithVatBase), expenseNoVat: r2(expenseNoVat), + profit: r2(profit), taxRate, tax: r2(tax), monthly, + excludedIncomeCategories, excludedExpenseCategories, + categoryMonthly, + }); + } catch (err) { + handleApiError(res, err); + } + }); + + app.post("/api/planning/forecast", async (req, res) => { + try { + const minMonths = Math.max(2, Math.min(12, Number(req.body?.minConsecutiveMonths) || 2)); + const tolerancePct = Math.max(0, Math.min(100, Number(req.body?.tolerancePercent) || 20)); + + const [incResult, expResult] = await Promise.all([ + cachedQuery("planning-inc-monthly", TTL_SHORT, () => + query(`SELECT zakazchik, dogovorzakazchika, TO_CHAR(data, 'YYYY-MM') as period, SUM(summa) as total, COUNT(*) as cnt FROM ${getIncomeEffectiveView()} WHERE zakazchik IS NOT NULL AND zakazchik != '0' AND dogovorzakazchika IS NOT NULL AND dogovorzakazchika != '0' GROUP BY zakazchik, dogovorzakazchika, TO_CHAR(data, 'YYYY-MM') ORDER BY period`) + ), + cachedQuery("planning-exp-monthly", TTL_SHORT, () => + query(`SELECT zakazchik, dogovorzakazchika, TO_CHAR(data, 'YYYY-MM') as period, SUM(summadokumenta) as total, COUNT(*) as cnt FROM ${getExpenseEffectiveView()} WHERE zakazchik IS NOT NULL AND zakazchik != '0' AND dogovorzakazchika IS NOT NULL AND dogovorzakazchika != '0' GROUP BY zakazchik, dogovorzakazchika, TO_CHAR(data, 'YYYY-MM') ORDER BY period`) + ), + ]); + + const allPeriods = new Set(); + for (const r of incResult.rows as any[]) allPeriods.add(r.period); + for (const r of expResult.rows as any[]) allPeriods.add(r.period); + const sortedPeriods = [...allPeriods].sort(); + if (sortedPeriods.length === 0) { + return res.json({ historical: [], forecast: [], recurringIncome: [], recurringExpenses: [], lastPeriod: null }); + } + + const lastPeriod = sortedPeriods[sortedPeriods.length - 1]; + + function detectRecurring(rows: any[], datasetPeriods: Set) { + const byKey = new Map>(); + for (const r of rows) { + const key = `${r.zakazchik}|||${r.dogovorzakazchika}`; + if (!byKey.has(key)) byKey.set(key, new Map()); + const existing = byKey.get(key)!.get(r.period) || 0; + byKey.get(key)!.set(r.period, existing + Number(r.total)); + } + + const dsLatest = [...datasetPeriods].sort(); + const dsLastPeriod = dsLatest.length > 0 ? dsLatest[dsLatest.length - 1] : lastPeriod; + const dsSecondLast = dsLatest.length > 1 ? dsLatest[dsLatest.length - 2] : null; + + const recurring: { name: string; contract: string; avgAmount: number; months: number; monthlyAmounts: Map }[] = []; + + for (const [key, periodMap] of byKey) { + const periods = [...periodMap.keys()].sort(); + if (periods.length < minMonths) continue; + + const runs: string[][] = []; + let currentRun: string[] = [periods[0]]; + for (let i = 1; i < periods.length; i++) { + const prevDate = new Date(periods[i - 1] + "-01"); + const expectedNext = new Date(prevDate); + expectedNext.setMonth(expectedNext.getMonth() + 1); + const expectedStr = expectedNext.toISOString().substring(0, 7); + if (periods[i] === expectedStr) { + currentRun.push(periods[i]); + } else { + runs.push(currentRun); + currentRun = [periods[i]]; + } + } + runs.push(currentRun); + + let selectedRun: string[] | null = null; + for (let i = runs.length - 1; i >= 0; i--) { + const run = runs[i]; + if (run.length < minMonths) continue; + const runEnd = run[run.length - 1]; + if (runEnd === dsLastPeriod || runEnd === dsSecondLast) { + selectedRun = run; + break; + } + } + if (!selectedRun) continue; + + const amounts = selectedRun.map(p => periodMap.get(p) || 0); + const avg = amounts.reduce((a, b) => a + b, 0) / amounts.length; + if (avg <= 0) continue; + + const withinTolerance = amounts.every(a => Math.abs(a - avg) / avg * 100 <= tolerancePct); + if (!withinTolerance) { + const median = [...amounts].sort((a, b) => a - b)[Math.floor(amounts.length / 2)]; + const medianCheck = amounts.every(a => Math.abs(a - median) / median * 100 <= tolerancePct); + if (!medianCheck) continue; + } + + const [zakazchik, contract] = key.split("|||"); + const monthlyAmounts = new Map(); + for (const p of selectedRun) monthlyAmounts.set(p, periodMap.get(p) || 0); + recurring.push({ name: zakazchik, contract, avgAmount: Math.round(avg * 100) / 100, months: selectedRun.length, monthlyAmounts }); + } + return recurring; + } + + const incPeriods = new Set((incResult.rows as any[]).map((r: any) => r.period)); + const expPeriods = new Set((expResult.rows as any[]).map((r: any) => r.period)); + const recurringIncome = detectRecurring(incResult.rows as any[], incPeriods); + const recurringExpenses = detectRecurring(expResult.rows as any[], expPeriods); + + const historical = new Map(); + for (const p of sortedPeriods) { + historical.set(p, { income: 0, expenses: 0 }); + } + for (const r of incResult.rows as any[]) { + const h = historical.get(r.period); + if (h) h.income += Number(r.total); + } + for (const r of expResult.rows as any[]) { + const h = historical.get(r.period); + if (h) h.expenses += Number(r.total); + } + + const historicalData = sortedPeriods.map(p => { + const h = historical.get(p)!; + return { + period: p, + income: Math.round(h.income * 100) / 100, + expenses: Math.round(h.expenses * 100) / 100, + net: Math.round((h.income - h.expenses) * 100) / 100, + type: "historical" as const, + }; + }); + + const forecastMonths: string[] = []; + const lastDate = new Date(lastPeriod + "-01"); + for (let i = 1; i <= 12; i++) { + const d = new Date(lastDate); + d.setMonth(d.getMonth() + i); + forecastMonths.push(d.toISOString().substring(0, 7)); + } + + const forecastData = forecastMonths.map(p => { + let forecastIncome = 0; + let forecastExpenses = 0; + for (const ri of recurringIncome) forecastIncome += ri.avgAmount; + for (const re of recurringExpenses) forecastExpenses += re.avgAmount; + return { + period: p, + income: Math.round(forecastIncome * 100) / 100, + expenses: Math.round(forecastExpenses * 100) / 100, + net: Math.round((forecastIncome - forecastExpenses) * 100) / 100, + type: "forecast" as const, + }; + }); + + res.json({ + historical: historicalData, + forecast: forecastData, + recurringIncome: recurringIncome.map(r => ({ name: r.name, contract: r.contract, avgAmount: r.avgAmount, months: r.months })), + recurringExpenses: recurringExpenses.map(r => ({ name: r.name, contract: r.contract, avgAmount: r.avgAmount, months: r.months })), + lastPeriod, + }); + } catch (err) { + handleApiError(res, err); + } + }); + + // --- AI Planning Analysis --- + const PLANNING_ITEMS_PATH = path.join(process.cwd(), "data", "planning-items.json"); + + interface PlanItem { + id: string; + type: "income" | "expense"; + name: string; + contract: string; + monthlyAmount: number; + confidence: string; + recurring?: boolean; + reasoning: string; + enabled: boolean; + source: "ai" | "manual"; + } + + function loadPlanItems(): PlanItem[] { + try { + if (fs.existsSync(PLANNING_ITEMS_PATH)) { + return JSON.parse(fs.readFileSync(PLANNING_ITEMS_PATH, "utf-8")); + } + } catch {} + return []; + } + + function savePlanItems(items: PlanItem[]): void { + fs.writeFileSync(PLANNING_ITEMS_PATH, JSON.stringify(items, null, 2)); + } + + app.get("/api/planning/saved", (_req, res) => { + res.json(loadPlanItems()); + }); + + app.post("/api/planning/saved", (req, res) => { + try { + const items = req.body; + if (!Array.isArray(items)) return res.status(400).json({ error: "Ожидается массив" }); + if (items.length > 200) return res.status(400).json({ error: "Слишком много элементов (макс. 200)" }); + const validated: PlanItem[] = items.map((it: any, idx: number) => ({ + id: String(it.id || `item-${idx}`), + type: it.type === "income" ? "income" : "expense", + name: String(it.name || "").slice(0, 500), + contract: String(it.contract || "").slice(0, 500), + monthlyAmount: typeof it.monthlyAmount === "number" ? it.monthlyAmount : 0, + confidence: ["high", "medium", "low"].includes(it.confidence) ? it.confidence : "medium", + recurring: typeof it.recurring === "boolean" ? it.recurring : false, + reasoning: String(it.reasoning || "").slice(0, 1000), + enabled: Boolean(it.enabled), + source: it.source === "manual" ? "manual" : "ai", + })); + savePlanItems(validated); + res.json({ ok: true }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка сохранения" }); + } + }); + + app.get("/api/planning/categories", async (_req, res) => { + try { + const incomeCats = loadCategories(); + const expenseCats = loadExpenseCategories(); + const mapCats = (cats: IncomeCategory[]): { id: string; name: string }[] => + cats.filter(c => !c.excluded).map(c => ({ id: c.id, name: c.name })); + res.json({ income: mapCats(incomeCats), expenses: mapCats(expenseCats) }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка загрузки категорий" }); + } + }); + + app.post("/api/planning/ai-analyze", async (req, res) => { + const startTime = Date.now(); + try { + const cfg = loadAiPlanningConfig(); + if (!cfg.apiKey) return res.status(400).json({ error: "AI Аналитик не настроен. Укажите API ключ в настройках AI (вкладка Аналитик)." }); + + const months = Math.max(3, Math.min(24, Number(req.body?.months) || 6)); + const selectedExpenseIds: string[] = Array.isArray(req.body?.expenseCategories) ? req.body.expenseCategories : []; + + const cutoffDate = new Date(); + cutoffDate.setMonth(cutoffDate.getMonth() - months); + const cutoff = cutoffDate.toISOString().substring(0, 10); + + const incomeCats = loadCategories(); + const expenseCats = loadExpenseCategories(); + + const topIncomeIds = incomeCats.filter(c => !c.excluded).map(c => c.id); + const effectiveExpenseIds = selectedExpenseIds.length > 0 + ? selectedExpenseIds + : expenseCats.filter(c => !c.excluded).map(c => c.id); + const useSelectedExpenseFilter = selectedExpenseIds.length > 0; + + async function queryCategoryData( + table: string, + sumCol: string, + cats: IncomeCategory[], + selectedIds: string[], + allowedCols: string[], + cutoffStr: string, + ): Promise<{ byCat: Map; names: Map }> { + const byCat = new Map(); + const names = new Map(); + const allFlat = flattenCategories(cats); + + const BATCH_SIZE = 10; + for (let i = 0; i < selectedIds.length; i += BATCH_SIZE) { + const batch = selectedIds.slice(i, i + BATCH_SIZE); + await Promise.all(batch.map(async (catId) => { + const cat = allFlat.find(c => c.id === catId); + if (!cat) return; + names.set(catId, cat.name); + const params: unknown[] = []; + const conds = buildCategoryWithAncestorsConditions(cats, catId, params, allowedCols); + params.push(cutoffStr); + const dateIdx = params.length; + const whereClause = conds.length > 0 + ? `WHERE data >= $${dateIdx}::date AND ${conds.join(" AND ")}` + : `WHERE data >= $${dateIdx}::date`; + const sql = `SELECT TO_CHAR(data, 'YYYY-MM') as period, SUM(${sumCol}::numeric) as total FROM ${table} ${whereClause} GROUP BY TO_CHAR(data, 'YYYY-MM') ORDER BY period`; + const result = await query(sql, params); + const entry = { periods: [] as string[], amounts: [] as number[], total: 0 }; + for (const r of result.rows as any[]) { + entry.periods.push(r.period); + const amt = Number(r.total); + entry.amounts.push(amt); + entry.total += amt; + } + if (entry.total > 0) byCat.set(catId, entry); + })); + } + return { byCat, names }; + } + + const baseQueries = [ + query(`SELECT TO_CHAR(data, 'YYYY-MM') as period, SUM(summa::numeric) as total FROM ${getIncomeEffectiveView()} WHERE data >= '${cutoff}'::date GROUP BY TO_CHAR(data, 'YYYY-MM') ORDER BY period`), + query(`SELECT TO_CHAR(data, 'YYYY-MM') as period, SUM(summadokumenta::numeric) as total FROM ${getExpenseEffectiveView()} WHERE data >= '${cutoff}'::date GROUP BY TO_CHAR(data, 'YYYY-MM') ORDER BY period`), + query(`SELECT COALESCE(SUM(summadolga::numeric),0) as total_debt, COALESCE(SUM(summaprosrocheno::numeric),0) as total_overdue, COUNT(DISTINCT kontragent) as cnt FROM ${getDebtTable()} WHERE period = (SELECT MAX(period) FROM ${getDebtTable()})`), + query(`SELECT kontragent, dogovor, COALESCE(SUM(summadokumenta::numeric),0) as doc_total, COALESCE(SUM(summaoplacheno::numeric),0) as paid_total, COALESCE(SUM(summadolga::numeric),0) as debt_total, COALESCE(SUM(summaprosrocheno::numeric),0) as overdue_total, MAX(COALESCE(dneiprosrocheno::numeric,0)) as max_days FROM ${getDebtTable()} WHERE period = (SELECT MAX(period) FROM ${getDebtTable()}) GROUP BY kontragent, dogovor ORDER BY debt_total DESC LIMIT 20`), + query(`SELECT CASE WHEN klassprosrochki IS NULL OR klassprosrochki = '0' THEN 'Не просрочено' ELSE klassprosrochki END as cls, COALESCE(SUM(summadolga::numeric),0) as total FROM ${getDebtTable()} WHERE period = (SELECT MAX(period) FROM ${getDebtTable()}) GROUP BY cls ORDER BY total DESC`), + ]; + + const baseResults = await Promise.all(baseQueries); + const [incTotals, expTotals, debtSummary, debtTop, debtByClass] = baseResults; + + console.log(`[AI Planning] Querying categories: income=${topIncomeIds.length}, expense=${effectiveExpenseIds.length}${useSelectedExpenseFilter ? " (выбраны)" : " (все)"}`); + + async function fetchSalarySummary(monthsCount: number): Promise { + try { + const status = getAuthStatus(); + if (!status.loggedIn) return ""; + const areas = loadAreas(); + if (areas.length === 0) return ""; + + let salaryText = `\n## Зарплаты по участкам (Google Sheets)\n`; + let hasData = false; + + for (const area of areas) { + try { + const sheetNames = await getSheetNames(area.spreadsheetId); + const monthSheets = sheetNames + .filter(name => MONTH_PATTERN.test(name)) + .sort((a, b) => { + const [, am, ay] = a.match(MONTH_PATTERN)!; + const [, bm, by] = b.match(MONTH_PATTERN)!; + return (Number(ay) * 12 + Number(am)) - (Number(by) * 12 + Number(bm)); + }) + .slice(-monthsCount); + + if (monthSheets.length === 0) continue; + + const fioIdx = colLetterToIndex(area.fioCol || "B"); + const bazaIdx = colLetterToIndex(area.bazaCol || "F"); + const kVydacheIdx = colLetterToIndex(area.kVydacheCol || "J"); + const lastCol = maxColLetter(area.fioCol || "B", area.bazaCol || "F", area.kVydacheCol || "J"); + + const monthlyTotals: { month: string; totalBaza: number; totalKVydache: number; empCount: number }[] = []; + for (const sheetName of monthSheets) { + const rows = await getSheetData(area.spreadsheetId, `'${sheetName}'!A4:${lastCol}500`); + let totalBaza = 0, totalKVydache = 0, empCount = 0; + for (const row of rows) { + if (!row || !row[fioIdx] || !String(row[fioIdx]).trim()) continue; + const fio = String(row[fioIdx] || "").replace(/[0-9]/g, "").replace(/[^\p{L}\s\-().]/gu, "").replace(/\s+/g, " ").trim(); + if (!fio) continue; + totalBaza += Number(String(row[bazaIdx] || "0").replace(/\s/g, "").replace(",", ".")) || 0; + totalKVydache += Number(String(row[kVydacheIdx] || "0").replace(/\s/g, "").replace(",", ".")) || 0; + empCount++; + } + if (empCount > 0) monthlyTotals.push({ month: sheetName, totalBaza, totalKVydache, empCount }); + } + + if (monthlyTotals.length > 0) { + hasData = true; + salaryText += `- ${area.name} | Месяцев: ${monthlyTotals.length} | Периоды: ${monthlyTotals.map(m => m.month).join(",")} | К выдаче: ${monthlyTotals.map(m => m.totalKVydache.toFixed(0)).join(",")} | Сотрудников: ${monthlyTotals.map(m => m.empCount).join(",")}\n`; + } + } catch (err: any) { + console.error(`[AI Planning] Salary area ${area.name} error:`, err?.message); + } + } + return hasData ? salaryText : ""; + } catch (err: any) { + console.error("[AI Planning] Salary fetch error:", err?.message); + return ""; + } + } + + const [incCatData, expCatData, salarySection] = await Promise.all([ + queryCategoryData(getIncomeEffectiveView(), "summa", incomeCats, topIncomeIds, ALLOWED_INCOME_COLUMNS(), cutoff), + queryCategoryData(getExpenseEffectiveView(), "summadokumenta", expenseCats, effectiveExpenseIds, ALLOWED_EXPENSE_COLUMNS(), cutoff), + fetchSalarySummary(months), + ]); + + let dataSummary = `Период анализа: последние ${months} месяцев (с ${cutoff})\n\n`; + dataSummary += `## Общие итоги по месяцам\n`; + dataSummary += `Доходы: ${(incTotals.rows as any[]).map((r: any) => `${r.period}: ${Number(r.total).toFixed(0)}`).join(", ")}\n`; + dataSummary += `Расходы: ${(expTotals.rows as any[]).map((r: any) => `${r.period}: ${Number(r.total).toFixed(0)}`).join(", ")}\n\n`; + + { + const sortedInc = [...incCatData.byCat.entries()].sort((a, b) => b[1].total - a[1].total); + dataSummary += `## Доходы по категориям (${sortedInc.length} из ${topIncomeIds.length}, excluded исключены)\n`; + for (const [catId, data] of sortedInc) { + dataSummary += `- ${incCatData.names.get(catId) || catId} | Месяцев: ${data.periods.length} | Итого: ${data.total.toFixed(0)} | Периоды: ${data.periods.join(",")} | Суммы: ${data.amounts.map(a => a.toFixed(0)).join(",")}\n`; + } + } + + { + const sortedExp = [...expCatData.byCat.entries()].sort((a, b) => b[1].total - a[1].total); + dataSummary += `\n## Расходы по категориям (${sortedExp.length} из ${effectiveExpenseIds.length}${useSelectedExpenseFilter ? ", выбраны вручную" : ", excluded исключены"})\n`; + for (const [catId, data] of sortedExp) { + dataSummary += `- ${expCatData.names.get(catId) || catId} | Месяцев: ${data.periods.length} | Итого: ${data.total.toFixed(0)} | Периоды: ${data.periods.join(",")} | Суммы: ${data.amounts.map(a => a.toFixed(0)).join(",")}\n`; + } + } + + const ds = debtSummary.rows[0] as any; + dataSummary += `\n## Дебиторская задолженность\n`; + dataSummary += `Общий долг: ${Number(ds?.total_debt || 0).toFixed(0)} руб | Просрочено: ${Number(ds?.total_overdue || 0).toFixed(0)} руб | Контрагентов: ${ds?.cnt || 0}\n\n`; + dataSummary += `Распределение по классам просрочки:\n`; + for (const r of debtByClass.rows as any[]) { + dataSummary += `- ${r.cls}: ${Number(r.total).toFixed(0)} руб\n`; + } + dataSummary += `\nТоп-20 должников:\n`; + for (const r of debtTop.rows as any[]) { + dataSummary += `- ${r.kontragent} | ${r.dogovor || "б/д"} | Выставлено: ${Number(r.doc_total).toFixed(0)} | Оплачено: ${Number(r.paid_total).toFixed(0)} | Долг: ${Number(r.debt_total).toFixed(0)} | Просрочка: ${Number(r.overdue_total).toFixed(0)} | Дней: ${Number(r.max_days).toFixed(0)}\n`; + } + + if (salarySection) { + dataSummary += salarySection; + } + + console.log(`[AI Planning] Data summary: ${dataSummary.length} chars, inc=кат:${incCatData.byCat.size}, exp=кат:${expCatData.byCat.size}${useSelectedExpenseFilter ? "(выбраны)" : ""}${salarySection ? ", +зарплаты" : ""}, sending to AI...`); + + const planningPrompt = cfg.systemPrompt || "Ты — финансовый аналитик. Проанализируй данные и верни JSON-массив прогнозных позиций."; + + const messages = [ + { role: "system", content: planningPrompt }, + { role: "user", content: dataSummary }, + ]; + + const apiUrl = cfg.apiUrl.replace(/\/+$/, ""); + const resp = await fetch(`${apiUrl}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${cfg.apiKey}`, + }, + body: JSON.stringify({ + model: cfg.model, + messages, + temperature: 0.1, + max_tokens: 6000, + }), + signal: AbortSignal.timeout(180000), + }); + + if (!resp.ok) { + const errText = await resp.text(); + const errorMsg = `Ошибка AI API (${resp.status}): ${errText.slice(0, 300)}`; + saveLogEntry({ + timestamp: new Date().toISOString(), + chatId: "web", + chatName: "Планирование", + question: `[Аналитик] Анализ ${months} мес. (${dataSummary.length} символов)`, + answer: errorMsg, + sqlQueries: [], + }); + return res.status(502).json({ error: errorMsg }); + } + + const json = await resp.json() as any; + const content = json.choices?.[0]?.message?.content || ""; + + let items: any[] = []; + try { + const cleanContent = content + .replace(/```json\s*/gi, "") + .replace(/```\s*/g, "") + .replace(/[\s\S]*?<\/think>/g, "") + .trim(); + const jsonMatch = cleanContent.match(/\[[\s\S]*\]/); + if (jsonMatch) { + items = JSON.parse(jsonMatch[0]); + } + } catch (parseErr) { + console.error("[AI Planning] Failed to parse AI response:", parseErr); + saveLogEntry({ + timestamp: new Date().toISOString(), + chatId: "web", + chatName: "Планирование", + question: `[Аналитик] Анализ ${months} мес. (${dataSummary.length} символов)`, + answer: `Ошибка парсинга: ${content.slice(0, 300)}`, + sqlQueries: [], + }); + return res.status(502).json({ error: "AI вернул некорректный формат. Попробуйте ещё раз.", raw: content.slice(0, 500) }); + } + + const validated = items + .filter((it: any) => it && (it.type === "income" || it.type === "expense") && it.name && typeof it.monthlyAmount === "number") + .map((it: any, idx: number) => ({ + id: `ai-${Date.now()}-${idx}`, + type: it.type, + name: it.name, + contract: it.contract || "", + monthlyAmount: Math.round(it.monthlyAmount * 100) / 100, + confidence: ["high", "medium", "low"].includes(it.confidence) ? it.confidence : "medium", + recurring: typeof it.recurring === "boolean" ? it.recurring : false, + reasoning: it.reasoning || "", + enabled: true, + source: "ai" as const, + })); + + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + console.log(`[AI Planning] Analyzed ${months} months, got ${validated.length} plan items in ${elapsed}s`); + + saveLogEntry({ + timestamp: new Date().toISOString(), + chatId: "web", + chatName: "Планирование", + question: `[Аналитик] Анализ ${months} мес. (${dataSummary.length} символов, кат.доходов: ${incCatData.byCat.size}, кат.расходов: ${expCatData.byCat.size}${useSelectedExpenseFilter ? " (выбраны)" : ""}, должников: ${(debtTop.rows as any[]).length})`, + answer: `Найдено ${validated.length} позиций (${validated.filter(i => i.type === "income").length} доходных, ${validated.filter(i => i.type === "expense").length} расходных). Время: ${elapsed}с`, + sqlQueries: [], + }); + + res.json(validated); + } catch (err: any) { + console.error("[AI Planning]", err?.message); + const elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + const errorMsg = err?.message?.includes("abort") || err?.message?.includes("timeout") + ? `Превышено время ожидания ответа от AI (${elapsed}с). Попробуйте ещё раз.` + : `Ошибка анализа: ${err?.message || "неизвестная ошибка"}`; + saveLogEntry({ + timestamp: new Date().toISOString(), + chatId: "web", + chatName: "Планирование", + question: `[Аналитик] Анализ (ошибка)`, + answer: errorMsg, + sqlQueries: [], + }); + if (err?.message?.includes("abort") || err?.message?.includes("timeout")) { + return res.status(504).json({ error: errorMsg }); + } + res.status(500).json({ error: errorMsg }); + } + }); + + // --- Google Auth routes --- + app.get("/api/auth/google", (_req, res) => { + try { + const url = getAuthUrl(); + res.redirect(url); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка авторизации Google" }); + } + }); + + app.get("/api/auth/google/callback", async (req, res) => { + try { + const code = req.query.code as string; + if (!code) return res.status(400).send("Код авторизации не получен"); + await handleCallback(code); + res.redirect("/salary"); + } catch (err) { + console.error(err); + res.status(500).send("Ошибка при авторизации Google. Попробуйте ещё раз."); + } + }); + + app.get("/api/auth/google/status", (_req, res) => { + res.json(getAuthStatus()); + }); + + app.post("/api/auth/google/logout", (_req, res) => { + logout(); + res.json({ success: true }); + }); + + // --- Salary areas CRUD --- + const SALARY_AREAS_PATH = path.join(process.cwd(), "data", "salary-areas.json"); + + interface SalaryArea { + id: string; + name: string; + spreadsheetId: string; + fioCol: string; + bazaCol: string; + kVydacheCol: string; + } + + function colLetterToIndex(col: string): number { + const c = col.toUpperCase(); + let idx = 0; + for (let i = 0; i < c.length; i++) { + idx = idx * 26 + (c.charCodeAt(i) - 64); + } + return idx - 1; + } + + function maxColLetter(...cols: string[]): string { + let max = cols[0]; + for (const c of cols) { + if (colLetterToIndex(c) > colLetterToIndex(max)) max = c; + } + return max.toUpperCase(); + } + + function loadAreas(): SalaryArea[] { + try { + if (fs.existsSync(SALARY_AREAS_PATH)) { + const raw = JSON.parse(fs.readFileSync(SALARY_AREAS_PATH, "utf-8")); + return raw.map((a: any) => ({ + fioCol: "B", + bazaCol: "F", + kVydacheCol: "J", + ...a, + })); + } + } catch {} + return []; + } + + function saveAreas(areas: any[]): void { + fs.writeFileSync(SALARY_AREAS_PATH, JSON.stringify(areas, null, 2)); + } + + function extractSpreadsheetId(url: string): string | null { + const m = url.match(/\/spreadsheets\/d\/([a-zA-Z0-9_-]+)/); + return m ? m[1] : null; + } + + app.get("/api/salary/areas", (_req, res) => { + res.json(loadAreas()); + }); + + const COL_PATTERN = /^[A-Za-z]{1,2}$/; + + app.post("/api/salary/areas", (req, res) => { + try { + const { name, spreadsheetUrl, fioCol = "B", bazaCol = "F", kVydacheCol = "J" } = req.body || {}; + if (!name || !spreadsheetUrl) return res.status(400).json({ error: "Укажите название участка и ссылку на таблицу" }); + const spreadsheetId = extractSpreadsheetId(spreadsheetUrl); + if (!spreadsheetId) return res.status(400).json({ error: "Неверная ссылка на Google Таблицу" }); + if (!COL_PATTERN.test(fioCol) || !COL_PATTERN.test(bazaCol) || !COL_PATTERN.test(kVydacheCol)) { + return res.status(400).json({ error: "Столбцы должны быть буквами (A-Z)" }); + } + const areas = loadAreas(); + const id = Date.now().toString(36) + Math.random().toString(36).slice(2, 6); + areas.push({ id, name, spreadsheetId, fioCol: fioCol.toUpperCase(), bazaCol: bazaCol.toUpperCase(), kVydacheCol: kVydacheCol.toUpperCase() }); + saveAreas(areas); + res.json({ id, name, spreadsheetId, fioCol: fioCol.toUpperCase(), bazaCol: bazaCol.toUpperCase(), kVydacheCol: kVydacheCol.toUpperCase() }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка при добавлении участка" }); + } + }); + + app.patch("/api/salary/areas/:id", (req, res) => { + try { + const areas = loadAreas(); + const area = areas.find(a => a.id === req.params.id); + if (!area) return res.status(404).json({ error: "Участок не найден" }); + const { fioCol, bazaCol, kVydacheCol } = req.body || {}; + if (fioCol !== undefined) { + if (!COL_PATTERN.test(fioCol)) return res.status(400).json({ error: "Столбец ФИО должен быть буквой (A-Z)" }); + area.fioCol = fioCol.toUpperCase(); + } + if (bazaCol !== undefined) { + if (!COL_PATTERN.test(bazaCol)) return res.status(400).json({ error: "Столбец База должен быть буквой (A-Z)" }); + area.bazaCol = bazaCol.toUpperCase(); + } + if (kVydacheCol !== undefined) { + if (!COL_PATTERN.test(kVydacheCol)) return res.status(400).json({ error: "Столбец К выдаче должен быть буквой (A-Z)" }); + area.kVydacheCol = kVydacheCol.toUpperCase(); + } + saveAreas(areas); + res.json(area); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка при обновлении участка" }); + } + }); + + app.delete("/api/salary/areas/:id", (req, res) => { + try { + let areas = loadAreas(); + areas = areas.filter(a => a.id !== req.params.id); + saveAreas(areas); + res.json({ success: true }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка при удалении участка" }); + } + }); + + // --- opneof table init --- + let opneofReady = false; + + async function tryInitOpneof(): Promise { + if (opneofReady) return true; + try { + await query(` + CREATE TABLE IF NOT EXISTS opneof ( + id SERIAL PRIMARY KEY, + area_id TEXT NOT NULL, + area_name TEXT NOT NULL, + month TEXT NOT NULL, + fio TEXT NOT NULL, + baza NUMERIC NOT NULL DEFAULT 0, + k_vydache NUMERIC NOT NULL DEFAULT 0, + synced_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deleted_at TIMESTAMPTZ + ) + `); + await query(` + CREATE UNIQUE INDEX IF NOT EXISTS opneof_ukey ON opneof (area_id, month, fio) + `); + opneofReady = true; + console.log("[opneof] Таблица готова"); + return true; + } catch (err: any) { + console.error("[opneof] Ошибка создания таблицы:", err?.message); + return false; + } + } + tryInitOpneof(); + + async function ensureOpneof(): Promise { + return tryInitOpneof(); + } + + // ── opof_manual_areas: persistent manual FIO → area overrides ────────────── + let opofManualAreasReady = false; + async function ensureOpofManualAreas(): Promise { + if (opofManualAreasReady) return true; + try { + await query(` + CREATE TABLE IF NOT EXISTS opof_manual_areas ( + fio_letters TEXT PRIMARY KEY, + fio_display TEXT NOT NULL, + area_id TEXT NOT NULL, + area_name TEXT NOT NULL, + canonical_fio TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + `); + opofManualAreasReady = true; + console.log("[opof_manual_areas] Таблица готова"); + return true; + } catch (err: any) { + console.error("[opof_manual_areas] Ошибка создания таблицы:", err?.message); + return false; + } + } + ensureOpofManualAreas(); + + async function opneofHasData(): Promise { + try { + if (!(await ensureOpneof())) return false; + const r = await query("SELECT 1 FROM opneof WHERE deleted_at IS NULL LIMIT 1"); + return r.rows.length > 0; + } catch { + return false; + } + } + + async function readNeofFromDb(areaId?: string): Promise { + let sql = `SELECT area_id, area_name, month, fio, baza, k_vydache FROM opneof WHERE deleted_at IS NULL`; + const params: unknown[] = []; + if (areaId) { + params.push(areaId); + sql += ` AND area_id = $${params.length}`; + } + sql += ` ORDER BY area_id, month, fio`; + const r = await query(sql, params); + + const areasMap = new Map }>(); + for (const row of r.rows) { + if (!areasMap.has(row.area_id)) { + areasMap.set(row.area_id, { id: row.area_id, name: row.area_name, monthsMap: new Map() }); + } + const a = areasMap.get(row.area_id)!; + if (!a.monthsMap.has(row.month)) a.monthsMap.set(row.month, []); + a.monthsMap.get(row.month)!.push({ + fio: row.fio, + baza: Number(row.baza), + kVydache: Number(row.k_vydache), + }); + } + + const areas = [...areasMap.values()].map(a => ({ + id: a.id, + name: a.name, + months: [...a.monthsMap.entries()].map(([month, employees]) => ({ month, employees })), + })); + + return { areas, source: "db" }; + } + + // --- Salary data fetch from Google Sheets --- + const MONTH_PATTERN = /^(\d{1,2})\.(\d{2})$/; + + function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + async function withRetry(fn: () => Promise, maxRetries = 4, baseDelayMs = 1000): Promise { + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await fn(); + } catch (err: any) { + const msg = String(err?.message || ""); + const status = err?.statusCode || err?.code || err?.response?.status; + const isQuota = msg.includes("Quota exceeded") || msg.includes("quota") || status === 429 || msg.includes("429") || msg.includes("RATE_LIMIT") || msg.includes("rate limit"); + if (!isQuota || attempt === maxRetries) throw err; + const delayMs = baseDelayMs * Math.pow(2, attempt); + console.log(`[opneof] retry ${attempt + 1}/${maxRetries} after ${delayMs}ms — ${msg.slice(0, 120)}`); + await sleep(delayMs); + } + } + throw new Error("withRetry: unreachable"); + } + + async function fetchNeofFromSheets(targetAreas: SalaryArea[]): Promise { + const results: any[] = []; + for (const area of targetAreas) { + try { + const sheetNames = await withRetry(() => getSheetNames(area.spreadsheetId)); + const monthSheets = sheetNames + .filter(name => MONTH_PATTERN.test(name)) + .sort((a, b) => { + const [, am, ay] = a.match(MONTH_PATTERN)!; + const [, bm, by] = b.match(MONTH_PATTERN)!; + return (Number(ay) * 12 + Number(am)) - (Number(by) * 12 + Number(bm)); + }); + + const months: { month: string; employees: any[] }[] = []; + for (const sheetName of monthSheets) { + const fioIdx = colLetterToIndex(area.fioCol || "B"); + const bazaIdx = colLetterToIndex(area.bazaCol || "F"); + const kVydacheIdx = colLetterToIndex(area.kVydacheCol || "J"); + const lastCol = maxColLetter(area.fioCol || "B", area.bazaCol || "F", area.kVydacheCol || "J"); + const rows = await withRetry(() => getSheetData(area.spreadsheetId, `'${sheetName}'!A4:${lastCol}500`)); + const employees = rows + .filter(row => row && row[fioIdx] && String(row[fioIdx]).trim()) + .map(row => { + const rawFio = String(row[fioIdx] || ""); + const fio = rawFio.replace(/[0-9]/g, "").replace(/[^\p{L}\s\-().]/gu, "").replace(/\s+/g, " ").trim(); + if (!fio) return null; + const baza = Number(String(row[bazaIdx] || "0").replace(/\s/g, "").replace(",", ".")) || 0; + const kVydache = Number(String(row[kVydacheIdx] || "0").replace(/\s/g, "").replace(",", ".")) || 0; + return { fio, baza, kVydache }; + }) + .filter(Boolean); + + months.push({ month: sheetName, employees }); + await sleep(200); + } + + results.push({ id: area.id, name: area.name, months: months.filter(m => m.employees.length > 0) }); + } catch (err: any) { + console.error(`Error fetching area ${area.name}:`, err?.message); + results.push({ id: area.id, name: area.name, months: [], error: err?.message || "Ошибка загрузки" }); + } + await sleep(600); + } + return results; + } + + app.get("/api/salary/neof", async (req, res) => { + try { + const areaId = req.query.areaId as string | undefined; + + const hasDb = await opneofHasData(); + if (hasDb) { + const result = await readNeofFromDb(areaId); + return res.json(result); + } + + const status = getAuthStatus(); + if (!status.loggedIn) return res.status(401).json({ error: "Не авторизован в Google", authRequired: true }); + + const areas = loadAreas(); + const targetAreas = areaId ? areas.filter(a => a.id === areaId) : areas; + if (targetAreas.length === 0) return res.json({ areas: [], source: "sheets" }); + + const results = await fetchNeofFromSheets(targetAreas); + res.json({ areas: results, source: "sheets" }); + } catch (err: any) { + if (err?.message?.includes("авторизован") || err?.message?.includes("Токен")) { + return res.status(401).json({ error: err.message, authRequired: true }); + } + console.error(err); + res.status(500).json({ error: "Ошибка загрузки данных зарплат" }); + } + }); + + // --- ОФ tab: opof table --- + // GET /api/salary/zof/areas — unique areas from opneof for the assign dropdown + app.get("/api/salary/zof/areas", async (_req, res) => { + try { + if (!(await ensureOpneof())) return res.json({ areas: [] }); + const r = await query( + `SELECT DISTINCT area_id, area_name FROM opneof WHERE deleted_at IS NULL ORDER BY area_name` + ); + const areas = r.rows.map((row: any) => ({ id: row.area_id, name: row.area_name })); + res.json({ areas }); + } catch (err: any) { + if (isConnectionError(err)) return res.json({ areas: [] }); + console.error("[zof/areas]", err); + res.status(500).json({ error: "Ошибка загрузки участков" }); + } + }); + + // POST /api/salary/zof/assign — upsert a manual FIO → area override + app.post("/api/salary/zof/assign", async (req, res) => { + try { + await ensureOpofManualAreas(); + const { fioLetters, fioDisplay, areaId, areaName, canonicalFio } = req.body || {}; + if (!fioLetters || !areaId || !areaName) { + return res.status(400).json({ error: "fioLetters, areaId, areaName обязательны" }); + } + await query(` + INSERT INTO opof_manual_areas (fio_letters, fio_display, area_id, area_name, canonical_fio) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (fio_letters) + DO UPDATE SET fio_display = EXCLUDED.fio_display, + area_id = EXCLUDED.area_id, + area_name = EXCLUDED.area_name, + canonical_fio = EXCLUDED.canonical_fio, + created_at = NOW() + `, [fioLetters, fioDisplay || fioLetters, areaId, areaName, canonicalFio || ""]); + res.json({ ok: true }); + } catch (err: any) { + console.error("[zof/assign]", err); + res.status(500).json({ error: "Ошибка сохранения назначения" }); + } + }); + + // DELETE /api/salary/zof/assign/:fioLetters — remove a manual override + app.delete("/api/salary/zof/assign/:fioLetters", async (req, res) => { + try { + await ensureOpofManualAreas(); + await query( + `DELETE FROM opof_manual_areas WHERE fio_letters = $1`, + [req.params.fioLetters] + ); + res.json({ ok: true }); + } catch (err: any) { + console.error("[zof/assign/delete]", err); + res.status(500).json({ error: "Ошибка удаления назначения" }); + } + }); + + app.get("/api/salary/zof", async (_req, res) => { + try { + // Format periodregistrator → "MM.YY" + // The column may be a date/timestamp or a string like "01.2026" / "2026-01-01" + const opofRows = await query(` + SELECT + sotrudnik, + nachislenobezndfl, + summandfl, + strahov, + periodregistrator + FROM opof + ORDER BY periodregistrator, sotrudnik + `); + + // Keep only Cyrillic letters, lowercase — for fuzzy FIO matching + function lettersOnly(s: string): string { + return s.toLowerCase().replace(/[^а-яё]/g, ""); + } + + // Levenshtein distance (edit distance) + function levenshtein(a: string, b: string): number { + const m = a.length, n = b.length; + if (m === 0) return n; + if (n === 0) return m; + const dp: number[] = Array.from({ length: n + 1 }, (_, i) => i); + for (let i = 1; i <= m; i++) { + let prev = dp[0]; + dp[0] = i; + for (let j = 1; j <= n; j++) { + const tmp = dp[j]; + dp[j] = a[i - 1] === b[j - 1] + ? prev + : 1 + Math.min(prev, dp[j], dp[j - 1]); + prev = tmp; + } + } + return dp[n]; + } + + // Similarity score 0..1: 1 = identical, 0 = completely different + function similarity(a: string, b: string): number { + const maxLen = Math.max(a.length, b.length); + if (maxLen === 0) return 1; + return 1 - levenshtein(a, b) / maxLen; + } + + // Load manual overrides (opof_manual_areas) — persist across restarts + await ensureOpofManualAreas(); + const manualRows = await query(`SELECT fio_letters, fio_display, area_id, area_name, canonical_fio FROM opof_manual_areas`).catch(() => ({ rows: [] })); + const manualMap = new Map(); + for (const row of manualRows.rows) { + manualMap.set(String(row.fio_letters), { + id: String(row.area_id), + name: String(row.area_name), + canonicalFio: String(row.canonical_fio || row.fio_display || ""), + }); + } + + // Build global FIO index from opneof (no month constraint) + const neofRows = await query( + `SELECT DISTINCT fio, area_id, area_name FROM opneof WHERE deleted_at IS NULL` + ); + const neofIndex: Array<{ letters: string; id: string; name: string; canonicalFio: string }> = []; + for (const row of neofRows.rows) { + const fio = String(row.fio || "").trim(); + const letters = lettersOnly(fio); + if (letters) { + neofIndex.push({ letters, id: row.area_id, name: row.area_name, canonicalFio: fio }); + } + } + + // Find best match in neofIndex for a given letters string (threshold 0.9) + function findBestMatch(srcLetters: string): { id: string; name: string; canonicalFio: string } | null { + if (!srcLetters) return null; + let best: { id: string; name: string; canonicalFio: string } | null = null; + let bestSim = 0; + for (const entry of neofIndex) { + const sim = similarity(srcLetters, entry.letters); + if (sim > bestSim) { + bestSim = sim; + best = entry; + } + } + return bestSim >= 0.9 ? best : null; + } + + function formatPeriod(raw: unknown): string { + if (raw == null) return "??.??"; + const s = String(raw).trim(); + // Already in MM.YY or MM.YYYY format + const mmYY = s.match(/^(\d{1,2})\.(\d{2})$/); + if (mmYY) return `${mmYY[1].padStart(2, "0")}.${mmYY[2]}`; + const mmYYYY = s.match(/^(\d{1,2})\.(\d{4})$/); + if (mmYYYY) return `${mmYYYY[1].padStart(2, "0")}.${mmYYYY[2].slice(2)}`; + // ISO date or timestamp: 2026-01-01 or 2026-01-01T... + const isoDate = s.match(/^(\d{4})-(\d{2})-(\d{2})/); + if (isoDate) return `${isoDate[2]}.${isoDate[1].slice(2)}`; + // Try JS Date + const d = new Date(s); + if (!isNaN(d.getTime())) { + const mm = String(d.getMonth() + 1).padStart(2, "0"); + const yy = String(d.getFullYear()).slice(2); + return `${mm}.${yy}`; + } + return s; + } + + // Group by area → month → employees + const areasMap = new Map; + }>(); + + const ensureArea = (id: string, name: string) => { + if (!areasMap.has(id)) areasMap.set(id, { id, name, monthsMap: new Map() }); + return areasMap.get(id)!; + }; + + for (const row of opofRows.rows) { + const rawFio = String(row.sotrudnik || "").trim(); + const period = formatPeriod(row.periodregistrator); + const srcLetters = lettersOnly(rawFio); + // Priority: manual override > fuzzy match > unassigned + const manual = manualMap.get(srcLetters); + const fuzzy = manual ? null : findBestMatch(srcLetters); + const resolved = manual ?? fuzzy; + const displayFio = resolved?.canonicalFio || rawFio; + const area = ensureArea( + resolved?.id ?? "unassigned", + resolved?.name ?? "Нераспределённые" + ); + if (!area.monthsMap.has(period)) area.monthsMap.set(period, []); + area.monthsMap.get(period)!.push({ + fio: displayFio, + fioLetters: srcLetters, + nachislenobezndfl: Number(row.nachislenobezndfl) || 0, + summandfl: Number(row.summandfl) || 0, + strahov: Number(row.strahov) || 0, + }); + } + + // Sort: assigned areas first, unassigned last + const areas = [...areasMap.values()] + .sort((a, b) => { + if (a.id === "unassigned") return 1; + if (b.id === "unassigned") return -1; + return a.name.localeCompare(b.name, "ru"); + }) + .map(a => ({ + id: a.id, + name: a.name, + months: [...a.monthsMap.entries()] + .sort((x, y) => { + const [xm, xy] = x[0].split(".").map(Number); + const [ym, yy] = y[0].split(".").map(Number); + return (yy * 12 + ym) - (xy * 12 + xm); + }) + .map(([month, employees]) => ({ month, employees })), + })); + + res.json({ areas }); + } catch (err: any) { + if (isConnectionError(err)) { + return res.json({ areas: [] }); + } + console.error("[zof]", err); + res.status(500).json({ error: "Ошибка загрузки данных ОФ" }); + } + }); + + app.get("/api/salary/db-status", async (_req, res) => { + try { + if (!(await ensureOpneof())) return res.json({ totalRows: 0, lastSync: null, areas: [], monthCount: 0 }); + const countRes = await query("SELECT COUNT(*) as cnt FROM opneof WHERE deleted_at IS NULL"); + const lastSyncRes = await query("SELECT MAX(synced_at) as last_sync FROM opneof"); + const areasRes = await query("SELECT DISTINCT area_name FROM opneof WHERE deleted_at IS NULL ORDER BY area_name"); + const monthsRes = await query("SELECT COUNT(DISTINCT month) as cnt FROM opneof WHERE deleted_at IS NULL"); + res.json({ + totalRows: Number(countRes.rows[0]?.cnt || 0), + lastSync: lastSyncRes.rows[0]?.last_sync || null, + areas: areasRes.rows.map(r => r.area_name), + monthCount: Number(monthsRes.rows[0]?.cnt || 0), + }); + } catch (err: any) { + if (isConnectionError(err)) { + return res.json({ totalRows: 0, lastSync: null, areas: [], monthCount: 0 }); + } + handleApiError(res, err, "Ошибка получения статуса opneof"); + } + }); + + let syncInProgress = false; + + app.post("/api/salary/sync", async (_req, res) => { + try { + if (syncInProgress) return res.status(409).json({ error: "Синхронизация уже выполняется" }); + syncInProgress = true; + + const status = getAuthStatus(); + if (!status.loggedIn) { syncInProgress = false; return res.status(401).json({ error: "Не авторизован в Google", authRequired: true }); } + + if (!(await ensureOpneof())) { syncInProgress = false; return res.status(503).json({ error: "Таблица opneof не готова" }); } + + const areas = loadAreas(); + if (areas.length === 0) { syncInProgress = false; return res.json({ added: 0, updated: 0, deleted: 0, total: 0 }); } + + const sheetsData = await fetchNeofFromSheets(areas); + + const failedAreas = sheetsData.filter(a => a.error); + if (failedAreas.length === sheetsData.length) { + syncInProgress = false; + const firstErr = failedAreas[0]?.error || "Неизвестная ошибка"; + if (firstErr.includes("авторизован") || firstErr.includes("Токен") || firstErr.includes("401")) { + return res.status(401).json({ error: "Ошибка авторизации Google: " + firstErr, authRequired: true }); + } + return res.status(500).json({ error: "Все участки вернули ошибку: " + firstErr }); + } + + const now = new Date().toISOString(); + + const client = await getPoolClient(); + try { + await client.query("BEGIN"); + let added = 0, updated = 0, deleted = 0; + const errors: string[] = failedAreas.map(a => `${a.name}: ${a.error}`); + + const sheetsKeys = new Set(); + + for (const areaData of sheetsData) { + if (areaData.error) continue; + for (const monthData of areaData.months) { + for (const emp of monthData.employees) { + sheetsKeys.add(`${areaData.id}||${monthData.month}||${emp.fio}`); + + const r = await client.query( + `INSERT INTO opneof (area_id, area_name, month, fio, baza, k_vydache, synced_at, deleted_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, NULL) + ON CONFLICT (area_id, month, fio) DO UPDATE + SET baza = EXCLUDED.baza, k_vydache = EXCLUDED.k_vydache, + area_name = EXCLUDED.area_name, synced_at = EXCLUDED.synced_at, deleted_at = NULL + WHERE opneof.baza IS DISTINCT FROM EXCLUDED.baza + OR opneof.k_vydache IS DISTINCT FROM EXCLUDED.k_vydache + OR opneof.area_name IS DISTINCT FROM EXCLUDED.area_name + OR opneof.deleted_at IS NOT NULL + RETURNING (xmax = 0) AS inserted`, + [areaData.id, areaData.name, monthData.month, emp.fio, emp.baza, emp.kVydache, now] + ); + if (r.rows.length === 0) continue; + if (r.rows[0]?.inserted) added++; else updated++; + } + } + } + + for (const areaData of sheetsData) { + if (areaData.error) continue; + const existingRows = await client.query( + "SELECT id, area_id, month, fio FROM opneof WHERE area_id = $1 AND deleted_at IS NULL", + [areaData.id] + ); + for (const row of existingRows.rows) { + if (!sheetsKeys.has(`${row.area_id}||${row.month}||${row.fio}`)) { + await client.query("UPDATE opneof SET deleted_at = $1, synced_at = $1 WHERE id = $2", [now, row.id]); + deleted++; + } + } + } + + await client.query("COMMIT"); + client.release(); + + const totalRes = await query("SELECT COUNT(*) as cnt FROM opneof WHERE deleted_at IS NULL"); + const total = Number(totalRes.rows[0]?.cnt || 0); + + console.log(`[opneof sync] added=${added} updated=${updated} deleted=${deleted} total=${total} errors=${errors.length}`); + syncInProgress = false; + res.json({ added, updated, deleted, total, errors: errors.length > 0 ? errors : undefined }); + } catch (txErr) { + await client.query("ROLLBACK").catch(() => {}); + client.release(); + throw txErr; + } + } catch (err: any) { + syncInProgress = false; + if (err?.message?.includes("авторизован") || err?.message?.includes("Токен")) { + return res.status(401).json({ error: err.message, authRequired: true }); + } + console.error("[salary sync] Error:", err); + res.status(500).json({ error: "Ошибка синхронизации: " + (err?.message || "неизвестная ошибка") }); + } + }); + + app.get("/api/planning/salary-monthly", async (_req, res) => { + try { + const hasDb = await opneofHasData(); + if (hasDb) { + const r = await query(` + SELECT month, SUM(k_vydache) as total + FROM opneof + WHERE deleted_at IS NULL + GROUP BY month + ORDER BY month + `); + const MONTH_NAMES_MAP: Record = { + "1": "01", "2": "02", "3": "03", "4": "04", "5": "05", "6": "06", + "7": "07", "8": "08", "9": "09", "10": "10", "11": "11", "12": "12", + }; + const result = r.rows.map(row => { + const m = row.month.match(/^(\d{1,2})\.(\d{2})$/); + if (!m) return null; + const fullYear = Number(m[2]) < 50 ? 2000 + Number(m[2]) : 1900 + Number(m[2]); + const period = `${fullYear}-${(MONTH_NAMES_MAP[m[1]] || m[1])}`; + return { period, total: Math.round(Number(row.total) * 100) / 100 }; + }).filter(Boolean).sort((a: any, b: any) => a.period.localeCompare(b.period)); + return res.json(result); + } + + const status = getAuthStatus(); + if (!status.loggedIn) return res.json([]); + + const areas = loadAreas(); + if (areas.length === 0) return res.json([]); + + const result = await cachedQuery("planning-salary-monthly", TTL_SHORT, async () => { + const monthTotals = new Map(); + + for (const area of areas) { + try { + const sheetNames = await getSheetNames(area.spreadsheetId); + const monthSheets = sheetNames.filter(name => MONTH_PATTERN.test(name)); + + for (const sheetName of monthSheets) { + const [, mm, yy] = sheetName.match(MONTH_PATTERN)!; + const fullYear = Number(yy) < 50 ? 2000 + Number(yy) : 1900 + Number(yy); + const period = `${fullYear}-${String(mm).padStart(2, "0")}`; + + const kVydacheIdx = colLetterToIndex(area.kVydacheCol || "J"); + const fioIdx = colLetterToIndex(area.fioCol || "B"); + const lastCol = maxColLetter(area.fioCol || "B", area.bazaCol || "F", area.kVydacheCol || "J"); + const rows = await getSheetData(area.spreadsheetId, `'${sheetName}'!A4:${lastCol}500`); + + let total = 0; + for (const row of rows) { + if (!row || !row[fioIdx] || !String(row[fioIdx]).trim()) continue; + const fio = String(row[fioIdx] || "").replace(/[0-9]/g, "").replace(/[^\p{L}\s\-().]/gu, "").replace(/\s+/g, " ").trim(); + if (!fio) continue; + total += Number(String(row[kVydacheIdx] || "0").replace(/\s/g, "").replace(",", ".")) || 0; + } + + monthTotals.set(period, (monthTotals.get(period) || 0) + total); + } + } catch (err: any) { + console.error(`[salary-monthly] Error area ${area.name}:`, err?.message); + } + } + + return [...monthTotals.entries()] + .sort((a, b) => a[0].localeCompare(b[0])) + .map(([period, total]) => ({ period, total: Math.round(total * 100) / 100 })); + }); + + res.json(result); + } catch (err: any) { + console.error("[salary-monthly] Error:", err?.message); + res.json([]); + } + }); + + app.get("/api/planning/receivables", async (_req, res) => { + try { + const result = await cachedQuery("planning-receivables", TTL_SHORT, async () => { + const cutoffDate = new Date(); + cutoffDate.setMonth(cutoffDate.getMonth() - 6); + const cutoff6m = cutoffDate.toISOString().substring(0, 10); + + const [summaryRes, byKontragentRes, byClassRes, dynamicsRes, debtDetailRes, incomeByKontragentRes] = await Promise.all([ + query(`SELECT COALESCE(SUM(summadolga::numeric),0) as total_debt, COALESCE(SUM(summaprosrocheno::numeric),0) as total_overdue, COUNT(DISTINCT kontragent) as kontragent_count FROM ${getDebtTable()} WHERE period = (SELECT MAX(period) FROM ${getDebtTable()})`), + query(`SELECT kontragent, COALESCE(SUM(summadokumenta::numeric),0) as doc_total, COALESCE(SUM(summaoplacheno::numeric),0) as paid_total, COALESCE(SUM(summadolga::numeric),0) as debt_total, COALESCE(SUM(summaprosrocheno::numeric),0) as overdue_total, MAX(COALESCE(dneiprosrocheno::numeric,0)) as max_days_overdue FROM ${getDebtTable()} WHERE period = (SELECT MAX(period) FROM ${getDebtTable()}) GROUP BY kontragent ORDER BY debt_total DESC LIMIT 30`), + query(`SELECT CASE WHEN klassprosrochki IS NULL OR klassprosrochki = '0' THEN 'Не просрочено' WHEN klassprosrochki = '1-30' THEN '1-30 дней' WHEN klassprosrochki = '31-60' THEN '31-60 дней' WHEN klassprosrochki = '61-90' THEN '61-90 дней' WHEN klassprosrochki = '91-365' THEN CASE WHEN COALESCE(dneiprosrocheno::numeric,0) <= 120 THEN '91-120 дней' WHEN COALESCE(dneiprosrocheno::numeric,0) <= 180 THEN '121-180 дней' WHEN COALESCE(dneiprosrocheno::numeric,0) <= 270 THEN '181-270 дней' ELSE '271-365 дней' END END as cls, COALESCE(SUM(summadolga::numeric),0) as total FROM ${getDebtTable()} WHERE period = (SELECT MAX(period) FROM ${getDebtTable()}) AND (klassprosrochki IS NULL OR klassprosrochki != '> 365') GROUP BY cls ORDER BY total DESC`), + query(`SELECT TO_CHAR(period, 'YYYY-MM') as month, COALESCE(SUM(summadolga::numeric),0) as total_debt, COALESCE(SUM(summaprosrocheno::numeric),0) as total_overdue FROM ${getDebtTable()} WHERE period >= '${cutoff6m}'::date GROUP BY TO_CHAR(period, 'YYYY-MM') ORDER BY month`), + query(`SELECT kontragent, dogovor, COALESCE(summadokumenta::numeric,0) as doc_amount, COALESCE(summaoplacheno::numeric,0) as paid_amount, COALESCE(summadolga::numeric,0) as debt_amount, COALESCE(summaprosrocheno::numeric,0) as overdue_amount, COALESCE(dneiprosrocheno::numeric,0) as days_overdue, dokumentraschetov FROM ${getDebtTable()} WHERE period = (SELECT MAX(period) FROM ${getDebtTable()}) AND summadolga::numeric > 0 ORDER BY debt_amount DESC LIMIT 100`), + query(`SELECT zakazchik as kontragent, SUM(summa::numeric) as income_total FROM ${getIncomeEffectiveView()} WHERE data >= '${cutoff6m}'::date AND zakazchik IS NOT NULL AND zakazchik != '0' GROUP BY zakazchik`), + ]); + + const summary = { + totalDebt: Number(summaryRes.rows[0]?.total_debt || 0), + totalOverdue: Number(summaryRes.rows[0]?.total_overdue || 0), + kontragentCount: Number(summaryRes.rows[0]?.kontragent_count || 0), + }; + + const incomeMap = new Map(); + for (const r of incomeByKontragentRes.rows as any[]) { + incomeMap.set(r.kontragent, Number(r.income_total)); + } + + const byKontragent = (byKontragentRes.rows as any[]).map((r: any) => ({ + kontragent: r.kontragent, + docTotal: Number(r.doc_total), + paidTotal: Number(r.paid_total), + debtTotal: Number(r.debt_total), + overdueTotal: Number(r.overdue_total), + maxDaysOverdue: Number(r.max_days_overdue), + incomeReceived: incomeMap.get(r.kontragent) || 0, + })); + + const byClass = (byClassRes.rows as any[]).map((r: any) => ({ + cls: r.cls, + total: Number(r.total), + })); + + const dynamics = (dynamicsRes.rows as any[]).map((r: any) => ({ + month: r.month, + totalDebt: Number(r.total_debt), + totalOverdue: Number(r.total_overdue), + })); + + const expectedPayments = (debtDetailRes.rows as any[]).map((r: any) => ({ + kontragent: r.kontragent, + dogovor: r.dogovor || "", + document: r.dokumentraschetov || "", + docAmount: Number(r.doc_amount), + paidAmount: Number(r.paid_amount), + debtAmount: Number(r.debt_amount), + overdueAmount: Number(r.overdue_amount), + daysOverdue: Number(r.days_overdue), + })); + + return { summary, byKontragent, byClass, dynamics, expectedPayments }; + }); + res.json(result); + } catch (err: any) { + console.error("[Planning Receivables]", err); + res.status(500).json({ error: "Ошибка загрузки данных дебиторки" }); + } + }); + + // --- Calc sections CRUD --- + const CALC_SECTIONS_PATH = path.join(process.cwd(), "data", "calc-sections.json"); + + interface CalcSection { + id: string; + name: string; + spreadsheetUrl: string; + spreadsheetId: string; + } + + function loadCalcSections(): CalcSection[] { + try { + if (fs.existsSync(CALC_SECTIONS_PATH)) { + return JSON.parse(fs.readFileSync(CALC_SECTIONS_PATH, "utf-8")); + } + } catch {} + return []; + } + + function saveCalcSections(sections: CalcSection[]): void { + fs.writeFileSync(CALC_SECTIONS_PATH, JSON.stringify(sections, null, 2)); + } + + app.get("/api/calculations/sections", (_req, res) => { + res.json(loadCalcSections()); + }); + + app.post("/api/calculations/sections", (req, res) => { + try { + const { name, spreadsheetUrl } = req.body || {}; + if (!name || !spreadsheetUrl) return res.status(400).json({ error: "Укажите название участка и ссылку на таблицу" }); + const spreadsheetId = extractSpreadsheetId(spreadsheetUrl); + if (!spreadsheetId) return res.status(400).json({ error: "Неверная ссылка на Google Таблицу" }); + try { + const parsed = new URL(spreadsheetUrl); + if (parsed.hostname !== "docs.google.com" || parsed.protocol !== "https:") { + return res.status(400).json({ error: "Ссылка должна вести на docs.google.com" }); + } + } catch { + return res.status(400).json({ error: "Неверный формат ссылки" }); + } + const canonicalUrl = `https://docs.google.com/spreadsheets/d/${spreadsheetId}`; + const sections = loadCalcSections(); + const id = Date.now().toString(36) + Math.random().toString(36).slice(2, 6); + sections.push({ id, name, spreadsheetUrl: canonicalUrl, spreadsheetId }); + saveCalcSections(sections); + res.json({ id, name, spreadsheetUrl: canonicalUrl, spreadsheetId }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка при добавлении участка" }); + } + }); + + app.delete("/api/calculations/sections/:id", (req, res) => { + try { + const sections = loadCalcSections(); + const idx = sections.findIndex(s => s.id === req.params.id); + if (idx === -1) return res.status(404).json({ error: "Участок не найден" }); + sections.splice(idx, 1); + saveCalcSections(sections); + res.json({ ok: true }); + } catch (err) { + console.error(err); + res.status(500).json({ error: "Ошибка при удалении участка" }); + } + }); + + app.get("/api/audit/integrity", async (_req, res) => { + try { + const result = await runIntegrityAudit(); + res.json(result); + } catch (err) { + handleApiError(res, err, "Ошибка при аудите данных"); + } + }); + + app.get("/api/overrides", async (_req, res) => { + try { + const type = _req.query.type as string; + if (type === "income") { + const rows = await getAllIncomeOverrides(); + res.json(Object.fromEntries(rows.map(r => [r.id, r.categoryId]))); + } else if (type === "expense") { + const rows = await getAllExpenseOverrides(); + res.json(Object.fromEntries(rows.map(r => [r.id, r.categoryId]))); + } else { + const [incRows, expRows] = await Promise.all([getAllIncomeOverrides(), getAllExpenseOverrides()]); + res.json({ + income: Object.fromEntries(incRows.map(r => [r.id, r.categoryId])), + expense: Object.fromEntries(expRows.map(r => [r.id, r.categoryId])), + }); + } + } catch (err) { + handleApiError(res, err, "Ошибка при загрузке переопределений"); + } + }); + + app.post("/api/overrides", async (req, res) => { + try { + const { type, recordId, categoryId } = req.body; + if (!type || !recordId || !categoryId) { + return res.status(400).json({ error: "type, recordId, categoryId обязательны" }); + } + if (type !== "expense" && type !== "income") { + return res.status(400).json({ error: "type must be expense or income" }); + } + const rid = String(recordId).trim(); + const cid = String(categoryId).trim(); + if (!rid || !cid || rid.length > 100 || cid.length > 100) { + return res.status(400).json({ error: "Некорректные recordId или categoryId" }); + } + const cats = type === "income" ? loadCategories() : loadExpenseCategories(); + const foundCat = findCategoryById(cats, cid); + if (!foundCat) { + return res.status(400).json({ error: "Категория не найдена" }); + } + if (type === "income") { + const overrideCols = extractColumnValuesFromCategoryPath(foundCat.path, INCOME_OVERRIDE_COLS); + await saveIncomeOverride(rid, overrideCols, cid); + } else { + const overrideCols = extractColumnValuesFromCategoryPath(foundCat.path, EXPENSE_OVERRIDE_COLS); + await saveExpenseOverride(rid, overrideCols, cid); + } + invalidateServerCache(); + scheduleTreeRebuild(); + res.json({ ok: true }); + } catch (err) { + handleApiError(res, err, "Ошибка при сохранении переопределения"); + } + }); + + app.delete("/api/overrides/:type/:recordId", async (req, res) => { + try { + const { type, recordId } = req.params; + if (type !== "expense" && type !== "income") { + return res.status(400).json({ error: "type must be expense or income" }); + } + if (type === "income") { + await deleteIncomeOverride(recordId); + } else { + await deleteExpenseOverride(recordId); + } + invalidateServerCache(); + scheduleTreeRebuild(); + res.json({ ok: true }); + } catch (err) { + handleApiError(res, err, "Ошибка при удалении переопределения"); + } + }); + + app.post("/api/admin/rebuild-tree-tables", async (_req, res) => { + try { + scheduleTreeRebuild(); + res.json({ ok: true, message: "Перестройка таблиц деревьев запущена в фоне" }); + } catch (err) { + handleApiError(res, err, "Ошибка запуска перестройки"); + } + }); + + app.get("/api/tags", async (_req, res) => { + try { + const result = await query(`SELECT id, name, color FROM node_tags ORDER BY name`); + res.json(result.rows); + } catch (err) { + handleApiError(res, err, "Ошибка при загрузке тегов"); + } + }); + + app.post("/api/tags", async (req, res) => { + try { + const { name, color } = req.body; + if (!name || !color) return res.status(400).json({ error: "name и color обязательны" }); + const id = randomUUID(); + await query(`INSERT INTO node_tags (id, name, color) VALUES ($1, $2, $3)`, [id, String(name).trim(), String(color)]); + res.json({ id, name: String(name).trim(), color: String(color) }); + } catch (err) { + handleApiError(res, err, "Ошибка при создании тега"); + } + }); + + app.delete("/api/tags/:id", async (req, res) => { + try { + const { id } = req.params; + await query(`DELETE FROM node_tag_assignments WHERE tag_id = $1`, [id]); + await query(`DELETE FROM node_tags WHERE id = $1`, [id]); + res.json({ ok: true }); + } catch (err) { + handleApiError(res, err, "Ошибка при удалении тега"); + } + }); + + app.get("/api/tag-assignments", async (req, res) => { + try { + const context = req.query.context as string | undefined; + if (context && context !== "income" && context !== "expense") { + return res.status(400).json({ error: "context должен быть income или expense" }); + } + const rawNodeIds = req.query["nodeIds[]"]; + const nodeIds: string[] = rawNodeIds + ? (Array.isArray(rawNodeIds) ? rawNodeIds : [rawNodeIds]).map(String) + : []; + const conditions: string[] = []; + const params: unknown[] = []; + if (context) { params.push(context); conditions.push(`context = $${params.length}`); } + if (nodeIds.length > 0) { params.push(nodeIds); conditions.push(`node_id = ANY($${params.length})`); } + const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + const result = await query( + `SELECT id, node_id as "nodeId", tag_id as "tagId" FROM node_tag_assignments ${where}`, + params + ); + res.json(result.rows); + } catch (err) { + handleApiError(res, err, "Ошибка при загрузке назначений тегов"); + } + }); + + app.post("/api/tag-assignments", async (req, res) => { + try { + const { nodeId, tagId, context } = req.body; + if (!nodeId || !tagId || !context) return res.status(400).json({ error: "nodeId, tagId, context обязательны" }); + const id = randomUUID(); + await query( + `INSERT INTO node_tag_assignments (id, node_id, tag_id, context) VALUES ($1, $2, $3, $4) ON CONFLICT (node_id, tag_id, context) DO NOTHING`, + [id, String(nodeId), String(tagId), String(context)] + ); + res.json({ ok: true }); + } catch (err) { + handleApiError(res, err, "Ошибка при назначении тега"); + } + }); + + app.delete("/api/tag-assignments", async (req, res) => { + try { + const { nodeId, tagId, context } = req.body; + if (!nodeId || !tagId || !context) return res.status(400).json({ error: "nodeId, tagId, context обязательны" }); + await query( + `DELETE FROM node_tag_assignments WHERE node_id = $1 AND tag_id = $2 AND context = $3`, + [String(nodeId), String(tagId), String(context)] + ); + res.json({ ok: true }); + } catch (err) { + handleApiError(res, err, "Ошибка при снятии тега"); + } + }); + + app.get("/api/admin/tree-tables", async (_req, res) => { + try { + const incConfig = loadIncomeAutoDetectConfig(); + const expConfig = loadAutoDetectConfig(); + const allTables: { name: string; type: string; isPrimary: boolean; exists: boolean }[] = []; + const incomeTrees = [{ name: "Ручные категории", isPrimary: true }, { name: "Нераспределённые", isPrimary: false }, ...incConfig.trees.map(t => ({ name: t.name, isPrimary: t.isPrimary ?? true }))]; + const expenseTrees = [{ name: "Ручные категории", isPrimary: true }, { name: "Нераспределённые", isPrimary: false }, ...expConfig.trees.map(t => ({ name: t.name, isPrimary: t.isPrimary ?? true }))]; + for (const t of incomeTrees) { + const tableName = getTreeTableName("income", t.name); + allTables.push({ name: tableName, type: "income", isPrimary: t.isPrimary, exists: await treeTableExists(tableName) }); + } + for (const t of expenseTrees) { + const tableName = getTreeTableName("expense", t.name); + allTables.push({ name: tableName, type: "expense", isPrimary: t.isPrimary, exists: await treeTableExists(tableName) }); + } + res.json({ tables: allTables, rebuildInProgress: treeRebuildInProgress }); + } catch (err) { + handleApiError(res, err); + } + }); + + app.post("/api/audit/run", async (_req, res) => { + try { + const result = await runIntegrityAudit(); + res.json(result); + } catch (err) { + handleApiError(res, err, "Ошибка при запуске аудита"); + } + }); + + app.get("/api/audit/history", (_req, res) => { + res.json(getAuditHistory()); + }); + + scheduleTreeRebuild(); + + return httpServer; +} diff --git a/server/finance-di2/schema.ts b/server/finance-di2/schema.ts new file mode 100644 index 0000000..575052e --- /dev/null +++ b/server/finance-di2/schema.ts @@ -0,0 +1,366 @@ +import { z } from "zod"; + +export const dbConnectionSchema = z.object({ + host: z.string().min(1), + port: z.coerce.number().int().positive(), + database: z.string().min(1), + user: z.string().min(1), + password: z.string().min(1), +}); + +export type DbConnectionConfig = z.infer; + +export interface TableConfig { + incomeTable: string; + expenseTable: string; + debtTable: string; + incomeSumColumn: string; + expenseSumColumn: string; + incomeColumns: string[]; + expenseColumns: string[]; + columnLabels: Record; +} + +export interface IncomeRow { + id: string; + data: string | null; + summa: number | null; + stavkands: string | null; + dogovorkontragenta: string | null; + zakazchik: string | null; + dogovorzakazchika: string | null; + otvetstvenniipokontraktu: string | null; + statyadop: string | null; + kommentarii: string | null; + statya: string | null; + dochka: string | null; + vnuchka: string | null; + naznachenieplatezha: string | null; + schetnaoplatu: string | null; + overrideCategoryId?: string; + overrideCategoryName?: string; +} + +export interface ExpenseRow { + id: string; + data: string | null; + summadokumenta: number | null; + stavkands: string | null; + dogovorkontragenta: string | null; + zakazchik: string | null; + dogovorzakazchika: string | null; + otvetstvenniipokontraktu: string | null; + statyadop: string | null; + kommentarii: string | null; + statya: string | null; + dochka: string | null; + vnuchka: string | null; + naznachenieplatezha: string | null; + ssilkanomervhodyaschegodokumenta: string | null; + kontragent: string | null; + dataobnovleniyazapisi: string | null; + overrideCategoryId?: string; + overrideCategoryName?: string; +} + +export interface DebtRow { + period: string | null; + schet: string | null; + kontragent: string | null; + dogovor: string | null; + dokumentraschetov: string | null; + organizaciya: string | null; + summadolga: number | null; + summadolgaval: number | null; + valyuta: string | null; + otsrochkapodogovoru: number | null; + kontragentinn: string | null; + dataobrazovaniyadolga: string | null; + summadokumenta: number | null; + planovayadatapogasheniya: string | null; + dneiprosrocheno: number | null; + dneidolga: number | null; + otvetstvennii: string | null; + klassprosrochki: string | null; + klassdolga: string | null; + summaprosrocheno: number | null; + summaoplacheno: number | null; +} + +export interface DashboardStats { + incomeTotal: number; + incomeCount: number; + expenseTotal: number; + expenseCount: number; + debtTotal: number; + debtOverdue: number; + debtCounterparts: number; + monthlyIncome: { month: string; total: number }[]; + monthlyExpense: { month: string; total: number }[]; + debtByClass: { class: string; total: number }[]; + billedVsPaid: { month: string; billed: number; paid: number; debt: number }[]; +} + +export interface FilterOptions { + statya: string[]; + zakazchik: string[]; + dochka: string[]; + vnuchka: string[]; + otvetstvennii?: string[]; + organizaciya?: string[]; + klassprosrochki?: string[]; +} + +export interface PaginatedResult { + rows: T[]; + total: number; + page: number; + pageSize: number; + totalPages: number; + totalSum?: number; +} + +export type RuleOperator = "equals" | "not_equals" | "contains" | "greater_than" | "less_than" | "equal_number"; + +export interface IncomeCategoryRule { + column: string; + operator: RuleOperator; + value: string; +} + +export const MAX_CATEGORY_DEPTH = 5; + +export interface IncomeCategory { + id: string; + name: string; + rules: IncomeCategoryRule[]; + ruleLogic?: "and" | "or"; + excluded?: boolean; + children?: IncomeCategory[]; + autoDetected?: boolean; + treeId?: string; +} + +export const NUMERIC_OPERATORS: { value: RuleOperator; label: string }[] = [ + { value: "equal_number", label: "равно (=)" }, + { value: "greater_than", label: "больше (>)" }, + { value: "less_than", label: "меньше (<)" }, +]; + +export const TEXT_OPERATORS: { value: RuleOperator; label: string }[] = [ + { value: "equals", label: "равно (=)" }, + { value: "not_equals", label: "не равно (≠)" }, + { value: "contains", label: "содержит (∋)" }, +]; + +export interface ProfitabilityRow { + zakazchik: string; + totalIncome: number; + totalExpense: number; + profit: number; + margin: number; + contractCount: number; +} + +export interface ContractProfitRow { + dogovorzakazchika: string; + totalIncome: number; + totalExpense: number; + profit: number; +} + +export interface MonthlyProfitRow { + month: string; + income: number; + expense: number; + profit: number; +} + +export interface ProfitabilitySummary { + totalIncome: number; + totalExpense: number; + totalProfit: number; + counterpartCount: number; +} + +export interface ProfitabilityResponse { + rows: ProfitabilityRow[]; + total: number; + page: number; + pageSize: number; + totalPages: number; + summary: ProfitabilitySummary; +} + +export interface DebtByEntity { + name: string; + total: number; + cls0: number; + cls1_30: number; + cls31_60: number; + cls61_90: number; + cls91_365: number; + cls365plus: number; +} + +export interface DebtOverdueClass { + cls: string; + total: number; +} + +export interface DebtAnalytics { + summary: { + updatedAt: string | null; + totalDebt: number; + totalOverdue: number; + }; + byKontragent: DebtByEntity[]; + byManager: DebtByEntity[]; + byOverdueClass: DebtOverdueClass[]; + table: { + rows: DebtTableGroup[]; + total: number; + page: number; + pageSize: number; + totalPages: number; + }; +} + +export interface DebtTableRow { + dogovor: string; + summadokumenta: number; + dokumentraschetov: string; + summaoplacheno: number; + summadolga: number; + summaprosrocheno: number; + otsrochkapodogovoru: number; + dneiprosrocheno: number; +} + +export interface DebtTableGroup { + kontragent: string; + totalDebt: number; + totalOverdue: number; + rows: DebtTableRow[]; +} + +export interface DebtDynamicsRow { + period: string; + totalDebt: number; + totalOverdue: number; +} + +export interface DebtDynamicsKontragent { + kontragent: string; + summadokumenta: number; + summaoplacheno: number; + summadolga: number; + summaprosrocheno: number; + dneiprosrocheno: number; +} + +export interface DebtFiltersResponse { + periods: string[]; + klassprosrochki: string[]; + organizaciya: string[]; + otvetstvennii: string[]; + kontragenty: string[]; +} + +export const INCOME_FILTERABLE_COLUMNS = [ + { key: "data", label: "Дата" }, + { key: "summa", label: "Сумма" }, + { key: "statya", label: "Статья" }, + { key: "zakazchik", label: "Заказчик" }, + { key: "dochka", label: "Дочка" }, + { key: "vnuchka", label: "Внучка" }, + { key: "otvetstvenniipokontraktu", label: "Ответственный" }, + { key: "stavkands", label: "Ставка НДС" }, + { key: "kommentarii", label: "Комментарий" }, + { key: "dogovorkontragenta", label: "Договор контрагента" }, + { key: "schetnaoplatu", label: "Счет на оплату" }, + { key: "naznachenieplatezha", label: "Назначение платежа" }, + { key: "dogovorzakazchika", label: "Договор заказчика" }, + { key: "statyadop", label: "Статья (доп.)" }, + { key: "nomervhodyaschegodokumenta", label: "Номер вх. документа" }, +] as const; + +export const ALL_INCOME_COLUMNS: { key: string; label: string; type: "date" | "number" | "text" }[] = [ + { key: "data", label: "Дата", type: "date" }, + { key: "summa", label: "Сумма", type: "number" }, + { key: "statya", label: "Статья", type: "text" }, + { key: "zakazchik", label: "Заказчик", type: "text" }, + { key: "dochka", label: "Дочка", type: "text" }, + { key: "vnuchka", label: "Внучка", type: "text" }, + { key: "otvetstvenniipokontraktu", label: "Ответственный", type: "text" }, + { key: "stavkands", label: "Ставка НДС", type: "text" }, + { key: "kommentarii", label: "Комментарий", type: "text" }, + { key: "dogovorkontragenta", label: "Договор контрагента", type: "text" }, + { key: "schetnaoplatu", label: "Счет на оплату", type: "text" }, + { key: "naznachenieplatezha", label: "Назначение платежа", type: "text" }, + { key: "dogovorzakazchika", label: "Договор заказчика", type: "text" }, + { key: "statyadop", label: "Статья (доп.)", type: "text" }, + { key: "nomervhodyaschegodokumenta", label: "Номер вх. документа", type: "text" }, + { key: "id", label: "ID", type: "text" }, +]; + +export const DEFAULT_VISIBLE_COLUMNS = ["data", "summa", "statya", "zakazchik", "dochka", "vnuchka", "kommentarii"]; + +export const ALL_EXPENSE_COLUMNS: { key: string; label: string; type: "date" | "number" | "text" }[] = [ + { key: "data", label: "Дата", type: "date" }, + { key: "summadokumenta", label: "Сумма документа", type: "number" }, + { key: "statya", label: "Статья", type: "text" }, + { key: "zakazchik", label: "Заказчик", type: "text" }, + { key: "dochka", label: "Дочка", type: "text" }, + { key: "vnuchka", label: "Внучка", type: "text" }, + { key: "kommentarii", label: "Комментарий", type: "text" }, + { key: "stavkands", label: "Ставка НДС", type: "text" }, + { key: "naznachenieplatezha", label: "Назначение платежа", type: "text" }, + { key: "otvetstvenniipokontraktu", label: "Ответственный", type: "text" }, + { key: "dogovorkontragenta", label: "Договор контрагента", type: "text" }, + { key: "dogovorzakazchika", label: "Договор заказчика", type: "text" }, + { key: "statyadop", label: "Статья (доп.)", type: "text" }, + { key: "ssilkanomervhodyaschegodokumenta", label: "Номер вх. документа", type: "text" }, + { key: "kontragent", label: "Контрагент", type: "text" }, +]; + +export const EXPENSE_FILTERABLE_COLUMNS = [ + { key: "data", label: "Дата" }, + { key: "summadokumenta", label: "Сумма документа" }, + { key: "statya", label: "Статья" }, + { key: "zakazchik", label: "Заказчик" }, + { key: "dochka", label: "Дочка" }, + { key: "vnuchka", label: "Внучка" }, + { key: "kommentarii", label: "Комментарий" }, + { key: "stavkands", label: "Ставка НДС" }, + { key: "naznachenieplatezha", label: "Назначение платежа" }, + { key: "otvetstvenniipokontraktu", label: "Ответственный" }, + { key: "dogovorkontragenta", label: "Договор контрагента" }, + { key: "dogovorzakazchika", label: "Договор заказчика" }, + { key: "statyadop", label: "Статья (доп.)" }, + { key: "ssilkanomervhodyaschegodokumenta", label: "Номер вх. документа" }, + { key: "kontragent", label: "Контрагент" }, +] as const; + +export const DEFAULT_VISIBLE_EXPENSE_COLUMNS = ["data", "summadokumenta", "statya", "zakazchik", "dochka", "vnuchka", "kommentarii"]; + +export const CATEGORY_RULE_COLUMNS = [ + ...INCOME_FILTERABLE_COLUMNS.map(c => ({ ...c, type: "text" as const })), + { key: "summa", label: "Сумма", type: "number" as const }, +]; + +export const EXPENSE_CATEGORY_RULE_COLUMNS = [ + ...EXPENSE_FILTERABLE_COLUMNS.filter(c => c.key !== "summadokumenta").map(c => ({ ...c, type: "text" as const })), + { key: "summadokumenta", label: "Сумма документа", type: "number" as const }, +]; + +export interface ExpenseCategory { + id: string; + name: string; + rules: IncomeCategoryRule[]; + ruleLogic?: "and" | "or"; + excluded?: boolean; + children?: ExpenseCategory[]; + autoDetected?: boolean; + treeId?: string; +} diff --git a/server/finance-di2/table-config.ts b/server/finance-di2/table-config.ts new file mode 100644 index 0000000..af95f49 --- /dev/null +++ b/server/finance-di2/table-config.ts @@ -0,0 +1,154 @@ +import fs from "fs"; +import path from "path"; +import type { TableConfig } from "./schema"; + +const TABLE_CONFIG_FILE = path.join(process.cwd(), "data", "table-config.json"); + +const SYSTEM_COLUMNS = ["id", "dataobnovleniyazapisi"]; + +const DEFAULT_COLUMN_LABELS: Record = { + data: "Дата", + summa: "Сумма", + summadokumenta: "Сумма документа", + statya: "Статья", + zakazchik: "Заказчик", + dochka: "Дочка", + vnuchka: "Внучка", + kommentarii: "Комментарий", + stavkands: "Ставка НДС", + naznachenieplatezha: "Назначение платежа", + otvetstvenniipokontraktu: "Ответственный", + dogovorkontragenta: "Договор контрагента", + dogovorzakazchika: "Договор заказчика", + statyadop: "Статья (доп.)", + schetnaoplatu: "Счет на оплату", + nomervhodyaschegodokumenta: "Номер вх. документа", + ssilkanomervhodyaschegodokumenta: "Номер вх. документа", + kontragent: "Контрагент", + dogovor: "Договор", + summadolga: "Сумма долга", + summaprosrocheno: "Сумма просрочено", + period: "Период", + klassprosrochki: "Класс просрочки", + otvetstvennii: "Ответственный", + organizaciya: "Организация", + dokumentraschetov: "Документ расчётов", + summaoplacheno: "Сумма оплачено", + dneiprosrocheno: "Дней просрочено", + dneidolga: "Дней долга", + dataobrazovaniyadolga: "Дата образования долга", + planovayadatapogasheniya: "Плановая дата погашения", + otsrochkapodogovoru: "Отсрочка по договору", + kontragentinn: "ИНН контрагента", + valyuta: "Валюта", + summadolgaval: "Сумма долга (вал.)", + schet: "Счёт", + klassdolga: "Класс долга", + dataobnovleniyazapisi: "Дата обновления", +}; + +export const DEFAULT_TABLE_CONFIG: TableConfig = { + incomeTable: "opdohodi", + expenseTable: "oprashodi", + debtTable: "vw_debt_active", + incomeSumColumn: "summa", + expenseSumColumn: "summadokumenta", + incomeColumns: ["id", "data", "summa", "statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands", "naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta", "schetnaoplatu", "dogovorzakazchika", "statyadop", "nomervhodyaschegodokumenta", "dataobnovleniyazapisi"], + expenseColumns: ["id", "data", "summadokumenta", "statya", "zakazchik", "dochka", "vnuchka", "kommentarii", "stavkands", "naznachenieplatezha", "otvetstvenniipokontraktu", "dogovorkontragenta", "dogovorzakazchika", "statyadop", "ssilkanomervhodyaschegodokumenta", "kontragent", "dataobnovleniyazapisi"], + columnLabels: { ...DEFAULT_COLUMN_LABELS }, +}; + +let cachedConfig: TableConfig | null = null; + +export function loadTableConfig(): TableConfig { + if (cachedConfig) return cachedConfig; + try { + const raw = fs.readFileSync(TABLE_CONFIG_FILE, "utf-8"); + const parsed = JSON.parse(raw); + cachedConfig = { ...DEFAULT_TABLE_CONFIG, ...parsed, columnLabels: { ...DEFAULT_COLUMN_LABELS, ...(parsed.columnLabels || {}) } }; + return cachedConfig!; + } catch { + cachedConfig = { ...DEFAULT_TABLE_CONFIG }; + return cachedConfig; + } +} + +export function saveTableConfig(cfg: Partial) { + const current = loadTableConfig(); + const merged: TableConfig = { + ...current, + ...cfg, + columnLabels: { ...current.columnLabels, ...(cfg.columnLabels || {}) }, + }; + if (!fs.existsSync("data")) fs.mkdirSync("data", { recursive: true }); + fs.writeFileSync(TABLE_CONFIG_FILE, JSON.stringify(merged, null, 2), "utf-8"); + cachedConfig = merged; +} + +export function getIncomeTable(): string { + return "vw_income_effective"; +} + +export function getExpenseTable(): string { + return "vw_expense_effective"; +} + +export function getIncomeBaseTable(): string { + return loadTableConfig().incomeTable; +} + +export function getExpenseBaseTable(): string { + return loadTableConfig().expenseTable; +} + +export function getIncomeEffectiveView(): string { + return "vw_income_effective"; +} + +export function getExpenseEffectiveView(): string { + return "vw_expense_effective"; +} + +export function getDebtTable(): string { + return loadTableConfig().debtTable; +} + +export function getIncomeSumColumn(): string { + return loadTableConfig().incomeSumColumn; +} + +export function getExpenseSumColumn(): string { + return loadTableConfig().expenseSumColumn; +} + +export function getAllowedIncomeColumns(): string[] { + const cfg = loadTableConfig(); + return cfg.incomeColumns.filter(c => !SYSTEM_COLUMNS.includes(c)); +} + +export function getAllowedExpenseColumns(): string[] { + const cfg = loadTableConfig(); + return cfg.expenseColumns.filter(c => !SYSTEM_COLUMNS.includes(c)); +} + +export function getAutoDetectAllowedColumns(type: "income" | "expense"): string[] { + const allowed = type === "income" ? getAllowedIncomeColumns() : getAllowedExpenseColumns(); + return allowed.filter(c => c !== "data" && c !== "id" && c !== "summa" && c !== "summadokumenta" && c !== "dataobnovleniyazapisi"); +} + +export function getColumnLabel(col: string): string { + const cfg = loadTableConfig(); + return cfg.columnLabels[col] || col; +} + +export function getAllIncomeColumns(): string[] { + return loadTableConfig().incomeColumns; +} + +export function getAllExpenseColumns(): string[] { + return loadTableConfig().expenseColumns; +} + +export function invalidateTableConfigCache() { + cachedConfig = null; +} diff --git a/server/finance/db-client.ts b/server/finance/db-client.ts index 75ece90..b0543c2 100644 --- a/server/finance/db-client.ts +++ b/server/finance/db-client.ts @@ -125,3 +125,15 @@ export async function isConnected(): Promise { return false; } } + +const CONNECTION_ERROR_CODES = new Set(["57P01", "57P02", "57P03", "08000", "08001", "08003", "08006"]); + +export function isConnectionError(err: unknown): boolean { + if (!(err instanceof Error)) return false; + const code = (err as any).code; + if (code && CONNECTION_ERROR_CODES.has(code)) return true; + const msg = err.message.toLowerCase(); + return msg.includes("econnrefused") || msg.includes("etimedout") || msg.includes("enotfound") || + msg.includes("connection terminated") || msg.includes("connection timeout") || + msg.includes("закрытие подключения") || msg.includes("the database system is shutting down"); +} diff --git a/server/index.ts b/server/index.ts index d321921..8ccadde 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1057,6 +1057,14 @@ async function runStartupDataPatches() { // Static files for VPN client apps (olcbox APK/IPA) app.use('/apps', express.static('/app/apps')); + // Data-Insight2 sub-app (финансы v2) — сборка кладётся в dist/public-di2 (Task 2). + // Блок стоит ПЕРЕД setupVite/serveStatic, чтобы и в dev, и в prod /di2/* не уходил в main index.html. + const di2Dist = path.resolve(import.meta.dirname, "public-di2"); + if (fs.existsSync(di2Dist)) { + app.use("/di2", express.static(di2Dist)); + app.use("/di2", (_req, res) => res.sendFile(path.resolve(di2Dist, "index.html"))); + } + // Seed application roles and permissions try { await storage.seedAppRolesAndPermissions(); diff --git a/server/routes/index.ts b/server/routes/index.ts index 84002ff..2ae9788 100644 --- a/server/routes/index.ts +++ b/server/routes/index.ts @@ -31,6 +31,7 @@ import { registerPushRoutes } from "./push.routes"; import { registerMcpRagRoutes } from "./mcp-rag.routes"; import { registerFieldTemplateInsertRoutes } from "./field-template-insert.routes"; import { registerFinanceRoutes } from "../finance/routes"; +import { registerDi2Routes } from "../finance-di2"; import { registerDocumentRoutes } from "../documents/routes"; import { registerDashboardRoutes } from "./dashboard.routes"; import formAccessRouter from "./form-access.routes"; @@ -110,6 +111,8 @@ export async function registerRoutes(app: Express): Promise { registerPushRoutes(app); registerMcpRagRoutes(app); registerFieldTemplateInsertRoutes(app); + // Data-Insight2 (финансы v2) — СТРОГО перед старым модулем финансов: при совпадении путей приоритет у DI2 + await registerDi2Routes(app); await registerFinanceRoutes(app); await registerDocumentRoutes(app); registerDashboardRoutes(app);