feat: чанкинг batch-create >100 задач + журнал ошибок API

B: BatchCreateDialog режет список на чанки ≤100 (лимит сервера), каждый
чанк со своим idempotency-key (requestId-N) — повтор после ошибки не
создаёт дублей; 10-минутный лок армируется только при полном успехе,
а не до запроса (раньше блокировал даже после 400).

D: таблица error_logs (миграция 0066), захват ответов >=400 в request
middleware (без 401/health/events, дедуп 60с), admin API GET/DELETE
/api/error-logs и вкладка «Журнал ошибок» в Настройках.
This commit is contained in:
2026-08-03 14:55:29 +03:00
parent 1bb6a53226
commit 1d5321ed0c
8 changed files with 406 additions and 21 deletions

View File

@@ -54,6 +54,8 @@ interface LockRecord {
}
const LOCK_DURATION_MS = 10 * 60 * 1000;
// Сервер принимает максимум 100 задач за один batch — большие пачки режем на чанки
const MAX_BATCH_SIZE = 100;
function getLockKey(formId: number) {
return `batch-create-lock-${formId}`;
@@ -199,12 +201,6 @@ export function BatchCreateDialog({ formId, open, onOpenChange, onSuccess }: Bat
const currentRequestId = requestId || crypto.randomUUID();
setRequestId(currentRequestId);
// Lock immediately so repeated clicks (or retries after a network error) cannot double-create
const lockRecord: LockRecord = { requestId: currentRequestId, startedAt: Date.now() };
localStorage.setItem(getLockKey(formId), JSON.stringify(lockRecord));
setIsLocked(true);
setLockMinutes(getRemainingLockMinutes(lockRecord.startedAt));
try {
setCreating(true);
const taskList: Array<{
@@ -229,24 +225,48 @@ export function BatchCreateDialog({ formId, open, onOpenChange, onSuccess }: Bat
taskList.push(task);
}
const res = await apiRequest('POST', `/api/forms/${formId}/tasks/batch`, {
idempotencyKey: currentRequestId,
tasks: taskList,
});
const data = await res.json();
if (data.success) {
toast({ title: data.cached ? `Задачи уже созданы (${data.created})` : `Создано задач: ${data.created}` });
onOpenChange(false);
onSuccess?.();
queryClient.invalidateQueries({ queryKey: ['/api/dashboard'] });
} else {
toast({ title: data.error || 'Ошибка создания', variant: 'destructive' });
// Разбиваем на чанки ≤ MAX_BATCH_SIZE (лимит сервера — 100 задач за batch).
// У каждого чанка свой idempotency-key, производный от requestId: повторный запуск
// после ошибки безопасен — уже созданные чанки вернутся из серверного кэша.
const chunks: typeof taskList[] = [];
for (let i = 0; i < taskList.length; i += MAX_BATCH_SIZE) {
chunks.push(taskList.slice(i, i + MAX_BATCH_SIZE));
}
let totalCreated = 0;
let allCached = chunks.length > 0;
for (let i = 0; i < chunks.length; i++) {
const res = await apiRequest('POST', `/api/forms/${formId}/tasks/batch`, {
idempotencyKey: `${currentRequestId}-${i}`,
tasks: chunks[i],
});
const data = await res.json();
if (!data.success) {
toast({
title: chunks.length > 1 ? `Ошибка в пачке ${i + 1} из ${chunks.length}` : 'Ошибка создания',
description: data.error || undefined,
variant: 'destructive',
});
return; // без лока — можно исправить и повторить, созданные чанки не дублируются
}
totalCreated += typeof data.created === 'number' ? data.created : chunks[i].length;
if (!data.cached) allCached = false;
}
// Лок против повторного создания армируем только при полном успехе
const lockRecord: LockRecord = { requestId: currentRequestId, startedAt: Date.now() };
localStorage.setItem(getLockKey(formId), JSON.stringify(lockRecord));
setIsLocked(true);
setLockMinutes(getRemainingLockMinutes(lockRecord.startedAt));
toast({ title: allCached ? `Задачи уже созданы (${totalCreated})` : `Создано задач: ${totalCreated}` });
onOpenChange(false);
onSuccess?.();
queryClient.invalidateQueries({ queryKey: ['/api/dashboard'] });
} catch (e) {
toast({ title: 'Ошибка создания задач', variant: 'destructive' });
} finally {
setCreating(false);
// Lock stays active for 10 minutes regardless of success or failure
}
};

View File

@@ -0,0 +1,186 @@
import { useState } from 'react';
import { useQuery, useMutation } from '@tanstack/react-query';
import { RefreshCw, Trash2, Search } from 'lucide-react';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { Badge } from '@/components/ui/badge';
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table';
import { apiRequest, queryClient } from '@/lib/queryClient';
import { useToast } from '@/hooks/use-toast';
interface ErrorLogItem {
id: number;
method: string;
path: string;
statusCode: number;
message: string | null;
createdAt: string;
userId: number | null;
userFirstName: string | null;
userLastName: string | null;
userEmail: string | null;
}
const PAGE_SIZE = 50;
export function ErrorLogsTab() {
const { toast } = useToast();
const [status, setStatus] = useState('all');
const [days, setDays] = useState('7');
const [searchInput, setSearchInput] = useState('');
const [search, setSearch] = useState('');
const [offset, setOffset] = useState(0);
const params = new URLSearchParams();
params.set('limit', String(PAGE_SIZE));
params.set('offset', String(offset));
if (status === '4xx') params.set('status', '400');
if (status === '5xx') params.set('status', '500');
if (days !== 'all') params.set('days', days);
if (search) params.set('search', search);
const { data, isFetching, refetch } = useQuery<{ success: boolean; logs: ErrorLogItem[]; total: number }>({
queryKey: ['/api/error-logs', status, days, search, offset],
queryFn: async () => {
const res = await apiRequest('GET', `/api/error-logs?${params.toString()}`);
return res.json();
},
});
const clearMutation = useMutation({
mutationFn: async () => {
const res = await apiRequest('DELETE', '/api/error-logs');
return res.json();
},
onSuccess: () => {
toast({ title: 'Журнал ошибок очищен' });
setOffset(0);
queryClient.invalidateQueries({ queryKey: ['/api/error-logs'] });
},
onError: () => toast({ title: 'Не удалось очистить журнал', variant: 'destructive' }),
});
const logs = data?.logs ?? [];
const total = data?.total ?? 0;
const formatUser = (l: ErrorLogItem) => {
if (!l.userId) return '—';
const name = `${l.userFirstName || ''} ${l.userLastName || ''}`.trim();
return name || l.userEmail || `#${l.userId}`;
};
return (
<div className="space-y-3">
<div className="flex flex-wrap items-center gap-2">
<div className="relative">
<Search className="absolute left-2 top-1/2 -translate-y-1/2 w-3.5 h-3.5 text-muted-foreground" />
<Input
className="pl-7 h-8 text-xs w-56"
placeholder="Поиск по пути или тексту..."
value={searchInput}
onChange={(e) => setSearchInput(e.target.value)}
onKeyDown={(e) => { if (e.key === 'Enter') { setOffset(0); setSearch(searchInput.trim()); } }}
/>
</div>
<Select value={status} onValueChange={(v) => { setOffset(0); setStatus(v); }}>
<SelectTrigger className="h-8 w-32 text-xs"><SelectValue /></SelectTrigger>
<SelectContent>
<SelectItem value="all">Все ошибки</SelectItem>
<SelectItem value="4xx">Только 4xx</SelectItem>
<SelectItem value="5xx">Только 5xx</SelectItem>
</SelectContent>
</Select>
<Select value={days} onValueChange={(v) => { setOffset(0); setDays(v); }}>
<SelectTrigger className="h-8 w-36 text-xs"><SelectValue /></SelectTrigger>
<SelectContent>
<SelectItem value="1">За сутки</SelectItem>
<SelectItem value="7">За 7 дней</SelectItem>
<SelectItem value="30">За 30 дней</SelectItem>
<SelectItem value="all">За всё время</SelectItem>
</SelectContent>
</Select>
<Button variant="outline" size="sm" className="h-8 text-xs gap-1" onClick={() => refetch()} disabled={isFetching}>
<RefreshCw className={`w-3.5 h-3.5 ${isFetching ? 'animate-spin' : ''}`} />
Обновить
</Button>
<div className="flex-1" />
<Button
variant="ghost"
size="sm"
className="h-8 text-xs gap-1 text-destructive hover:text-destructive"
disabled={clearMutation.isPending || total === 0}
onClick={() => {
if (window.confirm(`Очистить весь журнал ошибок (${total} записей)?`)) clearMutation.mutate();
}}
>
<Trash2 className="w-3.5 h-3.5" />
Очистить
</Button>
</div>
<div className="text-[11px] text-muted-foreground">
Всего записей: {total}. Одинаковые ошибки записываются не чаще раза в минуту.
</div>
<div className="border rounded-md overflow-hidden">
<Table>
<TableHeader>
<TableRow>
<TableHead className="w-[150px]">Время</TableHead>
<TableHead className="w-[70px]">Код</TableHead>
<TableHead className="w-[220px]">Запрос</TableHead>
<TableHead>Сообщение</TableHead>
<TableHead className="w-[160px]">Пользователь</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{logs.length === 0 ? (
<TableRow>
<TableCell colSpan={5} className="text-center text-xs text-muted-foreground py-6">
{isFetching ? 'Загрузка...' : 'Ошибок нет'}
</TableCell>
</TableRow>
) : logs.map((l) => (
<TableRow key={l.id}>
<TableCell className="text-xs whitespace-nowrap">
{new Date(l.createdAt).toLocaleString('ru-RU')}
</TableCell>
<TableCell>
<Badge variant={l.statusCode >= 500 ? 'destructive' : 'secondary'} className="text-[10px]">
{l.statusCode}
</Badge>
</TableCell>
<TableCell className="text-xs font-mono break-all">
<span className="font-semibold">{l.method}</span> {l.path}
</TableCell>
<TableCell className="text-xs break-all">{l.message || '—'}</TableCell>
<TableCell className="text-xs">{formatUser(l)}</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
<div className="flex items-center gap-2">
<Button
variant="outline" size="sm" className="h-7 text-xs"
disabled={offset === 0 || isFetching}
onClick={() => setOffset(Math.max(0, offset - PAGE_SIZE))}
>
Назад
</Button>
<span className="text-[11px] text-muted-foreground">
{total === 0 ? 0 : offset + 1}–{Math.min(offset + PAGE_SIZE, total)} из {total}
</span>
<Button
variant="outline" size="sm" className="h-7 text-xs"
disabled={offset + PAGE_SIZE >= total || isFetching}
onClick={() => setOffset(offset + PAGE_SIZE)}
>
Вперёд
</Button>
</div>
</div>
);
}

View File

@@ -10,7 +10,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
import { Input } from '@/components/ui/input';
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select';
import { useToast } from '@/hooks/use-toast';
import { Settings as SettingsIcon, Loader2, ArrowLeft, Key, Copy, Check, Bot, BookOpen, ExternalLink, Trash2, Zap, Link2, Plus, Database, RefreshCw, BrainCircuit, Eye, EyeOff, CheckCircle2, XCircle, ServerCog, Pencil, Download, HardDrive, ChevronDown, ChevronUp, Palette } from 'lucide-react';
import { Settings as SettingsIcon, Loader2, ArrowLeft, Key, Copy, Check, Bot, BookOpen, ExternalLink, Trash2, Zap, Link2, Plus, Database, RefreshCw, BrainCircuit, Eye, EyeOff, CheckCircle2, XCircle, ServerCog, Pencil, Download, HardDrive, ChevronDown, ChevronUp, Palette, ScrollText } from 'lucide-react';
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from '@/components/ui/dialog';
import { Label } from '@/components/ui/label';
import { Switch } from '@/components/ui/switch';
@@ -18,6 +18,7 @@ import { Link } from 'wouter';
import { BotsContent } from '@/pages/Bots';
import { AutomationsContent } from '@/pages/Automations';
import { AppearanceSettings } from '@/pages/AppearanceSettings';
import { ErrorLogsTab } from '@/components/settings/ErrorLogsTab';
import { Badge } from '@/components/ui/badge';
import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
import { Checkbox } from '@/components/ui/checkbox';
@@ -1715,6 +1716,10 @@ const Settings = () => {
<BookOpen className="w-3 h-3" />
API Docs
</TabsTrigger>
<TabsTrigger value="error-logs" className="h-6 px-2.5 text-[11px] gap-1">
<ScrollText className="w-3 h-3" />
Журнал ошибок
</TabsTrigger>
</TabsList>
<TabsContent value="bots">
@@ -1949,6 +1954,18 @@ const Settings = () => {
</div>
</div>
</TabsContent>
<TabsContent value="error-logs" className="space-y-4">
<div className="border border-border rounded">
<div className="flex items-center gap-2 px-3 py-2 border-b border-border">
<ScrollText className="w-4 h-4 text-muted-foreground" />
<span className="text-sm font-medium">Журнал ошибок API</span>
</div>
<div className="p-3">
<ErrorLogsTab />
</div>
</div>
</TabsContent>
</Tabs>
</div>
</main>

View File

@@ -0,0 +1,16 @@
-- Серверный журнал ошибок API (просмотр админом в Настройках)
-- Без RLS: вставки выполняет только серверный middleware (best-effort, в т.ч. до/без auth),
-- чтение — только через admin-роут с явным фильтром organization_id.
CREATE TABLE IF NOT EXISTS error_logs (
id SERIAL PRIMARY KEY,
organization_id INTEGER REFERENCES organizations(id) ON DELETE CASCADE,
user_id INTEGER,
method VARCHAR(10) NOT NULL,
path TEXT NOT NULL,
status_code INTEGER NOT NULL,
message TEXT,
created_at TIMESTAMP NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS error_logs_org_created_idx ON error_logs(organization_id, created_at);
CREATE INDEX IF NOT EXISTS error_logs_status_idx ON error_logs(status_code);

View File

@@ -18,7 +18,7 @@ import { fileURLToPath } from "url";
import { isS3Enabled, ensureS3Bucket, streamFromS3 } from "./utils/s3";
import { getKnownEmbeddingDim, detectEmbeddingDimension, type LlmProviderConfig } from "./services/llm-provider";
import { decrypt as decryptSecret } from "./crypto";
import { fileUploads } from "@shared/schema";
import { fileUploads, errorLogs } from "@shared/schema";
import { authenticateToken, authenticateFileToken, type AuthenticatedRequest } from "./middleware/auth.middleware";
import { EXT_TO_MIME, getFileExt } from "./utils/upload";
import crypto from "crypto";
@@ -287,12 +287,55 @@ app.use((req, res, next) => {
}
log(logLine);
// Журнал ошибок API (просмотр админом в Настройках → «Журнал ошибок»)
captureErrorLog(req, res.statusCode, path, capturedJsonResponse);
}
});
next();
});
// --- Захват ошибок API в таблицу error_logs ---
// Шум исключаем: 401 (просроченные токены/SSE), health-check и SSE-поток.
// Дедупликация: одинаковые (method, path, status, message) не чаще раза в 60 сек.
const ERROR_LOG_EXCLUDED_PATHS = new Set(['/api/health', '/api/events']);
const errorLogDedupe = new Map<string, number>();
function captureErrorLog(req: Request, statusCode: number, path: string, body?: Record<string, any>) {
try {
if (statusCode < 400 || statusCode === 401) return;
if (ERROR_LOG_EXCLUDED_PATHS.has(path)) return;
const rawMessage = (body && (body.error || body.message)) || '';
const message = String(rawMessage).slice(0, 2000) || `HTTP ${statusCode}`;
const key = `${req.method} ${path} ${statusCode} ${message.slice(0, 200)}`;
const now = Date.now();
const lastSeen = errorLogDedupe.get(key) ?? 0;
if (now - lastSeen < 60_000) return;
errorLogDedupe.set(key, now);
// Периодическая чистка map, чтобы не разрастался
if (errorLogDedupe.size > 5000) {
for (const [k, ts] of errorLogDedupe) {
if (now - ts > 60_000) errorLogDedupe.delete(k);
}
}
const authReq = req as AuthenticatedRequest;
db.insert(errorLogs).values({
organizationId: authReq.organizationId ?? null,
userId: authReq.user?.id ?? null,
method: req.method,
path: path.slice(0, 500),
statusCode,
message,
}).catch((e: unknown) => console.error('[error-logs] insert failed:', e));
} catch (e) {
console.error('[error-logs] capture failed:', e);
}
}
/**
* Page seeds — canonical JS code stored in server/page-seeds/ as fallback/recovery.
* Seeds are applied ONLY when the current DB code contains known critical bugs.

View File

@@ -0,0 +1,80 @@
import { Router } from "express";
import { desc, eq, and, gte, or, ilike, sql } from "drizzle-orm";
import { db } from "../db";
import { errorLogs, users } from "@shared/schema";
import { authenticateToken, type AuthenticatedRequest } from "../middleware/auth.middleware";
const router = Router();
// Только администратор приложения
function requireAppAdmin(req: AuthenticatedRequest, res: any, next: any) {
if (req.user?.appRole !== 'admin') {
return res.status(403).json({ success: false, error: 'Доступно только администратору' });
}
next();
}
// GET /api/error-logs?limit=50&offset=0&status=500&search=batch&days=7
router.get('/api/error-logs', authenticateToken, requireAppAdmin, async (req: AuthenticatedRequest, res) => {
try {
const organizationId = req.organizationId!;
const limit = Math.min(parseInt(req.query.limit as string) || 50, 200);
const offset = parseInt(req.query.offset as string) || 0;
const status = req.query.status ? parseInt(req.query.status as string) : undefined;
const search = (req.query.search as string || '').trim();
const days = req.query.days ? parseInt(req.query.days as string) : undefined;
const conditions = [eq(errorLogs.organizationId, organizationId)];
if (status && !isNaN(status)) {
if (status === 400) conditions.push(and(gte(errorLogs.statusCode, 400), sql`${errorLogs.statusCode} < 500`)!);
else conditions.push(gte(errorLogs.statusCode, status));
}
if (search) {
conditions.push(or(ilike(errorLogs.path, `%${search}%`), ilike(errorLogs.message, `%${search}%`))!);
}
if (days && !isNaN(days) && days > 0) {
conditions.push(gte(errorLogs.createdAt, new Date(Date.now() - days * 24 * 60 * 60 * 1000)));
}
const where = and(...conditions);
const [rows, [{ count }]] = await Promise.all([
db.select({
id: errorLogs.id,
method: errorLogs.method,
path: errorLogs.path,
statusCode: errorLogs.statusCode,
message: errorLogs.message,
createdAt: errorLogs.createdAt,
userId: errorLogs.userId,
userFirstName: users.firstName,
userLastName: users.lastName,
userEmail: users.email,
})
.from(errorLogs)
.leftJoin(users, eq(errorLogs.userId, users.id))
.where(where)
.orderBy(desc(errorLogs.createdAt))
.limit(limit)
.offset(offset),
db.select({ count: sql<number>`count(*)::int` }).from(errorLogs).where(where),
]);
res.json({ success: true, logs: rows, total: count, limit, offset });
} catch (error) {
console.error('Get error logs error:', error);
res.status(500).json({ success: false, error: 'Ошибка при получении журнала ошибок' });
}
});
// DELETE /api/error-logs — очистить журнал организации
router.delete('/api/error-logs', authenticateToken, requireAppAdmin, async (req: AuthenticatedRequest, res) => {
try {
await db.delete(errorLogs).where(eq(errorLogs.organizationId, req.organizationId!));
res.json({ success: true });
} catch (error) {
console.error('Clear error logs error:', error);
res.status(500).json({ success: false, error: 'Ошибка при очистке журнала ошибок' });
}
});
export default router;

View File

@@ -20,6 +20,7 @@ import taskInboxHiddenRouter from "./task-inbox-hidden.routes";
import taskBulkRouter from "./task-bulk.routes";
import ragRouter from "./rag.routes";
import llmProvidersRouter from "./llm-providers.routes";
import errorLogsRouter from "./error-logs.routes";
import { registerBotRoutes } from "./bots.routes";
import { registerExternalRoutes } from "./external.routes";
import { registerBotApiRoutes } from "./bot-api.routes";
@@ -96,6 +97,7 @@ export async function registerRoutes(app: Express): Promise<Server> {
app.use(taskInboxHiddenRouter);
app.use(ragRouter);
app.use(llmProvidersRouter);
app.use(errorLogsRouter);
// Domain route registrations
registerBotRoutes(app);

View File

@@ -3698,6 +3698,27 @@ export const idempotencyKeys = pgTable("idempotency_keys", {
export type IdempotencyKey = typeof idempotencyKeys.$inferSelect;
export type InsertIdempotencyKey = typeof idempotencyKeys.$inferInsert;
// =====================
// Error Logs (серверный журнал ошибок API, просмотр админом в Настройках)
// =====================
export const errorLogs = pgTable("error_logs", {
id: serial("id").primaryKey(),
organizationId: integer("organization_id").references(() => organizations.id, { onDelete: "cascade" }),
userId: integer("user_id"),
method: varchar("method", { length: 10 }).notNull(),
path: text("path").notNull(),
statusCode: integer("status_code").notNull(),
message: text("message"),
createdAt: timestamp("created_at").defaultNow(),
}, (table) => ({
orgCreatedIdx: index("error_logs_org_created_idx").on(table.organizationId, table.createdAt),
statusIdx: index("error_logs_status_idx").on(table.statusCode),
}));
export type ErrorLog = typeof errorLogs.$inferSelect;
export type InsertErrorLog = typeof errorLogs.$inferInsert;
export const FINANCE_COLORS = [
"#2563eb", "#e11d48", "#16a34a", "#f59e0b",
"#8b5cf6", "#06b6d4", "#ea580c", "#84cc16",