si/iistwin
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

RAG: ночная пакетная индексация эмбеддингов, очередь embedding_queue, MCP get_organization_context, снижение OLLAMA_KEEP_ALIVE

Browse Source
This commit is contained in:
Ильяс Султанов
2026-07-08 15:57:25 +03:00
parent 1e9131753d
commit 70dfe07fa6
11 changed files with 440 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
server/routes/chat.messages.routes.ts
View File

@@ -13,7 +13,7 @@ import { webhookService } from "../services/webhook.service";
import { generateBotServiceToken } from "../utils/jwt";
import { sendWebhook } from "../utils/webhook";
import { eventBus } from "./shared";
import { buildMessageText, upsertEmbedding } from "../services/embedding.service";
import { handleAiBotMention, checkBotAccess } from "../services/ai-bot.service";
import { db } from "../db";
import { fileUploads } from "@shared/schema";
@@ -300,22 +300,8 @@ export function registerChatMessageRoutes(router: Router): void {
}
if (messageData.messageType === 'comment') {
(async () => {
try {
const users = await storage.getUsersByOrganization(req.organizationId!);
const usersMap = new Map(users.map(u => [u.id, u]));
const authorUser = req.user?.id ? usersMap.get(req.user.id) : null;
const authorName = authorUser
? `${authorUser.firstName ?? ''} ${authorUser.middleName ?? ''} ${authorUser.lastName ?? ''}`.trim() || authorUser.email
: 'Пользователь';
const content = buildMessageText(createdMessage, authorName, task.title);
await upsertEmbedding(req.organizationId!, 'task_message', createdMessage.id, content, {
taskId, formId: task.formId, authorId: req.user?.id,
});
} catch (err) {
console.error('[RAG] indexMessage error:', err);
}
})();
storage.enqueueEmbedding(req.organizationId!, 'task_message', createdMessage.id, 'upsert')
.catch(err => console.error('[RAG] enqueue message embedding error:', err));
}
if (req.user?.id) {

8
server/routes/mcp-rag.routes.ts
View File

@@ -230,13 +230,7 @@ export function registerMcpRagRoutes(app: import("express").Express): void {
...(aiSummaryIsAuto !== undefined ? { aiSummaryIsAuto } : {}),
});
const { buildFormText, upsertEmbedding } = await import('../services/embedding.service');
const [fields, statuses] = await Promise.all([
storage.getFormFields(formId, req.organizationId!),
storage.getFormStatuses(formId, req.organizationId!),
]);
const content = buildFormText(updated, fields, statuses);
upsertEmbedding(req.organizationId!, 'form', formId, content, { name: updated.name }).catch(() => {});
storage.enqueueEmbedding(req.organizationId!, 'form', formId, 'upsert').catch(() => {});
res.json({ success: true, form: updated });
} catch (error) {

69
server/routes/task-helpers.ts
View File

@@ -1,22 +1,10 @@
import type { Request } from 'express';
import { storage } from "../storage";
import {
buildFormText, generateFormSummary, upsertEmbedding, buildTaskText,
} from "../services/embedding.service";
import { generateFormSummary } from "../services/embedding.service";
async function indexFormAsync(formId: number, organizationId: number): Promise<void> {
try {
const [form, fields, statuses] = await Promise.all([
storage.getForm(formId, organizationId),
storage.getFormFields(formId, organizationId),
storage.getFormStatuses(formId, organizationId),
]);
if (!form) return;
const content = buildFormText(form, fields, statuses);
await upsertEmbedding(organizationId, 'form', formId, content, {
name: form.name,
description: form.description,
});
await storage.enqueueEmbedding(organizationId, 'form', formId, 'upsert');
} catch (err) {
console.error('[RAG] indexFormAsync error:', err);
}
@@ -24,22 +12,20 @@ async function indexFormAsync(formId: number, organizationId: number): Promise<v
async function indexFormWithSummaryAsync(formId: number, organizationId: number): Promise<void> {
try {
// Summary generation uses external chat provider (Kimi), so it can stay synchronous.
const [form, fields, statuses] = await Promise.all([
storage.getForm(formId, organizationId),
storage.getFormFields(formId, organizationId),
storage.getFormStatuses(formId, organizationId),
]);
if (!form) return;
const summary = await generateFormSummary(form, fields, statuses);
if (summary) {
await storage.updateForm(formId, organizationId, { aiSummary: summary, aiSummaryIsAuto: true });
if (form) {
const summary = await generateFormSummary(form, fields, statuses);
if (summary) {
await storage.updateForm(formId, organizationId, { aiSummary: summary, aiSummaryIsAuto: true });
}
}
const indexedForm = summary ? { ...form, aiSummary: summary } : form;
const content = buildFormText(indexedForm, fields, statuses);
await upsertEmbedding(organizationId, 'form', formId, content, {
name: form.name,
description: form.description,
});
// Actual embedding is deferred to the nightly batch.
await storage.enqueueEmbedding(organizationId, 'form', formId, 'upsert');
} catch (err) {
console.error('[RAG] indexFormWithSummaryAsync error:', err);
}
@@ -47,40 +33,7 @@ async function indexFormWithSummaryAsync(formId: number, organizationId: number)
async function indexTaskAsync(taskId: number, organizationId: number): Promise<void> {
try {
const task = await storage.getTask(taskId, organizationId);
if (!task) return;
const [form, statuses, users, fieldValues, formFields] = await Promise.all([
storage.getForm(task.formId, organizationId),
storage.getFormStatuses(task.formId, organizationId),
storage.getUsersByOrganization(organizationId),
storage.getTaskFieldValues(taskId, organizationId),
storage.getFormFields(task.formId, organizationId),
]);
if (!form) return;
const statusMap = new Map(statuses.map(s => [s.id, s.name]));
const usersMap = new Map(users.map(u => [u.id, u]));
const assigneeUser = task.assignedTo ? usersMap.get(task.assignedTo) : null;
const assigneeName = assigneeUser
? `${assigneeUser.firstName ?? ''} ${assigneeUser.middleName ?? ''} ${assigneeUser.lastName ?? ''}`.trim() || assigneeUser.email
: null;
const fieldMap = new Map(formFields.map(f => [f.id, f]));
const enrichedFieldValues = fieldValues.map(fv => ({
name: fieldMap.get(fv.fieldId)?.name ?? '',
value: fv.value,
type: fieldMap.get(fv.fieldId)?.type ?? 'text',
}));
const content = buildTaskText(
task,
form.name,
statusMap.get(task.currentStatusId) ?? '',
assigneeName,
enrichedFieldValues
);
await upsertEmbedding(organizationId, 'task', taskId, content, {
formId: task.formId,
formName: form.name,
title: task.title,
});
await storage.enqueueEmbedding(organizationId, 'task', taskId, 'upsert');
} catch (err) {
console.error('[RAG] indexTaskAsync error:', err);
}