RAG: ночная пакетная индексация эмбеддингов, очередь embedding_queue, MCP get_organization_context, снижение OLLAMA_KEEP_ALIVE

server/services/embedding.service.ts

@@ -471,6 +471,166 @@ export async function deleteRagEmbeddingsByForm(
   `);
 }
 
+interface QueueItem {
+  id: number;
+  organizationId: number;
+  entityType: EntityType;
+  entityId: number;
+  operation: 'upsert' | 'delete';
+}
+
+export async function processEmbeddingQueue(
+  stor: IStorage,
+  items: QueueItem[],
+  onProgress?: (indexed: number, total: number) => void
+): Promise<{ attempted: number; succeeded: number; failed: number }> {
+  if (items.length === 0) return { attempted: 0, succeeded: 0, failed: 0 };
+
+  // Group by organization to reuse config and users cache
+  const byOrg = new Map<number, QueueItem[]>();
+  for (const item of items) {
+    const list = byOrg.get(item.organizationId) || [];
+    list.push(item);
+    byOrg.set(item.organizationId, list);
+  }
+
+  let attempted = 0;
+  let succeeded = 0;
+  let failed = 0;
+
+  for (const [organizationId, orgItems] of byOrg.entries()) {
+    const config = await resolveOrgConfig(organizationId);
+    const hasAccess = config.provider === 'ollama' || !!config.apiKey;
+    if (!hasAccess) {
+      console.warn(`[RAG] Skipping queue for org ${organizationId}: no API key configured`);
+      failed += orgItems.length;
+      continue;
+    }
+
+    const users = await stor.getUsersByOrganization(organizationId);
+    const usersMap = new Map(users.map(u => [u.id, u]));
+    const formCache = new Map<number, { form: Awaited<ReturnType<typeof stor.getForm>>; statuses: Awaited<ReturnType<typeof stor.getFormStatuses>>; fields: Awaited<ReturnType<typeof stor.getFormFields>> }>();
+
+    async function getFormMeta(formId: number) {
+      if (formCache.has(formId)) return formCache.get(formId)!;
+      const [form, statuses, formFields] = await Promise.all([
+        stor.getForm(formId, organizationId),
+        stor.getFormStatuses(formId, organizationId),
+        stor.getFormFields(formId, organizationId),
+      ]);
+      const meta = { form, statuses, fields: formFields };
+      formCache.set(formId, meta);
+      return meta;
+    }
+
+    const embeddingItems: EmbeddingItem[] = [];
+
+    for (const item of orgItems) {
+      attempted++;
+      try {
+        if (item.operation === 'delete') {
+          await deleteRagEmbedding(organizationId, item.entityType, item.entityId);
+          succeeded++;
+          if (onProgress) onProgress(succeeded + failed, attempted);
+          continue;
+        }
+
+        if (item.entityType === 'form') {
+          const form = await stor.getForm(item.entityId, organizationId);
+          if (!form) {
+            failed++;
+            continue;
+          }
+          const [fields, statuses] = await Promise.all([
+            stor.getFormFields(item.entityId, organizationId),
+            stor.getFormStatuses(item.entityId, organizationId),
+          ]);
+          const content = buildFormText(form, fields, statuses);
+          embeddingItems.push({
+            organizationId,
+            entityType: 'form',
+            entityId: item.entityId,
+            content,
+            metadata: { name: form.name },
+          });
+        } else if (item.entityType === 'task') {
+          const task = await stor.getTask(item.entityId, organizationId);
+          if (!task) {
+            failed++;
+            continue;
+          }
+          const { form, statuses, fields: formFields } = await getFormMeta(task.formId);
+          if (!form) {
+            failed++;
+            continue;
+          }
+          const statusMap = new Map(statuses.map(s => [s.id, s.name]));
+          const assigneeUser = task.assignedTo ? usersMap.get(task.assignedTo) : null;
+          const assigneeName = assigneeUser
+            ? `${assigneeUser.firstName ?? ''} ${assigneeUser.middleName ?? ''} ${assigneeUser.lastName ?? ''}`.trim() || assigneeUser.email
+            : null;
+          const fieldMap = new Map(formFields.map(f => [f.id, f]));
+          const fieldValues = await stor.getTaskFieldValues(task.id, organizationId);
+          const enrichedFieldValues = fieldValues.map(fv => ({
+            name: fieldMap.get(fv.fieldId)?.name ?? '',
+            value: fv.value,
+            type: fieldMap.get(fv.fieldId)?.type ?? 'text',
+          }));
+          const content = buildTaskText(
+            task,
+            form.name,
+            statusMap.get(task.currentStatusId) ?? '',
+            assigneeName,
+            enrichedFieldValues
+          );
+          embeddingItems.push({
+            organizationId,
+            entityType: 'task',
+            entityId: task.id,
+            content,
+            metadata: { formId: task.formId, formName: form.name, title: task.title },
+          });
+        } else if (item.entityType === 'task_message') {
+          const message = await stor.getTaskMessage(item.entityId, organizationId);
+          if (!message) {
+            failed++;
+            continue;
+          }
+          const task = await stor.getTask(message.taskId, organizationId);
+          if (!task) {
+            failed++;
+            continue;
+          }
+          const authorUser = message.authorId ? usersMap.get(message.authorId) : null;
+          const authorName = authorUser
+            ? `${authorUser.firstName ?? ''} ${authorUser.middleName ?? ''} ${authorUser.lastName ?? ''}`.trim() || authorUser.email
+            : 'Бот';
+          const content = buildMessageText(message, authorName, task.title);
+          embeddingItems.push({
+            organizationId,
+            entityType: 'task_message',
+            entityId: message.id,
+            content,
+            metadata: { taskId: task.id, formId: task.formId, authorId: message.authorId },
+          });
+        }
+      } catch (err) {
+        console.error(`[RAG] Failed to prepare queue item ${item.id}:`, err);
+        failed++;
+        if (onProgress) onProgress(succeeded + failed, attempted);
+      }
+    }
+
+    const batchSucceeded = await processBatchedItems(embeddingItems, config, (batchIndexed, total) => {
+      if (onProgress) onProgress(succeeded + failed + batchIndexed, attempted);
+    });
+    succeeded += batchSucceeded;
+    failed += embeddingItems.length - batchSucceeded;
+  }
+
+  return { attempted, succeeded, failed };
+}
+
 const TASK_CHUNK_SIZE = 100;
 
 export async function reindexOrganization(