|
@@ -2,8 +2,9 @@ import { Injectable, Logger, NotFoundException, Inject, forwardRef } from '@nest
|
|
|
import { ConfigService } from '@nestjs/config';
|
|
import { ConfigService } from '@nestjs/config';
|
|
|
import { I18nService } from '../i18n/i18n.service';
|
|
import { I18nService } from '../i18n/i18n.service';
|
|
|
import { InjectRepository } from '@nestjs/typeorm';
|
|
import { InjectRepository } from '@nestjs/typeorm';
|
|
|
-import { Repository } from 'typeorm';
|
|
|
|
|
|
|
+import { Repository, In } from 'typeorm';
|
|
|
import { FileStatus, KnowledgeBase, ProcessingMode } from './knowledge-base.entity';
|
|
import { FileStatus, KnowledgeBase, ProcessingMode } from './knowledge-base.entity';
|
|
|
|
|
+import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
|
|
|
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
|
|
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
|
|
|
import { TikaService } from '../tika/tika.service';
|
|
import { TikaService } from '../tika/tika.service';
|
|
|
import * as fs from 'fs';
|
|
import * as fs from 'fs';
|
|
@@ -29,6 +30,9 @@ export class KnowledgeBaseService {
|
|
|
constructor(
|
|
constructor(
|
|
|
@InjectRepository(KnowledgeBase)
|
|
@InjectRepository(KnowledgeBase)
|
|
|
private kbRepository: Repository<KnowledgeBase>,
|
|
private kbRepository: Repository<KnowledgeBase>,
|
|
|
|
|
+ @InjectRepository(KnowledgeGroup)
|
|
|
|
|
+ private groupRepository: Repository<KnowledgeGroup>,
|
|
|
|
|
+ @Inject(forwardRef(() => ElasticsearchService))
|
|
|
private elasticsearchService: ElasticsearchService,
|
|
private elasticsearchService: ElasticsearchService,
|
|
|
private tikaService: TikaService,
|
|
private tikaService: TikaService,
|
|
|
private embeddingService: EmbeddingService,
|
|
private embeddingService: EmbeddingService,
|
|
@@ -52,6 +56,7 @@ export class KnowledgeBaseService {
|
|
|
async createAndIndex(
|
|
async createAndIndex(
|
|
|
fileInfo: any,
|
|
fileInfo: any,
|
|
|
userId: string,
|
|
userId: string,
|
|
|
|
|
+ tenantId: string,
|
|
|
config?: any,
|
|
config?: any,
|
|
|
): Promise<KnowledgeBase> {
|
|
): Promise<KnowledgeBase> {
|
|
|
const mode = config?.mode || 'fast';
|
|
const mode = config?.mode || 'fast';
|
|
@@ -64,24 +69,60 @@ export class KnowledgeBaseService {
|
|
|
mimetype: fileInfo.mimetype,
|
|
mimetype: fileInfo.mimetype,
|
|
|
status: FileStatus.PENDING,
|
|
status: FileStatus.PENDING,
|
|
|
userId: userId,
|
|
userId: userId,
|
|
|
|
|
+ tenantId: tenantId,
|
|
|
chunkSize: config?.chunkSize || 200,
|
|
chunkSize: config?.chunkSize || 200,
|
|
|
chunkOverlap: config?.chunkOverlap || 40,
|
|
chunkOverlap: config?.chunkOverlap || 40,
|
|
|
embeddingModelId: config?.embeddingModelId || null,
|
|
embeddingModelId: config?.embeddingModelId || null,
|
|
|
processingMode: processingMode,
|
|
processingMode: processingMode,
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
|
|
+ // 分類(グループ)の関連付け
|
|
|
|
|
+ if (config?.groupIds && config.groupIds.length > 0) {
|
|
|
|
|
+ const groups = await this.groupRepository.find({
|
|
|
|
|
+ where: { id: In(config.groupIds), tenantId: tenantId }
|
|
|
|
|
+ });
|
|
|
|
|
+ kb.groups = groups;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
const savedKb = await this.kbRepository.save(kb);
|
|
const savedKb = await this.kbRepository.save(kb);
|
|
|
|
|
|
|
|
this.logger.log(
|
|
this.logger.log(
|
|
|
`Created KB record: ${savedKb.id}, mode: ${mode}, file: ${fileInfo.originalname}`
|
|
`Created KB record: ${savedKb.id}, mode: ${mode}, file: ${fileInfo.originalname}`
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
|
|
+ // ---------------------------------------------------------
|
|
|
|
|
+ // Move the file to the final partitioned directory
|
|
|
|
|
+ // source: uploads/{tenantId}/{filename} (or wherever it was)
|
|
|
|
|
+ // target: uploads/{tenantId}/{savedKb.id}/{filename}
|
|
|
|
|
+ // ---------------------------------------------------------
|
|
|
|
|
+ const fs = await import('fs');
|
|
|
|
|
+ const path = await import('path');
|
|
|
|
|
+ const uploadPath = process.env.UPLOAD_FILE_PATH || './uploads';
|
|
|
|
|
+ const targetDir = path.join(uploadPath, tenantId || 'default', savedKb.id);
|
|
|
|
|
+ const targetPath = path.join(targetDir, fileInfo.filename);
|
|
|
|
|
+
|
|
|
|
|
+ try {
|
|
|
|
|
+ if (!fs.existsSync(targetDir)) {
|
|
|
|
|
+ fs.mkdirSync(targetDir, { recursive: true });
|
|
|
|
|
+ }
|
|
|
|
|
+ if (fs.existsSync(fileInfo.path)) {
|
|
|
|
|
+ fs.renameSync(fileInfo.path, targetPath);
|
|
|
|
|
+ // Update the DB record with the new path
|
|
|
|
|
+ savedKb.storagePath = targetPath;
|
|
|
|
|
+ await this.kbRepository.save(savedKb);
|
|
|
|
|
+ this.logger.log(`Moved file to partitioned storage: ${targetPath}`);
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (fsError) {
|
|
|
|
|
+ this.logger.error(`Failed to move file ${savedKb.id} to partitioned storage`, fsError);
|
|
|
|
|
+ // We will let it continue, but the file might be stuck in the temp/root folder
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
// If queue processing is requested, await completion
|
|
// If queue processing is requested, await completion
|
|
|
if (config?.waitForCompletion) {
|
|
if (config?.waitForCompletion) {
|
|
|
- await this.processFile(savedKb.id, userId, config);
|
|
|
|
|
|
|
+ await this.processFile(savedKb.id, userId, tenantId, config);
|
|
|
} else {
|
|
} else {
|
|
|
// Otherwise trigger asynchronously (default)
|
|
// Otherwise trigger asynchronously (default)
|
|
|
- this.processFile(savedKb.id, userId, config).catch((err) => {
|
|
|
|
|
|
|
+ this.processFile(savedKb.id, userId, tenantId, config).catch((err) => {
|
|
|
this.logger.error(`Error processing file ${savedKb.id}`, err);
|
|
this.logger.error(`Error processing file ${savedKb.id}`, err);
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
@@ -89,14 +130,21 @@ export class KnowledgeBaseService {
|
|
|
return savedKb;
|
|
return savedKb;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- async findAll(userId: string): Promise<KnowledgeBase[]> {
|
|
|
|
|
|
|
+ async findAll(userId: string, tenantId?: string): Promise<KnowledgeBase[]> {
|
|
|
|
|
+ const where: any = {};
|
|
|
|
|
+ if (tenantId) {
|
|
|
|
|
+ where.tenantId = tenantId;
|
|
|
|
|
+ } else {
|
|
|
|
|
+ where.userId = userId;
|
|
|
|
|
+ }
|
|
|
return this.kbRepository.find({
|
|
return this.kbRepository.find({
|
|
|
|
|
+ where,
|
|
|
relations: ['groups'], // グループリレーションをロード
|
|
relations: ['groups'], // グループリレーションをロード
|
|
|
order: { createdAt: 'DESC' },
|
|
order: { createdAt: 'DESC' },
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- async searchKnowledge(userId: string, query: string, topK: number = 5) {
|
|
|
|
|
|
|
+ async searchKnowledge(userId: string, tenantId: string, query: string, topK: number = 5) {
|
|
|
try {
|
|
try {
|
|
|
// 環境変数のデフォルト次元数を使用してシミュレーションベクトルを生成
|
|
// 環境変数のデフォルト次元数を使用してシミュレーションベクトルを生成
|
|
|
const defaultDimensions = parseInt(
|
|
const defaultDimensions = parseInt(
|
|
@@ -113,6 +161,7 @@ export class KnowledgeBaseService {
|
|
|
queryVector,
|
|
queryVector,
|
|
|
userId,
|
|
userId,
|
|
|
topK,
|
|
topK,
|
|
|
|
|
+ tenantId, // Ensure shared visibility within tenant
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
// 3. Get file information from database
|
|
// 3. Get file information from database
|
|
@@ -144,14 +193,14 @@ export class KnowledgeBaseService {
|
|
|
};
|
|
};
|
|
|
} catch (error) {
|
|
} catch (error) {
|
|
|
this.logger.error(
|
|
this.logger.error(
|
|
|
- this.i18nService.formatMessage('searchMetadataFailed', { userId }),
|
|
|
|
|
- error,
|
|
|
|
|
|
|
+ `Metadata search failed for tenant ${tenantId}:`,
|
|
|
|
|
+ error.stack || error.message,
|
|
|
);
|
|
);
|
|
|
throw error;
|
|
throw error;
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- async ragSearch(userId: string, query: string, settings: any) {
|
|
|
|
|
|
|
+ async ragSearch(userId: string, tenantId: string, query: string, settings: any) {
|
|
|
this.logger.log(
|
|
this.logger.log(
|
|
|
`RAG search request: userId=${userId}, query="${query}", settings=${JSON.stringify(settings)}`,
|
|
`RAG search request: userId=${userId}, query="${query}", settings=${JSON.stringify(settings)}`,
|
|
|
);
|
|
);
|
|
@@ -169,6 +218,7 @@ export class KnowledgeBaseService {
|
|
|
undefined,
|
|
undefined,
|
|
|
undefined,
|
|
undefined,
|
|
|
settings.rerankSimilarityThreshold,
|
|
settings.rerankSimilarityThreshold,
|
|
|
|
|
+ tenantId, // Ensure shared visibility within tenant for RAG
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
const sources = this.ragService.extractSources(ragResults);
|
|
const sources = this.ragService.extractSources(ragResults);
|
|
@@ -204,13 +254,13 @@ export class KnowledgeBaseService {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- async deleteFile(fileId: string, userId: string): Promise<void> {
|
|
|
|
|
|
|
+ async deleteFile(fileId: string, userId: string, tenantId: string): Promise<void> {
|
|
|
this.logger.log(`Deleting file ${fileId} for user ${userId}`);
|
|
this.logger.log(`Deleting file ${fileId} for user ${userId}`);
|
|
|
|
|
|
|
|
try {
|
|
try {
|
|
|
// 1. Get file info
|
|
// 1. Get file info
|
|
|
const file = await this.kbRepository.findOne({
|
|
const file = await this.kbRepository.findOne({
|
|
|
- where: { id: fileId },
|
|
|
|
|
|
|
+ where: { id: fileId, tenantId }, // Filter by tenantId
|
|
|
});
|
|
});
|
|
|
if (!file) {
|
|
if (!file) {
|
|
|
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
|
throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
|
|
@@ -229,7 +279,7 @@ export class KnowledgeBaseService {
|
|
|
|
|
|
|
|
// 3. Delete from Elasticsearch
|
|
// 3. Delete from Elasticsearch
|
|
|
try {
|
|
try {
|
|
|
- await this.elasticsearchService.deleteByFileId(fileId, userId);
|
|
|
|
|
|
|
+ await this.elasticsearchService.deleteByFileId(fileId, userId, tenantId);
|
|
|
this.logger.log(`Deleted ES documents for file ${fileId}`);
|
|
this.logger.log(`Deleted ES documents for file ${fileId}`);
|
|
|
} catch (error) {
|
|
} catch (error) {
|
|
|
this.logger.warn(
|
|
this.logger.warn(
|
|
@@ -240,7 +290,7 @@ export class KnowledgeBaseService {
|
|
|
|
|
|
|
|
// 4. Remove from all groups (cleanup M2M relations)
|
|
// 4. Remove from all groups (cleanup M2M relations)
|
|
|
const fileWithGroups = await this.kbRepository.findOne({
|
|
const fileWithGroups = await this.kbRepository.findOne({
|
|
|
- where: { id: fileId },
|
|
|
|
|
|
|
+ where: { id: fileId, tenantId },
|
|
|
relations: ['groups'],
|
|
relations: ['groups'],
|
|
|
});
|
|
});
|
|
|
|
|
|
|
@@ -261,15 +311,15 @@ export class KnowledgeBaseService {
|
|
|
|
|
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- async clearAll(userId: string): Promise<void> {
|
|
|
|
|
- this.logger.log(`Clearing all knowledge base data for user ${userId}`);
|
|
|
|
|
|
|
+ async clearAll(userId: string, tenantId: string): Promise<void> {
|
|
|
|
|
+ this.logger.log(`Clearing all knowledge base data for user ${userId} in tenant ${tenantId}`);
|
|
|
|
|
|
|
|
try {
|
|
try {
|
|
|
// Get all files and delete them one by one
|
|
// Get all files and delete them one by one
|
|
|
const files = await this.kbRepository.find();
|
|
const files = await this.kbRepository.find();
|
|
|
|
|
|
|
|
for (const file of files) {
|
|
for (const file of files) {
|
|
|
- await this.deleteFile(file.id, userId);
|
|
|
|
|
|
|
+ await this.deleteFile(file.id, userId, tenantId);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
this.logger.log(`Cleared all knowledge base data for user ${userId}`);
|
|
this.logger.log(`Cleared all knowledge base data for user ${userId}`);
|
|
@@ -282,7 +332,7 @@ export class KnowledgeBaseService {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- private async processFile(kbId: string, userId: string, config?: any) {
|
|
|
|
|
|
|
+ private async processFile(kbId: string, userId: string, tenantId: string, config?: any) {
|
|
|
this.logger.log(`Starting processing for file ${kbId}, mode: ${config?.mode || 'fast'}`);
|
|
this.logger.log(`Starting processing for file ${kbId}, mode: ${config?.mode || 'fast'}`);
|
|
|
await this.updateStatus(kbId, FileStatus.INDEXING);
|
|
await this.updateStatus(kbId, FileStatus.INDEXING);
|
|
|
|
|
|
|
@@ -302,10 +352,10 @@ export class KnowledgeBaseService {
|
|
|
|
|
|
|
|
if (mode === 'precise') {
|
|
if (mode === 'precise') {
|
|
|
// 精密モード - Vision Pipeline を使用
|
|
// 精密モード - Vision Pipeline を使用
|
|
|
- await this.processPreciseMode(kb, userId, config);
|
|
|
|
|
|
|
+ await this.processPreciseMode(kb, userId, tenantId, config);
|
|
|
} else {
|
|
} else {
|
|
|
// 高速モード - Tika を使用
|
|
// 高速モード - Tika を使用
|
|
|
- await this.processFastMode(kb, userId, config);
|
|
|
|
|
|
|
+ await this.processFastMode(kb, userId, tenantId, config);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
this.logger.log(`File ${kbId} processed successfully in ${mode} mode.`);
|
|
this.logger.log(`File ${kbId} processed successfully in ${mode} mode.`);
|
|
@@ -318,7 +368,7 @@ export class KnowledgeBaseService {
|
|
|
/**
|
|
/**
|
|
|
* 高速モード処理(既存フロー)
|
|
* 高速モード処理(既存フロー)
|
|
|
*/
|
|
*/
|
|
|
- private async processFastMode(kb: KnowledgeBase, userId: string, config?: any) {
|
|
|
|
|
|
|
+ private async processFastMode(kb: KnowledgeBase, userId: string, tenantId: string, config?: any) {
|
|
|
// 1. Tika を使用してテキストを抽出
|
|
// 1. Tika を使用してテキストを抽出
|
|
|
let text = await this.tikaService.extractText(kb.storagePath);
|
|
let text = await this.tikaService.extractText(kb.storagePath);
|
|
|
|
|
|
|
@@ -329,6 +379,7 @@ export class KnowledgeBaseService {
|
|
|
const visionModel = await this.modelConfigService.findOne(
|
|
const visionModel = await this.modelConfigService.findOne(
|
|
|
visionModelId,
|
|
visionModelId,
|
|
|
userId,
|
|
userId,
|
|
|
|
|
+ tenantId,
|
|
|
);
|
|
);
|
|
|
if (visionModel && visionModel.type === 'vision' && visionModel.isEnabled !== false) {
|
|
if (visionModel && visionModel.type === 'vision' && visionModel.isEnabled !== false) {
|
|
|
text = await this.visionService.extractImageContent(kb.storagePath, {
|
|
text = await this.visionService.extractImageContent(kb.storagePath, {
|
|
@@ -355,7 +406,7 @@ export class KnowledgeBaseService {
|
|
|
await this.updateStatus(kb.id, FileStatus.EXTRACTED);
|
|
await this.updateStatus(kb.id, FileStatus.EXTRACTED);
|
|
|
|
|
|
|
|
// 非同期ベクトル化
|
|
// 非同期ベクトル化
|
|
|
- await this.vectorizeToElasticsearch(kb.id, userId, text, config).catch((err) => {
|
|
|
|
|
|
|
+ await this.vectorizeToElasticsearch(kb.id, userId, tenantId, text, config).catch((err) => {
|
|
|
this.logger.error(`Error vectorizing file ${kb.id}`, err);
|
|
this.logger.error(`Error vectorizing file ${kb.id}`, err);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
@@ -365,7 +416,7 @@ export class KnowledgeBaseService {
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
// 非同期的に PDF 変換をトリガー(ドキュメントファイルの場合)
|
|
// 非同期的に PDF 変換をトリガー(ドキュメントファイルの場合)
|
|
|
- this.ensurePDFExists(kb.id, userId).catch((err) => {
|
|
|
|
|
|
|
+ this.ensurePDFExists(kb.id, userId, tenantId).catch((err) => {
|
|
|
this.logger.warn(this.i18nService.formatMessage('pdfConversionFailedDetail', { id: kb.id }), err);
|
|
this.logger.warn(this.i18nService.formatMessage('pdfConversionFailedDetail', { id: kb.id }), err);
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
@@ -373,7 +424,7 @@ export class KnowledgeBaseService {
|
|
|
/**
|
|
/**
|
|
|
* 精密モード処理(新規フロー)
|
|
* 精密モード処理(新規フロー)
|
|
|
*/
|
|
*/
|
|
|
- private async processPreciseMode(kb: KnowledgeBase, userId: string, config?: any) {
|
|
|
|
|
|
|
+ private async processPreciseMode(kb: KnowledgeBase, userId: string, tenantId: string, config?: any) {
|
|
|
// 精密モードがサポートされているか確認
|
|
// 精密モードがサポートされているか確認
|
|
|
const preciseFormats = ['.pdf', '.doc', '.docx', '.ppt', '.pptx'];
|
|
const preciseFormats = ['.pdf', '.doc', '.docx', '.ppt', '.pptx'];
|
|
|
const ext = kb.originalName.toLowerCase().substring(kb.originalName.lastIndexOf('.'));
|
|
const ext = kb.originalName.toLowerCase().substring(kb.originalName.lastIndexOf('.'));
|
|
@@ -382,7 +433,7 @@ export class KnowledgeBaseService {
|
|
|
this.logger.warn(
|
|
this.logger.warn(
|
|
|
this.i18nService.formatMessage('preciseModeUnsupported', { ext })
|
|
this.i18nService.formatMessage('preciseModeUnsupported', { ext })
|
|
|
);
|
|
);
|
|
|
- return this.processFastMode(kb, userId, config);
|
|
|
|
|
|
|
+ return this.processFastMode(kb, userId, tenantId, config);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Vision モデルが設定されているか確認
|
|
// Vision モデルが設定されているか確認
|
|
@@ -391,18 +442,19 @@ export class KnowledgeBaseService {
|
|
|
this.logger.warn(
|
|
this.logger.warn(
|
|
|
this.i18nService.getMessage('visionModelNotConfiguredFallback')
|
|
this.i18nService.getMessage('visionModelNotConfiguredFallback')
|
|
|
);
|
|
);
|
|
|
- return this.processFastMode(kb, userId, config);
|
|
|
|
|
|
|
+ return this.processFastMode(kb, userId, tenantId, config);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const visionModel = await this.modelConfigService.findOne(
|
|
const visionModel = await this.modelConfigService.findOne(
|
|
|
visionModelId,
|
|
visionModelId,
|
|
|
userId,
|
|
userId,
|
|
|
|
|
+ tenantId,
|
|
|
);
|
|
);
|
|
|
if (!visionModel || visionModel.type !== 'vision' || visionModel.isEnabled === false) {
|
|
if (!visionModel || visionModel.type !== 'vision' || visionModel.isEnabled === false) {
|
|
|
this.logger.warn(
|
|
this.logger.warn(
|
|
|
this.i18nService.getMessage('visionModelInvalidFallback')
|
|
this.i18nService.getMessage('visionModelInvalidFallback')
|
|
|
);
|
|
);
|
|
|
- return this.processFastMode(kb, userId, config);
|
|
|
|
|
|
|
+ return this.processFastMode(kb, userId, tenantId, config);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Vision Pipeline を呼び出し
|
|
// Vision Pipeline を呼び出し
|
|
@@ -411,6 +463,7 @@ export class KnowledgeBaseService {
|
|
|
kb.storagePath,
|
|
kb.storagePath,
|
|
|
{
|
|
{
|
|
|
userId,
|
|
userId,
|
|
|
|
|
+ tenantId, // New
|
|
|
modelId: visionModelId,
|
|
modelId: visionModelId,
|
|
|
fileId: kb.id,
|
|
fileId: kb.id,
|
|
|
fileName: kb.originalName,
|
|
fileName: kb.originalName,
|
|
@@ -421,7 +474,7 @@ export class KnowledgeBaseService {
|
|
|
if (!result.success) {
|
|
if (!result.success) {
|
|
|
this.logger.error(`Vision pipeline failed, falling back to fast mode`);
|
|
this.logger.error(`Vision pipeline failed, falling back to fast mode`);
|
|
|
this.logger.warn(this.i18nService.getMessage('visionPipelineFailed'));
|
|
this.logger.warn(this.i18nService.getMessage('visionPipelineFailed'));
|
|
|
- return this.processFastMode(kb, userId, config);
|
|
|
|
|
|
|
+ return this.processFastMode(kb, userId, tenantId, config);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// テキスト内容をデータベースに保存
|
|
// テキスト内容をデータベースに保存
|
|
@@ -450,12 +503,12 @@ export class KnowledgeBaseService {
|
|
|
|
|
|
|
|
// 非同期でベクトル化し、Elasticsearch にインデックス
|
|
// 非同期でベクトル化し、Elasticsearch にインデックス
|
|
|
// 各ページを独立したドキュメントとして作成し、メタデータを保持
|
|
// 各ページを独立したドキュメントとして作成し、メタデータを保持
|
|
|
- this.indexPreciseResults(kb, userId, kb.embeddingModelId, result.results).catch((err) => {
|
|
|
|
|
|
|
+ this.indexPreciseResults(kb, userId, tenantId, kb.embeddingModelId, result.results).catch((err) => {
|
|
|
this.logger.error(`Error indexing precise results for ${kb.id}`, err);
|
|
this.logger.error(`Error indexing precise results for ${kb.id}`, err);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
// 非同期で PDF 変換をトリガー
|
|
// 非同期で PDF 変換をトリガー
|
|
|
- this.ensurePDFExists(kb.id, userId).catch((err) => {
|
|
|
|
|
|
|
+ this.ensurePDFExists(kb.id, userId, tenantId).catch((err) => {
|
|
|
this.logger.warn(`Initial PDF conversion failed for ${kb.id}`, err);
|
|
this.logger.warn(`Initial PDF conversion failed for ${kb.id}`, err);
|
|
|
});
|
|
});
|
|
|
|
|
|
|
@@ -466,7 +519,7 @@ export class KnowledgeBaseService {
|
|
|
|
|
|
|
|
} catch (error) {
|
|
} catch (error) {
|
|
|
this.logger.error(`Vision pipeline error: ${error.message}, falling back to fast mode`);
|
|
this.logger.error(`Vision pipeline error: ${error.message}, falling back to fast mode`);
|
|
|
- return this.processFastMode(kb, userId, config);
|
|
|
|
|
|
|
+ return this.processFastMode(kb, userId, tenantId, config);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -476,13 +529,14 @@ export class KnowledgeBaseService {
|
|
|
private async indexPreciseResults(
|
|
private async indexPreciseResults(
|
|
|
kb: KnowledgeBase,
|
|
kb: KnowledgeBase,
|
|
|
userId: string,
|
|
userId: string,
|
|
|
|
|
+ tenantId: string,
|
|
|
embeddingModelId: string,
|
|
embeddingModelId: string,
|
|
|
results: any[]
|
|
results: any[]
|
|
|
): Promise<void> {
|
|
): Promise<void> {
|
|
|
this.logger.log(`Indexing ${results.length} precise results for ${kb.id}`);
|
|
this.logger.log(`Indexing ${results.length} precise results for ${kb.id}`);
|
|
|
|
|
|
|
|
// インデックスの存在を確認 - 実際のモデル次元数を取得
|
|
// インデックスの存在を確認 - 実際のモデル次元数を取得
|
|
|
- const actualDimensions = await this.getActualModelDimensions(embeddingModelId, userId);
|
|
|
|
|
|
|
+ const actualDimensions = await this.getActualModelDimensions(embeddingModelId, userId, tenantId);
|
|
|
await this.elasticsearchService.createIndexIfNotExists(actualDimensions);
|
|
await this.elasticsearchService.createIndexIfNotExists(actualDimensions);
|
|
|
|
|
|
|
|
// ベクトル化とインデックスをバッチ処理
|
|
// ベクトル化とインデックスをバッチ処理
|
|
@@ -519,6 +573,7 @@ export class KnowledgeBaseService {
|
|
|
originalName: kb.originalName,
|
|
originalName: kb.originalName,
|
|
|
mimetype: kb.mimetype,
|
|
mimetype: kb.mimetype,
|
|
|
userId: userId,
|
|
userId: userId,
|
|
|
|
|
+ tenantId: tenantId, // New
|
|
|
pageNumber: result.pageIndex,
|
|
pageNumber: result.pageIndex,
|
|
|
images: result.images,
|
|
images: result.images,
|
|
|
layout: result.layout,
|
|
layout: result.layout,
|
|
@@ -542,8 +597,8 @@ export class KnowledgeBaseService {
|
|
|
/**
|
|
/**
|
|
|
* PDF の特定ページの画像を取得
|
|
* PDF の特定ページの画像を取得
|
|
|
*/
|
|
*/
|
|
|
- async getPageAsImage(fileId: string, pageIndex: number, userId: string): Promise<string> {
|
|
|
|
|
- const pdfPath = await this.ensurePDFExists(fileId, userId);
|
|
|
|
|
|
|
+ async getPageAsImage(fileId: string, pageIndex: number, userId: string, tenantId: string): Promise<string> {
|
|
|
|
|
+ const pdfPath = await this.ensurePDFExists(fileId, userId, tenantId);
|
|
|
|
|
|
|
|
// 特定のページを変換
|
|
// 特定のページを変換
|
|
|
const result = await this.pdf2ImageService.convertToImages(pdfPath, {
|
|
const result = await this.pdf2ImageService.convertToImages(pdfPath, {
|
|
@@ -564,11 +619,12 @@ export class KnowledgeBaseService {
|
|
|
private async vectorizeToElasticsearch(
|
|
private async vectorizeToElasticsearch(
|
|
|
kbId: string,
|
|
kbId: string,
|
|
|
userId: string,
|
|
userId: string,
|
|
|
|
|
+ tenantId: string,
|
|
|
text: string,
|
|
text: string,
|
|
|
config?: any,
|
|
config?: any,
|
|
|
) {
|
|
) {
|
|
|
try {
|
|
try {
|
|
|
- const kb = await this.kbRepository.findOne({ where: { id: kbId } });
|
|
|
|
|
|
|
+ const kb = await this.kbRepository.findOne({ where: { id: kbId, tenantId } });
|
|
|
if (!kb) return;
|
|
if (!kb) return;
|
|
|
|
|
|
|
|
// メモリ監視 - ベクトル化前チェック
|
|
// メモリ監視 - ベクトル化前チェック
|
|
@@ -643,6 +699,7 @@ export class KnowledgeBaseService {
|
|
|
const recommendedBatchSize = await this.chunkConfigService.getRecommendedBatchSize(
|
|
const recommendedBatchSize = await this.chunkConfigService.getRecommendedBatchSize(
|
|
|
kb.embeddingModelId,
|
|
kb.embeddingModelId,
|
|
|
userId,
|
|
userId,
|
|
|
|
|
+ tenantId,
|
|
|
parseInt(process.env.CHUNK_BATCH_SIZE || '100'),
|
|
parseInt(process.env.CHUNK_BATCH_SIZE || '100'),
|
|
|
);
|
|
);
|
|
|
|
|
|
|
@@ -656,7 +713,7 @@ export class KnowledgeBaseService {
|
|
|
this.logger.log(`推定メモリ使用量: ${estimatedMemory}MB (バッチサイズ: ${recommendedBatchSize})`);
|
|
this.logger.log(`推定メモリ使用量: ${estimatedMemory}MB (バッチサイズ: ${recommendedBatchSize})`);
|
|
|
|
|
|
|
|
// 6. 実際のモデル次元数を取得し、インデックスの存在を確認
|
|
// 6. 実際のモデル次元数を取得し、インデックスの存在を確認
|
|
|
- const actualDimensions = await this.getActualModelDimensions(kb.embeddingModelId, userId);
|
|
|
|
|
|
|
+ const actualDimensions = await this.getActualModelDimensions(kb.embeddingModelId, userId, tenantId);
|
|
|
await this.elasticsearchService.createIndexIfNotExists(actualDimensions);
|
|
await this.elasticsearchService.createIndexIfNotExists(actualDimensions);
|
|
|
|
|
|
|
|
// 7. ベクトル化とインデックス作成をバッチ処理
|
|
// 7. ベクトル化とインデックス作成をバッチ処理
|
|
@@ -713,8 +770,8 @@ export class KnowledgeBaseService {
|
|
|
userId: userId,
|
|
userId: userId,
|
|
|
chunkIndex: chunk.index,
|
|
chunkIndex: chunk.index,
|
|
|
startPosition: chunk.startPosition,
|
|
startPosition: chunk.startPosition,
|
|
|
- endPosition: chunk.endPosition,
|
|
|
|
|
- },
|
|
|
|
|
|
|
+ tenantId, // Passing tenantId to ES
|
|
|
|
|
+ }
|
|
|
);
|
|
);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -763,7 +820,8 @@ export class KnowledgeBaseService {
|
|
|
chunkIndex: chunk.index,
|
|
chunkIndex: chunk.index,
|
|
|
startPosition: chunk.startPosition,
|
|
startPosition: chunk.startPosition,
|
|
|
endPosition: chunk.endPosition,
|
|
endPosition: chunk.endPosition,
|
|
|
- },
|
|
|
|
|
|
|
+ tenantId,
|
|
|
|
|
+ }
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
if ((i + 1) % 10 === 0) {
|
|
if ((i + 1) % 10 === 0) {
|
|
@@ -824,7 +882,8 @@ export class KnowledgeBaseService {
|
|
|
chunkIndex: chunk.index,
|
|
chunkIndex: chunk.index,
|
|
|
startPosition: chunk.startPosition,
|
|
startPosition: chunk.startPosition,
|
|
|
endPosition: chunk.endPosition,
|
|
endPosition: chunk.endPosition,
|
|
|
- },
|
|
|
|
|
|
|
+ tenantId, // Passing tenantId to ES metadata
|
|
|
|
|
+ }
|
|
|
);
|
|
);
|
|
|
}
|
|
}
|
|
|
},
|
|
},
|
|
@@ -1064,12 +1123,12 @@ export class KnowledgeBaseService {
|
|
|
/**
|
|
/**
|
|
|
* 失敗したファイルのベクトル化を再試行
|
|
* 失敗したファイルのベクトル化を再試行
|
|
|
*/
|
|
*/
|
|
|
- async retryFailedFile(fileId: string, userId: string): Promise<KnowledgeBase> {
|
|
|
|
|
- this.logger.log(`Retrying failed file ${fileId} for user ${userId}`);
|
|
|
|
|
|
|
+ async retryFailedFile(fileId: string, userId: string, tenantId: string): Promise<KnowledgeBase> {
|
|
|
|
|
+ this.logger.log(`Retrying failed file ${fileId} for user ${userId} in tenant ${tenantId}`);
|
|
|
|
|
|
|
|
- // 1. Get file without user restriction (now allowing access to all files)
|
|
|
|
|
|
|
+ // 1. Get file with tenant restriction
|
|
|
const kb = await this.kbRepository.findOne({
|
|
const kb = await this.kbRepository.findOne({
|
|
|
- where: { id: fileId },
|
|
|
|
|
|
|
+ where: { id: fileId, tenantId },
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
if (!kb) {
|
|
if (!kb) {
|
|
@@ -1091,6 +1150,7 @@ export class KnowledgeBaseService {
|
|
|
this.vectorizeToElasticsearch(
|
|
this.vectorizeToElasticsearch(
|
|
|
fileId,
|
|
fileId,
|
|
|
userId,
|
|
userId,
|
|
|
|
|
+ tenantId,
|
|
|
kb.content,
|
|
kb.content,
|
|
|
{
|
|
{
|
|
|
chunkSize: kb.chunkSize,
|
|
chunkSize: kb.chunkSize,
|
|
@@ -1102,7 +1162,7 @@ export class KnowledgeBaseService {
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
// 4. 更新後のファイルステータスを返却
|
|
// 4. 更新後のファイルステータスを返却
|
|
|
- const updatedKb = await this.kbRepository.findOne({ where: { id: fileId } });
|
|
|
|
|
|
|
+ const updatedKb = await this.kbRepository.findOne({ where: { id: fileId, tenantId } });
|
|
|
if (!updatedKb) {
|
|
if (!updatedKb) {
|
|
|
throw new NotFoundException('ファイルが存在しません');
|
|
throw new NotFoundException('ファイルが存在しません');
|
|
|
}
|
|
}
|
|
@@ -1112,12 +1172,12 @@ export class KnowledgeBaseService {
|
|
|
/**
|
|
/**
|
|
|
* ファイルのすべてのチャンク情報を取得
|
|
* ファイルのすべてのチャンク情報を取得
|
|
|
*/
|
|
*/
|
|
|
- async getFileChunks(fileId: string, userId: string) {
|
|
|
|
|
- this.logger.log(`Getting chunks for file ${fileId}, user ${userId}`);
|
|
|
|
|
|
|
+ async getFileChunks(fileId: string, userId: string, tenantId: string) {
|
|
|
|
|
+ this.logger.log(`Getting chunks for file ${fileId}, user ${userId}, tenant ${tenantId}`);
|
|
|
|
|
|
|
|
- // 1. Get file without user ownership check (now allowing access to all files)
|
|
|
|
|
|
|
+ // 1. Get file with tenant check
|
|
|
const kb = await this.kbRepository.findOne({
|
|
const kb = await this.kbRepository.findOne({
|
|
|
- where: { id: fileId },
|
|
|
|
|
|
|
+ where: { id: fileId, tenantId },
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
if (!kb) {
|
|
if (!kb) {
|
|
@@ -1125,7 +1185,7 @@ export class KnowledgeBaseService {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// 2. Elasticsearch からすべてのチャンクを取得
|
|
// 2. Elasticsearch からすべてのチャンクを取得
|
|
|
- const chunks = await this.elasticsearchService.getFileChunks(fileId);
|
|
|
|
|
|
|
+ const chunks = await this.elasticsearchService.getFileChunks(fileId, userId, tenantId);
|
|
|
|
|
|
|
|
// 3. チャンク情報を返却
|
|
// 3. チャンク情報を返却
|
|
|
return {
|
|
return {
|
|
@@ -1149,9 +1209,9 @@ export class KnowledgeBaseService {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// PDF プレビュー関連メソッド
|
|
// PDF プレビュー関連メソッド
|
|
|
- async ensurePDFExists(fileId: string, userId: string, force: boolean = false): Promise<string> {
|
|
|
|
|
|
|
+ async ensurePDFExists(fileId: string, userId: string, tenantId: string, force: boolean = false): Promise<string> {
|
|
|
const kb = await this.kbRepository.findOne({
|
|
const kb = await this.kbRepository.findOne({
|
|
|
- where: { id: fileId },
|
|
|
|
|
|
|
+ where: { id: fileId, tenantId },
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
if (!kb) {
|
|
if (!kb) {
|
|
@@ -1225,9 +1285,9 @@ export class KnowledgeBaseService {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- async getPDFStatus(fileId: string, userId: string) {
|
|
|
|
|
|
|
+ async getPDFStatus(fileId: string, userId: string, tenantId: string) {
|
|
|
const kb = await this.kbRepository.findOne({
|
|
const kb = await this.kbRepository.findOne({
|
|
|
- where: { id: fileId },
|
|
|
|
|
|
|
+ where: { id: fileId, tenantId },
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
if (!kb) {
|
|
if (!kb) {
|
|
@@ -1236,7 +1296,7 @@ export class KnowledgeBaseService {
|
|
|
|
|
|
|
|
// 元ファイルが PDF の場合
|
|
// 元ファイルが PDF の場合
|
|
|
if (kb.mimetype === 'application/pdf') {
|
|
if (kb.mimetype === 'application/pdf') {
|
|
|
- const token = this.generateTempToken(fileId, userId);
|
|
|
|
|
|
|
+ const token = this.generateTempToken(fileId, userId, tenantId);
|
|
|
return {
|
|
return {
|
|
|
status: 'ready',
|
|
status: 'ready',
|
|
|
url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
|
|
url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
|
|
@@ -1256,7 +1316,7 @@ export class KnowledgeBaseService {
|
|
|
kb.pdfPath = pdfPath;
|
|
kb.pdfPath = pdfPath;
|
|
|
await this.kbRepository.save(kb);
|
|
await this.kbRepository.save(kb);
|
|
|
}
|
|
}
|
|
|
- const token = this.generateTempToken(fileId, userId);
|
|
|
|
|
|
|
+ const token = this.generateTempToken(fileId, userId, tenantId);
|
|
|
return {
|
|
return {
|
|
|
status: 'ready',
|
|
status: 'ready',
|
|
|
url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
|
|
url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
|
|
@@ -1269,7 +1329,7 @@ export class KnowledgeBaseService {
|
|
|
};
|
|
};
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- private generateTempToken(fileId: string, userId: string): string {
|
|
|
|
|
|
|
+ private generateTempToken(fileId: string, userId: string, tenantId: string): string {
|
|
|
const jwt = require('jsonwebtoken');
|
|
const jwt = require('jsonwebtoken');
|
|
|
|
|
|
|
|
const secret = process.env.JWT_SECRET;
|
|
const secret = process.env.JWT_SECRET;
|
|
@@ -1278,7 +1338,7 @@ export class KnowledgeBaseService {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
return jwt.sign(
|
|
return jwt.sign(
|
|
|
- { fileId, userId, type: 'pdf-access' },
|
|
|
|
|
|
|
+ { fileId, userId, tenantId, type: 'pdf-access' },
|
|
|
secret,
|
|
secret,
|
|
|
{ expiresIn: '1h' }
|
|
{ expiresIn: '1h' }
|
|
|
);
|
|
);
|
|
@@ -1287,7 +1347,7 @@ export class KnowledgeBaseService {
|
|
|
/**
|
|
/**
|
|
|
* モデルの実際の次元数を取得(キャッシュ確認とプローブロジック付き)
|
|
* モデルの実際の次元数を取得(キャッシュ確認とプローブロジック付き)
|
|
|
*/
|
|
*/
|
|
|
- private async getActualModelDimensions(embeddingModelId: string, userId: string): Promise<number> {
|
|
|
|
|
|
|
+ private async getActualModelDimensions(embeddingModelId: string, userId: string, tenantId: string): Promise<number> {
|
|
|
const defaultDimensions = parseInt(
|
|
const defaultDimensions = parseInt(
|
|
|
process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
|
|
process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
|
|
|
);
|
|
);
|
|
@@ -1297,6 +1357,7 @@ export class KnowledgeBaseService {
|
|
|
const modelConfig = await this.modelConfigService.findOne(
|
|
const modelConfig = await this.modelConfigService.findOne(
|
|
|
embeddingModelId,
|
|
embeddingModelId,
|
|
|
userId,
|
|
userId,
|
|
|
|
|
+ tenantId,
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
if (modelConfig && modelConfig.dimensions) {
|
|
if (modelConfig && modelConfig.dimensions) {
|
|
@@ -1319,7 +1380,7 @@ export class KnowledgeBaseService {
|
|
|
// 次回利用のためにモデル設定を更新
|
|
// 次回利用のためにモデル設定を更新
|
|
|
if (modelConfig) {
|
|
if (modelConfig) {
|
|
|
try {
|
|
try {
|
|
|
- await this.modelConfigService.update(modelConfig.id, userId, {
|
|
|
|
|
|
|
+ await this.modelConfigService.update(userId, tenantId, modelConfig.id, {
|
|
|
dimensions: actualDimensions,
|
|
dimensions: actualDimensions,
|
|
|
});
|
|
});
|
|
|
this.logger.log(`モデル ${modelConfig.name} の次元数設定を ${actualDimensions} に更新しました`);
|
|
this.logger.log(`モデル ${modelConfig.name} の次元数設定を ${actualDimensions} に更新しました`);
|
|
@@ -1351,6 +1412,7 @@ export class KnowledgeBaseService {
|
|
|
if (!kb || !kb.content || kb.content.trim().length === 0) {
|
|
if (!kb || !kb.content || kb.content.trim().length === 0) {
|
|
|
return null;
|
|
return null;
|
|
|
}
|
|
}
|
|
|
|
|
+ const tenantId = kb.tenantId;
|
|
|
|
|
|
|
|
// すでにタイトルがある場合はスキップ
|
|
// すでにタイトルがある場合はスキップ
|
|
|
if (kb.title) {
|
|
if (kb.title) {
|
|
@@ -1368,10 +1430,17 @@ export class KnowledgeBaseService {
|
|
|
const prompt = this.i18nService.getDocumentTitlePrompt(language, contentSample);
|
|
const prompt = this.i18nService.getDocumentTitlePrompt(language, contentSample);
|
|
|
|
|
|
|
|
// LLMを呼び出してタイトルを生成
|
|
// LLMを呼び出してタイトルを生成
|
|
|
- const generatedTitle = await this.chatService.generateSimpleChat(
|
|
|
|
|
- [{ role: 'user', content: prompt }],
|
|
|
|
|
- kb.userId
|
|
|
|
|
- );
|
|
|
|
|
|
|
+ let generatedTitle: string | undefined;
|
|
|
|
|
+ try {
|
|
|
|
|
+ generatedTitle = await this.chatService.generateSimpleChat(
|
|
|
|
|
+ [{ role: 'user', content: prompt }],
|
|
|
|
|
+ kb.userId,
|
|
|
|
|
+ kb.tenantId
|
|
|
|
|
+ );
|
|
|
|
|
+ } catch (err) {
|
|
|
|
|
+ this.logger.warn(`Failed to generate title for document ${kbId} due to LLM configuration issue: ${err.message}`);
|
|
|
|
|
+ return null; // Skip title generation if LLM is not configured for this tenant
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
if (generatedTitle && generatedTitle.trim().length > 0) {
|
|
if (generatedTitle && generatedTitle.trim().length > 0) {
|
|
|
// 余分な引用符や改行を除去
|
|
// 余分な引用符や改行を除去
|
|
@@ -1379,7 +1448,7 @@ export class KnowledgeBaseService {
|
|
|
await this.kbRepository.update(kbId, { title: cleanedTitle });
|
|
await this.kbRepository.update(kbId, { title: cleanedTitle });
|
|
|
|
|
|
|
|
// Elasticsearch のチャンクも更新
|
|
// Elasticsearch のチャンクも更新
|
|
|
- await this.elasticsearchService.updateTitleByFileId(kbId, cleanedTitle).catch((err) => {
|
|
|
|
|
|
|
+ await this.elasticsearchService.updateTitleByFileId(kbId, cleanedTitle, tenantId).catch((err) => {
|
|
|
this.logger.error(`Failed to update title in Elasticsearch for ${kbId}`, err);
|
|
this.logger.error(`Failed to update title in Elasticsearch for ${kbId}`, err);
|
|
|
});
|
|
});
|
|
|
|
|
|