|
@@ -20,18 +20,18 @@ import { I18nService } from '../i18n/i18n.service';
|
|
|
export class ChunkConfigService {
|
|
export class ChunkConfigService {
|
|
|
private readonly logger = new Logger(ChunkConfigService.name);
|
|
private readonly logger = new Logger(ChunkConfigService.name);
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
private readonly DEFAULTS = {
|
|
private readonly DEFAULTS = {
|
|
|
chunkSize: DEFAULT_CHUNK_SIZE,
|
|
chunkSize: DEFAULT_CHUNK_SIZE,
|
|
|
chunkOverlap: DEFAULT_CHUNK_OVERLAP,
|
|
chunkOverlap: DEFAULT_CHUNK_OVERLAP,
|
|
|
minChunkSize: MIN_CHUNK_SIZE,
|
|
minChunkSize: MIN_CHUNK_SIZE,
|
|
|
minChunkOverlap: MIN_CHUNK_OVERLAP,
|
|
minChunkOverlap: MIN_CHUNK_OVERLAP,
|
|
|
- maxOverlapRatio: DEFAULT_MAX_OVERLAP_RATIO,
|
|
|
|
|
- maxBatchSize: DEFAULT_MAX_BATCH_SIZE,
|
|
|
|
|
- expectedDimensions: DEFAULT_VECTOR_DIMENSIONS,
|
|
|
|
|
|
|
+ maxOverlapRatio: DEFAULT_MAX_OVERLAP_RATIO,
|
|
|
|
|
+ maxBatchSize: DEFAULT_MAX_BATCH_SIZE,
|
|
|
|
|
+ expectedDimensions: DEFAULT_VECTOR_DIMENSIONS,
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
private readonly envMaxChunkSize: number;
|
|
private readonly envMaxChunkSize: number;
|
|
|
private readonly envMaxOverlapSize: number;
|
|
private readonly envMaxOverlapSize: number;
|
|
|
|
|
|
|
@@ -42,7 +42,7 @@ export class ChunkConfigService {
|
|
|
private tenantService: TenantService,
|
|
private tenantService: TenantService,
|
|
|
private userSettingService: UserSettingService,
|
|
private userSettingService: UserSettingService,
|
|
|
) {
|
|
) {
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
this.envMaxChunkSize = parseInt(
|
|
this.envMaxChunkSize = parseInt(
|
|
|
this.configService.get<string>('MAX_CHUNK_SIZE', '8191')
|
|
this.configService.get<string>('MAX_CHUNK_SIZE', '8191')
|
|
|
);
|
|
);
|
|
@@ -55,7 +55,7 @@ export class ChunkConfigService {
|
|
|
);
|
|
);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
async getModelLimits(modelId: string, userId: string, tenantId?: string): Promise<{
|
|
async getModelLimits(modelId: string, userId: string, tenantId?: string): Promise<{
|
|
|
maxInputTokens: number;
|
|
maxInputTokens: number;
|
|
|
maxBatchSize: number;
|
|
maxBatchSize: number;
|
|
@@ -69,20 +69,20 @@ export class ChunkConfigService {
|
|
|
throw new BadRequestException(this.i18nService.formatMessage('embeddingModelNotFound', { id: modelId }));
|
|
throw new BadRequestException(this.i18nService.formatMessage('embeddingModelNotFound', { id: modelId }));
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
const maxInputTokens = modelConfig.maxInputTokens || this.envMaxChunkSize;
|
|
const maxInputTokens = modelConfig.maxInputTokens || this.envMaxChunkSize;
|
|
|
const maxBatchSize = modelConfig.maxBatchSize || this.DEFAULTS.maxBatchSize;
|
|
const maxBatchSize = modelConfig.maxBatchSize || this.DEFAULTS.maxBatchSize;
|
|
|
const expectedDimensions = modelConfig.dimensions || parseInt(this.configService.get('DEFAULT_VECTOR_DIMENSIONS', String(this.DEFAULTS.expectedDimensions)));
|
|
const expectedDimensions = modelConfig.dimensions || parseInt(this.configService.get('DEFAULT_VECTOR_DIMENSIONS', String(this.DEFAULTS.expectedDimensions)));
|
|
|
- const providerName = modelConfig.providerName || '不明';
|
|
|
|
|
|
|
+ const providerName = modelConfig.providerName || 'Unknown';
|
|
|
const isVectorModel = modelConfig.isVectorModel || false;
|
|
const isVectorModel = modelConfig.isVectorModel || false;
|
|
|
|
|
|
|
|
this.logger.log(
|
|
this.logger.log(
|
|
|
this.i18nService.formatMessage('configLoaded', { name: modelConfig.name, id: modelConfig.modelId }) + '\n' +
|
|
this.i18nService.formatMessage('configLoaded', { name: modelConfig.name, id: modelConfig.modelId }) + '\n' +
|
|
|
- ` - プロバイダー: ${providerName}\n` +
|
|
|
|
|
- ` - Token制限: ${maxInputTokens}\n` +
|
|
|
|
|
|
|
+ ` - Provider: ${providerName}\n` +
|
|
|
|
|
+ ` - Token Limit: ${maxInputTokens}\n` +
|
|
|
` - Batch Limit: ${maxBatchSize}\n` +
|
|
` - Batch Limit: ${maxBatchSize}\n` +
|
|
|
` - Vector Dimensions: ${expectedDimensions}\n` +
|
|
` - Vector Dimensions: ${expectedDimensions}\n` +
|
|
|
- ` - ベクトルモデルか: ${isVectorModel}`,
|
|
|
|
|
|
|
+ ` - Is Vector Model: ${isVectorModel}`,
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
return {
|
|
return {
|
|
@@ -94,7 +94,7 @@ export class ChunkConfigService {
|
|
|
};
|
|
};
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
async validateChunkConfig(
|
|
async validateChunkConfig(
|
|
|
chunkSize: number,
|
|
chunkSize: number,
|
|
|
chunkOverlap: number,
|
|
chunkOverlap: number,
|
|
@@ -111,7 +111,7 @@ export class ChunkConfigService {
|
|
|
const warnings: string[] = [];
|
|
const warnings: string[] = [];
|
|
|
const limits = await this.getModelLimits(modelId, userId, tenantId);
|
|
const limits = await this.getModelLimits(modelId, userId, tenantId);
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
const effectiveMaxChunkSize = Math.min(
|
|
const effectiveMaxChunkSize = Math.min(
|
|
|
this.envMaxChunkSize,
|
|
this.envMaxChunkSize,
|
|
|
limits.maxInputTokens,
|
|
limits.maxInputTokens,
|
|
@@ -122,7 +122,7 @@ export class ChunkConfigService {
|
|
|
Math.floor(effectiveMaxChunkSize * this.DEFAULTS.maxOverlapRatio),
|
|
Math.floor(effectiveMaxChunkSize * this.DEFAULTS.maxOverlapRatio),
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if (chunkSize > effectiveMaxChunkSize) {
|
|
if (chunkSize > effectiveMaxChunkSize) {
|
|
|
const reason =
|
|
const reason =
|
|
|
this.envMaxChunkSize < limits.maxInputTokens
|
|
this.envMaxChunkSize < limits.maxInputTokens
|
|
@@ -139,7 +139,7 @@ export class ChunkConfigService {
|
|
|
chunkSize = effectiveMaxChunkSize;
|
|
chunkSize = effectiveMaxChunkSize;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if (chunkSize < this.DEFAULTS.minChunkSize) {
|
|
if (chunkSize < this.DEFAULTS.minChunkSize) {
|
|
|
warnings.push(
|
|
warnings.push(
|
|
|
this.i18nService.formatMessage('chunkUnderflow', {
|
|
this.i18nService.formatMessage('chunkUnderflow', {
|
|
@@ -150,7 +150,7 @@ export class ChunkConfigService {
|
|
|
chunkSize = this.DEFAULTS.minChunkSize;
|
|
chunkSize = this.DEFAULTS.minChunkSize;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
if (chunkOverlap > effectiveMaxOverlapSize) {
|
|
if (chunkOverlap > effectiveMaxOverlapSize) {
|
|
|
warnings.push(
|
|
warnings.push(
|
|
|
this.i18nService.formatMessage('overlapOverflow', {
|
|
this.i18nService.formatMessage('overlapOverflow', {
|
|
@@ -161,7 +161,7 @@ export class ChunkConfigService {
|
|
|
chunkOverlap = effectiveMaxOverlapSize;
|
|
chunkOverlap = effectiveMaxOverlapSize;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
const maxOverlapByRatio = Math.floor(
|
|
const maxOverlapByRatio = Math.floor(
|
|
|
chunkSize * this.DEFAULTS.maxOverlapRatio,
|
|
chunkSize * this.DEFAULTS.maxOverlapRatio,
|
|
|
);
|
|
);
|
|
@@ -185,9 +185,9 @@ export class ChunkConfigService {
|
|
|
chunkOverlap = this.DEFAULTS.minChunkOverlap;
|
|
chunkOverlap = this.DEFAULTS.minChunkOverlap;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
- const safetyMargin = 0.8;
|
|
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ const safetyMargin = 0.8;
|
|
|
const safeChunkSize = Math.floor(effectiveMaxChunkSize * safetyMargin);
|
|
const safeChunkSize = Math.floor(effectiveMaxChunkSize * safetyMargin);
|
|
|
|
|
|
|
|
if (chunkSize > safeChunkSize) {
|
|
if (chunkSize > safeChunkSize) {
|
|
@@ -200,9 +200,9 @@ export class ChunkConfigService {
|
|
|
);
|
|
);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
const estimatedChunkCount = this.estimateChunkCount(
|
|
const estimatedChunkCount = this.estimateChunkCount(
|
|
|
- 1000000,
|
|
|
|
|
|
|
+ 1000000,
|
|
|
chunkSize,
|
|
chunkSize,
|
|
|
);
|
|
);
|
|
|
|
|
|
|
@@ -221,7 +221,7 @@ export class ChunkConfigService {
|
|
|
};
|
|
};
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
async getRecommendedBatchSize(
|
|
async getRecommendedBatchSize(
|
|
|
modelId: string,
|
|
modelId: string,
|
|
|
userId: string,
|
|
userId: string,
|
|
@@ -230,11 +230,11 @@ export class ChunkConfigService {
|
|
|
): Promise<number> {
|
|
): Promise<number> {
|
|
|
const limits = await this.getModelLimits(modelId, userId, tenantId);
|
|
const limits = await this.getModelLimits(modelId, userId, tenantId);
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
const recommended = Math.min(
|
|
const recommended = Math.min(
|
|
|
currentBatchSize,
|
|
currentBatchSize,
|
|
|
limits.maxBatchSize,
|
|
limits.maxBatchSize,
|
|
|
- 200,
|
|
|
|
|
|
|
+ 200,
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
if (recommended < currentBatchSize) {
|
|
if (recommended < currentBatchSize) {
|
|
@@ -247,16 +247,16 @@ export class ChunkConfigService {
|
|
|
);
|
|
);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- return Math.max(10, recommended);
|
|
|
|
|
|
|
+ return Math.max(10, recommended);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
estimateChunkCount(textLength: number, chunkSize: number): number {
|
|
estimateChunkCount(textLength: number, chunkSize: number): number {
|
|
|
const chunkSizeInChars = chunkSize * 4; // 1 token ≈ 4 chars
|
|
const chunkSizeInChars = chunkSize * 4; // 1 token ≈ 4 chars
|
|
|
return Math.ceil(textLength / chunkSizeInChars);
|
|
return Math.ceil(textLength / chunkSizeInChars);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
async validateDimensions(
|
|
async validateDimensions(
|
|
|
modelId: string,
|
|
modelId: string,
|
|
|
userId: string,
|
|
userId: string,
|
|
@@ -279,7 +279,7 @@ export class ChunkConfigService {
|
|
|
return true;
|
|
return true;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
async getConfigSummary(
|
|
async getConfigSummary(
|
|
|
chunkSize: number,
|
|
chunkSize: number,
|
|
|
chunkOverlap: number,
|
|
chunkOverlap: number,
|
|
@@ -291,14 +291,14 @@ export class ChunkConfigService {
|
|
|
|
|
|
|
|
return [
|
|
return [
|
|
|
`Model: ${modelId}`,
|
|
`Model: ${modelId}`,
|
|
|
- `Chunk size: ${chunkSize} tokens (制限: ${limits.maxInputTokens})`,
|
|
|
|
|
- `重なりサイズ: ${chunkOverlap} tokens`,
|
|
|
|
|
- `バッチサイズ: ${limits.maxBatchSize}`,
|
|
|
|
|
|
|
+ `Chunk size: ${chunkSize} tokens (Limit: ${limits.maxInputTokens})`,
|
|
|
|
|
+ `Overlap size: ${chunkOverlap} tokens`,
|
|
|
|
|
+ `Batch size: ${limits.maxBatchSize}`,
|
|
|
`Vector Dimensions: ${limits.expectedDimensions}`,
|
|
`Vector Dimensions: ${limits.expectedDimensions}`,
|
|
|
].join(', ');
|
|
].join(', ');
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
async getFrontendLimits(
|
|
async getFrontendLimits(
|
|
|
modelId: string,
|
|
modelId: string,
|
|
|
userId: string,
|
|
userId: string,
|
|
@@ -318,18 +318,18 @@ export class ChunkConfigService {
|
|
|
}> {
|
|
}> {
|
|
|
const limits = await this.getModelLimits(modelId, userId, tenantId);
|
|
const limits = await this.getModelLimits(modelId, userId, tenantId);
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
const maxChunkSize = Math.min(this.envMaxChunkSize, limits.maxInputTokens);
|
|
const maxChunkSize = Math.min(this.envMaxChunkSize, limits.maxInputTokens);
|
|
|
const maxOverlapSize = Math.min(
|
|
const maxOverlapSize = Math.min(
|
|
|
this.envMaxOverlapSize,
|
|
this.envMaxOverlapSize,
|
|
|
Math.floor(maxChunkSize * this.DEFAULTS.maxOverlapRatio),
|
|
Math.floor(maxChunkSize * this.DEFAULTS.maxOverlapRatio),
|
|
|
);
|
|
);
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
const modelConfig = await this.modelConfigService.findOne(modelId, userId, tenantId || '');
|
|
const modelConfig = await this.modelConfigService.findOne(modelId, userId, tenantId || '');
|
|
|
const modelName = modelConfig?.name || 'Unknown';
|
|
const modelName = modelConfig?.name || 'Unknown';
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
|
+
|
|
|
let defaultChunkSize = this.DEFAULTS.chunkSize;
|
|
let defaultChunkSize = this.DEFAULTS.chunkSize;
|
|
|
let defaultOverlapSize = this.DEFAULTS.chunkOverlap;
|
|
let defaultOverlapSize = this.DEFAULTS.chunkOverlap;
|
|
|
|
|
|