import { Injectable, Logger, NotFoundException, Inject, forwardRef } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { I18nService } from '../i18n/i18n.service';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository, In } from 'typeorm';
import { FileStatus, KnowledgeBase, ProcessingMode } from './knowledge-base.entity';
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
import { TikaService } from '../tika/tika.service';
import * as fs from 'fs';
import * as path from 'path';
import { EmbeddingService } from './embedding.service';
import { TextChunkerService } from './text-chunker.service';
import { ModelConfigService } from '../model-config/model-config.service';
import { RagService } from '../rag/rag.service';
import { VisionService } from '../vision/vision.service';
import { UserSettingService } from '../user-setting/user-setting.service';
import { MemoryMonitorService } from './memory-monitor.service';
import { ChunkConfigService } from './chunk-config.service';
import { VisionPipelineService } from '../vision-pipeline/vision-pipeline.service';
import { LibreOfficeService } from '../libreoffice/libreoffice.service';
import { Pdf2ImageService } from '../pdf2image/pdf2image.service';
import { DOC_EXTENSIONS, IMAGE_EXTENSIONS } from '../common/file-support.constants';
import { ChatService } from '../chat/chat.service';

export interface PaginatedKnowledgeBase {
  items: KnowledgeBase[];
  total: number;
  page: number;
  limit: number;
}

@Injectable()
export class KnowledgeBaseService {
  private readonly logger = new Logger(KnowledgeBaseService.name);

  constructor(
    @InjectRepository(KnowledgeBase)
    private kbRepository: Repository<KnowledgeBase>,
    @InjectRepository(KnowledgeGroup)
    private groupRepository: Repository<KnowledgeGroup>,
    @Inject(forwardRef(() => ElasticsearchService))
    private elasticsearchService: ElasticsearchService,
    private tikaService: TikaService,
    private embeddingService: EmbeddingService,
    private textChunkerService: TextChunkerService,
    private modelConfigService: ModelConfigService,
    @Inject(forwardRef(() => RagService))
    private ragService: RagService,
    private visionService: VisionService,
    private userSettingService: UserSettingService,
    private memoryMonitor: MemoryMonitorService,
    private chunkConfigService: ChunkConfigService,
    private visionPipelineService: VisionPipelineService,
    private libreOfficeService: LibreOfficeService,
    private pdf2ImageService: Pdf2ImageService,
    private configService: ConfigService,
    private i18nService: I18nService,
    @Inject(forwardRef(() => ChatService))
    private chatService: ChatService,
  ) { }

  async createAndIndex(
    fileInfo: any,
    userId: string,
    tenantId: string,
    config?: any,
  ): Promise<KnowledgeBase> {
    const mode = config?.mode || 'fast';
    const processingMode = mode === 'precise' ? ProcessingMode.PRECISE : ProcessingMode.FAST;

    const kb = this.kbRepository.create({
      originalName: fileInfo.originalname,
      storagePath: fileInfo.path,
      size: fileInfo.size,
      mimetype: fileInfo.mimetype,
      status: FileStatus.PENDING,
      userId: userId,
      tenantId: tenantId,
      chunkSize: config?.chunkSize || 200,
      chunkOverlap: config?.chunkOverlap || 40,
      embeddingModelId: config?.embeddingModelId || null,
      processingMode: processingMode,
    });

    
    if (config?.groupIds && config.groupIds.length > 0) {
      const groups = await this.groupRepository.find({
        where: { id: In(config.groupIds), tenantId: tenantId }
      });
      kb.groups = groups;
    }

    const savedKb = await this.kbRepository.save(kb);

    this.logger.log(
      `Created KB record: ${savedKb.id}, mode: ${mode}, file: ${fileInfo.originalname}`
    );

    // ---------------------------------------------------------
    // Move the file to the final partitioned directory
    // source: uploads/{tenantId}/{filename} (or wherever it was)
    // target: uploads/{tenantId}/{savedKb.id}/{filename}
    // ---------------------------------------------------------
    const fs = await import('fs');
    const path = await import('path');
    const uploadPath = process.env.UPLOAD_FILE_PATH || './uploads';
    const targetDir = path.join(uploadPath, tenantId || 'default', savedKb.id);
    const targetPath = path.join(targetDir, fileInfo.filename);

    try {
      if (!fs.existsSync(targetDir)) {
        fs.mkdirSync(targetDir, { recursive: true });
      }
      if (fs.existsSync(fileInfo.path)) {
        fs.renameSync(fileInfo.path, targetPath);
        // Update the DB record with the new path
        savedKb.storagePath = targetPath;
        await this.kbRepository.save(savedKb);
        this.logger.log(`Moved file to partitioned storage: ${targetPath}`);
      }
    } catch (fsError) {
      this.logger.error(`Failed to move file ${savedKb.id} to partitioned storage`, fsError);
      // We will let it continue, but the file might be stuck in the temp/root folder
    }

    // If queue processing is requested, await completion
    if (config?.waitForCompletion) {
      await this.processFile(savedKb.id, userId, tenantId, config);
    } else {
      // Otherwise trigger asynchronously (default)
      this.processFile(savedKb.id, userId, tenantId, config).catch((err) => {
        this.logger.error(`Error processing file ${savedKb.id}`, err);
      });
    }

    return savedKb;
  }

  async findAll(
    userId: string,
    tenantId: string,
    options: {
      page?: number;
      limit?: number;
      name?: string;
      status?: string;
      groupId?: string;
    } = {},
  ): Promise<PaginatedKnowledgeBase> {
    const { page = 1, limit = 12, name, status, groupId } = options;
    const skip = (page - 1) * limit;

    const queryBuilder = this.kbRepository.createQueryBuilder('kb')
      .leftJoinAndSelect('kb.groups', 'groups')
      .where('kb.tenantId = :tenantId', { tenantId });

    if (name) {
      queryBuilder.andWhere('kb.originalName LIKE :name', { name: `%${name}%` });
    }

    if (status && status !== 'all') {
      if (status === 'ready') {
        queryBuilder.andWhere('kb.status IN (:...statuses)', { statuses: [FileStatus.VECTORIZED] });
      } else if (status === 'indexing') {
        queryBuilder.andWhere('kb.status IN (:...statuses)', { statuses: [FileStatus.INDEXING, FileStatus.PENDING, FileStatus.EXTRACTED] });
      } else if (status === 'failed') {
        queryBuilder.andWhere('kb.status IN (:...statuses)', { statuses: [FileStatus.FAILED] });
      }
    }

    if (groupId) {
      // Find files that belong to this group or its descendants
      const group = await this.groupRepository.findOne({ where: { id: groupId, tenantId } });
      if (group) {
        const groupIds = [groupId];
        const collectChildren = async (parentId: string) => {
          const children = await this.groupRepository.find({ where: { parentId, tenantId } });
          for (const child of children) {
            groupIds.push(child.id);
            await collectChildren(child.id);
          }
        };
        await collectChildren(groupId);

        queryBuilder.innerJoin('kb.groups', 'filterGroup', 'filterGroup.id IN (:...groupIds)', { groupIds });
      }
    }

    const [items, total] = await queryBuilder
      .orderBy('kb.createdAt', 'DESC')
      .skip(skip)
      .take(limit)
      .getManyAndCount();

    return {
      items,
      total,
      page,
      limit,
    };
  }

  async getStatuses(ids: string[], userId: string, tenantId: string): Promise<any[]> {
    if (!ids || ids.length === 0) return [];

    return this.kbRepository.find({
      select: ['id', 'status', 'updatedAt'],
      where: {
        id: In(ids),
        tenantId
      }
    });
  }

  async getStats(userId: string, tenantId: string): Promise<{ total: number, uncategorized: number }> {
    const total = await this.kbRepository.count({ where: { tenantId } });

    // Count uncategorized files by finding files with NO groups
    // Using query builder for more complex relation queries
    const uncategorized = await this.kbRepository.createQueryBuilder('kb')
      .leftJoin('kb.groups', 'groups')
      .where('kb.tenantId = :tenantId', { tenantId })
      .andWhere('groups.id IS NULL')
      .getCount();

    return {
      total,
      uncategorized
    };
  }

  async searchKnowledge(userId: string, tenantId: string, query: string, topK: number = 5) {
    try {
      
      const defaultDimensions = parseInt(
        process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
      );
      const mockEmbedding = Array.from(
        { length: defaultDimensions },
        () => Math.random() - 0.5,
      );
      const queryVector = mockEmbedding;

      // 2. Search in Elasticsearch
      const searchResults = await this.elasticsearchService.searchSimilar(
        queryVector,
        userId,
        topK,
        tenantId, // Ensure shared visibility within tenant
      );

      // 3. Get file information from database
      const fileIds = [...new Set(searchResults.map((r) => r.fileId))];
      const files = await this.kbRepository.findByIds(fileIds);
      const fileMap = new Map(files.map((f) => [f.id, f]));

      // 4. Combine results with file info
      const results = searchResults.map((result) => {
        const file = fileMap.get(result.fileId);
        return {
          ...result,
          file: file
            ? {
              id: file.id,
              name: file.originalName,
              mimetype: file.mimetype,
              size: file.size,
              createdAt: file.createdAt,
            }
            : null,
        };
      });

      return {
        query,
        results,
        total: results.length,
      };
    } catch (error) {
      this.logger.error(
        `Metadata search failed for tenant ${tenantId}:`,
        error.stack || error.message,
      );
      throw error;
    }
  }

  async ragSearch(userId: string, tenantId: string, query: string, settings: any) {
    this.logger.log(
      `RAG search request: userId=${userId}, query="${query}", settings=${JSON.stringify(settings)}`,
    );

    try {
      const ragResults = await this.ragService.searchKnowledge(
        query,
        userId,
        settings.topK,
        settings.similarityThreshold,
        settings.selectedEmbeddingId,
        settings.enableFullTextSearch,
        settings.enableRerank,
        settings.selectedRerankId,
        undefined,
        undefined,
        settings.rerankSimilarityThreshold,
        tenantId, // Ensure shared visibility within tenant for RAG
      );

      const sources = this.ragService.extractSources(ragResults);
      const ragPrompt = this.ragService.buildRagPrompt(
        query,
        ragResults,
        settings.language || 'ja',
      );

      const result = {
        searchResults: ragResults,
        sources,
        ragPrompt,
        hasRelevantContent: ragResults.length > 0,
      };

      this.logger.log(
        `RAG search completed: found ${ragResults.length} results`,
      );
      return result;
    } catch (error) {
      this.logger.error(
        `RAG search failed for user ${userId}:`,
        error.stack || error.message,
      );
      
      return {
        searchResults: [],
        sources: [],
        ragPrompt: query, 
        hasRelevantContent: false,
      };
    }
  }

  async deleteFile(fileId: string, userId: string, tenantId: string): Promise<void> {
    this.logger.log(`Deleting file ${fileId} for user ${userId}`);

    try {
      // 1. Get file info
      const file = await this.kbRepository.findOne({
        where: { id: fileId, tenantId }, // Filter by tenantId
      });
      if (!file) {
        throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
      }

      // 2. Delete file from filesystem
      const fs = await import('fs');
      try {
        if (fs.existsSync(file.storagePath)) {
          fs.unlinkSync(file.storagePath);
          this.logger.log(`Deleted file: ${file.storagePath}`);
        }
      } catch (error) {
        this.logger.warn(`Failed to delete file ${file.storagePath}:`, error);
      }

      // 3. Delete from Elasticsearch
      try {
        await this.elasticsearchService.deleteByFileId(fileId, userId, tenantId);
        this.logger.log(`Deleted ES documents for file ${fileId}`);
      } catch (error) {
        this.logger.warn(
          `Failed to delete ES documents for file ${fileId}:`,
          error,
        );
      }

      // 4. Remove from all groups (cleanup M2M relations)
      const fileWithGroups = await this.kbRepository.findOne({
        where: { id: fileId, tenantId },
        relations: ['groups'],
      });

      if (fileWithGroups && fileWithGroups.groups && fileWithGroups.groups.length > 0) {
        // Clear groups to remove entries from join table
        fileWithGroups.groups = [];
        await this.kbRepository.save(fileWithGroups);
        this.logger.log(`Cleared group associations for file ${fileId}`);
      }

      // 5. Delete from SQLite
      await this.kbRepository.delete({ id: fileId });
      this.logger.log(`Deleted database record for file ${fileId}`);
    } catch (error) {
      this.logger.error(`Failed to delete file ${fileId}`, error);
      throw error;
    }

  }

  async clearAll(userId: string, tenantId: string): Promise<void> {
    this.logger.log(`Clearing all knowledge base data for user ${userId} in tenant ${tenantId}`);

    try {
      // Get all files and delete them one by one
      const files = await this.kbRepository.find();

      for (const file of files) {
        await this.deleteFile(file.id, userId, tenantId);
      }

      this.logger.log(`Cleared all knowledge base data for user ${userId}`);
    } catch (error) {
      this.logger.error(
        `Failed to clear knowledge base for user ${userId}`,
        error,
      );
      throw error;
    }
  }

  private async processFile(kbId: string, userId: string, tenantId: string, config?: any) {
    this.logger.log(`Starting processing for file ${kbId}, mode: ${config?.mode || 'fast'}`);
    await this.updateStatus(kbId, FileStatus.INDEXING);

    try {
      const kb = await this.kbRepository.findOne({ where: { id: kbId } });
      if (!kb) {
        this.logger.error(`KB not found: ${kbId}`);
        return;
      }

      
      const memBefore = this.memoryMonitor.getMemoryUsage();
      this.logger.log(`Memory state - Before processing: ${memBefore.heapUsed}/${memBefore.heapTotal}MB`);

      
      const mode = config?.mode || 'fast';

      if (mode === 'precise') {
        
        await this.processPreciseMode(kb, userId, tenantId, config);
      } else {
        
        await this.processFastMode(kb, userId, tenantId, config);
      }

      this.logger.log(`File ${kbId} processed successfully in ${mode} mode.`);
    } catch (error) {
      this.logger.error(`Failed to process file ${kbId}`, error);
      await this.updateStatus(kbId, FileStatus.FAILED);
    }
  }

  
  private async processFastMode(kb: KnowledgeBase, userId: string, tenantId: string, config?: any) {
    
    let text = await this.tikaService.extractText(kb.storagePath);

    
    if (this.visionService.isImageFile(kb.mimetype)) {
      const visionModelId = await this.userSettingService.getVisionModelId(userId);
      if (visionModelId) {
        const visionModel = await this.modelConfigService.findOne(
          visionModelId,
          userId,
          tenantId,
        );
        if (visionModel && visionModel.type === 'vision' && visionModel.isEnabled !== false) {
          text = await this.visionService.extractImageContent(kb.storagePath, {
            baseUrl: visionModel.baseUrl || '',
            apiKey: visionModel.apiKey || '',
            modelId: visionModel.modelId,
          });
        }
      }
    }

    if (!text || text.trim().length === 0) {
      this.logger.warn(this.i18nService.getMessage('noTextExtracted'));
    }

    
    const textSizeMB = Math.round(text.length / 1024 / 1024);
    if (textSizeMB > 50) {
      this.logger.warn(this.i18nService.formatMessage('extractedTextTooLarge', { size: textSizeMB }));
    }

    
    await this.kbRepository.update(kb.id, { content: text });
    await this.updateStatus(kb.id, FileStatus.EXTRACTED);

    
    await this.vectorizeToElasticsearch(kb.id, userId, tenantId, text, config).catch((err) => {
      this.logger.error(`Error vectorizing file ${kb.id}`, err);
    });

    
    this.generateTitle(kb.id).catch((err) => {
      this.logger.error(`Error generating title for file ${kb.id}`, err);
    });

    
    this.ensurePDFExists(kb.id, userId, tenantId).catch((err) => {
      this.logger.warn(this.i18nService.formatMessage('pdfConversionFailedDetail', { id: kb.id }), err);
    });
  }

  
  private async processPreciseMode(kb: KnowledgeBase, userId: string, tenantId: string, config?: any) {
    
    const preciseFormats = ['.pdf', '.doc', '.docx', '.ppt', '.pptx'];
    const ext = kb.originalName.toLowerCase().substring(kb.originalName.lastIndexOf('.'));

    if (!preciseFormats.includes(ext)) {
      this.logger.warn(
        this.i18nService.formatMessage('preciseModeUnsupported', { ext })
      );
      return this.processFastMode(kb, userId, tenantId, config);
    }

    
    const visionModelId = await this.userSettingService.getVisionModelId(userId);
    if (!visionModelId) {
      this.logger.warn(
        this.i18nService.getMessage('visionModelNotConfiguredFallback')
      );
      return this.processFastMode(kb, userId, tenantId, config);
    }

    const visionModel = await this.modelConfigService.findOne(
      visionModelId,
      userId,
      tenantId,
    );
    if (!visionModel || visionModel.type !== 'vision' || visionModel.isEnabled === false) {
      this.logger.warn(
        this.i18nService.getMessage('visionModelInvalidFallback')
      );
      return this.processFastMode(kb, userId, tenantId, config);
    }

    
    try {
      const result = await this.visionPipelineService.processPreciseMode(
        kb.storagePath,
        {
          userId,
          tenantId, // New
          modelId: visionModelId,
          fileId: kb.id,
          fileName: kb.originalName,
          skipQualityCheck: false,
        }
      );

      if (!result.success) {
        this.logger.error(`Vision pipeline failed, falling back to fast mode`);
        this.logger.warn(this.i18nService.getMessage('visionPipelineFailed'));
        return this.processFastMode(kb, userId, tenantId, config);
      }

      
      const combinedText = result.results.map(r => r.text).join('\n\n');
      const metadata = {
        processedPages: result.processedPages,
        failedPages: result.failedPages,
        cost: result.cost,
        duration: result.duration,
        results: result.results.map(r => ({
          pageIndex: r.pageIndex,
          confidence: r.confidence,
          layout: r.layout,
          imageCount: r.images.length,
        })),
      };
      await this.kbRepository.update(kb.id, {
        content: combinedText,
        metadata: metadata as any,
      });

      await this.updateStatus(kb.id, FileStatus.EXTRACTED);
      this.logger.log(
        this.i18nService.formatMessage('preciseModeComplete', { pages: result.processedPages, cost: result.cost.toFixed(2) })
      );

      
      
      this.indexPreciseResults(kb, userId, tenantId, kb.embeddingModelId, result.results).catch((err) => {
        this.logger.error(`Error indexing precise results for ${kb.id}`, err);
      });

      
      this.ensurePDFExists(kb.id, userId, tenantId).catch((err) => {
        this.logger.warn(`Initial PDF conversion failed for ${kb.id}`, err);
      });

      
      this.generateTitle(kb.id).catch((err) => {
        this.logger.error(`Error generating title for file ${kb.id}`, err);
      });

    } catch (error) {
      this.logger.error(`Vision pipeline error: ${error.message}, falling back to fast mode`);
      return this.processFastMode(kb, userId, tenantId, config);
    }
  }

  
  private async indexPreciseResults(
    kb: KnowledgeBase,
    userId: string,
    tenantId: string,
    embeddingModelId: string,
    results: any[]
  ): Promise<void> {
    this.logger.log(`Indexing ${results.length} precise results for ${kb.id}`);

    
    const actualDimensions = await this.getActualModelDimensions(embeddingModelId, userId, tenantId);
    await this.elasticsearchService.createIndexIfNotExists(actualDimensions);

    
    const batchSize = parseInt(process.env.CHUNK_BATCH_SIZE || '50');

    for (let i = 0; i < results.length; i += batchSize) {
      const batch = results.slice(i, i + batchSize);
      const texts = batch.map(r => r.text);

      try {
        
        const embeddings = await this.embeddingService.getEmbeddings(
          texts,
          userId,
          embeddingModelId
        );

        
        for (let j = 0; j < batch.length; j++) {
          const result = batch[j];
          const embedding = embeddings[j];

          if (!embedding || embedding.length === 0) {
            this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorPage', { page: result.pageIndex }));
            continue;
          }

          await this.elasticsearchService.indexDocument(
            `${kb.id}_page_${result.pageIndex}`,
            result.text,
            embedding,
            {
              fileId: kb.id,
              originalName: kb.originalName,
              mimetype: kb.mimetype,
              userId: userId,
              tenantId: tenantId, // New
              pageNumber: result.pageIndex,
              images: result.images,
              layout: result.layout,
              confidence: result.confidence,
              source: 'precise',
              mode: 'vision',
            }
          );
        }

        this.logger.log(`Batch ${Math.floor(i / batchSize) + 1} complete: ${batch.length} pages`);
      } catch (error) {
        this.logger.error(`Processing of batch ${Math.floor(i / batchSize) + 1} failed`, error);
      }
    }

    await this.updateStatus(kb.id, FileStatus.VECTORIZED);
    this.logger.log(`Precise mode indexing complete: ${results.length} pages`);
  }

  
  async getPageAsImage(fileId: string, pageIndex: number, userId: string, tenantId: string): Promise<string> {
    const pdfPath = await this.ensurePDFExists(fileId, userId, tenantId);

    
    const result = await this.pdf2ImageService.convertToImages(pdfPath, {
      density: 150,
      quality: 75,
      format: 'jpeg',
    });

    
    const pageImage = result.images.find(img => img.pageIndex === pageIndex + 1);
    if (!pageImage) {
      throw new NotFoundException(this.i18nService.formatMessage('pageImageNotFoundDetail', { page: pageIndex + 1 }));
    }

    return pageImage.path;
  }

  private async vectorizeToElasticsearch(
    kbId: string,
    userId: string,
    tenantId: string,
    text: string,
    config?: any,
  ) {
    try {
      const kb = await this.kbRepository.findOne({ where: { id: kbId, tenantId } });
      if (!kb) return;

      
      const memBeforeChunk = this.memoryMonitor.getMemoryUsage();
      this.logger.log(
        `Memory before vectorization: ${memBeforeChunk.heapUsed}/${memBeforeChunk.heapTotal}MB`,
      );

      this.logger.debug(`File ${kbId}: Validating chunk config...`);
      
      const validatedConfig = await this.chunkConfigService.validateChunkConfig(
        kb.chunkSize,
        kb.chunkOverlap,
        kb.embeddingModelId,
        userId,
      );
      this.logger.debug(`File ${kbId}: Chunk config validated.`);

      
      if (validatedConfig.warnings.length > 0) {
        this.logger.warn(
          this.i18nService.formatMessage('chunkConfigCorrection', { warnings: validatedConfig.warnings.join(', ') })
        );

        
        if (validatedConfig.chunkSize !== kb.chunkSize ||
          validatedConfig.chunkOverlap !== kb.chunkOverlap) {
          await this.kbRepository.update(kbId, {
            chunkSize: validatedConfig.chunkSize,
            chunkOverlap: validatedConfig.chunkOverlap,
          });
        }
      }

      
      this.logger.debug(`File ${kbId}: Getting config summary...`);
      const configSummary = await this.chunkConfigService.getConfigSummary(
        validatedConfig.chunkSize,
        validatedConfig.chunkOverlap,
        kb.embeddingModelId,
        userId,
      );
      this.logger.log(`Chunk configuration: ${configSummary}`);
      this.logger.log(`Configuration limits: Chunk=${validatedConfig.effectiveMaxChunkSize}, Overlap=${validatedConfig.effectiveMaxOverlapSize}`);

      
      const chunks = this.textChunkerService.chunkText(
        text,
        validatedConfig.chunkSize,
        validatedConfig.chunkOverlap,
      );
      this.logger.log(`Split ${chunks.length} text blocks from file ${kbId}`);

      if (chunks.length === 0) {
        this.logger.warn(this.i18nService.formatMessage('noChunksGenerated', { id: kbId }));
        await this.updateStatus(kbId, FileStatus.VECTORIZED);
        return;
      }

      
      const estimatedChunkCount = this.chunkConfigService.estimateChunkCount(
        text.length,
        validatedConfig.chunkSize,
      );
      if (chunks.length > estimatedChunkCount * 1.2) {
        this.logger.warn(
          this.i18nService.formatMessage('chunkCountAnomaly', { actual: chunks.length, estimated: estimatedChunkCount })
        );
      }

      
      const recommendedBatchSize = await this.chunkConfigService.getRecommendedBatchSize(
        kb.embeddingModelId,
        userId,
        tenantId,
        parseInt(process.env.CHUNK_BATCH_SIZE || '100'),
      );

      
      const avgChunkSize = chunks.reduce((sum, c) => sum + c.content.length, 0) / chunks.length;
      const estimatedMemory = this.memoryMonitor.estimateMemoryUsage(
        chunks.length,
        avgChunkSize,
        parseInt(process.env.DEFAULT_VECTOR_DIMENSIONS || '2560'),
      );
      this.logger.log(`Estimated memory usage: ${estimatedMemory}MB (Batch Size: ${recommendedBatchSize})`);

      
      const actualDimensions = await this.getActualModelDimensions(kb.embeddingModelId, userId, tenantId);
      await this.elasticsearchService.createIndexIfNotExists(actualDimensions);

      
      const useBatching = this.memoryMonitor.shouldUseBatching(
        chunks.length,
        avgChunkSize,
        actualDimensions,
      );

      if (useBatching) {
        try {
          await this.processInBatches(
            chunks,
            async (batch, batchIndex) => {
              
              if (batch.length > recommendedBatchSize) {
                this.logger.warn(
                  this.i18nService.formatMessage('batchSizeExceeded', { index: batchIndex, actual: batch.length, limit: recommendedBatchSize })
                );
              }

              const chunkTexts = batch.map((chunk) => chunk.content);
              const embeddings = await this.embeddingService.getEmbeddings(
                chunkTexts,
                userId,
                kb.embeddingModelId,
              );

              
              if (embeddings.length > 0 && embeddings[0].length !== actualDimensions) {
                this.logger.warn(
                  `Vector dimension mismatch: Expected ${actualDimensions}, Actual ${embeddings[0].length}`
                );
              }

              
              for (let i = 0; i < batch.length; i++) {
                const chunk = batch[i];
                const embedding = embeddings[i];

                if (!embedding || embedding.length === 0) {
                  this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorChunk', { index: chunk.index }));
                  continue;
                }

                await this.elasticsearchService.indexDocument(
                  `${kb.id}_chunk_${chunk.index}`,
                  chunk.content,
                  embedding,
                  {
                    fileId: kb.id,
                    originalName: kb.originalName,
                    mimetype: kb.mimetype,
                    userId: userId,
                    chunkIndex: chunk.index,
                    startPosition: chunk.startPosition,
                    tenantId, // Passing tenantId to ES
                  }
                );
              }

              this.logger.log(`Batch ${batchIndex} complete: ${batch.length} chunks`);
            },
            {
              batchSize: recommendedBatchSize,
              onBatchComplete: (batchIndex, totalBatches) => {
                const mem = this.memoryMonitor.getMemoryUsage();
                this.logger.log(
                  `Batch ${batchIndex}/${totalBatches} complete, Memory: ${mem.heapUsed}MB`,
                );
              },
            },
          );
        } catch (error) {
          
          if (error.message && (error.message.includes('context length') || error.message.includes('context length exceeds limit ') || error.message.includes('context length exceeds limit '))) {
            this.logger.warn(this.i18nService.getMessage('contextLengthErrorFallback'));

            
            for (let i = 0; i < chunks.length; i++) {
              const chunk = chunks[i];

              try {
                const embeddings = await this.embeddingService.getEmbeddings(
                  [chunk.content], 
                  userId,
                  kb.embeddingModelId,
                );

                if (!embeddings[0] || embeddings[0].length === 0) {
                  this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorChunk', { index: chunk.index }));
                  continue;
                }

                await this.elasticsearchService.indexDocument(
                  `${kb.id}_chunk_${chunk.index}`,
                  chunk.content,
                  embeddings[0],
                  {
                    fileId: kb.id,
                    originalName: kb.originalName,
                    mimetype: kb.mimetype,
                    userId: userId,
                    chunkIndex: chunk.index,
                    startPosition: chunk.startPosition,
                    endPosition: chunk.endPosition,
                    tenantId,
                  }
                );

                if ((i + 1) % 10 === 0) {
                  this.logger.log(`Single processing progress: ${i + 1}/${chunks.length}`);
                }
              } catch (chunkError) {
                this.logger.error(
                  `Failed to process text block ${chunk.index}. Skipping: ${chunkError.message}`
                );
                continue;
              }
            }

            this.logger.log(`Single text processing complete: ${chunks.length} chunks`);
          } else {
            
            throw error;
          }
        }
      } else {
        
        const chunkTexts = chunks.map((chunk) => chunk.content);

        
        if (chunks.length > recommendedBatchSize) {
          this.logger.warn(
            this.i18nService.formatMessage('chunkLimitExceededForceBatch', { actual: chunks.length, limit: recommendedBatchSize })
          );
          try {
            await this.processInBatches(
              chunks,
              async (batch, batchIndex) => {
                const batchTexts = batch.map((c) => c.content);
                const embeddings = await this.embeddingService.getEmbeddings(
                  batchTexts,
                  userId,
                  kb.embeddingModelId,
                );

                for (let i = 0; i < batch.length; i++) {
                  const chunk = batch[i];
                  const embedding = embeddings[i];

                  if (!embedding || embedding.length === 0) {
                    this.logger.warn(`Skipping text block ${chunk.index} with empty vector`);
                    continue;
                  }

                  await this.elasticsearchService.indexDocument(
                    `${kb.id}_chunk_${chunk.index}`,
                    chunk.content,
                    embedding,
                    {
                      fileId: kb.id,
                      originalName: kb.originalName,
                      mimetype: kb.mimetype,
                      userId: userId,
                      chunkIndex: chunk.index,
                      startPosition: chunk.startPosition,
                      endPosition: chunk.endPosition,
                      tenantId, // Passing tenantId to ES metadata
                    }
                  );
                }
              },
            );
          } catch (error) {
            
            if (error.message && (error.message.includes('context length') || error.message.includes('context length exceeds limit ') || error.message.includes('context length exceeds limit '))) {
              this.logger.warn(this.i18nService.getMessage('batchContextLengthErrorFallback'));

              
              for (let i = 0; i < chunks.length; i++) {
                const chunk = chunks[i];

                try {
                  const embeddings = await this.embeddingService.getEmbeddings(
                    [chunk.content], 
                    userId,
                    kb.embeddingModelId,
                  );

                  if (!embeddings[0] || embeddings[0].length === 0) {
                    this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorChunk', { index: chunk.index }));
                    continue;
                  }

                  await this.elasticsearchService.indexDocument(
                    `${kb.id}_chunk_${chunk.index}`,
                    chunk.content,
                    embeddings[0],
                    {
                      fileId: kb.id,
                      originalName: kb.originalName,
                      mimetype: kb.mimetype,
                      userId: userId,
                      tenantId, // Added tenantId
                      chunkIndex: chunk.index,
                      startPosition: chunk.startPosition,
                      endPosition: chunk.endPosition,
                    },
                  );

                  if ((i + 1) % 10 === 0) {
                    this.logger.log(`Single processing progress: ${i + 1}/${chunks.length}`);
                  }
                } catch (chunkError) {
                  this.logger.error(
                    this.i18nService.formatMessage('chunkProcessingFailed', { index: chunk.index, message: chunkError.message })
                  );
                  continue;
                }
              }

              this.logger.log(this.i18nService.formatMessage('singleTextProcessingComplete', { count: chunks.length }));
            } else {
              
              throw error;
            }
          }
        } else {
          
          try {
            const embeddings = await this.embeddingService.getEmbeddings(
              chunkTexts,
              userId,
              kb.embeddingModelId,
            );

            for (let i = 0; i < chunks.length; i++) {
              const chunk = chunks[i];
              const embedding = embeddings[i];

              if (!embedding || embedding.length === 0) {
                this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorChunk', { index: chunk.index }));
                continue;
              }

              await this.elasticsearchService.indexDocument(
                `${kb.id}_chunk_${chunk.index}`,
                chunk.content,
                embedding,
                {
                  fileId: kb.id,
                  originalName: kb.originalName,
                  mimetype: kb.mimetype,
                  userId: userId,
                  tenantId, // Added tenantId
                  chunkIndex: chunk.index,
                  startPosition: chunk.startPosition,
                  endPosition: chunk.endPosition,
                },
              );
            }
          } catch (error) {
            
            if (error.message && (error.message.includes('context length') || error.message.includes('context length exceeds limit ') || error.message.includes('context length exceeds limit '))) {
              this.logger.warn(this.i18nService.getMessage('batchContextLengthErrorFallback'));

              
              for (let i = 0; i < chunks.length; i++) {
                const chunk = chunks[i];

                try {
                  const embeddings = await this.embeddingService.getEmbeddings(
                    [chunk.content], 
                    userId,
                    kb.embeddingModelId,
                  );

                  if (!embeddings[0] || embeddings[0].length === 0) {
                    this.logger.warn(`Skipping text block ${chunk.index} with empty vector`);
                    continue;
                  }

                  await this.elasticsearchService.indexDocument(
                    `${kb.id}_chunk_${chunk.index}`,
                    chunk.content,
                    embeddings[0],
                    {
                      fileId: kb.id,
                      originalName: kb.originalName,
                      mimetype: kb.mimetype,
                      userId: userId,
                      tenantId, // Added tenantId
                      chunkIndex: chunk.index,
                      startPosition: chunk.startPosition,
                      endPosition: chunk.endPosition,
                    },
                  );

                  if ((i + 1) % 10 === 0) {
                    this.logger.log(`Single processing progress: ${i + 1}/${chunks.length}`);
                  }
                } catch (chunkError) {
                  this.logger.error(
                    `Failed to process text block ${chunk.index}. Skipping: ${chunkError.message}`
                  );
                  continue;
                }
              }

              this.logger.log(this.i18nService.formatMessage('singleTextProcessingComplete', { count: chunks.length }));
            } else {
              
              throw error;
            }
          }
        }
      }

      await this.updateStatus(kbId, FileStatus.VECTORIZED);
      const memAfter = this.memoryMonitor.getMemoryUsage();
      this.logger.log(
        this.i18nService.formatMessage('fileVectorizationComplete', { id: kbId, count: chunks.length, memory: memAfter.heapUsed })
      );
    } catch (error) {
      this.logger.error(this.i18nService.formatMessage('fileVectorizationFailed', { id: kbId }), error);

      
      try {
        const kb = await this.kbRepository.findOne({ where: { id: kbId } });
        if (kb) {
          const metadata = kb.metadata || {};
          metadata.lastError = error.message;
          metadata.failedAt = new Date().toISOString();
          await this.kbRepository.update(kbId, { metadata });
        }
      } catch (e) {
        this.logger.warn(`Failed to update metadata for failed file ${kbId}`, e);
      }

      await this.updateStatus(kbId, FileStatus.FAILED);
    }
  }

  
  private async processInBatches<T>(
    items: T[],
    processor: (batch: T[], batchIndex: number) => Promise<void>,
    options?: {
      batchSize?: number;
      onBatchComplete?: (batchIndex: number, totalBatches: number) => void;
    },
  ): Promise<void> {
    const totalItems = items.length;
    if (totalItems === 0) return;

    const startTime = Date.now();
    this.logger.log(this.i18nService.formatMessage('batchProcessingStarted', { count: totalItems }));

    // Use provided batch size or fallback to env/default
    const initialBatchSize = options?.batchSize || parseInt(process.env.CHUNK_BATCH_SIZE || '100');
    const totalBatches = Math.ceil(totalItems / initialBatchSize);

    for (let i = 0; i < totalItems;) {
      
      await this.memoryMonitor.waitForMemoryAvailable();

      
      
      
      const currentMem = this.memoryMonitor.getMemoryUsage().heapUsed;
      const dynamicBatchSize = this.memoryMonitor.getDynamicBatchSize(currentMem);

      // Ensure we don't exceed the model's limit (initialBatchSize) even if memory allows more
      const batchSize = Math.min(dynamicBatchSize, initialBatchSize);

      
      const batch = items.slice(i, i + batchSize);
      const batchIndex = Math.floor(i / batchSize) + 1;

      this.logger.log(
        this.i18nService.formatMessage('batchProcessingProgress', { index: batchIndex, total: totalBatches, count: batch.length })
      );

      
      await processor(batch, batchIndex);

      
      if (options?.onBatchComplete) {
        options.onBatchComplete(batchIndex, totalBatches);
      }

      
      if (currentMem > 800) {
        this.memoryMonitor.forceGC();
      }

      
      batch.length = 0;

      i += batchSize;
    }

    const duration = ((Date.now() - startTime) / 1000).toFixed(2);
    this.logger.log(this.i18nService.formatMessage('batchProcessingComplete', { count: totalItems, duration }));
  }

  
  async retryFailedFile(fileId: string, userId: string, tenantId: string): Promise<KnowledgeBase> {
    this.logger.log(`Retrying failed file ${fileId} for user ${userId} in tenant ${tenantId}`);

    // 1. Get file with tenant restriction
    const kb = await this.kbRepository.findOne({
      where: { id: fileId, tenantId },
    });

    if (!kb) {
      throw new NotFoundException('File does not exist');
    }

    if (kb.status !== FileStatus.FAILED) {
      throw new Error(this.i18nService.formatMessage('onlyFailedFilesRetryable', { status: kb.status }));
    }

    if (!kb.content || kb.content.trim().length === 0) {
      throw new Error(this.i18nService.getMessage('emptyFileRetryFailed'));
    }

    
    await this.updateStatus(fileId, FileStatus.INDEXING);

    
    this.vectorizeToElasticsearch(
      fileId,
      userId,
      tenantId,
      kb.content,
      {
        chunkSize: kb.chunkSize,
        chunkOverlap: kb.chunkOverlap,
        embeddingModelId: kb.embeddingModelId,
      }
    ).catch((err) => {
      this.logger.error(`Retry vectorization failed for file ${fileId}`, err);
    });

    
    const updatedKb = await this.kbRepository.findOne({ where: { id: fileId, tenantId } });
    if (!updatedKb) {
      throw new NotFoundException('File does not exist');
    }
    return updatedKb;
  }

  
  async getFileChunks(fileId: string, userId: string, tenantId: string) {
    this.logger.log(`Getting chunks for file ${fileId}, user ${userId}, tenant ${tenantId}`);

    // 1. Get file with tenant check
    const kb = await this.kbRepository.findOne({
      where: { id: fileId, tenantId },
    });

    if (!kb) {
      throw new NotFoundException('File does not exist');
    }

    
    const chunks = await this.elasticsearchService.getFileChunks(fileId, userId, tenantId);

    
    return {
      fileId: kb.id,
      fileName: kb.originalName,
      totalChunks: chunks.length,
      chunkSize: kb.chunkSize,
      chunkOverlap: kb.chunkOverlap,
      chunks: chunks.map(chunk => ({
        index: chunk.chunkIndex,
        content: chunk.content,
        contentLength: chunk.content.length,
        startPosition: chunk.startPosition,
        endPosition: chunk.endPosition,
      })),
    };
  }

  private async updateStatus(id: string, status: FileStatus) {
    await this.kbRepository.update(id, { status });
  }

  
  async ensurePDFExists(fileId: string, userId: string, tenantId: string, force: boolean = false): Promise<string> {
    const kb = await this.kbRepository.findOne({
      where: { id: fileId, tenantId },
    });

    if (!kb) {
      throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
    }

    
    if (kb.mimetype === 'application/pdf') {
      return kb.storagePath;
    }

    
    const ext = kb.originalName.toLowerCase().split('.').pop() || '';
    const isConvertible = [...DOC_EXTENSIONS, ...IMAGE_EXTENSIONS].includes(ext);

    if (!isConvertible) {
      this.logger.log(`Skipping PDF conversion for unsupported format: .${ext} (${kb.originalName})`);
      throw new Error(this.i18nService.getMessage('pdfPreviewNotSupported'));
    }

    
    const path = await import('path');
    const fs = await import('fs');
    const uploadDir = path.dirname(kb.storagePath);
    const baseName = path.basename(kb.storagePath, path.extname(kb.storagePath));
    const pdfPath = path.join(uploadDir, `${baseName}.pdf`);

    
    if (force && fs.existsSync(pdfPath)) {
      try {
        fs.unlinkSync(pdfPath);
        this.logger.log(`Forced regeneration: Deleted existing PDF ${pdfPath}`);
      } catch (e) {
        this.logger.warn(`Failed to delete existing PDF for regeneration: ${e.message}`);
      }
    }

    
    if (fs.existsSync(pdfPath) && !force) {
      if (!kb.pdfPath) {
        await this.kbRepository.update(kb.id, { pdfPath: pdfPath });
      }
      return pdfPath;
    }

    
    try {
      this.logger.log(`Starting PDF conversion for ${kb.originalName} at ${kb.storagePath}`);

      
      await this.libreOfficeService.convertToPDF(kb.storagePath);

      
      if (!fs.existsSync(pdfPath)) {
        throw new Error(`PDF conversion completed but file not found at ${pdfPath}`);
      }

      const stats = fs.statSync(pdfPath);
      if (stats.size === 0) {
        fs.unlinkSync(pdfPath);
        throw new Error(`PDF conversion failed: output file is empty`);
      }

      await this.kbRepository.update(kb.id, { pdfPath: pdfPath });

      this.logger.log(`PDF conversion successful: ${pdfPath}`);
      return pdfPath;
    } catch (error) {
      this.logger.error(`PDF conversion failed for ${fileId}: ${error.message}`, error.stack);
      throw new Error(this.i18nService.formatMessage('pdfConversionFailedDetail', { id: fileId }));
    }
  }

  async getPDFStatus(fileId: string, userId: string, tenantId: string) {
    const kb = await this.kbRepository.findOne({
      where: { id: fileId, tenantId },
    });

    if (!kb) {
      throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
    }

    
    if (kb.mimetype === 'application/pdf') {
      const token = this.generateTempToken(fileId, userId, tenantId);
      return {
        status: 'ready',
        url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
      };
    }

    
    const path = await import('path');
    const fs = await import('fs');
    const uploadDir = path.dirname(kb.storagePath);
    const baseName = path.basename(kb.storagePath, path.extname(kb.storagePath));
    const pdfPath = path.join(uploadDir, `${baseName}.pdf`);

    
    if (fs.existsSync(pdfPath)) {
      if (!kb.pdfPath) {
        kb.pdfPath = pdfPath;
        await this.kbRepository.save(kb);
      }
      const token = this.generateTempToken(fileId, userId, tenantId);
      return {
        status: 'ready',
        url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
      };
    }

    
    return {
      status: 'pending',
    };
  }

  private generateTempToken(fileId: string, userId: string, tenantId: string): string {
    const jwt = require('jsonwebtoken');

    const secret = process.env.JWT_SECRET;
    if (!secret) {
      throw new Error('JWT_SECRET environment variable is required but not set');
    }

    return jwt.sign(
      { fileId, userId, tenantId, type: 'pdf-access' },
      secret,
      { expiresIn: '1h' }
    );
  }

  
  private async getActualModelDimensions(embeddingModelId: string, userId: string, tenantId: string): Promise<number> {
    const defaultDimensions = parseInt(
      process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
    );

    try {
      
      const modelConfig = await this.modelConfigService.findOne(
        embeddingModelId,
        userId,
        tenantId,
      );

      if (modelConfig && modelConfig.dimensions) {
        this.logger.log(`Retrieved dimensions for ${modelConfig.name} from config: ${modelConfig.dimensions}`);
        return modelConfig.dimensions;
      }

      
      this.logger.log(`Probing model dimensions: ${embeddingModelId}`);
      const probeEmbeddings = await this.embeddingService.getEmbeddings(
        ['probe'],
        userId,
        embeddingModelId,
      );

      if (probeEmbeddings.length > 0) {
        const actualDimensions = probeEmbeddings[0].length;
        this.logger.log(`Detected actual model dimensions: ${actualDimensions}`);

        
        if (modelConfig) {
          try {
            await this.modelConfigService.update(userId, tenantId, modelConfig.id, {
              dimensions: actualDimensions,
            });
            this.logger.log(`Updated dimension config for model ${modelConfig.name} to ${actualDimensions}`);
          } catch (updateErr) {
            this.logger.warn(`Failed to update model dimension setting: ${updateErr.message}`);
          }
        }

        return actualDimensions;
      }
    } catch (err) {
      this.logger.warn(
        `Failed to get dimensions. Using default dimensions: ${defaultDimensions}`,
        err.message,
      );
    }

    return defaultDimensions;
  }

  
  async generateTitle(kbId: string): Promise<string | null> {
    this.logger.log(`Generating automatic title for file ${kbId}`);

    try {
      const kb = await this.kbRepository.findOne({ where: { id: kbId } });
      if (!kb || !kb.content || kb.content.trim().length === 0) {
        return null;
      }
      const tenantId = kb.tenantId;

      
      if (kb.title) {
        return kb.title;
      }

      
      const contentSample = kb.content.substring(0, 2500);

      
      const settings = await this.userSettingService.findOrCreate(kb.userId);
      const language = settings.language || 'ja';

      
      const prompt = this.i18nService.getDocumentTitlePrompt(language, contentSample);

      
      let generatedTitle: string | undefined;
      try {
        generatedTitle = await this.chatService.generateSimpleChat(
          [{ role: 'user', content: prompt }],
          kb.userId,
          kb.tenantId
        );
      } catch (err) {
        this.logger.warn(`Failed to generate title for document ${kbId} due to LLM configuration issue: ${err.message}`);
        return null; // Skip title generation if LLM is not configured for this tenant
      }

      if (generatedTitle && generatedTitle.trim().length > 0) {
        
        const cleanedTitle = generatedTitle.trim().replace(/^["']|["']$/g, '').substring(0, 100);
        await this.kbRepository.update(kbId, { title: cleanedTitle });

        
        await this.elasticsearchService.updateTitleByFileId(kbId, cleanedTitle, tenantId).catch((err) => {
          this.logger.error(`Failed to update title in Elasticsearch for ${kbId}`, err);
        });

        this.logger.log(`Successfully generated title for ${kbId}: ${cleanedTitle}`);
        return cleanedTitle;
      }
    } catch (error) {
      this.logger.error(`Failed to generate title for ${kbId}`, error);
    }

    return null;
  }
}
