import { Injectable, Logger, NotFoundException, Inject, forwardRef } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { I18nService } from '../i18n/i18n.service';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository, In } from 'typeorm';
import { FileStatus, KnowledgeBase, ProcessingMode } from './knowledge-base.entity';
import { KnowledgeGroup } from '../knowledge-group/knowledge-group.entity';
import { ElasticsearchService } from '../elasticsearch/elasticsearch.service';
import { TikaService } from '../tika/tika.service';
import * as fs from 'fs';
import * as path from 'path';
import { EmbeddingService } from './embedding.service';
import { TextChunkerService } from './text-chunker.service';
import { ModelConfigService } from '../model-config/model-config.service';
import { RagService } from '../rag/rag.service';
import { VisionService } from '../vision/vision.service';
import { TenantService } from '../tenant/tenant.service';
import { MemoryMonitorService } from './memory-monitor.service';
import { ChunkConfigService } from './chunk-config.service';
import { VisionPipelineService } from '../vision-pipeline/vision-pipeline.service';
import { LibreOfficeService } from '../libreoffice/libreoffice.service';
import { Pdf2ImageService } from '../pdf2image/pdf2image.service';
import { DOC_EXTENSIONS, IMAGE_EXTENSIONS } from '../common/file-support.constants';
import { ChatService } from '../chat/chat.service';
import { UserSettingService } from '../user/user-setting.service';
import { ModelType } from '../types';

@Injectable()
export class KnowledgeBaseService {
  private readonly logger = new Logger(KnowledgeBaseService.name);

  constructor(
    @InjectRepository(KnowledgeBase)
    private kbRepository: Repository<KnowledgeBase>,
    @InjectRepository(KnowledgeGroup)
    private groupRepository: Repository<KnowledgeGroup>,
    @Inject(forwardRef(() => ElasticsearchService))
    private elasticsearchService: ElasticsearchService,
    private tikaService: TikaService,
    private embeddingService: EmbeddingService,
    private textChunkerService: TextChunkerService,
    private modelConfigService: ModelConfigService,
    @Inject(forwardRef(() => RagService))
    private ragService: RagService,
    private visionService: VisionService,
    private tenantService: TenantService,
    private memoryMonitor: MemoryMonitorService,
    private chunkConfigService: ChunkConfigService,
    private visionPipelineService: VisionPipelineService,
    private libreOfficeService: LibreOfficeService,
    private pdf2ImageService: Pdf2ImageService,
    private configService: ConfigService,
    private i18nService: I18nService,
    @Inject(forwardRef(() => ChatService))
    private chatService: ChatService,
    private userSettingService: UserSettingService,
  ) { }

  async createAndIndex(
    fileInfo: any,
    userId: string,
    tenantId: string,
    config?: any,
  ): Promise<KnowledgeBase> {
    const mode = config?.mode || 'fast';
    const processingMode = mode === 'precise' ? ProcessingMode.PRECISE : ProcessingMode.FAST;

    // Strict validation: Ensure tenant has configured an embedding model in "Index Chat Config"
    // If config.embeddingModelId is specifically passed (e.g. from a task), we use it.
    // Otherwise, we lookup the tenant's default embedding model in strict mode.
    let embeddingModelId = config?.embeddingModelId;
    if (!embeddingModelId) {
      const defaultEmbedding = await this.modelConfigService.findDefaultByType(tenantId, ModelType.EMBEDDING, true);
      embeddingModelId = defaultEmbedding.id;
    }

    const kb = this.kbRepository.create({
      originalName: fileInfo.originalname,
      storagePath: fileInfo.path,
      size: fileInfo.size,
      mimetype: fileInfo.mimetype,
      status: FileStatus.PENDING,
      userId: userId,
      tenantId: tenantId,
      chunkSize: config?.chunkSize || 200,
      chunkOverlap: config?.chunkOverlap || 40,
      embeddingModelId: embeddingModelId,
      processingMode: processingMode,
    });

    // Associate groups
    if (config?.groupIds && config.groupIds.length > 0) {
      const groups = await this.groupRepository.find({
        where: { id: In(config.groupIds), tenantId: tenantId }
      });
      kb.groups = groups;
    }

    const savedKb = await this.kbRepository.save(kb);

    this.logger.log(
      `Created KB record: ${savedKb.id}, mode: ${mode}, file: ${fileInfo.originalname}`
    );

    // ---------------------------------------------------------
    // Move the file to the final partitioned directory
    // source: uploads/{tenantId}/{filename} (or wherever it was)
    // target: uploads/{tenantId}/{savedKb.id}/{filename}
    // ---------------------------------------------------------
    const fs = await import('fs');
    const path = await import('path');
    const uploadPath = process.env.UPLOAD_FILE_PATH || './uploads';
    const targetDir = path.join(uploadPath, tenantId || 'default', savedKb.id);
    const targetPath = path.join(targetDir, fileInfo.filename);

    try {
      if (!fs.existsSync(targetDir)) {
        fs.mkdirSync(targetDir, { recursive: true });
      }
      if (fs.existsSync(fileInfo.path)) {
        fs.renameSync(fileInfo.path, targetPath);
        // Update the DB record with the new path
        savedKb.storagePath = targetPath;
        await this.kbRepository.save(savedKb);
        this.logger.log(`Moved file to partitioned storage: ${targetPath}`);
      }
    } catch (fsError) {
      this.logger.error(`Failed to move file ${savedKb.id} to partitioned storage`, fsError);
      // We will let it continue, but the file might be stuck in the temp/root folder
    }

    // If queue processing is requested, await completion
    if (config?.waitForCompletion) {
      await this.processFile(savedKb.id, userId, tenantId, config);
    } else {
      // Otherwise trigger asynchronously (default)
      this.processFile(savedKb.id, userId, tenantId, config).catch((err) => {
        this.logger.error(`Error processing file ${savedKb.id}`, err);
      });
    }

    return savedKb;
  }

  async findAll(userId: string, tenantId?: string): Promise<KnowledgeBase[]> {
    const where: any = {};
    if (tenantId) {
      where.tenantId = tenantId;
    } else {
      where.userId = userId;
    }
    return this.kbRepository.find({
      where,
      relations: ['groups'], // Load group relations
      order: { createdAt: 'DESC' },
    });
  }

  async searchKnowledge(userId: string, tenantId: string, query: string, topK: number = 5) {
    try {
      // Generate simulation vector using default dimensions from environment variable
      const defaultDimensions = parseInt(
        process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
      );
      const mockEmbedding = Array.from(
        { length: defaultDimensions },
        () => Math.random() - 0.5,
      );
      const queryVector = mockEmbedding;

      // 2. Search in Elasticsearch
      const searchResults = await this.elasticsearchService.searchSimilar(
        queryVector,
        userId,
        topK,
        tenantId, // Ensure shared visibility within tenant
      );

      // 3. Get file information from database
      const fileIds = [...new Set(searchResults.map((r) => r.fileId))];
      const files = await this.kbRepository.findByIds(fileIds);
      const fileMap = new Map(files.map((f) => [f.id, f]));

      // 4. Combine results with file info
      const results = searchResults.map((result) => {
        const file = fileMap.get(result.fileId);
        return {
          ...result,
          file: file
            ? {
              id: file.id,
              name: file.originalName,
              mimetype: file.mimetype,
              size: file.size,
              createdAt: file.createdAt,
            }
            : null,
        };
      });

      return {
        query,
        results,
        total: results.length,
      };
    } catch (error) {
      this.logger.error(
        `Metadata search failed for tenant ${tenantId}:`,
        error.stack || error.message,
      );
      throw error;
    }
  }

  async ragSearch(userId: string, tenantId: string, query: string, settings: any) {
    this.logger.log(
      `RAG search request: userId=${userId}, query="${query}", settings=${JSON.stringify(settings)}`,
    );

    try {
      const ragResults = await this.ragService.searchKnowledge(
        query,
        userId,
        settings.topK,
        settings.similarityThreshold,
        settings.selectedEmbeddingId,
        settings.enableFullTextSearch,
        settings.enableRerank,
        settings.selectedRerankId,
        undefined,
        undefined,
        settings.rerankSimilarityThreshold,
        tenantId, // Ensure shared visibility within tenant for RAG
      );

      const sources = this.ragService.extractSources(ragResults);
      const ragPrompt = this.ragService.buildRagPrompt(
        query,
        ragResults,
        settings.language || 'ja',
      );

      const result = {
        searchResults: ragResults,
        sources,
        ragPrompt,
        hasRelevantContent: ragResults.length > 0,
      };

      this.logger.log(
        `RAG search completed: found ${ragResults.length} results`,
      );
      return result;
    } catch (error) {
      this.logger.error(
        `RAG search failed for user ${userId}:`,
        error.stack || error.message,
      );
      // Return empty result instead of throwing error to keep system running
      return {
        searchResults: [],
        sources: [],
        ragPrompt: query, // Use original query
        hasRelevantContent: false,
      };
    }
  }

  async deleteFile(fileId: string, userId: string, tenantId: string): Promise<void> {
    this.logger.log(`Deleting file ${fileId} for user ${userId}`);

    try {
      // 1. Get file info
      const file = await this.kbRepository.findOne({
        where: { id: fileId, tenantId }, // Filter by tenantId
      });
      if (!file) {
        throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
      }

      // 2. Delete file from filesystem
      const fs = await import('fs');
      try {
        if (fs.existsSync(file.storagePath)) {
          fs.unlinkSync(file.storagePath);
          this.logger.log(`Deleted file: ${file.storagePath}`);
        }
      } catch (error) {
        this.logger.warn(`Failed to delete file ${file.storagePath}:`, error);
      }

      // 3. Delete from Elasticsearch
      try {
        await this.elasticsearchService.deleteByFileId(fileId, userId, tenantId);
        this.logger.log(`Deleted ES documents for file ${fileId}`);
      } catch (error) {
        this.logger.warn(
          `Failed to delete ES documents for file ${fileId}:`,
          error,
        );
      }

      // 4. Remove from all groups (cleanup M2M relations)
      const fileWithGroups = await this.kbRepository.findOne({
        where: { id: fileId, tenantId },
        relations: ['groups'],
      });

      if (fileWithGroups && fileWithGroups.groups && fileWithGroups.groups.length > 0) {
        // Clear groups to remove entries from join table
        fileWithGroups.groups = [];
        await this.kbRepository.save(fileWithGroups);
        this.logger.log(`Cleared group associations for file ${fileId}`);
      }

      // 5. Delete from SQLite
      await this.kbRepository.delete({ id: fileId });
      this.logger.log(`Deleted database record for file ${fileId}`);
    } catch (error) {
      this.logger.error(`Failed to delete file ${fileId}`, error);
      throw error;
    }

  }

  async clearAll(userId: string, tenantId: string): Promise<void> {
    this.logger.log(`Clearing all knowledge base data for user ${userId} in tenant ${tenantId}`);

    try {
      // Get all files and delete them one by one
      const files = await this.kbRepository.find();

      for (const file of files) {
        await this.deleteFile(file.id, userId, tenantId);
      }

      this.logger.log(`Cleared all knowledge base data for user ${userId}`);
    } catch (error) {
      this.logger.error(
        `Failed to clear knowledge base for user ${userId}`,
        error,
      );
      throw error;
    }
  }

  private async processFile(kbId: string, userId: string, tenantId: string, config?: any) {
    this.logger.log(`Starting processing for file ${kbId}, mode: ${config?.mode || 'fast'}`);
    await this.updateStatus(kbId, FileStatus.INDEXING);

    try {
      const kb = await this.kbRepository.findOne({ where: { id: kbId } });
      if (!kb) {
        this.logger.error(`KB not found: ${kbId}`);
        return;
      }

      // Memory monitor - pre-processing check
      const memBefore = this.memoryMonitor.getMemoryUsage();
      this.logger.log(`Memory state - before processing: ${memBefore.heapUsed}/${memBefore.heapTotal}MB`);

      // Select processing flow based on mode
      const mode = config?.mode || 'fast';

      if (mode === 'precise') {
        // Precise mode - use Vision Pipeline
        await this.processPreciseMode(kb, userId, tenantId, config);
      } else {
        // Fast mode - use Tika
        await this.processFastMode(kb, userId, tenantId, config);
      }

      this.logger.log(`File ${kbId} processed successfully in ${mode} mode.`);
    } catch (error) {
      this.logger.error(`Failed to process file ${kbId}`, error);
      await this.updateStatus(kbId, FileStatus.FAILED);
    }
  }

  /**
   * Fast mode processing (existing flow)
   */
  private async processFastMode(kb: KnowledgeBase, userId: string, tenantId: string, config?: any) {
    // 1. Extract text using Tika
    let text = await this.tikaService.extractText(kb.storagePath);

    // Use vision model for image files
    if (this.visionService.isImageFile(kb.mimetype)) {
      const settings = await this.tenantService.getSettings(tenantId || 'default');
      const visionModelId = settings?.selectedVisionId;
      if (visionModelId) {
        const visionModel = await this.modelConfigService.findOne(
          visionModelId,
        );
        if (visionModel && visionModel.type === 'vision' && visionModel.isEnabled !== false) {
          text = await this.visionService.extractImageContent(kb.storagePath, {
            baseUrl: visionModel.baseUrl || '',
            apiKey: visionModel.apiKey || '',
            modelId: visionModel.modelId,
          });
        }
      }
    }

    if (!text || text.trim().length === 0) {
      this.logger.warn(this.i18nService.getMessage('noTextExtracted'));
    }

    // Check text size
    const textSizeMB = Math.round(text.length / 1024 / 1024);
    if (textSizeMB > 50) {
      this.logger.warn(this.i18nService.formatMessage('extractedTextTooLarge', { size: textSizeMB }));
    }

    // Save text to database
    await this.kbRepository.update(kb.id, { content: text });
    await this.updateStatus(kb.id, FileStatus.EXTRACTED);

    // Async vectorization
    await this.vectorizeToElasticsearch(kb.id, userId, tenantId, text, config).catch((err) => {
      this.logger.error(`Error vectorizing file ${kb.id}`, err);
    });

    // Auto-generate title (async execution)
    this.generateTitle(kb.id).catch((err) => {
      this.logger.error(`Error generating title for file ${kb.id}`, err);
    });

    // Trigger PDF conversion asynchronously (for document files)
    this.ensurePDFExists(kb.id, userId, tenantId).catch((err) => {
      this.logger.warn(this.i18nService.formatMessage('pdfConversionFailedDetail', { id: kb.id }), err);
    });
  }

  /**
   * Precise mode processing (new flow)
   */
  private async processPreciseMode(kb: KnowledgeBase, userId: string, tenantId: string, config?: any) {
    // Check if precise mode is supported
    const preciseFormats = ['.pdf', '.doc', '.docx', '.ppt', '.pptx'];
    const ext = kb.originalName.toLowerCase().substring(kb.originalName.lastIndexOf('.'));

    if (!preciseFormats.includes(ext)) {
      this.logger.warn(
        this.i18nService.formatMessage('preciseModeUnsupported', { ext })
      );
      return this.processFastMode(kb, userId, tenantId, config);
    }

    // Check if Vision model is configured
      const settings = await this.tenantService.getSettings(tenantId || 'default');
    const visionModelId = settings?.selectedVisionId;
    if (!visionModelId) {
      this.logger.warn(
        this.i18nService.getMessage('visionModelNotConfiguredFallback')
      );
      return this.processFastMode(kb, userId, tenantId, config);
    }

    const visionModel = await this.modelConfigService.findOne(
      visionModelId,
    );
    if (!visionModel || visionModel.type !== 'vision' || visionModel.isEnabled === false) {
      this.logger.warn(
        this.i18nService.getMessage('visionModelInvalidFallback')
      );
      return this.processFastMode(kb, userId, tenantId, config);
    }

    // Call Vision Pipeline
    try {
      const result = await this.visionPipelineService.processPreciseMode(
        kb.storagePath,
        {
          userId,
          tenantId, // New
          modelId: visionModelId,
          fileId: kb.id,
          fileName: kb.originalName,
          skipQualityCheck: false,
        }
      );

      if (!result.success) {
        this.logger.error(`Vision pipeline failed, falling back to fast mode`);
        this.logger.warn(this.i18nService.getMessage('visionPipelineFailed'));
        return this.processFastMode(kb, userId, tenantId, config);
      }

      // Save text content to database
      const combinedText = result.results.map(r => r.text).join('\n\n');
      const metadata = {
        processedPages: result.processedPages,
        failedPages: result.failedPages,
        cost: result.cost,
        duration: result.duration,
        results: result.results.map(r => ({
          pageIndex: r.pageIndex,
          confidence: r.confidence,
          layout: r.layout,
          imageCount: r.images.length,
        })),
      };
      await this.kbRepository.update(kb.id, {
        content: combinedText,
        metadata: metadata as any,
      });

      await this.updateStatus(kb.id, FileStatus.EXTRACTED);
      this.logger.log(
        this.i18nService.formatMessage('preciseModeComplete', { pages: result.processedPages, cost: result.cost.toFixed(2) })
      );

      // Async vectorization and Elasticsearch indexing
      // Create each page as separate document with metadata
      this.indexPreciseResults(kb, userId, tenantId, kb.embeddingModelId, result.results).catch((err) => {
        this.logger.error(`Error indexing precise results for ${kb.id}`, err);
      });

      // Trigger PDF conversion asynchronously
      this.ensurePDFExists(kb.id, userId, tenantId).catch((err) => {
        this.logger.warn(`Initial PDF conversion failed for ${kb.id}`, err);
      });

      // Auto-generate title (async execution)
      this.generateTitle(kb.id).catch((err) => {
        this.logger.error(`Error generating title for file ${kb.id}`, err);
      });

    } catch (error) {
      this.logger.error(`Vision pipeline error: ${error.message}`, error.stack);
      this.logger.error(`Falling back to fast mode for file ${kb.id}`);
      return this.processFastMode(kb, userId, tenantId, config);
    }
  }

  /**
   * Index precise mode results
   */
  private async indexPreciseResults(
    kb: KnowledgeBase,
    userId: string,
    tenantId: string,
    embeddingModelId: string,
    results: any[]
  ): Promise<void> {
    this.logger.log(`Indexing ${results.length} precise results for ${kb.id}`);

    // Check index existence - get actual model dimensions
    const actualDimensions = await this.getActualModelDimensions(embeddingModelId, userId, tenantId);
    await this.elasticsearchService.createIndexIfNotExists(actualDimensions);

    // Batch vectorization and indexing
    const batchSize = parseInt(process.env.CHUNK_BATCH_SIZE || '50');

    for (let i = 0; i < results.length; i += batchSize) {
      const batch = results.slice(i, i + batchSize);
      const texts = batch.map(r => r.text);

      try {
        // Generate vectors
        const embeddings = await this.embeddingService.getEmbeddings(
          texts,
          userId,
          embeddingModelId
        );

        // Index each result
        for (let j = 0; j < batch.length; j++) {
          const result = batch[j];
          const embedding = embeddings[j];

          if (!embedding || embedding.length === 0) {
            this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorPage', { page: result.pageIndex }));
            continue;
          }

          await this.elasticsearchService.indexDocument(
            `${kb.id}_page_${result.pageIndex}`,
            result.text,
            embedding,
            {
              fileId: kb.id,
              originalName: kb.originalName,
              mimetype: kb.mimetype,
              userId: userId,
              tenantId: tenantId, // New
              pageNumber: result.pageIndex,
              images: result.images,
              layout: result.layout,
              confidence: result.confidence,
              source: 'precise',
              mode: 'vision',
            }
          );
        }

        this.logger.log(`Batch ${Math.floor(i / batchSize) + 1} completed: ${batch.length} pages`);
      } catch (error) {
        this.logger.error(`Batch ${Math.floor(i / batchSize) + 1} processing failed`, error);
      }
    }

    await this.updateStatus(kb.id, FileStatus.VECTORIZED);
    this.logger.log(`Precise mode indexing completed: ${results.length} pages`);
  }

  /**
   * Get specific page of PDF as image
   */
  async getPageAsImage(fileId: string, pageIndex: number, userId: string, tenantId: string): Promise<string> {
    const pdfPath = await this.ensurePDFExists(fileId, userId, tenantId);

    // Convert specific pages
    const result = await this.pdf2ImageService.convertToImages(pdfPath, {
      density: 150,
      quality: 75,
      format: 'jpeg',
    });

    // Find images for corresponding page numbers
    const pageImage = result.images.find(img => img.pageIndex === pageIndex + 1);
    if (!pageImage) {
      throw new NotFoundException(this.i18nService.formatMessage('pageImageNotFoundDetail', { page: pageIndex + 1 }));
    }

    return pageImage.path;
  }

  private async vectorizeToElasticsearch(
    kbId: string,
    userId: string,
    tenantId: string,
    text: string,
    config?: any,
  ) {
    try {
      const kb = await this.kbRepository.findOne({ where: { id: kbId, tenantId } });
      if (!kb) return;

      // Memory monitor - pre-vectorization check
      const memBeforeChunk = this.memoryMonitor.getMemoryUsage();
      this.logger.log(
        `Pre-vectorization memory: ${memBeforeChunk.heapUsed}/${memBeforeChunk.heapTotal}MB`,
      );

      this.logger.debug(`File ${kbId}: Validating chunk config...`);
      // 1. Validate and fix chunk config (based on model limits and env vars)
      const validatedConfig = await this.chunkConfigService.validateChunkConfig(
        kb.chunkSize,
        kb.chunkOverlap,
        kb.embeddingModelId,
      );
      this.logger.debug(`File ${kbId}: Chunk config validated.`);

      // If config modified, log warning and update database
      if (validatedConfig.warnings.length > 0) {
        this.logger.warn(
          this.i18nService.formatMessage('chunkConfigCorrection', { warnings: validatedConfig.warnings.join(', ') })
        );

        // Update config in database
        if (validatedConfig.chunkSize !== kb.chunkSize ||
          validatedConfig.chunkOverlap !== kb.chunkOverlap) {
          await this.kbRepository.update(kbId, {
            chunkSize: validatedConfig.chunkSize,
            chunkOverlap: validatedConfig.chunkOverlap,
          });
        }
      }

      // Display config summary (including actual limits applied)
      this.logger.debug(`File ${kbId}: Getting config summary...`);
      const configSummary = await this.chunkConfigService.getConfigSummary(
        validatedConfig.chunkSize,
        validatedConfig.chunkOverlap,
        kb.embeddingModelId,
      );
      this.logger.log(`Chunk config: ${configSummary}`);
      this.logger.log(`Config limits: chunk=${validatedConfig.effectiveMaxChunkSize}, overlap=${validatedConfig.effectiveMaxOverlapSize}`);

      // 2. Split text using validated config
      const chunks = this.textChunkerService.chunkText(
        text,
        validatedConfig.chunkSize,
        validatedConfig.chunkOverlap,
      );
      this.logger.log(`File ${kbId} split into ${chunks.length} text blocks`);

      if (chunks.length === 0) {
        this.logger.warn(this.i18nService.formatMessage('noChunksGenerated', { id: kbId }));
        await this.updateStatus(kbId, FileStatus.VECTORIZED);
        return;
      }

      // 3. Validate chunk count is reasonable
      const estimatedChunkCount = this.chunkConfigService.estimateChunkCount(
        text.length,
        validatedConfig.chunkSize,
      );
      if (chunks.length > estimatedChunkCount * 1.2) {
        this.logger.warn(
          this.i18nService.formatMessage('chunkCountAnomaly', { actual: chunks.length, estimated: estimatedChunkCount })
        );
      }

      // 4. Get recommended batch size (based on model limits)
      const recommendedBatchSize = await this.chunkConfigService.getRecommendedBatchSize(
        kb.embeddingModelId,
        parseInt(process.env.CHUNK_BATCH_SIZE || '100'),
      );

      // 5. Estimate memory usage
      const avgChunkSize = chunks.reduce((sum, c) => sum + c.content.length, 0) / chunks.length;
      const estimatedMemory = this.memoryMonitor.estimateMemoryUsage(
        chunks.length,
        avgChunkSize,
        parseInt(process.env.DEFAULT_VECTOR_DIMENSIONS || '2560'),
      );
      this.logger.log(`Estimated memory usage: ${estimatedMemory}MB (batch size: ${recommendedBatchSize})`);

      // 6. Get actual model dimensions and check index exists
      const actualDimensions = await this.getActualModelDimensions(kb.embeddingModelId, userId, tenantId);
      await this.elasticsearchService.createIndexIfNotExists(actualDimensions);

      // 7. Batch vectorization and indexing
      const useBatching = this.memoryMonitor.shouldUseBatching(
        chunks.length,
        avgChunkSize,
        actualDimensions,
      );

      if (useBatching) {
        try {
          await this.processInBatches(
            chunks,
            async (batch, batchIndex) => {
              // Verify batch size not exceeding model limit
              if (batch.length > recommendedBatchSize) {
                this.logger.warn(
                  this.i18nService.formatMessage('batchSizeExceeded', { index: batchIndex, actual: batch.length, limit: recommendedBatchSize })
                );
              }

              const chunkTexts = batch.map((chunk) => chunk.content);
              const embeddings = await this.embeddingService.getEmbeddings(
                chunkTexts,
                userId,
                kb.embeddingModelId,
              );

              // Validate dimension consistency
              if (embeddings.length > 0 && embeddings[0].length !== actualDimensions) {
                this.logger.warn(
                  `Vector dimension mismatch: expected ${actualDimensions}, got ${embeddings[0].length}`
                );
              }

              // Index this batch data immediately
              for (let i = 0; i < batch.length; i++) {
                const chunk = batch[i];
                const embedding = embeddings[i];

                if (!embedding || embedding.length === 0) {
                  this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorChunk', { index: chunk.index }));
                  continue;
                }

                await this.elasticsearchService.indexDocument(
                  `${kb.id}_chunk_${chunk.index}`,
                  chunk.content,
                  embedding,
                  {
                    fileId: kb.id,
                    originalName: kb.originalName,
                    mimetype: kb.mimetype,
                    userId: userId,
                    chunkIndex: chunk.index,
                    startPosition: chunk.startPosition,
                    tenantId, // Passing tenantId to ES
                  }
                );
              }

              this.logger.log(`Batch ${batchIndex} completed: ${batch.length} chunks`);
            },
            {
              batchSize: recommendedBatchSize,
              onBatchComplete: (batchIndex, totalBatches) => {
                const mem = this.memoryMonitor.getMemoryUsage();
                this.logger.log(
                  `Batch ${batchIndex}/${totalBatches} completed, memory: ${mem.heapUsed}MB`,
                );
              },
            },
          );
        } catch (error) {
          // Detect context length error (supports Japanese/Chinese/English)
          if (error.message && (error.message.includes('context length') || error.message.includes('context length exceeded'))) {
            this.logger.warn(this.i18nService.getMessage('contextLengthErrorFallback'));

            // Downgrade to single text processing
            for (let i = 0; i < chunks.length; i++) {
              const chunk = chunks[i];

              try {
                const embeddings = await this.embeddingService.getEmbeddings(
                  [chunk.content], // Single text
                  userId,
                  kb.embeddingModelId,
                );

                if (!embeddings[0] || embeddings[0].length === 0) {
                  this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorChunk', { index: chunk.index }));
                  continue;
                }

                await this.elasticsearchService.indexDocument(
                  `${kb.id}_chunk_${chunk.index}`,
                  chunk.content,
                  embeddings[0],
                  {
                    fileId: kb.id,
                    originalName: kb.originalName,
                    mimetype: kb.mimetype,
                    userId: userId,
                    chunkIndex: chunk.index,
                    startPosition: chunk.startPosition,
                    endPosition: chunk.endPosition,
                    tenantId,
                  }
                );

                if ((i + 1) % 10 === 0) {
                  this.logger.log(`Single processing progress: ${i + 1}/${chunks.length}`);
                }
              } catch (chunkError) {
                this.logger.error(
                  `Failed to process text block ${chunk.index}. Skipping: ${chunkError.message}`
                );
                continue;
              }
            }

            this.logger.log(`Single text processing completed: ${chunks.length} chunks`);
          } else {
            // Throw other errors directly
            throw error;
          }
        }
      } else {
        // Small files, batch processing (but need to check batch limits)
        const chunkTexts = chunks.map((chunk) => chunk.content);

        // Force batch processing if chunk count exceeds model batch limit
        if (chunks.length > recommendedBatchSize) {
          this.logger.warn(
            this.i18nService.formatMessage('chunkLimitExceededForceBatch', { actual: chunks.length, limit: recommendedBatchSize })
          );
          try {
            await this.processInBatches(
              chunks,
              async (batch, batchIndex) => {
                const batchTexts = batch.map((c) => c.content);
                const embeddings = await this.embeddingService.getEmbeddings(
                  batchTexts,
                  userId,
                  kb.embeddingModelId,
                );

                for (let i = 0; i < batch.length; i++) {
                  const chunk = batch[i];
                  const embedding = embeddings[i];

                  if (!embedding || embedding.length === 0) {
                    this.logger.warn(`Skipping empty vector text block ${chunk.index}`);
                    continue;
                  }

                  await this.elasticsearchService.indexDocument(
                    `${kb.id}_chunk_${chunk.index}`,
                    chunk.content,
                    embedding,
                    {
                      fileId: kb.id,
                      originalName: kb.originalName,
                      mimetype: kb.mimetype,
                      userId: userId,
                      chunkIndex: chunk.index,
                      startPosition: chunk.startPosition,
                      endPosition: chunk.endPosition,
                      tenantId, // Passing tenantId to ES metadata
                    }
                  );
                }
              },
            );
          } catch (error) {
            // Detect context length error (supports Japanese/Chinese/English)
            if (error.message && (error.message.includes('context length') || error.message.includes('context length exceeded'))) {
              this.logger.warn(this.i18nService.getMessage('batchContextLengthErrorFallback'));

              // Downgrade to single text processing
              for (let i = 0; i < chunks.length; i++) {
                const chunk = chunks[i];

                try {
                  const embeddings = await this.embeddingService.getEmbeddings(
                    [chunk.content], // Single text
                    userId,
                    kb.embeddingModelId,
                  );

                  if (!embeddings[0] || embeddings[0].length === 0) {
                    this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorChunk', { index: chunk.index }));
                    continue;
                  }

                  await this.elasticsearchService.indexDocument(
                    `${kb.id}_chunk_${chunk.index}`,
                    chunk.content,
                    embeddings[0],
                    {
                      fileId: kb.id,
                      originalName: kb.originalName,
                      mimetype: kb.mimetype,
                      userId: userId,
                      tenantId, // Added tenantId
                      chunkIndex: chunk.index,
                      startPosition: chunk.startPosition,
                      endPosition: chunk.endPosition,
                    },
                  );

                  if ((i + 1) % 10 === 0) {
                    this.logger.log(`Single processing progress: ${i + 1}/${chunks.length}`);
                  }
                } catch (chunkError) {
                  this.logger.error(
                    this.i18nService.formatMessage('chunkProcessingFailed', { index: chunk.index, message: chunkError.message })
                  );
                  continue;
                }
              }

              this.logger.log(this.i18nService.formatMessage('singleTextProcessingComplete', { count: chunks.length }));
            } else {
              // Throw other errors directly
              throw error;
            }
          }
        } else {
          // Process if file is small enough
          try {
            const embeddings = await this.embeddingService.getEmbeddings(
              chunkTexts,
              userId,
              kb.embeddingModelId,
            );

            for (let i = 0; i < chunks.length; i++) {
              const chunk = chunks[i];
              const embedding = embeddings[i];

              if (!embedding || embedding.length === 0) {
                this.logger.warn(this.i18nService.formatMessage('skippingEmptyVectorChunk', { index: chunk.index }));
                continue;
              }

              await this.elasticsearchService.indexDocument(
                `${kb.id}_chunk_${chunk.index}`,
                chunk.content,
                embedding,
                {
                  fileId: kb.id,
                  originalName: kb.originalName,
                  mimetype: kb.mimetype,
                  userId: userId,
                  tenantId, // Added tenantId
                  chunkIndex: chunk.index,
                  startPosition: chunk.startPosition,
                  endPosition: chunk.endPosition,
                },
              );
            }
          } catch (error) {
            // Detect context length error (supports Japanese/Chinese/English)
            if (error.message && (error.message.includes('context length') || error.message.includes('context length exceeded'))) {
              this.logger.warn(this.i18nService.getMessage('batchContextLengthErrorFallback'));

              // Downgrade to single text processing
              for (let i = 0; i < chunks.length; i++) {
                const chunk = chunks[i];

                try {
                  const embeddings = await this.embeddingService.getEmbeddings(
                    [chunk.content], // Single text
                    userId,
                    kb.embeddingModelId,
                  );

                  if (!embeddings[0] || embeddings[0].length === 0) {
                    this.logger.warn(`Skipping empty vector text block ${chunk.index}`);
                    continue;
                  }

                  await this.elasticsearchService.indexDocument(
                    `${kb.id}_chunk_${chunk.index}`,
                    chunk.content,
                    embeddings[0],
                    {
                      fileId: kb.id,
                      originalName: kb.originalName,
                      mimetype: kb.mimetype,
                      userId: userId,
                      tenantId, // Added tenantId
                      chunkIndex: chunk.index,
                      startPosition: chunk.startPosition,
                      endPosition: chunk.endPosition,
                    },
                  );

                  if ((i + 1) % 10 === 0) {
                    this.logger.log(`Single processing progress: ${i + 1}/${chunks.length}`);
                  }
                } catch (chunkError) {
                  this.logger.error(
                    `Failed to process text block ${chunk.index}. Skipping: ${chunkError.message}`
                  );
                  continue;
                }
              }

              this.logger.log(this.i18nService.formatMessage('singleTextProcessingComplete', { count: chunks.length }));
            } else {
              // Throw other errors directly
              throw error;
            }
          }
        }
      }

      await this.updateStatus(kbId, FileStatus.VECTORIZED);
      const memAfter = this.memoryMonitor.getMemoryUsage();
      this.logger.log(
        this.i18nService.formatMessage('fileVectorizationComplete', { id: kbId, count: chunks.length, memory: memAfter.heapUsed })
      );
    } catch (error) {
      this.logger.error(this.i18nService.formatMessage('fileVectorizationFailed', { id: kbId }), error);

      // Save error info to metadata
      try {
        const kb = await this.kbRepository.findOne({ where: { id: kbId } });
        if (kb) {
          const metadata = kb.metadata || {};
          metadata.lastError = error.message;
          metadata.failedAt = new Date().toISOString();
          await this.kbRepository.update(kbId, { metadata });
        }
      } catch (e) {
        this.logger.warn(`Failed to update metadata for failed file ${kbId}`, e);
      }

      await this.updateStatus(kbId, FileStatus.FAILED);
    }
  }

  /**
   * Batch processing with memory control
   */
  private async processInBatches<T>(
    items: T[],
    processor: (batch: T[], batchIndex: number) => Promise<void>,
    options?: {
      batchSize?: number;
      onBatchComplete?: (batchIndex: number, totalBatches: number) => void;
    },
  ): Promise<void> {
    const totalItems = items.length;
    if (totalItems === 0) return;

    const startTime = Date.now();
    this.logger.log(this.i18nService.formatMessage('batchProcessingStarted', { count: totalItems }));

    // Use provided batch size or fallback to env/default
    const initialBatchSize = options?.batchSize || parseInt(process.env.CHUNK_BATCH_SIZE || '100');
    const totalBatches = Math.ceil(totalItems / initialBatchSize);

    for (let i = 0; i < totalItems;) {
      // Check memory and wait
      await this.memoryMonitor.waitForMemoryAvailable();

      // Dynamically adjust batch size (start from initialBatchSize, memory monitor can reduce if needed)
      // Note: memoryMonitor.getDynamicBatchSize may return larger values based on memory situation,
      // but we must respect model limits (initialBatchSize)
      const currentMem = this.memoryMonitor.getMemoryUsage().heapUsed;
      const dynamicBatchSize = this.memoryMonitor.getDynamicBatchSize(currentMem);

      // Ensure we don't exceed the model's limit (initialBatchSize) even if memory allows more
      const batchSize = Math.min(dynamicBatchSize, initialBatchSize);

      // Get current batch
      const batch = items.slice(i, i + batchSize);
      const batchIndex = Math.floor(i / batchSize) + 1;

      this.logger.log(
        this.i18nService.formatMessage('batchProcessingProgress', { index: batchIndex, total: totalBatches, count: batch.length })
      );

      // Process batch
      await processor(batch, batchIndex);

      // Callback notification
      if (options?.onBatchComplete) {
        options.onBatchComplete(batchIndex, totalBatches);
      }

      // Force GC (if memory is near threshold)
      if (currentMem > 800) {
        this.memoryMonitor.forceGC();
      }

      // Clear references to help GC
      batch.length = 0;

      i += batchSize;
    }

    const duration = ((Date.now() - startTime) / 1000).toFixed(2);
    this.logger.log(this.i18nService.formatMessage('batchProcessingComplete', { count: totalItems, duration }));
  }

  /**
   * Retry vectorization for failed files
   */
  async retryFailedFile(fileId: string, userId: string, tenantId: string): Promise<KnowledgeBase> {
    this.logger.log(`Retrying failed file ${fileId} for user ${userId} in tenant ${tenantId}`);

    // 1. Get file with tenant restriction
    const kb = await this.kbRepository.findOne({
      where: { id: fileId, tenantId },
    });

    if (!kb) {
      throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
    }

    if (kb.status !== FileStatus.FAILED) {
      throw new Error(this.i18nService.formatMessage('onlyFailedFilesRetryable', { status: kb.status }));
    }

    if (!kb.content || kb.content.trim().length === 0) {
      throw new Error(this.i18nService.getMessage('emptyFileRetryFailed'));
    }

    // 2. Reset status to INDEXING
    await this.updateStatus(fileId, FileStatus.INDEXING);

    // 3. Trigger vectorization asynchronously (reuse existing logic)
    this.vectorizeToElasticsearch(
      fileId,
      userId,
      tenantId,
      kb.content,
      {
        chunkSize: kb.chunkSize,
        chunkOverlap: kb.chunkOverlap,
        embeddingModelId: kb.embeddingModelId,
      }
    ).catch((err) => {
      this.logger.error(`Retry vectorization failed for file ${fileId}`, err);
    });

    // 4. Return updated file status
    const updatedKb = await this.kbRepository.findOne({ where: { id: fileId, tenantId } });
    if (!updatedKb) {
      throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
    }
    return updatedKb;
  }

  /**
   * Get all chunk information for a file
   */
  async getFileChunks(fileId: string, userId: string, tenantId: string) {
    this.logger.log(`Getting chunks for file ${fileId}, user ${userId}, tenant ${tenantId}`);

    // 1. Get file with tenant check
    const kb = await this.kbRepository.findOne({
      where: { id: fileId, tenantId },
    });

    if (!kb) {
      throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
    }

    // 2. Get all chunks from Elasticsearch
    const chunks = await this.elasticsearchService.getFileChunks(fileId, userId, tenantId);

    // 3. Return chunk info
    return {
      fileId: kb.id,
      fileName: kb.originalName,
      totalChunks: chunks.length,
      chunkSize: kb.chunkSize,
      chunkOverlap: kb.chunkOverlap,
      chunks: chunks.map(chunk => ({
        index: chunk.chunkIndex,
        content: chunk.content,
        contentLength: chunk.content.length,
        startPosition: chunk.startPosition,
        endPosition: chunk.endPosition,
      })),
    };
  }

  private async updateStatus(id: string, status: FileStatus) {
    await this.kbRepository.update(id, { status });
  }

  // PDF preview related methods
  async ensurePDFExists(fileId: string, userId: string, tenantId: string, force: boolean = false): Promise<string> {
    const kb = await this.kbRepository.findOne({
      where: { id: fileId, tenantId },
    });

    if (!kb) {
      throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
    }

    // If original file is PDF, return the original file path directly
    if (kb.mimetype === 'application/pdf') {
      return kb.storagePath;
    }

    // Check if preview conversion is supported (only documents or images allowed)
    const ext = kb.originalName.toLowerCase().split('.').pop() || '';
    const isConvertible = [...DOC_EXTENSIONS, ...IMAGE_EXTENSIONS].includes(ext);

    if (!isConvertible) {
      this.logger.log(`Skipping PDF conversion for unsupported format: .${ext} (${kb.originalName})`);
      throw new Error(this.i18nService.getMessage('pdfPreviewNotSupported'));
    }

    // Generate PDF field path
    const path = await import('path');
    const fs = await import('fs');
    const uploadDir = path.dirname(kb.storagePath);
    const baseName = path.basename(kb.storagePath, path.extname(kb.storagePath));
    const pdfPath = path.join(uploadDir, `${baseName}.pdf`);

    // Delete if forced regeneration specified and file exists
    if (force && fs.existsSync(pdfPath)) {
      try {
        fs.unlinkSync(pdfPath);
        this.logger.log(`Forced regeneration: Deleted existing PDF ${pdfPath}`);
      } catch (e) {
        this.logger.warn(`Failed to delete existing PDF for regeneration: ${e.message}`);
      }
    }

    // Check if already converted and regeneration not needed
    if (fs.existsSync(pdfPath) && !force) {
      if (!kb.pdfPath) {
        await this.kbRepository.update(kb.id, { pdfPath: pdfPath });
      }
      return pdfPath;
    }

    // Need to convert to PDF
    try {
      this.logger.log(`Starting PDF conversion for ${kb.originalName} at ${kb.storagePath}`);

      // Convert file
      await this.libreOfficeService.convertToPDF(kb.storagePath);

      // Check conversion result
      if (!fs.existsSync(pdfPath)) {
        throw new Error(`PDF conversion completed but file not found at ${pdfPath}`);
      }

      const stats = fs.statSync(pdfPath);
      if (stats.size === 0) {
        fs.unlinkSync(pdfPath);
        throw new Error(`PDF conversion failed: output file is empty`);
      }

      await this.kbRepository.update(kb.id, { pdfPath: pdfPath });

      this.logger.log(`PDF conversion successful: ${pdfPath}`);
      return pdfPath;
    } catch (error) {
      this.logger.error(`PDF conversion failed for ${fileId}: ${error.message}`, error.stack);
      throw new Error(this.i18nService.formatMessage('pdfConversionFailedDetail', { id: fileId }));
    }
  }

  async getPDFStatus(fileId: string, userId: string, tenantId: string) {
    const kb = await this.kbRepository.findOne({
      where: { id: fileId, tenantId },
    });

    if (!kb) {
      throw new NotFoundException(this.i18nService.getMessage('fileNotFound'));
    }

    // If original file is PDF
    if (kb.mimetype === 'application/pdf') {
      const token = this.generateTempToken(fileId, userId, tenantId);
      return {
        status: 'ready',
        url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
      };
    }

    // Generate PDF file path
    const path = await import('path');
    const fs = await import('fs');
    const uploadDir = path.dirname(kb.storagePath);
    const baseName = path.basename(kb.storagePath, path.extname(kb.storagePath));
    const pdfPath = path.join(uploadDir, `${baseName}.pdf`);

    // Check if converted
    if (fs.existsSync(pdfPath)) {
      if (!kb.pdfPath) {
        kb.pdfPath = pdfPath;
        await this.kbRepository.save(kb);
      }
      const token = this.generateTempToken(fileId, userId, tenantId);
      return {
        status: 'ready',
        url: `/api/knowledge-bases/${fileId}/pdf?token=${token}`,
      };
    }

    // Conversion needed
    return {
      status: 'pending',
    };
  }

  private generateTempToken(fileId: string, userId: string, tenantId: string): string {
    const jwt = require('jsonwebtoken');

    const secret = process.env.JWT_SECRET;
    if (!secret) {
      throw new Error(this.i18nService.getMessage('jwtSecretRequired'));
    }

    return jwt.sign(
      { fileId, userId, tenantId, type: 'pdf-access' },
      secret,
      { expiresIn: '1h' }
    );
  }

  /**
   * Get actual model dimensions (with cache check and probe logic)
   */
  private async getActualModelDimensions(embeddingModelId: string, userId: string, tenantId: string): Promise<number> {
    const defaultDimensions = parseInt(
      process.env.DEFAULT_VECTOR_DIMENSIONS || '2560',
    );

    try {
      // 1. Prioritize getting from model config
      const modelConfig = await this.modelConfigService.findOne(
        embeddingModelId,
      );

      if (modelConfig && modelConfig.dimensions) {
        this.logger.log(`Got dimensions from ${modelConfig.name} config: ${modelConfig.dimensions}`);
        return modelConfig.dimensions;
      }

      // 2. Otherwise probe for dimensions
      this.logger.log(`Probing model dimensions: ${embeddingModelId}`);
      const probeEmbeddings = await this.embeddingService.getEmbeddings(
        ['probe'],
        userId,
        embeddingModelId,
      );

      if (probeEmbeddings.length > 0) {
        const actualDimensions = probeEmbeddings[0].length;
        this.logger.log(`Detected actual model dimensions: ${actualDimensions}`);

        // Update model config for next use
        if (modelConfig) {
          try {
            await this.modelConfigService.update(modelConfig.id, {
              dimensions: actualDimensions,
            });
            this.logger.log(`Updated model ${modelConfig.name} dimension config to ${actualDimensions}`);
          } catch (updateErr) {
            this.logger.warn(`Failed to update model dimension config: ${updateErr.message}`);
          }
        }

        return actualDimensions;
      }
    } catch (err) {
      this.logger.warn(
        `Failed to get dimensions. Using default: ${defaultDimensions}`,
        err.message,
      );
    }

    return defaultDimensions;
  }

  /**
   * Auto-generate document title using AI
   */
  async generateTitle(kbId: string): Promise<string | null> {
    this.logger.log(`Generating automatic title for file ${kbId}`);

    try {
      const kb = await this.kbRepository.findOne({ where: { id: kbId } });
      if (!kb || !kb.content || kb.content.trim().length === 0) {
        return null;
      }
      const tenantId = kb.tenantId;

      // Skip if title already exists
      if (kb.title) {
        return kb.title;
      }

      // Get content sample (max 2500 characters)
      const contentSample = kb.content.substring(0, 2500);

      // Get language from org settings, or use default
      const userSettings = await this.userSettingService.getByUser(kb.userId);
      const language = userSettings.language || 'zh';

      // Build prompt
      const prompt = this.i18nService.getDocumentTitlePrompt(language, contentSample);

      // Call LLM to generate title
      let generatedTitle: string | undefined;
      try {
        generatedTitle = await this.chatService.generateSimpleChat(
          [{ role: 'user', content: prompt }],
          kb.userId,
          kb.tenantId
        );
      } catch (err) {
        this.logger.warn(`Failed to generate title for document ${kbId} due to LLM configuration issue: ${err.message}`);
        return null; // Skip title generation if LLM is not configured for this tenant
      }

      if (generatedTitle && generatedTitle.trim().length > 0) {
        // Remove extra quotes and newlines
        const cleanedTitle = generatedTitle.trim().replace(/^["']|["']$/g, '').substring(0, 100);
        await this.kbRepository.update(kbId, { title: cleanedTitle });

        // Also update ES chunks
        await this.elasticsearchService.updateTitleByFileId(kbId, cleanedTitle, tenantId).catch((err) => {
          this.logger.error(`Failed to update title in Elasticsearch for ${kbId}`, err);
        });

        this.logger.log(`Successfully generated title for ${kbId}: ${cleanedTitle}`);
        return cleanedTitle;
      }
    } catch (error) {
      this.logger.error(`Failed to generate title for ${kbId}`, error);
    }

    return null;
  }
}
