KnowledgeBase/server/src/upload/upload.service.ts

import { Injectable, Logger, BadRequestException } from '@nestjs/common';
import { KnowledgeBaseService } from '../knowledge-base/knowledge-base.service';
import { KnowledgeGroupService } from '../knowledge-group/knowledge-group.service';
import * as fs from 'fs';
import * as path from 'path';

@Injectable()
export class UploadService {
  private readonly logger = new Logger(UploadService.name);

  constructor(
    private kbService: KnowledgeBaseService,
    private groupService: KnowledgeGroupService,
  ) {}

  async processUploadedFile(file: Express.Multer.File) {
    // Add more business logic here. Example:
    // - Save file info to database
    // - Call other services to process file (Tika text extraction, ES indexing etc.)
    // - Validate file format or analyze content

    // Currently only return basic file info
    return {
      filename: file.filename,
      originalname: file.originalname,
      size: file.size,
      mimetype: file.mimetype,
      path: file.path, // After Multer saves file, full path is in file.path
    };
  }

  async importLocalFolder(
    sourcePath: string,
    userId: string,
    tenantId: string,
    config: any,
  ) {
    if (!fs.existsSync(sourcePath)) {
      throw new BadRequestException(`Directory not found: ${sourcePath}`);
    }

    const stat = fs.statSync(sourcePath);
    if (!stat.isDirectory()) {
      throw new BadRequestException(`Path is not a directory: ${sourcePath}`);
    }

    // Determine root group for hierarchy or single group
    let rootGroupId: string | null = null;
    if (config.groupIds && config.groupIds.length > 0) {
      rootGroupId = config.groupIds[0];
    }

    this.logger.log(
      `Starting local folder import: ${sourcePath} for user ${userId}, tenant ${tenantId}`,
    );

    // Trigger scanning and processing asynchronously to not block the request
    this.executeLocalImport(
      sourcePath,
      userId,
      tenantId,
      config,
      rootGroupId,
    ).catch((err) => {
      this.logger.error(`Local folder import failed for ${sourcePath}`, err);
    });

    return {
      sourcePath,
      status: 'PROCESSING',
    };
  }

  private async executeLocalImport(
    sourcePath: string,
    userId: string,
    tenantId: string,
    config: any,
    rootGroupId: string | null,
  ) {
    const files = this.scanDir(sourcePath);
    this.logger.log(`Found ${files.length} files in ${sourcePath}`);

    const dirToGroupId = new Map<string, string>();
    if (rootGroupId) {
      dirToGroupId.set('.', rootGroupId);
    } else {
      // Create a root group based on folder name if none provided
      const rootName = path.basename(sourcePath);
      const rootGroup = await this.groupService.create(userId, tenantId, {
        name: rootName,
        description: `Imported from local path: ${sourcePath}`,
      });
      rootGroupId = rootGroup.id;
      dirToGroupId.set('.', rootGroupId);
    }

    const uploadBaseDir = process.env.UPLOAD_FILE_PATH || './uploads';

    for (const filePath of files) {
      try {
        const relativeDir = path.relative(sourcePath, path.dirname(filePath));
        const normalizedDir = relativeDir || '.';

        let targetGroupId = rootGroupId;
        if (config.useHierarchy) {
          targetGroupId = await this.ensureHierarchy(
            userId,
            tenantId,
            normalizedDir,
            dirToGroupId,
            rootGroupId,
          );
        }

        const filename = path.basename(filePath);
        const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1e9);
        const storedFilename = `local-${uniqueSuffix}-${filename}`;

        // Ensure tenant directory exists
        const tenantDir = path.join(uploadBaseDir, tenantId);
        if (!fs.existsSync(tenantDir)) {
          fs.mkdirSync(tenantDir, { recursive: true });
        }

        const targetPath = path.join(tenantDir, storedFilename);
        fs.copyFileSync(filePath, targetPath);

        const stats = fs.statSync(targetPath);
        const fileInfo = {
          filename: storedFilename,
          originalname: filename,
          path: targetPath,
          size: stats.size,
          mimetype: this.getMimeType(filename),
        };

        await this.kbService.createAndIndex(fileInfo, userId, tenantId, {
          ...config,
          groupIds: [targetGroupId],
        });
      } catch (err) {
        this.logger.error(`Failed to process local file: ${filePath}`, err);
      }
    }

    this.logger.log(`Local folder import completed: ${sourcePath}`);
  }

  private async ensureHierarchy(
    userId: string,
    tenantId: string,
    relativeDir: string,
    dirToGroupId: Map<string, string>,
    rootGroupId: string,
  ): Promise<string> {
    if (dirToGroupId.has(relativeDir)) {
      return dirToGroupId.get(relativeDir)!;
    }

    const segments = relativeDir.split(path.sep);
    let currentPath = '';
    let parentId = rootGroupId;

    for (const segment of segments) {
      if (!segment || segment === '.') continue;
      currentPath = currentPath ? path.join(currentPath, segment) : segment;

      if (dirToGroupId.has(currentPath)) {
        parentId = dirToGroupId.get(currentPath)!;
        continue;
      }

      const group = await this.groupService.findOrCreate(
        userId,
        tenantId,
        segment,
        parentId,
        `Sub-folder from local import: ${currentPath}`,
      );
      dirToGroupId.set(currentPath, group.id);
      parentId = group.id;
    }

    return parentId;
  }

  private scanDir(directory: string): string[] {
    let results: string[] = [];
    if (!fs.existsSync(directory)) return results;

    const items = fs.readdirSync(directory);
    for (const item of items) {
      const fullPath = path.join(directory, item);
      const stat = fs.statSync(fullPath);
      if (stat.isDirectory()) {
        results = results.concat(this.scanDir(fullPath));
      } else {
        // Only include supported document and code extensions
        const ext = path.extname(item).toLowerCase().slice(1);
        if (
          [
            'pdf',
            'doc',
            'docx',
            'xls',
            'xlsx',
            'ppt',
            'pptx',
            'rtf',
            'csv',
            'txt',
            'md',
            'html',
            'json',
            'xml',
            'js',
            'ts',
            'py',
            'java',
            'sql',
          ].includes(ext)
        ) {
          results.push(fullPath);
        }
      }
    }
    return results;
  }

  private getMimeType(filename: string): string {
    const ext = path.extname(filename).toLowerCase();
    const mimeMap: Record<string, string> = {
      '.pdf': 'application/pdf',
      '.doc': 'application/msword',
      '.docx':
        'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
      '.md': 'text/markdown',
      '.txt': 'text/plain',
      '.json': 'application/json',
      '.html': 'text/html',
      '.csv': 'text/csv',
    };
    return mimeMap[ext] || 'application/octet-stream';
  }
}