KnowledgeBase/clean_translations.js

const fs = require('fs');
const path = require('path');

const filePath = process.argv[2];

if (!filePath) {
    console.error('Please provide a file path');
    process.exit(1);
}

const content = fs.readFileSync(filePath, 'utf8');

// These are missing keys that we want to ensure exist in each language block
const missingKeysData = {
    kbSettingsSaved: { zh: "检索与对话配置已保存", en: "Knowledge base settings saved", ja: "設定を保存しました" },
    failedToSaveSettings: { zh: "保存设置失败", en: "Failed to save settings", ja: "設定の保存に失敗しました" },
    actionFailed: { zh: "操作失败", en: "Action failed", ja: "操作に失敗しました" },
    userAddedToOrganization: { zh: "用户已添加到组织", en: "User added to organization", ja: "ユーザーが組織に追加されました" },
    featureUpdated: { zh: "功能已更新", en: "Feature updated", ja: "機能が更新されました" },
    roleTenantAdmin: { zh: "租户管理员", en: "Tenant Administrator", ja: "テナント管理者" },
    roleRegularUser: { zh: "普通用户", en: "Regular User", ja: "一般ユーザー" },
    creatingRegularUser: { zh: "正在创建普通用户", en: "Creating regular user", ja: "一般ユーザーを作成中" },
    editUserRole: { zh: "修改用户角色", en: "Edit user role", ja: "ユーザーロールを編集" },
    targetRole: { zh: "目标角色", en: "Target Role", ja: "対象のロール" },
    editCategory: { zh: "编辑分类", en: "Edit category", ja: "カテゴリを編集" },
    totalTenants: { zh: "总租户数", en: "Total Tenants", ja: "総テナント数" },
    systemUsers: { zh: "系统用户", en: "System Users", ja: "システムユーザー" },
    systemHealth: { zh: "系统健康", en: "System Health", ja: "システムヘルス" },
    operational: { zh: "运行正常", en: "Operational", ja: "正常稼働中" },
    orgManagement: { zh: "组织管理", en: "Organization Management", ja: "組織管理" },
    globalTenantControl: { zh: "全局租户控制", en: "Global Tenant Control", ja: "グローバルテナントコントロール" },
    newTenant: { zh: "新租户", en: "New Tenant", ja: "新規テナント" },
    domainOptional: { zh: "域名 (可选)", en: "Domain (Optional)", ja: "ドメイン (任意)" },
    saveChanges: { zh: "保存修改", en: "Save changes", ja: "変更を保存" },
    modelConfiguration: { zh: "模型配置", en: "Model Configuration", ja: "モデル設定" },
    defaultLLMModel: { zh: "默认推理模型", en: "Default LLM Model", ja: "デフォルト推論モデル" },
    selectLLM: { zh: "选择 LLM", en: "Select LLM", ja: "LLMを選択" },
    selectEmbedding: { zh: "选择 Embedding", en: "Select Embedding", ja: "埋め込みを選択" },
    rerankModel: { zh: "Rerank 模型", en: "Rerank Model", ja: "リランクモデル" },
    none: { zh: "无", en: "None", ja: "なし" },
    indexingChunkingConfig: { zh: "索引与切片配置", en: "Indexing & Chunking Config", ja: "インデックスとチャンク設定" },
    chatHyperparameters: { zh: "聊天超参数", en: "Chat Hyperparameters", ja: "チャットハイパーパラメータ" },
    temperature: { zh: "随机性 (Temperature)", en: "Temperature", ja: "温度" },
    precise: { zh: "精确", en: "Precise", ja: "精密" },
    creative: { zh: "创意", en: "Creative", ja: "クリエイティブ" },
    maxResponseTokens: { zh: "最大响应标识 (Max Tokens)", en: "Max Response Tokens", ja: "最大応答トークン数" },
    retrievalSearchSettings: { zh: "检索与搜索设置", en: "Retrieval & Search Settings", ja: "検索設定" },
    topK: { zh: "召回数量 (Top K)", en: "Top K", ja: "Top K" },
    similarityThreshold: { zh: "相似度阈值", en: "Similarity Threshold", ja: "類似度しきい値" },
    enableHybridSearch: { zh: "启用混合检索", en: "Enable Hybrid Search", ja: "ハイブリッド検索を有効にする" },
    hybridSearchDesc: { zh: "同时使用向量和全文检索以提高召回率", en: "Use both vector and full-text search to improve recall", ja: "ベクトル検索と全文検索を併用して検索精度を向上させます" },
    hybridWeight: { zh: "混合权重 (0.0=全文, 1.0=向量)", en: "Hybrid Weight (0.0=Fulltext, 1.0=Vector)", ja: "ハイブリッド重み (0.0=全文, 1.0=ベクトル)" },
    pureText: { zh: "纯文本", en: "Pure Text", ja: "純粋なテキスト" },
    pureVector: { zh: "纯向量", en: "Pure Vector", ja: "純粋なベクトル" },
    enableQueryExpansion: { zh: "启用查询扩展", en: "Enable Query Expansion", ja: "クエリ拡張を有効にする" },
    queryExpansionDesc: { zh: "生成多个查询变体以提高覆盖率", en: "Generate multiple query variations for better coverage", ja: "複数のクエリバリアントを生成してカバレッジを向上させます" },
    enableHyDE: { zh: "启用 HyDE", en: "Enable HyDE", ja: "HyDEを有効にする" },
    hydeDesc: { zh: "生成假设回答以改善语义搜索", en: "Generate hypothetical answers to improve semantic search", ja: "仮想的な回答を生成してセマンティック検索を改善します" },
    enableReranking: { zh: "启用重排序 (Rerank)", en: "Enable Reranking", ja: "リランクを有効にする" },
    rerankingDesc: { zh: "使用 Rerank 模型对结果进行二次排序", en: "Use Rerank model to re-sort results", ja: "リランクモデルを使用して結果を再ソートします" },
    broad: { zh: "宽泛", en: "Broad", ja: "広範" },
    strict: { zh: "严格", en: "Strict", ja: "厳格" },
    maxInput: { zh: "最大输入", en: "Max Input", ja: "最大入力" },
    dimensions: { zh: "维度", en: "Dimensions", ja: "次元" },
    defaultBadge: { zh: "默认", en: "Default", ja: "デフォルト" },
    dims: { zh: "维度: $1", en: "Dims: $1", ja: "次元: $1" },
    ctx: { zh: "上下文: $1", en: "Ctx: $1", ja: "コンテキスト: $1" },
    baseApi: { zh: "Base API: $1", en: "Base API: $1", ja: "Base API: $1" },
    configured: { zh: "已配置", en: "Configured", ja: "設定済み" },
    groupUpdated: { zh: "分组已更新", en: "Group updated", ja: "グループが更新されました" },
    groupDeleted: { zh: "分组已删除", en: "Group deleted", ja: "グループが削除されました" },
    groupCreated: { zh: "分组已创建", en: "Group created", ja: "グループが作成されました" },
    navCatalog: { zh: "目录", en: "Catalog", ja: "カタログ" },
    allDocuments: { zh: "所有文档", en: "All Documents", ja: "すべてのドキュメント" },
    categories: { zh: "分类", en: "Categories", ja: "カテゴリ" },
    uncategorizedFiles: { zh: "未分类文件", en: "Uncategorized Files", ja: "未分類ファイル" },
    category: { zh: "分类", en: "Category", ja: "カテゴリ" },
    statusReadyDesc: { zh: "已索引可查询", en: "Indexed and searchable", ja: "インデックス済みで検索可能" },
    statusIndexingDesc: { zh: "正在建立词向量索引", en: "Building vector index", ja: "ベクトルインデックスを作成中" },
    selectCategory: { zh: "选择分类", en: "Select Category", ja: "カテゴリを選択" },
    noneUncategorized: { zh: "无未分类文件", en: "No uncategorized files", ja: "未分類ファイルなし" },
    previous: { zh: "上一页", en: "Previous", ja: "前へ" },
    next: { zh: "下一页", en: "Next", ja: "次へ" },
    createCategory: { zh: "创建分类", en: "Create Category", ja: "カテゴリを作成" },
    categoryDesc: { zh: "描述您的知识分类", en: "Describe your knowledge category", ja: "ナレッジカテゴリを説明します" },
    categoryName: { zh: "分类名称", en: "Category Name", ja: "カテゴリ名" },
    createCategoryBtn: { zh: "立即创建", en: "Create Now", ja: "今すぐ作成" },
    newGroup: { zh: "新建分组", en: "New Group", ja: "新規グループ" },
    noKnowledgeGroups: { zh: "暂无知识库分组", en: "No knowledge groups yet", ja: "ナレッジグループがまだありません" },
    createGroupDesc: { zh: "开始创建您的第一个知识库分组并上传相关文档。", en: "Start by creating your first knowledge group and uploading documents.", ja: "最初のナレッジグループを作成してドキュメントをアップロードしてください。" },
    noDescriptionProvided: { zh: "未提供描述", en: "No description provided", ja: "説明なし" },
    browseManageFiles: { zh: "浏览并管理该分组下的文件和笔记。", en: "Browse and manage files and notes in this group.", ja: "このグループ内のファイルとメモを閲覧・管理します。" },
    filterGroupFiles: { zh: "根据名称搜索分组内文件...", en: "Search files in group by name...", ja: "名前でグループ内のファイルを検索..." },
    generalSettingsSubtitle: { zh: "管理您的应用程序首选项。", en: "Manage your application preferences.", ja: "アプリケーションの設定を管理します。" },
    userManagementSubtitle: { zh: "管理访问权限和帐户。", en: "Manage access and accounts.", ja: "アクセス権限とアカウントを管理します。" },
    modelManagementSubtitle: { zh: "配置全局 AI 模型。", en: "Configure global AI models.", ja: "グローバルなAIモデルを設定します。" },
    kbSettingsSubtitle: { zh: "索引和聊天参数的技术配置。", en: "Technical configuration for indexing and chat parameters.", ja: "インデックス作成とチャットパラメータの技術設定。" },
    tenantsSubtitle: { zh: "全局系统概览。", en: "Global system overview.", ja: "グローバルシステムの概要。" },
    allNotes: { zh: "所有笔记", en: "All Notes", ja: "すべてのノート" },
    filterNotesPlaceholder: { zh: "筛选笔记...", en: "Filter notes...", ja: "ノートをフィルタリング..." },
    noteTitlePlaceholder: { zh: "标题...", en: "Title...", ja: "タイトル..." },
    startWritingPlaceholder: { zh: "开始写作...", en: "Start writing...", ja: "書き始める..." },
    previewHeader: { zh: "预览", en: "Preview", ja: "プレビュー" },
    noContentToPreview: { zh: "没有可预览的内容", en: "No content to preview", ja: "プレビューするコンテンツがありません" },
    hidePreview: { zh: "隐藏预览", en: "Hide Preview", ja: "プレビューを非表示" },
    showPreview: { zh: "显示预览", en: "Show Preview", ja: "プレビューを表示" },
    directoryLabel: { zh: "目录", en: "Directory", ja: "ディレクトリ" },
    uncategorized: { zh: "未分类", en: "Uncategorized", ja: "未分類" },
    enterNamePlaceholder: { zh: "输入名称...", en: "Enter name...", ja: "名前を入力..." },
    subFolderPlaceholder: { zh: "子文件夹...", en: "Sub-folder...", ja: "サブフォルダ..." },
    categoryCreated: { zh: "分类已创建", en: "Category created", ja: "カテゴリが作成されました" },
    failedToCreateCategory: { zh: "创建分类失败", en: "Failed to create category", ja: "カテゴリの作成に失敗しました" },
    failedToDeleteCategory: { zh: "删除分类失败", en: "Failed to delete category", ja: "カテゴリの削除に失敗しました" },
    confirmDeleteCategory: { zh: "您确定要删除此分类吗？", en: "Are you sure you want to delete this category?", ja: "このカテゴリを削除してもよろしいですか？" }
};

const lines = content.split('\n');
let currentLang = null;
let resultLines = [];
let keysSeen = new Set();

const langStartRegex = /^\s+(\w+): \{/;
const keyRegex = /^\s+([a-zA-Z0-9_-]+):/;

for (let i = 0; i < lines.length; i++) {
    const line = lines[i];

    const langMatch = line.match(langStartRegex);
    if (langMatch) {
        // If we were in a language block, append missing keys before finishing it
        if (currentLang) {
            addMissingKeys(currentLang, resultLines, keysSeen);
        }
        currentLang = langMatch[1];
        keysSeen = new Set();
        resultLines.push(line);
        continue;
    }

    if (currentLang) {
        const keyMatch = line.match(keyRegex);
        if (keyMatch) {
            const key = keyMatch[1];
            if (keysSeen.has(key)) {
                // Duplicate key, skip it
                continue;
            }
            keysSeen.add(key);
        }

        // If the line ends the block
        if (line.trim() === '},') {
            addMissingKeys(currentLang, resultLines, keysSeen);
            currentLang = null;
            resultLines.push(line);
            continue;
        }

        // Also handle the very last block which might not have a comma
        if (line.trim() === '}' && i > lines.length - 5) {
            addMissingKeys(currentLang, resultLines, keysSeen);
            currentLang = null;
            resultLines.push(line);
            continue;
        }
    }

    resultLines.push(line);
}

function addMissingKeys(lang, targetLines, seen) {
    for (const [key, translations] of Object.entries(missingKeysData)) {
        if (!seen.has(key)) {
            const val = translations[lang] || translations['en'] || key;
            const escapedVal = JSON.stringify(val);
            targetLines.push(`    ${key}: ${escapedVal},`);
            seen.add(key);
        }
    }
}

fs.writeFileSync(filePath, resultLines.join('\n'), 'utf8');
console.log('Translations file cleaned and updated successfully!');