fix: Exact plural forms for basic MT translators (tolgee#2454)

Author: JanCizmar

backend/data/src/main/kotlin/io/tolgee/service/machineTranslation/PluralTranslationUtil.kt

@@ -12,21 +12,14 @@ class PluralTranslationUtil(
   private val item: MtBatchItemParams,
   private val translateFn: (String) -> MtTranslatorResult,
 ) {
-  val forms by lazy {
-    context.getPluralFormsReplacingReplaceParam(baseTranslationText)
-      ?: throw IllegalStateException("Plural forms are null")
-  }
-
   fun translate(): MtTranslatorResult {
     return result
   }
 
   private val preparedFormSourceStrings: Sequence<Pair<String, String>> by lazy {
-    return@lazy targetExamples.asSequence().map {
-      val form = sourceRules?.select(it.value.toDouble())
-      val formValue = forms.forms[form] ?: forms.forms[PluralRules.KEYWORD_OTHER] ?: ""
-      it.key to formValue.replaceReplaceNumberPlaceholderWithExample(it.value)
-    }
+    val targetLanguageTag = context.getLanguage(item.targetLanguageId).tag
+    val sourceLanguageTag = context.baseLanguage.tag
+    getPreparedSourceStrings(sourceLanguageTag, targetLanguageTag, forms)
   }
 
   private val translated by lazy {
@@ -35,6 +28,11 @@ class PluralTranslationUtil(
     }
   }
 
+  private val forms by lazy {
+    context.getPluralFormsReplacingReplaceParam(baseTranslationText)
+      ?: throw IllegalStateException("Plural forms are null")
+  }
+
   private val result: MtTranslatorResult by lazy {
     val result =
       translated.map { (form, result) ->
@@ -59,18 +57,6 @@ class PluralTranslationUtil(
     )
   }
 
-  private val targetExamples by lazy {
-    val targetLanguageTag = context.getLanguage(item.targetLanguageId).tag
-    val targetULocale = getULocaleFromTag(targetLanguageTag)
-    val targetRules = PluralRules.forLocale(targetULocale)
-    getPluralFormExamples(targetRules)
-  }
-
-  private val sourceRules by lazy {
-    val sourceLanguageTag = context.baseLanguage.tag
-    getRulesByTag(sourceLanguageTag)
-  }
-
   private fun String.replaceNumberTags(): String {
     return this.replace(TOLGEE_TAG_REGEX, "#")
   }
@@ -126,5 +112,43 @@ class PluralTranslationUtil(
       val sourceULocale = getULocaleFromTag(languageTag)
       return PluralRules.forLocale(sourceULocale)
     }
+
+    fun getPreparedSourceStrings(
+      sourceLanguageTag: String,
+      targetLanguageTag: String,
+      forms: PluralForms,
+    ): Sequence<Pair<String, String>> {
+      val sourceRules = getRulesByTag(sourceLanguageTag)
+      val keywordCases =
+        getTargetExamples(targetLanguageTag).asSequence().map {
+          val form = sourceRules?.select(it.value.toDouble())
+          val formValue = forms.forms[form] ?: forms.forms[PluralRules.KEYWORD_OTHER] ?: ""
+          it.key to formValue.replaceReplaceNumberPlaceholderWithExample(it.value)
+        }
+
+      val exactCases =
+        forms.forms.asSequence().filter {
+          it.key.startsWith("=")
+        }.mapNotNull {
+          val number = it.key.substring(1).toDoubleOrNull() ?: return@mapNotNull null
+          it.key to it.value.replaceReplaceNumberPlaceholderWithExample(number)
+        }
+
+      return keywordCases + exactCases
+    }
+
+    private fun String.toDoubleOrNull(): Number? {
+      return try {
+        this.toBigDecimalOrNull()
+      } catch (e: NumberFormatException) {
+        null
+      }
+    }
+
+    private fun getTargetExamples(targetLanguageTag: String): Map<String, Number> {
+      val targetULocale = getULocaleFromTag(targetLanguageTag)
+      val targetRules = PluralRules.forLocale(targetULocale)
+      return getPluralFormExamples(targetRules)
+    }
   }
 }

backend/data/src/test/kotlin/io/tolgee/unit/util/PluralTranslationUtilTest.kt

@@ -0,0 +1,31 @@
+package io.tolgee.unit.util
+
+import io.tolgee.formats.getPluralFormsReplacingReplaceParam
+import io.tolgee.service.machineTranslation.PluralTranslationUtil
+import io.tolgee.testing.assert
+import org.junit.jupiter.api.Test
+
+class PluralTranslationUtilTest {
+  @Test
+  fun `provides correct forms for basic MT providers`() {
+    val baseString = """{number, plural, one {# apple} =1 {one apple} =2 {Two apples} =5 {# apples} other {# apples}}"""
+    val result =
+      PluralTranslationUtil.getPreparedSourceStrings(
+        "en",
+        "cs",
+        getPluralFormsReplacingReplaceParam(baseString, PluralTranslationUtil.REPLACE_NUMBER_PLACEHOLDER)!!,
+      )
+
+    result.toMap().assert.isEqualTo(
+      mapOf(
+        "one" to "<x id=\"tolgee-number\">1</x> apple",
+        "few" to "<x id=\"tolgee-number\">2</x> apples",
+        "many" to "<x id=\"tolgee-number\">0.5</x> apples",
+        "other" to "<x id=\"tolgee-number\">10</x> apples",
+        "=1" to "one apple",
+        "=2" to "Two apples",
+        "=5" to "<x id=\"tolgee-number\">5</x> apples",
+      ),
+    )
+  }
+}

e2e/cypress/e2e/import/importApplication.cy.ts

@@ -22,7 +22,7 @@ describe('Import application', () => {
     'Applies import',
     {
       retries: {
-        runMode: 4,
+        runMode: 10,
       },
     },
     () => {

e2e/cypress/e2e/import/importResultManupulation.cy.ts

@@ -188,7 +188,7 @@ describe('Import result & manipulation', () => {
       }
     );
 
-    it('imports with selects namespaces', () => {
+    it('imports with selects namespaces', { retries: { runMode: 5 } }, () => {
       gcy('import_apply_import_button').click();
       assertMessage('Import successful');
       gcy('import-result-row').should('have.length', 0);

e2e/cypress/e2e/translations/plurals.cy.ts

@@ -8,7 +8,7 @@ import {
 } from '../../common/translations';
 import { waitForGlobalLoading } from '../../common/loading';
 import { createKey, deleteProject } from '../../common/apiCalls/common';
-import { confirmStandard } from '../../common/shared';
+import { confirmStandard, gcyAdvanced } from '../../common/shared';
 
 describe('Translations Base', () => {
   let project: ProjectDTO = null;
@@ -60,6 +60,27 @@ describe('Translations Base', () => {
       .should('be.visible');
   });
 
+  it('shows base and existing exact forms', () => {
+    createKey(
+      project.id,
+      'Test key',
+      {
+        en: 'You have {testValue, plural, one {# item} =2 {Two items} other {# items}}',
+        cs: 'Máte {testValue, plural, one {# položku} =4 {# položky } few {# položky} other {# položek}}',
+      },
+      { isPlural: true }
+    );
+    visitTranslations(project.id);
+    waitForGlobalLoading();
+    getTranslationCell('Test key', 'cs').click();
+    gcyAdvanced({ value: 'translation-editor', variant: '=2' }).should(
+      'be.visible'
+    );
+    gcyAdvanced({ value: 'translation-editor', variant: '=4' }).should(
+      'be.visible'
+    );
+  });
+
   it('will change plural parameter name for all translations', () => {
     createKey(
       project.id,

webapp/src/views/projects/translations/TranslationEditor.tsx

@@ -18,6 +18,7 @@ export const TranslationEditor = ({ mode, tools, editorRef }: Props) => {
     handleSave,
     handleClose,
     handleInsertBase,
+    baseValue,
   } = tools;
 
   return (
@@ -30,6 +31,7 @@ export const TranslationEditor = ({ mode, tools, editorRef }: Props) => {
       autofocus={true}
       activeEditorRef={editorRef}
       mode={mode}
+      baseValue={baseValue}
       editorProps={{
         shortcuts: [
           { key: 'Escape', run: () => (handleClose(true), true) },

webapp/src/views/projects/translations/TranslationsTable/TranslationWrite.tsx

@@ -56,6 +56,7 @@ export const TranslationWrite: React.FC<Props> = ({ tools }) => {
     handleInsertBase,
     editEnabled,
     disabled,
+    baseText,
   } = tools;
   const editVal = tools.editVal!;
   const state = translation?.state || 'UNTRANSLATED';
@@ -68,7 +69,7 @@ export const TranslationWrite: React.FC<Props> = ({ tools }) => {
 
   const baseTranslation = useBaseTranslation(
     activeVariant,
-    keyData.translations[baseLanguage]?.text,
+    baseText,
     keyData.keyIsPlural
   );
 

webapp/src/views/projects/translations/translationVisual/PluralEditor.tsx

@@ -17,6 +17,7 @@ type Props = {
   autofocus?: boolean;
   activeEditorRef?: RefObject<EditorView | null>;
   mode: 'placeholders' | 'syntax';
+  baseValue?: TolgeeFormat;
 };
 
 export const PluralEditor = ({
@@ -29,6 +30,7 @@ export const PluralEditor = ({
   activeEditorRef,
   editorProps,
   mode,
+  baseValue,
 }: Props) => {
   function handleChange(text: string, variant: string) {
     onChange?.({ ...value, variants: { ...value.variants, [variant]: text } });
@@ -38,13 +40,25 @@ export const PluralEditor = ({
 
   const editorMode = project.icuPlaceholders ? mode : 'plain';
 
+  function getExactForms() {
+    if (!baseValue) {
+      return [];
+    }
+    return Object.keys(baseValue.variants)
+      .filter((key) => /^=\d+(\.\d+)?$/.test(key))
+      .map((key) => parseFloat(key.substring(1)));
+  }
+
+  const exactForms = getExactForms();
+
   return (
     <TranslationPlurals
       value={value}
       locale={locale}
       showEmpty
       activeVariant={activeVariant}
       variantPaddingTop="8px"
+      exactForms={exactForms}
       render={({ content, variant, exampleValue }) => {
         const variantOrOther = variant || 'other';
         return (

webapp/src/views/projects/translations/translationVisual/TranslationPlurals.tsx

@@ -1,10 +1,9 @@
-import { useMemo } from 'react';
+import React, { useMemo } from 'react';
 import { styled } from '@mui/material';
-import React from 'react';
 import {
-  TolgeeFormat,
   getPluralVariants,
   getVariantExample,
+  TolgeeFormat,
 } from '@tginternal/editor';
 
 const StyledContainer = styled('div')`
@@ -67,6 +66,7 @@ type Props = {
   showEmpty?: boolean;
   activeVariant?: string;
   variantPaddingTop?: number | string;
+  exactForms?: number[];
 };
 
 export const TranslationPlurals = ({
@@ -76,19 +76,12 @@ export const TranslationPlurals = ({
   showEmpty,
   activeVariant,
   variantPaddingTop,
+  exactForms,
 }: Props) => {
-  const variants = useMemo(() => {
-    const existing = new Set(Object.keys(value.variants));
-    const required = getPluralVariants(locale);
-    required.forEach((val) => existing.delete(val));
-    const result = Array.from(existing).map((value) => {
-      return [value, getVariantExample(locale, value)] as const;
-    });
-    required.forEach((value) => {
-      result.push([value, getVariantExample(locale, value)]);
-    });
-    return result;
-  }, [locale]);
+  const variants = useMemo(
+    () => getForms(locale, value, exactForms),
+    [locale, exactForms, value]
+  );
 
   if (value.parameter) {
     return (
@@ -137,3 +130,27 @@ export const TranslationPlurals = ({
     </StyledContainerSimple>
   );
 };
+
+function getForms(locale: string, value: TolgeeFormat, exactForms?: number[]) {
+  const forms: Set<string> = new Set();
+  getPluralVariants(locale).forEach((value) => forms.add(value));
+  Object.keys(value.variants).forEach((value) => forms.add(value));
+  (exactForms || [])
+    .map((value) => `=${value.toString()}`)
+    .forEach((value) => forms.add(value));
+
+  const formsArray = sortExactForms(forms);
+
+  return formsArray.map((value) => {
+    return [value, getVariantExample(locale, value)] as const;
+  });
+}
+
+function sortExactForms(forms: Set<string>) {
+  return [...forms].sort((a, b) => {
+    if (a.startsWith('=') && b.startsWith('=')) {
+      return Number(a.substring(1)) - Number(b.substring(1));
+    }
+    return 0;
+  });
+}