TEDEFO-5133: Select preferred language only when translating view templates

src/main/java/eu/europa/ted/efx/sdk1/EfxTemplateTranslatorV1.java

@@ -359,7 +359,7 @@ private void shorthandIndirectLabelReference(ParserRuleContext ctx, final String
         ? this.script.composeFieldAttributeReference(
             this.script.contextualizePath(this.symbols.getAbsolutePathOfFieldWithoutTheAttribute(fieldId), currentContext.absolutePath()),
             this.symbols.getAttributeNameFromAttributeField(fieldId), StringPath.class)
-        : this.script.composeFieldValueReference(
+        : this.composeFieldValueReference(
         this.symbols.getRelativePathOfField(fieldId, currentContext.symbol()));
     Variable loopVariable = new Variable("item",
         this.script.composeVariableDeclaration("item", StringExpression.class), StringExpression.empty(),
@@ -532,12 +532,60 @@ public void exitShorthandFieldValueReferenceFromContextField(
     if (!this.efxContext.isFieldContext()) {
       throw InvalidUsageException.shorthandRequiresFieldContext(ctx, "$value");
     }
-    this.stack.push(this.script.composeFieldValueReference(
+    this.stack.push(this.composeFieldValueReference(
         this.symbols.getRelativePathOfField(this.efxContext.symbol(), this.efxContext.symbol())));
   }
 
   // #endregion Expression Blocks ${...} --------------------------------------
-
+
+  // #region Value References -------------------------------------------------
+
+  /***
+   * Multilingual fields are handled by this class, so the value reference is composed here instead
+   * of directly by the script generator. Anything else is left to the inherited behaviour.
+   *
+   * @see #composeFieldValueReference(PathExpression)
+   */
+  @Override
+  public void exitScalarFromFieldReference(final ScalarFromFieldReferenceContext ctx) {
+    if (!this.stack.peekType().is(EfxDataType.MultilingualString.class)) {
+      super.exitScalarFromFieldReference(ctx);
+      return;
+    }
+    this.stack.push(this.composeFieldValueReference(this.stack.pop(PathExpression.class)));
+  }
+
+  /***
+   * @see #exitScalarFromFieldReference(ScalarFromFieldReferenceContext)
+   */
+  @Override
+  public void exitSequenceFromFieldReference(final SequenceFromFieldReferenceContext ctx) {
+    if (!this.stack.peekType().is(EfxDataType.MultilingualString.class)) {
+      super.exitSequenceFromFieldReference(ctx);
+      return;
+    }
+    this.stack.push(this.composeFieldValueReference(this.stack.pop(PathExpression.class)));
+  }
+
+  /***
+   * In a view template the value of a multilingual field must be rendered in the language preferred
+   * by the reader, which EFX-1 gives the template author no syntax to ask for. Template translation
+   * therefore selects the preferred language implicitly, for every multilingual field it renders.
+   *
+   * Outside of view templates no such selection is possible: the function that performs it is
+   * provided by the XSL of the notice viewer and exists nowhere else. There the value of a
+   * multilingual field is retrieved like that of any other text field.
+   */
+  private PathExpression composeFieldValueReference(final PathExpression fieldReference) {
+    if (fieldReference.is(EfxDataType.MultilingualString.class)) {
+      return Expression.from(this.script.getTextInPreferredLanguage(fieldReference),
+          fieldReference.getClass());
+    }
+    return this.script.composeFieldValueReference(fieldReference);
+  }
+
+  // #endregion Value References ----------------------------------------------
+
   // #region Context Declaration Blocks {...} ---------------------------------
 
   /**

src/main/java/eu/europa/ted/efx/sdk1/xpath/XPathScriptGeneratorV1.java

@@ -25,7 +25,6 @@
 import eu.europa.ted.efx.model.expressions.scalar.NumericExpression;
 import eu.europa.ted.efx.model.expressions.scalar.StringExpression;
 import eu.europa.ted.efx.model.expressions.scalar.StringLiteral;
-import eu.europa.ted.efx.model.types.EfxDataType;
 import eu.europa.ted.efx.xpath.XPathScriptGenerator;
 
 @SdkComponent(versions = {"1"}, componentType = SdkComponentType.SCRIPT_GENERATOR)
@@ -48,29 +47,6 @@ public StringExpression composeToStringConversion(NumericExpression number) {
         return new StringExpression("format-number(" + number.getScript() + ", '" + formatString + "')");
     }
 
-    /***
-     * This method is overridden to workaround a limitation of EFX 1.
-     * 
-     * When a multilingual text field is referenced, then a special XPath expression
-     * is generated to retrieve the value in the "preferred" language.
-     * Preferred language is the first language among the languages listed in the
-     * translator options for which a text value is available in the field.
-     * 
-     * The logic of the workaround is as follows:
-     * if the fieldReference is a multilingual text field and it does not
-     * already come with a predicate that filters by @languageID, then we add a
-     * predicate which, using a for loop, will find the first language for which a
-     * value is available in the field.
-     * 
-     * In EFX 1 therefore the selection of the appropriate (preferred) language is
-     * done implicitly, whereas in EFX 2 it is done explicitly by calling a special
-     * function designed to perform this task.
-     * 
-     * Both EFX-1 and EFX-2 implementations of the feature rely on the existence of a
-     * $PREFERRED_LANGUAGES variable in the XSLT.
-     * This function returns the list of languages used in the visualisation in the
-     * order of preference (visualisation language followed by notice language(s)).
-     */
     /**
      * Preserved V1 behavior: pass EFX string literal through as-is without converting
      * escape sequences to XPath format.
@@ -91,13 +67,36 @@ public BooleanExpression composePatternMatchCondition(StringExpression expressio
             String.format("fn:matches(normalize-space(%s), %s)", expression.getScript(), pattern));
     }
 
+    /***
+     * Retrieves the value of a multilingual text field in the "preferred" language.
+     * Preferred language is the first language among the languages listed in the
+     * translator options for which a text value is available in the field.
+     *
+     * This is a workaround for a limitation of EFX 1: the language cannot be selected
+     * explicitly by the template author, so template translation applies this
+     * implicitly to every multilingual field it renders. In EFX 2 the selection is
+     * done explicitly, by calling a function designed to perform this task.
+     *
+     * If the reference already comes with a predicate that filters by @languageID,
+     * then the template author has already pinned a language and the value is
+     * retrieved as-is.
+     *
+     * Both EFX-1 and EFX-2 implementations of the feature rely on the existence of a
+     * $PREFERRED_LANGUAGES variable in the XSLT.
+     * This function returns the list of languages used in the visualisation in the
+     * order of preference (visualisation language followed by notice language(s)).
+     */
     @Override
-    public PathExpression composeFieldValueReference(PathExpression fieldReference) {
-        XPathInfo xpathInfo = XPathProcessor.parse(fieldReference.getScript());
-        if (fieldReference.is(EfxDataType.MultilingualString.class) && !xpathInfo.hasPredicate("@languageID")) {
-            return Expression.instantiate("efx:preferred-language-text(" + fieldReference.getScript() + ")", fieldReference.getClass());
+    public StringExpression getTextInPreferredLanguage(final PathExpression fieldReference) {
+        final XPathInfo xpathInfo = XPathProcessor.parse(fieldReference.getScript());
+        if (xpathInfo.hasPredicate("@languageID")) {
+            // The value reference is a PathExpression, which is not a StringExpression and cannot
+            // be returned as such. Only the generated script matters here: the caller re-creates
+            // the expression using the type of the field reference it started from.
+            return Expression.from(super.composeFieldValueReference(fieldReference),
+                StringExpression.class);
         }
-        return super.composeFieldValueReference(fieldReference);
+        return super.getTextInPreferredLanguage(fieldReference);
     }
 
     @Override

src/test/java/eu/europa/ted/efx/EfxTestsBase.java

@@ -5,15 +5,10 @@
 import eu.europa.ted.efx.interfaces.TranslatorOptions;
 import eu.europa.ted.efx.mock.DependencyFactoryMock;
 import eu.europa.ted.efx.model.DecimalFormat;
-import net.sf.saxon.s9api.ExtensionFunction;
-import net.sf.saxon.s9api.ItemType;
-import net.sf.saxon.s9api.OccurrenceIndicator;
 import net.sf.saxon.s9api.Processor;
 import net.sf.saxon.s9api.QName;
 import net.sf.saxon.s9api.SaxonApiException;
-import net.sf.saxon.s9api.SequenceType;
 import net.sf.saxon.s9api.XPathCompiler;
-import net.sf.saxon.s9api.XdmValue;
 
 public abstract class EfxTestsBase {
 
@@ -26,51 +21,17 @@ public abstract class EfxTestsBase {
   static {
     Processor processor = new Processor(false);
 
-    // Register custom EFX extension functions so Saxon can validate XPath syntax.
-    // These are only needed for V1 expression tests, where multilingual field references
-    // are implicitly wrapped in efx:preferred-language-text() by XPathScriptGeneratorV1.
-    // V2 bans these functions in expression context (template-only).
-    processor.registerExtensionFunction(efxFunction("preferred-language"));
-    processor.registerExtensionFunction(efxFunction("preferred-language-text"));
-
     XPATH_COMPILER = processor.newXPathCompiler();
     XPATH_COMPILER.setLanguageVersion("3.1");
     XPATH_COMPILER.declareNamespace("fn", "http://www.w3.org/2005/xpath-functions");
     XPATH_COMPILER.declareNamespace("xs", "http://www.w3.org/2001/XMLSchema");
+    // The functions of the EFX namespace are provided by the XSL of the notice viewer and are
+    // available to view templates only. None of them is registered here, so an expression that
+    // calls one fails to compile: that is what keeps them out of validation rules.
     XPATH_COMPILER.declareNamespace("efx", EFX_NAMESPACE);
     XPATH_COMPILER.declareVariable(new QName("urlPrefix"));
   }
 
-  /**
-   * Creates a dummy extension function stub for XPath syntax validation.
-   * Accepts one argument (node) and returns a string.
-   */
-  private static ExtensionFunction efxFunction(String localName) {
-    return new ExtensionFunction() {
-      @Override
-      public QName getName() {
-        return new QName(EFX_NAMESPACE, localName);
-      }
-
-      @Override
-      public SequenceType getResultType() {
-        return SequenceType.makeSequenceType(ItemType.STRING, OccurrenceIndicator.ONE);
-      }
-
-      @Override
-      public SequenceType[] getArgumentTypes() {
-        return new SequenceType[] {
-            SequenceType.makeSequenceType(ItemType.ANY_ITEM, OccurrenceIndicator.ONE_OR_MORE)
-        };
-      }
-
-      @Override
-      public XdmValue call(XdmValue[] arguments) {
-        throw new UnsupportedOperationException("Stub for XPath validation only");
-      }
-    };
-  }
-
   protected abstract String getSdkVersion();
 
   protected void testExpressionTranslationWithContext(final String expectedTranslation,

src/test/java/eu/europa/ted/efx/sdk1/EfxExpressionTranslatorV1Test.java

@@ -90,7 +90,7 @@ void testLikePatternCondition_WithNot() {
   @Test
   void testFieldValueComparison_UsingTextFields() {
     testExpressionTranslationWithContext(
-        "PathNode/TextField/normalize-space(text()) = efx:preferred-language-text(PathNode/TextMultilingualField)",
+        "PathNode/TextField/normalize-space(text()) = PathNode/TextMultilingualField/normalize-space(text())",
         "ND-Root", "BT-00-Text == BT-00-Text-Multilingual");
   }
 
@@ -1135,12 +1135,24 @@ void testFieldReference_WithAxis() {
         "ND-Root::preceding::BT-00-Integer");
   }
 
+  /**
+   * Outside of view templates there is no preferred language to select: efx:preferred-language-text()
+   * is defined by the notice viewer's XSL and is unavailable anywhere else, so a multilingual field
+   * value is retrieved like any other text value.
+   */
   @Test
   void testMultilingualTextFieldReference() {
-    testExpressionTranslationWithContext("efx:preferred-language-text(PathNode/TextMultilingualField)",
+    testExpressionTranslationWithContext("PathNode/TextMultilingualField/normalize-space(text())",
         "ND-Root", "BT-00-Text-Multilingual");
   }
 
+  @Test
+  void testMultilingualTextFieldReference_AsSequence() {
+    testExpressionTranslationWithContext(
+        "for $t in PathNode/TextMultilingualField/normalize-space(text()) return $t", "ND-Root",
+        "for text:$t in BT-00-Text-Multilingual return $t");
+  }
+
   @Test
   void testMultilingualTextFieldReference_WithLanguagePredicate() {
     testExpressionTranslationWithContext("PathNode/TextMultilingualField[./@languageID = 'eng']/normalize-space(text())",

src/test/java/eu/europa/ted/efx/sdk1/EfxTemplateTranslatorV1Test.java

@@ -377,4 +377,51 @@ void testImplicitFormatting_Dates() {
   void testImplicitFormatting_Times() {
     assertEquals("TEMPLATES:\nlet block01() -> { eval(for $item in PathNode/StartTimeField/xs:time(text()) return format-time($item, '[H01]:[m01] [Z]')) }\nMAIN:\nfor-each(/*).call(block01())", translateTemplate("{ND-Root} ${BT-00-StartTime}"));
   }
+
+  /*** Multilingual fields ***/
+
+  /**
+   * In a view template the value of a multilingual field must be retrieved in the preferred
+   * language, so the reference is wrapped in a call to efx:preferred-language-text().
+   * This is the behaviour that template translation must keep.
+   */
+  @Test
+  void testMultilingualTextField_IsRetrievedInPreferredLanguage() {
+    assertEquals(
+        "TEMPLATES:\nlet block01() -> { eval(efx:preferred-language-text(PathNode/TextMultilingualField)) }\nMAIN:\nfor-each(/*).call(block01())",
+        translateTemplate("{ND-Root} ${BT-00-Text-Multilingual}"));
+  }
+
+  /**
+   * When the reference already pins a language, the value is retrieved as-is: adding the preferred
+   * language call on top would override the language the template author asked for.
+   */
+  @Test
+  void testMultilingualTextField_WithLanguagePredicate_IsRetrievedAsIs() {
+    assertEquals(
+        "TEMPLATES:\nlet block01() -> { eval(PathNode/TextMultilingualField[./@languageID = 'eng']/normalize-space(text())) }\nMAIN:\nfor-each(/*).call(block01())",
+        translateTemplate(
+            "{ND-Root} ${BT-00-Text-Multilingual[BT-00-Text-Multilingual/@languageID == 'eng']}"));
+  }
+
+  /**
+   * A multilingual field used as a sequence goes through its own code path, so it needs its own
+   * coverage.
+   */
+  @Test
+  void testMultilingualTextField_AsSequence_IsRetrievedInPreferredLanguage() {
+    assertEquals(
+        "TEMPLATES:\nlet block01() -> { eval(for $t in efx:preferred-language-text(PathNode/TextMultilingualField) return $t) }\nMAIN:\nfor-each(/*).call(block01())",
+        translateTemplate("{ND-Root} ${for text:$t in BT-00-Text-Multilingual return $t}"));
+  }
+
+  /**
+   * The $value shorthand goes through a template-only code path, so it needs its own coverage.
+   */
+  @Test
+  void testMultilingualTextField_ShorthandValueReference_IsRetrievedInPreferredLanguage() {
+    assertEquals(
+        "TEMPLATES:\nlet block01() -> { eval(efx:preferred-language-text(.)) }\nMAIN:\nfor-each(/*/PathNode/TextMultilingualField).call(block01())",
+        translateTemplate("{BT-00-Text-Multilingual} $value"));
+  }
 }