Merge pull request #35877 from vespa-engine/arnej/locale-for-linguistics

Improve locale handling in linguistics code

Author: arnej27959

linguistics-components/src/main/java/com/yahoo/language/huggingface/ModelInfo.java

@@ -2,6 +2,7 @@
 
 package com.yahoo.language.huggingface;
 
+import com.yahoo.text.Text;
 import java.util.Arrays;
 
 /**
@@ -21,7 +22,7 @@ public static TruncationStrategy fromString(String v) {
             else if ("false".equals(v)) return DO_NOT_TRUNCATE;
             return Arrays.stream(values())
                     .filter(s -> s.name().equalsIgnoreCase(v))
-                    .findAny().orElseThrow(() -> new IllegalArgumentException("Invalid strategy '%s'".formatted(v)));
+                    .findAny().orElseThrow(() -> new IllegalArgumentException(Text.format("Invalid strategy '%s'", v)));
         }
     }
 
@@ -35,7 +36,7 @@ public static PaddingStrategy fromString(String v) {
             else if ("false".equals(v)) return DO_NOT_PAD;
             return Arrays.stream(values())
                     .filter(s -> s.name().equalsIgnoreCase(v))
-                    .findAny().orElseThrow(() -> new IllegalArgumentException("Invalid strategy '%s'".formatted(v)));
+                    .findAny().orElseThrow(() -> new IllegalArgumentException(Text.format("Invalid strategy '%s'", v)));
         }
     }
 }

linguistics-components/src/main/java/com/yahoo/language/wordpiece/Model.java

@@ -18,6 +18,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.TreeMap;
@@ -67,7 +68,7 @@ class Model {
 
     List<Integer> embed(String text, Tokenizer tokenizer, LinguisticsParameters parameters) {
         List<Integer> ids = new ArrayList<>();
-        text = text.toLowerCase();
+        text = text.toLowerCase(Locale.ROOT);
         for (Token t : tokenizer.tokenize(text, parameters)) {
             String originalToken = t.getTokenString();
             String candidate = originalToken;

linguistics-components/src/test/java/com/yahoo/language/huggingface/HuggingFaceTokenizerTest.java

@@ -3,6 +3,7 @@
 package com.yahoo.language.huggingface;
 
 import com.yahoo.language.tools.EmbedderTester;
+import com.yahoo.text.Text;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
@@ -129,7 +130,7 @@ private static void assertMaxLengthRespected(int maxLength, Encoding encoding) {
     }
 
     private static Path decompressModelFile(Path tmp, String model) throws IOException {
-        var source = Paths.get("src/test/models/huggingface/%s.json.gz".formatted(model));
+        var source = Paths.get(Text.format("src/test/models/huggingface/%s.json.gz", model));
         Path destination = tmp.resolve(source.getFileName().toString().replace(".gz", ""));
         try (InputStream in = new GZIPInputStream(Files.newInputStream(source));
              OutputStream out = Files.newOutputStream(destination, StandardOpenOption.CREATE)) {

linguistics/src/main/java/com/yahoo/language/significance/impl/DefaultSignificanceModelRegistry.java

@@ -7,6 +7,7 @@
 import com.yahoo.language.significance.SignificanceModel;
 import com.yahoo.language.significance.SignificanceModelRegistry;
 import com.yahoo.search.significance.config.SignificanceConfig;
+import com.yahoo.text.Text;
 import io.airlift.compress.zstd.ZstdInputStream;
 
 import java.io.IOException;
@@ -61,12 +62,12 @@ public void addModel(Path path) {
             for (var pair : file.languages().entrySet()) {
 
                 var languagesStr = pair.getKey();
-                log.fine(() -> "Found model for languages '%s'".formatted(languagesStr));
+                log.fine(() -> Text.format("Found model for languages '%s'", languagesStr));
                 String[] languageTags = languagesStr.split(",");
 
                 for (var languageTag : languageTags) {
                     var language = Language.fromLanguageTag(languageTag);
-                    log.fine(() -> "Adding model for language %s with id %s".formatted(language, file.id()));
+                    log.fine(() -> Text.format("Adding model for language %s with id %s", language, file.id()));
                     this.models.put(language, new DefaultSignificanceModel(pair.getValue(), file.id()));
                 }
             }

linguistics/src/test/java/com/yahoo/language/process/GenWordCharsBitVector.java

@@ -2,6 +2,7 @@
 
 package com.yahoo.language.process;
 import java.nio.charset.StandardCharsets;
+import java.util.Locale;
 
 // program to generate code tables used by Fast_UnicodeUtil::IsWordChar()
 
@@ -36,7 +37,7 @@ static String genTable() {
                 while (s.length() < nextpos) s.append(' ');
                 nextpos += 18;
                 s.append("0x");
-                String hex = Long.toHexString(val).toUpperCase();
+                String hex = Long.toHexString(val).toUpperCase(Locale.ROOT);
                 while (s.length() + hex.length() < nextpos) s.append('0');
                 s.append(hex);
                 if (codepoint + 1 < maxCodeBlocks * 0x100) s.append(",");

lucene-linguistics/src/test/java/com/yahoo/language/lucene/LuceneTokenizerTest.java

@@ -35,6 +35,7 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 
@@ -386,7 +387,7 @@ protected DuplicateTokenFilter(TokenStream input) {
         public boolean incrementToken() throws IOException {
             if (emitUppercase) {
                 restoreState(savedState);
-                String value = term.toString().toUpperCase();
+                String value = term.toString().toUpperCase(Locale.ROOT);
                 term.setEmpty();
                 term.append(value);
                 position.setPositionIncrement(0); // same position