Merge pull request #35869 from vespa-engine/arnej/locale-config-application-package

bratseth · web-flow · commit ae982d58a4bf · 2026-02-12T21:19:52.000+01:00
Ensure UTF-8 in config application package
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java b/config-application-package/src/main/java/com/yahoo/config/application/ConfigDefinitionDir.java
@@ -8,6 +8,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.List;
+import java.nio.charset.StandardCharsets;
 
 /**
  * A @{link ConfigDefinitionDir} contains a set of config definitions. New definitions may be added,
@@ -41,7 +42,7 @@ private void checkAndCopyUserDefs(Bundle bundle, List<Bundle> bundlesAdded) thro
                         " (got " + outFile.getAbsolutePath() + ")");
             }
             OutputStream out = new FileOutputStream(outFile);
-            out.write(def.contents.getBytes());
+            out.write(def.contents.getBytes(StandardCharsets.UTF_8));
             out.close();
         }
     }
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java
@@ -7,6 +7,7 @@
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Tags;
+import com.yahoo.text.Utf8;
 import com.yahoo.text.XML;
 import org.w3c.dom.Document;
 import org.xml.sax.InputSource;
@@ -51,7 +52,7 @@ public XmlPreProcessor(File applicationDir,
                            RegionName region,
                            CloudName cloud,
                            Tags tags) throws IOException {
-        this(applicationDir, new FileReader(xmlInput), instance, environment, region, cloud, tags);
+        this(applicationDir, Utf8.createReader(xmlInput), instance, environment, region, cloud, tags);
     }
 
     public XmlPreProcessor(File applicationDir,
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java
@@ -83,9 +83,8 @@ public static boolean deleteFile(File path) {
 
     @Override
     public Reader createReader() throws FileNotFoundException {
-        return new FileReader(file);
+        return Utf8.createReader(file);
     }
-
     @Override
     public InputStream createInputStream() throws FileNotFoundException {
         return new FileInputStream(file);
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java
@@ -21,6 +21,7 @@
 import com.yahoo.io.IOUtils;
 import com.yahoo.io.reader.NamedReader;
 import com.yahoo.path.Path;
+import com.yahoo.text.Utf8;
 import com.yahoo.vespa.config.ConfigDefinition;
 import com.yahoo.vespa.config.ConfigDefinitionBuilder;
 import com.yahoo.vespa.config.ConfigDefinitionKey;
@@ -145,7 +146,7 @@ private List<NamedReader> getFiles(Path relativePath, String namePrefix, String
                             readers.addAll(getFiles(relativePath.append(file.getName()), namePrefix + "/" + file.getName(), suffix, recurse));
                     } else {
                         if (suffix == null || file.getName().endsWith(suffix))
-                            readers.add(new NamedReader(file.getName(), new FileReader(file)));
+                            readers.add(new NamedReader(file.getName(), Utf8.createReader(file)));
                     }
                 }
             }
@@ -175,7 +176,7 @@ public Reader getHosts() {
         try {
             File hostsFile = applicationFile(HOSTS);
             if (!hostsFile.exists()) return null;
-            return new FileReader(hostsFile);
+            return Utf8.createReader(hostsFile);
         } catch (Exception e) {
             throw new IllegalArgumentException(e);
         }
@@ -238,7 +239,7 @@ public Collection<NamedReader> getSchemas() {
         Set<NamedReader> ret = new LinkedHashSet<>();
         try {
             for (File f : getSchemaFiles()) {
-                ret.add(new NamedReader(f.getName(), new FileReader(f)));
+                ret.add(new NamedReader(f.getName(), Utf8.createReader(f)));
             }
         } catch (Exception e) {
             throw new IllegalArgumentException("Couldn't get schema contents.", e);
@@ -254,7 +255,7 @@ public Collection<NamedReader> getSchemas() {
      */
     private Reader retrieveConfigDefReaderFromThis(File defPath) {
         try {
-            return new NamedReader(defPath.getPath(), new FileReader(defPath));
+            return new NamedReader(defPath.getPath(), Utf8.createReader(defPath));
         } catch (IOException e) {
             throw new IllegalArgumentException("Could not read config definition file '" + defPath + "'", e);
         }
@@ -335,8 +336,7 @@ public String getUnparsedContent() {
     @Override
     public Reader getServices() {
         try {
-            return new FileReader(applicationFile(SERVICES).getPath());
-        } catch (Exception e) {
+            return Utf8.createReader(applicationFile(SERVICES).getPath());        } catch (Exception e) {
             throw new IllegalArgumentException(e);
         }
     }
@@ -403,7 +403,7 @@ private ApplicationMetaData readMetaData(File appDir) {
         if ( ! metaFile.exists()) {
             return defaultMetaData;
         }
-        try (FileReader reader = new FileReader(metaFile)) {
+        try (Reader reader = Utf8.createReader(metaFile)) {
             return ApplicationMetaData.fromJsonString(IOUtils.readAll(reader));
         } catch (Exception e) {
             // Not a big deal, return default
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java
@@ -3,6 +3,7 @@
 
 import com.yahoo.component.Version;
 import com.yahoo.io.IOUtils;
+import com.yahoo.text.Text;
 import org.osgi.framework.Bundle;
 import org.xml.sax.SAXException;
 import java.io.File;
@@ -16,6 +17,8 @@
 import java.util.jar.JarFile;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
 
 import static com.yahoo.vespa.defaults.Defaults.getDefaults;
 import static java.nio.file.Files.createTempDirectory;
@@ -123,7 +126,7 @@ private File saveSchemasFromJar(File tmpBase, Version vespaVersion) throws IOExc
                     schemasFound = true;
                     copySchemas(schemaPath, tmpDir);
                 } else {
-                    log.log(Level.FINE, () -> String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle.getVersion()));
+                    log.log(Level.FINE, () -> Text.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle.getVersion()));
                     for (Enumeration<URL> entries = bundle.findEntries("schema", "*.rnc", true); entries.hasMoreElements(); ) {
                         URL url = entries.nextElement();
                         writeContentsToFile(tmpDir, url.getFile(), url.openStream());
@@ -154,7 +157,7 @@ private static void copySchemas(File from, File to) throws IOException {
     }
 
     private static void writeContentsToFile(File outDir, String outFile, InputStream inputStream) throws IOException {
-        String contents = IOUtils.readAll(new InputStreamReader(inputStream));
+        String contents = IOUtils.readAll(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
 
         // Validate and sanitize the output path
         File out = new File(outDir, outFile).getCanonicalFile();
diff --git a/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/OverrideProcessorTest.java
@@ -410,8 +410,8 @@ public void testSearchersReferredWithIdRefPerCloud() throws TransformerException
                         </container>
                         "</services>""";
 
-        assertOverride(input, "aws", expected.formatted("AwsSearcher"));
-        assertOverride(input, "gcp", expected.formatted("GcpSearcher"));
+        assertOverride(input, "aws", com.yahoo.text.Text.format(expected, "AwsSearcher"));
+        assertOverride(input, "gcp", com.yahoo.text.Text.format(expected, "GcpSearcher"));
     }
 
     private void assertOverride(Environment environment, RegionName region, String expected) throws TransformerException {
diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java
@@ -7,6 +7,7 @@
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.io.IOUtils;
+import com.yahoo.text.Utf8;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -96,7 +97,7 @@ public void testDeploymentXml() throws IOException {
         assertTrue(app.getDeployment().isPresent());
         assertFalse(app.getDeploymentSpec().isEmpty());
         assertFalse(app.getMajorVersion().isPresent());
-        assertEquals(IOUtils.readAll(app.getDeployment().get()), IOUtils.readAll(new FileReader(deployment)));
+        assertEquals(IOUtils.readAll(app.getDeployment().get()), IOUtils.readAll(Utf8.createReader(deployment)));
     }
 
     @Test
@@ -108,7 +109,7 @@ public void testPinningMajorVersion() throws IOException {
         assertTrue(app.getDeployment().isPresent());
         assertTrue(app.getMajorVersion().isPresent());
         assertEquals(6, (int)app.getMajorVersion().get());
-        assertEquals(IOUtils.readAll(app.getDeployment().get()), IOUtils.readAll(new FileReader(deployment)));
+        assertEquals(IOUtils.readAll(app.getDeployment().get()), IOUtils.readAll(Utf8.createReader(deployment)));
     }
 
     @Test

Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,7 @@`
`8`	`8`	`import java.io.IOException;`
`9`	`9`	`import java.io.OutputStream;`
`10`	`10`	`import java.util.List;`
	`11`	`+import java.nio.charset.StandardCharsets;`
`11`	`12`
`12`	`13`	`/**`
`13`	`14`	`* A @{link ConfigDefinitionDir} contains a set of config definitions. New definitions may be added,`
`@@ -41,7 +42,7 @@ private void checkAndCopyUserDefs(Bundle bundle, List<Bundle> bundlesAdded) thro`
`41`	`42`	`" (got " + outFile.getAbsolutePath() + ")");`
`42`	`43`	`}`
`43`	`44`	`OutputStream out = new FileOutputStream(outFile);`
`44`		`- out.write(def.contents.getBytes());`
	`45`	`+ out.write(def.contents.getBytes(StandardCharsets.UTF_8));`
`45`	`46`	`out.close();`
`46`	`47`	`}`
`47`	`48`	`}`
Original file line number	Diff line number	Diff line change
`@@ -410,8 +410,8 @@ public void testSearchersReferredWithIdRefPerCloud() throws TransformerException`
`410`	`410`	`</container>`
`411`	`411`	`"</services>""";`
`412`	`412`
`413`		`- assertOverride(input, "aws", expected.formatted("AwsSearcher"));`
`414`		`- assertOverride(input, "gcp", expected.formatted("GcpSearcher"));`
	`413`	`+ assertOverride(input, "aws", com.yahoo.text.Text.format(expected, "AwsSearcher"));`
	`414`	`+ assertOverride(input, "gcp", com.yahoo.text.Text.format(expected, "GcpSearcher"));`
`415`	`415`	`}`
`416`	`416`
`417`	`417`	`private void assertOverride(Environment environment, RegionName region, String expected) throws TransformerException {`