Skip to content

Commit de29f63

Browse files
arnej27959arnej27959
authored
Merge pull request #35904 from vespa-engine/arnej/locale-schemals
Standardize locale in schema-language-server
2 parents fb2efc0 + 4c3e5ec commit de29f63

14 files changed

Lines changed: 52 additions & 36 deletions

File tree

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaLanguageServer.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import java.io.PrintStream;
1111
import java.net.URL;
1212
import java.net.URLDecoder;
13+
import java.nio.charset.StandardCharsets;
1314
import java.nio.file.Files;
1415
import java.nio.file.Path;
1516
import java.nio.file.Paths;
@@ -184,9 +185,9 @@ public void connect(LanguageClient languageClient) {
184185
setupDocumentation(docPath);
185186
} catch (IOException ioex) {
186187
this.logger.error("Failed to set up documentation. Error: " + ioex.getMessage());
187-
try (ByteArrayOutputStream os = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(os)) {
188+
try (ByteArrayOutputStream os = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(os, true, StandardCharsets.UTF_8)) {
188189
ioex.printStackTrace(ps);
189-
logger.error(os.toString());
190+
logger.error(os.toString(StandardCharsets.UTF_8));
190191
} catch (IOException bruh) {
191192
logger.error("Error inside error " + bruh.getMessage());
192193
}
@@ -278,9 +279,9 @@ public boolean accept(File pathname) {
278279
}
279280
Files.createDirectories(destination.getParent());
280281
try (InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(entry.getName())) {
281-
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
282+
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
282283
String content = reader.lines().collect(Collectors.joining(System.lineSeparator()));
283-
Files.write(destination, content.getBytes(), StandardOpenOption.CREATE);
284+
Files.write(destination, content.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
284285
}
285286
}
286287
}

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaTextDocumentService.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.io.ByteArrayOutputStream;
44
import java.io.PrintStream;
5+
import java.nio.charset.StandardCharsets;
56
import java.util.ArrayList;
67
import java.util.List;
78
import java.util.concurrent.CancellationException;
@@ -97,13 +98,13 @@ public CompletableFuture<Either<List<CompletionItem>, CompletionList>> completio
9798
try {
9899

99100
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
100-
PrintStream errorLogger = new PrintStream(outputStream);
101+
PrintStream errorLogger = new PrintStream(outputStream, true, StandardCharsets.UTF_8);
101102
Either<List<CompletionItem>, CompletionList> result =
102103
Either.forLeft(CommonCompletion.getCompletionItems(eventContextCreator.createContext(completionParams), errorLogger));
103104

104105
if (outputStream.size() > 0) {
105-
schemaMessageHandler.logMessage(MessageType.Error,
106-
"Completion failed with errors: " + outputStream.toString()
106+
schemaMessageHandler.logMessage(MessageType.Error,
107+
"Completion failed with errors: " + outputStream.toString(StandardCharsets.UTF_8)
107108
);
108109
}
109110
return result;

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/ClientLogger.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.io.ByteArrayOutputStream;
44
import java.io.PrintStream;
5+
import java.nio.charset.StandardCharsets;
56

67
import org.eclipse.lsp4j.MessageType;
78

@@ -26,10 +27,10 @@ public void info(Object message) {
2627

2728
public static String errorToString(Exception ex) {
2829
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
29-
PrintStream logger = new PrintStream(outputStream);
30+
PrintStream logger = new PrintStream(outputStream, true, StandardCharsets.UTF_8);
3031
ex.printStackTrace(logger);
3132

32-
return outputStream.toString();
33+
return outputStream.toString(StandardCharsets.UTF_8);
3334
}
3435

3536
public void error(Exception ex) {

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/FileUtils.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import java.util.stream.Collectors;
2222

2323
import com.yahoo.io.IOUtils;
24+
import com.yahoo.text.Utf8;
2425

2526
public class FileUtils {
2627
public static String fileNameFromPath(String path) {
@@ -42,7 +43,7 @@ public static String schemaNameFromPath(String path) {
4243

4344
public static String readFromURI(String fileURI) throws IOException {
4445
File file = new File(URI.create(fileURI));
45-
return IOUtils.readAll(new FileReader(file));
46+
return IOUtils.readAll(Utf8.createReader(file));
4647
}
4748

4849
public static List<String> findSchemaFiles(String workspaceFolderUri, ClientLogger logger) {

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/documentation/FetchDocumentation.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@
33
import ai.vespa.schemals.common.FileUtils;
44

55
import java.io.IOException;
6+
import java.nio.charset.StandardCharsets;
67
import java.nio.file.Files;
78
import java.nio.file.Path;
89
import java.nio.file.Paths;
910
import java.nio.file.StandardOpenOption;
1011
import java.util.HashMap;
1112
import java.util.List;
13+
import java.util.Locale;
1214
import java.util.Map;
1315

1416
/**
@@ -78,7 +80,7 @@ public static void fetchServicesDocs(Path targetPath) throws IOException {
7880

7981
for (var entry : markdownContent.entrySet()) {
8082
if (entry.getKey().contains("/")) continue;
81-
String fileName = entry.getKey().toLowerCase();
83+
String fileName = entry.getKey().toLowerCase(Locale.ROOT);
8284
fileName = FileUtils.sanitizeFileName(fileName);
8385
writeMarkdown(writePath.resolve(fileName + ".md"), entry.getValue());
8486
}
@@ -87,11 +89,11 @@ public static void fetchServicesDocs(Path targetPath) throws IOException {
8789

8890

8991
private static void writeMarkdown(Path writePath, String markdown) throws IOException {
90-
Files.write(writePath, markdown.getBytes(), StandardOpenOption.CREATE);
92+
Files.write(writePath, markdown.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE);
9193
}
9294

9395
private static String convertToToken(String h2Id) {
94-
return h2Id.toUpperCase().replaceAll("-", "_");
96+
return h2Id.toUpperCase(Locale.ROOT).replaceAll("-", "_");
9597
}
9698

9799
// Runs during build

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/index/SchemaIndex.java‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import ai.vespa.schemals.parser.ast.rootSchema;
3030
import ai.vespa.schemals.parser.ast.structDefinitionElm;
3131
import ai.vespa.schemals.parser.ast.structFieldDefinition;
32+
import com.yahoo.text.Text;
3233

3334
public class SchemaIndex {
3435
public static final HashMap<Class<?>, SymbolType> IDENTIFIER_TYPE_MAP = new HashMap<>() {{
@@ -69,7 +70,7 @@ public class SchemaIndex {
6970

7071
// This is an inheritance graph, even though it doesn't model *inheritance* per se.
7172
private InheritanceGraph<Symbol> documentReferenceGraph;
72-
73+
7374
public SchemaIndex(ClientLogger logger) {
7475
this.logger = logger;
7576
this.documentInheritanceGraph = new InheritanceGraph<>();
@@ -221,7 +222,7 @@ public List<Symbol> findSymbols(Symbol scope, SymbolType type, String shortIdent
221222
// Special case for schema and document because a schema can sometimes refer to a document and vice versa
222223
if (type == SymbolType.SCHEMA || type == SymbolType.DOCUMENT) {
223224
SymbolType firstCheck = (type == SymbolType.SCHEMA ? SymbolType.SCHEMA : SymbolType.DOCUMENT);
224-
List<Symbol> schemaDefinitions =
225+
List<Symbol> schemaDefinitions =
225226
symbolDefinitions.get(firstCheck)
226227
.stream()
227228
.filter(symbolDefinition -> symbolDefinition.getShortIdentifier().equals(shortIdentifier))
@@ -249,7 +250,7 @@ public List<Symbol> findSymbols(Symbol scope, SymbolType type, String shortIdent
249250
}
250251

251252
/*
252-
* Given a scope, type and short identifier, find definitions defined inside the actual scope.
253+
* Given a scope, type and short identifier, find definitions defined inside the actual scope.
253254
* Will not search inheritance graphs.
254255
*/
255256
private Optional<Symbol> findSymbolInConcreteScope(Symbol scope, SymbolType type, String shortIdentifier) {
@@ -309,24 +310,24 @@ public List<Symbol> findSymbolsInScope(Symbol reference) {
309310
public boolean isInScope(Symbol symbol, Symbol scope) {
310311
if (scope == null) return true; // lets say every symbol is in the empty scope
311312
if (scope.getType() == SymbolType.RANK_PROFILE) {
312-
return !rankProfileInheritanceGraph.findFirstMatches(scope,
313+
return !rankProfileInheritanceGraph.findFirstMatches(scope,
313314
rankProfileDefinitionSymbol -> {
314315
if (rankProfileDefinitionSymbol.equals(symbol.getScope())) {
315316
return Boolean.valueOf(true);
316317
}
317318
return null;
318319
}).isEmpty();
319320
} else if (scope.getType() == SymbolType.STRUCT) {
320-
return !structInheritanceGraph.findFirstMatches(scope,
321+
return !structInheritanceGraph.findFirstMatches(scope,
321322
structDefinitionSymbol -> {
322323
if (structDefinitionSymbol.equals(symbol.getScope())) {
323324
return Boolean.valueOf(true);
324325
}
325326
return null;
326327
}).isEmpty();
327-
} else if ((symbol.getScope() == null || symbol.getScope().getType() == SymbolType.SCHEMA || symbol.getScope().getType() == SymbolType.DOCUMENT) &&
328+
} else if ((symbol.getScope() == null || symbol.getScope().getType() == SymbolType.SCHEMA || symbol.getScope().getType() == SymbolType.DOCUMENT) &&
328329
(scope.getType() == SymbolType.SCHEMA || scope.getType() == SymbolType.DOCUMENT)) {
329-
return !documentInheritanceGraph.findFirstMatches(scope.getFileURI(),
330+
return !documentInheritanceGraph.findFirstMatches(scope.getFileURI(),
330331
ancestorURI -> {
331332
if (symbol.getFileURI().equals(ancestorURI)) {
332333
return Boolean.valueOf(true);
@@ -341,8 +342,8 @@ public boolean isInScope(Symbol symbol, Symbol scope) {
341342

342343
/**
343344
* Retrieves the definition of a symbol from a map.
344-
* We have three versions of this, because definitions can reference other definitions.
345-
* For example, struct-field is both a definition (you want to jump there if you go-to-definition on field.structfield somewhere else),
345+
* We have three versions of this, because definitions can reference other definitions.
346+
* For example, struct-field is both a definition (you want to jump there if you go-to-definition on field.structfield somewhere else),
346347
* and a reference (to the field inside the struct).
347348
* {@link getSymbolDefinition} returns identity if you supply a definition.
348349
* {@link getNextSymbolDefinition} always tries to interpret the argument as a reference (jumping one step even if argument is a definition).
@@ -592,7 +593,7 @@ public void dumpIndex() {
592593

593594
logger.info("\n === REFERENCES TO DEFINITIONS ===");
594595
for (var entry : definitionOfReference.entrySet()) {
595-
String toPrint = String.format("%-50s -> %s", entry.getKey(), entry.getValue());
596+
String toPrint = Text.format("%-50s -> %s", entry.getKey(), entry.getValue());
596597
logger.info(toPrint);
597598
}
598599

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/command/commandtypes/RunVespaQuery.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@
33
import java.io.BufferedReader;
44
import java.io.IOException;
55
import java.io.InputStreamReader;
6+
import java.nio.charset.StandardCharsets;
67
import java.nio.file.Path;
78
import java.nio.file.Paths;
89
import java.time.ZonedDateTime;
910
import java.time.format.DateTimeFormatter;
1011
import java.util.List;
12+
import java.util.Locale;
1113
import java.util.concurrent.CompletableFuture;
1214

1315
import org.eclipse.lsp4j.ApplyWorkspaceEditParams;
@@ -55,7 +57,7 @@ public Object execute(EventExecuteCommandContext context) {
5557

5658
runVespaQuery(queryCommand, context.logger).thenAccept(result -> {
5759
if (!result.success()) {
58-
if (result.result().toLowerCase().contains("cannot run program")) {
60+
if (result.result().toLowerCase(Locale.ROOT).contains("cannot run program")) {
5961
context.messageHandler.sendMessage(MessageType.Error, "Could not find vespa CLI. Make sure vespa CLI is installed and added to path. Download vespa CLI here: https://docs.vespa.ai/en/clients/vespa-cli.html");
6062
return;
6163
}
@@ -102,7 +104,7 @@ private record QueryResult(boolean success, String result) {};
102104

103105
private CompletableFuture<QueryResult> runVespaQuery(String query, ClientLogger logger) {
104106

105-
boolean isWindows = System.getProperty("os.name").toLowerCase().startsWith("windows");
107+
boolean isWindows = System.getProperty("os.name").toLowerCase(Locale.ROOT).startsWith("windows");
106108

107109
ProcessBuilder builder = new ProcessBuilder();
108110

@@ -117,7 +119,7 @@ private CompletableFuture<QueryResult> runVespaQuery(String query, ClientLogger
117119

118120
Process process = builder.start();
119121

120-
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
122+
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8));
121123

122124
String line;
123125
StringBuilder output = new StringBuilder();
@@ -131,7 +133,7 @@ private CompletableFuture<QueryResult> runVespaQuery(String query, ClientLogger
131133
return new QueryResult(true, output.toString());
132134
}
133135

134-
BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream()));
136+
BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream(), StandardCharsets.UTF_8));
135137
StringBuilder error = new StringBuilder();
136138
while ((line = errorReader.readLine()) != null) {
137139
error.append(line).append("\n");

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/completion/provider/BodyKeywordCompletion.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import java.nio.file.Path;
44
import java.util.HashMap;
55
import java.util.List;
6+
import java.util.Locale;
67
import java.util.Map;
78
import java.util.Optional;
89

@@ -231,7 +232,7 @@ public class BodyKeywordCompletion implements CompletionProvider {
231232
// compute docs
232233
for (var entry : this.entrySet()) {
233234
for (CompletionItem item : entry.getValue()) {
234-
String markdownKey = item.getLabel().toUpperCase().replaceAll("-", "_");
235+
String markdownKey = item.getLabel().toUpperCase(Locale.ROOT).replaceAll("-", "_");
235236
Optional<Hover> hover = SchemaHover.getFileHoverInformation(schemaHoverPath, markdownKey, new Range());
236237
if (hover.isPresent() && hover.get().getContents().isRight()) {
237238
item.setDocumentation(hover.get().getContents().getRight());

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/hover/SchemaHover.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import java.util.HashMap;
77
import java.util.LinkedList;
88
import java.util.List;
9+
import java.util.Locale;
910
import java.util.Map;
1011
import java.util.Optional;
1112

@@ -149,7 +150,7 @@ private static Hover getFieldHover(Node node, EventPositionContext context) {
149150
for (int i = 0; i < indexingTypes.length; ++i) {
150151
if (i == 0) hoverText += "\n\t";
151152
else hoverText += " | ";
152-
hoverText += indexingTypes[i].toString().toLowerCase();
153+
hoverText += indexingTypes[i].toString().toLowerCase(Locale.ROOT);
153154
}
154155

155156
String fieldDefinitionDocumentContent = context.scheduler.getDocument(fieldDefinitionSymbol.get().getFileURI()).getCurrentContent();

‎integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/completion/provider/GroupingExpressionProvider.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.util.Arrays;
44
import java.util.List;
5+
import java.util.Locale;
56

67
import org.eclipse.lsp4j.CompletionItem;
78
import org.eclipse.lsp4j.Position;
@@ -27,13 +28,13 @@ public class GroupingExpressionProvider implements CompletionProvider {
2728

2829

2930
private final static List<String> mathFunctions = Arrays.stream(MathFunctions.Function.values())
30-
.map(func -> func.name().toLowerCase())
31+
.map(func -> func.name().toLowerCase(Locale.ROOT))
3132
.toList();
3233

3334
private final static List<String> timeFunctions = Arrays.stream(TimeFunctions.Type.values())
3435
.map(func -> func.name()
3536
.replaceAll("_", "")
36-
.toLowerCase())
37+
.toLowerCase(Locale.ROOT))
3738
.toList();
3839

3940
private final static List<CompletionItem> expressionCompletions = List.of(

0 commit comments

Comments
 (0)
⚡