russell@unturf.com
0a580b313d
Authors: russell@unturf.com · brackishbert@gmail.com · foxhop.net · TimeHexOn.com Patches, unit tests, benchmarks, whitepaper, and outreach briefs. Public domain — no copyright claimed. Use freely.
371 lines
15 KiB
Java
371 lines
15 KiB
Java
/*
|
|
* Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
|
|
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
*
|
|
* This code is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License version 2 only, as
|
|
* published by the Free Software Foundation. Oracle designates this
|
|
* particular file as subject to the "Classpath" exception as provided
|
|
* by Oracle in the LICENSE file that accompanied this code.
|
|
*
|
|
* This code is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
* version 2 for more details (a copy is included in the LICENSE file that
|
|
* accompanied this code).
|
|
*
|
|
* You should have received a copy of the GNU General Public License version
|
|
* 2 along with this work; if not, write to the Free Software Foundation,
|
|
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
*
|
|
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
* or visit www.oracle.com if you need additional information or have any
|
|
* questions.
|
|
*/
|
|
|
|
package build.tools.generatelsrequivmaps;
|
|
|
|
import java.io.BufferedWriter;
|
|
import java.io.IOException;
|
|
import java.nio.file.Files;
|
|
import java.nio.file.Paths;
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.Calendar;
|
|
import java.util.Date;
|
|
import java.util.GregorianCalendar;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
import java.util.Map;
|
|
import java.util.TimeZone;
|
|
import java.util.TreeMap;
|
|
import java.util.regex.Pattern;
|
|
|
|
/**
|
|
* This tool reads the IANA Language Subtag Registry data file downloaded from
|
|
* http://www.iana.org/assignments/language-subtag-registry, which is specified
|
|
* in the command line and generates a .java source file as specified in
|
|
* command line. The generated .java source file contains equivalent language
|
|
* maps. These equivalent language maps are used by LocaleMatcher.java
|
|
* for the locale matching mechanism specified in RFC 4647 "Matching of Language
|
|
* Tags".
|
|
*/
|
|
public class EquivMapsGenerator {
|
|
|
|
public static void main(String[] args) throws Exception {
|
|
int i = 0;
|
|
boolean valid = true;
|
|
if (args.length != 5 && args.length !=3) {
|
|
valid = false;
|
|
} else if (args.length == 5) {
|
|
if ("-jdk-header-template".equals(args[i])) {
|
|
jdkHeaderTemplate = Files.readString(Paths.get(args[++i]));
|
|
i++;
|
|
} else {
|
|
valid = false;
|
|
}
|
|
}
|
|
if (!valid) {
|
|
System.err.println("Usage: java EquivMapsGenerator"
|
|
+ " [-jdk-header-template <file>]"
|
|
+ " language-subtag-registry.txt LocaleEquivalentMaps.java copyrightYear");
|
|
System.exit(1);
|
|
}
|
|
String lsrFile = args[i++];
|
|
String outputFile = args[i++];
|
|
copyrightYear = Integer.parseInt(args[i++]);
|
|
|
|
readLSRfile(lsrFile);
|
|
// Builds the maps from the IANA data
|
|
generateEquivalentMap();
|
|
// Writes the maps out to LocaleEquivalentMaps.java
|
|
generateSourceCode(outputFile);
|
|
}
|
|
|
|
private static String LSRrevisionDate;
|
|
private static int copyrightYear;
|
|
private static String jdkHeaderTemplate;
|
|
private static Map<String, StringBuilder> initialLanguageMap =
|
|
new TreeMap<>();
|
|
private static Map<String, StringBuilder> initialRegionVariantMap =
|
|
new TreeMap<>();
|
|
|
|
private static Map<String, String> sortedLanguageMap1 = new TreeMap<>();
|
|
private static Map<String, String[]> sortedLanguageMap2 = new TreeMap<>();
|
|
private static Map<String, String> sortedRegionVariantMap =
|
|
new TreeMap<>();
|
|
|
|
private static void readLSRfile(String filename) throws Exception {
|
|
String type = null;
|
|
String tag = null;
|
|
String preferred = null;
|
|
String prefix = null;
|
|
|
|
for (String line : Files.readAllLines(Paths.get(filename))) {
|
|
line = line.toLowerCase(Locale.ROOT);
|
|
int index = line.indexOf(' ') + 1;
|
|
if (line.startsWith("file-date:")) {
|
|
LSRrevisionDate = line.substring(index);
|
|
} else if (line.startsWith("type:")) {
|
|
type = line.substring(index);
|
|
} else if (line.startsWith("tag:") || line.startsWith("subtag:")) {
|
|
tag = line.substring(index);
|
|
} else if (line.startsWith("preferred-value:")) {
|
|
preferred = line.substring(index);
|
|
} else if (line.startsWith("prefix:")) {
|
|
prefix = line.substring(index);
|
|
} else if (line.equals("%%")) {
|
|
processDeprecatedData(type, tag, preferred, prefix);
|
|
type = null;
|
|
tag = null;
|
|
preferred = null;
|
|
prefix = null;
|
|
}
|
|
}
|
|
|
|
// Last entry
|
|
processDeprecatedData(type, tag, preferred, prefix);
|
|
}
|
|
|
|
private static void processDeprecatedData(String type,
|
|
String tag,
|
|
String preferred,
|
|
String prefix) {
|
|
StringBuilder sb;
|
|
|
|
if (type == null || tag == null || preferred == null) {
|
|
return;
|
|
}
|
|
|
|
if (type.equals("extlang") && prefix != null) {
|
|
tag = prefix + "-" + tag;
|
|
}
|
|
|
|
if (type.equals("region") || type.equals("variant")) {
|
|
if (!initialRegionVariantMap.containsKey(preferred)) {
|
|
sb = new StringBuilder("-");
|
|
sb.append(preferred);
|
|
sb.append(",-");
|
|
sb.append(tag);
|
|
initialRegionVariantMap.put("-"+preferred, sb);
|
|
} else {
|
|
throw new RuntimeException("New case, need implementation."
|
|
+ " A region/variant subtag \"" + preferred
|
|
+ "\" is registered for more than one subtags.");
|
|
}
|
|
} else { // language, extlang, legacy, and redundant
|
|
if (!initialLanguageMap.containsKey(preferred)) {
|
|
// IANA update 4/13 introduced case where a preferred value
|
|
// can have a preferred value itself.
|
|
// eg: ar-ajp has pref ajp which has pref apc
|
|
boolean foundInOther = false;
|
|
Pattern pattern = Pattern.compile(","+preferred+"(,|$)");
|
|
// Check if current pref exists inside a value for another pref
|
|
List<StringBuilder> doublePrefs = initialLanguageMap
|
|
.values()
|
|
.stream()
|
|
.filter(e -> pattern.matcher(e.toString()).find())
|
|
.toList();
|
|
for (StringBuilder otherPrefVal : doublePrefs) {
|
|
otherPrefVal.append(",");
|
|
otherPrefVal.append(tag);
|
|
foundInOther = true;
|
|
}
|
|
if (!foundInOther) {
|
|
// does not exist in any other pref's values, so add as new entry
|
|
sb = new StringBuilder(preferred);
|
|
sb.append(',');
|
|
sb.append(tag);
|
|
initialLanguageMap.put(preferred, sb);
|
|
}
|
|
} else {
|
|
sb = initialLanguageMap.get(preferred);
|
|
sb.append(',');
|
|
sb.append(tag);
|
|
initialLanguageMap.put(preferred, sb);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void generateEquivalentMap() {
|
|
String[] subtags;
|
|
for (String preferred : initialLanguageMap.keySet()) {
|
|
// There are cases where the same tag may appear in two entries, e.g.,
|
|
// "yue" is defined both as extlang and redundant. Remove the dup.
|
|
subtags = Arrays.stream(initialLanguageMap.get(preferred).toString().split(","))
|
|
.distinct()
|
|
.toList()
|
|
.toArray(new String[0]);
|
|
|
|
if (subtags.length == 2) {
|
|
sortedLanguageMap1.put(subtags[0], subtags[1]);
|
|
sortedLanguageMap1.put(subtags[1], subtags[0]);
|
|
} else if (subtags.length > 2) {
|
|
for (int i = 0; i < subtags.length; i++) {
|
|
sortedLanguageMap2.put(subtags[i], createLangArray(i, subtags));
|
|
}
|
|
} else {
|
|
throw new RuntimeException("New case, need implementation."
|
|
+ " A language subtag \"" + preferred
|
|
+ "\" is registered for more than two subtags. ");
|
|
}
|
|
}
|
|
|
|
for (String preferred : initialRegionVariantMap.keySet()) {
|
|
subtags =
|
|
initialRegionVariantMap.get(preferred).toString().split(",");
|
|
|
|
sortedRegionVariantMap.put(subtags[0], subtags[1]);
|
|
sortedRegionVariantMap.put(subtags[1], subtags[0]);
|
|
}
|
|
|
|
}
|
|
|
|
/* create the array of subtags excluding the subtag at index location */
|
|
private static String[] createLangArray(int index, String[] subtags) {
|
|
List<String> list = new ArrayList<>();
|
|
for (int i = 0; i < subtags.length; i++) {
|
|
if (i != index) {
|
|
list.add(subtags[i]);
|
|
}
|
|
}
|
|
return list.toArray(new String[list.size()]);
|
|
}
|
|
|
|
/**
|
|
* The input lsr data file is in UTF-8, so theoretically for the characters
|
|
* beyond US-ASCII, the generated Java String literals need to be Unicode
|
|
* escaped (\\uXXXX) while writing to a file. But as of now, there is not
|
|
* the case since we don't use "description", "comment" or alike.
|
|
*/
|
|
private static void generateSourceCode(String fileName) {
|
|
try (BufferedWriter writer = Files.newBufferedWriter(
|
|
Paths.get(fileName))) {
|
|
writer.write(getOpenJDKCopyright());
|
|
writer.write("\n");
|
|
writer.write(HEADER_TEXT);
|
|
writer.write(getMapsText());
|
|
writer.write(getLSRText());
|
|
writeEquiv(writer, "singleEquivMap", sortedLanguageMap1);
|
|
writer.newLine();
|
|
writeMultiEquiv(writer);
|
|
writer.newLine();
|
|
writeEquiv(writer, "regionVariantEquivMap", sortedRegionVariantMap);
|
|
writer.write(FOOTER_TEXT);
|
|
} catch (IOException ex) {
|
|
ex.printStackTrace(System.err);
|
|
System.exit(1);
|
|
}
|
|
}
|
|
|
|
private static String getOpenJDKCopyright() {
|
|
return String.format(Locale.US,
|
|
(jdkHeaderTemplate != null ? jdkHeaderTemplate : COPYRIGHT), copyrightYear);
|
|
}
|
|
|
|
private static final String COPYRIGHT =
|
|
"""
|
|
/*
|
|
* Copyright (c) 2012, %d, Oracle and/or its affiliates. All rights reserved.
|
|
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
|
*
|
|
* This code is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License version 2 only, as
|
|
* published by the Free Software Foundation. Oracle designates this
|
|
* particular file as subject to the \"Classpath\" exception as provided
|
|
* by Oracle in the LICENSE file that accompanied this code.
|
|
*
|
|
* This code is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
* version 2 for more details (a copy is included in the LICENSE file that
|
|
* accompanied this code).
|
|
*
|
|
* You should have received a copy of the GNU General Public License version
|
|
* 2 along with this work; if not, write to the Free Software Foundation,
|
|
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
*
|
|
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
|
* or visit www.oracle.com if you need additional information or have any
|
|
* questions.
|
|
*/
|
|
""";
|
|
|
|
private static final String HEADER_TEXT =
|
|
"""
|
|
package sun.util.locale;
|
|
|
|
import java.util.HashMap;
|
|
import java.util.Map;
|
|
|
|
final class LocaleEquivalentMaps {
|
|
|
|
static final Map<String, String> singleEquivMap;
|
|
static final Map<String, String[]> multiEquivsMap;
|
|
static final Map<String, String> regionVariantEquivMap;
|
|
|
|
""";
|
|
|
|
private static final String FOOTER_TEXT =
|
|
"""
|
|
}
|
|
|
|
}
|
|
""";
|
|
|
|
private static String getMapsText() {
|
|
return """
|
|
static {
|
|
singleEquivMap = HashMap.newHashMap(%s);
|
|
multiEquivsMap = HashMap.newHashMap(%s);
|
|
regionVariantEquivMap = HashMap.newHashMap(%s);
|
|
|
|
""".formatted(
|
|
sortedLanguageMap1.size(),
|
|
sortedLanguageMap2.size(),
|
|
sortedRegionVariantMap.size());
|
|
}
|
|
|
|
private static final String getLSRText(){
|
|
return """
|
|
// This is an auto-generated file and should not be manually edited.
|
|
// LSR Revision: %s
|
|
""".formatted(LSRrevisionDate);
|
|
}
|
|
|
|
private static void writeMultiEquiv(BufferedWriter writer) throws IOException {
|
|
for (String key : sortedLanguageMap2.keySet()) {
|
|
String[] values = sortedLanguageMap2.get(key);
|
|
if (values.length >= 2) {
|
|
writer.write(String.format(
|
|
" multiEquivsMap.put(\"%s\", new String[] {%s});"
|
|
, key, generateValuesString(values)));
|
|
writer.newLine();
|
|
}
|
|
}
|
|
}
|
|
|
|
// Use for writing an equivlancy map with one value
|
|
private static void writeEquiv(BufferedWriter writer, String name, Map<String, String> map) throws IOException {
|
|
for (String key : map.keySet()) {
|
|
String value = map.get(key);
|
|
writer.write(String.format(
|
|
" %s.put(\"%s\", \"%s\");"
|
|
, name, key, value));
|
|
writer.newLine();
|
|
}
|
|
}
|
|
|
|
private static String generateValuesString(String[] values) {
|
|
String outputStr = "";
|
|
for (int i = 0; i < values.length; i++) {
|
|
if (i != values.length - 1) {
|
|
outputStr = String.format("%s\"%s\", ", outputStr, values[i]);
|
|
} else {
|
|
outputStr = String.format("%s\"%s\"", outputStr, values[i]);
|
|
}
|
|
|
|
}
|
|
return outputStr;
|
|
}
|
|
}
|