engineering/java-topology
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

undefect. CWE-407 — 63 sites patched across 27 ecosystems

Browse source 
Authors: russell@unturf.com · brackishbert@gmail.com · foxhop.net · TimeHexOn.com

Patches, unit tests, benchmarks, whitepaper, and outreach briefs.
Public domain — no copyright claimed. Use freely.
This commit is contained in:
russell@unturf.com 2026-03-26 17:11:57 -04:00
commit 0a580b313d
70422 changed files with 17213626 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

527
test/jdk/java/text/Normalizer/ConformanceTest.java Normal file
View file

@ -0,0 +1,527 @@
/*
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
* @test
* @bug 4221795 6565620 6959267 7070436 7198195 8032446 8174270 8221431 8239383
* @summary Confirm Normalizer's fundamental behavior
* @library /lib/testlibrary/java/lang
* @modules java.base/sun.text java.base/jdk.internal.icu.text
* @compile -XDignore.symbol.file ConformanceTest.java
* @run main/timeout=3000 ConformanceTest
*/
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.util.BitSet;
import java.util.StringTokenizer;
import jdk.internal.icu.text.NormalizerBase;
/*
* Conformance test for java.text.Normalizer and sun.text.Normalizer.
*/
public class ConformanceTest {
//
// Options to be used with sun.text.Normalizer
//
/*
* Default Unicode 3.2.0 normalization. (Provided for IDNA/StringPrep)
*
* - Without Corrigendum 4 fix
* (Different from ICU4J 3.2's Normalizer.)
* - Without Public Review Issue #29 fix
* (Different from ICU4J 3.2's Normalizer.)
*/
private static final int UNICODE_3_2_0 = sun.text.Normalizer.UNICODE_3_2;
/*
* Original Unicode 3.2.0 normalization. (Provided for testing only)
*
* - With Corrigendum 4 fix
* - With Public Revilew Issue #29 fix
*/
private static final int UNICODE_3_2_0_ORIGINAL =
NormalizerBase.UNICODE_3_2;
/*
* Default normalization. In JDK 6,
* - Unicode 4.0.0
* - With Corrigendum 4 fix
* - Without Public Review Issue #29 fix
*
* In JDK 7,
* - Unicode 5.1.0
* (Different from ICU4J 3.2's Normalizer.)
* - With Corrigendum 4 fix
* - With Public Review Issue #29 fix
*
* In JDK 8,
* - Unicode 6.1.0
* - With Corrigendum 4 fix
* - With Public Review Issue #29 fix
*
* When we support Unicode 4.1.0 or later, we need to do normalization
* with Public Review Issue #29 fix. For more details of PRI #29, see
* http://unicode.org/review/pr-29.html .
*/
private static final int UNICODE_LATEST = NormalizerBase.UNICODE_LATEST;
//
// Conformance test datafiles
//
/*
* Conformance test datafile for Unicode 3.2.0 with Corrigendum4
* corrections.
* This testdata is for sun.text.Normalize(UNICODE_3_2)
*
* This is NOT an original Conformace test data. Some inconvenient test
* cases are commented out. About corrigendum 4, please refer
* http://www.unicode.org/review/resolved-pri.html#pri29
*
*/
static final String DATA_3_2_0_CORRIGENDUM =
"NormalizationTest-3.2.0.Corrigendum4.txt";
/*
* Conformance test datafile for Unicode 3.2.0 without Corrigendum4
* corrections. This is the original Conformace test data.
*
* This testdata is for sun.text.Normalize(UNICODE_3_2_IDNA)
*/
static final String DATA_3_2_0 = "NormalizationTest-3.2.0.txt";
/*
* Conformance test datafile for the latest Unicode which is supported
* by J2SE.
* Unicode 4.0.0 is the latest version in JDK 5.0 and JDK 6. Unicode 5.1.0
* in JDK 7, and 6.1.0 in JDK 8. This Unicode can be used via both
* java.text.Normalizer and sun.text.Normalizer.
*
* This testdata is for sun.text.Normalize(UNICODE_LATEST)
*/
static final String DATA_LATEST = "NormalizationTest.txt";
/*
* Conformance test datafile in ICU4J 3.2.
*/
static final String DATA_ICU = "ICUNormalizationTest.txt";
/*
* Decorder
*/
static final CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder();
/*
* List to pick up characters which are not listed in Part1
*/
static BitSet charList = new BitSet(Character.MAX_CODE_POINT+1);
/*
* Shortcuts
*/
private static final java.text.Normalizer.Form NFC =
java.text.Normalizer.Form.NFC;
private static final java.text.Normalizer.Form NFD =
java.text.Normalizer.Form.NFD;
private static final java.text.Normalizer.Form NFKC =
java.text.Normalizer.Form.NFKC;
private static final java.text.Normalizer.Form NFKD =
java.text.Normalizer.Form.NFKD;
static final java.text.Normalizer.Form[] forms = {NFC, NFD, NFKC, NFKD};
static TestNormalizer normalizer;
public static void main(String[] args) throws Exception {
ConformanceTest ct = new ConformanceTest();
ct.test();
}
void test() throws Exception {
normalizer = new testJavaNormalizer();
test(DATA_LATEST, UNICODE_LATEST);
normalizer = new testSunNormalizer();
test(DATA_3_2_0_CORRIGENDUM, UNICODE_3_2_0);
test(DATA_LATEST, UNICODE_LATEST);
test(DATA_ICU, UNICODE_LATEST);
/* Unconformity test */
// test(DATA_3_2_0, UNICODE_LATEST);
// test(DATA_LATEST, UNICODE_3_2_0);
}
/*
* Main routine of conformance test
*/
private static void test(String filename, int unicodeVer) throws Exception {
File f = filename.equals(DATA_LATEST) ?
UCDFiles.NORMALIZATION_TEST.toFile() :
new File(System.getProperty("test.src", "."), filename);
FileInputStream fis = new FileInputStream(f);
BufferedReader in =
new BufferedReader(new InputStreamReader(fis, decoder));
System.out.println("\nStart testing for " + normalizer.name +
" with " + filename + " for options: " +
(((unicodeVer & NormalizerBase.UNICODE_3_2) != 0) ?
"Unicode 3.2.0" : "the latest Unicode"));
int lineNo = 0;
String text;
boolean part1test = false;
boolean part1testExists = false;
String[] columns = new String[6];
while ((text = in.readLine()) != null) {
lineNo ++;
char c = text.charAt(0);
if (c == '#') {
continue;
} else if (c == '@') {
if (text.startsWith("@Part")) {
System.out.println("# Testing data in " + text);
if (text.startsWith("@Part1 ")) {
part1test = true;
part1testExists = true;
} else {
part1test = false;
}
continue;
}
}
prepareColumns(columns, text, filename, lineNo, part1test);
testNFC(columns, unicodeVer, filename, lineNo);
testNFD(columns, unicodeVer, filename, lineNo);
testNFKC(columns, unicodeVer, filename, lineNo);
testNFKD(columns, unicodeVer, filename, lineNo);
}
in.close();
fis.close();
if (part1testExists) {
System.out.println("# Testing characters which are not listed in Part1");
testRemainingChars(filename, unicodeVer);
part1testExists = false;
}
}
/*
* Test for NFC
*
* c2 == NFC(c1) == NFC(c2) == NFC(c3)
* c4 == NFC(c4) == NFC(c5)
*/
private static void testNFC(String[] c, int unicodeVer,
String file, int line) throws Exception {
test(2, c, 1, 3, NFC, unicodeVer, file, line);
test(4, c, 4, 5, NFC, unicodeVer, file, line);
}
/*
* Test for NFD
*
* c3 == NFD(c1) == NFD(c2) == NFD(c3)
* c5 == NFD(c4) == NFD(c5)
*/
private static void testNFD(String[] c, int unicodeVer,
String file, int line) throws Exception {
test(3, c, 1, 3, NFD, unicodeVer, file, line);
test(5, c, 4, 5, NFD, unicodeVer, file, line);
}
/*
* Test for NFKC
*
* c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
*/
private static void testNFKC(String[] c, int unicodeVer,
String file, int line) throws Exception {
test(4, c, 1, 5, NFKC, unicodeVer, file, line);
}
/*
* Test for NFKD
*
* c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
*/
private static void testNFKD(String[] c, int unicodeVer,
String file, int line) throws Exception {
test(5, c, 1, 5, NFKD, unicodeVer, file, line);
}
/*
* Test for characters which aren't listed in Part1
*
* X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
*/
private static void testRemainingChars(String file,
int unicodeVer) throws Exception {
for (int i = Character.MIN_CODE_POINT;
i <= Character.MAX_CODE_POINT;
i++) {
if (!charList.get(i)) {
String from = String.valueOf(Character.toChars(i));
String to;
for (int j = 0; j < forms.length; j++) {
java.text.Normalizer.Form form = forms[j];
to = normalizer.normalize(from, form, unicodeVer);
if (!from.equals(to)) {
error(form, from, from, to, file, -1);
// } else {
// okay(form, from, from, to, file, -1);
}
if (!normalizer.isNormalized(from, form, unicodeVer)) {
error(form, from, file, -1);
// } else {
// okay(form, from, file, -1);
}
}
}
}
}
/*
* Test normalize() and isNormalized()
*/
private static void test(int col, String[] c,
int FROM, int TO,
java.text.Normalizer.Form form, int unicodeVer,
String file, int line) throws Exception {
for (int i = FROM; i <= TO; i++) {
String got = normalizer.normalize(c[i], form, unicodeVer);
if (!c[col].equals(got)) {
error(form, c[i], c[col], got, file, line);
// } else {
// okay(form, c[i], c[col], got, file, line);
}
/*
* If the original String equals its normalized String, it means
* that the original String is normalizerd. Thus, isNormalized()
* should return true. And, vice versa!
*/
if (c[col].equals(c[i])) {
if (!normalizer.isNormalized(c[i], form, unicodeVer)) {
error(form, c[i], file, line);
// } else {
// okay(form, c[i], file, line);
}
} else {
if (normalizer.isNormalized(c[i], form, unicodeVer)) {
error(form, c[i], file, line);
// } else {
// okay(form, c[i], file, line);
}
}
}
}
/*
* Generate an array of String from a line of conformance datafile.
*/
private static void prepareColumns(String[] cols, String text,
String file, int line,
boolean part1test) throws Exception {
int index = text.indexOf('#');
if (index != -1) {
text = text.substring(0, index);
}
StringTokenizer st = new StringTokenizer(text, ";");
int tokenCount = st.countTokens();
if (tokenCount < 5) {
throw new RuntimeException("# of tokens in datafile should be 6, but got: " + tokenCount + " at line " + line + " in " + file);
}
StringBuffer sb = new StringBuffer();
for (int i = 1; i <= 5; i++) {
StringTokenizer tst = new StringTokenizer(st.nextToken(), " ");
while (tst.hasMoreTokens()) {
int code = Integer.parseInt(tst.nextToken(), 16);
sb.append(Character.toChars(code));
}
cols[i] = sb.toString();
sb.setLength(0);
}
if (part1test) {
charList.set(cols[1].codePointAt(0));
}
}
/*
* Show an error message when normalize() didn't return the expected value.
* (An exception is sometimes convenient. Therefore, it is commented out
* for the moment.)
*/
private static void error(java.text.Normalizer.Form form,
String from, String to, String got,
String file, int line) throws Exception {
System.err.println("-\t" + form.toString() + ": normalize(" +
toHexString(from) + ") doesn't equal <" + toHexString(to) +
"> at line " + line + " in " + file + ". Got [" +
toHexString(got) + "]");
throw new RuntimeException("Normalization(" + form.toString() + ") failed");
}
/*
* Show an error message when isNormalize() didn't return the expected
* value.
* (An exception is sometimes convenient. Therefore, it is commented out
* for the moment.)
*/
private static void error(java.text.Normalizer.Form form, String s,
String file, int line) throws Exception {
System.err.println("\t" + form.toString() + ": isNormalized(" +
toHexString(s) + ") returned the wrong value at line " + line +
" in " + file);
throw new RuntimeException("Normalization(" + form.toString() +") failed");
}
/*
* (For debugging)
* Shows a message when normalize() returned the expected value.
*/
private static void okay(java.text.Normalizer.Form form,
String from, String to, String got,
String file, int line) {
System.out.println("\t" + form.toString() + ": normalize(" +
toHexString(from) + ") equals <" + toHexString(to) +
"> at line " + line + " in " + file + ". Got [" +
toHexString(got) + "]");
}
/*
* (For debugging)
* Shows a message when isNormalized() returned the expected value.
*/
private static void okay(java.text.Normalizer.Form form, String s,
String file, int line) {
System.out.println("\t" + form.toString() + ": isNormalized(" +
toHexString(s) + ") returned the correct value at line " +
line + " in " + file);
}
/*
* Returns a spece-delimited hex String
*/
private static String toHexString(String s) {
StringBuffer sb = new StringBuffer(" ");
for (int i = 0; i < s.length(); i++) {
sb.append(Integer.toHexString(s.charAt(i)));
sb.append(' ');
}
return sb.toString();
}
/*
* Abstract class to call each Normalizer in java.text or sun.text.
*/
private abstract class TestNormalizer {
String name;
TestNormalizer(String str) {
name = str;
}
String getNormalizerName() {
return name;
}
abstract String normalize(CharSequence cs,
java.text.Normalizer.Form form,
int option);
abstract boolean isNormalized(CharSequence cs,
java.text.Normalizer.Form form,
int option);
}
/*
* For java.text.Normalizer
* - normalize(CharSequence, Normalizer.Form)
* - isNormalized(CharSequence, Normalizer.Form)
*/
private class testJavaNormalizer extends TestNormalizer {
testJavaNormalizer() {
super("java.text.Normalizer");
}
String normalize(CharSequence cs,
java.text.Normalizer.Form form,
int option) {
return java.text.Normalizer.normalize(cs, form);
}
boolean isNormalized(CharSequence cs,
java.text.Normalizer.Form form,
int option) {
return java.text.Normalizer.isNormalized(cs, form);
}
}
/*
* For sun.text.Normalizer
* - normalize(CharSequence, Normalizer.Form, int)
* - isNormalized(CharSequence, Normalizer.Form, int)
*/
private class testSunNormalizer extends TestNormalizer {
testSunNormalizer() {
super("sun.text.Normalizer");
}
String normalize(CharSequence cs,
java.text.Normalizer.Form form,
int option) {
return sun.text.Normalizer.normalize(cs, form, option);
}
boolean isNormalized(CharSequence cs,
java.text.Normalizer.Form form,
int option) {
return sun.text.Normalizer.isNormalized(cs, form, option);
}
}
}

447
test/jdk/java/text/Normalizer/DataValidationTest.java Normal file
View file

@ -0,0 +1,447 @@
/*
* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
* test
* bug 4221795
* summary Confirm *.icu data using ICU4J Normalizer
*/
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.util.BitSet;
import java.util.StringTokenizer;
import com.ibm.icu.text.Normalizer;
import com.ibm.icu.impl.NormalizerImpl;
/**
* This is not a test program but a data validation utility.
* Two datafiles for Normalizer, unorm.icu and uprops.icu under
* sun/text/resouces, are generated using generators in ICU4C 3.2 on a
* BIG-ENDIAN machine. Before using them with java.text.Normalizer and
* sun.text.Normalizer, you may want to check these test datafile's validation.
* You can test datafiles using Normalizer in ICU4J 3.2. Download ICU4J 3.2 and
* run this test program with -cp <ICU4J 3.2>.
*/
public class DataValidationTest {
//
// Options to be used with com.ibm.icu.text.Normalizer
//
/*
* Default Unicode 3.2.0 normalization.
*
* - With Corrigendum 4 fix
* (Different from Mustang's Normalizer.)
* - With Public Review Issue #29 fix
* (Different from Mustang's Normalizer.)
*/
private static final int UNICODE_3_2_0 = Normalizer.UNICODE_3_2;
/*
* *Incomplete* Unicode 3.2.0 normalization for IDNA/StringPrep.
*
* - With Corrigendum 4 fix
* - Without Public Review Issue #29 fix
*
* ICU4J's Normalizer itself doesn't support normalization for Unicode 3.2.0
* without Corrigendum 4 fix, which is necessary for IDNA/StringPrep. It is
* done in StringPrep. Therefore, we don't test the normlaization in this
* test program. We merely test normalization for Unicode 3.2.0 without
* Public Review Issue #29 fix with this test program.
*/
private static final int UNICODE_3_2_0_BEFORE_PRI_29 =
Normalizer.UNICODE_3_2 |
NormalizerImpl.BEFORE_PRI_29;
/*
* Default normalization.
*
* - Unicode 4.0.1
* (Different from Mustang's Normalizer.)
* - With Corrigendum 4 fix
* - With Public Review Issue #29 fix
* (Different from Mustang's Normalizer.)
*
* Because Public Review Issue #29 is fixed in Unicode 4.1.0. I think that
* IUC4J 3.2 should not support it. But it actually supports PRI #29 fix
* as default....
*/
private static final int UNICODE_LATEST = 0x00;
/*
* Normalization without Public Review Issue #29 fix.
*
* - Unicode 4.0.1
* - Without Corrigendum 4 fix
* - Without Public Review Issue #29 fix
*/
static final int UNICODE_LATEST_BEFORE_PRI_29 =
NormalizerImpl.BEFORE_PRI_29;
//
// Conformance test datafiles
//
/*
* Conformance test datafile for normalization for Unicode 3.2.0 with
* Corrigendum 4 corrections. This is NOT an original Conformace test
* data. Some inconvenient test cases are commented out.
* About corrigendum 4, please refer
* http://www.unicode.org/versions/corrigendum4.html
*
* ICU4J 3.2's Normalizer itself doesn't support normalization for Unicode
* 3.2.0 without Corrigendum 4 corrections. StringPrep helps it. So, we
* don't test the normalization with this test program.
*/
static final String DATA_3_2_0 = "NormalizationTest-3.2.0.Corrigendum4.txt";
/*
* Conformance test datafile for the latest Unicode which is supported
* by J2SE.
*/
static final String DATA_LATEST = "NormalizationTest-Latest.txt";
/*
* Decorder
*/
static final CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder();
/*
* List to pick up characters which are not listed in Part1
*/
static BitSet charList = new BitSet(Character.MAX_CODE_POINT+1);
/*
* Shortcuts
*/
static final Normalizer.Mode NFC = com.ibm.icu.text.Normalizer.NFC;
static final Normalizer.Mode NFD = com.ibm.icu.text.Normalizer.NFD;
static final Normalizer.Mode NFKC = com.ibm.icu.text.Normalizer.NFKC;
static final Normalizer.Mode NFKD = com.ibm.icu.text.Normalizer.NFKD;
static final Normalizer.Mode[] modes = {NFC, NFD, NFKC, NFKD};
public static void main(String[] args) throws Exception {
test(DATA_3_2_0, UNICODE_3_2_0);
test(DATA_3_2_0, UNICODE_3_2_0_BEFORE_PRI_29);
test(DATA_LATEST, UNICODE_LATEST);
// This test started failing since ICU4J 3.6.
// test(DATA_LATEST, UNICODE_LATEST_BEFORE_PRI_29);
/* Unconformity test */
// test(DATA_3_2_0, UNICODE_LATEST);
// test(DATA_LATEST, UNICODE_3_2);
}
private static void test(String filename, int unicodeVer) throws Exception {
FileInputStream fis = new FileInputStream(filename);
BufferedReader in =
new BufferedReader(new InputStreamReader(fis, decoder));
System.out.println("\nStart testing with " + filename +
" for options: " +
(((unicodeVer & Normalizer.UNICODE_3_2) != 0) ?
"Unicode 3.2.0" : "the latest Unicode") + ", " +
(((unicodeVer & NormalizerImpl.BEFORE_PRI_29) != 0) ?
"with" : "without") + " PRI #29 fix");
int lineNo = 0;
String text;
String[] columns = new String[6];
boolean part1test = false;
while ((text = in.readLine()) != null) {
lineNo ++;
char c = text.charAt(0);
if (c == '#') {
continue;
} else if (c == '@') {
if (text.startsWith("@Part")) {
System.out.println("# Testing data in " + text);
if (text.startsWith("@Part1 ")) {
part1test = true;
} else {
part1test = false;
}
continue;
}
}
prepareColumns(columns, text, filename, lineNo, part1test);
testNFC(columns, unicodeVer, filename, lineNo);
testNFD(columns, unicodeVer, filename, lineNo);
testNFKC(columns, unicodeVer, filename, lineNo);
testNFKD(columns, unicodeVer, filename, lineNo);
}
in.close();
fis.close();
if (unicodeVer == UNICODE_LATEST) {
System.out.println("# Testing characters which are not listed in Part1");
testRemainingChars(filename, unicodeVer);
}
}
/*
* Test for NFC
*
* c2 == NFC(c1) == NFC(c2) == NFC(c3)
* c4 == NFC(c4) == NFC(c5)
*/
private static void testNFC(String[] c, int unicodeVer,
String file, int line) throws Exception {
test(2, c, 1, 3, NFC, unicodeVer, file, line);
test(4, c, 4, 5, NFC, unicodeVer, file, line);
}
/*
* Test for NFD
*
* c3 == NFD(c1) == NFD(c2) == NFD(c3)
* c5 == NFD(c4) == NFD(c5)
*/
private static void testNFD(String[] c, int unicodeVer,
String file, int line) throws Exception {
test(3, c, 1, 3, NFD, unicodeVer, file, line);
test(5, c, 4, 5, NFD, unicodeVer, file, line);
}
/*
* Test for NFKC
*
* c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
*/
private static void testNFKC(String[] c, int unicodeVer,
String file, int line) throws Exception {
test(4, c, 1, 5, NFKC, unicodeVer, file, line);
}
/*
* Test for NFKD
*
* c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
*/
private static void testNFKD(String[] c, int unicodeVer,
String file, int line) throws Exception {
test(5, c, 1, 5, NFKD, unicodeVer, file, line);
}
/*
* Test for characters which aren't listed in Part1
*
* X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
*/
private static void testRemainingChars(String file,
int unicodeVer) throws Exception {
for (int i = Character.MIN_CODE_POINT;
i <= Character.MAX_CODE_POINT;
i++) {
if (!charList.get(i)) {
String from = String.valueOf(Character.toChars(i));
String to;
for (int j = 0; j < modes.length; j++) {
Normalizer.Mode mode = modes[j];
to = Normalizer.normalize(from, mode, unicodeVer);
if (!from.equals(to)) {
error(mode, from, from, to, file, -1);
// } else {
// okay(mode, from, from, to, file, -1);
}
if (!Normalizer.isNormalized(from, mode, unicodeVer)) {
error(mode, from, file, -1);
// } else {
// okay(mode, from, file, -1);
}
}
}
}
}
/*
* Test normalize() and isNormalized()
*/
private static void test(int col, String[] c,
int FROM, int TO,
Normalizer.Mode mode, int unicodeVer,
String file, int line) throws Exception {
for (int i = FROM; i <= TO; i++) {
String got = Normalizer.normalize(c[i], mode, unicodeVer);
if (!c[col].equals(got)) {
error(mode, c[i], c[col], got, file, line);
// } else {
// okay(mode, c[i], c[col], got, file, line);
}
/*
* If the original String equals its normalized String, it means
* that the original String is normalizerd. Thus, isNormalized()
* should return true. And, vice versa!
*/
if (c[col].equals(c[i])) {
if (!Normalizer.isNormalized(c[i], mode, unicodeVer)) {
error(mode, c[i], file, line);
// } else {
// okay(mode, c[i], file, line);
}
} else {
if (Normalizer.isNormalized(c[i], mode, unicodeVer)) {
error(mode, c[i], file, line);
// } else {
// okay(mode, c[i], file, line);
}
}
}
}
/*
* Generate an array of String from a line of conformance datafile.
*/
private static void prepareColumns(String[] col, String text,
String file, int line,
boolean part1test) throws Exception {
int index = text.indexOf('#');
if (index != -1) {
text = text.substring(0, index);
}
StringTokenizer st = new StringTokenizer(text, ";");
int tokenCount = st.countTokens();
if (tokenCount < 5) {
throw new RuntimeException("# of tokens in datafile should be 6, but got: " + tokenCount + " at line " + line + " in " + file);
}
StringBuffer sb = new StringBuffer();
for (int i = 1; i <= 5; i++) {
StringTokenizer tst = new StringTokenizer(st.nextToken(), " ");
while (tst.hasMoreTokens()) {
int code = Integer.parseInt(tst.nextToken(), 16);
sb.append(Character.toChars(code));
}
col[i] = sb.toString();
sb.setLength(0);
}
if (part1test) {
charList.set(col[1].codePointAt(0));
}
}
/*
* Show an error message when normalize() didn't return the expected value.
* (An exception is sometimes convenient. Therefore, it is commented out
* for the moment.)
*/
private static void error(Normalizer.Mode mode,
String from, String to, String got,
String file, int line) throws Exception {
System.err.println("\t" + toString(mode) + ": normalize(" +
toHexString(from) + ") doesn't equal <" + toHexString(to) +
"> at line " + line + " in " + file + ". Got <" +
toHexString(got) + ">.");
// throw new RuntimeException("Normalization(" + toString(mode) + ") failed");
}
/*
* Show an error message when isNormalize() didn't return the expected value.
* (An exception is sometimes convenient. Therefore, it is commented out
* for the moment.)
*/
private static void error(Normalizer.Mode mode, String orig,
String file, int line) throws Exception {
System.err.println("\t" + toString(mode) + ": isNormalized(" +
toHexString(orig) + ") returned the wrong value at line " + line +
" in " + file + ".");
// throw new RuntimeException("Normalization(" + toString(mode) +") failed");
}
/*
* (For debugging)
* Shows a message when normalize() returned the expected value.
*/
private static void okay(Normalizer.Mode mode,
String from, String to, String got,
String file, int line) {
System.out.println("\t" + toString(mode) + ": normalize(" +
toHexString(from) + ") equals <" + toHexString(to) +
"> at line " + line + " in " + file + ". Got <" +
toHexString(got) + ">.");
}
/*
* (For debugging)
* Shows a message when isNormalized() returned the expected value.
*/
private static void okay(Normalizer.Mode mode, String orig,
String file, int line) {
System.out.println("\t" + toString(mode) + ": isNormalized(" +
toHexString(orig) + ") returned the correct value at line " +
line + " in " + file + ".");
}
/*
* Returns a spece-delimited hex String
*/
private static String toHexString(String s) {
StringBuffer sb = new StringBuffer(" ");
for (int i = 0; i < s.length(); i++) {
sb.append(Integer.toHexString(s.charAt(i)));
sb.append(' ');
}
return sb.toString();
}
/*
* Returns the name of Normalizer.Mode
*/
private static String toString(Normalizer.Mode mode) {
if (mode == Normalizer.NFC) {
return "NFC";
} else if (mode == Normalizer.NFD) {
return "NFD";
} else if (mode == Normalizer.NFKC) {
return "NFKC";
} else if (mode == Normalizer.NFKD) {
return "NFKD";
}
return "unknown";
}
}

632
test/jdk/java/text/Normalizer/ICUBasicTest.java Normal file
View file

@ -0,0 +1,632 @@
/*
* Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
* @test
* @bug 4221795 8032446 8174270
* @summary Confirm Normalizer's fundamental behavior. Imported from ICU4J 3.2's
* src/com/ibm/icu/dev/test and modified.
* @modules java.base/sun.text java.base/jdk.internal.icu.text
* @compile -XDignore.symbol.file ICUBasicTest.java
* @run junit/timeout=30 ICUBasicTest
*/
/*
*******************************************************************************
* Copyright (C) 1996-2004, International Business Machines Corporation and *
* others. All Rights Reserved. *
*******************************************************************************
*/
import sun.text.Normalizer;
import jdk.internal.icu.text.NormalizerBase;
import java.util.HexFormat;
import static java.text.Normalizer.Form.*;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.fail;
public class ICUBasicTest {
/*
* Normalization modes
*/
private static final NormalizerBase.Mode NFCmode = NormalizerBase.NFC;
private static final NormalizerBase.Mode NFDmode = NormalizerBase.NFD;
private static final NormalizerBase.Mode NFKCmode = NormalizerBase.NFKC;
private static final NormalizerBase.Mode NFKDmode = NormalizerBase.NFKD;
private static final NormalizerBase.Mode NONEmode = NormalizerBase.NONE;
/*
* Normalization options
*/
/* Normal Unicode versions */
private static final int UNICODE_3_2_0 = Normalizer.UNICODE_3_2;
private static final int UNICODE_LATEST = NormalizerBase.UNICODE_LATEST;
/*
* Special cases for UAX #15 bug
* see Unicode Public Review Issue #29
* at http://www.unicode.org/review/resolved-pri.html#pri29
*
* Note:
* PRI #29 is supported in Unicode 4.1.0. Therefore, expected results are
* different for earlier Unicode versions.
*/
@Test
public void TestComposition() {
final TestCompositionCase cases[] = new TestCompositionCase[] {
new TestCompositionCase(NFC, UNICODE_3_2_0,
"\u1100\u0300\u1161\u0327",
"\u1100\u0300\u1161\u0327"),
new TestCompositionCase(NFC, UNICODE_LATEST,
"\u1100\u0300\u1161\u0327",
"\u1100\u0300\u1161\u0327"),
new TestCompositionCase(NFC, UNICODE_3_2_0,
"\u1100\u0300\u1161\u0327\u11a8",
"\u1100\u0300\u1161\u0327\u11a8"),
new TestCompositionCase(NFC, UNICODE_LATEST,
"\u1100\u0300\u1161\u0327\u11a8",
"\u1100\u0300\u1161\u0327\u11a8"),
new TestCompositionCase(NFC, UNICODE_3_2_0,
"\uac00\u0300\u0327\u11a8",
"\uac00\u0327\u0300\u11a8"),
new TestCompositionCase(NFC, UNICODE_LATEST,
"\uac00\u0300\u0327\u11a8",
"\uac00\u0327\u0300\u11a8"),
new TestCompositionCase(NFC, UNICODE_3_2_0,
"\u0b47\u0300\u0b3e",
"\u0b47\u0300\u0b3e"),
new TestCompositionCase(NFC, UNICODE_LATEST,
"\u0b47\u0300\u0b3e",
"\u0b47\u0300\u0b3e"),
};
String output;
int i, length;
for (i=0; i<cases.length; ++i) {
output = Normalizer.normalize(cases[i].input,
cases[i].form, cases[i].options);
if (!output.equals(cases[i].expect)) {
fail("unexpected result for case " + i + ". Expected="
+ cases[i].expect + ", Actual=" + output);
}
}
}
private final static class TestCompositionCase {
public java.text.Normalizer.Form form;
public int options;
public String input, expect;
TestCompositionCase(java.text.Normalizer.Form form,
int options,
String input,
String expect) {
this.form = form;
this.options = options;
this.input = input;
this.expect = expect;
}
}
/*
* Added in order to detect a regression.
*/
@Test
public void TestCombiningMarks() {
String src = "\u0f71\u0f72\u0f73\u0f74\u0f75";
String expected = "\u0F71\u0F71\u0F71\u0F72\u0F72\u0F74\u0F74";
String result = NormalizerBase.normalize(src, NFD);
if (!expected.equals(result)) {
fail("Reordering of combining marks failed. Expected: " +
HexFormat.of().withDelimiter(" ").formatHex(expected.getBytes())
+ " Got: "+ HexFormat.of().withDelimiter(" ").formatHex(result.getBytes()));
}
}
/*
* Added in order to detect a regression.
*/
@Test
public void TestBengali() throws Exception {
String input = "\u09bc\u09be\u09cd\u09be";
String output=NormalizerBase.normalize(input, NFC);
if (!input.equals(output)) {
fail("ERROR in NFC of string");
}
return;
}
/*
* Added in order to detect a regression.
*/
/**
* Test for a problem found by Verisign. Problem is that
* characters at the start of a string are not put in canonical
* order correctly by compose() if there is no starter.
*/
@Test
public void TestVerisign() throws Exception {
String[] inputs = {
"\u05b8\u05b9\u05b1\u0591\u05c3\u05b0\u05ac\u059f",
"\u0592\u05b7\u05bc\u05a5\u05b0\u05c0\u05c4\u05ad"
};
String[] outputs = {
"\u05b1\u05b8\u05b9\u0591\u05c3\u05b0\u05ac\u059f",
"\u05b0\u05b7\u05bc\u05a5\u0592\u05c0\u05ad\u05c4"
};
for (int i = 0; i < inputs.length; ++i) {
String input = inputs[i];
String output = outputs[i];
String result = NormalizerBase.normalize(input, NFD);
if (!result.equals(output)) {
fail("FAIL input: " + HexFormat.of().withDelimiter(" ")
.formatHex(input.getBytes()) + "\n" +
" decompose: " + HexFormat.of().withDelimiter(" ")
.formatHex(result.getBytes()) + "\n" +
" expected: " + HexFormat.of().withDelimiter(" ")
.formatHex(output.getBytes()));
}
result = NormalizerBase.normalize(input, NFC);
if (!result.equals(output)) {
fail("FAIL input: " + HexFormat.of().withDelimiter(" ")
.formatHex(input.getBytes()) + "\n" +
" compose: " + HexFormat.of().withDelimiter(" ")
.formatHex(output.getBytes()) + "\n" +
" expected: " + HexFormat.of().withDelimiter(" ")
.formatHex(output.getBytes()));
}
}
}
/**
* Test for a problem that showed up just before ICU 1.6 release
* having to do with combining characters with an index of zero.
* Such characters do not participate in any canonical
* decompositions. However, having an index of zero means that
* they all share one typeMask[] entry, that is, they all have to
* map to the same canonical class, which is not the case, in
* reality.
*/
@Test
public void TestZeroIndex() throws Exception {
String[] DATA = {
// Expect col1 x COMPOSE_COMPAT => col2
// Expect col2 x DECOMP => col3
"A\u0316\u0300", "\u00C0\u0316", "A\u0316\u0300",
"A\u0300\u0316", "\u00C0\u0316", "A\u0316\u0300",
"A\u0327\u0300", "\u00C0\u0327", "A\u0327\u0300",
"c\u0321\u0327", "c\u0321\u0327", "c\u0321\u0327",
"c\u0327\u0321", "\u00E7\u0321", "c\u0327\u0321",
};
for (int i=0; i<DATA.length; i+=3) {
String a = DATA[i];
String b = NormalizerBase.normalize(a, NFKC);
String exp = DATA[i+1];
if (b.equals(exp)) {
System.out.println("Ok: " + HexFormat.of().withDelimiter(" ")
.formatHex(a.getBytes()) + " x COMPOSE_COMPAT => " +
HexFormat.of().withDelimiter(" ")
.formatHex(b.getBytes()));
} else {
fail("FAIL: " + HexFormat.of().withDelimiter(" ")
.formatHex(b.getBytes()) + " x COMPOSE_COMPAT => " +
HexFormat.of().withDelimiter(" ")
.formatHex(a.getBytes()) + ", expect " +
HexFormat.of().withDelimiter(" ")
.formatHex(exp.getBytes()));
}
a = NormalizerBase.normalize(b, NFD);
exp = DATA[i+2];
if (a.equals(exp)) {
System.out.println("Ok: " + HexFormat.of().withDelimiter(" ").formatHex(b.getBytes()) + " x DECOMP => " +
HexFormat.of().withDelimiter(" ").formatHex(a.getBytes()));
} else {
fail("FAIL: " + HexFormat.of().withDelimiter(" ").formatHex(b.getBytes()) + " x DECOMP => " +
HexFormat.of().withDelimiter(" ").formatHex(a.getBytes()) + ", expect " + HexFormat.of().withDelimiter(" ").formatHex(exp.getBytes()));
}
}
}
/**
* Make sure characters in the CompositionExclusion.txt list do not get
* composed to.
*/
@Test
public void TestCompositionExclusion() throws Exception {
// This list is generated from CompositionExclusion.txt.
// Update whenever the normalizer tables are updated. Note
// that we test all characters listed, even those that can be
// derived from the Unicode DB and are therefore commented
// out.
/*
* kyuka's note:
* Original data seemed to be based on Unicode 3.0.0(the initial
* Composition Exclusions list) and seemed to have some mistakes.
* Updated in order to correct mistakes and to support Unicode 4.0.0.
* And, this table can be used also for Unicode 3.2.0.
*/
String[][] EXCLUDED_UNICODE_3_2_0 = {
{"\u0340"},
{"\u0341"},
{"\u0343"},
{"\u0344"},
{"\u0374"},
{"\u037E"},
{"\u0387"},
{"\u0958"},
{"\u0959", "\u095F"},
{"\u09DC"},
{"\u09DD"},
{"\u09DF"},
{"\u0A33"},
{"\u0A36"},
{"\u0A59", "\u0A5B"},
{"\u0A5E"},
{"\u0B5C"},
{"\u0B5D"},
{"\u0F43"},
{"\u0F4D"},
{"\u0F52"},
{"\u0F57"},
{"\u0F5C"},
{"\u0F69"},
{"\u0F73"},
{"\u0F75"},
{"\u0F76"},
{"\u0F78"},
{"\u0F81"},
{"\u0F93"},
{"\u0F9D"},
{"\u0FA2"},
{"\u0FA7"},
{"\u0FAC"},
{"\u0FB9"},
{"\u1F71"},
{"\u1F73"},
{"\u1F75"},
{"\u1F77"},
{"\u1F79"},
{"\u1F7B"},
{"\u1F7D"},
{"\u1FBB"},
{"\u1FBE"},
{"\u1FC9"},
{"\u1FCB"},
{"\u1FD3"},
{"\u1FDB"},
{"\u1FE3"},
{"\u1FEB"},
{"\u1FEE"},
{"\u1FEF"},
{"\u1FF9"},
{"\u1FFB"},
{"\u1FFD"},
{"\u2000"},
{"\u2001"},
{"\u2126"},
{"\u212A"},
{"\u212B"},
{"\u2329"},
{"\u232A"},
{"\u2ADC"},
{"\uF900", "\uFA0D"},
{"\uFA10"},
{"\uFA12"},
{"\uFA15", "\uFA1E"},
{"\uFA20"},
{"\uFA22"},
{"\uFA25"},
{"\uFA26"},
{"\uFA2A", "\uFA2D"},
{"\uFA30", "\uFA6A"},
{"\uFB1D"},
{"\uFB1F"},
{"\uFB2A", "\uFB36"},
{"\uFB38", "\uFB3C"},
{"\uFB3E"},
{"\uFB40"},
{"\uFB41"},
{"\uFB43"},
{"\uFB44"},
{"\uFB46", "\uFB4E"},
{"\uD834\uDD5E", "\uD834\uDD64"},
{"\uD834\uDDBB", "\uD834\uDDC0"},
{"\uD87E\uDC00", "\uD87E\uDE1D"}
};
String[][] EXCLUDED_LATEST = {
};
for (int i = 0; i < EXCLUDED_UNICODE_3_2_0.length; ++i) {
if (EXCLUDED_UNICODE_3_2_0[i].length == 1) {
checkCompositionExclusion_320(EXCLUDED_UNICODE_3_2_0[i][0]);
} else {
int from, to;
from = Character.codePointAt(EXCLUDED_UNICODE_3_2_0[i][0], 0);
to = Character.codePointAt(EXCLUDED_UNICODE_3_2_0[i][1], 0);
for (int j = from; j <= to; j++) {
checkCompositionExclusion_320(String.valueOf(Character.toChars(j)));
}
}
}
}
private void checkCompositionExclusion_320(String s) throws Exception {
String a = String.valueOf(s);
String b = NormalizerBase.normalize(a, NFKD);
String c = NormalizerBase.normalize(b, NFC);
if (c.equals(a)) {
fail("FAIL: " + HexFormat.of().withDelimiter(" ")
.formatHex(a.getBytes()) + " x DECOMP_COMPAT => " +
HexFormat.of().withDelimiter(" ")
.formatHex(b.getBytes()) + " x COMPOSE => " +
HexFormat.of().withDelimiter(" ")
.formatHex(c.getBytes()) + " for the latest Unicode");
}
b = NormalizerBase.normalize(a, NFKD, Normalizer.UNICODE_3_2);
c = NormalizerBase.normalize(b, NFC, Normalizer.UNICODE_3_2);
if (c.equals(a)) {
fail("FAIL: " + HexFormat.of().withDelimiter(" ")
.formatHex(a.getBytes()) + " x DECOMP_COMPAT => " +
HexFormat.of().withDelimiter(" ").formatHex(b.getBytes()) + " x COMPOSE => " +
HexFormat.of().withDelimiter(" ").formatHex(c.getBytes()) + " for Unicode 3.2.0");
}
}
@Test
public void TestTibetan() throws Exception {
String[][] decomp = {
{ "\u0f77", "\u0f77", "\u0fb2\u0f71\u0f80" }
};
String[][] compose = {
{ "\u0fb2\u0f71\u0f80", "\u0fb2\u0f71\u0f80", "\u0fb2\u0f71\u0f80" }
};
staticTest(NFD, decomp, 1);
staticTest(NFKD,decomp, 2);
staticTest(NFC, compose, 1);
staticTest(NFKC,compose, 2);
}
@Test
public void TestExplodingBase() throws Exception{
// \u017f - Latin small letter long s
// \u0307 - combining dot above
// \u1e61 - Latin small letter s with dot above
// \u1e9b - Latin small letter long s with dot above
String[][] canon = {
// Input Decomposed Composed
{ "Tschu\u017f", "Tschu\u017f", "Tschu\u017f" },
{ "Tschu\u1e9b", "Tschu\u017f\u0307", "Tschu\u1e9b" },
};
String[][] compat = {
// Input Decomposed Composed
{ "\u017f", "s", "s" },
{ "\u1e9b", "s\u0307", "\u1e61" },
};
staticTest(NFD, canon, 1);
staticTest(NFC, canon, 2);
staticTest(NFKD, compat, 1);
staticTest(NFKC, compat, 2);
}
private String[][] canonTests = {
// Input Decomposed Composed
{ "cat", "cat", "cat" },
{ "\u00e0ardvark", "a\u0300ardvark", "\u00e0ardvark", },
// D-dot_above
{ "\u1e0a", "D\u0307", "\u1e0a" },
// D dot_above
{ "D\u0307", "D\u0307", "\u1e0a" },
// D-dot_below dot_above
{ "\u1e0c\u0307", "D\u0323\u0307", "\u1e0c\u0307" },
// D-dot_above dot_below
{ "\u1e0a\u0323", "D\u0323\u0307", "\u1e0c\u0307" },
// D dot_below dot_above
{ "D\u0307\u0323", "D\u0323\u0307", "\u1e0c\u0307" },
// D dot_below cedilla dot_above
{ "\u1e10\u0307\u0323", "D\u0327\u0323\u0307", "\u1e10\u0323\u0307"},
// D dot_above ogonek dot_below
{ "D\u0307\u0328\u0323","D\u0328\u0323\u0307", "\u1e0c\u0328\u0307"},
// E-macron-grave
{ "\u1E14", "E\u0304\u0300", "\u1E14" },
// E-macron + grave
{ "\u0112\u0300", "E\u0304\u0300", "\u1E14" },
// E-grave + macron
{ "\u00c8\u0304", "E\u0300\u0304", "\u00c8\u0304" },
// angstrom_sign
{ "\u212b", "A\u030a", "\u00c5" },
// A-ring
{ "\u00c5", "A\u030a", "\u00c5" },
{ "\u00c4ffin", "A\u0308ffin", "\u00c4ffin" },
{ "\u00c4\uFB03n", "A\u0308\uFB03n", "\u00c4\uFB03n" },
//updated with 3.0
{ "\u00fdffin", "y\u0301ffin", "\u00fdffin" },
{ "\u00fd\uFB03n", "y\u0301\uFB03n", "\u00fd\uFB03n" },
{ "Henry IV", "Henry IV", "Henry IV" },
{ "Henry \u2163", "Henry \u2163", "Henry \u2163" },
// ga(Zenkaku-Katakana)
{ "\u30AC", "\u30AB\u3099", "\u30AC" },
// ka(Zenkaku-Katakana) + ten(Zenkaku)
{ "\u30AB\u3099", "\u30AB\u3099", "\u30AC" },
// ka(Hankaku-Katakana) + ten(Hankaku-Katakana)
{ "\uFF76\uFF9E", "\uFF76\uFF9E", "\uFF76\uFF9E" },
// ka(Zenkaku-Katakana) + ten(Hankaku)
{ "\u30AB\uFF9E", "\u30AB\uFF9E", "\u30AB\uFF9E" },
// ka(Hankaku-Katakana) + ten(Zenkaku)
{ "\uFF76\u3099", "\uFF76\u3099", "\uFF76\u3099" },
{ "A\u0300\u0316", "A\u0316\u0300", "\u00C0\u0316" },
{ "\ud834\udd5e\ud834\udd57\ud834\udd65\ud834\udd5e",
"\ud834\udd57\ud834\udd65\ud834\udd57\ud834\udd65\ud834\udd57\ud834\udd65",
"\ud834\udd57\ud834\udd65\ud834\udd57\ud834\udd65\ud834\udd57\ud834\udd65" },
};
private String[][] compatTests = {
// Input Decomposed Composed
{ "cat", "cat", "cat" },
// Alef-Lamed vs. Alef, Lamed
{ "\uFB4f", "\u05D0\u05DC", "\u05D0\u05DC", },
{ "\u00C4ffin", "A\u0308ffin", "\u00C4ffin" },
// ffi ligature -> f + f + i
{ "\u00C4\uFB03n", "A\u0308ffin", "\u00C4ffin" },
//updated for 3.0
{ "\u00fdffin", "y\u0301ffin", "\u00fdffin" },
// ffi ligature -> f + f + i
{ "\u00fd\uFB03n", "y\u0301ffin", "\u00fdffin" },
{ "Henry IV", "Henry IV", "Henry IV" },
{ "Henry \u2163", "Henry IV", "Henry IV" },
// ga(Zenkaku-Katakana)
{ "\u30AC", "\u30AB\u3099", "\u30AC" },
// ka(Zenkaku-Katakana) + ten(Zenkaku)
{ "\u30AB\u3099", "\u30AB\u3099", "\u30AC" },
// ka(Hankaku-Katakana) + ten(Zenkaku)
{ "\uFF76\u3099", "\u30AB\u3099", "\u30AC" },
/* These two are broken in Unicode 2.1.2 but fixed in 2.1.5 and later*/
// ka(Hankaku-Katakana) + ten(Hankaku)
{ "\uFF76\uFF9E", "\u30AB\u3099", "\u30AC" },
// ka(Zenkaku-Katakana) + ten(Hankaku)
{ "\u30AB\uFF9E", "\u30AB\u3099", "\u30AC" },
};
@Test
public void TestNFD() throws Exception{
staticTest(NFD, canonTests, 1);
}
@Test
public void TestNFC() throws Exception{
staticTest(NFC, canonTests, 2);
}
@Test
public void TestNFKD() throws Exception{
staticTest(NFKD, compatTests, 1);
}
@Test
public void TestNFKC() throws Exception{
staticTest(NFKC, compatTests, 2);
}
private void staticTest(java.text.Normalizer.Form form,
String[][] tests,
int outCol) throws Exception {
for (int i = 0; i < tests.length; i++) {
String input = tests[i][0];
System.out.println("Normalizing '" + input + "' (" + HexFormat.of()
.withDelimiter(" ").formatHex(input.getBytes()) + ")" );
String expect =tests[i][outCol];
String output = java.text.Normalizer.normalize(input, form);
if (!output.equals(expect)) {
fail("FAIL: case " + i
+ " expected '" + expect + "' (" + HexFormat.of()
.withDelimiter(" ").formatHex(expect.getBytes()) + ")"
+ " but got '" + output + "' (" + HexFormat.of()
.withDelimiter(" ").formatHex(output.getBytes()) + ")"
);
}
}
}
// With Canonical decomposition, Hangul syllables should get decomposed
// into Jamo, but Jamo characters should not be decomposed into
// conjoining Jamo
private String[][] hangulCanon = {
// Input Decomposed Composed
{ "\ud4db", "\u1111\u1171\u11b6", "\ud4db" },
{ "\u1111\u1171\u11b6", "\u1111\u1171\u11b6", "\ud4db" },
};
@Test
public void TestHangulCompose() throws Exception{
System.out.println("Canonical composition...");
staticTest(NFC, hangulCanon, 2);
}
@Test
public void TestHangulDecomp() throws Exception{
System.out.println("Canonical decomposition...");
staticTest(NFD, hangulCanon, 1);
}
}

11
test/jdk/java/text/Normalizer/ICUNormalizationTest.txt Normal file
View file

@ -0,0 +1,11 @@
#
#
#
# Conformance test data which was added by Markus Scherer of IBM.
# According to him, "more interesting conformance test cases, not in the
# unicode.org NormalizationTest.txt".
# Found in ConformanceTest.java in
# src/com/ibm/icu/dev/test/normalizer/ConformanceTest.java of ICU4J 3.2.
@Part0 # Other (ICU4J test data)
0061 0332 0308;00E4 0332;0061 0332 0308;00E4 0332;0061 0332 0308; # Markus 0",
0061 0301 0F73;00E1 0F71 0F72;0061 0F71 0F72 0301;00E1 0F71 0F72;0061 0F71 0F72 0301; # Markus 1"

17035
test/jdk/java/text/Normalizer/NormalizationTest-3.2.0.Corrigendum4.txt Normal file
View file

File diff suppressed because it is too large Load diff

17035
test/jdk/java/text/Normalizer/NormalizationTest-3.2.0.txt Normal file
View file

File diff suppressed because it is too large Load diff

351
test/jdk/java/text/Normalizer/NormalizerAPITest.java Normal file
View file

@ -0,0 +1,351 @@
/*
* Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
* @test
* @bug 4221795 8174270
* @summary Confirm Normalizer's fundamental behavior
* @modules java.base/sun.text java.base/jdk.internal.icu.text
* @compile -XDignore.symbol.file NormalizerAPITest.java
* @run junit/timeout=30 NormalizerAPITest
*/
import java.text.Normalizer;
import java.nio.CharBuffer;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.fail;
/*
* Tests around null/"" arguments for public methods.
*
* You may think that so elaborate testing for such a part is not necessary.
* But I actually detected a bug by this program during my porting work.
*/
public class NormalizerAPITest {
//
// Shortcuts
//
/*
* Normalization forms
*/
static final Normalizer.Form NFC = Normalizer.Form.NFC;
static final Normalizer.Form NFD = Normalizer.Form.NFD;
static final Normalizer.Form NFKC = Normalizer.Form.NFKC;
static final Normalizer.Form NFKD = Normalizer.Form.NFKD;
static final Normalizer.Form[] forms = {NFC, NFD, NFKC, NFKD};
static final Normalizer.Form NULL = null;
/*
* Option
*/
static final int[] options = {
0x00,
sun.text.Normalizer.UNICODE_3_2,
jdk.internal.icu.text.NormalizerBase.UNICODE_3_2,
jdk.internal.icu.text.NormalizerBase.UNICODE_LATEST,
};
static final String nonNullStr = "testdata";
/*
* Check if normalize(null) throws NullPointerException as expected.
*/
@Test
public void Test_NullPointerException_java_normalize() {
boolean error = false;
/* Check null as String to be normalized */
for (int i = 0; i < forms.length; i++) {
try {
String s = Normalizer.normalize(null, forms[i]);
error = true;
}
catch (NullPointerException e) {
}
}
/* Check null as a Normalization form */
try {
String s = Normalizer.normalize(nonNullStr, NULL);
error = true;
}
catch (NullPointerException e) {
}
if (error) {
fail("normalize(null) should throw NullPointerException.");
}
}
/*
* Check if normalize(null) throws NullPointerException as expected.
*/
@Test
public void Test_NullPointerException_sun_normalize() {
boolean error = false;
for (int j = 0; j < options.length; j++) {
for (int i = 0; i < forms.length; i++) {
/* Check null as a String to be normalized */
try {
String s = sun.text.Normalizer.normalize(null, forms[i], options[j]);
error = true;
}
catch (NullPointerException e) {
}
}
/* Check null as a Normalization form */
try {
String s = sun.text.Normalizer.normalize(nonNullStr, NULL, options[j]);
error = true;
}
catch (NullPointerException e) {
}
}
if (error) {
fail("normalize(null) should throw NullPointerException.");
}
}
/*
* Check if isNormalized(null) throws NullPointerException as expected.
*/
@Test
public void Test_NullPointerException_java_isNormalized() {
boolean error = false;
for (int i = 0; i < forms.length; i++) {
try {
/* Check null as a String to be scanned */
boolean b = Normalizer.isNormalized(null, forms[i]);
error = true;
}
catch (NullPointerException e) {
}
}
/* Check null as a String to be scanned */
try {
boolean b = Normalizer.isNormalized(nonNullStr, NULL);
error = true;
}
catch (NullPointerException e) {
}
if (error) {
fail("isNormalized(null) should throw NullPointerException.");
}
}
/*
* Check if isNormalized(null) throws NullPointerException as expected.
*/
@Test
public void Test_NullPointerException_sun_isNormalized() {
boolean error = false;
for (int j = 0; j < options.length; j++) {
for (int i = 0; i < forms.length; i++) {
try {
/* Check null as a String to be scanned */
boolean b = sun.text.Normalizer.isNormalized(null, forms[i], options[j]);
error = true;
}
catch (NullPointerException e) {
}
}
/* Check null as a String to be scanned */
try {
boolean b = sun.text.Normalizer.isNormalized(nonNullStr, NULL, options[j]);
error = true;
}
catch (NullPointerException e) {
}
}
if (error) {
fail("isNormalized(null) should throw NullPointerException.");
}
}
/*
* Check if isNormalized("") doesn't throw NullPointerException and returns
* "" as expected.
*/
@Test
public void Test_No_NullPointerException_java_normalize() {
boolean error = false;
for (int i = 0; i < forms.length; i++) {
try {
String s = Normalizer.normalize("", forms[i]);
if (!s.equals("")) {
error = true;
}
}
catch (NullPointerException e) {
error = true;
}
}
if (error) {
fail("normalize() for String(\"\") should return \"\".");
}
}
/*
* Check if isNormalized("") doesn't throw NullPointerException and returns
* "" as expected.
*/
@Test
public void Test_No_NullPointerException_sun_normalize() {
boolean error = false;
for (int j = 0; j < options.length; j++) {
for (int i = 0; i < forms.length; i++) {
try {
String s = sun.text.Normalizer.normalize("", forms[i], options[j]);
if (!s.equals("")) {
error = true;
}
}
catch (NullPointerException e) {
error = true;
}
}
}
if (error) {
fail("normalize() for String(\"\") should return \"\".");
}
}
/*
* Check if isNormalized("") doesn't throw NullPointerException and returns
* "" as expected.
*/
@Test
public void Test_No_NullPointerException_java_isNormalized() {
boolean error = false;
for (int i = 0; i < forms.length; i++) {
try {
boolean b = Normalizer.isNormalized("", forms[i]);
if (!b) {
error = true;
}
}
catch (NullPointerException e) {
error = true;
}
}
if (error) {
fail("isNormalized() for String(\"\") should not return true.");
}
}
/*
* Check if isNormalized("") doesn't throw NullPointerException and returns
* "" as expected.
*/
@Test
public void Test_No_NullPointerException_sun_isNormalized() {
boolean error = false;
for (int j = 0; j < options.length; j++) {
for (int i = 0; i < forms.length; i++) {
try {
boolean b = sun.text.Normalizer.isNormalized("", forms[i], options[j]);
if (!b) {
error = true;
}
}
catch (NullPointerException e) {
error = true;
}
}
}
if (error) {
fail("isNormalized() for String(\"\") should not return true.");
}
}
/*
* Check if normalize() and isNormalized() work as expected for every
* known class which implement CharSequence Interface.
*/
@Test
public void Test_CharSequence() {
check_CharSequence(String.valueOf(inputData),
String.valueOf(outputData));
check_CharSequence(new StringBuffer(original),
new StringBuffer(expected));
check_CharSequence(new StringBuilder(original),
new StringBuilder(expected));
check_CharSequence(CharBuffer.wrap(inputData),
CharBuffer.wrap(outputData));
}
void check_CharSequence(CharSequence in, CharSequence expected) {
String out = Normalizer.normalize(in, NFD);
if (!out.equals(expected.toString())) {
fail("java.text.Normalizer.normalize(" +
in.getClass().getSimpleName() + ") failed.");
}
out = sun.text.Normalizer.normalize(in, NFD,
jdk.internal.icu.text.NormalizerBase.UNICODE_LATEST);
if (!out.equals(expected.toString())) {
fail("sun.text.Normalizer.normalize(" +
in.getClass().getSimpleName() + ") failed.");
}
if (!Normalizer.isNormalized(expected, NFD)) {
fail("java.text.Normalizer.isNormalize(" +
in.getClass().getSimpleName() + ") failed.");
}
if (!sun.text.Normalizer.isNormalized(expected, NFD,
jdk.internal.icu.text.NormalizerBase.UNICODE_LATEST)) {
fail("sun.text.Normalizer.isNormalize(" +
in.getClass().getSimpleName() + ") failed.");
}
}
static final char[] inputData = {'T', 's', 'c', 'h', 'u', '\u1e9b'};
static final char[] outputData = {'T', 's', 'c', 'h', 'u', '\u017f', '\u0307'};
static final String original = String.valueOf(inputData);
static final String expected = String.valueOf(outputData);
}

60
test/jdk/java/text/Normalizer/SquareEraCharacterTest.java Normal file
View file

@ -0,0 +1,60 @@
/*
* Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
* @test
* @bug 8221431
* @summary Tests decomposition of Japanese square era characters.
* @run junit/othervm SquareEraCharacterTest
*/
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import java.text.Normalizer;
import static org.junit.jupiter.api.Assertions.assertEquals;
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class SquareEraCharacterTest {
Object[][] squareEras() {
return new Object[][] {
// square era character, expected decomposed string
{'\u337e', "\u660e\u6cbb"}, // Meizi
{'\u337d', "\u5927\u6b63"}, // Taisho
{'\u337c', "\u662d\u548c"}, // Showa
{'\u337b', "\u5e73\u6210"}, // Heisei
{'\u32ff', "\u4ee4\u548c"}, // Reiwa
};
}
@ParameterizedTest
@MethodSource("squareEras")
void test_normalize(char squareChar, String expected) {
assertEquals(expected, Normalizer.normalize(Character.toString(squareChar), Normalizer.Form.NFKD),
"decomposing " + Character.getName(squareChar) + ".");
}
}

142
test/jdk/java/text/Normalizer/ThreadSafeTest.java Normal file
View file

@ -0,0 +1,142 @@
/*
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
* @test
* @bug 4221795 8032446 8174270
* @summary Confirm that java.text.Normalizer and sun.text.Normalizer are
* thread-safe.
* @modules java.base/sun.text java.base/jdk.internal.icu.text
* @compile -XDignore.symbol.file ThreadSafeTest.java
* @run main/othervm -esa ThreadSafeTest 5 10
*/
// Usage: java ThreadSafeTest [threadsFactor [duration]]
public class ThreadSafeTest {
static volatile boolean runrun = true;
static volatile boolean error = false;
public static void main(String[] args) throws Exception {
int threadsFactor = 5;
if (args.length > 0) {
threadsFactor = Math.max(Integer.parseInt(args[0]), 5);
}
int duration = 180;
if (args.length > 1) {
duration = Math.max(5, Integer.parseInt(args[1]));
}
int nProcessors = Runtime.getRuntime().availableProcessors();
int nTasks = nProcessors * threadsFactor;
Thread[] tasks = new Thread[nTasks];
System.out.println("Testing with " + nTasks + " threads on " +
nProcessors + " processors for " + duration +
" seconds.");
for (int i = 0; i < nTasks; i++) {
tasks[i] = new Thread(new Worker());
}
for (int i = 0; i < nTasks; i++) {
tasks[i].start();
}
try {
for (int i = 0; runrun && i < duration; i++) {
Thread.sleep(1000); // 1 second
}
runrun = false;
for (int i = 0; i < nTasks; i++) {
tasks[i].join();
}
}
catch (InterruptedException e) {
}
if (error) {
throw new RuntimeException("Normalizer is not thread-safe.");
}
}
static void testJavaNormalize(int num, java.text.Normalizer.Form form) {
String got = java.text.Normalizer.normalize(data[num][0], form);
if (!got.equals(data[num][1])) {
System.err.println("java.text.Normalizer.normalize(" +
form.toString() + ") failed.");
error = true;
}
}
static void testSunNormalize(int num, java.text.Normalizer.Form form,
int option) {
String got = sun.text.Normalizer.normalize(data[num][0], form, option);
if (!got.equals(data[num][1])) {
System.err.println("sun.text.Normalizer.normalize(" +
form.toString() + ", " +
Integer.toHexString(option) + ") failed.");
error = true;
}
}
static void testIsNormalized(int num, java.text.Normalizer.Form form) {
boolean normalized = java.text.Normalizer.isNormalized(data[num][1], form);
if (!normalized) {
System.err.println("java.text.Normalizer.isNormalized(" +
form.toString() + ") failed.");
error = true;
}
}
static class Worker implements Runnable {
public void run() {
while (runrun) {
testJavaNormalize(0, java.text.Normalizer.Form.NFKC);
testSunNormalize(1, java.text.Normalizer.Form.NFC,
sun.text.Normalizer.UNICODE_3_2);
testJavaNormalize(2, java.text.Normalizer.Form.NFKD);
testSunNormalize(3, java.text.Normalizer.Form.NFC,
jdk.internal.icu.text.NormalizerBase.UNICODE_LATEST);
testJavaNormalize(4, java.text.Normalizer.Form.NFD);
testIsNormalized(0, java.text.Normalizer.Form.NFKC);
testIsNormalized(2, java.text.Normalizer.Form.NFKD);
testIsNormalized(4, java.text.Normalizer.Form.NFD);
if (error) {
runrun = false;
return;
}
}
}
}
static final String[][] data = {
/* From: To: */
{"A\u0300\u0316", "\u00C0\u0316"},
{"\u0071\u0307\u0323\u0072", "\u0071\u0323\u0307\u0072"},
{"\u0f77", "\u0fb2\u0f71\u0f80"},
{"D\u0307\u0328\u0323", "\u1e0c\u0328\u0307"},
{"\u0f71\u0f72\u0f73\u0f74\u0f75",
"\u0F71\u0F71\u0F71\u0F72\u0F72\u0F74\u0F74"},
};
}