undefect. CWE-407 — 63 sites patched across 27 ecosystems
Authors: russell@unturf.com · brackishbert@gmail.com · foxhop.net · TimeHexOn.com Patches, unit tests, benchmarks, whitepaper, and outreach briefs. Public domain — no copyright claimed. Use freely.
This commit is contained in:
commit
0a580b313d
70422 changed files with 17213626 additions and 0 deletions
527
test/jdk/java/text/Normalizer/ConformanceTest.java
Normal file
527
test/jdk/java/text/Normalizer/ConformanceTest.java
Normal file
|
|
@ -0,0 +1,527 @@
|
|||
/*
|
||||
* Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
|
||||
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
||||
*
|
||||
* This code is free software; you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License version 2 only, as
|
||||
* published by the Free Software Foundation.
|
||||
*
|
||||
* This code is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
||||
* version 2 for more details (a copy is included in the LICENSE file that
|
||||
* accompanied this code).
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License version
|
||||
* 2 along with this work; if not, write to the Free Software Foundation,
|
||||
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*
|
||||
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
|
||||
* or visit www.oracle.com if you need additional information or have any
|
||||
* questions.
|
||||
*/
|
||||
/*
|
||||
* @test
|
||||
* @bug 4221795 6565620 6959267 7070436 7198195 8032446 8174270 8221431 8239383
|
||||
* @summary Confirm Normalizer's fundamental behavior
|
||||
* @library /lib/testlibrary/java/lang
|
||||
* @modules java.base/sun.text java.base/jdk.internal.icu.text
|
||||
* @compile -XDignore.symbol.file ConformanceTest.java
|
||||
* @run main/timeout=3000 ConformanceTest
|
||||
*/
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.CharsetDecoder;
|
||||
import java.util.BitSet;
|
||||
import java.util.StringTokenizer;
|
||||
|
||||
import jdk.internal.icu.text.NormalizerBase;
|
||||
|
||||
/*
|
||||
* Conformance test for java.text.Normalizer and sun.text.Normalizer.
|
||||
*/
|
||||
public class ConformanceTest {
|
||||
|
||||
//
|
||||
// Options to be used with sun.text.Normalizer
|
||||
//
|
||||
|
||||
/*
|
||||
* Default Unicode 3.2.0 normalization. (Provided for IDNA/StringPrep)
|
||||
*
|
||||
* - Without Corrigendum 4 fix
|
||||
* (Different from ICU4J 3.2's Normalizer.)
|
||||
* - Without Public Review Issue #29 fix
|
||||
* (Different from ICU4J 3.2's Normalizer.)
|
||||
*/
|
||||
private static final int UNICODE_3_2_0 = sun.text.Normalizer.UNICODE_3_2;
|
||||
|
||||
/*
|
||||
* Original Unicode 3.2.0 normalization. (Provided for testing only)
|
||||
*
|
||||
* - With Corrigendum 4 fix
|
||||
* - With Public Revilew Issue #29 fix
|
||||
*/
|
||||
private static final int UNICODE_3_2_0_ORIGINAL =
|
||||
NormalizerBase.UNICODE_3_2;
|
||||
|
||||
/*
|
||||
* Default normalization. In JDK 6,
|
||||
* - Unicode 4.0.0
|
||||
* - With Corrigendum 4 fix
|
||||
* - Without Public Review Issue #29 fix
|
||||
*
|
||||
* In JDK 7,
|
||||
* - Unicode 5.1.0
|
||||
* (Different from ICU4J 3.2's Normalizer.)
|
||||
* - With Corrigendum 4 fix
|
||||
* - With Public Review Issue #29 fix
|
||||
*
|
||||
* In JDK 8,
|
||||
* - Unicode 6.1.0
|
||||
* - With Corrigendum 4 fix
|
||||
* - With Public Review Issue #29 fix
|
||||
*
|
||||
* When we support Unicode 4.1.0 or later, we need to do normalization
|
||||
* with Public Review Issue #29 fix. For more details of PRI #29, see
|
||||
* http://unicode.org/review/pr-29.html .
|
||||
*/
|
||||
private static final int UNICODE_LATEST = NormalizerBase.UNICODE_LATEST;
|
||||
|
||||
//
|
||||
// Conformance test datafiles
|
||||
//
|
||||
|
||||
/*
|
||||
* Conformance test datafile for Unicode 3.2.0 with Corrigendum4
|
||||
* corrections.
|
||||
* This testdata is for sun.text.Normalize(UNICODE_3_2)
|
||||
*
|
||||
* This is NOT an original Conformace test data. Some inconvenient test
|
||||
* cases are commented out. About corrigendum 4, please refer
|
||||
* http://www.unicode.org/review/resolved-pri.html#pri29
|
||||
*
|
||||
*/
|
||||
static final String DATA_3_2_0_CORRIGENDUM =
|
||||
"NormalizationTest-3.2.0.Corrigendum4.txt";
|
||||
|
||||
/*
|
||||
* Conformance test datafile for Unicode 3.2.0 without Corrigendum4
|
||||
* corrections. This is the original Conformace test data.
|
||||
*
|
||||
* This testdata is for sun.text.Normalize(UNICODE_3_2_IDNA)
|
||||
*/
|
||||
static final String DATA_3_2_0 = "NormalizationTest-3.2.0.txt";
|
||||
|
||||
/*
|
||||
* Conformance test datafile for the latest Unicode which is supported
|
||||
* by J2SE.
|
||||
* Unicode 4.0.0 is the latest version in JDK 5.0 and JDK 6. Unicode 5.1.0
|
||||
* in JDK 7, and 6.1.0 in JDK 8. This Unicode can be used via both
|
||||
* java.text.Normalizer and sun.text.Normalizer.
|
||||
*
|
||||
* This testdata is for sun.text.Normalize(UNICODE_LATEST)
|
||||
*/
|
||||
static final String DATA_LATEST = "NormalizationTest.txt";
|
||||
|
||||
/*
|
||||
* Conformance test datafile in ICU4J 3.2.
|
||||
*/
|
||||
static final String DATA_ICU = "ICUNormalizationTest.txt";
|
||||
|
||||
/*
|
||||
* Decorder
|
||||
*/
|
||||
static final CharsetDecoder decoder = Charset.forName("UTF-8").newDecoder();
|
||||
|
||||
/*
|
||||
* List to pick up characters which are not listed in Part1
|
||||
*/
|
||||
static BitSet charList = new BitSet(Character.MAX_CODE_POINT+1);
|
||||
|
||||
/*
|
||||
* Shortcuts
|
||||
*/
|
||||
private static final java.text.Normalizer.Form NFC =
|
||||
java.text.Normalizer.Form.NFC;
|
||||
private static final java.text.Normalizer.Form NFD =
|
||||
java.text.Normalizer.Form.NFD;
|
||||
private static final java.text.Normalizer.Form NFKC =
|
||||
java.text.Normalizer.Form.NFKC;
|
||||
private static final java.text.Normalizer.Form NFKD =
|
||||
java.text.Normalizer.Form.NFKD;
|
||||
static final java.text.Normalizer.Form[] forms = {NFC, NFD, NFKC, NFKD};
|
||||
|
||||
|
||||
static TestNormalizer normalizer;
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
ConformanceTest ct = new ConformanceTest();
|
||||
ct.test();
|
||||
}
|
||||
|
||||
void test() throws Exception {
|
||||
normalizer = new testJavaNormalizer();
|
||||
test(DATA_LATEST, UNICODE_LATEST);
|
||||
|
||||
normalizer = new testSunNormalizer();
|
||||
test(DATA_3_2_0_CORRIGENDUM, UNICODE_3_2_0);
|
||||
test(DATA_LATEST, UNICODE_LATEST);
|
||||
test(DATA_ICU, UNICODE_LATEST);
|
||||
|
||||
/* Unconformity test */
|
||||
// test(DATA_3_2_0, UNICODE_LATEST);
|
||||
// test(DATA_LATEST, UNICODE_3_2_0);
|
||||
}
|
||||
|
||||
/*
|
||||
* Main routine of conformance test
|
||||
*/
|
||||
private static void test(String filename, int unicodeVer) throws Exception {
|
||||
|
||||
File f = filename.equals(DATA_LATEST) ?
|
||||
UCDFiles.NORMALIZATION_TEST.toFile() :
|
||||
new File(System.getProperty("test.src", "."), filename);
|
||||
FileInputStream fis = new FileInputStream(f);
|
||||
BufferedReader in =
|
||||
new BufferedReader(new InputStreamReader(fis, decoder));
|
||||
|
||||
System.out.println("\nStart testing for " + normalizer.name +
|
||||
" with " + filename + " for options: " +
|
||||
(((unicodeVer & NormalizerBase.UNICODE_3_2) != 0) ?
|
||||
"Unicode 3.2.0" : "the latest Unicode"));
|
||||
|
||||
int lineNo = 0;
|
||||
String text;
|
||||
boolean part1test = false;
|
||||
boolean part1testExists = false;
|
||||
String[] columns = new String[6];
|
||||
|
||||
while ((text = in.readLine()) != null) {
|
||||
lineNo ++;
|
||||
|
||||
char c = text.charAt(0);
|
||||
if (c == '#') {
|
||||
continue;
|
||||
} else if (c == '@') {
|
||||
if (text.startsWith("@Part")) {
|
||||
System.out.println("# Testing data in " + text);
|
||||
|
||||
if (text.startsWith("@Part1 ")) {
|
||||
part1test = true;
|
||||
part1testExists = true;
|
||||
} else {
|
||||
part1test = false;
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
prepareColumns(columns, text, filename, lineNo, part1test);
|
||||
|
||||
testNFC(columns, unicodeVer, filename, lineNo);
|
||||
testNFD(columns, unicodeVer, filename, lineNo);
|
||||
testNFKC(columns, unicodeVer, filename, lineNo);
|
||||
testNFKD(columns, unicodeVer, filename, lineNo);
|
||||
}
|
||||
|
||||
in.close();
|
||||
fis.close();
|
||||
|
||||
if (part1testExists) {
|
||||
System.out.println("# Testing characters which are not listed in Part1");
|
||||
testRemainingChars(filename, unicodeVer);
|
||||
part1testExists = false;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Test for NFC
|
||||
*
|
||||
* c2 == NFC(c1) == NFC(c2) == NFC(c3)
|
||||
* c4 == NFC(c4) == NFC(c5)
|
||||
*/
|
||||
private static void testNFC(String[] c, int unicodeVer,
|
||||
String file, int line) throws Exception {
|
||||
test(2, c, 1, 3, NFC, unicodeVer, file, line);
|
||||
test(4, c, 4, 5, NFC, unicodeVer, file, line);
|
||||
}
|
||||
|
||||
/*
|
||||
* Test for NFD
|
||||
*
|
||||
* c3 == NFD(c1) == NFD(c2) == NFD(c3)
|
||||
* c5 == NFD(c4) == NFD(c5)
|
||||
*/
|
||||
private static void testNFD(String[] c, int unicodeVer,
|
||||
String file, int line) throws Exception {
|
||||
test(3, c, 1, 3, NFD, unicodeVer, file, line);
|
||||
test(5, c, 4, 5, NFD, unicodeVer, file, line);
|
||||
}
|
||||
|
||||
/*
|
||||
* Test for NFKC
|
||||
*
|
||||
* c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
|
||||
*/
|
||||
private static void testNFKC(String[] c, int unicodeVer,
|
||||
String file, int line) throws Exception {
|
||||
test(4, c, 1, 5, NFKC, unicodeVer, file, line);
|
||||
}
|
||||
|
||||
/*
|
||||
* Test for NFKD
|
||||
*
|
||||
* c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
|
||||
*/
|
||||
private static void testNFKD(String[] c, int unicodeVer,
|
||||
String file, int line) throws Exception {
|
||||
test(5, c, 1, 5, NFKD, unicodeVer, file, line);
|
||||
}
|
||||
|
||||
/*
|
||||
* Test for characters which aren't listed in Part1
|
||||
*
|
||||
* X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
|
||||
*/
|
||||
private static void testRemainingChars(String file,
|
||||
int unicodeVer) throws Exception {
|
||||
for (int i = Character.MIN_CODE_POINT;
|
||||
i <= Character.MAX_CODE_POINT;
|
||||
i++) {
|
||||
if (!charList.get(i)) {
|
||||
String from = String.valueOf(Character.toChars(i));
|
||||
String to;
|
||||
|
||||
for (int j = 0; j < forms.length; j++) {
|
||||
java.text.Normalizer.Form form = forms[j];
|
||||
|
||||
to = normalizer.normalize(from, form, unicodeVer);
|
||||
if (!from.equals(to)) {
|
||||
error(form, from, from, to, file, -1);
|
||||
// } else {
|
||||
// okay(form, from, from, to, file, -1);
|
||||
}
|
||||
|
||||
if (!normalizer.isNormalized(from, form, unicodeVer)) {
|
||||
error(form, from, file, -1);
|
||||
// } else {
|
||||
// okay(form, from, file, -1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Test normalize() and isNormalized()
|
||||
*/
|
||||
private static void test(int col, String[] c,
|
||||
int FROM, int TO,
|
||||
java.text.Normalizer.Form form, int unicodeVer,
|
||||
String file, int line) throws Exception {
|
||||
for (int i = FROM; i <= TO; i++) {
|
||||
String got = normalizer.normalize(c[i], form, unicodeVer);
|
||||
if (!c[col].equals(got)) {
|
||||
error(form, c[i], c[col], got, file, line);
|
||||
// } else {
|
||||
// okay(form, c[i], c[col], got, file, line);
|
||||
}
|
||||
|
||||
/*
|
||||
* If the original String equals its normalized String, it means
|
||||
* that the original String is normalizerd. Thus, isNormalized()
|
||||
* should return true. And, vice versa!
|
||||
*/
|
||||
if (c[col].equals(c[i])) {
|
||||
if (!normalizer.isNormalized(c[i], form, unicodeVer)) {
|
||||
error(form, c[i], file, line);
|
||||
// } else {
|
||||
// okay(form, c[i], file, line);
|
||||
}
|
||||
} else {
|
||||
if (normalizer.isNormalized(c[i], form, unicodeVer)) {
|
||||
error(form, c[i], file, line);
|
||||
// } else {
|
||||
// okay(form, c[i], file, line);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Generate an array of String from a line of conformance datafile.
|
||||
*/
|
||||
private static void prepareColumns(String[] cols, String text,
|
||||
String file, int line,
|
||||
boolean part1test) throws Exception {
|
||||
int index = text.indexOf('#');
|
||||
if (index != -1) {
|
||||
text = text.substring(0, index);
|
||||
}
|
||||
|
||||
StringTokenizer st = new StringTokenizer(text, ";");
|
||||
int tokenCount = st.countTokens();
|
||||
if (tokenCount < 5) {
|
||||
throw new RuntimeException("# of tokens in datafile should be 6, but got: " + tokenCount + " at line " + line + " in " + file);
|
||||
}
|
||||
|
||||
StringBuffer sb = new StringBuffer();
|
||||
for (int i = 1; i <= 5; i++) {
|
||||
StringTokenizer tst = new StringTokenizer(st.nextToken(), " ");
|
||||
|
||||
while (tst.hasMoreTokens()) {
|
||||
int code = Integer.parseInt(tst.nextToken(), 16);
|
||||
sb.append(Character.toChars(code));
|
||||
}
|
||||
|
||||
cols[i] = sb.toString();
|
||||
sb.setLength(0);
|
||||
}
|
||||
|
||||
if (part1test) {
|
||||
charList.set(cols[1].codePointAt(0));
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Show an error message when normalize() didn't return the expected value.
|
||||
* (An exception is sometimes convenient. Therefore, it is commented out
|
||||
* for the moment.)
|
||||
*/
|
||||
private static void error(java.text.Normalizer.Form form,
|
||||
String from, String to, String got,
|
||||
String file, int line) throws Exception {
|
||||
System.err.println("-\t" + form.toString() + ": normalize(" +
|
||||
toHexString(from) + ") doesn't equal <" + toHexString(to) +
|
||||
"> at line " + line + " in " + file + ". Got [" +
|
||||
toHexString(got) + "]");
|
||||
throw new RuntimeException("Normalization(" + form.toString() + ") failed");
|
||||
}
|
||||
|
||||
/*
|
||||
* Show an error message when isNormalize() didn't return the expected
|
||||
* value.
|
||||
* (An exception is sometimes convenient. Therefore, it is commented out
|
||||
* for the moment.)
|
||||
*/
|
||||
private static void error(java.text.Normalizer.Form form, String s,
|
||||
String file, int line) throws Exception {
|
||||
System.err.println("\t" + form.toString() + ": isNormalized(" +
|
||||
toHexString(s) + ") returned the wrong value at line " + line +
|
||||
" in " + file);
|
||||
throw new RuntimeException("Normalization(" + form.toString() +") failed");
|
||||
}
|
||||
|
||||
/*
|
||||
* (For debugging)
|
||||
* Shows a message when normalize() returned the expected value.
|
||||
*/
|
||||
private static void okay(java.text.Normalizer.Form form,
|
||||
String from, String to, String got,
|
||||
String file, int line) {
|
||||
System.out.println("\t" + form.toString() + ": normalize(" +
|
||||
toHexString(from) + ") equals <" + toHexString(to) +
|
||||
"> at line " + line + " in " + file + ". Got [" +
|
||||
toHexString(got) + "]");
|
||||
}
|
||||
|
||||
/*
|
||||
* (For debugging)
|
||||
* Shows a message when isNormalized() returned the expected value.
|
||||
*/
|
||||
private static void okay(java.text.Normalizer.Form form, String s,
|
||||
String file, int line) {
|
||||
System.out.println("\t" + form.toString() + ": isNormalized(" +
|
||||
toHexString(s) + ") returned the correct value at line " +
|
||||
line + " in " + file);
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns a spece-delimited hex String
|
||||
*/
|
||||
private static String toHexString(String s) {
|
||||
StringBuffer sb = new StringBuffer(" ");
|
||||
|
||||
for (int i = 0; i < s.length(); i++) {
|
||||
sb.append(Integer.toHexString(s.charAt(i)));
|
||||
sb.append(' ');
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/*
|
||||
* Abstract class to call each Normalizer in java.text or sun.text.
|
||||
*/
|
||||
private abstract class TestNormalizer {
|
||||
String name;
|
||||
|
||||
TestNormalizer(String str) {
|
||||
name = str;
|
||||
}
|
||||
|
||||
String getNormalizerName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
abstract String normalize(CharSequence cs,
|
||||
java.text.Normalizer.Form form,
|
||||
int option);
|
||||
|
||||
abstract boolean isNormalized(CharSequence cs,
|
||||
java.text.Normalizer.Form form,
|
||||
int option);
|
||||
}
|
||||
|
||||
/*
|
||||
* For java.text.Normalizer
|
||||
* - normalize(CharSequence, Normalizer.Form)
|
||||
* - isNormalized(CharSequence, Normalizer.Form)
|
||||
*/
|
||||
private class testJavaNormalizer extends TestNormalizer {
|
||||
testJavaNormalizer() {
|
||||
super("java.text.Normalizer");
|
||||
}
|
||||
|
||||
String normalize(CharSequence cs,
|
||||
java.text.Normalizer.Form form,
|
||||
int option) {
|
||||
return java.text.Normalizer.normalize(cs, form);
|
||||
}
|
||||
|
||||
boolean isNormalized(CharSequence cs,
|
||||
java.text.Normalizer.Form form,
|
||||
int option) {
|
||||
return java.text.Normalizer.isNormalized(cs, form);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* For sun.text.Normalizer
|
||||
* - normalize(CharSequence, Normalizer.Form, int)
|
||||
* - isNormalized(CharSequence, Normalizer.Form, int)
|
||||
*/
|
||||
private class testSunNormalizer extends TestNormalizer {
|
||||
testSunNormalizer() {
|
||||
super("sun.text.Normalizer");
|
||||
}
|
||||
|
||||
String normalize(CharSequence cs,
|
||||
java.text.Normalizer.Form form,
|
||||
int option) {
|
||||
return sun.text.Normalizer.normalize(cs, form, option);
|
||||
}
|
||||
|
||||
boolean isNormalized(CharSequence cs,
|
||||
java.text.Normalizer.Form form,
|
||||
int option) {
|
||||
return sun.text.Normalizer.isNormalized(cs, form, option);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue