From d24dbe78c56546941da73080837161d97d57b3d9 Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sun, 24 Jul 2016 07:31:55 +0200 Subject: [PATCH 1/5] add a real test for native2ascii --- .../taskdefs/optional/native2ascii-test.xml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml index 7a30df795..1f7aa7fe1 100644 --- a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml +++ b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml @@ -65,4 +65,50 @@ public class Adapter implements Native2AsciiAdapter { + + + + + äöü=ÄÖÜ + + \u00e4\u00f6\u00fc=\u00c4\u00d6\u00dc + + + + + + + + + + + + + + + + + \u00e4\u00f6\u00fc=\u00c4\u00d6\u00dc + + äöü=ÄÖÜ + + + + + + + + From 270fb2cf382a09ef8abe21879f8b0acb1eb36bf5 Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sun, 24 Jul 2016 11:00:15 +0200 Subject: [PATCH 2/5] try to get better error reporting from Jenkins --- .../taskdefs/optional/native2ascii-test.xml | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml index 1f7aa7fe1..a7e6b5731 100644 --- a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml +++ b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml @@ -66,49 +66,62 @@ public class Adapter implements Native2AsciiAdapter { - + + + + + + + + + + + + + + + + + + + + - äöü=ÄÖÜ - - \u00e4\u00f6\u00fc=\u00c4\u00d6\u00dc - + äöü=ÄÖÜ + - - + + - - + + - + - \u00e4\u00f6\u00fc=\u00c4\u00d6\u00dc - - äöü=ÄÖÜ - + \u00e4\u00f6\u00fc=\u00c4\u00d6\u00dc + - - + + From 0319f8134564bb4a4912608e046728fd72a9fcf4 Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sun, 24 Jul 2016 12:19:51 +0200 Subject: [PATCH 3/5] built-in implementation of native2ascii https://bz.apache.org/bugzilla/show_bug.cgi?id=59855 --- WHATSNEW | 5 ++ manual/Tasks/native2ascii.html | 11 ++- .../native2ascii/BuiltinNative2Ascii.java | 82 +++++++++++++++++++ .../Native2AsciiAdapterFactory.java | 29 +++++-- .../tools/ant/util/Native2AsciiUtils.java | 51 ++++++++++++ .../taskdefs/optional/native2ascii-test.xml | 7 ++ .../tools/ant/util/Native2AsciiUtilsTest.java | 46 +++++++++++ 7 files changed, 221 insertions(+), 10 deletions(-) create mode 100644 src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java create mode 100644 src/main/org/apache/tools/ant/util/Native2AsciiUtils.java create mode 100644 src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java diff --git a/WHATSNEW b/WHATSNEW index bd90733d6..0253e7e59 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -39,6 +39,11 @@ Other changes: * now supports compression via the optional compressed attribute. Bugzilla Report 47552 + * a new implementation "builtin" has been added to and + is the default when running on JDK9+ since the tool itself has been + removed from the JDK. + Bugzilla Report 59855 + Changes from Ant 1.9.6 TO Ant 1.9.7 =================================== diff --git a/manual/Tasks/native2ascii.html b/manual/Tasks/native2ascii.html index c50483b9f..acbd46085 100644 --- a/manual/Tasks/native2ascii.html +++ b/manual/Tasks/native2ascii.html @@ -61,9 +61,14 @@ with the implementation attribute or a nested element. Here are the choices of the attribute:

    -
  • default - the default converter (kaffe or sun) for the platform.
    • -
  • sun (the standard converter of the JDK)
    • -
  • kaffe (the standard converter of Kaffe)
    • +
  • default - the default converter for the platform - kaffee + when run on Kaffee, builtin if JDK9 or newer is detected, sun + otherwise.
    • +
  • sun (the standard converter of the JDK < 9)
    • +
  • kaffe (the standard converter + of Kaffe)
    • +
  • builtin - Ant's internal implementation used for + JDK9+. since ant 1.9.8
diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java b/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java new file mode 100644 index 000000000..b9e600ced --- /dev/null +++ b/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.tools.ant.taskdefs.optional.native2ascii; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Writer; + +import org.apache.tools.ant.BuildException; +import org.apache.tools.ant.taskdefs.optional.Native2Ascii; +import org.apache.tools.ant.util.FileUtils; +import org.apache.tools.ant.util.Native2AsciiUtils; +import org.apache.tools.ant.util.StringUtils; + +/** + * Encapsulates the built-in Native2Ascii implementation. + * + * @since Ant 1.9.8 + */ +public class BuiltinNative2Ascii implements Native2AsciiAdapter { + + static final String IMPLEMENTATION_NAME = "builtin"; + + public final boolean convert(Native2Ascii args, File srcFile, + File destFile) throws BuildException { + BufferedReader input = null; + try { + if (args.getEncoding() != null) { + input = new BufferedReader(new InputStreamReader( + new FileInputStream(srcFile), args.getEncoding())); + } else { + input = new BufferedReader(new FileReader(srcFile)); + } + try { + BufferedWriter output = new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(destFile), + "ASCII")); + try { + translate(input, output, args.getReverse()); + } finally { + FileUtils.close(output); + } + } finally { + FileUtils.close(input); + } + return true; + } catch (IOException ex) { + throw new BuildException("Exception trying to translate data", ex); + } + } + + private void translate(BufferedReader input, Writer output, + boolean reverse) throws IOException { + String line = null; + while ((line = input.readLine()) != null) { + output.write(Native2AsciiUtils.native2ascii(line)); + output.write(StringUtils.LINE_SEP); + } + } +} diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/Native2AsciiAdapterFactory.java b/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/Native2AsciiAdapterFactory.java index ae4190395..e8bd74d16 100644 --- a/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/Native2AsciiAdapterFactory.java +++ b/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/Native2AsciiAdapterFactory.java @@ -40,10 +40,13 @@ public class Native2AsciiAdapterFactory { * vendor */ public static String getDefault() { - if (JavaEnvUtils.isKaffe() || JavaEnvUtils.isClasspathBased()) { + if (shouldUseKaffee()) { return KaffeNative2Ascii.IMPLEMENTATION_NAME; } - return SunNative2Ascii.IMPLEMENTATION_NAME; + if (shouldUseSun()) { + return SunNative2Ascii.IMPLEMENTATION_NAME; + } + return BuiltinNative2Ascii.IMPLEMENTATION_NAME; } /** @@ -79,11 +82,14 @@ public class Native2AsciiAdapterFactory { ProjectComponent log, Path classpath) throws BuildException { - if (((JavaEnvUtils.isKaffe() || JavaEnvUtils.isClasspathBased()) && choice == null) + if ((shouldUseKaffee() && choice == null) || KaffeNative2Ascii.IMPLEMENTATION_NAME.equals(choice)) { return new KaffeNative2Ascii(); - } else if (SunNative2Ascii.IMPLEMENTATION_NAME.equals(choice)) { + } else if ((shouldUseSun() && choice == null) + || SunNative2Ascii.IMPLEMENTATION_NAME.equals(choice)) { return new SunNative2Ascii(); + } else if (BuiltinNative2Ascii.IMPLEMENTATION_NAME.equals(choice)) { + return new BuiltinNative2Ascii(); } else if (choice != null) { return resolveClassName(choice, // Memory leak in line below @@ -91,9 +97,7 @@ public class Native2AsciiAdapterFactory { .createClassLoader(classpath)); } - // This default has been good enough until Ant 1.6.3, so stick - // with it - return new SunNative2Ascii(); + return new BuiltinNative2Ascii(); } /** @@ -113,4 +117,15 @@ public class Native2AsciiAdapterFactory { Native2AsciiAdapterFactory.class.getClassLoader(), Native2AsciiAdapter.class); } + + private static final boolean shouldUseKaffee() { + return JavaEnvUtils.isKaffe() || JavaEnvUtils.isClasspathBased(); + } + + private static final boolean shouldUseSun() { + return JavaEnvUtils.isJavaVersion(JavaEnvUtils.JAVA_1_5) + || JavaEnvUtils.isJavaVersion(JavaEnvUtils.JAVA_1_6) + || JavaEnvUtils.isJavaVersion(JavaEnvUtils.JAVA_1_7) + || JavaEnvUtils.isJavaVersion(JavaEnvUtils.JAVA_1_8); + } } diff --git a/src/main/org/apache/tools/ant/util/Native2AsciiUtils.java b/src/main/org/apache/tools/ant/util/Native2AsciiUtils.java new file mode 100644 index 000000000..acdfaaa92 --- /dev/null +++ b/src/main/org/apache/tools/ant/util/Native2AsciiUtils.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.tools.ant.util; + +/** + * Contains helper methods for Ant's built-in implementation of native2ascii. + * + * @since Ant 1.9.8 + */ +public class Native2AsciiUtils { + + private static final int MAX_ASCII = 127; + + /** + * Replaces non-ASCII characters with their Unicode-Escapes. + *

Expects to be called once per line if applied to a file.

+ * @param line the input line + * @return the translated line + */ + public static String native2ascii(String line) { + StringBuilder sb = new StringBuilder(); + for (char c : line.toCharArray()) { + if (c <= MAX_ASCII) { + sb.append(c); + } else { + sb.append("\\u"); + String encoded = Integer.toHexString(c); + for (int i = encoded.length(); i < 4; i++) { + sb.append("0"); + } + sb.append(encoded); + } + } + return sb.toString(); + } +} diff --git a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml index a7e6b5731..8705f6a1c 100644 --- a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml +++ b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml @@ -109,6 +109,13 @@ public class Adapter implements Native2AsciiAdapter { expected="${umlauts.expected}"/> + + + + + diff --git a/src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java b/src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java new file mode 100644 index 000000000..75719f2ab --- /dev/null +++ b/src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.tools.ant.util; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class Native2AsciiUtilsTest { + + @Test + public void doesntTouchAscii() { + StringBuilder sb = new StringBuilder(); + for (char i = 0; i < 128; i++) { + sb.append(i); + } + assertEquals(sb.toString(), Native2AsciiUtils.native2ascii(sb.toString())); + } + + @Test + public void escapes() { + assertEquals("\\u00e4\\u00f6\\u00fc", + Native2AsciiUtils.native2ascii("\u00e4\u00f6\u00fc")); + } + + @Test + public void pads() { + assertEquals("\\u00e4\\u01f6\\u12fc", + Native2AsciiUtils.native2ascii("\u00e4\u01f6\u12fc")); + } +} From 51bd1cbff7615c621dfa94b6ad2b938e9b747d49 Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sun, 24 Jul 2016 12:21:10 +0200 Subject: [PATCH 4/5] try to make tests pass with default encoding != UTF-8 --- src/tests/antunit/taskdefs/optional/native2ascii-test.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml index 8705f6a1c..59030e947 100644 --- a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml +++ b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml @@ -82,7 +82,7 @@ public class Adapter implements Native2AsciiAdapter { - From 4ef2554a53506a186bc11b5b505ec64cf902595f Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sun, 24 Jul 2016 12:51:53 +0200 Subject: [PATCH 5/5] builtin implementation of ascii2native --- manual/Tasks/native2ascii.html | 2 +- .../native2ascii/BuiltinNative2Ascii.java | 44 ++++++++++++++----- .../tools/ant/util/Native2AsciiUtils.java | 38 ++++++++++++++++ .../taskdefs/optional/native2ascii-test.xml | 13 +++--- .../tools/ant/util/Native2AsciiUtilsTest.java | 30 +++++++++++++ 5 files changed, 109 insertions(+), 18 deletions(-) diff --git a/manual/Tasks/native2ascii.html b/manual/Tasks/native2ascii.html index acbd46085..ecc065a3d 100644 --- a/manual/Tasks/native2ascii.html +++ b/manual/Tasks/native2ascii.html @@ -81,7 +81,7 @@ + sun and builtin converters diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java b/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java index b9e600ced..db91a2638 100644 --- a/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java +++ b/src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; @@ -45,20 +46,15 @@ public class BuiltinNative2Ascii implements Native2AsciiAdapter { public final boolean convert(Native2Ascii args, File srcFile, File destFile) throws BuildException { + boolean reverse = args.getReverse(); + String encoding = args.getEncoding(); BufferedReader input = null; try { - if (args.getEncoding() != null) { - input = new BufferedReader(new InputStreamReader( - new FileInputStream(srcFile), args.getEncoding())); - } else { - input = new BufferedReader(new FileReader(srcFile)); - } + input = getReader(srcFile, encoding, reverse); try { - BufferedWriter output = new BufferedWriter( - new OutputStreamWriter(new FileOutputStream(destFile), - "ASCII")); + Writer output = getWriter(destFile, encoding, reverse); try { - translate(input, output, args.getReverse()); + translate(input, output, reverse); } finally { FileUtils.close(output); } @@ -71,11 +67,37 @@ public class BuiltinNative2Ascii implements Native2AsciiAdapter { } } + private BufferedReader getReader(File srcFile, String encoding, + boolean reverse) throws IOException { + if (!reverse && encoding != null) { + return new BufferedReader(new InputStreamReader( + new FileInputStream(srcFile), encoding)); + } + return new BufferedReader(new FileReader(srcFile)); + } + + private Writer getWriter(File destFile, String encoding, + boolean reverse) throws IOException { + if (!reverse) { + encoding = "ASCII"; + } + if (encoding != null) { + return new BufferedWriter( + new OutputStreamWriter(new FileOutputStream(destFile), + encoding)); + } + return new BufferedWriter(new FileWriter(destFile)); + } + private void translate(BufferedReader input, Writer output, boolean reverse) throws IOException { String line = null; while ((line = input.readLine()) != null) { - output.write(Native2AsciiUtils.native2ascii(line)); + if (!reverse) { + output.write(Native2AsciiUtils.native2ascii(line)); + } else { + output.write(Native2AsciiUtils.ascii2native(line)); + } output.write(StringUtils.LINE_SEP); } } diff --git a/src/main/org/apache/tools/ant/util/Native2AsciiUtils.java b/src/main/org/apache/tools/ant/util/Native2AsciiUtils.java index acdfaaa92..a69d0313d 100644 --- a/src/main/org/apache/tools/ant/util/Native2AsciiUtils.java +++ b/src/main/org/apache/tools/ant/util/Native2AsciiUtils.java @@ -48,4 +48,42 @@ public class Native2AsciiUtils { } return sb.toString(); } + + /** + * Replaces Unicode-Escapes. + *

Expects to be called once per line if applied to a file.

+ * @param line the input line + * @return the translated line + */ + public static String ascii2native(String line) { + StringBuilder sb = new StringBuilder(); + int inputLen = line.length(); + for (int i = 0; i < inputLen; i++) { + char c = line.charAt(i); + if (c != '\\' || i >= inputLen - 5) { + sb.append(c); + } else { // backslash with enough remaining characters + char u = line.charAt(++i); + if (u == 'u') { + int unescaped = tryParse(line, i + 1); + if (unescaped >= 0) { + sb.append((char) unescaped); + i += 4; + continue; + } + } + // not a unicode escape + sb.append(c).append(u); + } + } + return sb.toString(); + } + + private static int tryParse(String line, int startIdx) { + try { + return Integer.parseInt(line.substring(startIdx, startIdx + 4), 16); + } catch (NumberFormatException ex) { + return -1; + } + } } diff --git a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml index 59030e947..be521de7f 100644 --- a/src/tests/antunit/taskdefs/optional/native2ascii-test.xml +++ b/src/tests/antunit/taskdefs/optional/native2ascii-test.xml @@ -95,12 +95,6 @@ public class Adapter implements Native2AsciiAdapter { value="\u00e4\u00f6\u00fc=\u00c4\u00d6\u00dc"/> - - - - - @@ -131,4 +125,11 @@ public class Adapter implements Native2AsciiAdapter { expected="${umlauts.expected}"/> + + + + + diff --git a/src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java b/src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java index 75719f2ab..f3b54e619 100644 --- a/src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java +++ b/src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java @@ -43,4 +43,34 @@ public class Native2AsciiUtilsTest { assertEquals("\\u00e4\\u01f6\\u12fc", Native2AsciiUtils.native2ascii("\u00e4\u01f6\u12fc")); } + + @Test + public void doesntTouchNonEscapes() { + StringBuilder sb = new StringBuilder(); + for (char i = 0; i < 128; i++) { + sb.append(i); + } + assertEquals(sb.toString(), Native2AsciiUtils.ascii2native(sb.toString())); + } + + @Test + public void unescapes() { + assertEquals("\u00e4\u00f6\u00fc", + Native2AsciiUtils.ascii2native("\\u00e4\\u00f6\\u00fc")); + } + + @Test + public void leavesNonUnicodeBackslashesAlone() { + assertEquals("\\abcdef", Native2AsciiUtils.ascii2native("\\abcdef")); + assertEquals("\\u012j", Native2AsciiUtils.ascii2native("\\u012j")); + } + + @Test + public void dealsWithUnfinishedEscapes() { + assertEquals("\u00e4", Native2AsciiUtils.ascii2native("\\u00e4")); + assertEquals("\\u00e", Native2AsciiUtils.ascii2native("\\u00e")); + assertEquals("\\u00", Native2AsciiUtils.ascii2native("\\u00")); + assertEquals("\\u0", Native2AsciiUtils.ascii2native("\\u0")); + } + }
reverse Reverse the sense of the conversion, i.e. convert from ASCII to native only supported by the - sun converter No