diff --git a/docs/manual/CoreTypes/filterchain.html b/docs/manual/CoreTypes/filterchain.html
index 53640f3cc..2001d4f7b 100644
--- a/docs/manual/CoreTypes/filterchain.html
+++ b/docs/manual/CoreTypes/filterchain.html
@@ -1256,7 +1256,7 @@ Convert to uppercase.
Custom string filters and tokenizers may be plugged in by
extending the interfaces org.apache.tools.ant.filters.TokenFilter.Filter
-and org.apache.tools.ant.filters.TokenFilter.Tokenizer respectly.
+and org.apache.tools.ant.util.Tokenizer respectly.
They are defined the build file using <typedef/>. For
example a string filter that capitalizes words may be declared as:
diff --git a/src/main/org/apache/tools/ant/filters/HeadFilter.java b/src/main/org/apache/tools/ant/filters/HeadFilter.java
index ae4ea80e7..eb31e68c1 100644
--- a/src/main/org/apache/tools/ant/filters/HeadFilter.java
+++ b/src/main/org/apache/tools/ant/filters/HeadFilter.java
@@ -55,6 +55,7 @@ package org.apache.tools.ant.filters;
import java.io.IOException;
import java.io.Reader;
+import org.apache.tools.ant.util.LineTokenizer;
import org.apache.tools.ant.types.Parameter;
/**
@@ -91,7 +92,7 @@ public final class HeadFilter extends BaseParamFilterReader
private long skip = 0;
/** A line tokenizer */
- private TokenFilter.LineTokenizer lineTokenizer = null;
+ private LineTokenizer lineTokenizer = null;
/** the current line from the input stream */
private String line = null;
@@ -115,7 +116,7 @@ public final class HeadFilter extends BaseParamFilterReader
*/
public HeadFilter(final Reader in) {
super(in);
- lineTokenizer = new TokenFilter.LineTokenizer();
+ lineTokenizer = new LineTokenizer();
lineTokenizer.setIncludeDelims(true);
}
diff --git a/src/main/org/apache/tools/ant/filters/TailFilter.java b/src/main/org/apache/tools/ant/filters/TailFilter.java
index 9792fe8e6..ab0034597 100644
--- a/src/main/org/apache/tools/ant/filters/TailFilter.java
+++ b/src/main/org/apache/tools/ant/filters/TailFilter.java
@@ -57,6 +57,7 @@ import java.io.IOException;
import java.io.Reader;
import java.util.LinkedList;
import org.apache.tools.ant.types.Parameter;
+import org.apache.tools.ant.util.LineTokenizer;
/**
* Reads the last n lines of a stream. (Default is last10 lines.)
@@ -100,7 +101,7 @@ public final class TailFilter extends BaseParamFilterReader
private int bufferPos = 0;
/** A line tokenizer */
- private TokenFilter.LineTokenizer lineTokenizer = null;
+ private LineTokenizer lineTokenizer = null;
/** the current line from the input stream */
private String line = null;
@@ -126,7 +127,7 @@ public final class TailFilter extends BaseParamFilterReader
*/
public TailFilter(final Reader in) {
super(in);
- lineTokenizer = new TokenFilter.LineTokenizer();
+ lineTokenizer = new LineTokenizer();
lineTokenizer.setIncludeDelims(true);
}
diff --git a/src/main/org/apache/tools/ant/filters/TokenFilter.java b/src/main/org/apache/tools/ant/filters/TokenFilter.java
index e1529519c..e993c83ad 100644
--- a/src/main/org/apache/tools/ant/filters/TokenFilter.java
+++ b/src/main/org/apache/tools/ant/filters/TokenFilter.java
@@ -63,6 +63,8 @@ import org.apache.tools.ant.ProjectComponent;
import org.apache.tools.ant.types.RegularExpression;
import org.apache.tools.ant.types.Substitution;
import org.apache.tools.ant.util.FileUtils;
+import org.apache.tools.ant.util.Tokenizer;
+import org.apache.tools.ant.util.LineTokenizer;
import org.apache.tools.ant.util.regexp.Regexp;
/**
@@ -77,28 +79,6 @@ import org.apache.tools.ant.util.regexp.Regexp;
*/
public class TokenFilter extends BaseFilterReader
implements ChainableReader {
- /**
- * input stream tokenizers implement this interface
- */
- public interface Tokenizer {
- /**
- * get the next token from the input stream
- * @param in the input stream
- * @return the next token, or null for the end
- * of the stream
- * @throws IOException if an error occurs
- */
- String getToken(Reader in)
- throws IOException;
-
- /**
- * return the string between tokens, after the
- * previous token.
- * @return the intra-token string
- */
- String getPostToken();
- }
-
/**
* string filters implement this interface
*/
@@ -361,96 +341,6 @@ public class TokenFilter extends BaseFilterReader
}
}
-
- /**
- * class to tokenize the input as lines seperated
- * by \r (mac style), \r\n (dos/windows style) or \n (unix style)
- */
- public static class LineTokenizer extends ProjectComponent
- implements Tokenizer {
- private String lineEnd = "";
- private int pushed = -2;
- private boolean includeDelims = false;
-
- /**
- * attribute includedelims - whether to include
- * the line ending with the line, or to return
- * it in the posttoken
- * default false
- * @param includeDelims if true include /r and /n in the line
- */
-
- public void setIncludeDelims(boolean includeDelims) {
- this.includeDelims = includeDelims;
- }
-
- /**
- * get the next line from the input
- *
- * @param in the input reader
- * @return the line excluding /r or /n, unless includedelims is set
- * @exception IOException if an error occurs reading
- */
- public String getToken(Reader in) throws IOException {
- int ch = -1;
- if (pushed != -2) {
- ch = pushed;
- pushed = -2;
- } else {
- ch = in.read();
- }
- if (ch == -1) {
- return null;
- }
-
- lineEnd = "";
- StringBuffer line = new StringBuffer();
-
- int state = 0;
- while (ch != -1) {
- if (state == 0) {
- if (ch == '\r') {
- state = 1;
- } else if (ch == '\n') {
- lineEnd = "\n";
- break;
- } else {
- line.append((char) ch);
- }
- } else {
- state = 0;
- if (ch == '\n') {
- lineEnd = "\r\n";
- } else {
- pushed = ch;
- lineEnd = "\r";
- }
- break;
- }
- ch = in.read();
- }
- if (ch == -1 && state == 1) {
- lineEnd = "\r";
- }
-
- if (includeDelims) {
- line.append(lineEnd);
- }
- return line.toString();
- }
-
- /**
- * @return the line ending character(s) for the current line
- */
- public String getPostToken() {
- if (includeDelims) {
- return "";
- }
- return lineEnd;
- }
-
- }
-
/**
* class to tokenize the input as areas seperated
* by white space, or by a specified list of
diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/i18n/Translate.java b/src/main/org/apache/tools/ant/taskdefs/optional/i18n/Translate.java
index 1d947d967..2f09ae3b9 100644
--- a/src/main/org/apache/tools/ant/taskdefs/optional/i18n/Translate.java
+++ b/src/main/org/apache/tools/ant/taskdefs/optional/i18n/Translate.java
@@ -71,6 +71,7 @@ import org.apache.tools.ant.filters.TokenFilter;
import org.apache.tools.ant.taskdefs.MatchingTask;
import org.apache.tools.ant.types.FileSet;
import org.apache.tools.ant.util.FileUtils;
+import org.apache.tools.ant.util.LineTokenizer;
/**
* Translates text embedded in files using Resource Bundle files.
@@ -573,7 +574,7 @@ public class Translate extends MatchingTask {
BufferedReader in
= new BufferedReader(new InputStreamReader(fis, srcEncoding));
String line;
- TokenFilter.LineTokenizer lineTokenizer = new TokenFilter.LineTokenizer();
+ LineTokenizer lineTokenizer = new LineTokenizer();
lineTokenizer.setIncludeDelims(true);
line = lineTokenizer.getToken(in);
while ((line) != null) {
diff --git a/src/main/org/apache/tools/ant/util/FileUtils.java b/src/main/org/apache/tools/ant/util/FileUtils.java
index 4aa96ad23..5089a5846 100644
--- a/src/main/org/apache/tools/ant/util/FileUtils.java
+++ b/src/main/org/apache/tools/ant/util/FileUtils.java
@@ -583,7 +583,7 @@ public class FileUtils {
in = new BufferedReader(rdr);
}
- TokenFilter.LineTokenizer lineTokenizer = new TokenFilter.LineTokenizer();
+ LineTokenizer lineTokenizer = new LineTokenizer();
lineTokenizer.setIncludeDelims(true);
String newline = null;
String line = lineTokenizer.getToken(in);
diff --git a/src/main/org/apache/tools/ant/util/LineTokenizer.java b/src/main/org/apache/tools/ant/util/LineTokenizer.java
new file mode 100644
index 000000000..4553ad832
--- /dev/null
+++ b/src/main/org/apache/tools/ant/util/LineTokenizer.java
@@ -0,0 +1,151 @@
+/*
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2003 The Apache Software Foundation. All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. The end-user documentation included with the redistribution, if
+ * any, must include the following acknowlegement:
+ * "This product includes software developed by the
+ * Apache Software Foundation (http://www.apache.org/)."
+ * Alternately, this acknowlegement may appear in the software itself,
+ * if and wherever such third-party acknowlegements normally appear.
+ *
+ * 4. The names "Ant" and "Apache Software
+ * Foundation" must not be used to endorse or promote products derived
+ * from this software without prior written permission. For written
+ * permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache"
+ * nor may "Apache" appear in their names without prior written
+ * permission of the Apache Group.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * .
+ */
+package org.apache.tools.ant.util;
+
+import java.io.Reader;
+import java.io.IOException;
+
+import org.apache.tools.ant.ProjectComponent;
+
+/**
+ * class to tokenize the input as lines seperated
+ * by \r (mac style), \r\n (dos/windows style) or \n (unix style)
+ * @author Peter Reilly
+ * @since Ant 1.6
+ */
+public class LineTokenizer extends ProjectComponent
+ implements Tokenizer {
+ private String lineEnd = "";
+ private int pushed = -2;
+ private boolean includeDelims = false;
+
+ /**
+ * attribute includedelims - whether to include
+ * the line ending with the line, or to return
+ * it in the posttoken
+ * default false
+ * @param includeDelims if true include /r and /n in the line
+ */
+
+ public void setIncludeDelims(boolean includeDelims) {
+ this.includeDelims = includeDelims;
+ }
+
+ /**
+ * get the next line from the input
+ *
+ * @param in the input reader
+ * @return the line excluding /r or /n, unless includedelims is set
+ * @exception IOException if an error occurs reading
+ */
+ public String getToken(Reader in) throws IOException {
+ int ch = -1;
+ if (pushed != -2) {
+ ch = pushed;
+ pushed = -2;
+ } else {
+ ch = in.read();
+ }
+ if (ch == -1) {
+ return null;
+ }
+
+ lineEnd = "";
+ StringBuffer line = new StringBuffer();
+
+ int state = 0;
+ while (ch != -1) {
+ if (state == 0) {
+ if (ch == '\r') {
+ state = 1;
+ } else if (ch == '\n') {
+ lineEnd = "\n";
+ break;
+ } else {
+ line.append((char) ch);
+ }
+ } else {
+ state = 0;
+ if (ch == '\n') {
+ lineEnd = "\r\n";
+ } else {
+ pushed = ch;
+ lineEnd = "\r";
+ }
+ break;
+ }
+ ch = in.read();
+ }
+ if (ch == -1 && state == 1) {
+ lineEnd = "\r";
+ }
+
+ if (includeDelims) {
+ line.append(lineEnd);
+ }
+ return line.toString();
+ }
+
+ /**
+ * @return the line ending character(s) for the current line
+ */
+ public String getPostToken() {
+ if (includeDelims) {
+ return "";
+ }
+ return lineEnd;
+ }
+
+}
+
diff --git a/src/main/org/apache/tools/ant/util/Tokenizer.java b/src/main/org/apache/tools/ant/util/Tokenizer.java
new file mode 100644
index 000000000..43d525e7c
--- /dev/null
+++ b/src/main/org/apache/tools/ant/util/Tokenizer.java
@@ -0,0 +1,82 @@
+/*
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2003 The Apache Software Foundation. All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. The end-user documentation included with the redistribution, if
+ * any, must include the following acknowlegement:
+ * "This product includes software developed by the
+ * Apache Software Foundation (http://www.apache.org/)."
+ * Alternately, this acknowlegement may appear in the software itself,
+ * if and wherever such third-party acknowlegements normally appear.
+ *
+ * 4. The names "Ant" and "Apache Software
+ * Foundation" must not be used to endorse or promote products derived
+ * from this software without prior written permission. For written
+ * permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache"
+ * nor may "Apache" appear in their names without prior written
+ * permission of the Apache Group.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation. For more
+ * information on the Apache Software Foundation, please see
+ * .
+ */
+package org.apache.tools.ant.util;
+
+import java.io.Reader;
+import java.io.IOException;
+
+/**
+ * input stream tokenizers implement this interface
+ *
+ * @author Peter Reilly
+ * @version Ant 1.6
+ */
+public interface Tokenizer {
+ /**
+ * get the next token from the input stream
+ * @param in the input stream
+ * @return the next token, or null for the end
+ * of the stream
+ * @throws IOException if an error occurs
+ */
+ String getToken(Reader in)
+ throws IOException;
+
+ /**
+ * return the string between tokens, after the
+ * previous token.
+ * @return the intra-token string
+ */
+ String getPostToken();
+}