Refactor: move LineTokenizer out of TokenFilter

git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@274905 13f79535-47bb-0310-9956-ffa450edef68

docs/manual/CoreTypes/filterchain.html

@@ -1256,7 +1256,7 @@ Convert to uppercase.
 
 Custom string filters and tokenizers may be plugged in by
 extending the interfaces org.apache.tools.ant.filters.TokenFilter.Filter
-and org.apache.tools.ant.filters.TokenFilter.Tokenizer respectly.
+and org.apache.tools.ant.util.Tokenizer respectly.
 
 They are defined the build file using <typedef/>. For
 example a string filter that capitalizes words may be declared as:

src/main/org/apache/tools/ant/filters/HeadFilter.java

@@ -55,6 +55,7 @@ package org.apache.tools.ant.filters;
 
 import java.io.IOException;
 import java.io.Reader;
+import org.apache.tools.ant.util.LineTokenizer;
 import org.apache.tools.ant.types.Parameter;
 
 /**
@@ -91,7 +92,7 @@ public final class HeadFilter extends BaseParamFilterReader
     private long skip = 0;
 
     /** A line tokenizer */
-    private TokenFilter.LineTokenizer lineTokenizer = null;
+    private LineTokenizer lineTokenizer = null;
 
     /** the current line from the input stream */
     private String    line      = null;
@@ -115,7 +116,7 @@ public final class HeadFilter extends BaseParamFilterReader
      */
     public HeadFilter(final Reader in) {
         super(in);
-        lineTokenizer = new TokenFilter.LineTokenizer();
+        lineTokenizer = new LineTokenizer();
         lineTokenizer.setIncludeDelims(true);
     }
 

src/main/org/apache/tools/ant/filters/TailFilter.java

@@ -57,6 +57,7 @@ import java.io.IOException;
 import java.io.Reader;
 import java.util.LinkedList;
 import org.apache.tools.ant.types.Parameter;
+import org.apache.tools.ant.util.LineTokenizer;
 
 /**
  * Reads the last <code>n</code> lines of a stream. (Default is last10 lines.)
@@ -100,7 +101,7 @@ public final class TailFilter extends BaseParamFilterReader
     private int bufferPos = 0;
 
     /** A line tokenizer */
-    private TokenFilter.LineTokenizer lineTokenizer = null;
+    private LineTokenizer lineTokenizer = null;
 
     /** the current line from the input stream */
     private String    line      = null;
@@ -126,7 +127,7 @@ public final class TailFilter extends BaseParamFilterReader
      */
     public TailFilter(final Reader in) {
         super(in);
-        lineTokenizer = new TokenFilter.LineTokenizer();
+        lineTokenizer = new LineTokenizer();
         lineTokenizer.setIncludeDelims(true);
     }
 

src/main/org/apache/tools/ant/filters/TokenFilter.java

@@ -63,6 +63,8 @@ import org.apache.tools.ant.ProjectComponent;
 import org.apache.tools.ant.types.RegularExpression;
 import org.apache.tools.ant.types.Substitution;
 import org.apache.tools.ant.util.FileUtils;
+import org.apache.tools.ant.util.Tokenizer;
+import org.apache.tools.ant.util.LineTokenizer;
 import org.apache.tools.ant.util.regexp.Regexp;
 
 /**
@@ -77,28 +79,6 @@ import org.apache.tools.ant.util.regexp.Regexp;
  */
 public class TokenFilter extends BaseFilterReader
     implements ChainableReader {
-    /**
-     * input stream tokenizers implement this interface
-     */
-    public interface Tokenizer {
-        /**
-         * get the next token from the input stream
-         * @param in the input stream
-         * @return the next token, or null for the end
-         *         of the stream
-         * @throws IOException if an error occurs
-         */
-        String getToken(Reader in)
-            throws IOException;
-
-        /**
-         * return the string between tokens, after the
-         * previous token.
-         * @return the intra-token string
-         */
-        String getPostToken();
-    }
-
     /**
      * string filters implement this interface
      */
@@ -361,96 +341,6 @@ public class TokenFilter extends BaseFilterReader
         }
     }
 
-
-    /**
-     * class to tokenize the input as lines seperated
-     * by \r (mac style), \r\n (dos/windows style) or \n (unix style)
-     */
-    public static class LineTokenizer extends ProjectComponent
-        implements Tokenizer {
-        private String  lineEnd = "";
-        private int     pushed = -2;
-        private boolean includeDelims = false;
-
-        /**
-         * attribute includedelims - whether to include
-         * the line ending with the line, or to return
-         * it in the posttoken
-         * default false
-         * @param includeDelims if true include /r and /n in the line
-         */
-
-        public void setIncludeDelims(boolean includeDelims) {
-            this.includeDelims = includeDelims;
-        }
-
-        /**
-         * get the next line from the input
-         *
-         * @param in the input reader
-         * @return the line excluding /r or /n, unless includedelims is set
-         * @exception IOException if an error occurs reading
-         */
-        public String getToken(Reader in) throws IOException {
-            int ch = -1;
-            if (pushed != -2) {
-                ch = pushed;
-                pushed = -2;
-            } else {
-                ch = in.read();
-            }
-            if (ch == -1) {
-                return null;
-            }
-
-            lineEnd = "";
-            StringBuffer line = new StringBuffer();
-
-            int state = 0;
-            while (ch != -1) {
-                if (state == 0) {
-                    if (ch == '\r') {
-                        state = 1;
-                    } else if (ch == '\n') {
-                        lineEnd = "\n";
-                        break;
-                    } else {
-                        line.append((char) ch);
-                    }
-                } else {
-                    state = 0;
-                    if (ch == '\n') {
-                        lineEnd = "\r\n";
-                    } else {
-                        pushed = ch;
-                        lineEnd = "\r";
-                    }
-                    break;
-                }
-                ch = in.read();
-            }
-            if (ch == -1 && state == 1) {
-                lineEnd = "\r";
-            }
-
-            if (includeDelims) {
-                line.append(lineEnd);
-            }
-            return line.toString();
-        }
-
-        /**
-         * @return the line ending character(s) for the current line
-         */
-        public String getPostToken() {
-            if (includeDelims) {
-                return "";
-            }
-            return lineEnd;
-        }
-
-    }
-
     /**
      * class to tokenize the input as areas seperated
      * by white space, or by a specified list of

src/main/org/apache/tools/ant/taskdefs/optional/i18n/Translate.java

@@ -71,6 +71,7 @@ import org.apache.tools.ant.filters.TokenFilter;
 import org.apache.tools.ant.taskdefs.MatchingTask;
 import org.apache.tools.ant.types.FileSet;
 import org.apache.tools.ant.util.FileUtils;
+import org.apache.tools.ant.util.LineTokenizer;
 
 /**
  * Translates text embedded in files using Resource Bundle files.
@@ -573,7 +574,7 @@ public class Translate extends MatchingTask {
                         BufferedReader in
                             = new BufferedReader(new InputStreamReader(fis, srcEncoding));
                         String line;
-                        TokenFilter.LineTokenizer lineTokenizer = new TokenFilter.LineTokenizer();
+                        LineTokenizer lineTokenizer = new LineTokenizer();
                         lineTokenizer.setIncludeDelims(true);
                         line = lineTokenizer.getToken(in);
                         while ((line) != null) {

src/main/org/apache/tools/ant/util/FileUtils.java

@@ -583,7 +583,7 @@ public class FileUtils {
                         in = new BufferedReader(rdr);
                     }
 
-                    TokenFilter.LineTokenizer lineTokenizer = new TokenFilter.LineTokenizer();
+                    LineTokenizer lineTokenizer = new LineTokenizer();
                     lineTokenizer.setIncludeDelims(true);
                     String newline = null;
                     String line = lineTokenizer.getToken(in);

src/main/org/apache/tools/ant/util/LineTokenizer.java

@@ -0,0 +1,151 @@
+/*
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2003 The Apache Software Foundation.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. The end-user documentation included with the redistribution, if
+ *    any, must include the following acknowlegement:
+ *       "This product includes software developed by the
+ *        Apache Software Foundation (http://www.apache.org/)."
+ *    Alternately, this acknowlegement may appear in the software itself,
+ *    if and wherever such third-party acknowlegements normally appear.
+ *
+ * 4. The names "Ant" and "Apache Software
+ *    Foundation" must not be used to endorse or promote products derived
+ *    from this software without prior written permission. For written
+ *    permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache"
+ *    nor may "Apache" appear in their names without prior written
+ *    permission of the Apache Group.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ */
+package org.apache.tools.ant.util;
+
+import java.io.Reader;
+import java.io.IOException;
+
+import org.apache.tools.ant.ProjectComponent;
+
+/**
+ * class to tokenize the input as lines seperated
+ * by \r (mac style), \r\n (dos/windows style) or \n (unix style)
+ * @author Peter Reilly
+ * @since Ant 1.6
+ */
+public class LineTokenizer extends ProjectComponent
+    implements Tokenizer {
+    private String  lineEnd = "";
+    private int     pushed = -2;
+    private boolean includeDelims = false;
+
+    /**
+     * attribute includedelims - whether to include
+     * the line ending with the line, or to return
+     * it in the posttoken
+     * default false
+     * @param includeDelims if true include /r and /n in the line
+     */
+
+    public void setIncludeDelims(boolean includeDelims) {
+        this.includeDelims = includeDelims;
+    }
+
+    /**
+     * get the next line from the input
+     *
+     * @param in the input reader
+     * @return the line excluding /r or /n, unless includedelims is set
+     * @exception IOException if an error occurs reading
+     */
+    public String getToken(Reader in) throws IOException {
+        int ch = -1;
+        if (pushed != -2) {
+            ch = pushed;
+            pushed = -2;
+        } else {
+            ch = in.read();
+        }
+        if (ch == -1) {
+            return null;
+        }
+
+        lineEnd = "";
+        StringBuffer line = new StringBuffer();
+
+        int state = 0;
+        while (ch != -1) {
+            if (state == 0) {
+                if (ch == '\r') {
+                    state = 1;
+                } else if (ch == '\n') {
+                    lineEnd = "\n";
+                    break;
+                } else {
+                    line.append((char) ch);
+                }
+            } else {
+                state = 0;
+                if (ch == '\n') {
+                    lineEnd = "\r\n";
+                } else {
+                    pushed = ch;
+                    lineEnd = "\r";
+                }
+                break;
+            }
+            ch = in.read();
+        }
+        if (ch == -1 && state == 1) {
+            lineEnd = "\r";
+        }
+
+        if (includeDelims) {
+            line.append(lineEnd);
+        }
+        return line.toString();
+    }
+
+    /**
+     * @return the line ending character(s) for the current line
+     */
+    public String getPostToken() {
+        if (includeDelims) {
+            return "";
+        }
+        return lineEnd;
+    }
+
+}
+

src/main/org/apache/tools/ant/util/Tokenizer.java

@@ -0,0 +1,82 @@
+/*
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2003 The Apache Software Foundation.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. The end-user documentation included with the redistribution, if
+ *    any, must include the following acknowlegement:
+ *       "This product includes software developed by the
+ *        Apache Software Foundation (http://www.apache.org/)."
+ *    Alternately, this acknowlegement may appear in the software itself,
+ *    if and wherever such third-party acknowlegements normally appear.
+ *
+ * 4. The names "Ant" and "Apache Software
+ *    Foundation" must not be used to endorse or promote products derived
+ *    from this software without prior written permission. For written
+ *    permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache"
+ *    nor may "Apache" appear in their names without prior written
+ *    permission of the Apache Group.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ */
+package org.apache.tools.ant.util;
+
+import java.io.Reader;
+import java.io.IOException;
+
+/**
+ * input stream tokenizers implement this interface
+ *
+ * @author Peter Reilly
+ * @version Ant 1.6
+ */
+public interface Tokenizer {
+    /**
+     * get the next token from the input stream
+     * @param in the input stream
+     * @return the next token, or null for the end
+     *         of the stream
+     * @throws IOException if an error occurs
+     */
+    String getToken(Reader in)
+        throws IOException;
+
+    /**
+     * return the string between tokens, after the
+     * previous token.
+     * @return the intra-token string
+     */
+    String getPostToken();
+}