Merge pull request #141 from TheConstructor/input-file-tokenizer

#64855 Add filterbeforeconcat option to Concat

manual/Tasks/concat.html

@@ -124,6 +124,15 @@
           be used.</td>
         <td>No; default is <q>false</q></td>
       </tr>
+      <tr>
+        <td>filterbeforeconcat</td>
+        <td>If this attribute is set to <q>true</q>, the task applies the filterchain to each
+          input after applying <code>fixlastline</code>. If this attribute is <q>false</q>, concat
+          will apply the filterchain only once to the already concatenated inputs. Filtering of
+          <code>header</code> and <code>footer</code> is not affected by this setting.
+          <em>Since Ant 1.10.10</em></td>
+        <td>No; default is <q>false</q></td>
+      </tr>
       <tr>
         <td>ignoreempty</td>
         <td><em>Since Ant 1.8.0</em> Specifies whether or not the file specified

src/main/org/apache/tools/ant/taskdefs/Concat.java

@@ -212,34 +212,17 @@ public class Concat extends Task implements ResourceCollection {
     }
 
     /**
-     * This class reads from each of the source files in turn.
-     * The concatenated result can then be filtered as
-     * a single stream.
+     * This class reads a Reader and ensures it ends with the desired linebreak
+     * @since Ant 1.10.10
      */
-    private final class MultiReader<S> extends Reader {
-        private Reader reader = null;
-        private int    lastPos = 0;
-        private char[] lastChars = new char[eolString.length()];
+    private final class LastLineFixingReader extends Reader {
+        private final Reader reader;
+        private int lastPos = 0;
+        private final char[] lastChars = new char[eolString.length()];
         private boolean needAddSeparator = false;
-        private Iterator<S> readerSources;
-        private ReaderFactory<S> factory;
-
-        private MultiReader(Iterator<S> readerSources, ReaderFactory<S> factory) {
-            this.readerSources = readerSources;
-            this.factory = factory;
-        }
-
-        private Reader getReader() throws IOException {
-            if (reader == null && readerSources.hasNext()) {
-                reader = factory.getReader(readerSources.next());
-                Arrays.fill(lastChars, (char) 0);
-            }
-            return reader;
-        }
 
-        private void nextReader() throws IOException {
-            close();
-            reader = null;
+        private LastLineFixingReader(Reader reader) {
+            this.reader = reader;
         }
 
         /**
@@ -253,25 +236,21 @@ public class Concat extends Task implements ResourceCollection {
         public int read() throws IOException {
             if (needAddSeparator) {
                 if (lastPos >= eolString.length()) {
-                    lastPos = 0;
-                    needAddSeparator = false;
+                    return -1;
                 } else {
                     return eolString.charAt(lastPos++);
                 }
             }
-            while (getReader() != null) {
-                int ch = getReader().read();
-                if (ch == -1) {
-                    nextReader();
-                    if (isFixLastLine() && isMissingEndOfLine()) {
-                        needAddSeparator = true;
-                        lastPos = 1;
-                        return eolString.charAt(0);
-                    }
-                } else {
-                    addLastChar((char) ch);
-                    return ch;
+            int ch = reader.read();
+            if (ch == -1) {
+                if (isMissingEndOfLine()) {
+                    needAddSeparator = true;
+                    lastPos = 1;
+                    return eolString.charAt(0);
                 }
+            } else {
+                addLastChar((char) ch);
+                return ch;
             }
             return -1;
         }
@@ -289,13 +268,12 @@ public class Concat extends Task implements ResourceCollection {
             throws IOException {
 
             int amountRead = 0;
-            while (getReader() != null || needAddSeparator) {
+            while (true) {
                 if (needAddSeparator) {
-                    cbuf[off] = eolString.charAt(lastPos++);
                     if (lastPos >= eolString.length()) {
-                        lastPos = 0;
-                        needAddSeparator = false;
+                        break;
                     }
+                    cbuf[off] = eolString.charAt(lastPos++);
                     len--;
                     off++;
                     amountRead++;
@@ -304,23 +282,22 @@ public class Concat extends Task implements ResourceCollection {
                     }
                     continue;
                 }
-                int nRead = getReader().read(cbuf, off, len);
+                int nRead = reader.read(cbuf, off, len);
                 if (nRead == -1 || nRead == 0) {
-                    nextReader();
-                    if (isFixLastLine() && isMissingEndOfLine()) {
+                    if (isMissingEndOfLine()) {
                         needAddSeparator = true;
                         lastPos = 0;
+                    } else {
+                        break;
                     }
                 } else {
-                    if (isFixLastLine()) {
-                        for (int i = nRead;
-                                 i > (nRead - lastChars.length);
-                                 --i) {
-                            if (i <= 0) {
-                                break;
-                            }
-                            addLastChar(cbuf[off + i - 1]);
+                    for (int i = nRead;
+                         i > (nRead - lastChars.length);
+                         --i) {
+                        if (i <= 0) {
+                            break;
                         }
+                        addLastChar(cbuf[off + i - 1]);
                     }
                     len -= nRead;
                     off += nRead;
@@ -341,9 +318,7 @@ public class Concat extends Task implements ResourceCollection {
          */
         @Override
         public void close() throws IOException {
-            if (reader != null) {
-                reader.close();
-            }
+            reader.close();
         }
 
         /**
@@ -367,6 +342,107 @@ public class Concat extends Task implements ResourceCollection {
             }
             return false;
         }
+    }
+
+    /**
+     * This class reads from each of the source files in turn.
+     * The concatenated result can then be filtered as
+     * a single stream.
+     */
+    private final class MultiReader<S> extends Reader {
+        private Reader reader = null;
+        private Iterator<S> readerSources;
+        private ReaderFactory<S> factory;
+        private final boolean filterBeforeConcat;
+
+        private MultiReader(Iterator<S> readerSources, ReaderFactory<S> factory,
+                boolean filterBeforeConcat) {
+            this.readerSources = readerSources;
+            this.factory = factory;
+            this.filterBeforeConcat = filterBeforeConcat;
+        }
+
+        private Reader getReader() throws IOException {
+            if (reader == null && readerSources.hasNext()) {
+                reader = factory.getReader(readerSources.next());
+                if(isFixLastLine())
+                {
+                    reader = new LastLineFixingReader(reader);
+                }
+                if(filterBeforeConcat)
+                {
+                    reader = getFilteredReader(reader);
+                }
+            }
+            return reader;
+        }
+
+        private void nextReader() throws IOException {
+            close();
+            reader = null;
+        }
+
+        /**
+         * Read a character from the current reader object. Advance
+         * to the next if the reader is finished.
+         * @return the character read, -1 for EOF on the last reader.
+         * @exception IOException - possibly thrown by the read for a reader
+         *            object.
+         */
+        @Override
+        public int read() throws IOException {
+            while (getReader() != null) {
+                int ch = getReader().read();
+                if (ch == -1) {
+                    nextReader();
+                } else {
+                    return ch;
+                }
+            }
+            return -1;
+        }
+
+        /**
+         * Read into the buffer <code>cbuf</code>.
+         * @param cbuf The array to be read into.
+         * @param off The offset.
+         * @param len The length to read.
+         * @exception IOException - possibly thrown by the reads to the
+         *            reader objects.
+         */
+        @Override
+        public int read(char[] cbuf, int off, int len)
+            throws IOException {
+
+            int amountRead = 0;
+            while (getReader() != null) {
+                int nRead = getReader().read(cbuf, off, len);
+                if (nRead == -1 || nRead == 0) {
+                    nextReader();
+                } else {
+                    len -= nRead;
+                    off += nRead;
+                    amountRead += nRead;
+                    if (len == 0) {
+                        return amountRead;
+                    }
+                }
+            }
+            if (amountRead == 0) {
+                return -1;
+            }
+            return amountRead;
+        }
+
+        /**
+         * Close the current reader
+         */
+        @Override
+        public void close() throws IOException {
+            if (reader != null) {
+                reader.close();
+            }
+        }
 
         private boolean isFixLastLine() {
             return fixLastLine && textBuffer == null;
@@ -386,8 +462,14 @@ public class Concat extends Task implements ResourceCollection {
                 result.setManagingComponent(this);
                 return result;
             }
-            Reader resourceReader = getFilteredReader(
-                    new MultiReader<>(c.iterator(), resourceReaderFactory));
+            Reader resourceReader;
+            if(filterBeforeConcat) {
+                resourceReader = new MultiReader<>(c.iterator(),
+                        resourceReaderFactory, true);
+            } else {
+                resourceReader = getFilteredReader(
+                        new MultiReader<>(c.iterator(), resourceReaderFactory, false));
+            }
             Reader rdr;
             if (header == null && footer == null) {
                 rdr = resourceReader;
@@ -416,7 +498,7 @@ public class Concat extends Task implements ResourceCollection {
                     }
                 }
                 rdr = new MultiReader<>(Arrays.asList(readers).iterator(),
-                        identityReaderFactory);
+                        identityReaderFactory, false);
             }
             return outputEncoding == null ? new ReaderInputStream(rdr)
                     : new ReaderInputStream(rdr, outputEncoding);
@@ -454,6 +536,9 @@ public class Concat extends Task implements ResourceCollection {
     /** Stores the binary attribute */
     private boolean binary;
 
+    /** Stores the filterBeforeConcat attribute */
+    private boolean filterBeforeConcat;
+
     // Child elements.
 
     /**
@@ -779,6 +864,17 @@ public class Concat extends Task implements ResourceCollection {
         this.binary = binary;
     }
 
+    /**
+     * Set the filterBeforeConcat attribute. If true, concat will filter each
+     * input through the filterchain before concatenating the results. This
+     * allows to e.g. use the FileTokenizer to tokenize each input.
+     * @param filterBeforeConcat if true, filter each input before concatenation
+     * @since Ant 1.10.10
+     */
+    public void setFilterBeforeConcat(final boolean filterBeforeConcat) {
+        this.filterBeforeConcat = filterBeforeConcat;
+    }
+
     /**
      * Execute the concat task.
      */

src/tests/antunit/taskdefs/concat-test.xml

@@ -118,6 +118,50 @@
     </au:assertTrue>
   </target>
 
+  <target name="testFilterIsAppliedToConcatenation"
+          depends="-fixlastline-setup"
+          description="https://bz.apache.org/bugzilla/show_bug.cgi?id=64855">
+    <au:assertTrue>
+      <resourcesmatch>
+        <string>1${line.separator}2${line.separator}1${line.separator}2${line.separator}</string>
+        <concat fixlastline="true">
+          <filelist dir="${input}">
+            <file name="1"/>
+            <file name="2"/>
+          </filelist>
+          <filterchain>
+	    <tokenfilter>
+          <filetokenizer/>
+          <replaceregex pattern="(.*)" flags="s" replace="\1\1"/>
+	    </tokenfilter>
+	  </filterchain>
+        </concat>
+      </resourcesmatch>
+    </au:assertTrue>
+  </target>
+
+  <target name="testFilterBeforeConcatActuallyAppliesFilterToEachInput"
+          depends="-fixlastline-setup"
+          description="https://bz.apache.org/bugzilla/show_bug.cgi?id=64855">
+    <au:assertTrue>
+      <resourcesmatch>
+        <string>1${line.separator}1${line.separator}2${line.separator}2${line.separator}</string>
+        <concat fixlastline="true" filterbeforeconcat="true">
+          <filelist dir="${input}">
+            <file name="1"/>
+            <file name="2"/>
+          </filelist>
+          <filterchain>
+	    <tokenfilter>
+          <filetokenizer/>
+          <replaceregex pattern="(.*)" flags="s" replace="\1\1"/>
+	    </tokenfilter>
+	  </filterchain>
+        </concat>
+      </resourcesmatch>
+    </au:assertTrue>
+  </target>
+
   <target name="testIgnoreEmptyFalseFileIsCreated">
     <mkdir dir="${input}" />
     <mkdir dir="${output}" />