diff --git a/WHATSNEW b/WHATSNEW index f23107f49..bd8b6a21d 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -32,6 +32,9 @@ Changes that could break older environments: Fixed bugs: ----------- +* Filter readers were not handling line endings properly. Bugzilla + Report 18476. + * Expand tasks did not behave as expected with PatternSets. * now works on OS/400. diff --git a/build.xml b/build.xml index 2667bec41..d2e7de21f 100644 --- a/build.xml +++ b/build.xml @@ -277,7 +277,7 @@ - + + --> - + @@ -59,4 +59,15 @@ + + This has no new lines + + + + + + + File was modified + + diff --git a/src/etc/testcases/filters/expected/head-tail.headtail.test b/src/etc/testcases/filters/expected/head-tail.headtail.test index 239d56bc8..13cb234bb 100644 --- a/src/etc/testcases/filters/expected/head-tail.headtail.test +++ b/src/etc/testcases/filters/expected/head-tail.headtail.test @@ -1 +1,2 @@ +Line 3 Line 4 diff --git a/src/etc/testcases/filters/input/head-tail.test b/src/etc/testcases/filters/input/head-tail.test index 75495256d..8fac760c2 100644 --- a/src/etc/testcases/filters/input/head-tail.test +++ b/src/etc/testcases/filters/input/head-tail.test @@ -57,4 +57,4 @@ Line 56 Line 57 Line 58 Line 59 -Line 60 \ No newline at end of file +Line 60 diff --git a/src/etc/testcases/filters/input/stripjavacomments.test b/src/etc/testcases/filters/input/stripjavacomments.test index f4bef4d6f..37535bccc 100644 --- a/src/etc/testcases/filters/input/stripjavacomments.test +++ b/src/etc/testcases/filters/input/stripjavacomments.test @@ -27,4 +27,4 @@ public class NormalLine { private String url = "http://ant.apache.org/"; // very difficult! private String url2 = "\"http://ant.apache.org/\""; // even worse -} \ No newline at end of file +} diff --git a/src/etc/testcases/taskdefs/copy.filterset b/src/etc/testcases/taskdefs/copy.filterset index 367acbb26..5563dd951 100644 --- a/src/etc/testcases/taskdefs/copy.filterset +++ b/src/etc/testcases/taskdefs/copy.filterset @@ -1 +1 @@ -This is the @TITLE@. \ No newline at end of file +This is the @TITLE@. diff --git a/src/main/org/apache/tools/ant/filters/HeadFilter.java b/src/main/org/apache/tools/ant/filters/HeadFilter.java index e2cfd625f..5584a744f 100644 --- a/src/main/org/apache/tools/ant/filters/HeadFilter.java +++ b/src/main/org/apache/tools/ant/filters/HeadFilter.java @@ -88,6 +88,14 @@ public final class HeadFilter /** Number of lines to be skipped. */ private long skip = 0; + /** A line tokenizer */ + private TokenFilter.LineTokenizer lineTokenizer = null; + + /** the current line from the input stream */ + private String line = null; + /** the position in the current line */ + private int linePos = 0; + /** * Constructor for "dummy" instances. * @@ -105,6 +113,8 @@ public final class HeadFilter */ public HeadFilter(final Reader in) { super(in); + lineTokenizer = new TokenFilter.LineTokenizer(); + lineTokenizer.setIncludeDelims(true); } /** @@ -125,23 +135,18 @@ public final class HeadFilter setInitialized(true); } - int ch = -1; - - // skip the lines (if set) - while (skip > 0) { - for (int tmp = in.read(); tmp != '\n'; tmp = in.read()); - skip--; - } - - if ( (linesRead < lines) || (lines < 0) ){ - - ch = in.read(); - - if (ch == '\n') { - linesRead++; - } + while (line == null || line.length() == 0) { + line = lineTokenizer.getToken(in); + if (line == null) + return -1; + line = headFilter(line); + linePos = 0; } + int ch = line.charAt(linePos); + linePos++; + if (linePos == line.length()) + line = null; return ch; } @@ -202,6 +207,7 @@ public final class HeadFilter /** * Scans the parameters list for the "lines" parameter and uses * it to set the number of lines to be returned in the filtered stream. + * also scan for skip parameter. */ private final void initialize() { Parameter[] params = getParameters(); @@ -209,13 +215,32 @@ public final class HeadFilter for (int i = 0; i < params.length; i++) { if (LINES_KEY.equals(params[i].getName())) { lines = new Long(params[i].getValue()).longValue(); - break; + continue; } if (SKIP_KEY.equals(params[i].getName())) { skip = new Long(params[i].getValue()).longValue(); - break; + continue; } } } } + + /** + * implements a head filter on the input stream + */ + private String headFilter(String line) { + linesRead++; + if (skip > 0) { + if ((linesRead - 1) < skip) { + return null; + } + } + + if (lines > 0) { + if (linesRead > (lines + skip)) { + return null; + } + } + return line; + } } diff --git a/src/main/org/apache/tools/ant/filters/StripJavaComments.java b/src/main/org/apache/tools/ant/filters/StripJavaComments.java index 9da58d59e..04d429ed2 100644 --- a/src/main/org/apache/tools/ant/filters/StripJavaComments.java +++ b/src/main/org/apache/tools/ant/filters/StripJavaComments.java @@ -66,28 +66,28 @@ import java.io.Reader; public final class StripJavaComments extends BaseFilterReader implements ChainableReader { - - /** + + /** * The read-ahead character, used for effectively pushing a single - * character back. A value of -1 indicates that no character is in the + * character back. A value of -1 indicates that no character is in the * buffer. */ private int readAheadCh = -1; - /** + /** * Whether or not the parser is currently in the middle of a string * literal. */ private boolean inString = false; - /** + /** * Whether or not the last char has been a backslash. */ private boolean quoted = false; /** * Constructor for "dummy" instances. - * + * * @see BaseFilterReader#BaseFilterReader() */ public StripJavaComments() { @@ -107,12 +107,12 @@ public final class StripJavaComments /** * Returns the next character in the filtered stream, not including * Java comments. - * + * * @return the next character in the resulting stream, or -1 * if the end of the resulting stream has been reached - * + * * @exception IOException if the underlying stream throws an IOException - * during reading + * during reading */ public final int read() throws IOException { int ch = -1; @@ -132,9 +132,15 @@ public final class StripJavaComments if (ch == '/') { ch = in.read(); if (ch == '/') { + int prevCh = -1; while (ch != '\n' && ch != -1) { + prevCh = ch; ch = in.read(); } + if ( ch == '\n' && prevCh == '\r' ) { + readAheadCh = ch; + ch = prevCh; + } } else if (ch == '*') { while (ch != -1) { ch = in.read(); @@ -165,10 +171,10 @@ public final class StripJavaComments /** * Creates a new StripJavaComments using the passed in * Reader for instantiation. - * + * * @param rdr A Reader object providing the underlying stream. * Must not be null. - * + * * @return a new filter based on this configuration, but filtering * the specified reader */ diff --git a/src/main/org/apache/tools/ant/filters/TailFilter.java b/src/main/org/apache/tools/ant/filters/TailFilter.java index 72a335b0c..276262f62 100644 --- a/src/main/org/apache/tools/ant/filters/TailFilter.java +++ b/src/main/org/apache/tools/ant/filters/TailFilter.java @@ -55,6 +55,7 @@ package org.apache.tools.ant.filters; import java.io.IOException; import java.io.Reader; +import java.util.LinkedList; import org.apache.tools.ant.types.Parameter; /** @@ -90,18 +91,22 @@ public final class TailFilter /** Number of lines to be skipped. */ private long skip = 0; - /** Buffer to hold in characters read ahead. */ - private char[] buffer = new char[4096]; - - /** The character position that has been returned from the buffer. */ - private int returnedCharPos = -1; - /** Whether or not read-ahead been completed. */ private boolean completedReadAhead = false; /** Current index position on the buffer. */ private int bufferPos = 0; + /** A line tokenizer */ + private TokenFilter.LineTokenizer lineTokenizer = null; + + /** the current line from the input stream */ + private String line = null; + /** the position in the current line */ + private int linePos = 0; + + private LinkedList lineList = new LinkedList(); + /** * Constructor for "dummy" instances. * @@ -119,6 +124,8 @@ public final class TailFilter */ public TailFilter(final Reader in) { super(in); + lineTokenizer = new TokenFilter.LineTokenizer(); + lineTokenizer.setIncludeDelims(true); } /** @@ -140,74 +147,19 @@ public final class TailFilter setInitialized(true); } - if (!completedReadAhead) { - int ch = -1; - while ((ch = in.read()) != -1) { - if (buffer.length == bufferPos) { - if (returnedCharPos != -1) { - final char[] tmpBuffer = new char[buffer.length]; - System.arraycopy(buffer, returnedCharPos + 1, tmpBuffer, - 0, buffer.length - (returnedCharPos + 1)); - buffer = tmpBuffer; - bufferPos = bufferPos - (returnedCharPos + 1); - returnedCharPos = -1; - } else { - final char[] tmpBuffer = new char[buffer.length * 2]; - System.arraycopy(buffer, 0, tmpBuffer, 0, bufferPos); - buffer = tmpBuffer; - } - } - - if (lines > 0) { - if (ch == '\n' || ch == -1) { - ++linesRead; - - if ((linesRead == lines + skip)) { - int i = 0; - for (i = returnedCharPos + 1; - buffer[i] != 0 && buffer[i] != '\n'; i++) { - } - returnedCharPos = i; - --linesRead; - } - } - } - if (ch == -1) { - break; - } - - buffer[bufferPos] = (char) ch; - bufferPos++; - } - completedReadAhead = true; - } - - // Because the complete stream is read into the buffer I can delete - // the "skip lines" from back to the beginning. - if (skip > 0) { - // searching... - int i; - for (i = buffer.length - 1; skip > 0; i--) { - if (buffer[i]=='\n') { - skip--; - } - } - - // cut the buffer to the new length - char[] newBuffer = new char[i]; - System.arraycopy(buffer, 0, newBuffer, 0, i); - buffer = newBuffer; - - // donīt forget to set the "lastposition" new - bufferPos = i; + while (line == null || line.length() == 0) { + line = lineTokenizer.getToken(in); + line = tailFilter(line); + if (line == null) + return -1; + linePos = 0; } - ++returnedCharPos; - if (returnedCharPos >= bufferPos) { - return -1; - } else { - return buffer[returnedCharPos]; - } + int ch = line.charAt(linePos); + linePos++; + if (linePos == line.length()) + line = null; + return ch; } /** @@ -267,6 +219,7 @@ public final class TailFilter /** * Scans the parameters list for the "lines" parameter and uses * it to set the number of lines to be returned in the filtered stream. + * also scan for "skip" parameter. */ private final void initialize() { Parameter[] params = getParameters(); @@ -274,13 +227,55 @@ public final class TailFilter for (int i = 0; i < params.length; i++) { if (LINES_KEY.equals(params[i].getName())) { setLines(new Long(params[i].getValue()).longValue()); - break; + continue; } if (SKIP_KEY.equals(params[i].getName())) { skip = new Long(params[i].getValue()).longValue(); - break; + continue; + } + } + } + } + + /** + * implement a tail filter on a stream of lines. + * line = null is the end of the stream. + * @return "" while reading in the lines, + * line while outputing the lines + * null at the end of outputting the lines + */ + private String tailFilter(String line) { + if (! completedReadAhead) { + if (line != null) { + lineList.add(line); + if (lines == -1) { + if (lineList.size() > skip) { + return (String) lineList.removeFirst(); + } + } + else { + long linesToKeep = lines + (skip > 0 ? skip : 0); + if (linesToKeep < lineList.size()) { + lineList.removeFirst(); + } } + return ""; } + completedReadAhead = true; + if (skip > 0) { + for (int i = 0; i < skip; ++i) { + lineList.removeLast(); + } + } + if (lines > -1) { + while (lineList.size() > lines) { + lineList.removeFirst(); + } + } + } + if (lineList.size() > 0) { + return (String) lineList.removeFirst(); } + return null; } } diff --git a/src/main/org/apache/tools/ant/filters/TokenFilter.java b/src/main/org/apache/tools/ant/filters/TokenFilter.java index b8a3681cc..73f873bf6 100644 --- a/src/main/org/apache/tools/ant/filters/TokenFilter.java +++ b/src/main/org/apache/tools/ant/filters/TokenFilter.java @@ -389,9 +389,20 @@ public class TokenFilter public static class LineTokenizer implements Tokenizer { - String lineEnd = ""; - int pushed = -2; + private String lineEnd = ""; + private int pushed = -2; + private boolean includeDelims = false; + /** + * attribute includedelims - whether to include + * the line ending with the line, or to return + * it in the posttoken + */ + + public void setIncludeDelims(boolean includeDelims) { + this.includeDelims = true; + } + public String getToken(Reader in) throws IOException { @@ -440,12 +451,19 @@ public class TokenFilter lineEnd = "\r"; } + if (includeDelims) { + line.append(lineEnd); + } return line.toString(); } public String getPostToken() { + if (includeDelims) { + return ""; + } return lineEnd; } + } /** @@ -464,17 +482,39 @@ public class TokenFilter private char[] delims = null; private boolean delimsAreTokens = false; private boolean suppressDelims = false; + private boolean includeDelims = false; + /** + * attribute delims - the delimeter characters + */ public void setDelims(String delims) { this.delims = resolveBackSlash(delims).toCharArray(); } + /** + * attribute delimsaretokens - treat delimiters as + * separate tokens. + */ + public void setDelimsAreTokens(boolean delimsAreTokens) { this.delimsAreTokens = delimsAreTokens; } + /** + * attribute suppressdelims - suppress delimiters. + * default - false + */ public void setSuppressDelims(boolean suppressDelims) { this.suppressDelims = suppressDelims; } + + /** + * attribute includedelims - treat delimiters as part + * of the token. + * default - false + */ + public void setIncludeDelims(boolean includeDelims) { + this.includeDelims = true; + } public String getToken(Reader in) throws IOException @@ -525,11 +565,14 @@ public class TokenFilter ch = in.read(); } intraString = padding.toString(); + if (includeDelims) { + word.append(intraString); + } return word.toString(); } public String getPostToken() { - if (suppressDelims) + if (suppressDelims || includeDelims) return ""; return intraString; } diff --git a/src/main/org/apache/tools/ant/util/FileUtils.java b/src/main/org/apache/tools/ant/util/FileUtils.java index 30d264187..4d5aa6a35 100644 --- a/src/main/org/apache/tools/ant/util/FileUtils.java +++ b/src/main/org/apache/tools/ant/util/FileUtils.java @@ -399,10 +399,7 @@ public class FileUtils { final boolean filterChainsAvailable = (filterChains != null && filterChains.size() > 0); - if (filterSetsAvailable || filterChainsAvailable - || (inputEncoding != null - && !inputEncoding.equals(outputEncoding)) - || (inputEncoding == null && outputEncoding != null)) { + if (filterSetsAvailable) { BufferedReader in = null; BufferedWriter out = null; @@ -459,6 +456,59 @@ public class FileUtils { in.close(); } } + } else if (filterChainsAvailable + || (inputEncoding != null + && !inputEncoding.equals(outputEncoding)) + || (inputEncoding == null && outputEncoding != null)) { + BufferedReader in = null; + BufferedWriter out = null; + + try { + if (inputEncoding == null) { + in = new BufferedReader(new FileReader(sourceFile)); + } else { + in = + new BufferedReader( + new InputStreamReader( + new FileInputStream(sourceFile), + inputEncoding)); + } + + if (outputEncoding == null) { + out = new BufferedWriter(new FileWriter(destFile)); + } else { + out = + new BufferedWriter( + new OutputStreamWriter( + new FileOutputStream(destFile), + outputEncoding)); + } + + if (filterChainsAvailable) { + ChainReaderHelper crh = new ChainReaderHelper(); + crh.setBufferSize(8192); + crh.setPrimaryReader(in); + crh.setFilterChains(filterChains); + crh.setProject(project); + Reader rdr = crh.getAssembledReader(); + in = new BufferedReader(rdr); + } + char buffer[] = new char[1024*8]; + while (true) { + int nRead = in.read(buffer, 0, buffer.length); + if (nRead == -1) { + break; + } + out.write(buffer, 0, nRead); + } + } finally { + if (out != null) { + out.close(); + } + if (in != null) { + in.close(); + } + } } else { FileInputStream in = null; FileOutputStream out = null; diff --git a/src/testcases/org/apache/tools/ant/filters/HeadTailTest.java b/src/testcases/org/apache/tools/ant/filters/HeadTailTest.java index 442887cb7..086363302 100644 --- a/src/testcases/org/apache/tools/ant/filters/HeadTailTest.java +++ b/src/testcases/org/apache/tools/ant/filters/HeadTailTest.java @@ -112,6 +112,19 @@ public class HeadTailTest extends BuildFileTest { assertTrue("testHeadLinesSkip: Result not like expected", fu.contentEquals(expected, result)); } +/* + public void testFilterReaderHeadLinesSkip() throws IOException { + executeTarget("testFilterReaderHeadLinesSkip"); + File expected = getProject().resolveFile( + "expected/head-tail.filterReaderHeadLinesSkip.test"); + File result = getProject().resolveFile( + "result/head-tail.headLinesSkip.test"); + FileUtils fu = FileUtils.newFileUtils(); + assertTrue("testFilterReaderHeadLinesSkip: Result not like expected", + fu.contentEquals(expected, result)); + } + +*/ public void testTail() throws IOException { executeTarget("testTail"); File expected = getProject().resolveFile("expected/head-tail.tail.test"); @@ -144,10 +157,23 @@ public class HeadTailTest extends BuildFileTest { assertTrue("testTailLinesSkip: Result not like expected", fu.contentEquals(expected, result)); } +/* + public void testFilterReaderTailLinesSkip() throws IOException { + executeTarget("testFilterReaderTailLinesSkip"); + File expected = getProject().resolveFile( + "expected/head-tail.filterReaderTailLinesSkip.test"); + File result = getProject().resolveFile( + "result/head-tail.tailLinesSkip.test"); + FileUtils fu = FileUtils.newFileUtils(); + assertTrue("testFilterReaderTailLinesSkip: Result not like expected", + fu.contentEquals(expected, result)); + } +*/ + public void testHeadTail() throws IOException { executeTarget("testHeadTail"); - File expected = getProject().resolveFile("expected/head-tail.headTail.test"); - File result = getProject().resolveFile("result/head-tail.headTail.test"); + File expected = getProject().resolveFile("expected/head-tail.headtail.test"); + File result = getProject().resolveFile("result/head-tail.headtail.test"); FileUtils fu = FileUtils.newFileUtils(); assertTrue("testHeadTail: Result not like expected", fu.contentEquals(expected, result)); } diff --git a/src/testcases/org/apache/tools/ant/filters/NoNewLineTest.java b/src/testcases/org/apache/tools/ant/filters/NoNewLineTest.java new file mode 100644 index 000000000..9151b488f --- /dev/null +++ b/src/testcases/org/apache/tools/ant/filters/NoNewLineTest.java @@ -0,0 +1,88 @@ +/* + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2003 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, if + * any, must include the following acknowlegement: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowlegement may appear in the software itself, + * if and wherever such third-party acknowlegements normally appear. + * + * 4. The names "Ant" and "Apache Software + * Foundation" must not be used to endorse or promote products derived + * from this software without prior written permission. For written + * permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache" + * nor may "Apache" appear in their names without prior written + * permission of the Apache Group. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + */ + +package org.apache.tools.ant.filters; + +import java.io.File; +import java.io.IOException; + +import org.apache.tools.ant.BuildFileTest; +import org.apache.tools.ant.util.FileUtils; + +/** JUnit Testcases for No new line when filterchain used + * @author Peter Reilly + */ + + +public class NoNewLineTest extends BuildFileTest { + + public NoNewLineTest(String name) { + super(name); + } + + public void setUp() { + configureProject("src/etc/testcases/filters/build.xml"); + } + + public void tearDown() { + executeTarget("cleanup"); + } + + public void testNoAddNewLine() throws IOException { + executeTarget("testNoAddNewLine"); + } + + +} +