Browse Source

filter readers modify lineendings.

PR: 18476

Submitted by: peter.reilly@corvil.com (peter reilly)


git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@274499 13f79535-47bb-0310-9956-ffa450edef68
master
Magesh Umasankar 22 years ago
parent
commit
e596d350ae
15 changed files with 396 additions and 121 deletions
  1. +3
    -0
      WHATSNEW
  2. +1
    -1
      build.xml
  3. +30
    -3
      docs/manual/CoreTypes/filterchain.html
  4. +14
    -3
      src/etc/testcases/filters/build.xml
  5. +1
    -0
      src/etc/testcases/filters/expected/head-tail.headtail.test
  6. +1
    -1
      src/etc/testcases/filters/input/head-tail.test
  7. +1
    -1
      src/etc/testcases/filters/input/stripjavacomments.test
  8. +1
    -1
      src/etc/testcases/taskdefs/copy.filterset
  9. +42
    -17
      src/main/org/apache/tools/ant/filters/HeadFilter.java
  10. +17
    -11
      src/main/org/apache/tools/ant/filters/StripJavaComments.java
  11. +69
    -74
      src/main/org/apache/tools/ant/filters/TailFilter.java
  12. +46
    -3
      src/main/org/apache/tools/ant/filters/TokenFilter.java
  13. +54
    -4
      src/main/org/apache/tools/ant/util/FileUtils.java
  14. +28
    -2
      src/testcases/org/apache/tools/ant/filters/HeadTailTest.java
  15. +88
    -0
      src/testcases/org/apache/tools/ant/filters/NoNewLineTest.java

+ 3
- 0
WHATSNEW View File

@@ -32,6 +32,9 @@ Changes that could break older environments:

Fixed bugs:
-----------
* Filter readers were not handling line endings properly. Bugzilla
Report 18476.

* Expand tasks did not behave as expected with PatternSets.

* <property environment=... /> now works on OS/400.


+ 1
- 1
build.xml View File

@@ -277,7 +277,7 @@
<patternset id="teststhatfail">
<exclude name="${optional.package}/BeanShellScriptTest.java"/>
<exclude name="${ant.package}/taskdefs/ImportTest.java"/>
<exclude name="${ant.package}/filters/HeadTailTest.java"/>
<!-- <exclude name="${ant.package}/filters/HeadTailTest.java"/> -->
</patternset>

<!--


+ 30
- 3
docs/manual/CoreTypes/filterchain.html View File

@@ -898,10 +898,24 @@ This tokenizer splits the input into lines.
The tokenizer delimits lines
by "\r", "\n" or "\r\n".
This is the default tokenizer.
<TABLE cellSpacing=0 cellPadding=2 border=1>
<TR>
<TD vAlign=top><B>Attribute</B></TD>
<TD vAlign=top><B>Description</B></TD>
<TD vAlign=top align="center"><B>Required</B></TD>
</TR>
<TR>
<TD vAlign=top>includeDelims</TD>
<TD vAlign=top>
Include the line endings in the token.
Default is false.
</TD>
<TD vAlign=top align="center">No</TD>
</TR>
</TABLE>
<H4>Examples:</H4>

Convert input current line endings to unix style line endings.
<em>This currently has no effect when used in the copy task.</em>
<BLOCKQUOTE><PRE>
&lt;tokenfilter delimoutput=&quot;\n&quot;/&gt;
</PRE></BLOCKQUOTE>
@@ -955,14 +969,27 @@ attribute is used).
<tr>
<td valign="top">delimsaretokens</td>
<td valign="top">If this is true,
each delimiter character is returned as a token</td>
each delimiter character is returned as a token.
Default is false.
</td>
<td valign="top" align="center">No</td>
</tr>
<tr>
<td valign="top">suppressdelims</td>
<td valign="top">If this is true, delimiters are not returned. </td>
<td valign="top">
If this is true, delimiters are not returned.
Default is false.
</td>
<td valign="top" align="center">No</td>
</tr>
<tr>
<td vAlign=top>includeDelims</td>
<td vAlign=top>
Include the delimiters in the token.
Default is false.
</td>
<td vAlign=top align="center">No</td>
</tr>
</TABLE>

<H4>Examples:</H4>


+ 14
- 3
src/etc/testcases/filters/build.xml View File

@@ -20,11 +20,11 @@
</filterreader>
</filterchain>
</copy>
<!--fixcrlf srcdir="result" eol="lf">
<!--<fixcrlf srcdir="result" eol="lf">
<include name="linecontains.test"/>
</fixcrlf-->
</fixcrlf>-->
</target>
<target name="testEscapeUnicode" depends="init">
<copy todir="result" encoding="UTF-8">
<fileset dir="input">
@@ -59,4 +59,15 @@
</copy>
</target>

<target name="testNoAddNewLine" depends="init">
<concat destfile="result/nonl">This has no new lines</concat>
<copy file="result/nonl" tofile="result/nonl-copyfilter">
<filterchain><tokenfilter/></filterchain>
</copy>
<condition property="filterchain.files.are.same">
<filesmatch file1="result/nonl" file2="result/nonl-copyfilter"/>
</condition>
<fail unless="filterchain.files.are.same">File was modified</fail>
</target>

</project>

+ 1
- 0
src/etc/testcases/filters/expected/head-tail.headtail.test View File

@@ -1 +1,2 @@
Line 3
Line 4

+ 1
- 1
src/etc/testcases/filters/input/head-tail.test View File

@@ -57,4 +57,4 @@ Line 56
Line 57
Line 58
Line 59
Line 60
Line 60

+ 1
- 1
src/etc/testcases/filters/input/stripjavacomments.test View File

@@ -27,4 +27,4 @@ public class NormalLine {
private String url = "http://ant.apache.org/"; // very difficult!
private String url2 = "\"http://ant.apache.org/\""; // even worse

}
}

+ 1
- 1
src/etc/testcases/taskdefs/copy.filterset View File

@@ -1 +1 @@
This is the @TITLE@.
This is the @TITLE@.

+ 42
- 17
src/main/org/apache/tools/ant/filters/HeadFilter.java View File

@@ -88,6 +88,14 @@ public final class HeadFilter
/** Number of lines to be skipped. */
private long skip = 0;

/** A line tokenizer */
private TokenFilter.LineTokenizer lineTokenizer = null;
/** the current line from the input stream */
private String line = null;
/** the position in the current line */
private int linePos = 0;

/**
* Constructor for "dummy" instances.
*
@@ -105,6 +113,8 @@ public final class HeadFilter
*/
public HeadFilter(final Reader in) {
super(in);
lineTokenizer = new TokenFilter.LineTokenizer();
lineTokenizer.setIncludeDelims(true);
}

/**
@@ -125,23 +135,18 @@ public final class HeadFilter
setInitialized(true);
}

int ch = -1;

// skip the lines (if set)
while (skip > 0) {
for (int tmp = in.read(); tmp != '\n'; tmp = in.read());
skip--;
}

if ( (linesRead < lines) || (lines < 0) ){

ch = in.read();

if (ch == '\n') {
linesRead++;
}
while (line == null || line.length() == 0) {
line = lineTokenizer.getToken(in);
if (line == null)
return -1;
line = headFilter(line);
linePos = 0;
}

int ch = line.charAt(linePos);
linePos++;
if (linePos == line.length())
line = null;
return ch;
}

@@ -202,6 +207,7 @@ public final class HeadFilter
/**
* Scans the parameters list for the "lines" parameter and uses
* it to set the number of lines to be returned in the filtered stream.
* also scan for skip parameter.
*/
private final void initialize() {
Parameter[] params = getParameters();
@@ -209,13 +215,32 @@ public final class HeadFilter
for (int i = 0; i < params.length; i++) {
if (LINES_KEY.equals(params[i].getName())) {
lines = new Long(params[i].getValue()).longValue();
break;
continue;
}
if (SKIP_KEY.equals(params[i].getName())) {
skip = new Long(params[i].getValue()).longValue();
break;
continue;
}
}
}
}

/**
* implements a head filter on the input stream
*/
private String headFilter(String line) {
linesRead++;
if (skip > 0) {
if ((linesRead - 1) < skip) {
return null;
}
}

if (lines > 0) {
if (linesRead > (lines + skip)) {
return null;
}
}
return line;
}
}

+ 17
- 11
src/main/org/apache/tools/ant/filters/StripJavaComments.java View File

@@ -66,28 +66,28 @@ import java.io.Reader;
public final class StripJavaComments
extends BaseFilterReader
implements ChainableReader {
/**
/**
* The read-ahead character, used for effectively pushing a single
* character back. A value of -1 indicates that no character is in the
* character back. A value of -1 indicates that no character is in the
* buffer.
*/
private int readAheadCh = -1;

/**
/**
* Whether or not the parser is currently in the middle of a string
* literal.
*/
private boolean inString = false;

/**
/**
* Whether or not the last char has been a backslash.
*/
private boolean quoted = false;

/**
* Constructor for "dummy" instances.
*
*
* @see BaseFilterReader#BaseFilterReader()
*/
public StripJavaComments() {
@@ -107,12 +107,12 @@ public final class StripJavaComments
/**
* Returns the next character in the filtered stream, not including
* Java comments.
*
*
* @return the next character in the resulting stream, or -1
* if the end of the resulting stream has been reached
*
*
* @exception IOException if the underlying stream throws an IOException
* during reading
* during reading
*/
public final int read() throws IOException {
int ch = -1;
@@ -132,9 +132,15 @@ public final class StripJavaComments
if (ch == '/') {
ch = in.read();
if (ch == '/') {
int prevCh = -1;
while (ch != '\n' && ch != -1) {
prevCh = ch;
ch = in.read();
}
if ( ch == '\n' && prevCh == '\r' ) {
readAheadCh = ch;
ch = prevCh;
}
} else if (ch == '*') {
while (ch != -1) {
ch = in.read();
@@ -165,10 +171,10 @@ public final class StripJavaComments
/**
* Creates a new StripJavaComments using the passed in
* Reader for instantiation.
*
*
* @param rdr A Reader object providing the underlying stream.
* Must not be <code>null</code>.
*
*
* @return a new filter based on this configuration, but filtering
* the specified reader
*/


+ 69
- 74
src/main/org/apache/tools/ant/filters/TailFilter.java View File

@@ -55,6 +55,7 @@ package org.apache.tools.ant.filters;

import java.io.IOException;
import java.io.Reader;
import java.util.LinkedList;
import org.apache.tools.ant.types.Parameter;

/**
@@ -90,18 +91,22 @@ public final class TailFilter
/** Number of lines to be skipped. */
private long skip = 0;

/** Buffer to hold in characters read ahead. */
private char[] buffer = new char[4096];

/** The character position that has been returned from the buffer. */
private int returnedCharPos = -1;

/** Whether or not read-ahead been completed. */
private boolean completedReadAhead = false;

/** Current index position on the buffer. */
private int bufferPos = 0;

/** A line tokenizer */
private TokenFilter.LineTokenizer lineTokenizer = null;
/** the current line from the input stream */
private String line = null;
/** the position in the current line */
private int linePos = 0;

private LinkedList lineList = new LinkedList();
/**
* Constructor for "dummy" instances.
*
@@ -119,6 +124,8 @@ public final class TailFilter
*/
public TailFilter(final Reader in) {
super(in);
lineTokenizer = new TokenFilter.LineTokenizer();
lineTokenizer.setIncludeDelims(true);
}

/**
@@ -140,74 +147,19 @@ public final class TailFilter
setInitialized(true);
}

if (!completedReadAhead) {
int ch = -1;
while ((ch = in.read()) != -1) {
if (buffer.length == bufferPos) {
if (returnedCharPos != -1) {
final char[] tmpBuffer = new char[buffer.length];
System.arraycopy(buffer, returnedCharPos + 1, tmpBuffer,
0, buffer.length - (returnedCharPos + 1));
buffer = tmpBuffer;
bufferPos = bufferPos - (returnedCharPos + 1);
returnedCharPos = -1;
} else {
final char[] tmpBuffer = new char[buffer.length * 2];
System.arraycopy(buffer, 0, tmpBuffer, 0, bufferPos);
buffer = tmpBuffer;
}
}

if (lines > 0) {
if (ch == '\n' || ch == -1) {
++linesRead;

if ((linesRead == lines + skip)) {
int i = 0;
for (i = returnedCharPos + 1;
buffer[i] != 0 && buffer[i] != '\n'; i++) {
}
returnedCharPos = i;
--linesRead;
}
}
}
if (ch == -1) {
break;
}

buffer[bufferPos] = (char) ch;
bufferPos++;
}
completedReadAhead = true;
}

// Because the complete stream is read into the buffer I can delete
// the "skip lines" from back to the beginning.
if (skip > 0) {
// searching...
int i;
for (i = buffer.length - 1; skip > 0; i--) {
if (buffer[i]=='\n') {
skip--;
}
}

// cut the buffer to the new length
char[] newBuffer = new char[i];
System.arraycopy(buffer, 0, newBuffer, 0, i);
buffer = newBuffer;

// don´t forget to set the "lastposition" new
bufferPos = i;
while (line == null || line.length() == 0) {
line = lineTokenizer.getToken(in);
line = tailFilter(line);
if (line == null)
return -1;
linePos = 0;
}

++returnedCharPos;
if (returnedCharPos >= bufferPos) {
return -1;
} else {
return buffer[returnedCharPos];
}
int ch = line.charAt(linePos);
linePos++;
if (linePos == line.length())
line = null;
return ch;
}

/**
@@ -267,6 +219,7 @@ public final class TailFilter
/**
* Scans the parameters list for the "lines" parameter and uses
* it to set the number of lines to be returned in the filtered stream.
* also scan for "skip" parameter.
*/
private final void initialize() {
Parameter[] params = getParameters();
@@ -274,13 +227,55 @@ public final class TailFilter
for (int i = 0; i < params.length; i++) {
if (LINES_KEY.equals(params[i].getName())) {
setLines(new Long(params[i].getValue()).longValue());
break;
continue;
}
if (SKIP_KEY.equals(params[i].getName())) {
skip = new Long(params[i].getValue()).longValue();
break;
continue;
}
}
}
}

/**
* implement a tail filter on a stream of lines.
* line = null is the end of the stream.
* @return "" while reading in the lines,
* line while outputing the lines
* null at the end of outputting the lines
*/
private String tailFilter(String line) {
if (! completedReadAhead) {
if (line != null) {
lineList.add(line);
if (lines == -1) {
if (lineList.size() > skip) {
return (String) lineList.removeFirst();
}
}
else {
long linesToKeep = lines + (skip > 0 ? skip : 0);
if (linesToKeep < lineList.size()) {
lineList.removeFirst();
}
}
return "";
}
completedReadAhead = true;
if (skip > 0) {
for (int i = 0; i < skip; ++i) {
lineList.removeLast();
}
}
if (lines > -1) {
while (lineList.size() > lines) {
lineList.removeFirst();
}
}
}
if (lineList.size() > 0) {
return (String) lineList.removeFirst();
}
return null;
}
}

+ 46
- 3
src/main/org/apache/tools/ant/filters/TokenFilter.java View File

@@ -389,9 +389,20 @@ public class TokenFilter
public static class LineTokenizer
implements Tokenizer
{
String lineEnd = "";
int pushed = -2;
private String lineEnd = "";
private int pushed = -2;
private boolean includeDelims = false;

/**
* attribute includedelims - whether to include
* the line ending with the line, or to return
* it in the posttoken
*/
public void setIncludeDelims(boolean includeDelims) {
this.includeDelims = true;
}
public String getToken(Reader in)
throws IOException
{
@@ -440,12 +451,19 @@ public class TokenFilter
lineEnd = "\r";
}

if (includeDelims) {
line.append(lineEnd);
}
return line.toString();
}

public String getPostToken() {
if (includeDelims) {
return "";
}
return lineEnd;
}

}

/**
@@ -464,17 +482,39 @@ public class TokenFilter
private char[] delims = null;
private boolean delimsAreTokens = false;
private boolean suppressDelims = false;
private boolean includeDelims = false;

/**
* attribute delims - the delimeter characters
*/
public void setDelims(String delims) {
this.delims = resolveBackSlash(delims).toCharArray();
}

/**
* attribute delimsaretokens - treat delimiters as
* separate tokens.
*/
public void setDelimsAreTokens(boolean delimsAreTokens) {
this.delimsAreTokens = delimsAreTokens;
}
/**
* attribute suppressdelims - suppress delimiters.
* default - false
*/
public void setSuppressDelims(boolean suppressDelims) {
this.suppressDelims = suppressDelims;
}
/**
* attribute includedelims - treat delimiters as part
* of the token.
* default - false
*/
public void setIncludeDelims(boolean includeDelims) {
this.includeDelims = true;
}

public String getToken(Reader in)
throws IOException
@@ -525,11 +565,14 @@ public class TokenFilter
ch = in.read();
}
intraString = padding.toString();
if (includeDelims) {
word.append(intraString);
}
return word.toString();
}

public String getPostToken() {
if (suppressDelims)
if (suppressDelims || includeDelims)
return "";
return intraString;
}


+ 54
- 4
src/main/org/apache/tools/ant/util/FileUtils.java View File

@@ -399,10 +399,7 @@ public class FileUtils {
final boolean filterChainsAvailable = (filterChains != null
&& filterChains.size() > 0);

if (filterSetsAvailable || filterChainsAvailable
|| (inputEncoding != null
&& !inputEncoding.equals(outputEncoding))
|| (inputEncoding == null && outputEncoding != null)) {
if (filterSetsAvailable) {
BufferedReader in = null;
BufferedWriter out = null;

@@ -459,6 +456,59 @@ public class FileUtils {
in.close();
}
}
} else if (filterChainsAvailable
|| (inputEncoding != null
&& !inputEncoding.equals(outputEncoding))
|| (inputEncoding == null && outputEncoding != null)) {
BufferedReader in = null;
BufferedWriter out = null;
try {
if (inputEncoding == null) {
in = new BufferedReader(new FileReader(sourceFile));
} else {
in =
new BufferedReader(
new InputStreamReader(
new FileInputStream(sourceFile),
inputEncoding));
}
if (outputEncoding == null) {
out = new BufferedWriter(new FileWriter(destFile));
} else {
out =
new BufferedWriter(
new OutputStreamWriter(
new FileOutputStream(destFile),
outputEncoding));
}
if (filterChainsAvailable) {
ChainReaderHelper crh = new ChainReaderHelper();
crh.setBufferSize(8192);
crh.setPrimaryReader(in);
crh.setFilterChains(filterChains);
crh.setProject(project);
Reader rdr = crh.getAssembledReader();
in = new BufferedReader(rdr);
}
char buffer[] = new char[1024*8];
while (true) {
int nRead = in.read(buffer, 0, buffer.length);
if (nRead == -1) {
break;
}
out.write(buffer, 0, nRead);
}
} finally {
if (out != null) {
out.close();
}
if (in != null) {
in.close();
}
}
} else {
FileInputStream in = null;
FileOutputStream out = null;


+ 28
- 2
src/testcases/org/apache/tools/ant/filters/HeadTailTest.java View File

@@ -112,6 +112,19 @@ public class HeadTailTest extends BuildFileTest {
assertTrue("testHeadLinesSkip: Result not like expected", fu.contentEquals(expected, result));
}

/*
public void testFilterReaderHeadLinesSkip() throws IOException {
executeTarget("testFilterReaderHeadLinesSkip");
File expected = getProject().resolveFile(
"expected/head-tail.filterReaderHeadLinesSkip.test");
File result = getProject().resolveFile(
"result/head-tail.headLinesSkip.test");
FileUtils fu = FileUtils.newFileUtils();
assertTrue("testFilterReaderHeadLinesSkip: Result not like expected",
fu.contentEquals(expected, result));
}

*/
public void testTail() throws IOException {
executeTarget("testTail");
File expected = getProject().resolveFile("expected/head-tail.tail.test");
@@ -144,10 +157,23 @@ public class HeadTailTest extends BuildFileTest {
assertTrue("testTailLinesSkip: Result not like expected", fu.contentEquals(expected, result));
}

/*
public void testFilterReaderTailLinesSkip() throws IOException {
executeTarget("testFilterReaderTailLinesSkip");
File expected = getProject().resolveFile(
"expected/head-tail.filterReaderTailLinesSkip.test");
File result = getProject().resolveFile(
"result/head-tail.tailLinesSkip.test");
FileUtils fu = FileUtils.newFileUtils();
assertTrue("testFilterReaderTailLinesSkip: Result not like expected",
fu.contentEquals(expected, result));
}
*/

public void testHeadTail() throws IOException {
executeTarget("testHeadTail");
File expected = getProject().resolveFile("expected/head-tail.headTail.test");
File result = getProject().resolveFile("result/head-tail.headTail.test");
File expected = getProject().resolveFile("expected/head-tail.headtail.test");
File result = getProject().resolveFile("result/head-tail.headtail.test");
FileUtils fu = FileUtils.newFileUtils();
assertTrue("testHeadTail: Result not like expected", fu.contentEquals(expected, result));
}


+ 88
- 0
src/testcases/org/apache/tools/ant/filters/NoNewLineTest.java View File

@@ -0,0 +1,88 @@
/*
* The Apache Software License, Version 1.1
*
* Copyright (c) 2003 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution, if
* any, must include the following acknowlegement:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowlegement may appear in the software itself,
* if and wherever such third-party acknowlegements normally appear.
*
* 4. The names "Ant" and "Apache Software
* Foundation" must not be used to endorse or promote products derived
* from this software without prior written permission. For written
* permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache"
* nor may "Apache" appear in their names without prior written
* permission of the Apache Group.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/

package org.apache.tools.ant.filters;

import java.io.File;
import java.io.IOException;

import org.apache.tools.ant.BuildFileTest;
import org.apache.tools.ant.util.FileUtils;

/** JUnit Testcases for No new line when filterchain used
* @author Peter Reilly
*/


public class NoNewLineTest extends BuildFileTest {

public NoNewLineTest(String name) {
super(name);
}

public void setUp() {
configureProject("src/etc/testcases/filters/build.xml");
}

public void tearDown() {
executeTarget("cleanup");
}

public void testNoAddNewLine() throws IOException {
executeTarget("testNoAddNewLine");
}


}


Loading…
Cancel
Save