Browse Source

PR 56584 allow ReplaceTokens filter to use multi-character token separators. Submitted by Ralf Hergert.

master
Stefan Bodewig 11 years ago
parent
commit
6e88f92ead
10 changed files with 188 additions and 105 deletions
  1. +1
    -0
      CONTRIBUTORS
  2. +9
    -0
      WHATSNEW
  3. +4
    -0
      contributors.xml
  4. +27
    -2
      manual/Types/filterchain.html
  5. +33
    -0
      src/etc/testcases/filters/build.xml
  6. +2
    -0
      src/etc/testcases/filters/expected/replacetokens.double.test
  7. +2
    -0
      src/etc/testcases/filters/input/replacetokens.double.test
  8. +2
    -0
      src/etc/testcases/filters/input/replacetokens.mustache.test
  9. +84
    -101
      src/main/org/apache/tools/ant/filters/ReplaceTokens.java
  10. +24
    -2
      src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java

+ 1
- 0
CONTRIBUTORS View File

@@ -300,6 +300,7 @@ Pierre Delisle
Pierre Dittgen
riasol
R Handerson
Ralf Hergert
Rami Ojares
Randy Watler
Raphael Pierquin


+ 9
- 0
WHATSNEW View File

@@ -4,6 +4,15 @@ Changes from Ant 1.9.4 TO Ant 1.9.5
Changes that could break older environments:
-------------------------------------------

* The ReplaceTokens filter can now use token-separators longer than
one character. This means it can be used to replace mustache-style
{{patterns}} and similar templates. This is going to break code
that invokes the setters on ReplaceTokens via the Java API as their
parameters have been changed from char to String. It may also
break build files that specified multi character tokens and relied
on Ant silently ignoring all but the first character.
Bugzilla Report 56584

Fixed bugs:
-----------



+ 4
- 0
contributors.xml View File

@@ -1213,6 +1213,10 @@
<first>R</first>
<last>Handerson</last>
</name>
<name>
<first>Ralf</first>
<last>Hergert</last>
</name>
<name>
<first>Rami</first>
<last>Ojares</last>


+ 27
- 2
manual/Types/filterchain.html View File

@@ -551,14 +551,14 @@ user defined values.
<tr>
<td vAlign=top>tokenchar</td>
<td vAlign=top>begintoken</td>
<td vAlign=top>Character marking the
<td vAlign=top>String marking the
beginning of a token. Defaults to @</td>
<td vAlign=top align="center">No</td>
</tr>
<tr>
<td vAlign=top>tokenchar</td>
<td vAlign=top>endtoken</td>
<td vAlign=top>Character marking the
<td vAlign=top>String marking the
end of a token. Defaults to @</td>
<td vAlign=top align="center">No</td>
</tr>
@@ -626,6 +626,31 @@ Convenience method:
&lt;/loadfile&gt;
</pre></blockquote>

This replaces occurrences of the string {{DATE}} in the data
with today's date and stores it in the property ${src.file.replaced}.
<blockquote><pre>
&lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;
&lt;filterchain&gt;
&lt;filterreader classname=&quot;org.apache.tools.ant.filters.ReplaceTokens&quot;&gt;
&lt;param type=&quot;tokenchar&quot; name=&quot;begintoken&quot; value=&quot;{{&quot;/&gt;
&lt;param type=&quot;tokenchar&quot; name=&quot;endtoken&quot; value=&quot;}}&quot;/&gt;
&lt;/filterreader&gt;
&lt;/filterchain&gt;
&lt;/loadfile&gt;
</pre></blockquote>

Convenience method:
<blockquote><pre>
&lt;tstamp/&gt;
&lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;
&lt;filterchain&gt;
&lt;replacetokens begintoken=&quot;{{&quot; endtoken=&quot;}}&quot;&gt;
&lt;token key=&quot;DATE&quot; value=&quot;${TODAY}&quot;/&gt;
&lt;/replacetokens&gt;
&lt;/filterchain&gt;
&lt;/loadfile&gt;
</pre></blockquote>

This will treat each properties file entry in sample.properties as a token/key pair :
<blockquote><pre>
&lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;


+ 33
- 0
src/etc/testcases/filters/build.xml View File

@@ -100,6 +100,39 @@
</copy>
</target>

<target name="testReplaceTokensDoubleEncoded" depends="setUp">
<copy todir="${output}">
<fileset dir="input" includes="replacetokens.double.test" />
<filterchain>
<replacetokens>
<token key="foo" value=""/>
</replacetokens>
</filterchain>
</copy>
</target>

<target name="testReplaceTokensDoubleEncodedToSimple" depends="setUp">
<copy todir="${output}">
<fileset dir="input" includes="replacetokens.double.test" />
<filterchain>
<replacetokens begintoken="@@" endtoken="@@">
<token key="foo" value=""/>
</replacetokens>
</filterchain>
</copy>
</target>

<target name="testReplaceTokensMustacheStyle" depends="setUp">
<copy todir="${output}">
<fileset dir="input" includes="replacetokens.mustache.test" />
<filterchain>
<replacetokens begintoken="{{" endtoken="}}">
<token key="foo" value=""/>
</replacetokens>
</filterchain>
</copy>
</target>

<target name="testNoAddNewLine" depends="setUp">
<concat destfile="${output}/nonl">This has no new lines</concat>
<copy file="${output}/nonl" tofile="${output}/nonl-copyfilter">


+ 2
- 0
src/etc/testcases/filters/expected/replacetokens.double.test View File

@@ -0,0 +1,2 @@
1@@2
3

+ 2
- 0
src/etc/testcases/filters/input/replacetokens.double.test View File

@@ -0,0 +1,2 @@
1@@foo@@2
3

+ 2
- 0
src/etc/testcases/filters/input/replacetokens.mustache.test View File

@@ -0,0 +1,2 @@
1{{foo}}2
3

+ 84
- 101
src/main/org/apache/tools/ant/filters/ReplaceTokens.java View File

@@ -24,7 +24,9 @@ import java.io.Reader;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Properties;
import org.apache.tools.ant.BuildException;
import java.util.SortedMap;
import java.util.TreeMap;

import org.apache.tools.ant.types.Parameter;
import org.apache.tools.ant.types.Resource;
import org.apache.tools.ant.types.resources.FileResource;
@@ -52,13 +54,19 @@ public final class ReplaceTokens
extends BaseParamFilterReader
implements ChainableReader {
/** Default "begin token" character. */
private static final char DEFAULT_BEGIN_TOKEN = '@';
private static final String DEFAULT_BEGIN_TOKEN = "@";

/** Default "end token" character. */
private static final char DEFAULT_END_TOKEN = '@';
private static final String DEFAULT_END_TOKEN = "@";

/** Hashtable to holds the original replacee-replacer pairs (String to String). */
private Hashtable<String, String> hash = new Hashtable<String, String>();

/** Data to be used before reading from stream again */
private String queuedData = null;
/** This map holds the "resolved" tokens (begin- and end-tokens are added to make searching simpler) */
private final TreeMap<String, String> resolvedTokens = new TreeMap<String, String>();
private boolean resolvedTokensBuilt = false;
/** Used for comparisons and lookup into the resolvedTokens map. */
private String readBuffer = "";

/** replacement test from a token */
private String replaceData = null;
@@ -66,26 +74,18 @@ public final class ReplaceTokens
/** Index into replacement data */
private int replaceIndex = -1;

/** Index into queue data */
private int queueIndex = -1;

/** Hashtable to hold the replacee-replacer pairs (String to String). */
private Hashtable<String, String> hash = new Hashtable<String, String>();

/** Character marking the beginning of a token. */
private char beginToken = DEFAULT_BEGIN_TOKEN;
private String beginToken = DEFAULT_BEGIN_TOKEN;

/** Character marking the end of a token. */
private char endToken = DEFAULT_END_TOKEN;
private String endToken = DEFAULT_END_TOKEN;

/**
* Constructor for "dummy" instances.
*
* @see BaseFilterReader#BaseFilterReader()
*/
public ReplaceTokens() {
super();
}
public ReplaceTokens() {}

/**
* Creates a new filtered reader.
@@ -97,18 +97,6 @@ public final class ReplaceTokens
super(in);
}

private int getNextChar() throws IOException {
if (queueIndex != -1) {
final int ch = queuedData.charAt(queueIndex++);
if (queueIndex >= queuedData.length()) {
queueIndex = -1;
}
return ch;
}

return in.read();
}

/**
* Returns the next character in the filtered stream, replacing tokens
* from the original stream.
@@ -125,63 +113,66 @@ public final class ReplaceTokens
setInitialized(true);
}

if (replaceIndex != -1) {
final int ch = replaceData.charAt(replaceIndex++);
if (replaceIndex >= replaceData.length()) {
replaceIndex = -1;
if (!resolvedTokensBuilt) {
// build the resolved tokens tree map.
for (String key : hash.keySet()) {
resolvedTokens.put(beginToken + key + endToken, hash.get(key));
}
return ch;
resolvedTokensBuilt = true;
}

int ch = getNextChar();

if (ch == beginToken) {
final StringBuffer key = new StringBuffer("");
do {
ch = getNextChar();
if (ch != -1) {
key.append((char) ch);
} else {
break;
}
} while (ch != endToken);

if (ch == -1) {
if (queuedData == null || queueIndex == -1) {
queuedData = key.toString();
} else {
queuedData
= key.toString() + queuedData.substring(queueIndex);
}
if (queuedData.length() > 0) {
queueIndex = 0;
} else {
queueIndex = -1;
}
return beginToken;
// are we currently serving replace data?
if (replaceData != null) {
if (replaceIndex < replaceData.length()) {
return replaceData.charAt(replaceIndex++);
} else {
key.setLength(key.length() - 1);
replaceData = null;
}
}

final String replaceWith = (String) hash.get(key.toString());
if (replaceWith != null) {
if (replaceWith.length() > 0) {
replaceData = replaceWith;
replaceIndex = 0;
}
return read();
// is the read buffer empty?
if (readBuffer.length() == 0) {
int next = in.read();
if (next == -1) {
return next; // end of stream. all buffers empty.
}
readBuffer += (char)next;
}

for (;;) {
// get the closest tokens
SortedMap<String,String> possibleTokens = resolvedTokens.tailMap(readBuffer);
if (possibleTokens.isEmpty() || !possibleTokens.firstKey().startsWith(readBuffer)) { // if there is none, then deliver the first char from the buffer.
return getFirstCharacterFromReadBuffer();
} else if (readBuffer.equals(possibleTokens.firstKey())) { // there exists a nearest token - is it an exact match?
// we have found a token. prepare the replaceData buffer.
replaceData = resolvedTokens.get(readBuffer);
replaceIndex = 0;
readBuffer = ""; // destroy the readBuffer - it's contents are being replaced entirely.
// get the first character via recursive call.
return read();
} else { // nearest token is not matching exactly - read one character more.
int next = in.read();
if (next != -1) {
readBuffer += (char)next;
} else {
String newData = key.toString() + endToken;
if (queuedData == null || queueIndex == -1) {
queuedData = newData;
} else {
queuedData = newData + queuedData.substring(queueIndex);
}
queueIndex = 0;
return beginToken;
return getFirstCharacterFromReadBuffer(); // end of stream. deliver remaining characters from buffer.
}
}
}
return ch;
}

/**
* @return the first character from the read buffer or -1 if read buffer is empty.
*/
private int getFirstCharacterFromReadBuffer() {
if (readBuffer.length() > 0) {
int chr = readBuffer.charAt(0);
readBuffer = readBuffer.substring(1);
return chr;
} else {
return -1;
}
}

/**
@@ -189,7 +180,7 @@ public final class ReplaceTokens
*
* @param beginToken the character used to denote the beginning of a token
*/
public void setBeginToken(final char beginToken) {
public void setBeginToken(final String beginToken) {
this.beginToken = beginToken;
}

@@ -198,7 +189,7 @@ public final class ReplaceTokens
*
* @return the character used to denote the beginning of a token
*/
private char getBeginToken() {
private String getBeginToken() {
return beginToken;
}

@@ -207,7 +198,7 @@ public final class ReplaceTokens
*
* @param endToken the character used to denote the end of a token
*/
public void setEndToken(final char endToken) {
public void setEndToken(final String endToken) {
this.endToken = endToken;
}

@@ -216,7 +207,7 @@ public final class ReplaceTokens
*
* @return the character used to denote the end of a token
*/
private char getEndToken() {
private String getEndToken() {
return endToken;
}

@@ -238,18 +229,19 @@ public final class ReplaceTokens
*/
public void addConfiguredToken(final Token token) {
hash.put(token.getKey(), token.getValue());
resolvedTokensBuilt = false; // invalidate to build them again if they have been built already.
}

/**
* Returns properties from a specified properties file.
*
* @param fileName The file to load properties from.
* @param resource The resource to load properties from.
*/
private Properties getProperties(Resource r) {
private Properties getProperties(Resource resource) {
InputStream in = null;
Properties props = new Properties();
try {
in = r.getInputStream();
in = resource.getInputStream();
props.load(in);
} catch (IOException ioe) {
ioe.printStackTrace();
@@ -305,32 +297,23 @@ public final class ReplaceTokens
private void initialize() {
Parameter[] params = getParameters();
if (params != null) {
for (int i = 0; i < params.length; i++) {
if (params[i] != null) {
final String type = params[i].getType();
for (Parameter param : params) {
if (param != null) {
final String type = param.getType();
if ("tokenchar".equals(type)) {
final String name = params[i].getName();
String value = params[i].getValue();
final String name = param.getName();
if ("begintoken".equals(name)) {
if (value.length() == 0) {
throw new BuildException("Begin token cannot "
+ "be empty");
}
beginToken = params[i].getValue().charAt(0);
beginToken = param.getValue();
} else if ("endtoken".equals(name)) {
if (value.length() == 0) {
throw new BuildException("End token cannot "
+ "be empty");
}
endToken = params[i].getValue().charAt(0);
endToken = param.getValue();
}
} else if ("token".equals(type)) {
final String name = params[i].getName();
final String value = params[i].getValue();
final String name = param.getName();
final String value = param.getValue();
hash.put(name, value);
} else if ("propertiesfile".equals(type)) {
makeTokensFromProperties(
new FileResource(new File(params[i].getValue())));
new FileResource(new File(param.getValue())));
}
}
}


+ 24
- 2
src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java View File

@@ -31,7 +31,6 @@ import static org.junit.Assert.assertEquals;

public class ReplaceTokensTest {


@Rule
public BuildFileRule buildRule = new BuildFileRule();

@@ -45,7 +44,7 @@ public class ReplaceTokensTest {
buildRule.executeTarget("testReplaceTokens");
File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.test");
assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
}

@Test
@@ -56,4 +55,27 @@ public class ReplaceTokensTest {
assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
}

@Test
public void testReplaceTokensDoubleEncoded() throws IOException {
buildRule.executeTarget("testReplaceTokensDoubleEncoded");
File expected = buildRule.getProject().resolveFile("expected/replacetokens.double.test");
File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
}

@Test
public void testReplaceTokensDoubleEncodedToSimple() throws IOException {
buildRule.executeTarget("testReplaceTokensDoubleEncodedToSimple");
File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
}

@Test
public void testReplaceTokensMustacheStyle() throws IOException {
buildRule.executeTarget("testReplaceTokensMustacheStyle");
File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.mustache.test");
assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
}
}

Loading…
Cancel
Save