From 6e88f92ead6a5f67935055661a9e8041f5ed8bae Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Tue, 24 Jun 2014 06:31:55 +0200 Subject: [PATCH] PR 56584 allow ReplaceTokens filter to use multi-character token separators. Submitted by Ralf Hergert. --- CONTRIBUTORS | 1 + WHATSNEW | 9 + contributors.xml | 4 + manual/Types/filterchain.html | 29 ++- src/etc/testcases/filters/build.xml | 33 ++++ .../expected/replacetokens.double.test | 2 + .../filters/input/replacetokens.double.test | 2 + .../filters/input/replacetokens.mustache.test | 2 + .../tools/ant/filters/ReplaceTokens.java | 185 ++++++++---------- .../tools/ant/filters/ReplaceTokensTest.java | 26 ++- 10 files changed, 188 insertions(+), 105 deletions(-) create mode 100644 src/etc/testcases/filters/expected/replacetokens.double.test create mode 100644 src/etc/testcases/filters/input/replacetokens.double.test create mode 100644 src/etc/testcases/filters/input/replacetokens.mustache.test diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 4a10c42b4..4d65e56e8 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -300,6 +300,7 @@ Pierre Delisle Pierre Dittgen riasol R Handerson +Ralf Hergert Rami Ojares Randy Watler Raphael Pierquin diff --git a/WHATSNEW b/WHATSNEW index 6ebed1eca..d80727dcb 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -4,6 +4,15 @@ Changes from Ant 1.9.4 TO Ant 1.9.5 Changes that could break older environments: ------------------------------------------- + * The ReplaceTokens filter can now use token-separators longer than + one character. This means it can be used to replace mustache-style + {{patterns}} and similar templates. This is going to break code + that invokes the setters on ReplaceTokens via the Java API as their + parameters have been changed from char to String. It may also + break build files that specified multi character tokens and relied + on Ant silently ignoring all but the first character. + Bugzilla Report 56584 + Fixed bugs: ----------- diff --git a/contributors.xml b/contributors.xml index a86950aff..8bdae0f70 100644 --- a/contributors.xml +++ b/contributors.xml @@ -1213,6 +1213,10 @@ R Handerson + + Ralf + Hergert + Rami Ojares diff --git a/manual/Types/filterchain.html b/manual/Types/filterchain.html index b7c95a189..4de10ea3c 100644 --- a/manual/Types/filterchain.html +++ b/manual/Types/filterchain.html @@ -551,14 +551,14 @@ user defined values. tokenchar begintoken - Character marking the + String marking the beginning of a token. Defaults to @ No tokenchar endtoken - Character marking the + String marking the end of a token. Defaults to @ No @@ -626,6 +626,31 @@ Convenience method: </loadfile> +This replaces occurrences of the string {{DATE}} in the data +with today's date and stores it in the property ${src.file.replaced}. +
+<loadfile srcfile="${src.file}" property="${src.file.replaced}">
+  <filterchain>
+    <filterreader classname="org.apache.tools.ant.filters.ReplaceTokens">
+      <param type="tokenchar" name="begintoken" value="{{"/>
+      <param type="tokenchar" name="endtoken" value="}}"/>
+    </filterreader>
+  </filterchain>
+</loadfile>
+
+ +Convenience method: +
+<tstamp/>
+<loadfile srcfile="${src.file}" property="${src.file.replaced}">
+  <filterchain>
+    <replacetokens begintoken="{{" endtoken="}}">
+      <token key="DATE" value="${TODAY}"/>
+    </replacetokens>
+  </filterchain>
+</loadfile>
+
+ This will treat each properties file entry in sample.properties as a token/key pair : 
 <loadfile srcfile="${src.file}" property="${src.file.replaced}">
diff --git a/src/etc/testcases/filters/build.xml b/src/etc/testcases/filters/build.xml
index cc3798ac1..b70b7786b 100644
--- a/src/etc/testcases/filters/build.xml
+++ b/src/etc/testcases/filters/build.xml
@@ -100,6 +100,39 @@
     
   
 
+  
+    
+      
+      
+        
+          
+        
+      
+    
+  
+
+  
+    
+      
+      
+        
+          
+        
+      
+    
+  
+
+  
+    
+      
+      
+        
+          
+        
+      
+    
+  
+
   
     This has no new lines
     
diff --git a/src/etc/testcases/filters/expected/replacetokens.double.test b/src/etc/testcases/filters/expected/replacetokens.double.test
new file mode 100644
index 000000000..72eaee7eb
--- /dev/null
+++ b/src/etc/testcases/filters/expected/replacetokens.double.test
@@ -0,0 +1,2 @@
+1@@2
+3
diff --git a/src/etc/testcases/filters/input/replacetokens.double.test b/src/etc/testcases/filters/input/replacetokens.double.test
new file mode 100644
index 000000000..163417d35
--- /dev/null
+++ b/src/etc/testcases/filters/input/replacetokens.double.test
@@ -0,0 +1,2 @@
+1@@foo@@2
+3
diff --git a/src/etc/testcases/filters/input/replacetokens.mustache.test b/src/etc/testcases/filters/input/replacetokens.mustache.test
new file mode 100644
index 000000000..62df44555
--- /dev/null
+++ b/src/etc/testcases/filters/input/replacetokens.mustache.test
@@ -0,0 +1,2 @@
+1{{foo}}2
+3
diff --git a/src/main/org/apache/tools/ant/filters/ReplaceTokens.java b/src/main/org/apache/tools/ant/filters/ReplaceTokens.java
index c414dcacb..efef83b72 100644
--- a/src/main/org/apache/tools/ant/filters/ReplaceTokens.java
+++ b/src/main/org/apache/tools/ant/filters/ReplaceTokens.java
@@ -24,7 +24,9 @@ import java.io.Reader;
 import java.util.Enumeration;
 import java.util.Hashtable;
 import java.util.Properties;
-import org.apache.tools.ant.BuildException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
 import org.apache.tools.ant.types.Parameter;
 import org.apache.tools.ant.types.Resource;
 import org.apache.tools.ant.types.resources.FileResource;
@@ -52,13 +54,19 @@ public final class ReplaceTokens
     extends BaseParamFilterReader
     implements ChainableReader {
     /** Default "begin token" character. */
-    private static final char DEFAULT_BEGIN_TOKEN = '@';
+    private static final String DEFAULT_BEGIN_TOKEN = "@";
 
     /** Default "end token" character. */
-    private static final char DEFAULT_END_TOKEN = '@';
+    private static final String DEFAULT_END_TOKEN = "@";
+
+    /** Hashtable to holds the original replacee-replacer pairs (String to String). */
+    private Hashtable hash = new Hashtable();
 
-    /** Data to be used before reading from stream again */
-    private String queuedData = null;
+    /** This map holds the "resolved" tokens (begin- and end-tokens are added to make searching simpler) */
+    private final TreeMap resolvedTokens = new TreeMap();
+    private boolean resolvedTokensBuilt = false;
+    /** Used for comparisons and lookup into the resolvedTokens map. */
+    private String readBuffer = "";
 
     /** replacement test from a token */
     private String replaceData = null;
@@ -66,26 +74,18 @@ public final class ReplaceTokens
     /** Index into replacement data */
     private int replaceIndex = -1;
 
-    /** Index into queue data */
-    private int queueIndex = -1;
-
-    /** Hashtable to hold the replacee-replacer pairs (String to String). */
-    private Hashtable hash = new Hashtable();
-
     /** Character marking the beginning of a token. */
-    private char beginToken = DEFAULT_BEGIN_TOKEN;
+    private String beginToken = DEFAULT_BEGIN_TOKEN;
 
     /** Character marking the end of a token. */
-    private char endToken = DEFAULT_END_TOKEN;
+    private String endToken = DEFAULT_END_TOKEN;
 
     /**
      * Constructor for "dummy" instances.
      *
      * @see BaseFilterReader#BaseFilterReader()
      */
-    public ReplaceTokens() {
-        super();
-    }
+    public ReplaceTokens() {}
 
     /**
      * Creates a new filtered reader.
@@ -97,18 +97,6 @@ public final class ReplaceTokens
         super(in);
     }
 
-    private int getNextChar() throws IOException {
-        if (queueIndex != -1) {
-            final int ch = queuedData.charAt(queueIndex++);
-            if (queueIndex >= queuedData.length()) {
-                queueIndex = -1;
-            }
-            return ch;
-        }
-
-        return in.read();
-    }
-
     /**
      * Returns the next character in the filtered stream, replacing tokens
      * from the original stream.
@@ -125,63 +113,66 @@ public final class ReplaceTokens
             setInitialized(true);
         }
 
-        if (replaceIndex != -1) {
-            final int ch = replaceData.charAt(replaceIndex++);
-            if (replaceIndex >= replaceData.length()) {
-                replaceIndex = -1;
+        if (!resolvedTokensBuilt) {
+            // build the resolved tokens tree map.
+            for (String key : hash.keySet()) {
+                resolvedTokens.put(beginToken + key + endToken, hash.get(key));
             }
-            return ch;
+            resolvedTokensBuilt = true;
         }
 
-        int ch = getNextChar();
-
-        if (ch == beginToken) {
-            final StringBuffer key = new StringBuffer("");
-            do  {
-                ch = getNextChar();
-                if (ch != -1) {
-                    key.append((char) ch);
-                } else {
-                    break;
-                }
-            } while (ch != endToken);
-
-            if (ch == -1) {
-                if (queuedData == null || queueIndex == -1) {
-                    queuedData = key.toString();
-                } else {
-                    queuedData
-                        = key.toString() + queuedData.substring(queueIndex);
-                }
-                if (queuedData.length() > 0) {
-                    queueIndex = 0;
-                } else {
-                    queueIndex = -1;
-                }
-                return beginToken;
+        // are we currently serving replace data?
+        if (replaceData != null) {
+            if (replaceIndex < replaceData.length()) {
+                return replaceData.charAt(replaceIndex++);
             } else {
-                key.setLength(key.length() - 1);
+                replaceData = null;
+            }
+        }
 
-                final String replaceWith = (String) hash.get(key.toString());
-                if (replaceWith != null) {
-                    if (replaceWith.length() > 0) {
-                        replaceData = replaceWith;
-                        replaceIndex = 0;
-                    }
-                    return read();
+        // is the read buffer empty?
+        if (readBuffer.length() == 0) {
+            int next = in.read();
+            if (next == -1) {
+                return next; // end of stream. all buffers empty.
+            }
+            readBuffer += (char)next;
+        }
+
+        for (;;) {
+            // get the closest tokens
+            SortedMap possibleTokens = resolvedTokens.tailMap(readBuffer);
+            if (possibleTokens.isEmpty() || !possibleTokens.firstKey().startsWith(readBuffer)) { // if there is none, then deliver the first char from the buffer.
+                return getFirstCharacterFromReadBuffer();
+            } else if (readBuffer.equals(possibleTokens.firstKey())) { // there exists a nearest token - is it an exact match?
+                // we have found a token. prepare the replaceData buffer.
+                replaceData = resolvedTokens.get(readBuffer);
+                replaceIndex = 0;
+                readBuffer = ""; // destroy the readBuffer - it's contents are being replaced entirely.
+                // get the first character via recursive call.
+                return read();
+            } else { // nearest token is not matching exactly - read one character more.
+                int next = in.read();
+                if (next != -1) {
+                    readBuffer += (char)next;
                 } else {
-                    String newData = key.toString() + endToken;
-                    if (queuedData == null || queueIndex == -1) {
-                        queuedData = newData;
-                    } else {
-                        queuedData = newData + queuedData.substring(queueIndex);
-                    }
-                    queueIndex = 0;
-                    return beginToken;
+                    return getFirstCharacterFromReadBuffer(); // end of stream. deliver remaining characters from buffer.
                 }
             }
         }
-        return ch;
+    }
+
+    /**
+     * @return the first character from the read buffer or -1 if read buffer is empty.
+     */
+    private int getFirstCharacterFromReadBuffer() {
+        if (readBuffer.length() > 0) {
+            int chr = readBuffer.charAt(0);
+            readBuffer = readBuffer.substring(1);
+            return chr;
+        } else {
+            return -1;
+        }
     }
 
     /**
@@ -189,7 +180,7 @@ public final class ReplaceTokens
      *
      * @param beginToken the character used to denote the beginning of a token
      */
-    public void setBeginToken(final char beginToken) {
+    public void setBeginToken(final String beginToken) {
         this.beginToken = beginToken;
     }
 
@@ -198,7 +189,7 @@ public final class ReplaceTokens
      *
      * @return the character used to denote the beginning of a token
      */
-    private char getBeginToken() {
+    private String getBeginToken() {
         return beginToken;
     }
 
@@ -207,7 +198,7 @@ public final class ReplaceTokens
      *
      * @param endToken the character used to denote the end of a token
      */
-    public void setEndToken(final char endToken) {
+    public void setEndToken(final String endToken) {
         this.endToken = endToken;
     }
 
@@ -216,7 +207,7 @@ public final class ReplaceTokens
      *
      * @return the character used to denote the end of a token
      */
-    private char getEndToken() {
+    private String getEndToken() {
         return endToken;
     }
 
@@ -238,18 +229,19 @@ public final class ReplaceTokens
      */
     public void addConfiguredToken(final Token token) {
         hash.put(token.getKey(), token.getValue());
+        resolvedTokensBuilt = false; // invalidate to build them again if they have been built already.
     }
 
     /**
      * Returns properties from a specified properties file.
      *
-     * @param fileName The file to load properties from.
+     * @param resource The resource to load properties from.
      */
-    private Properties getProperties(Resource r) {
+    private Properties getProperties(Resource resource) {
         InputStream in = null;
         Properties props = new Properties();
         try {
-            in = r.getInputStream();
+            in = resource.getInputStream();
             props.load(in);
         } catch (IOException ioe) {
             ioe.printStackTrace();
@@ -305,32 +297,23 @@ public final class ReplaceTokens
     private void initialize() {
         Parameter[] params = getParameters();
         if (params != null) {
-            for (int i = 0; i < params.length; i++) {
-                if (params[i] != null) {
-                    final String type = params[i].getType();
+            for (Parameter param : params) {
+                if (param != null) {
+                    final String type = param.getType();
                     if ("tokenchar".equals(type)) {
-                        final String name = params[i].getName();
-                        String value = params[i].getValue();
+                        final String name = param.getName();
                         if ("begintoken".equals(name)) {
-                            if (value.length() == 0) {
-                                throw new BuildException("Begin token cannot "
-                                    + "be empty");
-                            }
-                            beginToken = params[i].getValue().charAt(0);
+                            beginToken = param.getValue();
                         } else if ("endtoken".equals(name)) {
-                            if (value.length() == 0) {
-                                throw new BuildException("End token cannot "
-                                    + "be empty");
-                            }
-                            endToken = params[i].getValue().charAt(0);
+                            endToken = param.getValue();
                         }
                     } else if ("token".equals(type)) {
-                        final String name = params[i].getName();
-                        final String value = params[i].getValue();
+                        final String name = param.getName();
+                        final String value = param.getValue();
                         hash.put(name, value);
                     } else if ("propertiesfile".equals(type)) {
                         makeTokensFromProperties(
-                            new FileResource(new File(params[i].getValue())));
+                                new FileResource(new File(param.getValue())));
                     }
                 }
             }
diff --git a/src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java b/src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java
index c58dcc6d5..4db8d7a01 100644
--- a/src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java
+++ b/src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java
@@ -31,7 +31,6 @@ import static org.junit.Assert.assertEquals;
 
 public class ReplaceTokensTest {
 
-
     @Rule
     public BuildFileRule buildRule = new BuildFileRule();
 
@@ -45,7 +44,7 @@ public class ReplaceTokensTest {
         buildRule.executeTarget("testReplaceTokens");
         File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
         File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.test");
-       assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
     }
 
     @Test
@@ -56,4 +55,27 @@ public class ReplaceTokensTest {
         assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
     }
 
+    @Test
+    public void testReplaceTokensDoubleEncoded() throws IOException {
+        buildRule.executeTarget("testReplaceTokensDoubleEncoded");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.double.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
+
+    @Test
+    public void testReplaceTokensDoubleEncodedToSimple() throws IOException {
+        buildRule.executeTarget("testReplaceTokensDoubleEncodedToSimple");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
+
+    @Test
+    public void testReplaceTokensMustacheStyle() throws IOException {
+        buildRule.executeTarget("testReplaceTokensMustacheStyle");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.mustache.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
 }