diff --git a/src/main/org/apache/tools/ant/types/AbstractFileSet.java b/src/main/org/apache/tools/ant/types/AbstractFileSet.java
index 1f970032c..ae0ba0acf 100644
--- a/src/main/org/apache/tools/ant/types/AbstractFileSet.java
+++ b/src/main/org/apache/tools/ant/types/AbstractFileSet.java
@@ -586,6 +586,7 @@ public abstract class AbstractFileSet extends DataType
             throw noChildrenAllowed();
         }
         selectors.addElement(selector);
+        directoryScanner = null;
     }
 
     /* Methods below all add specific selectors */
@@ -784,7 +785,6 @@ public abstract class AbstractFileSet extends DataType
                         .addElement(((PatternSet) e.nextElement()).clone());
                 }
                 fs.selectors = (Vector) fs.selectors.clone();
-                fs.directoryScanner = null; // Clear the cached directory scanner
                 return fs;
             } catch (CloneNotSupportedException e) {
                 throw new BuildException(e);