Extend optional FTP task

Allow DirectoryScanner to work with remote directories.

Submitted by:	Glenn McAllister <glennm@ca.ibm.com>


git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@267934 13f79535-47bb-0310-9956-ffa450edef68

src/main/org/apache/tools/ant/DirectoryScanner.java

@@ -133,14 +133,14 @@ import java.util.*;
  *
  * @author Arnout J. Kuiper <a href="mailto:ajkuiper@wxs.nl">ajkuiper@wxs.nl</a>
  */
-public class DirectoryScanner {
+public class DirectoryScanner implements FileScanner {
 
     /**
      * Patterns that should be excluded by default.
      *
      * @see #addDefaultExcludes()
      */
-    private final static String[] DEFAULTEXCLUDES = {
+    protected final static String[] DEFAULTEXCLUDES = {
         "**/*~",
         "**/#*#",
         "**/%*%",
@@ -152,56 +152,56 @@ public class DirectoryScanner {
     /**
      * The base directory which should be scanned.
      */
-    private File basedir;
+    protected File basedir;
 
     /**
      * The patterns for the files that should be included.
      */
-    private String[] includes;
+    protected String[] includes;
 
     /**
      * The patterns for the files that should be excluded.
      */
-    private String[] excludes;
+    protected String[] excludes;
 
     /**
      * The files that where found and matched at least one includes, and matched
      * no excludes.
      */
-    private Vector filesIncluded;
+    protected Vector filesIncluded;
 
     /**
      * The files that where found and did not match any includes.
      */
-    private Vector filesNotIncluded;
+    protected Vector filesNotIncluded;
 
     /**
      * The files that where found and matched at least one includes, and also
      * matched at least one excludes.
      */
-    private Vector filesExcluded;
+    protected Vector filesExcluded;
 
     /**
      * The directories that where found and matched at least one includes, and
      * matched no excludes.
      */
-    private Vector dirsIncluded;
+    protected Vector dirsIncluded;
 
     /**
      * The directories that where found and did not match any includes.
      */
-    private Vector dirsNotIncluded;
+    protected Vector dirsNotIncluded;
 
     /**
      * The files that where found and matched at least one includes, and also
      * matched at least one excludes.
      */
-    private Vector dirsExcluded;
+    protected Vector dirsExcluded;
 
     /**
      * Have the Vectors holding our results been built by a slow scan?
      */
-    private boolean haveSlowResults = false;
+    protected boolean haveSlowResults = false;
 
     /**
      * Constructor.
@@ -221,7 +221,7 @@ public class DirectoryScanner {
      * @param pattern the (non-null) pattern to match against
      * @param str     the (non-null) string (path) to match
      */
-    private static boolean matchPatternStart(String pattern, String str) {
+    protected static boolean matchPatternStart(String pattern, String str) {
         // When str starts with a File.separator, pattern has to start with a
         // File.separator.
         // When pattern starts with a File.separator, str has to start with a
@@ -283,7 +283,7 @@ public class DirectoryScanner {
      * @return <code>true</code> when the pattern matches against the string.
      *         <code>false</code> otherwise.
      */
-    private static boolean matchPath(String pattern, String str) {
+    protected static boolean matchPath(String pattern, String str) {
         // When str starts with a File.separator, pattern has to start with a
         // File.separator.
         // When pattern starts with a File.separator, str has to start with a
@@ -423,7 +423,7 @@ strLoop:
      * @return <code>true</code> when the string matches against the pattern,
      *         <code>false</code> otherwise.
      */
-    private static boolean match(String pattern, String str) {
+    protected static boolean match(String pattern, String str) {
         char[] patArr = pattern.toCharArray();
         char[] strArr = str.toCharArray();
         int patIdxStart = 0;
@@ -682,7 +682,7 @@ strLoop:
      *
      * <p>Returns immediately if a slow scan has already been requested.
      */
-    private void slowScan() {
+    protected void slowScan() {
         if (haveSlowResults) {
             return;
         }
@@ -725,7 +725,7 @@ strLoop:
      * @see #dirsNotIncluded
      * @see #dirsExcluded
      */
-    private void scandir(File dir, String vpath, boolean fast) {
+    protected void scandir(File dir, String vpath, boolean fast) {
         String[] newfiles = dir.list();
 
         if (newfiles == null) {
@@ -785,7 +785,7 @@ strLoop:
      * @return <code>true</code> when the name matches against at least one
      *         include pattern, <code>false</code> otherwise.
      */
-    private boolean isIncluded(String name) {
+    protected boolean isIncluded(String name) {
         for (int i = 0; i < includes.length; i++) {
             if (matchPath(includes[i],name)) {
                 return true;
@@ -801,7 +801,7 @@ strLoop:
      * @return <code>true</code> when the name matches against at least one
      *         include pattern, <code>false</code> otherwise.
      */
-    private boolean couldHoldIncluded(String name) {
+    protected boolean couldHoldIncluded(String name) {
         for (int i = 0; i < includes.length; i++) {
             if (matchPatternStart(includes[i],name)) {
                 return true;
@@ -817,7 +817,7 @@ strLoop:
      * @return <code>true</code> when the name matches against at least one
      *         exclude pattern, <code>false</code> otherwise.
      */
-    private boolean isExcluded(String name) {
+    protected boolean isExcluded(String name) {
         for (int i = 0; i < excludes.length; i++) {
             if (matchPath(excludes[i],name)) {
                 return true;

src/main/org/apache/tools/ant/FileScanner.java

@@ -0,0 +1,102 @@
+package org.apache.tools.ant;
+
+import java.io.*;
+
+/**
+ * An interface used to describe the actions required by any type of 
+ * directory scanner.
+ */
+public interface FileScanner {
+	/**
+	 * Adds an array with default exclusions to the current exclusions set.
+	 *
+	 */
+	public void addDefaultExcludes();
+	/**
+	 * Gets the basedir that is used for scanning. This is the directory that
+	 * is scanned recursively.
+	 *
+	 * @return the basedir that is used for scanning
+	 */
+	public File getBasedir();
+	/**
+	 * Get the names of the directories that matched at least one of the include
+	 * patterns, an matched also at least one of the exclude patterns.
+	 * The names are relative to the basedir.
+	 *
+	 * @return the names of the directories
+	 */
+	public String[] getExcludedDirectories();
+	/**
+	 * Get the names of the files that matched at least one of the include
+	 * patterns, an matched also at least one of the exclude patterns.
+	 * The names are relative to the basedir.
+	 *
+	 * @return the names of the files
+	 */
+	public String[] getExcludedFiles();
+	/**
+	 * Get the names of the directories that matched at least one of the include
+	 * patterns, an matched none of the exclude patterns.
+	 * The names are relative to the basedir.
+	 *
+	 * @return the names of the directories
+	 */
+	public String[] getIncludedDirectories();
+	/**
+	 * Get the names of the files that matched at least one of the include
+	 * patterns, an matched none of the exclude patterns.
+	 * The names are relative to the basedir.
+	 *
+	 * @return the names of the files
+	 */
+	public String[] getIncludedFiles();
+	/**
+	 * Get the names of the directories that matched at none of the include
+	 * patterns.
+	 * The names are relative to the basedir.
+	 *
+	 * @return the names of the directories
+	 */
+	public String[] getNotIncludedDirectories();
+	/**
+	 * Get the names of the files that matched at none of the include patterns.
+	 * The names are relative to the basedir.
+	 *
+	 * @return the names of the files
+	 */
+	public String[] getNotIncludedFiles();
+	/**
+	 * Scans the base directory for files that match at least one include
+	 * pattern, and don't match any exclude patterns.
+	 *
+	 * @exception IllegalStateException when basedir was set incorrecly
+	 */
+	public void scan();
+	/**
+	 * Sets the basedir for scanning. This is the directory that is scanned
+	 * recursively. 
+	 *
+	 * @param basedir the (non-null) basedir for scanning
+	 */
+	public void setBasedir(String basedir);
+	/**
+	 * Sets the basedir for scanning. This is the directory that is scanned
+	 * recursively.
+	 *
+	 * @param basedir the basedir for scanning
+	 */
+	public void setBasedir(File basedir);
+	/**
+	 * Sets the set of exclude patterns to use.
+	 *
+	 * @param excludes list of exclude patterns
+	 */
+	public void setExcludes(String[] excludes);
+	/**
+	 * Sets the set of include patterns to use.
+	 *
+	 * @param includes list of include patterns
+	 */
+	public void setIncludes(String[] includes);
+}

src/main/org/apache/tools/ant/taskdefs/optional/FTP.java

@@ -62,21 +62,31 @@ import java.util.*;
 import com.oroinc.net.ftp.*;
 
 /**
- * Sends files to an FTP server.  Also intended to retrieve remote files,
- * but this has not yet been implemented
+ * Basic FTP client that performs the following actions:
+ * <ul>
+ *   <li><strong>send</strong> - send files to a remote server.  This is the
+ *              default action.</li>
+ *   <li><strong>get</strong> - retrive files from a remote server.</li>
+ *   <li><strong>del</strong> - delete files from a remote server.</li>
+ *   <li><strong>list</strong> - create a file listing.</li>
+ * </ul>
  *
  * @author Roger Vaughn <a href="mailto:rvaughn@seaconinc.com">rvaughn@seaconinc.com</a>
+ * @author Glenn McAllister <a href="mailto:glennm@ca.ibm.com">glennm@ca.ibm.com</a>
  */
 public class FTP
     extends Task
 {
     protected final static int SEND_FILES   = 0;
     protected final static int GET_FILES    = 1;
+    protected final static int DEL_FILES    = 2;	
+    protected final static int LIST_FILES   = 3;
     
     private String remotedir;
     private String server;
     private String userid;
     private String password;
+    private File listing;
     private boolean binary = true;
     private boolean verbose = false;
     private boolean newerOnly = false;
@@ -87,6 +97,107 @@ public class FTP
     private String remoteFileSep = "/";
     private int port = 21;
 
+    protected final static String[] ACTION_STRS = {
+        "sending",
+        "getting",
+        "deleting",
+        "listing"
+    };
+
+    protected final static String[] COMPLETED_ACTION_STRS = {
+        "sent",
+        "retrieved",
+        "deleted",
+        "listed"
+    };		
+	
+    protected class FTPDirectoryScanner extends DirectoryScanner {
+        protected FTPClient ftp = null;
+	
+        public FTPDirectoryScanner(FTPClient ftp) {
+            super();
+            this.ftp = ftp;
+        }
+
+        public void scan() {
+            if (includes == null) {
+                // No includes supplied, so set it to 'matches all'
+                includes = new String[1];
+                includes[0] = "**";
+            }
+            if (excludes == null) {
+                excludes = new String[0];
+            }
+
+            filesIncluded = new Vector();
+            filesNotIncluded = new Vector();
+            filesExcluded = new Vector();
+            dirsIncluded = new Vector();
+            dirsNotIncluded = new Vector();
+            dirsExcluded = new Vector();
+
+            try {
+                String cwd = ftp.printWorkingDirectory();
+                scandir(".", "", true); // always start from the current ftp working dir
+                ftp.changeWorkingDirectory(cwd);
+            } catch (IOException e) {
+                throw new BuildException("Unable to scan FTP server: ", e);
+            }
+        }
+
+        protected void scandir(String dir, String vpath, boolean fast) {
+            try {
+                if (!ftp.changeWorkingDirectory(dir)) {
+                    return;
+                }
+
+                FTPFile[] newfiles = ftp.listFiles();
+                if (newfiles == null) {
+                    return;    // no files in directory.
+                }
+
+                for (int i = 0; i < newfiles.length; i++) {
+                    FTPFile file = newfiles[i];
+                    String name = vpath + file.getName();
+                    if (file.isDirectory()) {
+                        if (isIncluded(name)) {
+                            if (!isExcluded(name)) {
+                                dirsIncluded.addElement(name);
+                                if (fast) {
+                                    scandir(name, name + File.separator, fast);
+                                }
+                            } else {
+                                dirsExcluded.addElement(name);
+                            }
+                        } else {
+                            dirsNotIncluded.addElement(name);
+                            if (fast && couldHoldIncluded(name)) {
+                                scandir(name, name + File.separator, fast);
+                            }
+                        }
+                        if (!fast) {
+                            scandir(name, name + File.separator, fast);
+                        }
+                    } else {
+                        if (file.isFile()) {
+                            if (isIncluded(name)) {
+                                if (!isExcluded(name)) {
+                                    filesIncluded.addElement(name);
+                                } else {
+                                    filesExcluded.addElement(name);
+                                }
+                            } else {
+                                filesNotIncluded.addElement(name);
+                            }
+                        }
+                    }
+                }
+            } catch (IOException e) {
+                throw new BuildException("Error while communicating with FTP server: ", e);
+            }
+        }
+    }	
+
     /**
      * Sets the remote directory where files will be placed.  This may
      * be a relative or absolute path, and must be in the path syntax
@@ -192,8 +303,8 @@ public class FTP
     }
 
     /**
-     * Sets the FTP action to be taken.  Currently accepts "put" and "get".
-     * "get" tasks are not yet supported and will effectively perform a noop.
+     * Sets the FTP action to be taken.  Currently accepts "put", "get",
+     * "del", and "list".
      */
     public void setAction(String action) throws BuildException
     {
@@ -207,11 +318,29 @@ public class FTP
         {
             this.action = GET_FILES;
         }
+        else if (action.toLowerCase().equals("del") ||
+                 action.toLowerCase().equals("delete" ))
+        {
+            this.action = DEL_FILES;
+        }
+        else if (action.toLowerCase().equals("list"))
+        {
+            this.action = LIST_FILES;
+        }
         else
         {
             throw new BuildException("action " + action + " is not supported");
         }
     }
+    
+    /**
+     * The output file for the "list" action.  This attribute is ignored for
+     * any other actions.
+     */
+    public void setListing(File listing) throws BuildException {
+        this.listing = listing;
+    }	
+    
 
     /**
      * Checks to see that all required parameters are set.
@@ -230,20 +359,81 @@ public class FTP
         {
             throw new BuildException("password attribute must be set!");
         }
+        
+        if ((action == LIST_FILES) && (listing == null))
+        {
+            throw new BuildException("listing attribute must be set for list action!");
+        }
     }
 
     /**
-     * Append all files found by a directory scanner to a vector.
+     * For each file in the fileset, do the appropriate action: send, get, delete,
+     * or list.
      */
-    protected int sendFiles(FTPClient ftp, DirectoryScanner ds)
+    protected int transferFiles(FTPClient ftp, FileSet fs)
         throws IOException, BuildException
     {
+        FileScanner ds;
+
+        if (action == SEND_FILES) {
+            ds = fs.getDirectoryScanner(project);
+        } else {
+            ds = new FTPDirectoryScanner(ftp);
+            fs.setupDirectoryScanner(ds, project);
+            ds.scan();
+        }			
+				
         String[] dsfiles = ds.getIncludedFiles();
-        String dir = ds.getBasedir().getAbsolutePath();
+        String dir = null;
+        if ((ds.getBasedir() == null) && ((action == SEND_FILES) || (action == GET_FILES))) {
+            throw new BuildException( "the dir attribute must be set for send and get actions" );
+        } else {
+            if ((action == SEND_FILES) || (action == GET_FILES)) {
+                dir = ds.getBasedir().getAbsolutePath();
+            }
+        }
+
+        // If we are doing a listing, we need the output stream created now.
+        BufferedWriter bw = null;
+        if (action == LIST_FILES) {
+            File pd = new File(listing.getParent());
+            if (!pd.exists()) {
+                pd.mkdirs();
+            }
+            bw = new BufferedWriter(new FileWriter(listing));
+        }
 
         for (int i = 0; i < dsfiles.length; i++)
         {
-            sendFile(ftp, dir, dsfiles[i]);
+            switch (action) {
+                case SEND_FILES: {
+                    sendFile(ftp, dir, dsfiles[i]);
+                    break;
+                }
+
+                case GET_FILES: {
+                    getFile(ftp, dir, dsfiles[i]);
+                    break;
+                }
+
+                case DEL_FILES: {
+                    delFile(ftp, dsfiles[i]);
+                    break;
+                }
+
+                case LIST_FILES: {
+                    listFile(ftp, bw, dsfiles[i]);
+                    break;
+                }
+
+                default: {
+                    throw new BuildException("unknown ftp action " + action );
+                }
+            }
+        }
+
+        if (action == LIST_FILES) {
+            bw.close();
         }
 
         return dsfiles.length;
@@ -253,7 +443,7 @@ public class FTP
      * Sends all files specified by the configured filesets to the remote
      * server.
      */
-    protected void sendFiles(FTPClient ftp)
+    protected void transferFiles(FTPClient ftp)
         throws IOException, BuildException
     {
         transferred = 0;
@@ -298,12 +488,12 @@ public class FTP
 
                 if (fs != null)
                 {
-                    sendFiles(ftp, fs.getDirectoryScanner(project));
+                    transferFiles(ftp, fs);
                 }
             }
         }
 
-        log(transferred + " files transferred");
+        log(transferred + " files " + COMPLETED_ACTION_STRS[action]);
     }
 
     /**
@@ -380,8 +570,13 @@ public class FTP
             return false;
         }
 
-        return files[0].getTimestamp().getTime().getTime() >
-            localFile.lastModified();
+        long remoteTimestamp = files[0].getTimestamp().getTime().getTime();
+        long localTimestamp = localFile.lastModified();
+        if (this.action == SEND_FILES) {
+            return remoteTimestamp > localTimestamp;
+        } else {
+            return localTimestamp > remoteTimestamp;
+        }
     }
 
     /**
@@ -444,6 +639,106 @@ public class FTP
         }
     }
 
+    /**
+     * Delete a file from the remote host.
+     */
+    protected void delFile(FTPClient ftp, String filename)
+            throws IOException, BuildException {
+        if (verbose) {
+            log("deleting " + filename);
+        }
+
+        if (!ftp.deleteFile(resolveFile(filename))) {
+            throw new BuildException("could not delete file: " + ftp.getReplyString());
+        }
+
+        log("File " + filename + " deleted from " + server, Project.MSG_VERBOSE); 
+
+        transferred++;
+    }
+
+    /**
+     * Retrieve a single file to the remote host.
+     * <code>filename</code> may contain a relative path specification.
+     * The file will then be retreived using the entire relative path spec - 
+     * no attempt is made to change directories.  It is anticipated that this may
+     * eventually cause problems with some FTP servers, but it simplifies
+     * the coding.
+     */
+    protected void getFile(FTPClient ftp, String dir, String filename)
+        throws IOException, BuildException
+    {
+        OutputStream outstream = null;
+        try
+        {
+            File file = project.resolveFile(new File(dir, filename).getPath());
+
+            if (newerOnly && isUpToDate(ftp, file, resolveFile(filename)))
+                return;
+
+            if (verbose)
+            {
+                log("transferring " + filename + " to " + file.getAbsolutePath());
+            }
+
+			
+            File pdir = new File(file.getParent());	// stay 1.1 compatible
+            if (!pdir.exists()) {
+                pdir.mkdirs();
+            }		
+            outstream = new BufferedOutputStream(new FileOutputStream(file));
+            ftp.retrieveFile(resolveFile(filename), outstream);
+			
+            if (!FTPReply.isPositiveCompletion(ftp.getReplyCode()))
+            {
+                throw new BuildException(
+"could not transfer file: " +
+ ftp.getReplyString());
+            }
+			
+            log("File " + file.getAbsolutePath() + " copied from " + server,
+ Project.MSG_VERBOSE);
+
+            transferred++;
+        }
+        finally
+        {            
+            if (outstream != null)
+            {
+                try
+                {
+                    outstream.close();
+                }
+                catch(IOException ex)
+                {
+                    // ignore it
+                }
+            }
+        }
+    }
+
+    /**
+     * List information about a single file from the remote host.
+     * <code>filename</code> may contain a relative path specification.
+     * The file listing will then be retrieved using the entire relative path spec 
+     * - no attempt is made to change directories.  It is anticipated that this may
+     * eventually cause problems with some FTP servers, but it simplifies
+     * the coding.
+     */
+    protected void listFile(FTPClient ftp, BufferedWriter bw, String filename)
+        throws IOException, BuildException 
+    {
+        if (verbose) {
+            log("listing " + filename);
+        }
+
+        FTPFile ftpfile = ftp.listFiles(resolveFile(filename))[0];
+        bw.write(ftpfile.toString());
+        bw.newLine();
+
+        transferred++;
+    }
+
     /**
      * Runs the task.
      */
@@ -499,16 +794,9 @@ public class FTP
                 }
             }
 
-            log("transferring files");
+            log(ACTION_STRS[action] + " files");
+            transferFiles(ftp);
 
-            if (action == SEND_FILES)
-            {
-                sendFiles(ftp);
-            }
-            else
-            {
-                throw new BuildException("getting files is not yet supported");
-            }
         }
         catch(IOException ex)
         {

src/main/org/apache/tools/ant/types/FileSet.java

@@ -55,6 +55,7 @@
 package org.apache.tools.ant.types;
 
 import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.FileScanner;
 import org.apache.tools.ant.DirectoryScanner;
 import org.apache.tools.ant.Project;
 
@@ -198,6 +199,16 @@ public class FileSet {
         }
 
         DirectoryScanner ds = new DirectoryScanner();
+        setupDirectoryScanner(ds, p);
+        ds.scan();
+        return ds;
+    }
+    
+    public void setupDirectoryScanner(FileScanner ds, Project p) {
+        if (ds == null) {
+            throw new IllegalArgumentException("ds cannot be null");
+        }
+        
         ds.setBasedir(dir);
 
         for (int i=0; i<additionalPatterns.size(); i++) {
@@ -219,8 +230,5 @@ public class FileSet {
         ds.setIncludes(defaultPatterns.getIncludePatterns(p));
         ds.setExcludes(defaultPatterns.getExcludePatterns(p));
         if (useDefaultExcludes) ds.addDefaultExcludes();
-        ds.scan();
-        return ds;
     }
-
 }