diff --git a/WHATSNEW b/WHATSNEW index d6687ac32..ae6d47abb 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -270,6 +270,11 @@ Other changes: between client and remote side. Bugzilla Report 19358. +* has been optimized to go directly to the include patterns. + This reduces scanning time under UNIX when followsymlinks="true" + and casesensitive="true" (the default) + Bugzilla Report 20103. + * The SOS and VSS tasks will no longer unconditionally prepend a $ to vsspath or projectpath. diff --git a/src/main/org/apache/tools/ant/DirectoryScanner.java b/src/main/org/apache/tools/ant/DirectoryScanner.java index d13130c68..ce62e282b 100644 --- a/src/main/org/apache/tools/ant/DirectoryScanner.java +++ b/src/main/org/apache/tools/ant/DirectoryScanner.java @@ -504,7 +504,17 @@ public class DirectoryScanner } /** - * Sets whether or not the file system should be regarded as case sensitive. + * Find out whether include exclude patterns are matched in a + * case sensitive way + * @return whether or not the scanning is case sensitive + * @since ant 1.6 + */ + public boolean isCaseSensitive() { + return isCaseSensitive; + } + /** + * Sets whether or not include and exclude patterns are matched + * in a case sensitive way * * @param isCaseSensitive whether or not the file system should be * regarded as a case sensitive one diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/net/FTP.java b/src/main/org/apache/tools/ant/taskdefs/optional/net/FTP.java index 2fe82ac61..738510512 100644 --- a/src/main/org/apache/tools/ant/taskdefs/optional/net/FTP.java +++ b/src/main/org/apache/tools/ant/taskdefs/optional/net/FTP.java @@ -66,9 +66,16 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.HashSet; import java.util.Locale; +import java.util.Map; +import java.util.Set; import java.util.StringTokenizer; import java.util.Vector; + import org.apache.tools.ant.BuildException; import org.apache.tools.ant.DirectoryScanner; import org.apache.tools.ant.Project; @@ -76,6 +83,7 @@ import org.apache.tools.ant.Task; import org.apache.tools.ant.taskdefs.Delete; import org.apache.tools.ant.types.EnumeratedAttribute; import org.apache.tools.ant.types.FileSet; +import org.apache.tools.ant.types.selectors.SelectorUtils; import org.apache.tools.ant.util.FileUtils; /** @@ -185,7 +193,13 @@ public class FTP */ protected class FTPDirectoryScanner extends DirectoryScanner { protected FTPClient ftp = null; - + private String rootPath = null; + /** + * since ant 1.6 + * this flag should be set to true on UNIX and can save scanning time + */ + private boolean remoteSystemCaseSensitive = false; + private boolean remoteSensitivityChecked = false; /** * constructor @@ -223,7 +237,8 @@ public class FTP String cwd = ftp.printWorkingDirectory(); // always start from the current ftp working dir - scandir(".", "", true); + checkIncludePatterns(); + clearCaches(); ftp.changeWorkingDirectory(cwd); } catch (IOException e) { throw new BuildException("Unable to scan FTP server: ", e); @@ -231,6 +246,106 @@ public class FTP } + /** + * this routine is actually checking all the include patterns in + * order to avoid scanning everything under base dir + * @since ant1.6 + */ + private void checkIncludePatterns() { + Hashtable newroots = new Hashtable(); + // put in the newroots vector the include patterns without + // wildcard tokens + for (int icounter = 0; icounter < includes.length; icounter++) { + String newpattern = + SelectorUtils.rtrimWildcardTokens(includes[icounter]); + newroots.put(newpattern, includes[icounter]); + } + if (remotedir == null) { + try { + remotedir = ftp.printWorkingDirectory(); + } catch (IOException e) { + throw new BuildException("could not read current ftp directory", + getLocation()); + } + } + AntFTPFile baseFTPFile = new AntFTPRootFile(ftp, remotedir); + rootPath = baseFTPFile.getAbsolutePath(); + // construct it + if (newroots.containsKey("")) { + // we are going to scan everything anyway + scandir(remotedir, "", true); + } else { + // only scan directories that can include matched files or + // directories + Enumeration enum2 = newroots.keys(); + + while (enum2.hasMoreElements()) { + String currentelement = (String) enum2.nextElement(); + String originalpattern = (String) newroots.get(currentelement); + AntFTPFile myfile = new AntFTPFile(baseFTPFile, currentelement); + boolean isOK = true; + boolean traversesSymlinks = false; + String path = null; + + if (myfile.exists()) { + if (remoteSensitivityChecked + && remoteSystemCaseSensitive && isFollowSymlinks()) { + // cool case, + //we do not need to scan all the subdirs in the relative path + path = myfile.getFastRelativePath(); + } else { + // may be on a case insensitive file system. We want + // the results to show what's really on the disk, so + // we need to double check. + try { + path = myfile.getRelativePath(); + traversesSymlinks = myfile.isTraverseSymlinks(); + } catch (IOException be) { + throw new BuildException(be, getLocation()); + } catch (BuildException be) { + isOK = false; + + } + } + } else { + isOK = false; + } + if (isOK) { + currentelement = path.replace(remoteFileSep.charAt(0), File.separatorChar); + if (!isFollowSymlinks() + && traversesSymlinks) { + continue; + } + + if (myfile.isDirectory()) { + if (isIncluded(currentelement) + && currentelement.length() > 0) { + accountForIncludedDir(currentelement, myfile, true); + } else { + if (currentelement.length() > 0) { + if (currentelement.charAt(currentelement + .length() - 1) + != File.separatorChar) { + currentelement = + currentelement + File.separatorChar; + } + } + scandir(myfile.getAbsolutePath(), currentelement, true); + } + } else { + if (isCaseSensitive + && originalpattern.equals(currentelement)) { + accountForIncludedFile(currentelement); + } else if (!isCaseSensitive + && originalpattern + .equalsIgnoreCase(currentelement)) { + accountForIncludedFile(currentelement); + } + } + } + } + } + } /** * scans a particular directory * @param dir directory to scan @@ -239,12 +354,22 @@ public class FTP * @param fast seems to be always true in practice */ protected void scandir(String dir, String vpath, boolean fast) { + // avoid double scanning of directories, can only happen in fast mode + if (fast && hasBeenScanned(vpath)) { + return; + } try { if (!ftp.changeWorkingDirectory(dir)) { return; } - - FTPFile[] newfiles = ftp.listFiles(); + String completePath = null; + if (!vpath.equals("")) { + completePath = rootPath + remoteFileSep + + vpath.replace(File.separatorChar, remoteFileSep.charAt(0)); + } else { + completePath = rootPath; + } + FTPFile[] newfiles = listFiles(completePath, false); if (newfiles == null) { ftp.changeToParentDirectory(); @@ -261,24 +386,8 @@ public class FTP dirsExcluded.addElement(name); slowScanAllowed = false; } else if (isIncluded(name)) { - if (!isExcluded(name)) { - if (fast) { - if (file.isSymbolicLink()) { - scandir(file.getLink(), - name + File.separator, fast); - } else { - scandir(file.getName(), - name + File.separator, fast); - } - } - dirsIncluded.addElement(name); - } else { - dirsExcluded.addElement(name); - if (fast && couldHoldIncluded(name)) { - scandir(file.getName(), - name + File.separator, fast); - } - } + accountForIncludedDir(name, + new AntFTPFile(ftp, file, completePath) , fast); } else { dirsNotIncluded.addElement(name); if (fast && couldHoldIncluded(name)) { @@ -295,15 +404,7 @@ public class FTP if (!isFollowSymlinks() && file.isSymbolicLink()) { filesExcluded.addElement(name); } else if (isFunctioningAsFile(ftp, dir, file)) { - if (isIncluded(name)) { - if (!isExcluded(name)) { - filesIncluded.addElement(name); - } else { - filesExcluded.addElement(name); - } - } else { - filesNotIncluded.addElement(name); - } + accountForIncludedFile(name); } } } @@ -314,6 +415,487 @@ public class FTP + "server: ", e); } } + /** + * process included file + * @param name path of the file relative to the directory of the fileset + */ + private void accountForIncludedFile(String name) { + if (!filesIncluded.contains(name) + && !filesExcluded.contains(name)) { + + if (isIncluded(name)) { + if (!isExcluded(name)) { + filesIncluded.addElement(name); + } else { + filesExcluded.addElement(name); + } + } else { + filesNotIncluded.addElement(name); + } + } + } + + /** + * + * @param name path of the directory relative to the directory of + * the fileset + * @param file directory as file + * @param fast + */ + private void accountForIncludedDir(String name, AntFTPFile file, boolean fast) { + if (!dirsIncluded.contains(name) + && !dirsExcluded.contains(name)) { + + if (!isExcluded(name)) { + if (fast) { + if (file.isSymbolicLink()) { + try { + file.getClient().changeWorkingDirectory(file.curpwd); + } catch (IOException ioe) { + throw new BuildException("could not change directory to curpwd"); + } + scandir(file.getLink(), + name + File.separator, fast); + } else { + try { + file.getClient().changeWorkingDirectory(file.curpwd); + } catch (IOException ioe) { + throw new BuildException("could not change directory to curpwd"); + } + scandir(file.getName(), + name + File.separator, fast); + } + } + dirsIncluded.addElement(name); + } else { + dirsExcluded.addElement(name); + if (fast && couldHoldIncluded(name)) { + try { + file.getClient().changeWorkingDirectory(file.curpwd); + } catch (IOException ioe) { + throw new BuildException("could not change directory to curpwd"); + } + scandir(file.getName(), + name + File.separator, fast); + } + } + } + } + /** + * temporary table to speed up the various scanning methods below + * + * @since Ant 1.6 + */ + private Map fileListMap = new HashMap(); + /** + * List of all scanned directories. + * + * @since Ant 1.6 + */ + private Set scannedDirs = new HashSet(); + + /** + * Has the directory with the given path relative to the base + * directory already been scanned? + * + *

Registers the given directory as scanned as a side effect.

+ * + * @since Ant 1.6 + */ + private boolean hasBeenScanned(String vpath) { + return !scannedDirs.add(vpath); + } + + /** + * Clear internal caches. + * + * @since Ant 1.6 + */ + private void clearCaches() { + fileListMap.clear(); + scannedDirs.clear(); + } + /** + * list the files present in one directory. + * @param directory full path on the remote side + * @param changedir if true change to directory directory before listing + * @return array of FTPFile + */ + public FTPFile[] listFiles(String directory, boolean changedir) { + //getProject().log("listing files in directory " + directory, Project.MSG_DEBUG); + String currentPath = directory; + if (changedir) { + try { + boolean result = ftp.changeWorkingDirectory(directory); + if (!result) { + return null; + } + currentPath = ftp.printWorkingDirectory(); + } catch (IOException ioe) { + throw new BuildException(ioe, getLocation()); + } + } + if (fileListMap.containsKey(currentPath)) { + getProject().log("filelist map used in listing files", Project.MSG_DEBUG); + return ((FTPFile[]) fileListMap.get(currentPath)); + } + FTPFile[] result = null; + try { + result = ftp.listFiles(); + } catch (IOException ioe) { + throw new BuildException(ioe, getLocation()); + } + fileListMap.put(currentPath, result); + if (!remoteSensitivityChecked) { + checkRemoteSensitivity(result, directory); + } + return result; + } + /** + * cd into one directory and + * list the files present in one directory. + * @param directory full path on the remote side + * @return array of FTPFile + */ + public FTPFile[] listFiles(String directory) { + return listFiles(directory, true); + } + private void checkRemoteSensitivity(FTPFile[] array, String directory) { + boolean candidateFound = false; + String target = null; + for (int icounter = 0; icounter < array.length; icounter++) { + if (array[icounter].isDirectory()) { + if (!array[icounter].getName().equals(".") + && !array[icounter].getName().equals("..")) { + candidateFound = true; + target = fiddleName(array[icounter].getName()); + getProject().log("will try to cd to " + + target + " where a directory called " + array[icounter].getName() + + " exists", Project.MSG_DEBUG); + for (int pcounter = 0; pcounter < array.length; pcounter++) { + if (array[pcounter].getName().equals(target) && pcounter != icounter) { + candidateFound = false; + } + } + if (candidateFound) { + break; + } + } + } + } + if (candidateFound) { + try { + getProject().log("testing case sensitivity, attempting to cd to " + + target, Project.MSG_DEBUG); + remoteSystemCaseSensitive = !ftp.changeWorkingDirectory(target); + } catch (IOException ioe) { + remoteSystemCaseSensitive = true; + } finally { + try { + ftp.changeWorkingDirectory(directory); + } catch (IOException ioe) { + throw new BuildException(ioe, getLocation()); + } + } + getProject().log("remote system is case sensitive : " + remoteSystemCaseSensitive, + Project.MSG_VERBOSE); + remoteSensitivityChecked = true; + } + } + private String fiddleName(String origin) { + StringBuffer result = new StringBuffer(); + for (int icounter = 0; icounter < origin.length(); icounter++) { + if (Character.isLowerCase(origin.charAt(icounter))) { + result.append(Character.toUpperCase(origin.charAt(icounter))); + } else if (Character.isUpperCase(origin.charAt(icounter))) { + result.append(Character.toLowerCase(origin.charAt(icounter))); + } else { + result.append(origin.charAt(icounter)); + } + } + return result.toString(); + } + /** + * an AntFTPFile is a representation of a remote file + * @since Ant 1.6 + */ + protected class AntFTPFile { + /** + * ftp client + */ + private FTPClient client; + /** + * parent directory of the file + */ + private String curpwd; + /** + * the file itself + */ + private FTPFile ftpFile; + /** + * + */ + private AntFTPFile parent = null; + private boolean relativePathCalculated = false; + private boolean traversesSymlinks = false; + private String relativePath = ""; + /** + * constructor + * @param client ftp client variable + * @param ftpFile the file + * @param curpwd absolute remote path where the file is found + */ + public AntFTPFile(FTPClient client, FTPFile ftpFile, String curpwd) { + this.client = client; + this.ftpFile = ftpFile; + this.curpwd = curpwd; + } + /** + * other constructor + * @param parent the parent file + * @param path a relative path to the parent file + */ + public AntFTPFile(AntFTPFile parent, String path) { + this.parent = parent; + this.client = parent.client; + Vector pathElements = SelectorUtils.tokenizePath(path); + try { + this.client.changeWorkingDirectory(parent.getAbsolutePath()); + this.curpwd = parent.getAbsolutePath(); + } catch (IOException ioe) { + throw new BuildException("could not change working dir to " + + parent.curpwd); + } + for (int fcount = 0; fcount < pathElements.size() - 1; fcount++) { + try { + this.client.changeWorkingDirectory((String) pathElements.elementAt(fcount)); + this.curpwd = this.curpwd + remoteFileSep + + (String) pathElements.elementAt(fcount); + } catch (IOException ioe) { + throw new BuildException("could not change working dir to " + + (String) pathElements.elementAt(fcount) + + " from " + this.curpwd); + } + + } + String lastpathelement = (String) pathElements.elementAt(pathElements.size() - 1); + FTPFile [] theFiles = listFiles(this.curpwd); + this.ftpFile = getFile(theFiles, lastpathelement); + } + /** + * find out if the file exists + * @return true if the file exists + */ + public boolean exists() { + return (ftpFile != null); + } + /** + * if the file is a symbolic link, find out to what it is pointing + * @return the target of the symbolic link + */ + public String getLink() { + return ftpFile.getLink(); + } + /** + * get the name of the file + * @return the name of the file + */ + public String getName() { + return ftpFile.getName(); + } + /** + * find out the absolute path of the file + * @return absolute path as string + */ + public String getAbsolutePath() { + return curpwd + remoteFileSep + ftpFile.getName(); + } + /** + * find out the relative path assuming that the path used to construct + * this AntFTPFile was spelled properly with regards to case. + * This is OK on a case sensitive system such as UNIX + * @return relative path + */ + public String getFastRelativePath() { + String absPath = getAbsolutePath(); + if (absPath.indexOf(rootPath + remoteFileSep) == 0) { + return absPath.substring(rootPath.length() + remoteFileSep.length()); + } + return null; + } + /** + * find out the relative path to the rootPath of the enclosing scanner. + * this relative path is spelled exactly like on disk, + * for instance if the AntFTPFile has been instantiated as ALPHA, + * but the file is really called alpha, this method will return alpha. + * If a symbolic link is encountered, it is followed, but the name of the link + * rather than the name of the target is returned. + * (ie does not behave like File.getCanonicalPath()) + * @return relative path, separated by remoteFileSep + * @throws IOException if a change directory fails, ... + * @throws BuildException if one of the components of the relative path cannot + * be found. + */ + public String getRelativePath() throws IOException, BuildException { + if (!relativePathCalculated) { + if (parent != null) { + traversesSymlinks = parent.isTraverseSymlinks(); + relativePath = getRelativePath(parent.getAbsolutePath(), + parent.getRelativePath()); + } else { + relativePath = getRelativePath(rootPath, ""); + relativePathCalculated = true; + } + } + return relativePath; + } + /** + * get thge relative path of this file + * @param currentPath base path + * @param currentRelativePath relative path of the base path with regards to remote dir + * @return relative path + */ + private String getRelativePath(String currentPath, String currentRelativePath) { + Vector pathElements = SelectorUtils.tokenizePath(getAbsolutePath(), remoteFileSep); + Vector pathElements2 = SelectorUtils.tokenizePath(currentPath, remoteFileSep); + String relPath = currentRelativePath; + for (int pcount = pathElements2.size(); pcount < pathElements.size(); pcount++) { + String currentElement = (String) pathElements.elementAt(pcount); + FTPFile[] theFiles = listFiles(currentPath); + FTPFile theFile = null; + if (theFiles != null) { + theFile = getFile(theFiles, currentElement); + } + if (theFile == null) { + throw new BuildException("could not find " + currentElement + + " from " + currentPath); + } else { + traversesSymlinks = traversesSymlinks || theFile.isSymbolicLink(); + if (!relPath.equals("")) { + relPath = relPath + remoteFileSep; + } + relPath = relPath + theFile.getName(); + currentPath = currentPath + remoteFileSep + theFile.getName(); + } + } + return relPath; + } + /** + * find a file matching a string in an array of FTPFile. + * This method will find "alpha" when requested for "ALPHA" + * if and only if the caseSensitive attribute is set to false. + * When caseSensitive is set to true, only the exact match is returned. + * @param theFiles array of files + * @param lastpathelement the file name being sought + * @return null if the file cannot be found, otherwise return the matching file. + */ + public FTPFile getFile(FTPFile[] theFiles, String lastpathelement) { + if (theFiles == null) { + return null; + } + for (int fcount = 0; fcount < theFiles.length; fcount++) { + if (theFiles[fcount].getName().equals(lastpathelement)) { + return theFiles[fcount]; + } else if (!isCaseSensitive() + && theFiles[fcount].getName().equalsIgnoreCase(lastpathelement)) { + return theFiles[fcount]; + } + } + return null; + } + /** + * tell if a file is a directory. + * note that it will return false for symbolic links pointing to directories. + * @return true for directories + */ + public boolean isDirectory() { + return ftpFile.isDirectory(); + } + /** + * tell if a file is a symbolic link + * @return true for symbolic links + */ + public boolean isSymbolicLink() { + return ftpFile.isSymbolicLink(); + } + /** + * return the attached FTP client object. + * Warning : this instance is really shared with the enclosing class. + * @return FTP client + */ + protected FTPClient getClient() { + return client; + } + + /** + * sets the current path of an AntFTPFile + * @param curpwd the current path one wants to set + */ + protected void setCurpwd(String curpwd) { + this.curpwd = curpwd; + } + /** + * returns the path of the directory containing the AntFTPFile. + * of the full path of the file itself in case of AntFTPRootFile + * @return parent directory of the AntFTPFile + */ + public String getCurpwd() { + return curpwd; + } + /** + * find out if a symbolic link is encountered in the relative path of this file + * from rootPath. + * @return true if a symbolic link is encountered in the relative path. + * @throws IOException if one of the change directory or directory listing operations + * fails + * @throws BuildException if a path component in the relative path cannot be found. + */ + public boolean isTraverseSymlinks() throws IOException, BuildException { + if (!relativePathCalculated) { + // getRelativePath also finds about symlinks + String relpath = getRelativePath(); + } + return traversesSymlinks; + } + } + /** + * special class to represent the remote directory itself + * @since Ant 1.6 + */ + protected class AntFTPRootFile extends AntFTPFile { + private String remotedir; + /** + * constructor + * @param aclient FTP client + * @param remotedir remote directory + */ + public AntFTPRootFile(FTPClient aclient, String remotedir) { + super(aclient, null, remotedir); + this.remotedir = remotedir; + try { + this.getClient().changeWorkingDirectory(this.remotedir); + this.setCurpwd(this.getClient().printWorkingDirectory()); + } catch (IOException ioe) { + throw new BuildException(ioe, getLocation()); + } + } + /** + * find the absolute path + * @return absolute path + */ + public String getAbsolutePath() { + return this.getCurpwd(); + } + /** + * find out the relative path to root + * @return empty string + * @throws BuildException actually never + * @throws IOException actually never + */ + public String getRelativePath() throws BuildException, IOException { + return ""; + } + } } /** * check FTPFiles to check whether they function as directories too diff --git a/src/testcases/org/apache/tools/ant/taskdefs/optional/net/FTPTest.java b/src/testcases/org/apache/tools/ant/taskdefs/optional/net/FTPTest.java index 8d4f78aa2..85a18d149 100644 --- a/src/testcases/org/apache/tools/ant/taskdefs/optional/net/FTPTest.java +++ b/src/testcases/org/apache/tools/ant/taskdefs/optional/net/FTPTest.java @@ -89,6 +89,7 @@ public class FTPTest extends BuildFileTest{ ftp = new FTPClient(); ftpFileSep = getProject().getProperty("ftp.filesep"); myFTPTask.setSeparator(ftpFileSep); + myFTPTask.setProject(getProject()); remoteTmpDir = myFTPTask.resolveFile(tmpDir); String remoteHost = getProject().getProperty("ftp.host"); int port = Integer.parseInt(getProject().getProperty("ftp.port"));