From d0cb590844449882c124f78aa671ef8750c75ea9 Mon Sep 17 00:00:00 2001 From: "Steven M. Cohen" Date: Mon, 30 May 2005 00:40:21 +0000 Subject: [PATCH] Based on a patch submitted by Neeme Praks, allow support for a retry count on FTP transfers. Some servers are unreliable for unknown - this allows for a retry count to be specified to accomodate work on such flaky servers. git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@278374 13f79535-47bb-0310-9956-ffa450edef68 --- docs/manual/OptionalTasks/ftp.html | 8 ++ .../testcases/taskdefs/optional/net/ftp.xml | 13 +++ .../tools/ant/taskdefs/optional/net/FTP.java | 105 ++++++++++++++---- .../apache/tools/ant/util/RetryHandler.java | 72 ++++++++++++ .../org/apache/tools/ant/util/Retryable.java | 32 ++++++ .../ant/taskdefs/optional/net/FTPTest.java | 86 ++++++++++++++ 6 files changed, 293 insertions(+), 23 deletions(-) create mode 100644 src/main/org/apache/tools/ant/util/RetryHandler.java create mode 100644 src/main/org/apache/tools/ant/util/Retryable.java diff --git a/docs/manual/OptionalTasks/ftp.html b/docs/manual/OptionalTasks/ftp.html index 918bdcbe3..93e45a5fd 100644 --- a/docs/manual/OptionalTasks/ftp.html +++ b/docs/manual/OptionalTasks/ftp.html @@ -192,6 +192,14 @@ coming from your ftp server (ls -l on the ftp prompt). (Note: Ignored on Java 1.1) No; defaults to false. + + retriesAllowed + Set the number of retries allowed on an file-transfer operation. + If a number > 0 specified, each file transfer can fail up to that + many times before the operation is failed. If -1 or "forever" specified, the + operation will keep trying until it succeeds. + No; defaults to 0 + diff --git a/src/etc/testcases/taskdefs/optional/net/ftp.xml b/src/etc/testcases/taskdefs/optional/net/ftp.xml index c66dfc4cf..e67947848 100644 --- a/src/etc/testcases/taskdefs/optional/net/ftp.xml +++ b/src/etc/testcases/taskdefs/optional/net/ftp.xml @@ -15,6 +15,7 @@ + @@ -272,5 +273,17 @@ + + + + + \ No newline at end of file diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/net/FTP.java b/src/main/org/apache/tools/ant/taskdefs/optional/net/FTP.java index 1081576a8..5640f7795 100644 --- a/src/main/org/apache/tools/ant/taskdefs/optional/net/FTP.java +++ b/src/main/org/apache/tools/ant/taskdefs/optional/net/FTP.java @@ -51,6 +51,8 @@ import org.apache.tools.ant.types.EnumeratedAttribute; import org.apache.tools.ant.types.FileSet; import org.apache.tools.ant.types.selectors.SelectorUtils; import org.apache.tools.ant.util.FileUtils; +import org.apache.tools.ant.util.RetryHandler; +import org.apache.tools.ant.util.Retryable; /** * Basic FTP client. Performs the following actions: @@ -126,6 +128,7 @@ public class FTP private String shortMonthNamesConfig = null; private Granularity timestampGranularity = Granularity.getDefault(); private boolean isConfigurationSet = false; + private int retriesAllowed = 0; protected static final String[] ACTION_STRS = { "sending", @@ -1360,6 +1363,37 @@ public class FTP } + + /** + * How many times to retry executing FTP command before giving up? + * Default is 0 - try once and if failure then give up. + * + * @param retriesAllowed number of retries to allow. -1 means + * keep trying forever. "forever" may also be specified as a + * synonym for -1. + */ + public void setRetriesAllowed(String retriesAllowed) { + if ("FOREVER".equalsIgnoreCase(retriesAllowed)) { + this.retriesAllowed = Retryable.RETRY_FOREVER; + } else { + try { + int retries = Integer.parseInt(retriesAllowed); + if (retries < Retryable.RETRY_FOREVER) { + throw new BuildException( + "Invalid value for retriesAllowed attribute: " + + retriesAllowed); + + } + this.retriesAllowed = retries; + } catch (NumberFormatException px) { + throw new BuildException( + "Invalid value for retriesAllowed attribute: " + + retriesAllowed); + + } + + } + } /** * @return Returns the systemTypeKey. */ @@ -1451,6 +1485,12 @@ public class FTP } } } + + protected void executeRetryable(RetryHandler h, Retryable r, String filename) + throws IOException + { + h.execute(r, filename); + } /** @@ -1465,7 +1505,7 @@ public class FTP * @throws IOException if there is a problem reading a file * @throws BuildException if there is a problem in the configuration. */ - protected int transferFiles(FTPClient ftp, FileSet fs) + protected int transferFiles(final FTPClient ftp, FileSet fs) throws IOException, BuildException { DirectoryScanner ds; if (action == SEND_FILES) { @@ -1512,38 +1552,51 @@ public class FTP } bw = new BufferedWriter(new FileWriter(listing)); } + RetryHandler h = new RetryHandler(this.retriesAllowed, this); if (action == RM_DIR) { // to remove directories, start by the end of the list // the trunk does not let itself be removed before the leaves for (int i = dsfiles.length - 1; i >= 0; i--) { - rmDir(ftp, dsfiles[i]); + final String dsfile = dsfiles[i]; + executeRetryable(h, new Retryable() { + public void execute() throws IOException { + rmDir(ftp, dsfile); + } + }, dsfile); } } else { + final BufferedWriter fbw = bw; + final String fdir = dir; if (this.newerOnly) { this.granularityMillis = this.timestampGranularity.getMilliseconds(action); } for (int i = 0; i < dsfiles.length; i++) { - switch (action) { - case SEND_FILES: - sendFile(ftp, dir, dsfiles[i]); - break; - case GET_FILES: - getFile(ftp, dir, dsfiles[i]); - break; - case DEL_FILES: - delFile(ftp, dsfiles[i]); - break; - case LIST_FILES: - listFile(ftp, bw, dsfiles[i]); - break; - case CHMOD: - doSiteCommand(ftp, "chmod " + chmod + " " + resolveFile(dsfiles[i])); - transferred++; - break; - default: - throw new BuildException("unknown ftp action " + action); - } + final String dsfile = dsfiles[i]; + executeRetryable(h, new Retryable() { + public void execute() throws IOException { + switch (action) { + case SEND_FILES: + sendFile(ftp, fdir, dsfile); + break; + case GET_FILES: + getFile(ftp, fdir, dsfile); + break; + case DEL_FILES: + delFile(ftp, dsfile); + break; + case LIST_FILES: + listFile(ftp, fbw, dsfile); + break; + case CHMOD: + doSiteCommand(ftp, "chmod " + chmod + " " + resolveFile(dsfile)); + transferred++; + break; + default: + throw new BuildException("unknown ftp action " + action); + } + } + }, dsfile); } } } finally { @@ -2198,7 +2251,13 @@ public class FTP // directory is the directory to create. if (action == MK_DIR) { - makeRemoteDir(ftp, remotedir); + RetryHandler h = new RetryHandler(this.retriesAllowed, this); + final FTPClient lftp = ftp; + executeRetryable(h, new Retryable() { + public void execute() throws IOException { + makeRemoteDir(lftp, remotedir); + } + }, remotedir); } else { if (remotedir != null) { log("changing the remote directory", Project.MSG_VERBOSE); diff --git a/src/main/org/apache/tools/ant/util/RetryHandler.java b/src/main/org/apache/tools/ant/util/RetryHandler.java new file mode 100644 index 000000000..6fc280869 --- /dev/null +++ b/src/main/org/apache/tools/ant/util/RetryHandler.java @@ -0,0 +1,72 @@ +/* + * Copyright 2005 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.tools.ant.util; + +import java.io.IOException; + +import org.apache.tools.ant.Project; +import org.apache.tools.ant.Task; + +/** + * A simple utility class to take a piece of code (that implements + * Retryable interface) and executes that with possibility to + * retry the execution in case of IOException. + */ +public class RetryHandler { + + private int retriesAllowed = 0; + private Task task; + + /** + * Create a new RetryingHandler. + * + * @param retriesAllowed how many times to retry + * @param task the Ant task that is is executed from, used for logging only + */ + public RetryHandler(int retriesAllowed, Task task) { + this.retriesAllowed = retriesAllowed; + this.task = task; + } + + /** + * Execute the Retryable code with specified number of retries. + * + * @param exe the code to execute + * @param desc some descriptive text for this piece of code, used for logging + * @throws IOException if the number of retries has exceeded the allowed limit + */ + public void execute(Retryable exe, String desc) throws IOException { + int retries = 0; + while (true) { + try { + exe.execute(); + break; + } catch (IOException e) { + retries++; + if (retries > this.retriesAllowed && this.retriesAllowed > -1) { + task.log("try #" + retries + ": IO error (" + + desc + "), number of maximum retries reached (" + + this.retriesAllowed + "), giving up", Project.MSG_WARN); + throw e; + } else { + task.log("try #" + retries + ": IO error (" + desc + "), retrying", Project.MSG_WARN); + } + } + } + } + +} diff --git a/src/main/org/apache/tools/ant/util/Retryable.java b/src/main/org/apache/tools/ant/util/Retryable.java new file mode 100644 index 000000000..b4bf024c3 --- /dev/null +++ b/src/main/org/apache/tools/ant/util/Retryable.java @@ -0,0 +1,32 @@ +/* + * Copyright 2005 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.tools.ant.util; + +import java.io.IOException; + + +/** + * Simple interface for executing a piece of code. Used for writing anonymous inner + * classes in FTP task for retry-on-IOException behaviour. + * + * @see RetryHandler + */ +public interface Retryable { + public static final int RETRY_FOREVER = -1; + void execute() throws IOException; + +} \ No newline at end of file diff --git a/src/testcases/org/apache/tools/ant/taskdefs/optional/net/FTPTest.java b/src/testcases/org/apache/tools/ant/taskdefs/optional/net/FTPTest.java index 5bb22bb50..86f1aaf54 100644 --- a/src/testcases/org/apache/tools/ant/taskdefs/optional/net/FTPTest.java +++ b/src/testcases/org/apache/tools/ant/taskdefs/optional/net/FTPTest.java @@ -21,17 +21,21 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Random; import java.util.Vector; import org.apache.commons.net.ftp.FTPClient; import org.apache.tools.ant.BuildEvent; import org.apache.tools.ant.BuildException; import org.apache.tools.ant.BuildFileTest; +import org.apache.tools.ant.ComponentHelper; import org.apache.tools.ant.DefaultLogger; import org.apache.tools.ant.DirectoryScanner; import org.apache.tools.ant.Project; import org.apache.tools.ant.taskdefs.condition.Os; import org.apache.tools.ant.types.FileSet; +import org.apache.tools.ant.util.RetryHandler; +import org.apache.tools.ant.util.Retryable; import org.apache.tools.ant.util.regexp.RegexpMatcher; import org.apache.tools.ant.util.regexp.RegexpMatcherFactory; @@ -779,6 +783,88 @@ public class FTPTest extends BuildFileTest{ public String resolveFile(String file) { return super.resolveFile(file); } + } + + + public abstract static class myRetryableFTP extends FTP { + private final int numberOfFailuresToSimulate; + private int simulatedFailuresLeft; + + protected myRetryableFTP(int numberOfFailuresToSimulate) { + this.numberOfFailuresToSimulate = numberOfFailuresToSimulate; + this.simulatedFailuresLeft = numberOfFailuresToSimulate; + } + protected void getFile(FTPClient ftp, String dir, String filename) + throws IOException, BuildException + { + if (this.simulatedFailuresLeft > 0) { + this.simulatedFailuresLeft--; + throw new IOException("Simulated failure for testing"); + } + super.getFile(ftp, dir, filename); + } + protected void executeRetryable(RetryHandler h, Retryable r, + String filename) throws IOException + { + this.simulatedFailuresLeft = this.numberOfFailuresToSimulate; + super.executeRetryable(h, r, filename); + } } + public static class oneFailureFTP extends myRetryableFTP { + public oneFailureFTP() { + super(1); + } + } + public static class twoFailureFTP extends myRetryableFTP { + public twoFailureFTP() { + super(2); + } + } + public static class threeFailureFTP extends myRetryableFTP { + public threeFailureFTP() { + super(3); + } + } + + public static class randomFailureFTP extends myRetryableFTP { + public randomFailureFTP() { + super(new Random(30000).nextInt()); + } + } + public void testGetWithSelectorRetryable1() { + getProject().addTaskDefinition("ftp", oneFailureFTP.class); + try { + getProject().executeTarget("ftp-get-with-selector-retryable"); + } catch (BuildException bx) { + fail("Two retries expected, failed after one."); + } + } + public void testGetWithSelectorRetryable2() { + getProject().addTaskDefinition("ftp", twoFailureFTP.class); + try { + getProject().executeTarget("ftp-get-with-selector-retryable"); + } catch (BuildException bx) { + fail("Two retries expected, failed after two."); + } + } + + public void testGetWithSelectorRetryable3() { + getProject().addTaskDefinition("ftp", threeFailureFTP.class); + try { + getProject().executeTarget("ftp-get-with-selector-retryable"); + fail("Two retries expected, continued after two."); + } catch (BuildException bx) { + } + } + public void testGetWithSelectorRetryableRandom() { + getProject().addTaskDefinition("ftp", threeFailureFTP.class); + try { + getProject().setProperty("ftp.retries", "forever"); + getProject().executeTarget("ftp-get-with-selector-retryable"); + } catch (BuildException bx) { + fail("Retry forever specified, but failed."); + } + } + }