antliscence/src/main/org/apache/tools/ant/taskdefs/Get.java

/*
 *  Licensed to the Apache Software Foundation (ASF) under one or more
 *  contributor license agreements.  See the NOTICE file distributed with
 *  this work for additional information regarding copyright ownership.
 *  The ASF licenses this file to You under the Apache License, Version 2.0
 *  (the "License"); you may not use this file except in compliance with
 *  the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 *
 */

package org.apache.tools.ant.taskdefs;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.Date;
import java.util.zip.GZIPInputStream;

import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.MagicNames;
import org.apache.tools.ant.Main;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.Task;
import org.apache.tools.ant.types.Mapper;
import org.apache.tools.ant.types.Resource;
import org.apache.tools.ant.types.ResourceCollection;
import org.apache.tools.ant.types.resources.Resources;
import org.apache.tools.ant.types.resources.URLProvider;
import org.apache.tools.ant.types.resources.URLResource;
import org.apache.tools.ant.util.FileNameMapper;
import org.apache.tools.ant.util.FileUtils;

/**
 * Gets a particular file from a URL source.
 * Options include verbose reporting, timestamp based fetches and controlling
 * actions on failures. NB: access through a firewall only works if the whole
 * Java runtime is correctly configured.
 *
 * @since Ant 1.1
 *
 * @ant.task category="network"
 */
public class Get extends Task {
    private static final int NUMBER_RETRIES = 3;
    private static final int DOTS_PER_LINE = 50;
    private static final int BIG_BUFFER_SIZE = 100 * 1024;
    private static final FileUtils FILE_UTILS = FileUtils.getFileUtils();
    private static final int REDIRECT_LIMIT = 25;
    // HttpURLConnection doesn't have a constant for this in Java5 and
    // what it calls HTTP_MOVED_TEMP would better be FOUND
    private static final int HTTP_MOVED_TEMP = 307;

    private static final String HTTP = "http";
    private static final String HTTPS = "https";

    private static final String DEFAULT_AGENT_PREFIX = "Apache Ant";
    private static final String GZIP_CONTENT_ENCODING = "gzip";

    private final Resources sources = new Resources();
    private File destination; // required
    private boolean verbose = false;
    private boolean quiet = false;
    private boolean useTimestamp = false; //off by default
    private boolean ignoreErrors = false;
    private String uname = null;
    private String pword = null;
    private long maxTime = 0;
    private int numberRetries = NUMBER_RETRIES;
    private boolean skipExisting = false;
    private boolean httpUseCaches = true; // on by default
    private boolean tryGzipEncoding = false;
    private Mapper mapperElement = null;
    private String userAgent =
        System.getProperty(MagicNames.HTTP_AGENT_PROPERTY,
                           DEFAULT_AGENT_PREFIX + "/"
                           + Main.getShortAntVersion());

    /**
     * Does the work.
     *
     * @exception BuildException Thrown in unrecoverable error.
     */
    @Override
    public void execute() throws BuildException {
        checkAttributes();

        for (final Resource r : sources) {
            final URLProvider up = r.as(URLProvider.class);
            final URL source = up.getURL();

            File dest = destination;
            if (destination.isDirectory()) {
                if (mapperElement == null) {
                    String path = source.getPath();
                    if (path.endsWith("/")) {
                        path = path.substring(0, path.length() - 1);
                    }
                    final int slash = path.lastIndexOf("/");
                    if (slash > -1) {
                        path = path.substring(slash + 1);
                    }
                    dest = new File(destination, path);
                } else {
                    final FileNameMapper mapper = mapperElement.getImplementation();
                    final String[] d = mapper.mapFileName(source.toString());
                    if (d == null) {
                        log("skipping " + r + " - mapper can't handle it",
                            Project.MSG_WARN);
                        continue;
                    } else if (d.length == 0) {
                        log("skipping " + r + " - mapper returns no file name",
                            Project.MSG_WARN);
                        continue;
                    } else if (d.length > 1) {
                        log("skipping " + r + " - mapper returns multiple file"
                            + " names", Project.MSG_WARN);
                        continue;
                    }
                    dest = new File(destination, d[0]);
                }
            }

            //set up logging
            final int logLevel = Project.MSG_INFO;
            DownloadProgress progress = null;
            if (verbose) {
                progress = new VerboseProgress(System.out);
            }

            //execute the get
            try {
                doGet(source, dest, logLevel, progress);
            } catch (final IOException ioe) {
                log("Error getting " + source + " to " + dest);
                if (!ignoreErrors) {
                    throw new BuildException(ioe, getLocation());
                }
            }
        }
    }

    /**
     * make a get request, with the supplied progress and logging info.
     * All the other config parameters are set at the task level,
     * source, dest, ignoreErrors, etc.
     * @param logLevel level to log at, see {@link Project#log(String, int)}
     * @param progress progress callback; null for no-callbacks
     * @return true for a successful download, false otherwise.
     * The return value is only relevant when {@link #ignoreErrors} is true, as
     * when false all failures raise BuildExceptions.
     * @throws IOException for network trouble
     * @throws BuildException for argument errors, or other trouble when ignoreErrors
     * is false.
     * @deprecated only gets the first configured resource
     */
    @Deprecated
    public boolean doGet(final int logLevel, final DownloadProgress progress)
            throws IOException {
        checkAttributes();
        for (final Resource r : sources) {
            final URLProvider up = r.as(URLProvider.class);
            final URL source = up.getURL();
            return doGet(source, destination, logLevel, progress);
        }
        /*NOTREACHED*/
        return false;
    }

    /**
     * make a get request, with the supplied progress and logging info.
     *
     * All the other config parameters like ignoreErrors are set at
     * the task level.
     * @param source the URL to get
     * @param dest the target file
     * @param logLevel level to log at, see {@link Project#log(String, int)}
     * @param progress progress callback; null for no-callbacks
     * @return true for a successful download, false otherwise.
     * The return value is only relevant when {@link #ignoreErrors} is true, as
     * when false all failures raise BuildExceptions.
     * @throws IOException for network trouble
     * @throws BuildException for argument errors, or other trouble when ignoreErrors
     * is false.
     * @since Ant 1.8.0
     */
    public boolean doGet(final URL source, final File dest, final int logLevel,
                         DownloadProgress progress)
        throws IOException {

        if (dest.exists() && skipExisting) {
            log("Destination already exists (skipping): "
                + dest.getAbsolutePath(), logLevel);
            return true;
        }

        //dont do any progress, unless asked
        if (progress == null) {
            progress = new NullProgress();
        }
        log("Getting: " + source, logLevel);
        log("To: " + dest.getAbsolutePath(), logLevel);

        //set the timestamp to the file date.
        long timestamp = 0;

        boolean hasTimestamp = false;
        if (useTimestamp && dest.exists()) {
            timestamp = dest.lastModified();
            if (verbose) {
                final Date t = new Date(timestamp);
                log("local file date : " + t.toString(), logLevel);
            }
            hasTimestamp = true;
        }

        final GetThread getThread = new GetThread(source, dest,
                                            hasTimestamp, timestamp, progress,
                                            logLevel, userAgent);
        getThread.setDaemon(true);
        getProject().registerThreadTask(getThread, this);
        getThread.start();
        try {
            getThread.join(maxTime * 1000);
        } catch (final InterruptedException ie) {
            log("interrupted waiting for GET to finish",
                Project.MSG_VERBOSE);
        }

        if (getThread.isAlive()) {
            final String msg = "The GET operation took longer than " + maxTime
                + " seconds, stopping it.";
            if (ignoreErrors) {
                log(msg);
            }
            getThread.closeStreams();
            if (!ignoreErrors) {
                throw new BuildException(msg);
            }
            return false;
        }

        return getThread.wasSuccessful();
    }

    @Override
    public void log(final String msg, final int msgLevel) {
        if (!quiet || msgLevel >= Project.MSG_ERR) {
            super.log(msg, msgLevel);
        }
    }

    /**
     * Check the attributes.
     */
    private void checkAttributes() {

        if (userAgent == null || userAgent.trim().length() == 0) {
            throw new BuildException("userAgent may not be null or empty");
        }

        if (sources.size() == 0) {
            throw new BuildException("at least one source is required",
                                     getLocation());
        }
        for (final Resource r : sources) {
            final URLProvider up = r.as(URLProvider.class);
            if (up == null) {
                throw new BuildException("Only URLProvider resources are"
                                         + " supported", getLocation());
            }
        }

        if (destination == null) {
            throw new BuildException("dest attribute is required", getLocation());
        }

        if (destination.exists() && sources.size() > 1
            && !destination.isDirectory()) {
            throw new BuildException("The specified destination is not a"
                                     + " directory",
                                     getLocation());
        }

        if (destination.exists() && !destination.canWrite()) {
            throw new BuildException("Can't write to "
                                     + destination.getAbsolutePath(),
                                     getLocation());
        }

        if (sources.size() > 1 && !destination.exists()) {
            destination.mkdirs();
        }
    }

    /**
     * Set an URL to get.
     *
     * @param u URL for the file.
     */
    public void setSrc(final URL u) {
        add(new URLResource(u));
    }

    /**
     * Adds URLs to get.
     * @since Ant 1.8.0
     */
    public void add(final ResourceCollection rc) {
        sources.add(rc);
    }

    /**
     * Where to copy the source file.
     *
     * @param dest Path to file.
     */
    public void setDest(final File dest) {
        this.destination = dest;
    }

    /**
     * If true, show verbose progress information.
     *
     * @param v if "true" then be verbose
     */
    public void setVerbose(final boolean v) {
        verbose = v;
    }

    /**
     * If true, set default log level to Project.MSG_ERR.
     *
     * @param v if "true" then be quiet
     * @since Ant 1.9.4
     */
    public void setQuiet(final boolean v){
        this.quiet = v;
    }

    /**
     * If true, log errors but do not treat as fatal.
     *
     * @param v if "true" then don't report download errors up to ant
     */
    public void setIgnoreErrors(final boolean v) {
        ignoreErrors = v;
    }

    /**
     * If true, conditionally download a file based on the timestamp
     * of the local copy.
     *
     * <p>In this situation, the if-modified-since header is set so
     * that the file is only fetched if it is newer than the local
     * file (or there is no local file) This flag is only valid on
     * HTTP connections, it is ignored in other cases.  When the flag
     * is set, the local copy of the downloaded file will also have
     * its timestamp set to the remote file time.</p>
     *
     * <p>Note that remote files of date 1/1/1970 (GMT) are treated as
     * 'no timestamp', and web servers often serve files with a
     * timestamp in the future by replacing their timestamp with that
     * of the current time. Also, inter-computer clock differences can
     * cause no end of grief.</p>
     * @param v "true" to enable file time fetching
     */
    public void setUseTimestamp(final boolean v) {
        useTimestamp = v;
    }


    /**
     * Username for basic auth.
     *
     * @param u username for authentication
     */
    public void setUsername(final String u) {
        this.uname = u;
    }

    /**
     * password for the basic authentication.
     *
     * @param p password for authentication
     */
    public void setPassword(final String p) {
        this.pword = p;
    }

    /**
     * The time in seconds the download is allowed to take before
     * being terminated.
     *
     * @since Ant 1.8.0
     */
    public void setMaxTime(final long maxTime) {
        this.maxTime = maxTime;
    }

    /**
     * The number of retries to attempt upon error, defaults to 3.
     *
     * @param r retry count
     *
     * @since Ant 1.8.0
     */
    public void setRetries(final int r) {
        this.numberRetries = r;
    }

    /**
     * Skip files that already exist locally.
     *
     * @param s "true" to skip existing destination files
     *
     * @since Ant 1.8.0
     */
    public void setSkipExisting(final boolean s) {
        this.skipExisting = s;
    }

    /**
     * HTTP connections only - set the user-agent to be used
     * when communicating with remote server. if null, then
     * the value is considered unset and the behaviour falls
     * back to the default of the http API.
     *
     * @since Ant 1.9.3
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * HTTP connections only - control caching on the
     * HttpUrlConnection: httpConnection.setUseCaches(); if false, do
     * not allow caching on the HttpUrlConnection.
     *
     * <p>Defaults to true (allow caching, which is also the
     * HttpUrlConnection default value.</p>
     *
     * @since Ant 1.8.0
     */
    public void setHttpUseCaches(final boolean httpUseCache) {
        this.httpUseCaches = httpUseCache;
    }

    /**
     * Whether to transparently try to reduce bandwidth by telling the
     * server ant would support gzip encoding.
     *
     * <p>Setting this to true also means Ant will uncompress
     * <code>.tar.gz</code> and similar files automatically.</p>
     *
     * @since Ant 1.9.5
     */
    public void setTryGzipEncoding(boolean b) {
        tryGzipEncoding = b;
    }

    /**
     * Define the mapper to map source to destination files.
     * @return a mapper to be configured.
     * @exception BuildException if more than one mapper is defined.
     * @since Ant 1.8.0
     */
    public Mapper createMapper() throws BuildException {
        if (mapperElement != null) {
            throw new BuildException("Cannot define more than one mapper",
                                     getLocation());
        }
        mapperElement = new Mapper(getProject());
        return mapperElement;
    }

    /**
     * Add a nested filenamemapper.
     * @param fileNameMapper the mapper to add.
     * @since Ant 1.8.0
     */
    public void add(final FileNameMapper fileNameMapper) {
        createMapper().add(fileNameMapper);
    }

    /**
     * Provide this for Backward Compatibility.
     */
    protected static class Base64Converter
        extends org.apache.tools.ant.util.Base64Converter {
    }

    /**
     * Interface implemented for reporting
     * progress of downloading.
     */
    public interface DownloadProgress {
        /**
         * begin a download
         */
        void beginDownload();

        /**
         * tick handler
         *
         */
        void onTick();

        /**
         * end a download
         */
        void endDownload();
    }

    /**
     * do nothing with progress info
     */
    public static class NullProgress implements DownloadProgress {

        /**
         * begin a download
         */
        public void beginDownload() {
        }

        /**
         * tick handler
         *
         */
        public void onTick() {
        }

        /**
         * end a download
         */
        public void endDownload() {
        }
    }

    /**
     * verbose progress system prints to some output stream
     */
    public static class VerboseProgress implements DownloadProgress  {
        private int dots = 0;
        // CheckStyle:VisibilityModifier OFF - bc
        PrintStream out;
        // CheckStyle:VisibilityModifier ON

        /**
         * Construct a verbose progress reporter.
         * @param out the output stream.
         */
        public VerboseProgress(final PrintStream out) {
            this.out = out;
        }

        /**
         * begin a download
         */
        public void beginDownload() {
            dots = 0;
        }

        /**
         * tick handler
         *
         */
        public void onTick() {
            out.print(".");
            if (dots++ > DOTS_PER_LINE) {
                out.flush();
                dots = 0;
            }
        }

        /**
         * end a download
         */
        public void endDownload() {
            out.println();
            out.flush();
        }
    }

    private class GetThread extends Thread {

        private final URL source;
        private final File dest;
        private final boolean hasTimestamp;
        private final long timestamp;
        private final DownloadProgress progress;
        private final int logLevel;

        private boolean success = false;
        private IOException ioexception = null;
        private BuildException exception = null;
        private InputStream is = null;
        private OutputStream os = null;
        private URLConnection connection;
        private int redirections = 0;
        private String userAgent = null;

        GetThread(final URL source, final File dest,
                  final boolean h, final long t, final DownloadProgress p, final int l, final String userAgent) {
            this.source = source;
            this.dest = dest;
            hasTimestamp = h;
            timestamp = t;
            progress = p;
            logLevel = l;
            this.userAgent = userAgent;
        }

        @Override
        public void run() {
            try {
                success = get();
            } catch (final IOException ioex) {
                ioexception = ioex;
            } catch (final BuildException bex) {
                exception = bex;
            }
        }

        private boolean get() throws IOException, BuildException {

            connection = openConnection(source);

            if (connection == null) {
                return false;
            }

            final boolean downloadSucceeded = downloadFile();

            //if (and only if) the use file time option is set, then
            //the saved file now has its timestamp set to that of the
            //downloaded file
            if (downloadSucceeded && useTimestamp)  {
                updateTimeStamp();
            }

            return downloadSucceeded;
        }


        private boolean redirectionAllowed(final URL aSource, final URL aDest) {
            if (!(aSource.getProtocol().equals(aDest.getProtocol()) || (HTTP
                    .equals(aSource.getProtocol()) && HTTPS.equals(aDest
                    .getProtocol())))) {
                final String message = "Redirection detected from "
                        + aSource.getProtocol() + " to " + aDest.getProtocol()
                        + ". Protocol switch unsafe, not allowed.";
                if (ignoreErrors) {
                    log(message, logLevel);
                    return false;
                } else {
                    throw new BuildException(message);
                }
            }

            redirections++;
            if (redirections > REDIRECT_LIMIT) {
                final String message = "More than " + REDIRECT_LIMIT
                        + " times redirected, giving up";
                if (ignoreErrors) {
                    log(message, logLevel);
                    return false;
                } else {
                    throw new BuildException(message);
                }
            }


            return true;
        }

        private URLConnection openConnection(final URL aSource) throws IOException {

            // set up the URL connection
            final URLConnection connection = aSource.openConnection();
            // modify the headers
            // NB: things like user authentication could go in here too.
            if (hasTimestamp) {
                connection.setIfModifiedSince(timestamp);
            }
            // Set the user agent
            connection.addRequestProperty("User-Agent", this.userAgent);

            // prepare Java 1.1 style credentials
            if (uname != null || pword != null) {
                final String up = uname + ":" + pword;
                String encoding;
                // we do not use the sun impl for portability,
                // and always use our own implementation for consistent
                // testing
                final Base64Converter encoder = new Base64Converter();
                encoding = encoder.encode(up.getBytes());
                connection.setRequestProperty("Authorization", "Basic "
                        + encoding);
            }

            if (tryGzipEncoding) {
                connection.setRequestProperty("Accept-Encoding", GZIP_CONTENT_ENCODING);
            }

            if (connection instanceof HttpURLConnection) {
                ((HttpURLConnection) connection)
                        .setInstanceFollowRedirects(false);
                ((HttpURLConnection) connection)
                        .setUseCaches(httpUseCaches);
            }
            // connect to the remote site (may take some time)
            try {
                connection.connect();
            } catch (final NullPointerException e) {
                //bad URLs can trigger NPEs in some JVMs
                throw new BuildException("Failed to parse " + source.toString(), e);
            }

            // First check on a 301 / 302 (moved) response (HTTP only)
            if (connection instanceof HttpURLConnection) {
                final HttpURLConnection httpConnection = (HttpURLConnection) connection;
                final int responseCode = httpConnection.getResponseCode();
                if (isMoved(responseCode)) {
                    final String newLocation = httpConnection.getHeaderField("Location");
                    final String message = aSource
                            + (responseCode == HttpURLConnection.HTTP_MOVED_PERM ? " permanently"
                                    : "") + " moved to " + newLocation;
                    log(message, logLevel);
                    final URL newURL = new URL(aSource, newLocation);
                    if (!redirectionAllowed(aSource, newURL)) {
                        return null;
                    }
                    return openConnection(newURL);
                }
                // next test for a 304 result (HTTP only)
                final long lastModified = httpConnection.getLastModified();
                if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED
                        || (lastModified != 0 && hasTimestamp && timestamp >= lastModified)) {
                    // not modified so no file download. just return
                    // instead and trace out something so the user
                    // doesn't think that the download happened when it
                    // didn't
                    log("Not modified - so not downloaded", logLevel);
                    return null;
                }
                // test for 401 result (HTTP only)
                if (responseCode == HttpURLConnection.HTTP_UNAUTHORIZED) {
                    final String message = "HTTP Authorization failure";
                    if (ignoreErrors) {
                        log(message, logLevel);
                        return null;
                    } else {
                        throw new BuildException(message);
                    }
                }
            }

            //REVISIT: at this point even non HTTP connections may
            //support the if-modified-since behaviour -we just check
            //the date of the content and skip the write if it is not
            //newer. Some protocols (FTP) don't include dates, of
            //course.
            return connection;
        }

        private boolean isMoved(final int responseCode) {
            return responseCode == HttpURLConnection.HTTP_MOVED_PERM ||
                responseCode == HttpURLConnection.HTTP_MOVED_TEMP ||
                responseCode == HttpURLConnection.HTTP_SEE_OTHER ||
                responseCode == HTTP_MOVED_TEMP;
        }

        private boolean downloadFile()
                throws FileNotFoundException, IOException {
            for (int i = 0; i < numberRetries; i++) {
                // this three attempt trick is to get round quirks in different
                // Java implementations. Some of them take a few goes to bind
                // properly; we ignore the first couple of such failures.
                try {
                    is = connection.getInputStream();
                    break;
                } catch (final IOException ex) {
                    log("Error opening connection " + ex, logLevel);
                }
            }
            if (is == null) {
                log("Can't get " + source + " to " + dest, logLevel);
                if (ignoreErrors) {
                    return false;
                }
                throw new BuildException("Can't get " + source + " to " + dest,
                        getLocation());
            }

            if (tryGzipEncoding
                && GZIP_CONTENT_ENCODING.equals(connection.getContentEncoding())) {
                is = new GZIPInputStream(is);
            }

            os = new FileOutputStream(dest);
            progress.beginDownload();
            boolean finished = false;
            try {
                final byte[] buffer = new byte[BIG_BUFFER_SIZE];
                int length;
                while (!isInterrupted() && (length = is.read(buffer)) >= 0) {
                    os.write(buffer, 0, length);
                    progress.onTick();
                }
                finished = !isInterrupted();
            } finally {
                FileUtils.close(os);
                FileUtils.close(is);

                // we have started to (over)write dest, but failed.
                // Try to delete the garbage we'd otherwise leave
                // behind.
                if (!finished) {
                    dest.delete();
                }
            }
            progress.endDownload();
            return true;
        }

        private void updateTimeStamp() {
            final long remoteTimestamp = connection.getLastModified();
            if (verbose)  {
                final Date t = new Date(remoteTimestamp);
                log("last modified = " + t.toString()
                    + ((remoteTimestamp == 0)
                       ? " - using current time instead"
                       : ""), logLevel);
            }
            if (remoteTimestamp != 0) {
                FILE_UTILS.setFileLastModified(dest, remoteTimestamp);
            }
        }

        /**
         * Has the download completed successfully?
         *
         * <p>Re-throws any exception caught during executaion.</p>
         */
        boolean wasSuccessful() throws IOException, BuildException {
            if (ioexception != null) {
                throw ioexception;
            }
            if (exception != null) {
                throw exception;
            }
            return success;
        }

        /**
         * Closes streams, interrupts the download, may delete the
         * output file.
         */
        void closeStreams() {
            interrupt();
            FileUtils.close(os);
            FileUtils.close(is);
            if (!success && dest.exists()) {
                dest.delete();
            }
        }
    }
}