Browse Source

port Kristian Rosenvold's write performance improvements from Commons Compress

master
Stefan Bodewig 10 years ago
parent
commit
2c04d7e833
9 changed files with 432 additions and 222 deletions
  1. +1
    -0
      CONTRIBUTORS
  2. +3
    -0
      WHATSNEW
  3. +4
    -0
      contributors.xml
  4. +22
    -9
      src/main/org/apache/tools/zip/GeneralPurposeBit.java
  5. +87
    -26
      src/main/org/apache/tools/zip/ZipEntry.java
  6. +20
    -4
      src/main/org/apache/tools/zip/ZipLong.java
  7. +258
    -178
      src/main/org/apache/tools/zip/ZipOutputStream.java
  8. +14
    -2
      src/main/org/apache/tools/zip/ZipShort.java
  9. +23
    -3
      src/main/org/apache/tools/zip/ZipUtil.java

+ 1
- 0
CONTRIBUTORS View File

@@ -206,6 +206,7 @@ Kevin Ross
Kevin Z Grey Kevin Z Grey
Kim Hansen Kim Hansen
Kirk Wylie Kirk Wylie
Kristian Rosenvold
Kyle Adams Kyle Adams
Lajos Veres Lajos Veres
Larry Shatzer Larry Shatzer


+ 3
- 0
WHATSNEW View File

@@ -75,6 +75,9 @@ Other changes:
variable. variable.
Bugzilla Report 57371 Bugzilla Report 57371


* ported some of the write-optimization of Commons Compress 1.10 to
the ZIP package

Changes from Ant 1.9.3 TO Ant 1.9.4 Changes from Ant 1.9.3 TO Ant 1.9.4
=================================== ===================================




+ 4
- 0
contributors.xml View File

@@ -851,6 +851,10 @@
<first>Kirk</first> <first>Kirk</first>
<last>Wylie</last> <last>Wylie</last>
</name> </name>
<name>
<first>Kristian</first>
<last>Rosenvold</last>
</name>
<name> <name>
<first>Kyle</first> <first>Kyle</first>
<last>Adams</last> <last>Adams</last>


+ 22
- 9
src/main/org/apache/tools/zip/GeneralPurposeBit.java View File

@@ -122,15 +122,28 @@ public final class GeneralPurposeBit implements Cloneable {
* Encodes the set bits in a form suitable for ZIP archives. * Encodes the set bits in a form suitable for ZIP archives.
*/ */
public byte[] encode() { public byte[] encode() {
return
ZipShort.getBytes((dataDescriptorFlag ? DATA_DESCRIPTOR_FLAG : 0)
|
(languageEncodingFlag ? UFT8_NAMES_FLAG : 0)
|
(encryptionFlag ? ENCRYPTION_FLAG : 0)
|
(strongEncryptionFlag ? STRONG_ENCRYPTION_FLAG : 0)
);
byte[] result = new byte[2];
encode(result, 0);
return result;
}

/**
* Encodes the set bits in a form suitable for ZIP archives.
*
* @param buf the output buffer
* @param offset
* The offset within the output buffer of the first byte to be written.
* must be non-negative and no larger than <tt>buf.length-2</tt>
*/
public void encode(byte[] buf, int offset) {
ZipShort.putShort((dataDescriptorFlag ? DATA_DESCRIPTOR_FLAG : 0)
|
(languageEncodingFlag ? UFT8_NAMES_FLAG : 0)
|
(encryptionFlag ? ENCRYPTION_FLAG : 0)
|
(strongEncryptionFlag ? STRONG_ENCRYPTION_FLAG : 0)
, buf, offset);
} }


/** /**


+ 87
- 26
src/main/org/apache/tools/zip/ZipEntry.java View File

@@ -22,7 +22,6 @@ import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.zip.ZipException; import java.util.zip.ZipException;


@@ -51,6 +50,7 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {


public static final int PLATFORM_UNIX = 3; public static final int PLATFORM_UNIX = 3;
public static final int PLATFORM_FAT = 0; public static final int PLATFORM_FAT = 0;
public static final int CRC_UNKNOWN = -1;
private static final int SHORT_MASK = 0xFFFF; private static final int SHORT_MASK = 0xFFFF;
private static final int SHORT_SHIFT = 16; private static final int SHORT_SHIFT = 16;
private static final byte[] EMPTY = new byte[0]; private static final byte[] EMPTY = new byte[0];
@@ -75,11 +75,12 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
private int internalAttributes = 0; private int internalAttributes = 0;
private int platform = PLATFORM_FAT; private int platform = PLATFORM_FAT;
private long externalAttributes = 0; private long externalAttributes = 0;
private LinkedHashMap<ZipShort, ZipExtraField> extraFields = null;
private ZipExtraField[] extraFields;
private UnparseableExtraFieldData unparseableExtra = null; private UnparseableExtraFieldData unparseableExtra = null;
private String name = null; private String name = null;
private byte[] rawName = null; private byte[] rawName = null;
private GeneralPurposeBit gpb = new GeneralPurposeBit(); private GeneralPurposeBit gpb = new GeneralPurposeBit();
private static final ZipExtraField[] noExtraFields = new ZipExtraField[0];


/** /**
* Creates a new zip entry with the specified name. * Creates a new zip entry with the specified name.
@@ -135,7 +136,7 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
this((java.util.zip.ZipEntry) entry); this((java.util.zip.ZipEntry) entry);
setInternalAttributes(entry.getInternalAttributes()); setInternalAttributes(entry.getInternalAttributes());
setExternalAttributes(entry.getExternalAttributes()); setExternalAttributes(entry.getExternalAttributes());
setExtraFields(entry.getExtraFields(true));
setExtraFields(getAllExtraFieldsNoCopy());
setPlatform(entry.getPlatform()); setPlatform(entry.getPlatform());
GeneralPurposeBit other = entry.getGeneralPurposeBit(); GeneralPurposeBit other = entry.getGeneralPurposeBit();
setGeneralPurposeBit(other == null ? null : setGeneralPurposeBit(other == null ? null :
@@ -179,7 +180,7 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {


e.setInternalAttributes(getInternalAttributes()); e.setInternalAttributes(getInternalAttributes());
e.setExternalAttributes(getExternalAttributes()); e.setExternalAttributes(getExternalAttributes());
e.setExtraFields(getExtraFields(true));
e.setExtraFields(getAllExtraFieldsNoCopy());
return e; return e;
} }


@@ -300,14 +301,15 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
* @since 1.1 * @since 1.1
*/ */
public void setExtraFields(final ZipExtraField[] fields) { public void setExtraFields(final ZipExtraField[] fields) {
extraFields = new LinkedHashMap<ZipShort, ZipExtraField>();
for (final ZipExtraField field : fields) {
List<ZipExtraField> newFields = new ArrayList<ZipExtraField>();
for (ZipExtraField field : fields) {
if (field instanceof UnparseableExtraFieldData) { if (field instanceof UnparseableExtraFieldData) {
unparseableExtra = (UnparseableExtraFieldData) field; unparseableExtra = (UnparseableExtraFieldData) field;
} else { } else {
extraFields.put(field.getHeaderId(), field);
newFields.add( field);
} }
} }
extraFields = newFields.toArray(new ZipExtraField[newFields.size()]);
setExtra(); setExtra();
} }


@@ -316,7 +318,7 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
* @return an array of the extra fields * @return an array of the extra fields
*/ */
public ZipExtraField[] getExtraFields() { public ZipExtraField[] getExtraFields() {
return getExtraFields(false);
return getParseableExtraFields();
} }


/** /**
@@ -328,17 +330,55 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
* @since 1.1 * @since 1.1
*/ */
public ZipExtraField[] getExtraFields(final boolean includeUnparseable) { public ZipExtraField[] getExtraFields(final boolean includeUnparseable) {
return includeUnparseable ?
getAllExtraFields() :
getParseableExtraFields();
}

private ZipExtraField[] getParseableExtraFieldsNoCopy() {
if (extraFields == null) { if (extraFields == null) {
return !includeUnparseable || unparseableExtra == null
? new ZipExtraField[0]
: new ZipExtraField[] {unparseableExtra};
return noExtraFields;
} }
final List<ZipExtraField> result =
new ArrayList<ZipExtraField>(extraFields.values());
if (includeUnparseable && unparseableExtra != null) {
result.add(unparseableExtra);
return extraFields;
}

private ZipExtraField[] getParseableExtraFields() {
final ZipExtraField[] parseableExtraFields = getParseableExtraFieldsNoCopy();
return (parseableExtraFields == extraFields)
? copyOf(parseableExtraFields) : parseableExtraFields;
}

private ZipExtraField[] copyOf(ZipExtraField[] src){
return Arrays.copyOf(src, src.length);
}

private ZipExtraField[] getMergedFields() {
final ZipExtraField[] zipExtraFields =
Arrays.copyOf(extraFields, extraFields.length + 1);
zipExtraFields[zipExtraFields.length] = unparseableExtra;
return zipExtraFields;
}

private ZipExtraField[] getUnparseableOnly() {
return unparseableExtra == null
? noExtraFields : new ZipExtraField[] { unparseableExtra };
}

private ZipExtraField[] getAllExtraFields() {
final ZipExtraField[] allExtraFieldsNoCopy = getAllExtraFieldsNoCopy();
return (allExtraFieldsNoCopy == extraFields)
? copyOf( allExtraFieldsNoCopy) : allExtraFieldsNoCopy;
}

/**
* Get all extra fields, including unparseable ones.
* @return An array of all extra fields. Not necessarily a copy of internal data structures, hence private method
*/
private ZipExtraField[] getAllExtraFieldsNoCopy() {
if (extraFields == null) {
return getUnparseableOnly();
} }
return result.toArray(new ZipExtraField[0]);
return unparseableExtra != null ? getMergedFields() : extraFields;
} }


/** /**
@@ -355,9 +395,16 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
unparseableExtra = (UnparseableExtraFieldData) ze; unparseableExtra = (UnparseableExtraFieldData) ze;
} else { } else {
if (extraFields == null) { if (extraFields == null) {
extraFields = new LinkedHashMap<ZipShort, ZipExtraField>();
extraFields = new ZipExtraField[] {ze};
} else {
if (getExtraField(ze.getHeaderId()) != null){
removeExtraField(ze.getHeaderId());
}
final ZipExtraField[] zipExtraFields =
Arrays.copyOf(extraFields, extraFields.length + 1);
zipExtraFields[extraFields.length] = ze;
extraFields = zipExtraFields;
} }
extraFields.put(ze.getHeaderId(), ze);
} }
setExtra(); setExtra();
} }
@@ -374,12 +421,15 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
if (ze instanceof UnparseableExtraFieldData) { if (ze instanceof UnparseableExtraFieldData) {
unparseableExtra = (UnparseableExtraFieldData) ze; unparseableExtra = (UnparseableExtraFieldData) ze;
} else { } else {
final LinkedHashMap<ZipShort, ZipExtraField> copy = extraFields;
extraFields = new LinkedHashMap<ZipShort, ZipExtraField>();
extraFields.put(ze.getHeaderId(), ze);
if (copy != null) {
copy.remove(ze.getHeaderId());
extraFields.putAll(copy);
if (getExtraField(ze.getHeaderId()) != null){
removeExtraField(ze.getHeaderId());
}
ZipExtraField[] copy = extraFields;
int newLen = extraFields != null ? extraFields.length + 1: 1;
extraFields = new ZipExtraField[newLen];
extraFields[0] = ze;
if (copy != null){
System.arraycopy(copy, 0, extraFields, 1, extraFields.length - 1);
} }
} }
setExtra(); setExtra();
@@ -394,9 +444,16 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
if (extraFields == null) { if (extraFields == null) {
throw new java.util.NoSuchElementException(); throw new java.util.NoSuchElementException();
} }
if (extraFields.remove(type) == null) {
List<ZipExtraField> newResult = new ArrayList<ZipExtraField>();
for (ZipExtraField extraField : extraFields) {
if (!type.equals(extraField.getHeaderId())){
newResult.add(extraField);
}
}
if (extraFields.length == newResult.size()) {
throw new java.util.NoSuchElementException(); throw new java.util.NoSuchElementException();
} }
extraFields = newResult.toArray(new ZipExtraField[newResult.size()]);
setExtra(); setExtra();
} }


@@ -418,7 +475,11 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
*/ */
public ZipExtraField getExtraField(final ZipShort type) { public ZipExtraField getExtraField(final ZipShort type) {
if (extraFields != null) { if (extraFields != null) {
return extraFields.get(type);
for (ZipExtraField extraField : extraFields) {
if (type.equals(extraField.getHeaderId())) {
return extraField;
}
}
} }
return null; return null;
} }


+ 20
- 4
src/main/org/apache/tools/zip/ZipLong.java View File

@@ -114,13 +114,29 @@ public final class ZipLong implements Cloneable {
*/ */
public static byte[] getBytes(long value) { public static byte[] getBytes(long value) {
byte[] result = new byte[WORD]; byte[] result = new byte[WORD];
result[0] = (byte) ((value & BYTE_MASK));
result[BYTE_1] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT);
result[BYTE_2] = (byte) ((value & BYTE_2_MASK) >> BYTE_2_SHIFT);
result[BYTE_3] = (byte) ((value & BYTE_3_MASK) >> BYTE_3_SHIFT);
putLong(value, result, 0);
return result; return result;
} }


/**
* put the value as four bytes in big endian byte order.
* @param value the Java long to convert to bytes
* @param buf the output buffer
* @param offset
* The offset within the output buffer of the first byte to be written.
* must be non-negative and no larger than <tt>buf.length-4</tt>
*/
public static void putLong(long value, byte[] buf, int offset) {
buf[offset++] = (byte) ((value & BYTE_MASK));
buf[offset++] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT);
buf[offset++] = (byte) ((value & BYTE_2_MASK) >> BYTE_2_SHIFT);
buf[offset] = (byte) ((value & BYTE_3_MASK) >> BYTE_3_SHIFT);
}

public void putLong(byte[] buf, int offset) {
putLong(value, buf, offset);
}

/** /**
* Helper method to get the value as a Java long from four bytes starting at given array offset * Helper method to get the value as a Java long from four bytes starting at given array offset
* @param bytes the array of bytes * @param bytes the array of bytes


+ 258
- 178
src/main/org/apache/tools/zip/ZipOutputStream.java View File

@@ -26,7 +26,10 @@ import static org.apache.tools.zip.ZipConstants.WORD;
import static org.apache.tools.zip.ZipConstants.ZIP64_MAGIC; import static org.apache.tools.zip.ZipConstants.ZIP64_MAGIC;
import static org.apache.tools.zip.ZipConstants.ZIP64_MAGIC_SHORT; import static org.apache.tools.zip.ZipConstants.ZIP64_MAGIC_SHORT;
import static org.apache.tools.zip.ZipConstants.ZIP64_MIN_VERSION; import static org.apache.tools.zip.ZipConstants.ZIP64_MIN_VERSION;
import static org.apache.tools.zip.ZipLong.putLong;
import static org.apache.tools.zip.ZipShort.putShort;


import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.FilterOutputStream; import java.io.FilterOutputStream;
@@ -34,6 +37,7 @@ import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Calendar;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
@@ -72,6 +76,34 @@ import java.util.zip.ZipException;
public class ZipOutputStream extends FilterOutputStream { public class ZipOutputStream extends FilterOutputStream {


private static final int BUFFER_SIZE = 512; private static final int BUFFER_SIZE = 512;
private static final int LFH_SIG_OFFSET = 0;
private static final int LFH_VERSION_NEEDED_OFFSET = 4;
private static final int LFH_GPB_OFFSET = 6;
private static final int LFH_METHOD_OFFSET = 8;
private static final int LFH_TIME_OFFSET = 10;
private static final int LFH_CRC_OFFSET = 14;
private static final int LFH_COMPRESSED_SIZE_OFFSET = 18;
private static final int LFH_ORIGINAL_SIZE_OFFSET = 22;
private static final int LFH_FILENAME_LENGTH_OFFSET = 26;
private static final int LFH_EXTRA_LENGTH_OFFSET = 28;
private static final int LFH_FILENAME_OFFSET = 30;
private static final int CFH_SIG_OFFSET = 0;
private static final int CFH_VERSION_MADE_BY_OFFSET = 4;
private static final int CFH_VERSION_NEEDED_OFFSET = 6;
private static final int CFH_GPB_OFFSET = 8;
private static final int CFH_METHOD_OFFSET = 10;
private static final int CFH_TIME_OFFSET = 12;
private static final int CFH_CRC_OFFSET = 16;
private static final int CFH_COMPRESSED_SIZE_OFFSET = 20;
private static final int CFH_ORIGINAL_SIZE_OFFSET = 24;
private static final int CFH_FILENAME_LENGTH_OFFSET = 28;
private static final int CFH_EXTRA_LENGTH_OFFSET = 30;
private static final int CFH_COMMENT_LENGTH_OFFSET = 32;
private static final int CFH_DISK_NUMBER_OFFSET = 34;
private static final int CFH_INTERNAL_ATTRIBUTES_OFFSET = 36;
private static final int CFH_EXTERNAL_ATTRIBUTES_OFFSET = 38;
private static final int CFH_LFH_OFFSET = 42;
private static final int CFH_FILENAME_OFFSET = 46;


/** /**
* indicates if this archive is finished. * indicates if this archive is finished.
@@ -208,6 +240,8 @@ public class ZipOutputStream extends FilterOutputStream {
*/ */
private static final byte[] LZERO = {0, 0, 0, 0}; private static final byte[] LZERO = {0, 0, 0, 0};


private static final byte[] ONE = ZipLong.getBytes(1L);

/** /**
* Holds the offsets of the LFH starts for each entry. * Holds the offsets of the LFH starts for each entry.
* *
@@ -287,6 +321,8 @@ public class ZipOutputStream extends FilterOutputStream {


private Zip64Mode zip64Mode = Zip64Mode.AsNeeded; private Zip64Mode zip64Mode = Zip64Mode.AsNeeded;


private final Calendar calendarInstance = Calendar.getInstance();

/** /**
* Creates a new ZIP OutputStream filtering the underlying stream. * Creates a new ZIP OutputStream filtering the underlying stream.
* @param out the outputstream to zip * @param out the outputstream to zip
@@ -459,9 +495,7 @@ public class ZipOutputStream extends FilterOutputStream {
} }


cdOffset = written; cdOffset = written;
for (ZipEntry ze : entries) {
writeCentralFileHeader(ze);
}
writeCentralDirectoryInChunks();
cdLength = written - cdOffset; cdLength = written - cdOffset;
writeZip64CentralDirectory(); writeZip64CentralDirectory();
writeCentralDirectoryEnd(); writeCentralDirectoryEnd();
@@ -471,6 +505,21 @@ public class ZipOutputStream extends FilterOutputStream {
finished = true; finished = true;
} }


private void writeCentralDirectoryInChunks() throws IOException {
final int NUM_PER_WRITE = 1000;
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(70 * NUM_PER_WRITE);
int count = 0;
for (ZipEntry ze : entries) {
byteArrayOutputStream.write(createCentralFileHeader(ze));
if (++count > NUM_PER_WRITE){
writeCounted(byteArrayOutputStream.toByteArray());
byteArrayOutputStream.reset();
count = 0;
}
}
writeCounted(byteArrayOutputStream.toByteArray());
}

/** /**
* Writes all necessary data for this entry. * Writes all necessary data for this entry.
* *
@@ -481,17 +530,7 @@ public class ZipOutputStream extends FilterOutputStream {
* is {@link Zip64Mode#Never}. * is {@link Zip64Mode#Never}.
*/ */
public void closeEntry() throws IOException { public void closeEntry() throws IOException {
if (finished) {
throw new IOException("Stream has already been finished");
}

if (entry == null) {
throw new IOException("No current entry to close");
}

if (!entry.hasWritten) {
write(EMPTY, 0, 0);
}
preClose();


flushDeflater(); flushDeflater();


@@ -503,6 +542,10 @@ public class ZipOutputStream extends FilterOutputStream {
final boolean actuallyNeedsZip64 = final boolean actuallyNeedsZip64 =
handleSizesAndCrc(bytesWritten, realCrc, effectiveMode); handleSizesAndCrc(bytesWritten, realCrc, effectiveMode);


closeEntry(actuallyNeedsZip64);
}

private void closeEntry(boolean actuallyNeedsZip64) throws IOException {
if (raf != null) { if (raf != null) {
rewriteSizesAndCrc(actuallyNeedsZip64); rewriteSizesAndCrc(actuallyNeedsZip64);
} }
@@ -511,6 +554,20 @@ public class ZipOutputStream extends FilterOutputStream {
entry = null; entry = null;
} }


private void preClose() throws IOException {
if (finished) {
throw new IOException("Stream has already been finished");
}

if (entry == null) {
throw new IOException("No current entry to close");
}

if (!entry.hasWritten) {
write(EMPTY, 0, 0);
}
}

/** /**
* Ensures all bytes sent to the deflater are written to the stream. * Ensures all bytes sent to the deflater are written to the stream.
*/ */
@@ -564,9 +621,19 @@ public class ZipOutputStream extends FilterOutputStream {
entry.entry.setCrc(crc); entry.entry.setCrc(crc);
} }


final boolean actuallyNeedsZip64 = effectiveMode == Zip64Mode.Always
|| entry.entry.getSize() >= ZIP64_MAGIC
|| entry.entry.getCompressedSize() >= ZIP64_MAGIC;
return checkIfNeedsZip64(effectiveMode);
}

/**
* Ensures the current entry's size and CRC information is set to
* the values just written, verifies it isn't too big in the
* Zip64Mode.Never case and returns whether the entry would
* require a Zip64 extra field.
*/
private boolean checkIfNeedsZip64(Zip64Mode effectiveMode)
throws ZipException {
final boolean actuallyNeedsZip64 = isZip64Required(entry.entry,
effectiveMode);
if (actuallyNeedsZip64 && effectiveMode == Zip64Mode.Never) { if (actuallyNeedsZip64 && effectiveMode == Zip64Mode.Never) {
throw new Zip64RequiredException(Zip64RequiredException throw new Zip64RequiredException(Zip64RequiredException
.getEntryTooBigMessage(entry.entry)); .getEntryTooBigMessage(entry.entry));
@@ -574,6 +641,15 @@ public class ZipOutputStream extends FilterOutputStream {
return actuallyNeedsZip64; return actuallyNeedsZip64;
} }


private boolean isZip64Required(ZipEntry entry1, Zip64Mode requestedMode) {
return requestedMode == Zip64Mode.Always || isTooLageForZip32(entry1);
}

private boolean isTooLageForZip32(ZipEntry zipArchiveEntry){
return zipArchiveEntry.getSize() >= ZIP64_MAGIC
|| zipArchiveEntry.getCompressedSize() >= ZIP64_MAGIC;
}

/** /**
* When using random access output, write the local file header * When using random access output, write the local file header
* and potentiall the ZIP64 extra containing the correct CRC and * and potentiall the ZIP64 extra containing the correct CRC and
@@ -654,13 +730,15 @@ public class ZipOutputStream extends FilterOutputStream {
// just a placeholder, real data will be in data // just a placeholder, real data will be in data
// descriptor or inserted later via RandomAccessFile // descriptor or inserted later via RandomAccessFile
ZipEightByteInteger size = ZipEightByteInteger.ZERO; ZipEightByteInteger size = ZipEightByteInteger.ZERO;
ZipEightByteInteger compressedSize = ZipEightByteInteger.ZERO;
if (entry.entry.getMethod() == STORED if (entry.entry.getMethod() == STORED
&& entry.entry.getSize() != -1) { && entry.entry.getSize() != -1) {
// actually, we already know the sizes // actually, we already know the sizes
size = new ZipEightByteInteger(entry.entry.getSize()); size = new ZipEightByteInteger(entry.entry.getSize());
compressedSize = size;
} }
z64.setSize(size); z64.setSize(size);
z64.setCompressedSize(size);
z64.setCompressedSize(compressedSize);
entry.entry.setExtra(); entry.entry.setExtra();
} }


@@ -794,17 +872,33 @@ public class ZipOutputStream extends FilterOutputStream {
*/ */
@Override @Override
public void write(byte[] b, int offset, int length) throws IOException { public void write(byte[] b, int offset, int length) throws IOException {
if (entry == null) {
throw new IllegalStateException("No current entry");
}
ZipUtil.checkRequestedFeatures(entry.entry); ZipUtil.checkRequestedFeatures(entry.entry);
entry.hasWritten = true; entry.hasWritten = true;
if (entry.entry.getMethod() == DEFLATED) { if (entry.entry.getMethod() == DEFLATED) {
writeDeflated(b, offset, length); writeDeflated(b, offset, length);
} else { } else {
writeOut(b, offset, length);
written += length;
writeCounted(b, offset, length);
} }
crc.update(b, offset, length); crc.update(b, offset, length);
} }


/**
* Write bytes to output or random access file.
* @param data the byte array to write
* @throws IOException on error
*/
private void writeCounted(byte[] data) throws IOException {
writeCounted(data, 0, data.length);
}

private void writeCounted(byte[] data, int offset, int length) throws IOException {
writeOut(data, offset, length);
written += length;
}

/** /**
* write implementation for DEFLATED entries. * write implementation for DEFLATED entries.
*/ */
@@ -906,8 +1000,7 @@ public class ZipOutputStream extends FilterOutputStream {
protected final void deflate() throws IOException { protected final void deflate() throws IOException {
int len = def.deflate(buf, 0, buf.length); int len = def.deflate(buf, 0, buf.length);
if (len > 0) { if (len > 0) {
writeOut(buf, 0, len);
written += len;
writeCounted(buf, 0, len);
} }
} }


@@ -927,76 +1020,71 @@ public class ZipOutputStream extends FilterOutputStream {
addUnicodeExtraFields(ze, encodable, name); addUnicodeExtraFields(ze, encodable, name);
} }


offsets.put(ze, Long.valueOf(written));
final byte[] localHeader = createLocalFileHeader(ze, name, encodable);
final long localHeaderStart = written;
offsets.put(ze, localHeaderStart);
entry.localDataStart = localHeaderStart + LFH_CRC_OFFSET; // At crc offset
writeCounted(localHeader);
entry.dataStart = written;
}

private byte[] createLocalFileHeader(ZipEntry ze, ByteBuffer name, boolean encodable) {
byte[] extra = ze.getLocalFileDataExtra();
final int nameLen = name.limit() - name.position();
int len= LFH_FILENAME_OFFSET + nameLen + extra.length;
byte[] buf = new byte[len];


writeOut(LFH_SIG);
written += WORD;
System.arraycopy(LFH_SIG, 0, buf, LFH_SIG_OFFSET, WORD);


//store method in local variable to prevent multiple method calls //store method in local variable to prevent multiple method calls
final int zipMethod = ze.getMethod(); final int zipMethod = ze.getMethod();


writeVersionNeededToExtractAndGeneralPurposeBits(zipMethod,
!encodable
&& fallbackToUTF8,
hasZip64Extra(ze));
written += WORD;
putShort(versionNeededToExtract(zipMethod, hasZip64Extra(ze)),
buf, LFH_VERSION_NEEDED_OFFSET);

GeneralPurposeBit generalPurposeBit =
getGeneralPurposeBits(zipMethod, !encodable && fallbackToUTF8);
generalPurposeBit.encode(buf, LFH_GPB_OFFSET);


// compression method // compression method
writeOut(ZipShort.getBytes(zipMethod));
written += SHORT;
putShort(zipMethod, buf, LFH_METHOD_OFFSET);


// last mod. time and date
writeOut(ZipUtil.toDosTime(ze.getTime()));
written += WORD;
ZipUtil.toDosTime(calendarInstance, ze.getTime(), buf, LFH_TIME_OFFSET);


// CRC // CRC
// compressed length
// uncompressed length
entry.localDataStart = written;
if (zipMethod == DEFLATED || raf != null) { if (zipMethod == DEFLATED || raf != null) {
writeOut(LZERO);
if (hasZip64Extra(entry.entry)) {
// point to ZIP64 extended information extra field for
// sizes, may get rewritten once sizes are known if
// stream is seekable
writeOut(ZipLong.ZIP64_MAGIC.getBytes());
writeOut(ZipLong.ZIP64_MAGIC.getBytes());
} else {
writeOut(LZERO);
writeOut(LZERO);
}
System.arraycopy(LZERO, 0, buf, LFH_CRC_OFFSET, WORD);
} else { } else {
writeOut(ZipLong.getBytes(ze.getCrc()));
byte[] size = ZipLong.ZIP64_MAGIC.getBytes();
if (!hasZip64Extra(ze)) {
size = ZipLong.getBytes(ze.getSize());
}
writeOut(size);
writeOut(size);
putLong(ze.getCrc(), buf, LFH_CRC_OFFSET);
} }
// CheckStyle:MagicNumber OFF
written += 12;
// CheckStyle:MagicNumber ON


// compressed length
// uncompressed length
if (hasZip64Extra(entry.entry)){
// point to ZIP64 extended information extra field for
// sizes, may get rewritten once sizes are known if
// stream is seekable
ZipLong.ZIP64_MAGIC.putLong(buf, LFH_COMPRESSED_SIZE_OFFSET);
ZipLong.ZIP64_MAGIC.putLong(buf, LFH_ORIGINAL_SIZE_OFFSET);
} else if (zipMethod == DEFLATED || raf != null) {
System.arraycopy(LZERO, 0, buf, LFH_COMPRESSED_SIZE_OFFSET, WORD);
System.arraycopy(LZERO, 0, buf, LFH_ORIGINAL_SIZE_OFFSET, WORD);
} else { // Stored
putLong(ze.getSize(), buf, LFH_COMPRESSED_SIZE_OFFSET);
putLong(ze.getSize(), buf, LFH_ORIGINAL_SIZE_OFFSET);
}
// file name length // file name length
writeOut(ZipShort.getBytes(name.limit()));
written += SHORT;
putShort(nameLen, buf, LFH_FILENAME_LENGTH_OFFSET);


// extra field length // extra field length
byte[] extra = ze.getLocalFileDataExtra();
writeOut(ZipShort.getBytes(extra.length));
written += SHORT;
putShort(extra.length, buf, LFH_EXTRA_LENGTH_OFFSET);


// file name // file name
writeOut(name.array(), name.arrayOffset(),
name.limit() - name.position());
written += name.limit();
System.arraycopy(name.array(), name.arrayOffset(), buf,
LFH_FILENAME_OFFSET, nameLen);


// extra field
writeOut(extra);
written += extra.length;

entry.dataStart = written;
System.arraycopy(extra, 0, buf, LFH_FILENAME_OFFSET + nameLen, extra.length);
return buf;
} }


/** /**
@@ -1045,18 +1133,15 @@ public class ZipOutputStream extends FilterOutputStream {
if (ze.getMethod() != DEFLATED || raf != null) { if (ze.getMethod() != DEFLATED || raf != null) {
return; return;
} }
writeOut(DD_SIG);
writeOut(ZipLong.getBytes(ze.getCrc()));
int sizeFieldSize = WORD;
writeCounted(DD_SIG);
writeCounted(ZipLong.getBytes(ze.getCrc()));
if (!hasZip64Extra(ze)) { if (!hasZip64Extra(ze)) {
writeOut(ZipLong.getBytes(ze.getCompressedSize()));
writeOut(ZipLong.getBytes(ze.getSize()));
writeCounted(ZipLong.getBytes(ze.getCompressedSize()));
writeCounted(ZipLong.getBytes(ze.getSize()));
} else { } else {
sizeFieldSize = DWORD;
writeOut(ZipEightByteInteger.getBytes(ze.getCompressedSize()));
writeOut(ZipEightByteInteger.getBytes(ze.getSize()));
writeCounted(ZipEightByteInteger.getBytes(ze.getCompressedSize()));
writeCounted(ZipEightByteInteger.getBytes(ze.getSize()));
} }
written += 2 * WORD + 2 * sizeFieldSize;
} }


/** /**
@@ -1068,114 +1153,116 @@ public class ZipOutputStream extends FilterOutputStream {
* Zip64Mode#Never}. * Zip64Mode#Never}.
*/ */
protected void writeCentralFileHeader(ZipEntry ze) throws IOException { protected void writeCentralFileHeader(ZipEntry ze) throws IOException {
writeOut(CFH_SIG);
written += WORD;
byte[] centralFileHeader = createCentralFileHeader(ze);
writeCounted(centralFileHeader);
}


final long lfhOffset = offsets.get(ze).longValue();
private byte[] createCentralFileHeader(ZipEntry ze) throws IOException {
final long lfhOffset = offsets.get(ze);
final boolean needsZip64Extra = hasZip64Extra(ze) final boolean needsZip64Extra = hasZip64Extra(ze)
|| ze.getCompressedSize() >= ZIP64_MAGIC
|| ze.getSize() >= ZIP64_MAGIC
|| lfhOffset >= ZIP64_MAGIC;
|| ze.getCompressedSize() >= ZIP64_MAGIC
|| ze.getSize() >= ZIP64_MAGIC
|| lfhOffset >= ZIP64_MAGIC;


if (needsZip64Extra && zip64Mode == Zip64Mode.Never) { if (needsZip64Extra && zip64Mode == Zip64Mode.Never) {
// must be the offset that is too big, otherwise an // must be the offset that is too big, otherwise an
// exception would have been throw in putNextEntry or
// closeEntry
// exception would have been throw in putArchiveEntry or
// closeArchiveEntry
throw new Zip64RequiredException(Zip64RequiredException throw new Zip64RequiredException(Zip64RequiredException
.ARCHIVE_TOO_BIG_MESSAGE);
.ARCHIVE_TOO_BIG_MESSAGE);
} }



handleZip64Extra(ze, lfhOffset, needsZip64Extra); handleZip64Extra(ze, lfhOffset, needsZip64Extra);


return createCentralFileHeader(ze, getName(ze), lfhOffset, needsZip64Extra);
}

/**
* Writes the central file header entry.
* @param ze the entry to write
* @param name The encoded name
* @param lfhOffset Local file header offset for this file
* @throws IOException on error
*/
private byte[] createCentralFileHeader(ZipEntry ze, ByteBuffer name, long lfhOffset,
boolean needsZip64Extra) throws IOException {
byte[] extra = ze.getCentralDirectoryExtra();

// file comment length
String comm = ze.getComment();
if (comm == null) {
comm = "";
}

ByteBuffer commentB = getEntryEncoding(ze).encode(comm);
final int nameLen = name.limit() - name.position();
final int commentLen = commentB.limit() - commentB.position();
int len= CFH_FILENAME_OFFSET + nameLen + extra.length + commentLen;
byte[] buf = new byte[len];

System.arraycopy(CFH_SIG, 0, buf, CFH_SIG_OFFSET, WORD);

// version made by // version made by
// CheckStyle:MagicNumber OFF // CheckStyle:MagicNumber OFF
writeOut(ZipShort.getBytes((ze.getPlatform() << 8) |
(!hasUsedZip64 ? DATA_DESCRIPTOR_MIN_VERSION
: ZIP64_MIN_VERSION)));
written += SHORT;
putShort((ze.getPlatform() << 8) | (!hasUsedZip64 ? DATA_DESCRIPTOR_MIN_VERSION : ZIP64_MIN_VERSION),
buf, CFH_VERSION_MADE_BY_OFFSET);


final int zipMethod = ze.getMethod(); final int zipMethod = ze.getMethod();
final boolean encodable = zipEncoding.canEncode(ze.getName()); final boolean encodable = zipEncoding.canEncode(ze.getName());
writeVersionNeededToExtractAndGeneralPurposeBits(zipMethod,
!encodable
&& fallbackToUTF8,
needsZip64Extra);
written += WORD;
putShort(versionNeededToExtract(zipMethod, needsZip64Extra), buf, CFH_VERSION_NEEDED_OFFSET);
getGeneralPurposeBits(zipMethod, !encodable && fallbackToUTF8).encode(buf, CFH_GPB_OFFSET);


// compression method // compression method
writeOut(ZipShort.getBytes(zipMethod));
written += SHORT;
putShort(zipMethod, buf, CFH_METHOD_OFFSET);


// last mod. time and date // last mod. time and date
writeOut(ZipUtil.toDosTime(ze.getTime()));
written += WORD;
ZipUtil.toDosTime(calendarInstance, ze.getTime(), buf, CFH_TIME_OFFSET);


// CRC // CRC
// compressed length // compressed length
// uncompressed length // uncompressed length
writeOut(ZipLong.getBytes(ze.getCrc()));
putLong(ze.getCrc(), buf, CFH_CRC_OFFSET);
if (ze.getCompressedSize() >= ZIP64_MAGIC if (ze.getCompressedSize() >= ZIP64_MAGIC
|| ze.getSize() >= ZIP64_MAGIC) {
writeOut(ZipLong.ZIP64_MAGIC.getBytes());
writeOut(ZipLong.ZIP64_MAGIC.getBytes());
|| ze.getSize() >= ZIP64_MAGIC) {
ZipLong.ZIP64_MAGIC.putLong(buf, CFH_COMPRESSED_SIZE_OFFSET);
ZipLong.ZIP64_MAGIC.putLong(buf, CFH_ORIGINAL_SIZE_OFFSET);
} else { } else {
writeOut(ZipLong.getBytes(ze.getCompressedSize()));
writeOut(ZipLong.getBytes(ze.getSize()));
putLong(ze.getCompressedSize(), buf, CFH_COMPRESSED_SIZE_OFFSET);
putLong(ze.getSize(), buf, CFH_ORIGINAL_SIZE_OFFSET);
} }
// CheckStyle:MagicNumber OFF
written += 12;
// CheckStyle:MagicNumber ON

ByteBuffer name = getName(ze);


writeOut(ZipShort.getBytes(name.limit()));
written += SHORT;
putShort(nameLen, buf, CFH_FILENAME_LENGTH_OFFSET);


// extra field length // extra field length
byte[] extra = ze.getCentralDirectoryExtra();
writeOut(ZipShort.getBytes(extra.length));
written += SHORT;
putShort(extra.length, buf, CFH_EXTRA_LENGTH_OFFSET);


// file comment length
String comm = ze.getComment();
if (comm == null) {
comm = "";
}

ByteBuffer commentB = getEntryEncoding(ze).encode(comm);

writeOut(ZipShort.getBytes(commentB.limit()));
written += SHORT;
putShort(commentLen, buf, CFH_COMMENT_LENGTH_OFFSET);


// disk number start // disk number start
writeOut(ZERO);
written += SHORT;
System.arraycopy(ZERO, 0, buf, CFH_DISK_NUMBER_OFFSET, SHORT);


// internal file attributes // internal file attributes
writeOut(ZipShort.getBytes(ze.getInternalAttributes()));
written += SHORT;
putShort(ze.getInternalAttributes(), buf, CFH_INTERNAL_ATTRIBUTES_OFFSET);


// external file attributes // external file attributes
writeOut(ZipLong.getBytes(ze.getExternalAttributes()));
written += WORD;
putLong(ze.getExternalAttributes(), buf, CFH_EXTERNAL_ATTRIBUTES_OFFSET);


// relative offset of LFH // relative offset of LFH
writeOut(ZipLong.getBytes(Math.min(lfhOffset, ZIP64_MAGIC)));
written += WORD;
putLong(Math.min(lfhOffset, ZIP64_MAGIC), buf, CFH_LFH_OFFSET);


// file name // file name
writeOut(name.array(), name.arrayOffset(),
name.limit() - name.position());
written += name.limit();
System.arraycopy(name.array(), name.arrayOffset(), buf, CFH_FILENAME_OFFSET, nameLen);


// extra field
writeOut(extra);
written += extra.length;
int extraStart = CFH_FILENAME_OFFSET + nameLen;
System.arraycopy(extra, 0, buf, extraStart, extra.length);

int commentStart = extraStart + commentLen;


// file comment // file comment
writeOut(commentB.array(), commentB.arrayOffset(),
commentB.limit() - commentB.position());
written += commentB.limit();
System.arraycopy(commentB.array(), commentB.arrayOffset(), buf, commentStart, commentLen);
return buf;
} }


/** /**
@@ -1210,11 +1297,11 @@ public class ZipOutputStream extends FilterOutputStream {
* and {@link Zip64Mode #setUseZip64} is {@link Zip64Mode#Never}. * and {@link Zip64Mode #setUseZip64} is {@link Zip64Mode#Never}.
*/ */
protected void writeCentralDirectoryEnd() throws IOException { protected void writeCentralDirectoryEnd() throws IOException {
writeOut(EOCD_SIG);
writeCounted(EOCD_SIG);


// disk numbers // disk numbers
writeOut(ZERO);
writeOut(ZERO);
writeCounted(ZERO);
writeCounted(ZERO);


// number of entries // number of entries
int numberOfEntries = entries.size(); int numberOfEntries = entries.size();
@@ -1230,18 +1317,18 @@ public class ZipOutputStream extends FilterOutputStream {


byte[] num = ZipShort.getBytes(Math.min(numberOfEntries, byte[] num = ZipShort.getBytes(Math.min(numberOfEntries,
ZIP64_MAGIC_SHORT)); ZIP64_MAGIC_SHORT));
writeOut(num);
writeOut(num);
writeCounted(num);
writeCounted(num);


// length and location of CD // length and location of CD
writeOut(ZipLong.getBytes(Math.min(cdLength, ZIP64_MAGIC)));
writeOut(ZipLong.getBytes(Math.min(cdOffset, ZIP64_MAGIC)));
writeCounted(ZipLong.getBytes(Math.min(cdLength, ZIP64_MAGIC)));
writeCounted(ZipLong.getBytes(Math.min(cdOffset, ZIP64_MAGIC)));


// ZIP file comment // ZIP file comment
ByteBuffer data = this.zipEncoding.encode(comment); ByteBuffer data = this.zipEncoding.encode(comment);
writeOut(ZipShort.getBytes(data.limit()));
writeOut(data.array(), data.arrayOffset(),
data.limit() - data.position());
int dataLen = data.limit() - data.position();
writeCounted(ZipShort.getBytes(dataLen));
writeCounted(data.array(), data.arrayOffset(), dataLen);
} }


/** /**
@@ -1292,8 +1379,6 @@ public class ZipOutputStream extends FilterOutputStream {
} }
} }


private static final byte[] ONE = ZipLong.getBytes(1L);

/** /**
* Writes the &quot;ZIP64 End of central dir record&quot; and * Writes the &quot;ZIP64 End of central dir record&quot; and
* &quot;ZIP64 End of central dir locator&quot;. * &quot;ZIP64 End of central dir locator&quot;.
@@ -1409,33 +1494,28 @@ public class ZipOutputStream extends FilterOutputStream {
} }
} }


private void writeVersionNeededToExtractAndGeneralPurposeBits(final int
zipMethod,
final boolean
utfFallback,
final boolean
zip64)
throws IOException {

// CheckStyle:MagicNumber OFF
int versionNeededToExtract = INITIAL_VERSION;
private GeneralPurposeBit getGeneralPurposeBits(final int zipMethod, final boolean utfFallback) {
GeneralPurposeBit b = new GeneralPurposeBit(); GeneralPurposeBit b = new GeneralPurposeBit();
b.useUTF8ForNames(useUTF8Flag || utfFallback); b.useUTF8ForNames(useUTF8Flag || utfFallback);
if (zipMethod == DEFLATED && raf == null) {
// requires version 2 as we are going to store length info
// in the data descriptor
versionNeededToExtract = DATA_DESCRIPTOR_MIN_VERSION;
if (isDeflatedToOutputStream(zipMethod)) {
b.useDataDescriptor(true); b.useDataDescriptor(true);
} }
return b;
}

private int versionNeededToExtract(final int zipMethod, final boolean zip64) {
if (zip64) { if (zip64) {
versionNeededToExtract = ZIP64_MIN_VERSION;
return ZIP64_MIN_VERSION;
} }
// CheckStyle:MagicNumber ON
// requires version 2 as we are going to store length info
// in the data descriptor
return (isDeflatedToOutputStream(zipMethod)) ?
DATA_DESCRIPTOR_MIN_VERSION :
INITIAL_VERSION;
}


// version needed to extract
writeOut(ZipShort.getBytes(versionNeededToExtract));
// general purpose bit flag
writeOut(b.encode());
private boolean isDeflatedToOutputStream(int zipMethod) {
return zipMethod == DEFLATED && raf == null;
} }


/** /**


+ 14
- 2
src/main/org/apache/tools/zip/ZipShort.java View File

@@ -66,11 +66,23 @@ public final class ZipShort implements Cloneable {
*/ */
public byte[] getBytes() { public byte[] getBytes() {
byte[] result = new byte[2]; byte[] result = new byte[2];
result[0] = (byte) (value & BYTE_MASK);
result[1] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT);
putShort(value, result, 0);
return result; return result;
} }


/**
* put the value as two bytes in big endian byte order.
* @param value the Java int to convert to bytes
* @param buf the output buffer
* @param offset
* The offset within the output buffer of the first byte to be written.
* must be non-negative and no larger than <tt>buf.length-2</tt>
*/
public static void putShort(int value, byte[] buf, int offset) {
buf[offset] = (byte) (value & BYTE_MASK);
buf[offset+1] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT);
}

/** /**
* Get value as Java int. * Get value as Java int.
* @return value as a Java int * @return value as a Java int


+ 23
- 3
src/main/org/apache/tools/zip/ZipUtil.java View File

@@ -49,12 +49,32 @@ public abstract class ZipUtil {
* @return the date as a byte array * @return the date as a byte array
*/ */
public static byte[] toDosTime(long t) { public static byte[] toDosTime(long t) {
Calendar c = Calendar.getInstance();
byte[] result = new byte[4];
toDosTime(t, result, 0);
return result;
}

/**
* Convert a Date object to a DOS date/time field.
*
* <p>Stolen from InfoZip's <code>fileio.c</code></p>
* @param t number of milliseconds since the epoch
* @param buf the output buffer
* @param offset
* The offset within the output buffer of the first byte to be written.
* must be non-negative and no larger than <tt>buf.length-4</tt>
*/
public static void toDosTime(long t, byte[] buf, int offset) {
toDosTime(Calendar.getInstance(), t, buf, offset);
}

static void toDosTime(Calendar c, long t, byte[] buf, int offset) {
c.setTimeInMillis(t); c.setTimeInMillis(t);


int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
if (year < 1980) { if (year < 1980) {
return copy(DOS_TIME_MIN); // stop callers from changing the array
System.arraycopy(DOS_TIME_MIN, 0, buf, offset, DOS_TIME_MIN.length);// stop callers from changing the array
return;
} }
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
long value = ((year - 1980) << 25) long value = ((year - 1980) << 25)
@@ -63,7 +83,7 @@ public abstract class ZipUtil {
| (c.get(Calendar.HOUR_OF_DAY) << 11) | (c.get(Calendar.HOUR_OF_DAY) << 11)
| (c.get(Calendar.MINUTE) << 5) | (c.get(Calendar.MINUTE) << 5)
| (c.get(Calendar.SECOND) >> 1); | (c.get(Calendar.SECOND) >> 1);
return ZipLong.getBytes(value);
ZipLong.putLong(value, buf, offset);
} }


/** /**


Loading…
Cancel
Save