Browse Source

port Kristian Rosenvold's write performance improvements from Commons Compress

master
Stefan Bodewig 10 years ago
parent
commit
2c04d7e833
9 changed files with 432 additions and 222 deletions
  1. +1
    -0
      CONTRIBUTORS
  2. +3
    -0
      WHATSNEW
  3. +4
    -0
      contributors.xml
  4. +22
    -9
      src/main/org/apache/tools/zip/GeneralPurposeBit.java
  5. +87
    -26
      src/main/org/apache/tools/zip/ZipEntry.java
  6. +20
    -4
      src/main/org/apache/tools/zip/ZipLong.java
  7. +258
    -178
      src/main/org/apache/tools/zip/ZipOutputStream.java
  8. +14
    -2
      src/main/org/apache/tools/zip/ZipShort.java
  9. +23
    -3
      src/main/org/apache/tools/zip/ZipUtil.java

+ 1
- 0
CONTRIBUTORS View File

@@ -206,6 +206,7 @@ Kevin Ross
Kevin Z Grey
Kim Hansen
Kirk Wylie
Kristian Rosenvold
Kyle Adams
Lajos Veres
Larry Shatzer


+ 3
- 0
WHATSNEW View File

@@ -75,6 +75,9 @@ Other changes:
variable.
Bugzilla Report 57371

* ported some of the write-optimization of Commons Compress 1.10 to
the ZIP package

Changes from Ant 1.9.3 TO Ant 1.9.4
===================================



+ 4
- 0
contributors.xml View File

@@ -851,6 +851,10 @@
<first>Kirk</first>
<last>Wylie</last>
</name>
<name>
<first>Kristian</first>
<last>Rosenvold</last>
</name>
<name>
<first>Kyle</first>
<last>Adams</last>


+ 22
- 9
src/main/org/apache/tools/zip/GeneralPurposeBit.java View File

@@ -122,15 +122,28 @@ public final class GeneralPurposeBit implements Cloneable {
* Encodes the set bits in a form suitable for ZIP archives.
*/
public byte[] encode() {
return
ZipShort.getBytes((dataDescriptorFlag ? DATA_DESCRIPTOR_FLAG : 0)
|
(languageEncodingFlag ? UFT8_NAMES_FLAG : 0)
|
(encryptionFlag ? ENCRYPTION_FLAG : 0)
|
(strongEncryptionFlag ? STRONG_ENCRYPTION_FLAG : 0)
);
byte[] result = new byte[2];
encode(result, 0);
return result;
}

/**
* Encodes the set bits in a form suitable for ZIP archives.
*
* @param buf the output buffer
* @param offset
* The offset within the output buffer of the first byte to be written.
* must be non-negative and no larger than <tt>buf.length-2</tt>
*/
public void encode(byte[] buf, int offset) {
ZipShort.putShort((dataDescriptorFlag ? DATA_DESCRIPTOR_FLAG : 0)
|
(languageEncodingFlag ? UFT8_NAMES_FLAG : 0)
|
(encryptionFlag ? ENCRYPTION_FLAG : 0)
|
(strongEncryptionFlag ? STRONG_ENCRYPTION_FLAG : 0)
, buf, offset);
}

/**


+ 87
- 26
src/main/org/apache/tools/zip/ZipEntry.java View File

@@ -22,7 +22,6 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.zip.ZipException;

@@ -51,6 +50,7 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {

public static final int PLATFORM_UNIX = 3;
public static final int PLATFORM_FAT = 0;
public static final int CRC_UNKNOWN = -1;
private static final int SHORT_MASK = 0xFFFF;
private static final int SHORT_SHIFT = 16;
private static final byte[] EMPTY = new byte[0];
@@ -75,11 +75,12 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
private int internalAttributes = 0;
private int platform = PLATFORM_FAT;
private long externalAttributes = 0;
private LinkedHashMap<ZipShort, ZipExtraField> extraFields = null;
private ZipExtraField[] extraFields;
private UnparseableExtraFieldData unparseableExtra = null;
private String name = null;
private byte[] rawName = null;
private GeneralPurposeBit gpb = new GeneralPurposeBit();
private static final ZipExtraField[] noExtraFields = new ZipExtraField[0];

/**
* Creates a new zip entry with the specified name.
@@ -135,7 +136,7 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
this((java.util.zip.ZipEntry) entry);
setInternalAttributes(entry.getInternalAttributes());
setExternalAttributes(entry.getExternalAttributes());
setExtraFields(entry.getExtraFields(true));
setExtraFields(getAllExtraFieldsNoCopy());
setPlatform(entry.getPlatform());
GeneralPurposeBit other = entry.getGeneralPurposeBit();
setGeneralPurposeBit(other == null ? null :
@@ -179,7 +180,7 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {

e.setInternalAttributes(getInternalAttributes());
e.setExternalAttributes(getExternalAttributes());
e.setExtraFields(getExtraFields(true));
e.setExtraFields(getAllExtraFieldsNoCopy());
return e;
}

@@ -300,14 +301,15 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
* @since 1.1
*/
public void setExtraFields(final ZipExtraField[] fields) {
extraFields = new LinkedHashMap<ZipShort, ZipExtraField>();
for (final ZipExtraField field : fields) {
List<ZipExtraField> newFields = new ArrayList<ZipExtraField>();
for (ZipExtraField field : fields) {
if (field instanceof UnparseableExtraFieldData) {
unparseableExtra = (UnparseableExtraFieldData) field;
} else {
extraFields.put(field.getHeaderId(), field);
newFields.add( field);
}
}
extraFields = newFields.toArray(new ZipExtraField[newFields.size()]);
setExtra();
}

@@ -316,7 +318,7 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
* @return an array of the extra fields
*/
public ZipExtraField[] getExtraFields() {
return getExtraFields(false);
return getParseableExtraFields();
}

/**
@@ -328,17 +330,55 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
* @since 1.1
*/
public ZipExtraField[] getExtraFields(final boolean includeUnparseable) {
return includeUnparseable ?
getAllExtraFields() :
getParseableExtraFields();
}

private ZipExtraField[] getParseableExtraFieldsNoCopy() {
if (extraFields == null) {
return !includeUnparseable || unparseableExtra == null
? new ZipExtraField[0]
: new ZipExtraField[] {unparseableExtra};
return noExtraFields;
}
final List<ZipExtraField> result =
new ArrayList<ZipExtraField>(extraFields.values());
if (includeUnparseable && unparseableExtra != null) {
result.add(unparseableExtra);
return extraFields;
}

private ZipExtraField[] getParseableExtraFields() {
final ZipExtraField[] parseableExtraFields = getParseableExtraFieldsNoCopy();
return (parseableExtraFields == extraFields)
? copyOf(parseableExtraFields) : parseableExtraFields;
}

private ZipExtraField[] copyOf(ZipExtraField[] src){
return Arrays.copyOf(src, src.length);
}

private ZipExtraField[] getMergedFields() {
final ZipExtraField[] zipExtraFields =
Arrays.copyOf(extraFields, extraFields.length + 1);
zipExtraFields[zipExtraFields.length] = unparseableExtra;
return zipExtraFields;
}

private ZipExtraField[] getUnparseableOnly() {
return unparseableExtra == null
? noExtraFields : new ZipExtraField[] { unparseableExtra };
}

private ZipExtraField[] getAllExtraFields() {
final ZipExtraField[] allExtraFieldsNoCopy = getAllExtraFieldsNoCopy();
return (allExtraFieldsNoCopy == extraFields)
? copyOf( allExtraFieldsNoCopy) : allExtraFieldsNoCopy;
}

/**
* Get all extra fields, including unparseable ones.
* @return An array of all extra fields. Not necessarily a copy of internal data structures, hence private method
*/
private ZipExtraField[] getAllExtraFieldsNoCopy() {
if (extraFields == null) {
return getUnparseableOnly();
}
return result.toArray(new ZipExtraField[0]);
return unparseableExtra != null ? getMergedFields() : extraFields;
}

/**
@@ -355,9 +395,16 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
unparseableExtra = (UnparseableExtraFieldData) ze;
} else {
if (extraFields == null) {
extraFields = new LinkedHashMap<ZipShort, ZipExtraField>();
extraFields = new ZipExtraField[] {ze};
} else {
if (getExtraField(ze.getHeaderId()) != null){
removeExtraField(ze.getHeaderId());
}
final ZipExtraField[] zipExtraFields =
Arrays.copyOf(extraFields, extraFields.length + 1);
zipExtraFields[extraFields.length] = ze;
extraFields = zipExtraFields;
}
extraFields.put(ze.getHeaderId(), ze);
}
setExtra();
}
@@ -374,12 +421,15 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
if (ze instanceof UnparseableExtraFieldData) {
unparseableExtra = (UnparseableExtraFieldData) ze;
} else {
final LinkedHashMap<ZipShort, ZipExtraField> copy = extraFields;
extraFields = new LinkedHashMap<ZipShort, ZipExtraField>();
extraFields.put(ze.getHeaderId(), ze);
if (copy != null) {
copy.remove(ze.getHeaderId());
extraFields.putAll(copy);
if (getExtraField(ze.getHeaderId()) != null){
removeExtraField(ze.getHeaderId());
}
ZipExtraField[] copy = extraFields;
int newLen = extraFields != null ? extraFields.length + 1: 1;
extraFields = new ZipExtraField[newLen];
extraFields[0] = ze;
if (copy != null){
System.arraycopy(copy, 0, extraFields, 1, extraFields.length - 1);
}
}
setExtra();
@@ -394,9 +444,16 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
if (extraFields == null) {
throw new java.util.NoSuchElementException();
}
if (extraFields.remove(type) == null) {
List<ZipExtraField> newResult = new ArrayList<ZipExtraField>();
for (ZipExtraField extraField : extraFields) {
if (!type.equals(extraField.getHeaderId())){
newResult.add(extraField);
}
}
if (extraFields.length == newResult.size()) {
throw new java.util.NoSuchElementException();
}
extraFields = newResult.toArray(new ZipExtraField[newResult.size()]);
setExtra();
}

@@ -418,7 +475,11 @@ public class ZipEntry extends java.util.zip.ZipEntry implements Cloneable {
*/
public ZipExtraField getExtraField(final ZipShort type) {
if (extraFields != null) {
return extraFields.get(type);
for (ZipExtraField extraField : extraFields) {
if (type.equals(extraField.getHeaderId())) {
return extraField;
}
}
}
return null;
}


+ 20
- 4
src/main/org/apache/tools/zip/ZipLong.java View File

@@ -114,13 +114,29 @@ public final class ZipLong implements Cloneable {
*/
public static byte[] getBytes(long value) {
byte[] result = new byte[WORD];
result[0] = (byte) ((value & BYTE_MASK));
result[BYTE_1] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT);
result[BYTE_2] = (byte) ((value & BYTE_2_MASK) >> BYTE_2_SHIFT);
result[BYTE_3] = (byte) ((value & BYTE_3_MASK) >> BYTE_3_SHIFT);
putLong(value, result, 0);
return result;
}

/**
* put the value as four bytes in big endian byte order.
* @param value the Java long to convert to bytes
* @param buf the output buffer
* @param offset
* The offset within the output buffer of the first byte to be written.
* must be non-negative and no larger than <tt>buf.length-4</tt>
*/
public static void putLong(long value, byte[] buf, int offset) {
buf[offset++] = (byte) ((value & BYTE_MASK));
buf[offset++] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT);
buf[offset++] = (byte) ((value & BYTE_2_MASK) >> BYTE_2_SHIFT);
buf[offset] = (byte) ((value & BYTE_3_MASK) >> BYTE_3_SHIFT);
}

public void putLong(byte[] buf, int offset) {
putLong(value, buf, offset);
}

/**
* Helper method to get the value as a Java long from four bytes starting at given array offset
* @param bytes the array of bytes


+ 258
- 178
src/main/org/apache/tools/zip/ZipOutputStream.java View File

@@ -26,7 +26,10 @@ import static org.apache.tools.zip.ZipConstants.WORD;
import static org.apache.tools.zip.ZipConstants.ZIP64_MAGIC;
import static org.apache.tools.zip.ZipConstants.ZIP64_MAGIC_SHORT;
import static org.apache.tools.zip.ZipConstants.ZIP64_MIN_VERSION;
import static org.apache.tools.zip.ZipLong.putLong;
import static org.apache.tools.zip.ZipShort.putShort;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilterOutputStream;
@@ -34,6 +37,7 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
@@ -72,6 +76,34 @@ import java.util.zip.ZipException;
public class ZipOutputStream extends FilterOutputStream {

private static final int BUFFER_SIZE = 512;
private static final int LFH_SIG_OFFSET = 0;
private static final int LFH_VERSION_NEEDED_OFFSET = 4;
private static final int LFH_GPB_OFFSET = 6;
private static final int LFH_METHOD_OFFSET = 8;
private static final int LFH_TIME_OFFSET = 10;
private static final int LFH_CRC_OFFSET = 14;
private static final int LFH_COMPRESSED_SIZE_OFFSET = 18;
private static final int LFH_ORIGINAL_SIZE_OFFSET = 22;
private static final int LFH_FILENAME_LENGTH_OFFSET = 26;
private static final int LFH_EXTRA_LENGTH_OFFSET = 28;
private static final int LFH_FILENAME_OFFSET = 30;
private static final int CFH_SIG_OFFSET = 0;
private static final int CFH_VERSION_MADE_BY_OFFSET = 4;
private static final int CFH_VERSION_NEEDED_OFFSET = 6;
private static final int CFH_GPB_OFFSET = 8;
private static final int CFH_METHOD_OFFSET = 10;
private static final int CFH_TIME_OFFSET = 12;
private static final int CFH_CRC_OFFSET = 16;
private static final int CFH_COMPRESSED_SIZE_OFFSET = 20;
private static final int CFH_ORIGINAL_SIZE_OFFSET = 24;
private static final int CFH_FILENAME_LENGTH_OFFSET = 28;
private static final int CFH_EXTRA_LENGTH_OFFSET = 30;
private static final int CFH_COMMENT_LENGTH_OFFSET = 32;
private static final int CFH_DISK_NUMBER_OFFSET = 34;
private static final int CFH_INTERNAL_ATTRIBUTES_OFFSET = 36;
private static final int CFH_EXTERNAL_ATTRIBUTES_OFFSET = 38;
private static final int CFH_LFH_OFFSET = 42;
private static final int CFH_FILENAME_OFFSET = 46;

/**
* indicates if this archive is finished.
@@ -208,6 +240,8 @@ public class ZipOutputStream extends FilterOutputStream {
*/
private static final byte[] LZERO = {0, 0, 0, 0};

private static final byte[] ONE = ZipLong.getBytes(1L);

/**
* Holds the offsets of the LFH starts for each entry.
*
@@ -287,6 +321,8 @@ public class ZipOutputStream extends FilterOutputStream {

private Zip64Mode zip64Mode = Zip64Mode.AsNeeded;

private final Calendar calendarInstance = Calendar.getInstance();

/**
* Creates a new ZIP OutputStream filtering the underlying stream.
* @param out the outputstream to zip
@@ -459,9 +495,7 @@ public class ZipOutputStream extends FilterOutputStream {
}

cdOffset = written;
for (ZipEntry ze : entries) {
writeCentralFileHeader(ze);
}
writeCentralDirectoryInChunks();
cdLength = written - cdOffset;
writeZip64CentralDirectory();
writeCentralDirectoryEnd();
@@ -471,6 +505,21 @@ public class ZipOutputStream extends FilterOutputStream {
finished = true;
}

private void writeCentralDirectoryInChunks() throws IOException {
final int NUM_PER_WRITE = 1000;
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(70 * NUM_PER_WRITE);
int count = 0;
for (ZipEntry ze : entries) {
byteArrayOutputStream.write(createCentralFileHeader(ze));
if (++count > NUM_PER_WRITE){
writeCounted(byteArrayOutputStream.toByteArray());
byteArrayOutputStream.reset();
count = 0;
}
}
writeCounted(byteArrayOutputStream.toByteArray());
}

/**
* Writes all necessary data for this entry.
*
@@ -481,17 +530,7 @@ public class ZipOutputStream extends FilterOutputStream {
* is {@link Zip64Mode#Never}.
*/
public void closeEntry() throws IOException {
if (finished) {
throw new IOException("Stream has already been finished");
}

if (entry == null) {
throw new IOException("No current entry to close");
}

if (!entry.hasWritten) {
write(EMPTY, 0, 0);
}
preClose();

flushDeflater();

@@ -503,6 +542,10 @@ public class ZipOutputStream extends FilterOutputStream {
final boolean actuallyNeedsZip64 =
handleSizesAndCrc(bytesWritten, realCrc, effectiveMode);

closeEntry(actuallyNeedsZip64);
}

private void closeEntry(boolean actuallyNeedsZip64) throws IOException {
if (raf != null) {
rewriteSizesAndCrc(actuallyNeedsZip64);
}
@@ -511,6 +554,20 @@ public class ZipOutputStream extends FilterOutputStream {
entry = null;
}

private void preClose() throws IOException {
if (finished) {
throw new IOException("Stream has already been finished");
}

if (entry == null) {
throw new IOException("No current entry to close");
}

if (!entry.hasWritten) {
write(EMPTY, 0, 0);
}
}

/**
* Ensures all bytes sent to the deflater are written to the stream.
*/
@@ -564,9 +621,19 @@ public class ZipOutputStream extends FilterOutputStream {
entry.entry.setCrc(crc);
}

final boolean actuallyNeedsZip64 = effectiveMode == Zip64Mode.Always
|| entry.entry.getSize() >= ZIP64_MAGIC
|| entry.entry.getCompressedSize() >= ZIP64_MAGIC;
return checkIfNeedsZip64(effectiveMode);
}

/**
* Ensures the current entry's size and CRC information is set to
* the values just written, verifies it isn't too big in the
* Zip64Mode.Never case and returns whether the entry would
* require a Zip64 extra field.
*/
private boolean checkIfNeedsZip64(Zip64Mode effectiveMode)
throws ZipException {
final boolean actuallyNeedsZip64 = isZip64Required(entry.entry,
effectiveMode);
if (actuallyNeedsZip64 && effectiveMode == Zip64Mode.Never) {
throw new Zip64RequiredException(Zip64RequiredException
.getEntryTooBigMessage(entry.entry));
@@ -574,6 +641,15 @@ public class ZipOutputStream extends FilterOutputStream {
return actuallyNeedsZip64;
}

private boolean isZip64Required(ZipEntry entry1, Zip64Mode requestedMode) {
return requestedMode == Zip64Mode.Always || isTooLageForZip32(entry1);
}

private boolean isTooLageForZip32(ZipEntry zipArchiveEntry){
return zipArchiveEntry.getSize() >= ZIP64_MAGIC
|| zipArchiveEntry.getCompressedSize() >= ZIP64_MAGIC;
}

/**
* When using random access output, write the local file header
* and potentiall the ZIP64 extra containing the correct CRC and
@@ -654,13 +730,15 @@ public class ZipOutputStream extends FilterOutputStream {
// just a placeholder, real data will be in data
// descriptor or inserted later via RandomAccessFile
ZipEightByteInteger size = ZipEightByteInteger.ZERO;
ZipEightByteInteger compressedSize = ZipEightByteInteger.ZERO;
if (entry.entry.getMethod() == STORED
&& entry.entry.getSize() != -1) {
// actually, we already know the sizes
size = new ZipEightByteInteger(entry.entry.getSize());
compressedSize = size;
}
z64.setSize(size);
z64.setCompressedSize(size);
z64.setCompressedSize(compressedSize);
entry.entry.setExtra();
}

@@ -794,17 +872,33 @@ public class ZipOutputStream extends FilterOutputStream {
*/
@Override
public void write(byte[] b, int offset, int length) throws IOException {
if (entry == null) {
throw new IllegalStateException("No current entry");
}
ZipUtil.checkRequestedFeatures(entry.entry);
entry.hasWritten = true;
if (entry.entry.getMethod() == DEFLATED) {
writeDeflated(b, offset, length);
} else {
writeOut(b, offset, length);
written += length;
writeCounted(b, offset, length);
}
crc.update(b, offset, length);
}

/**
* Write bytes to output or random access file.
* @param data the byte array to write
* @throws IOException on error
*/
private void writeCounted(byte[] data) throws IOException {
writeCounted(data, 0, data.length);
}

private void writeCounted(byte[] data, int offset, int length) throws IOException {
writeOut(data, offset, length);
written += length;
}

/**
* write implementation for DEFLATED entries.
*/
@@ -906,8 +1000,7 @@ public class ZipOutputStream extends FilterOutputStream {
protected final void deflate() throws IOException {
int len = def.deflate(buf, 0, buf.length);
if (len > 0) {
writeOut(buf, 0, len);
written += len;
writeCounted(buf, 0, len);
}
}

@@ -927,76 +1020,71 @@ public class ZipOutputStream extends FilterOutputStream {
addUnicodeExtraFields(ze, encodable, name);
}

offsets.put(ze, Long.valueOf(written));
final byte[] localHeader = createLocalFileHeader(ze, name, encodable);
final long localHeaderStart = written;
offsets.put(ze, localHeaderStart);
entry.localDataStart = localHeaderStart + LFH_CRC_OFFSET; // At crc offset
writeCounted(localHeader);
entry.dataStart = written;
}

private byte[] createLocalFileHeader(ZipEntry ze, ByteBuffer name, boolean encodable) {
byte[] extra = ze.getLocalFileDataExtra();
final int nameLen = name.limit() - name.position();
int len= LFH_FILENAME_OFFSET + nameLen + extra.length;
byte[] buf = new byte[len];

writeOut(LFH_SIG);
written += WORD;
System.arraycopy(LFH_SIG, 0, buf, LFH_SIG_OFFSET, WORD);

//store method in local variable to prevent multiple method calls
final int zipMethod = ze.getMethod();

writeVersionNeededToExtractAndGeneralPurposeBits(zipMethod,
!encodable
&& fallbackToUTF8,
hasZip64Extra(ze));
written += WORD;
putShort(versionNeededToExtract(zipMethod, hasZip64Extra(ze)),
buf, LFH_VERSION_NEEDED_OFFSET);

GeneralPurposeBit generalPurposeBit =
getGeneralPurposeBits(zipMethod, !encodable && fallbackToUTF8);
generalPurposeBit.encode(buf, LFH_GPB_OFFSET);

// compression method
writeOut(ZipShort.getBytes(zipMethod));
written += SHORT;
putShort(zipMethod, buf, LFH_METHOD_OFFSET);

// last mod. time and date
writeOut(ZipUtil.toDosTime(ze.getTime()));
written += WORD;
ZipUtil.toDosTime(calendarInstance, ze.getTime(), buf, LFH_TIME_OFFSET);

// CRC
// compressed length
// uncompressed length
entry.localDataStart = written;
if (zipMethod == DEFLATED || raf != null) {
writeOut(LZERO);
if (hasZip64Extra(entry.entry)) {
// point to ZIP64 extended information extra field for
// sizes, may get rewritten once sizes are known if
// stream is seekable
writeOut(ZipLong.ZIP64_MAGIC.getBytes());
writeOut(ZipLong.ZIP64_MAGIC.getBytes());
} else {
writeOut(LZERO);
writeOut(LZERO);
}
System.arraycopy(LZERO, 0, buf, LFH_CRC_OFFSET, WORD);
} else {
writeOut(ZipLong.getBytes(ze.getCrc()));
byte[] size = ZipLong.ZIP64_MAGIC.getBytes();
if (!hasZip64Extra(ze)) {
size = ZipLong.getBytes(ze.getSize());
}
writeOut(size);
writeOut(size);
putLong(ze.getCrc(), buf, LFH_CRC_OFFSET);
}
// CheckStyle:MagicNumber OFF
written += 12;
// CheckStyle:MagicNumber ON

// compressed length
// uncompressed length
if (hasZip64Extra(entry.entry)){
// point to ZIP64 extended information extra field for
// sizes, may get rewritten once sizes are known if
// stream is seekable
ZipLong.ZIP64_MAGIC.putLong(buf, LFH_COMPRESSED_SIZE_OFFSET);
ZipLong.ZIP64_MAGIC.putLong(buf, LFH_ORIGINAL_SIZE_OFFSET);
} else if (zipMethod == DEFLATED || raf != null) {
System.arraycopy(LZERO, 0, buf, LFH_COMPRESSED_SIZE_OFFSET, WORD);
System.arraycopy(LZERO, 0, buf, LFH_ORIGINAL_SIZE_OFFSET, WORD);
} else { // Stored
putLong(ze.getSize(), buf, LFH_COMPRESSED_SIZE_OFFSET);
putLong(ze.getSize(), buf, LFH_ORIGINAL_SIZE_OFFSET);
}
// file name length
writeOut(ZipShort.getBytes(name.limit()));
written += SHORT;
putShort(nameLen, buf, LFH_FILENAME_LENGTH_OFFSET);

// extra field length
byte[] extra = ze.getLocalFileDataExtra();
writeOut(ZipShort.getBytes(extra.length));
written += SHORT;
putShort(extra.length, buf, LFH_EXTRA_LENGTH_OFFSET);

// file name
writeOut(name.array(), name.arrayOffset(),
name.limit() - name.position());
written += name.limit();
System.arraycopy(name.array(), name.arrayOffset(), buf,
LFH_FILENAME_OFFSET, nameLen);

// extra field
writeOut(extra);
written += extra.length;

entry.dataStart = written;
System.arraycopy(extra, 0, buf, LFH_FILENAME_OFFSET + nameLen, extra.length);
return buf;
}

/**
@@ -1045,18 +1133,15 @@ public class ZipOutputStream extends FilterOutputStream {
if (ze.getMethod() != DEFLATED || raf != null) {
return;
}
writeOut(DD_SIG);
writeOut(ZipLong.getBytes(ze.getCrc()));
int sizeFieldSize = WORD;
writeCounted(DD_SIG);
writeCounted(ZipLong.getBytes(ze.getCrc()));
if (!hasZip64Extra(ze)) {
writeOut(ZipLong.getBytes(ze.getCompressedSize()));
writeOut(ZipLong.getBytes(ze.getSize()));
writeCounted(ZipLong.getBytes(ze.getCompressedSize()));
writeCounted(ZipLong.getBytes(ze.getSize()));
} else {
sizeFieldSize = DWORD;
writeOut(ZipEightByteInteger.getBytes(ze.getCompressedSize()));
writeOut(ZipEightByteInteger.getBytes(ze.getSize()));
writeCounted(ZipEightByteInteger.getBytes(ze.getCompressedSize()));
writeCounted(ZipEightByteInteger.getBytes(ze.getSize()));
}
written += 2 * WORD + 2 * sizeFieldSize;
}

/**
@@ -1068,114 +1153,116 @@ public class ZipOutputStream extends FilterOutputStream {
* Zip64Mode#Never}.
*/
protected void writeCentralFileHeader(ZipEntry ze) throws IOException {
writeOut(CFH_SIG);
written += WORD;
byte[] centralFileHeader = createCentralFileHeader(ze);
writeCounted(centralFileHeader);
}

final long lfhOffset = offsets.get(ze).longValue();
private byte[] createCentralFileHeader(ZipEntry ze) throws IOException {
final long lfhOffset = offsets.get(ze);
final boolean needsZip64Extra = hasZip64Extra(ze)
|| ze.getCompressedSize() >= ZIP64_MAGIC
|| ze.getSize() >= ZIP64_MAGIC
|| lfhOffset >= ZIP64_MAGIC;
|| ze.getCompressedSize() >= ZIP64_MAGIC
|| ze.getSize() >= ZIP64_MAGIC
|| lfhOffset >= ZIP64_MAGIC;

if (needsZip64Extra && zip64Mode == Zip64Mode.Never) {
// must be the offset that is too big, otherwise an
// exception would have been throw in putNextEntry or
// closeEntry
// exception would have been throw in putArchiveEntry or
// closeArchiveEntry
throw new Zip64RequiredException(Zip64RequiredException
.ARCHIVE_TOO_BIG_MESSAGE);
.ARCHIVE_TOO_BIG_MESSAGE);
}


handleZip64Extra(ze, lfhOffset, needsZip64Extra);

return createCentralFileHeader(ze, getName(ze), lfhOffset, needsZip64Extra);
}

/**
* Writes the central file header entry.
* @param ze the entry to write
* @param name The encoded name
* @param lfhOffset Local file header offset for this file
* @throws IOException on error
*/
private byte[] createCentralFileHeader(ZipEntry ze, ByteBuffer name, long lfhOffset,
boolean needsZip64Extra) throws IOException {
byte[] extra = ze.getCentralDirectoryExtra();

// file comment length
String comm = ze.getComment();
if (comm == null) {
comm = "";
}

ByteBuffer commentB = getEntryEncoding(ze).encode(comm);
final int nameLen = name.limit() - name.position();
final int commentLen = commentB.limit() - commentB.position();
int len= CFH_FILENAME_OFFSET + nameLen + extra.length + commentLen;
byte[] buf = new byte[len];

System.arraycopy(CFH_SIG, 0, buf, CFH_SIG_OFFSET, WORD);

// version made by
// CheckStyle:MagicNumber OFF
writeOut(ZipShort.getBytes((ze.getPlatform() << 8) |
(!hasUsedZip64 ? DATA_DESCRIPTOR_MIN_VERSION
: ZIP64_MIN_VERSION)));
written += SHORT;
putShort((ze.getPlatform() << 8) | (!hasUsedZip64 ? DATA_DESCRIPTOR_MIN_VERSION : ZIP64_MIN_VERSION),
buf, CFH_VERSION_MADE_BY_OFFSET);

final int zipMethod = ze.getMethod();
final boolean encodable = zipEncoding.canEncode(ze.getName());
writeVersionNeededToExtractAndGeneralPurposeBits(zipMethod,
!encodable
&& fallbackToUTF8,
needsZip64Extra);
written += WORD;
putShort(versionNeededToExtract(zipMethod, needsZip64Extra), buf, CFH_VERSION_NEEDED_OFFSET);
getGeneralPurposeBits(zipMethod, !encodable && fallbackToUTF8).encode(buf, CFH_GPB_OFFSET);

// compression method
writeOut(ZipShort.getBytes(zipMethod));
written += SHORT;
putShort(zipMethod, buf, CFH_METHOD_OFFSET);

// last mod. time and date
writeOut(ZipUtil.toDosTime(ze.getTime()));
written += WORD;
ZipUtil.toDosTime(calendarInstance, ze.getTime(), buf, CFH_TIME_OFFSET);

// CRC
// compressed length
// uncompressed length
writeOut(ZipLong.getBytes(ze.getCrc()));
putLong(ze.getCrc(), buf, CFH_CRC_OFFSET);
if (ze.getCompressedSize() >= ZIP64_MAGIC
|| ze.getSize() >= ZIP64_MAGIC) {
writeOut(ZipLong.ZIP64_MAGIC.getBytes());
writeOut(ZipLong.ZIP64_MAGIC.getBytes());
|| ze.getSize() >= ZIP64_MAGIC) {
ZipLong.ZIP64_MAGIC.putLong(buf, CFH_COMPRESSED_SIZE_OFFSET);
ZipLong.ZIP64_MAGIC.putLong(buf, CFH_ORIGINAL_SIZE_OFFSET);
} else {
writeOut(ZipLong.getBytes(ze.getCompressedSize()));
writeOut(ZipLong.getBytes(ze.getSize()));
putLong(ze.getCompressedSize(), buf, CFH_COMPRESSED_SIZE_OFFSET);
putLong(ze.getSize(), buf, CFH_ORIGINAL_SIZE_OFFSET);
}
// CheckStyle:MagicNumber OFF
written += 12;
// CheckStyle:MagicNumber ON

ByteBuffer name = getName(ze);

writeOut(ZipShort.getBytes(name.limit()));
written += SHORT;
putShort(nameLen, buf, CFH_FILENAME_LENGTH_OFFSET);

// extra field length
byte[] extra = ze.getCentralDirectoryExtra();
writeOut(ZipShort.getBytes(extra.length));
written += SHORT;
putShort(extra.length, buf, CFH_EXTRA_LENGTH_OFFSET);

// file comment length
String comm = ze.getComment();
if (comm == null) {
comm = "";
}

ByteBuffer commentB = getEntryEncoding(ze).encode(comm);

writeOut(ZipShort.getBytes(commentB.limit()));
written += SHORT;
putShort(commentLen, buf, CFH_COMMENT_LENGTH_OFFSET);

// disk number start
writeOut(ZERO);
written += SHORT;
System.arraycopy(ZERO, 0, buf, CFH_DISK_NUMBER_OFFSET, SHORT);

// internal file attributes
writeOut(ZipShort.getBytes(ze.getInternalAttributes()));
written += SHORT;
putShort(ze.getInternalAttributes(), buf, CFH_INTERNAL_ATTRIBUTES_OFFSET);

// external file attributes
writeOut(ZipLong.getBytes(ze.getExternalAttributes()));
written += WORD;
putLong(ze.getExternalAttributes(), buf, CFH_EXTERNAL_ATTRIBUTES_OFFSET);

// relative offset of LFH
writeOut(ZipLong.getBytes(Math.min(lfhOffset, ZIP64_MAGIC)));
written += WORD;
putLong(Math.min(lfhOffset, ZIP64_MAGIC), buf, CFH_LFH_OFFSET);

// file name
writeOut(name.array(), name.arrayOffset(),
name.limit() - name.position());
written += name.limit();
System.arraycopy(name.array(), name.arrayOffset(), buf, CFH_FILENAME_OFFSET, nameLen);

// extra field
writeOut(extra);
written += extra.length;
int extraStart = CFH_FILENAME_OFFSET + nameLen;
System.arraycopy(extra, 0, buf, extraStart, extra.length);

int commentStart = extraStart + commentLen;

// file comment
writeOut(commentB.array(), commentB.arrayOffset(),
commentB.limit() - commentB.position());
written += commentB.limit();
System.arraycopy(commentB.array(), commentB.arrayOffset(), buf, commentStart, commentLen);
return buf;
}

/**
@@ -1210,11 +1297,11 @@ public class ZipOutputStream extends FilterOutputStream {
* and {@link Zip64Mode #setUseZip64} is {@link Zip64Mode#Never}.
*/
protected void writeCentralDirectoryEnd() throws IOException {
writeOut(EOCD_SIG);
writeCounted(EOCD_SIG);

// disk numbers
writeOut(ZERO);
writeOut(ZERO);
writeCounted(ZERO);
writeCounted(ZERO);

// number of entries
int numberOfEntries = entries.size();
@@ -1230,18 +1317,18 @@ public class ZipOutputStream extends FilterOutputStream {

byte[] num = ZipShort.getBytes(Math.min(numberOfEntries,
ZIP64_MAGIC_SHORT));
writeOut(num);
writeOut(num);
writeCounted(num);
writeCounted(num);

// length and location of CD
writeOut(ZipLong.getBytes(Math.min(cdLength, ZIP64_MAGIC)));
writeOut(ZipLong.getBytes(Math.min(cdOffset, ZIP64_MAGIC)));
writeCounted(ZipLong.getBytes(Math.min(cdLength, ZIP64_MAGIC)));
writeCounted(ZipLong.getBytes(Math.min(cdOffset, ZIP64_MAGIC)));

// ZIP file comment
ByteBuffer data = this.zipEncoding.encode(comment);
writeOut(ZipShort.getBytes(data.limit()));
writeOut(data.array(), data.arrayOffset(),
data.limit() - data.position());
int dataLen = data.limit() - data.position();
writeCounted(ZipShort.getBytes(dataLen));
writeCounted(data.array(), data.arrayOffset(), dataLen);
}

/**
@@ -1292,8 +1379,6 @@ public class ZipOutputStream extends FilterOutputStream {
}
}

private static final byte[] ONE = ZipLong.getBytes(1L);

/**
* Writes the &quot;ZIP64 End of central dir record&quot; and
* &quot;ZIP64 End of central dir locator&quot;.
@@ -1409,33 +1494,28 @@ public class ZipOutputStream extends FilterOutputStream {
}
}

private void writeVersionNeededToExtractAndGeneralPurposeBits(final int
zipMethod,
final boolean
utfFallback,
final boolean
zip64)
throws IOException {

// CheckStyle:MagicNumber OFF
int versionNeededToExtract = INITIAL_VERSION;
private GeneralPurposeBit getGeneralPurposeBits(final int zipMethod, final boolean utfFallback) {
GeneralPurposeBit b = new GeneralPurposeBit();
b.useUTF8ForNames(useUTF8Flag || utfFallback);
if (zipMethod == DEFLATED && raf == null) {
// requires version 2 as we are going to store length info
// in the data descriptor
versionNeededToExtract = DATA_DESCRIPTOR_MIN_VERSION;
if (isDeflatedToOutputStream(zipMethod)) {
b.useDataDescriptor(true);
}
return b;
}

private int versionNeededToExtract(final int zipMethod, final boolean zip64) {
if (zip64) {
versionNeededToExtract = ZIP64_MIN_VERSION;
return ZIP64_MIN_VERSION;
}
// CheckStyle:MagicNumber ON
// requires version 2 as we are going to store length info
// in the data descriptor
return (isDeflatedToOutputStream(zipMethod)) ?
DATA_DESCRIPTOR_MIN_VERSION :
INITIAL_VERSION;
}

// version needed to extract
writeOut(ZipShort.getBytes(versionNeededToExtract));
// general purpose bit flag
writeOut(b.encode());
private boolean isDeflatedToOutputStream(int zipMethod) {
return zipMethod == DEFLATED && raf == null;
}

/**


+ 14
- 2
src/main/org/apache/tools/zip/ZipShort.java View File

@@ -66,11 +66,23 @@ public final class ZipShort implements Cloneable {
*/
public byte[] getBytes() {
byte[] result = new byte[2];
result[0] = (byte) (value & BYTE_MASK);
result[1] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT);
putShort(value, result, 0);
return result;
}

/**
* put the value as two bytes in big endian byte order.
* @param value the Java int to convert to bytes
* @param buf the output buffer
* @param offset
* The offset within the output buffer of the first byte to be written.
* must be non-negative and no larger than <tt>buf.length-2</tt>
*/
public static void putShort(int value, byte[] buf, int offset) {
buf[offset] = (byte) (value & BYTE_MASK);
buf[offset+1] = (byte) ((value & BYTE_1_MASK) >> BYTE_1_SHIFT);
}

/**
* Get value as Java int.
* @return value as a Java int


+ 23
- 3
src/main/org/apache/tools/zip/ZipUtil.java View File

@@ -49,12 +49,32 @@ public abstract class ZipUtil {
* @return the date as a byte array
*/
public static byte[] toDosTime(long t) {
Calendar c = Calendar.getInstance();
byte[] result = new byte[4];
toDosTime(t, result, 0);
return result;
}

/**
* Convert a Date object to a DOS date/time field.
*
* <p>Stolen from InfoZip's <code>fileio.c</code></p>
* @param t number of milliseconds since the epoch
* @param buf the output buffer
* @param offset
* The offset within the output buffer of the first byte to be written.
* must be non-negative and no larger than <tt>buf.length-4</tt>
*/
public static void toDosTime(long t, byte[] buf, int offset) {
toDosTime(Calendar.getInstance(), t, buf, offset);
}

static void toDosTime(Calendar c, long t, byte[] buf, int offset) {
c.setTimeInMillis(t);

int year = c.get(Calendar.YEAR);
if (year < 1980) {
return copy(DOS_TIME_MIN); // stop callers from changing the array
System.arraycopy(DOS_TIME_MIN, 0, buf, offset, DOS_TIME_MIN.length);// stop callers from changing the array
return;
}
int month = c.get(Calendar.MONTH) + 1;
long value = ((year - 1980) << 25)
@@ -63,7 +83,7 @@ public abstract class ZipUtil {
| (c.get(Calendar.HOUR_OF_DAY) << 11)
| (c.get(Calendar.MINUTE) << 5)
| (c.get(Calendar.SECOND) >> 1);
return ZipLong.getBytes(value);
ZipLong.putLong(value, buf, offset);
}

/**


Loading…
Cancel
Save