Streamline CRC and hash calculation when preparing stored entries

Closes gh-46202

Author: wilkinsona

spring-boot-project/spring-boot-tools/spring-boot-gradle-plugin/src/main/java/org/springframework/boot/gradle/tasks/bundling/BootZipCopyAction.java

@@ -22,10 +22,13 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HexFormat;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -350,7 +353,7 @@ private void writeJarModeLibrary(String location, JarModeLibrary library) throws
 			String name = location + library.getName();
 			writeEntry(name, ZipEntryContentWriter.fromInputStream(library.openStream()), false, (entry) -> {
 				try (InputStream in = library.openStream()) {
-					prepareStoredEntry(library.openStream(), entry);
+					prepareStoredEntry(library.openStream(), false, entry);
 				}
 			});
 			if (BootZipCopyAction.this.layerResolver != null) {
@@ -450,14 +453,13 @@ private void prepareEntry(ZipArchiveEntry entry, String name, Long time, int mod
 		}
 
 		private void prepareStoredEntry(FileCopyDetails details, ZipArchiveEntry archiveEntry) throws IOException {
-			prepareStoredEntry(details.open(), archiveEntry);
-			if (BootZipCopyAction.this.requiresUnpack.isSatisfiedBy(details)) {
-				archiveEntry.setComment("UNPACK:" + FileUtils.sha1Hash(details.getFile()));
-			}
+			prepareStoredEntry(details.open(), BootZipCopyAction.this.requiresUnpack.isSatisfiedBy(details),
+					archiveEntry);
 		}
 
-		private void prepareStoredEntry(InputStream input, ZipArchiveEntry archiveEntry) throws IOException {
-			new CrcAndSize(input).setUpStoredEntry(archiveEntry);
+		private void prepareStoredEntry(InputStream input, boolean unpack, ZipArchiveEntry archiveEntry)
+				throws IOException {
+			new StoredEntryPreparator(input, unpack).prepareStoredEntry(archiveEntry);
 		}
 
 		private Long getTime() {
@@ -569,36 +571,55 @@ static ZipEntryContentWriter fromLines(String encoding, Collection<String> lines
 	}
 
 	/**
-	 * Data holder for CRC and Size.
+	 * Prepares a {@link ZipEntry#STORED stored} {@link ZipArchiveEntry entry} with CRC
+	 * and size information. Also adds an {@code UNPACK} comment, if needed.
 	 */
-	private static class CrcAndSize {
+	private static class StoredEntryPreparator {
 
 		private static final int BUFFER_SIZE = 32 * 1024;
 
+		private final MessageDigest messageDigest;
+
 		private final CRC32 crc = new CRC32();
 
 		private long size;
 
-		CrcAndSize(InputStream inputStream) throws IOException {
+		StoredEntryPreparator(InputStream inputStream, boolean unpack) throws IOException {
+			this.messageDigest = (unpack) ? sha1Digest() : null;
 			try (inputStream) {
 				load(inputStream);
 			}
 		}
 
+		private static MessageDigest sha1Digest() {
+			try {
+				return MessageDigest.getInstance("SHA-1");
+			}
+			catch (NoSuchAlgorithmException ex) {
+				throw new IllegalStateException(ex);
+			}
+		}
+
 		private void load(InputStream inputStream) throws IOException {
 			byte[] buffer = new byte[BUFFER_SIZE];
 			int bytesRead;
 			while ((bytesRead = inputStream.read(buffer)) != -1) {
 				this.crc.update(buffer, 0, bytesRead);
+				if (this.messageDigest != null) {
+					this.messageDigest.update(buffer, 0, bytesRead);
+				}
 				this.size += bytesRead;
 			}
 		}
 
-		void setUpStoredEntry(ZipArchiveEntry entry) {
+		void prepareStoredEntry(ZipArchiveEntry entry) {
 			entry.setSize(this.size);
 			entry.setCompressedSize(this.size);
 			entry.setCrc(this.crc.getValue());
 			entry.setMethod(ZipEntry.STORED);
+			if (this.messageDigest != null) {
+				entry.setComment("UNPACK:" + HexFormat.of().formatHex(this.messageDigest.digest()));
+			}
 		}
 
 	}

spring-boot-project/spring-boot-tools/spring-boot-gradle-plugin/src/test/java/org/springframework/boot/gradle/tasks/buildinfo/BuildInfoIntegrationTests.java

@@ -29,7 +29,7 @@
 import org.junit.jupiter.api.TestTemplate;
 
 import org.springframework.boot.gradle.junit.GradleCompatibility;
-import org.springframework.boot.loader.tools.FileUtils;
+import org.springframework.boot.testsupport.FileUtils;
 import org.springframework.boot.testsupport.gradle.testkit.GradleBuild;
 
 import static org.assertj.core.api.Assertions.assertThat;

spring-boot-project/spring-boot-tools/spring-boot-gradle-plugin/src/test/java/org/springframework/boot/gradle/tasks/bundling/AbstractBootArchiveIntegrationTests.java

@@ -52,8 +52,8 @@
 import org.gradle.testkit.runner.TaskOutcome;
 import org.junit.jupiter.api.TestTemplate;
 
-import org.springframework.boot.loader.tools.FileUtils;
 import org.springframework.boot.loader.tools.JarModeLibrary;
+import org.springframework.boot.testsupport.FileUtils;
 import org.springframework.boot.testsupport.gradle.testkit.GradleBuild;
 import org.springframework.util.FileSystemUtils;
 import org.springframework.util.StringUtils;

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/main/java/org/springframework/boot/loader/tools/AbstractJarWriter.java

@@ -18,17 +18,18 @@
 
 import java.io.BufferedInputStream;
 import java.io.BufferedWriter;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Collection;
 import java.util.Enumeration;
 import java.util.HashSet;
+import java.util.HexFormat;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.jar.JarEntry;
@@ -40,6 +41,7 @@
 
 import org.apache.commons.compress.archivers.jar.JarArchiveEntry;
 import org.apache.commons.compress.archivers.zip.UnixStat;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 
 /**
  * Abstract base class for JAR writers.
@@ -97,20 +99,21 @@ final void writeEntries(JarFile jarFile, EntryTransformer entryTransformer, Unpa
 
 	private void writeEntry(JarFile jarFile, EntryTransformer entryTransformer, UnpackHandler unpackHandler,
 			JarArchiveEntry entry, Library library) throws IOException {
-		setUpEntry(jarFile, entry);
+		setUpEntry(jarFile, entry, unpackHandler);
 		try (ZipHeaderPeekInputStream inputStream = new ZipHeaderPeekInputStream(jarFile.getInputStream(entry))) {
 			EntryWriter entryWriter = new InputStreamEntryWriter(inputStream);
 			JarArchiveEntry transformedEntry = entryTransformer.transform(entry);
 			if (transformedEntry != null) {
-				writeEntry(transformedEntry, library, entryWriter, unpackHandler);
+				writeEntry(transformedEntry, library, entryWriter);
 			}
 		}
 	}
 
-	private void setUpEntry(JarFile jarFile, JarArchiveEntry entry) throws IOException {
+	private void setUpEntry(JarFile jarFile, JarArchiveEntry entry, UnpackHandler unpackHandler) throws IOException {
 		try (ZipHeaderPeekInputStream inputStream = new ZipHeaderPeekInputStream(jarFile.getInputStream(entry))) {
 			if (inputStream.hasZipHeader() && entry.getMethod() != ZipEntry.STORED) {
-				new CrcAndSize(inputStream).setupStoredEntry(entry);
+				new StoredEntryPreparator(inputStream, unpackHandler.requiresUnpack(entry.getName()))
+					.prepareStoredEntry(entry);
 			}
 			else {
 				entry.setCompressedSize(-1);
@@ -151,9 +154,10 @@ public void writeEntry(String entryName, EntryWriter entryWriter) throws IOExcep
 	public void writeNestedLibrary(String location, Library library) throws IOException {
 		JarArchiveEntry entry = new JarArchiveEntry(location + library.getName());
 		entry.setTime(getNestedLibraryTime(library));
-		new CrcAndSize(library::openStream).setupStoredEntry(entry);
+		new StoredEntryPreparator(library.openStream(), new LibraryUnpackHandler(library).requiresUnpack(location))
+			.prepareStoredEntry(entry);
 		try (InputStream inputStream = library.openStream()) {
-			writeEntry(entry, library, new InputStreamEntryWriter(inputStream), new LibraryUnpackHandler(library));
+			writeEntry(entry, library, new InputStreamEntryWriter(inputStream));
 		}
 	}
 
@@ -240,7 +244,7 @@ private boolean isServicesEntry(JarEntry entry) {
 	}
 
 	private void writeEntry(JarArchiveEntry entry, EntryWriter entryWriter) throws IOException {
-		writeEntry(entry, null, entryWriter, UnpackHandler.NEVER);
+		writeEntry(entry, null, entryWriter);
 	}
 
 	/**
@@ -249,11 +253,9 @@ private void writeEntry(JarArchiveEntry entry, EntryWriter entryWriter) throws I
 	 * @param entry the entry to write
 	 * @param library the library for the entry or {@code null}
 	 * @param entryWriter the entry writer or {@code null} if there is no content
-	 * @param unpackHandler handles possible unpacking for the entry
 	 * @throws IOException in case of I/O errors
 	 */
-	private void writeEntry(JarArchiveEntry entry, Library library, EntryWriter entryWriter,
-			UnpackHandler unpackHandler) throws IOException {
+	private void writeEntry(JarArchiveEntry entry, Library library, EntryWriter entryWriter) throws IOException {
 		String name = entry.getName();
 		if (this.writtenEntries.add(name)) {
 			writeParentDirectoryEntries(name);
@@ -263,7 +265,6 @@ private void writeEntry(JarArchiveEntry entry, Library library, EntryWriter entr
 				entryWriter = SizeCalculatingEntryWriter.get(entryWriter);
 				entry.setSize(entryWriter.size());
 			}
-			entryWriter = addUnpackCommentIfNecessary(entry, entryWriter, unpackHandler);
 			updateLayerIndex(entry, library);
 			writeToArchive(entry, entryWriter);
 		}
@@ -283,22 +284,11 @@ private void writeParentDirectoryEntries(String name) throws IOException {
 		while (parent.lastIndexOf('/') != -1) {
 			parent = parent.substring(0, parent.lastIndexOf('/'));
 			if (!parent.isEmpty()) {
-				writeEntry(new JarArchiveEntry(parent + "/"), null, null, UnpackHandler.NEVER);
+				writeEntry(new JarArchiveEntry(parent + "/"), null, null);
 			}
 		}
 	}
 
-	private EntryWriter addUnpackCommentIfNecessary(JarArchiveEntry entry, EntryWriter entryWriter,
-			UnpackHandler unpackHandler) throws IOException {
-		if (entryWriter == null || !unpackHandler.requiresUnpack(entry.getName())) {
-			return entryWriter;
-		}
-		ByteArrayOutputStream output = new ByteArrayOutputStream();
-		entryWriter.write(output);
-		entry.setComment("UNPACK:" + unpackHandler.sha1Hash(entry.getName()));
-		return new InputStreamEntryWriter(new ByteArrayInputStream(output.toByteArray()));
-	}
-
 	/**
 	 * {@link EntryWriter} that writes content from an {@link InputStream}.
 	 */
@@ -323,38 +313,55 @@ public void write(OutputStream outputStream) throws IOException {
 	}
 
 	/**
-	 * Data holder for CRC and Size.
+	 * Prepares a {@link ZipEntry#STORED stored} {@link ZipArchiveEntry entry} with CRC
+	 * and size information. Also adds an {@code UNPACK} comment, if needed.
 	 */
-	private static class CrcAndSize {
+	private static class StoredEntryPreparator {
+
+		private static final int BUFFER_SIZE = 32 * 1024;
+
+		private final MessageDigest messageDigest;
 
 		private final CRC32 crc = new CRC32();
 
 		private long size;
 
-		CrcAndSize(InputStreamSupplier supplier) throws IOException {
-			try (InputStream inputStream = supplier.openStream()) {
+		StoredEntryPreparator(InputStream inputStream, boolean unpack) throws IOException {
+			this.messageDigest = (unpack) ? sha1Digest() : null;
+			try (inputStream) {
 				load(inputStream);
 			}
 		}
 
-		CrcAndSize(InputStream inputStream) throws IOException {
-			load(inputStream);
+		private static MessageDigest sha1Digest() {
+			try {
+				return MessageDigest.getInstance("SHA-1");
+			}
+			catch (NoSuchAlgorithmException ex) {
+				throw new IllegalStateException(ex);
+			}
 		}
 
 		private void load(InputStream inputStream) throws IOException {
 			byte[] buffer = new byte[BUFFER_SIZE];
 			int bytesRead;
 			while ((bytesRead = inputStream.read(buffer)) != -1) {
 				this.crc.update(buffer, 0, bytesRead);
+				if (this.messageDigest != null) {
+					this.messageDigest.update(buffer, 0, bytesRead);
+				}
 				this.size += bytesRead;
 			}
 		}
 
-		void setupStoredEntry(JarArchiveEntry entry) {
+		void prepareStoredEntry(ZipArchiveEntry entry) {
 			entry.setSize(this.size);
 			entry.setCompressedSize(this.size);
 			entry.setCrc(this.crc.getValue());
 			entry.setMethod(ZipEntry.STORED);
+			if (this.messageDigest != null) {
+				entry.setComment("UNPACK:" + HexFormat.of().formatHex(this.messageDigest.digest()));
+			}
 		}
 
 	}
@@ -381,24 +388,10 @@ interface EntryTransformer {
 	 */
 	interface UnpackHandler {
 
-		UnpackHandler NEVER = new UnpackHandler() {
-
-			@Override
-			public boolean requiresUnpack(String name) {
-				return false;
-			}
-
-			@Override
-			public String sha1Hash(String name) throws IOException {
-				throw new UnsupportedOperationException();
-			}
-
-		};
+		UnpackHandler NEVER = (name) -> false;
 
 		boolean requiresUnpack(String name);
 
-		String sha1Hash(String name) throws IOException;
-
 	}
 
 	/**
@@ -417,11 +410,6 @@ public boolean requiresUnpack(String name) {
 			return this.library.isUnpackRequired();
 		}
 
-		@Override
-		public String sha1Hash(String name) throws IOException {
-			return Digest.sha1(this.library::openStream);
-		}
-
 	}
 
 }

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/main/java/org/springframework/boot/loader/tools/Digest.java

@@ -17,7 +17,7 @@
 package org.springframework.boot.loader.tools;
 
 import java.io.IOException;
-import java.security.DigestInputStream;
+import java.io.InputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.HexFormat;
@@ -26,9 +26,14 @@
  * Utility class used to calculate digests.
  *
  * @author Phillip Webb
+ * @author Andy Wilkinson
+ * @deprecated since 3.4.8 for removal in 4.0.0 without replacement
  */
+@Deprecated(since = "3.4.8", forRemoval = true)
 final class Digest {
 
+	private static final int BUFFER_SIZE = 32 * 1024;
+
 	private Digest() {
 	}
 
@@ -40,10 +45,14 @@ private Digest() {
 	 */
 	static String sha1(InputStreamSupplier supplier) throws IOException {
 		try {
-			try (DigestInputStream inputStream = new DigestInputStream(supplier.openStream(),
-					MessageDigest.getInstance("SHA-1"))) {
-				inputStream.readAllBytes();
-				return HexFormat.of().formatHex(inputStream.getMessageDigest().digest());
+			MessageDigest messageDigest = MessageDigest.getInstance("SHA-1");
+			try (InputStream inputStream = supplier.openStream()) {
+				byte[] buffer = new byte[BUFFER_SIZE];
+				int bytesRead;
+				while ((bytesRead = inputStream.read(buffer)) != -1) {
+					messageDigest.update(buffer, 0, bytesRead);
+				}
+				return HexFormat.of().formatHex(messageDigest.digest());
 			}
 		}
 		catch (NoSuchAlgorithmException ex) {

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/main/java/org/springframework/boot/loader/tools/FileUtils.java

@@ -60,7 +60,10 @@ public static void removeDuplicatesFromOutputDirectory(File outputDirectory, Fil
 	 * @param file the file to hash
 	 * @return the hash value as a String
 	 * @throws IOException if the file cannot be read
+	 * @deprecated since 3.4.8 for removal in 4.0.0 without replacement
 	 */
+	@Deprecated(since = "3.4.8", forRemoval = true)
+	@SuppressWarnings("removal")
 	public static String sha1Hash(File file) throws IOException {
 		return Digest.sha1(InputStreamSupplier.forFile(file));
 	}