Write archive contents in chunks (#589)

Fix a bug in ArchiveWriter where large files would only partly be
written into an archive.
This is done so by writing the data in chunks.

---------

Signed-off-by: Aditya Ramani <a_ramani@apple.com>

Author: adityaramani

Sources/ContainerizationArchive/ArchiveError.swift

@@ -36,6 +36,8 @@ public enum ArchiveError: Error, CustomStringConvertible {
     case failedToDetectFilter
     case failedToDetectFormat
     case failedToExtractArchive(String)
+    case failedToCreateArchive(String)
+    case invalidBaseAddressArchiveWrite
 
     /// Description of the error
     public var description: String {
@@ -74,6 +76,10 @@ public enum ArchiveError: Error, CustomStringConvertible {
             return "failed to detect format from archive."
         case .failedToExtractArchive(let reason):
             return "failed to extract archive: \(reason)"
+        case .failedToCreateArchive(let reason):
+            return "failed to create archive: \(reason)"
+        case .invalidBaseAddressArchiveWrite:
+            return "got an invalid base address for pointer when writing data to archive"
         }
     }
 }

Sources/ContainerizationArchive/ArchiveReader.swift

@@ -48,7 +48,7 @@ public struct ArchiveEntryReader: ReadableStream {
 
 /// A class responsible for reading entries from an archive file.
 public final class ArchiveReader {
-    private static let blockSize = 65536
+    private static let chunkSize = 4 * 1024 * 1024
 
     /// A pointer to the underlying `archive` C structure.
     var underlying: OpaquePointer?
@@ -375,7 +375,7 @@ extension ArchiveReader {
     }
 
     private static func copyDataReaderToFd(dataReader: ArchiveEntryReader, fileFd: Int32, memberPath: FilePath) throws {
-        var buffer = [UInt8](repeating: 0, count: ArchiveReader.blockSize)
+        var buffer = [UInt8](repeating: 0, count: ArchiveReader.chunkSize)
         while true {
             let bytesRead = buffer.withUnsafeMutableBufferPointer { bufferPtr in
                 guard let baseAddress = bufferPtr.baseAddress else { return 0 }

Sources/ContainerizationArchive/ArchiveWriter.swift

@@ -20,6 +20,8 @@ import SystemPackage
 
 /// A class responsible for writing archives in various formats.
 public final class ArchiveWriter {
+    private static let chunkSize = 4 * 1024 * 1024
+
     var underlying: OpaquePointer!
 
     /// Initialize a new `ArchiveWriter` with the given configuration.
@@ -166,9 +168,16 @@ extension ArchiveWriter {
     fileprivate func writeData(data: UnsafeRawBufferPointer) throws {
         guard let underlying = self.underlying else { throw ArchiveError.noUnderlyingArchive }
 
-        let result = archive_write_data(underlying, data.baseAddress, data.count)
-        guard result >= 0 else {
-            throw ArchiveError.unableToWriteData(result)
+        var offset = 0
+        while offset < data.count {
+            guard let baseAddress = data.baseAddress?.advanced(by: offset) else {
+                throw ArchiveError.invalidBaseAddressArchiveWrite
+            }
+            let result = archive_write_data(underlying, baseAddress, data.count - offset)
+            guard result > 0 else {
+                throw ArchiveError.unableToWriteData(result)
+            }
+            offset += Int(result)
         }
     }
 }
@@ -188,7 +197,7 @@ extension ArchiveWriter {
         var rootStat = stat()
         guard lstat(dirPath.string, &rootStat) == 0 else {
             let err = POSIXErrorCode(rawValue: errno) ?? .EINVAL
-            throw ArchiveError.failedToExtractArchive("lstat failed for '\(dirPath)': \(POSIXError(err))")
+            throw ArchiveError.failedToCreateArchive("lstat failed for '\(dirPath)': \(POSIXError(err))")
         }
         let rootEntry = WriteEntry()
         rootEntry.path = "./"
@@ -215,7 +224,7 @@ extension ArchiveWriter {
             guard lstat(fullPath.string, &statInfo) == 0 else {
                 let errNo = errno
                 let err = POSIXErrorCode(rawValue: errNo) ?? .EINVAL
-                throw ArchiveError.failedToExtractArchive("lstat failed for '\(fullPath)': \(POSIXError(err))")
+                throw ArchiveError.failedToCreateArchive("lstat failed for '\(fullPath)': \(POSIXError(err))")
             }
 
             let mode = statInfo.st_mode
@@ -267,8 +276,28 @@ extension ArchiveWriter {
             entry.owner = uid
             entry.permissions = mode
             if type == .regular {
-                let data = try Data(contentsOf: URL(fileURLWithPath: fullPath.string), options: .uncached)
-                try self.writeEntry(entry: entry, data: data)
+                let buf = UnsafeMutableRawBufferPointer.allocate(byteCount: Self.chunkSize, alignment: 1)
+                guard let baseAddress = buf.baseAddress else {
+                    throw ArchiveError.failedToCreateArchive("cannot create temporary buffer of size \(Self.chunkSize)")
+                }
+                defer { buf.deallocate() }
+                let fd = Foundation.open(fullPath.string, O_RDONLY)
+                guard fd >= 0 else {
+                    let err = POSIXErrorCode(rawValue: errno) ?? .EINVAL
+                    throw ArchiveError.failedToCreateArchive("cannot open file \(fullPath.string) for reading: \(err)")
+                }
+                defer { close(fd) }
+                try self.writeHeader(entry: entry)
+                while true {
+                    let n = read(fd, baseAddress, Self.chunkSize)
+                    if n == 0 { break }
+                    if n < 0 {
+                        let err = POSIXErrorCode(rawValue: errno) ?? .EIO
+                        throw ArchiveError.failedToCreateArchive("failed to read from file \(fullPath.string): \(err)")
+                    }
+                    try self.writeData(data: UnsafeRawBufferPointer(start: baseAddress, count: n))
+                }
+                try self.finishEntry()
             } else {
                 try self.writeEntry(entry: entry, data: nil)
             }