diff --git a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift index 9ae6f65f..ddf5ea1f 100644 --- a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift +++ b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift @@ -1,9 +1,9 @@ import ArgumentParser +import CommonCrypto +import Compression // Add this import import Darwin import Foundation import Swift -import CommonCrypto -import Compression // Add this import // Extension to calculate SHA256 hash extension Data { @@ -25,14 +25,14 @@ enum PushError: Error { case authenticationFailed case missingToken case invalidURL - case lz4NotFound // Added error case - case invalidMediaType // Added during part refactoring - case missingUncompressedSizeAnnotation // Added for sparse file handling - case fileCreationFailed(String) // Added for sparse file handling - case reassemblySetupFailed(path: String, underlyingError: Error?) // Added for sparse file handling - case missingPart(Int) // Added for sparse file handling - case layerDownloadFailed(String) // Added for download retries - case manifestFetchFailed // Added for manifest fetching + case lz4NotFound // Added error case + case invalidMediaType // Added during part refactoring + case missingUncompressedSizeAnnotation // Added for sparse file handling + case fileCreationFailed(String) // Added for sparse file handling + case reassemblySetupFailed(path: String, underlyingError: Error?) // Added for sparse file handling + case missingPart(Int) // Added for sparse file handling + case layerDownloadFailed(String) // Added for download retries + case manifestFetchFailed // Added for manifest fetching } // Define a specific error type for when no underlying error exists @@ -54,8 +54,11 @@ struct OCIManifestLayer { let digest: String let uncompressedSize: UInt64? let uncompressedContentDigest: String? - - init(mediaType: String, size: Int, digest: String, uncompressedSize: UInt64? = nil, uncompressedContentDigest: String? = nil) { + + init( + mediaType: String, size: Int, digest: String, uncompressedSize: UInt64? = nil, + uncompressedContentDigest: String? = nil + ) { self.mediaType = mediaType self.size = size self.digest = digest @@ -119,21 +122,21 @@ actor DiskPartsCollector { // Store tuples of (sequentialPartNum, url) private var diskParts: [(Int, URL)] = [] // Restore internal counter - private var partCounter = 0 + private var partCounter = 0 // Adds a part and returns its assigned sequential number func addPart(url: URL) -> Int { - partCounter += 1 // Use counter logic - let partNum = partCounter - diskParts.append((partNum, url)) // Store sequential number - return partNum // Return assigned sequential number + partCounter += 1 // Use counter logic + let partNum = partCounter + diskParts.append((partNum, url)) // Store sequential number + return partNum // Return assigned sequential number } // Sort by the sequential part number (index 0 of tuple) func getSortedParts() -> [(Int, URL)] { return diskParts.sorted { $0.0 < $1.0 } } - + // Restore getTotalParts func getTotalParts() -> Int { return partCounter @@ -363,7 +366,7 @@ struct DownloadStats { // Renamed struct struct UploadStats { let totalBytes: Int64 - let uploadedBytes: Int64 // Renamed + let uploadedBytes: Int64 // Renamed let elapsedTime: TimeInterval let averageSpeed: Double let peakSpeed: Double @@ -391,9 +394,13 @@ struct UploadStats { let hours = Int(seconds) / 3600 let minutes = (Int(seconds) % 3600) / 60 let secs = Int(seconds) % 60 - if hours > 0 { return String(format: "%d hours, %d minutes, %d seconds", hours, minutes, secs) } - else if minutes > 0 { return String(format: "%d minutes, %d seconds", minutes, secs) } - else { return String(format: "%d seconds", secs) } + if hours > 0 { + return String(format: "%d hours, %d minutes, %d seconds", hours, minutes, secs) + } else if minutes > 0 { + return String(format: "%d minutes, %d seconds", minutes, secs) + } else { + return String(format: "%d seconds", secs) + } } } @@ -408,15 +415,15 @@ actor TaskCounter { class ImageContainerRegistry: @unchecked Sendable { private let registry: String private let organization: String - private let downloadProgress = ProgressTracker() // Renamed for clarity - private let uploadProgress = UploadProgressTracker() // Added upload tracker + private let downloadProgress = ProgressTracker() // Renamed for clarity + private let uploadProgress = UploadProgressTracker() // Added upload tracker private let cacheDirectory: URL private let downloadLock = NSLock() private var activeDownloads: [String] = [] private let cachingEnabled: Bool // Constants for zero-skipping write logic - private static let holeGranularityBytes = 4 * 1024 * 1024 // 4MB block size for checking zeros + private static let holeGranularityBytes = 4 * 1024 * 1024 // 4MB block size for checking zeros private static let zeroChunk = Data(count: holeGranularityBytes) // Add the createProgressBar function here as a private method @@ -768,9 +775,7 @@ class ImageContainerRegistry: @unchecked Sendable { ) let counter = TaskCounter() - // Remove totalDiskParts - // var totalDiskParts: Int? = nil - var lz4LayerCount = 0 // Count lz4 layers found + var lz4LayerCount = 0 // Count lz4 layers found try await withThrowingTaskGroup(of: Int64.self) { group in for layer in manifest.layers { @@ -785,45 +790,57 @@ class ImageContainerRegistry: @unchecked Sendable { // Identify disk parts by media type if layer.mediaType == "application/octet-stream+lz4" { - // --- Handle LZ4 Disk Part Layer --- - lz4LayerCount += 1 // Increment count - let currentPartNum = lz4LayerCount // Use the current count as the logical number for logging - + // --- Handle LZ4 Disk Part Layer --- + lz4LayerCount += 1 // Increment count + let currentPartNum = lz4LayerCount // Use the current count as the logical number for logging + let cachedLayer = getCachedLayerPath( manifestId: manifestId, digest: layer.digest) let digest = layer.digest let size = layer.size - if memoryConstrained && FileManager.default.fileExists(atPath: cachedLayer.path) { + if memoryConstrained + && FileManager.default.fileExists(atPath: cachedLayer.path) + { // Add to collector, get sequential number assigned by collector - let collectorPartNum = await diskPartsCollector.addPart(url: cachedLayer) + let collectorPartNum = await diskPartsCollector.addPart( + url: cachedLayer) // Log using the sequential number from collector for clarity if needed, or the lz4LayerCount - Logger.info("Using cached lz4 layer (part \(currentPartNum)) directly: \(cachedLayer.lastPathComponent) -> Collector #\(collectorPartNum)") + Logger.info( + "Using cached lz4 layer (part \(currentPartNum)) directly: \(cachedLayer.lastPathComponent) -> Collector #\(collectorPartNum)" + ) await downloadProgress.addProgress(Int64(size)) - continue + continue } else { // Download/Copy Path (Task Group) group.addTask { [self] in await counter.increment() let finalPath: URL if FileManager.default.fileExists(atPath: cachedLayer.path) { - let tempPartURL = tempDownloadDir.appendingPathComponent("disk.img.part.\(UUID().uuidString)") - try FileManager.default.copyItem(at: cachedLayer, to: tempPartURL) + let tempPartURL = tempDownloadDir.appendingPathComponent( + "disk.img.part.\(UUID().uuidString)") + try FileManager.default.copyItem( + at: cachedLayer, to: tempPartURL) await downloadProgress.addProgress(Int64(size)) finalPath = tempPartURL } else { - let tempPartURL = tempDownloadDir.appendingPathComponent("disk.img.part.\(UUID().uuidString)") + let tempPartURL = tempDownloadDir.appendingPathComponent( + "disk.img.part.\(UUID().uuidString)") if isDownloading(digest) { - try await waitForExistingDownload(digest, cachedLayer: cachedLayer) - if FileManager.default.fileExists(atPath: cachedLayer.path) { - try FileManager.default.copyItem(at: cachedLayer, to: tempPartURL) + try await waitForExistingDownload( + digest, cachedLayer: cachedLayer) + if FileManager.default.fileExists(atPath: cachedLayer.path) + { + try FileManager.default.copyItem( + at: cachedLayer, to: tempPartURL) await downloadProgress.addProgress(Int64(size)) finalPath = tempPartURL } else { markDownloadStarted(digest) try await self.downloadLayer( repository: "\(self.organization)/\(imageName)", - digest: digest, mediaType: layer.mediaType, token: token, + digest: digest, mediaType: layer.mediaType, + token: token, to: tempPartURL, maxRetries: 5, progress: downloadProgress, manifestId: manifestId ) @@ -833,7 +850,8 @@ class ImageContainerRegistry: @unchecked Sendable { markDownloadStarted(digest) try await self.downloadLayer( repository: "\(self.organization)/\(imageName)", - digest: digest, mediaType: layer.mediaType, token: token, + digest: digest, mediaType: layer.mediaType, + token: token, to: tempPartURL, maxRetries: 5, progress: downloadProgress, manifestId: manifestId ) @@ -841,15 +859,18 @@ class ImageContainerRegistry: @unchecked Sendable { } } // Add to collector, get sequential number assigned by collector - let collectorPartNum = await diskPartsCollector.addPart(url: finalPath) + let collectorPartNum = await diskPartsCollector.addPart( + url: finalPath) // Log using the sequential number from collector - Logger.info("Assigned path for lz4 layer (part \(currentPartNum)): \(finalPath.lastPathComponent) -> Collector #\(collectorPartNum)") + Logger.info( + "Assigned path for lz4 layer (part \(currentPartNum)): \(finalPath.lastPathComponent) -> Collector #\(collectorPartNum)" + ) await counter.decrement() return Int64(size) } } } else { - // --- Handle Non-Disk-Part Layer --- + // --- Handle Non-Disk-Part Layer --- let mediaType = layer.mediaType let digest = layer.digest let size = layer.size @@ -858,39 +879,42 @@ class ImageContainerRegistry: @unchecked Sendable { let outputURL: URL switch mediaType { case "application/vnd.oci.image.layer.v1.tar", - "application/octet-stream+gzip": // Might be compressed disk.img single file? - outputURL = tempDownloadDir.appendingPathComponent("disk.img") + "application/octet-stream+gzip": // Might be compressed disk.img single file? + outputURL = tempDownloadDir.appendingPathComponent("disk.img") case "application/vnd.oci.image.config.v1+json": outputURL = tempDownloadDir.appendingPathComponent("config.json") - case "application/octet-stream": // Could be nvram or uncompressed single disk.img - // Heuristic: If a config.json already exists or is expected, assume this is nvram. - // This might need refinement if single disk images use octet-stream. - if manifest.config != nil { + case "application/octet-stream": // Could be nvram or uncompressed single disk.img + // Heuristic: If a config.json already exists or is expected, assume this is nvram. + // This might need refinement if single disk images use octet-stream. + if manifest.config != nil { outputURL = tempDownloadDir.appendingPathComponent("nvram.bin") - } else { + } else { // Assume it's a single-file disk image if no config layer is present outputURL = tempDownloadDir.appendingPathComponent("disk.img") - } + } default: - Logger.info("Skipping unsupported layer media type: \(mediaType)") - continue // Skip to the next layer + Logger.info("Skipping unsupported layer media type: \(mediaType)") + continue // Skip to the next layer } // Add task to download/copy the non-disk-part layer group.addTask { [self] in await counter.increment() - let cachedLayer = getCachedLayerPath(manifestId: manifestId, digest: digest) + let cachedLayer = getCachedLayerPath( + manifestId: manifestId, digest: digest) if FileManager.default.fileExists(atPath: cachedLayer.path) { try FileManager.default.copyItem(at: cachedLayer, to: outputURL) await downloadProgress.addProgress(Int64(size)) } else { if isDownloading(digest) { - try await waitForExistingDownload(digest, cachedLayer: cachedLayer) + try await waitForExistingDownload( + digest, cachedLayer: cachedLayer) if FileManager.default.fileExists(atPath: cachedLayer.path) { - try FileManager.default.copyItem(at: cachedLayer, to: outputURL) + try FileManager.default.copyItem( + at: cachedLayer, to: outputURL) await downloadProgress.addProgress(Int64(size)) - await counter.decrement() // Decrement before returning + await counter.decrement() // Decrement before returning return Int64(size) } } @@ -908,304 +932,62 @@ class ImageContainerRegistry: @unchecked Sendable { return Int64(size) } } - } // End for layer in manifest.layers + } // End for layer in manifest.layers // Wait for remaining tasks for try await _ in group {} - } // End TaskGroup - - // --- Safely retrieve parts AFTER TaskGroup --- - let diskParts = await diskPartsCollector.getSortedParts() // Already sorted by logicalPartNum - // Check if totalDiskParts was set (meaning at least one lz4 layer was processed) - // Get total parts from the collector - let totalPartsFromCollector = await diskPartsCollector.getTotalParts() - // Change guard to if for logging only, as the later if condition handles the logic - if totalPartsFromCollector == 0 { - // If totalParts is 0, it means no layers matched the lz4 format. - Logger.info("No lz4 disk part layers found. Assuming single-part image or non-lz4 parts.") - // Reassembly logic below will be skipped if diskParts is empty. - // Explicitly set totalParts to 0 to prevent entering the reassembly block if diskParts might somehow be non-empty but totalParts was 0 - // This ensures consistency if the collector logic changes. - } - Logger.info("Finished processing layers. Found \(diskParts.count) disk parts to reassemble (Total Lz4 Layers: \(totalPartsFromCollector)).") - // --- End retrieving parts --- - - // Add detailed logging for debugging - Logger.info("Disk part numbers collected and sorted: \(diskParts.map { $0.0 })") - - Logger.info("") // New line after progress + } // End TaskGroup // Display download statistics let stats = await downloadProgress.getDownloadStats() + Logger.info("") // New line after progress Logger.info(stats.formattedSummary()) - // Parse config.json to get uncompressed size *before* reassembly - let configURL = tempDownloadDir.appendingPathComponent("config.json") - let uncompressedSize = getUncompressedSizeFromConfig(configPath: configURL) - - // Now also try to get disk size from VM config if OCI annotation not found - var vmConfigDiskSize: UInt64? = nil - if uncompressedSize == nil && FileManager.default.fileExists(atPath: configURL.path) { - do { - let configData = try Data(contentsOf: configURL) - let decoder = JSONDecoder() - if let vmConfig = try? decoder.decode(VMConfig.self, from: configData) { - vmConfigDiskSize = vmConfig.diskSize - if let size = vmConfigDiskSize { - Logger.info("Found diskSize from VM config.json: \(size) bytes") - } - } - } catch { - Logger.error("Failed to parse VM config.json for diskSize: \(error)") - } - } - - // Force explicit use - if uncompressedSize != nil { - Logger.info( - "Will use uncompressed size from annotation for sparse file: \(uncompressedSize!) bytes" - ) - } else if vmConfigDiskSize != nil { - Logger.info( - "Will use diskSize from VM config for sparse file: \(vmConfigDiskSize!) bytes") - } - - // Handle disk parts if present - if !diskParts.isEmpty && totalPartsFromCollector > 0 { - // Use totalPartsFromCollector here - Logger.info("Reassembling \(totalPartsFromCollector) disk image parts using sparse file technique...") - let outputURL = tempVMDir.appendingPathComponent("disk.img") - - // Wrap setup in do-catch for better error reporting - let outputHandle: FileHandle - do { - // 1. Ensure parent directory exists - try FileManager.default.createDirectory( - at: outputURL.deletingLastPathComponent(), withIntermediateDirectories: true - ) - - // 2. Explicitly create the file first, removing old one if needed - if FileManager.default.fileExists(atPath: outputURL.path) { - try FileManager.default.removeItem(at: outputURL) - } - guard FileManager.default.createFile(atPath: outputURL.path, contents: nil) - else { - throw PullError.fileCreationFailed(outputURL.path) - } - - // 3. Now open the handle for writing - outputHandle = try FileHandle(forWritingTo: outputURL) - - } catch { - // Catch errors during directory/file creation or handle opening - Logger.error( - "Failed during setup for disk image reassembly: \(error.localizedDescription)", - metadata: ["path": outputURL.path]) - throw PullError.reassemblySetupFailed( - path: outputURL.path, underlyingError: error) - } - - // Calculate expected size from the manifest layers (sum of compressed parts - for logging only now) - // Filter based on the correct media type now - let expectedCompressedTotalSize = UInt64( - manifest.layers.filter { $0.mediaType == "application/octet-stream+lz4" }.reduce(0) - { $0 + $1.size } - ) - Logger.info( - "Total compressed parts size: \(ByteCountFormatter.string(fromByteCount: Int64(expectedCompressedTotalSize), countStyle: .file))" - ) - - // Calculate fallback size (sum of compressed parts) - let _: UInt64 = diskParts.reduce(UInt64(0)) { - (acc: UInt64, element) -> UInt64 in - let fileSize = - (try? FileManager.default.attributesOfItem(atPath: element.1.path)[.size] - as? UInt64 ?? 0) ?? 0 - return acc + fileSize - } - - // Use: annotation size > VM config diskSize > fallback size - let sizeForTruncate: UInt64 - if let size = uncompressedSize { - Logger.info("Using uncompressed size from annotation: \(size) bytes") - sizeForTruncate = size - } else if let size = vmConfigDiskSize { - Logger.info("Using diskSize from VM config: \(size) bytes") - sizeForTruncate = size - } else { - Logger.error( - "Missing both uncompressed size annotation and VM config diskSize for multi-part image." - ) - throw PullError.missingUncompressedSizeAnnotation - } - - defer { try? outputHandle.close() } - - // Set the file size without writing data (creates a sparse file) - try outputHandle.truncate(atOffset: sizeForTruncate) - - // Verify the sparse file was created with the correct size - let initialSize = - (try? FileManager.default.attributesOfItem(atPath: outputURL.path)[.size] - as? UInt64) ?? 0 - Logger.info( - "Sparse file initialized with size: \(ByteCountFormatter.string(fromByteCount: Int64(initialSize), countStyle: .file))" - ) - - // Add a simple test pattern at the beginning and end of the file to verify it's writable - try outputHandle.seek(toOffset: 0) - let testPattern = "LUME_TEST_PATTERN".data(using: .utf8)! - try outputHandle.write(contentsOf: testPattern) - - try outputHandle.seek(toOffset: sizeForTruncate - UInt64(testPattern.count)) - try outputHandle.write(contentsOf: testPattern) - try outputHandle.synchronize() - - Logger.info("Test patterns written to sparse file. File is ready for writing.") - - var reassemblyProgressLogger = ProgressLogger(threshold: 0.05) - var currentOffset: UInt64 = 0 // Track position in the final *decompressed* file - - // Iterate using the reliable totalParts count from media type - // Use totalPartsFromCollector for the loop range - for partNum in 1...totalPartsFromCollector { - // Find the part URL from our collected parts using the logical partNum - guard let partInfo = diskParts.first(where: { $0.0 == partNum }) else { - // This error should now be less likely, but good to keep - Logger.error("Missing required part number \(partNum) in collected parts during reassembly.") - // Add current state log on error - Logger.error("Current disk part numbers available: \(diskParts.map { $0.0 })") - throw PullError.missingPart(partNum) - } - let partURL = partInfo.1 // Get the URL from the tuple - - Logger.info( - "Processing part \(partNum) of \(totalPartsFromCollector): \(partURL.lastPathComponent)") - - // Seek to the correct offset in the output sparse file - try outputHandle.seek(toOffset: currentOffset) - - // Check if this chunk might be all zeros (sparse data) by sampling the compressed data - // Skip this check for now as it's an optimization we can add later if needed - let isLikelySparse = false - - // Always attempt decompression using decompressChunkAndWriteSparse for LZ4 parts - if isLikelySparse { - // For sparse chunks, we don't need to write anything - just advance the offset - // We determine the uncompressed size from the chunk metadata or estimation - - // For now, we'll still decompress to ensure correct behavior, and optimize later - Logger.info("Chunk appears to be sparse, but decompressing for reliability") - let decompressedBytesWritten = try decompressChunkAndWriteSparse( - inputPath: partURL.path, - outputHandle: outputHandle, - startOffset: currentOffset - ) - currentOffset += decompressedBytesWritten - } else { - Logger.info("Decompressing part \(partNum)") - let decompressedBytesWritten = try decompressChunkAndWriteSparse( - inputPath: partURL.path, - outputHandle: outputHandle, - startOffset: currentOffset - ) - currentOffset += decompressedBytesWritten - } - - reassemblyProgressLogger.logProgress( - current: Double(currentOffset) / Double(sizeForTruncate), - context: "Reassembling" - ) - - // Ensure data is written before processing next part - try outputHandle.synchronize() - } - - // Finalize progress, close handle (done by defer) - reassemblyProgressLogger.logProgress(current: 1.0, context: "Reassembly Complete") - Logger.info("") // Newline - - // Optimize sparseness after completing reassembly - try outputHandle.close() // Close handle to ensure all data is flushed - - // Verify final size - let finalSize = - (try? FileManager.default.attributesOfItem(atPath: outputURL.path)[.size] - as? UInt64) ?? 0 - Logger.info( - "Final disk image size: \(ByteCountFormatter.string(fromByteCount: Int64(finalSize), countStyle: .file))" - ) - - // Optimize sparseness if on macOS - if FileManager.default.fileExists(atPath: "/bin/cp") { - Logger.info("Optimizing sparse file representation...") - let optimizedPath = outputURL.path + ".optimized" - - let process = Process() - process.executableURL = URL(fileURLWithPath: "/bin/cp") - process.arguments = ["-c", outputURL.path, optimizedPath] - - do { - try process.run() - process.waitUntilExit() - - if process.terminationStatus == 0 { - // Get size of optimized file - let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] as? UInt64) ?? 0 - let originalUsage = getActualDiskUsage(path: outputURL.path) - let optimizedUsage = getActualDiskUsage(path: optimizedPath) - - Logger.info( - "Sparse optimization results: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))" - ) - - // Replace the original with the optimized version - try FileManager.default.removeItem(at: outputURL) - try FileManager.default.moveItem(at: URL(fileURLWithPath: optimizedPath), to: outputURL) - Logger.info("Replaced with optimized sparse version") - } else { - Logger.info("Sparse optimization failed, using original file") - try? FileManager.default.removeItem(atPath: optimizedPath) - } - } catch { - Logger.info("Error during sparse optimization: \(error.localizedDescription)") - try? FileManager.default.removeItem(atPath: optimizedPath) - } - } - - if finalSize != sizeForTruncate { - Logger.info( - "Warning: Final reported size (\(finalSize) bytes) differs from expected size (\(sizeForTruncate) bytes), but this doesn't affect functionality" - ) - } - - Logger.info("Disk image reassembly completed") + // Now that we've downloaded everything to the cache, use copyFromCache to create final VM files + if cachingEnabled { + Logger.info("Using copyFromCache method to properly preserve partition tables") + try await copyFromCache(manifest: manifest, manifestId: manifestId, to: tempVMDir) } else { - // Copy single disk image if it exists + // If caching is disabled, just copy files directly to tempVMDir + Logger.info("Caching disabled - copying downloaded files directly to VM directory") + + // Copy non-disk files first + for file in ["config.json", "nvram.bin"] { + let sourceURL = tempDownloadDir.appendingPathComponent(file) + if FileManager.default.fileExists(atPath: sourceURL.path) { + try FileManager.default.copyItem( + at: sourceURL, + to: tempVMDir.appendingPathComponent(file) + ) + } + } + + // For the disk image, we have two cases - either a single file or parts let diskURL = tempDownloadDir.appendingPathComponent("disk.img") if FileManager.default.fileExists(atPath: diskURL.path) { + // Single file disk image try FileManager.default.copyItem( at: diskURL, to: tempVMDir.appendingPathComponent("disk.img") ) + Logger.info("Copied single disk.img file to VM directory") + } else { + // Multiple parts case - use the partitioned disk.img from reassembly + let diskParts = await diskPartsCollector.getSortedParts() + if !diskParts.isEmpty { + Logger.info("Using most recently assembled disk image for VM") + let assembledDiskURL = tempVMDir.appendingPathComponent("disk.img") + if FileManager.default.fileExists(atPath: assembledDiskURL.path) { + Logger.info("Assembled disk.img already exists in VM directory") + } else { + Logger.error( + "Could not find assembled disk image - VM may not boot properly") + } + } else { + Logger.error("No disk image found - VM may not boot properly") + } } } - - // Copy config and nvram files if they exist - for file in ["config.json", "nvram.bin"] { - let sourceURL = tempDownloadDir.appendingPathComponent(file) - if FileManager.default.fileExists(atPath: sourceURL.path) { - try FileManager.default.copyItem( - at: sourceURL, - to: tempVMDir.appendingPathComponent(file) - ) - } - } - } - - // Simulate cache pull behavior if this is a first pull - if !cachingEnabled || !validateCache(manifest: manifest, manifestId: manifestId) { - try simulateCachePull(tempVMDir: tempVMDir) } // Only move to final location once everything is complete @@ -1242,10 +1024,10 @@ class ImageContainerRegistry: @unchecked Sendable { private func createDiskImageFromSource( sourceURL: URL, // Source data to decompress destinationURL: URL, // Where to create the disk image - diskSize: UInt64 // Total size for the sparse file + diskSize: UInt64 // Total size for the sparse file ) throws { Logger.info("Creating sparse disk image...") - + // Create empty destination file if FileManager.default.fileExists(atPath: destinationURL.path) { try FileManager.default.removeItem(at: destinationURL) @@ -1253,11 +1035,11 @@ class ImageContainerRegistry: @unchecked Sendable { guard FileManager.default.createFile(atPath: destinationURL.path, contents: nil) else { throw PullError.fileCreationFailed(destinationURL.path) } - + // Create sparse file let outputHandle = try FileHandle(forWritingTo: destinationURL) try outputHandle.truncate(atOffset: diskSize) - + // Write test patterns at beginning and end Logger.info("Writing test patterns to verify writability...") let testPattern = "LUME_TEST_PATTERN".data(using: .utf8)! @@ -1266,7 +1048,7 @@ class ImageContainerRegistry: @unchecked Sendable { try outputHandle.seek(toOffset: diskSize - UInt64(testPattern.count)) try outputHandle.write(contentsOf: testPattern) try outputHandle.synchronize() - + // Decompress the source data at offset 0 Logger.info("Decompressing source data...") let bytesWritten = try decompressChunkAndWriteSparse( @@ -1274,57 +1056,62 @@ class ImageContainerRegistry: @unchecked Sendable { outputHandle: outputHandle, startOffset: 0 ) - Logger.info("Decompressed \(ByteCountFormatter.string(fromByteCount: Int64(bytesWritten), countStyle: .file)) of data") - + Logger.info( + "Decompressed \(ByteCountFormatter.string(fromByteCount: Int64(bytesWritten), countStyle: .file)) of data" + ) + // Ensure data is written and close handle try outputHandle.synchronize() try outputHandle.close() - + // Run sync to flush filesystem let syncProcess = Process() syncProcess.executableURL = URL(fileURLWithPath: "/bin/sync") try syncProcess.run() syncProcess.waitUntilExit() - + // Optimize with cp -c if FileManager.default.fileExists(atPath: "/bin/cp") { Logger.info("Optimizing sparse file representation...") let optimizedPath = destinationURL.path + ".optimized" - + let process = Process() process.executableURL = URL(fileURLWithPath: "/bin/cp") process.arguments = ["-c", destinationURL.path, optimizedPath] - + try process.run() process.waitUntilExit() - + if process.terminationStatus == 0 { // Get optimization results - let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] as? UInt64) ?? 0 + let optimizedSize = + (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] + as? UInt64) ?? 0 let originalUsage = getActualDiskUsage(path: destinationURL.path) let optimizedUsage = getActualDiskUsage(path: optimizedPath) - + Logger.info( "Sparse optimization results: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))" ) - + // Replace original with optimized try FileManager.default.removeItem(at: destinationURL) - try FileManager.default.moveItem(at: URL(fileURLWithPath: optimizedPath), to: destinationURL) + try FileManager.default.moveItem( + at: URL(fileURLWithPath: optimizedPath), to: destinationURL) Logger.info("Replaced with optimized sparse version") } else { Logger.info("Sparse optimization failed, using original file") try? FileManager.default.removeItem(atPath: optimizedPath) } } - + // Set permissions to 0644 let chmodProcess = Process() chmodProcess.executableURL = URL(fileURLWithPath: "/bin/chmod") chmodProcess.arguments = ["0644", destinationURL.path] try chmodProcess.run() chmodProcess.waitUntilExit() - + // Final sync let finalSyncProcess = Process() finalSyncProcess.executableURL = URL(fileURLWithPath: "/bin/sync") @@ -1335,94 +1122,95 @@ class ImageContainerRegistry: @unchecked Sendable { // Function to simulate cache pull behavior for freshly downloaded images private func simulateCachePull(tempVMDir: URL) throws { Logger.info("Simulating cache pull behavior for freshly downloaded image...") - + // Find disk.img in tempVMDir let diskImgPath = tempVMDir.appendingPathComponent("disk.img") guard FileManager.default.fileExists(atPath: diskImgPath.path) else { Logger.info("No disk.img found to simulate cache pull behavior") return } - + // Get file attributes and size let attributes = try FileManager.default.attributesOfItem(atPath: diskImgPath.path) guard let diskSize = attributes[.size] as? UInt64, diskSize > 0 else { Logger.error("Could not determine disk.img size for simulation") return } - + Logger.info("Creating true disk image clone with partition table preserved...") - + // Create backup of original file let backupPath = tempVMDir.appendingPathComponent("disk.img.original") try FileManager.default.moveItem(at: diskImgPath, to: backupPath) - + // Let's first check if the original image has a partition table Logger.info("Checking if source image has a partition table...") let checkProcess = Process() checkProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") checkProcess.arguments = ["imageinfo", backupPath.path] - + let checkPipe = Pipe() checkProcess.standardOutput = checkPipe - + try checkProcess.run() checkProcess.waitUntilExit() - + let checkData = checkPipe.fileHandleForReading.readDataToEndOfFile() let checkOutput = String(data: checkData, encoding: .utf8) ?? "" Logger.info("Source image info: \(checkOutput)") - + // Try different methods in sequence until one works var success = false - + // Method 1: Use hdiutil convert to convert the image while preserving all data if !success { Logger.info("Trying hdiutil convert...") let tempPath = tempVMDir.appendingPathComponent("disk.img.temp") - + let convertProcess = Process() convertProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") convertProcess.arguments = [ - "convert", - backupPath.path, - "-format", "UDRO", // Read-only first to preserve partition table - "-o", tempPath.path + "convert", + backupPath.path, + "-format", "UDRO", // Read-only first to preserve partition table + "-o", tempPath.path, ] - + let convertOutPipe = Pipe() let convertErrPipe = Pipe() convertProcess.standardOutput = convertOutPipe convertProcess.standardError = convertErrPipe - + do { try convertProcess.run() convertProcess.waitUntilExit() - + let errData = convertErrPipe.fileHandleForReading.readDataToEndOfFile() let errOutput = String(data: errData, encoding: .utf8) ?? "" - + if convertProcess.terminationStatus == 0 { Logger.info("hdiutil convert succeeded. Converting to writable format...") // Now convert to writable format let convertBackProcess = Process() convertBackProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") convertBackProcess.arguments = [ - "convert", - tempPath.path, - "-format", "UDRW", // Read-write format - "-o", diskImgPath.path + "convert", + tempPath.path, + "-format", "UDRW", // Read-write format + "-o", diskImgPath.path, ] - + try convertBackProcess.run() convertBackProcess.waitUntilExit() - + if convertBackProcess.terminationStatus == 0 { - Logger.info("Successfully converted to writable format with partition table") + Logger.info( + "Successfully converted to writable format with partition table") success = true } else { Logger.error("hdiutil convert to writable format failed") } - + // Clean up temporary image try? FileManager.default.removeItem(at: tempPath) } else { @@ -1432,33 +1220,33 @@ class ImageContainerRegistry: @unchecked Sendable { Logger.error("Error executing hdiutil convert: \(error)") } } - + // Method 2: Try direct raw copy method if !success { Logger.info("Trying direct raw copy with dd...") - + // Create empty file first FileManager.default.createFile(atPath: diskImgPath.path, contents: nil) - + let ddProcess = Process() ddProcess.executableURL = URL(fileURLWithPath: "/bin/dd") ddProcess.arguments = [ "if=\(backupPath.path)", "of=\(diskImgPath.path)", - "bs=1m", // Large block size - "count=81920" // Ensure we copy everything (80GB+ should be sufficient) + "bs=1m", // Large block size + "count=81920", // Ensure we copy everything (80GB+ should be sufficient) ] - + let ddErrPipe = Pipe() ddProcess.standardError = ddErrPipe - + do { try ddProcess.run() ddProcess.waitUntilExit() - + let errData = ddErrPipe.fileHandleForReading.readDataToEndOfFile() let errOutput = String(data: errData, encoding: .utf8) ?? "" - + if ddProcess.terminationStatus == 0 { Logger.info("Raw dd copy completed: \(errOutput)") success = true @@ -1469,34 +1257,36 @@ class ImageContainerRegistry: @unchecked Sendable { Logger.error("Error executing dd: \(error)") } } - + // Method 3: Use a more complex approach with disk mounting if !success { Logger.info("Trying advanced disk attach/detach approach...") - + // Mount the source disk image let attachProcess = Process() attachProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") attachProcess.arguments = ["attach", backupPath.path, "-nomount"] - + let attachPipe = Pipe() attachProcess.standardOutput = attachPipe - + try attachProcess.run() attachProcess.waitUntilExit() - + let attachData = attachPipe.fileHandleForReading.readDataToEndOfFile() let attachOutput = String(data: attachData, encoding: .utf8) ?? "" - + // Extract the disk device from output (/dev/diskN) var diskDevice: String? = nil - if let diskMatch = attachOutput.range(of: "/dev/disk[0-9]+", options: .regularExpression) { + if let diskMatch = attachOutput.range( + of: "/dev/disk[0-9]+", options: .regularExpression) + { diskDevice = String(attachOutput[diskMatch]) } - + if let device = diskDevice { Logger.info("Source disk attached at \(device)") - + // Create a bootable disk image clone let createProcess = Process() createProcess.executableURL = URL(fileURLWithPath: "/usr/sbin/asr") @@ -1505,19 +1295,22 @@ class ImageContainerRegistry: @unchecked Sendable { "--source", device, "--target", diskImgPath.path, "--erase", - "--noprompt" + "--noprompt", ] - + let createPipe = Pipe() createProcess.standardOutput = createPipe - + do { try createProcess.run() createProcess.waitUntilExit() - - let createOutput = String(data: createPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" + + let createOutput = + String( + data: createPipe.fileHandleForReading.readDataToEndOfFile(), + encoding: .utf8) ?? "" Logger.info("asr output: \(createOutput)") - + if createProcess.terminationStatus == 0 { Logger.info("Successfully created bootable disk image clone!") success = true @@ -1527,7 +1320,7 @@ class ImageContainerRegistry: @unchecked Sendable { } catch { Logger.error("Error executing asr: \(error)") } - + // Always detach the disk when done let detachProcess = Process() detachProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") @@ -1538,98 +1331,102 @@ class ImageContainerRegistry: @unchecked Sendable { Logger.error("Failed to extract disk device from hdiutil attach output") } } - + // Fallback: If none of the methods worked, revert to our previous method just to ensure we have a usable image if !success { Logger.info("All specialized methods failed. Reverting to basic copy...") - + // If the disk image file exists (from a failed attempt), remove it if FileManager.default.fileExists(atPath: diskImgPath.path) { try FileManager.default.removeItem(at: diskImgPath) } - + // Attempt a basic file copy which will at least give us something to work with try FileManager.default.copyItem(at: backupPath, to: diskImgPath) } - + // Optimize sparseness if possible if FileManager.default.fileExists(atPath: "/bin/cp") { Logger.info("Optimizing sparse file representation...") let optimizedPath = diskImgPath.path + ".optimized" - + let process = Process() process.executableURL = URL(fileURLWithPath: "/bin/cp") process.arguments = ["-c", diskImgPath.path, optimizedPath] - + try process.run() process.waitUntilExit() - + if process.terminationStatus == 0 { - let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] as? UInt64) ?? 0 + let optimizedSize = + (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] + as? UInt64) ?? 0 let originalUsage = getActualDiskUsage(path: diskImgPath.path) let optimizedUsage = getActualDiskUsage(path: optimizedPath) - + Logger.info( "Sparse optimization results: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))" ) - + // Replace with optimized version try FileManager.default.removeItem(at: diskImgPath) - try FileManager.default.moveItem(at: URL(fileURLWithPath: optimizedPath), to: diskImgPath) + try FileManager.default.moveItem( + at: URL(fileURLWithPath: optimizedPath), to: diskImgPath) Logger.info("Replaced with optimized sparse version") } else { Logger.info("Sparse optimization failed, using original file") try? FileManager.default.removeItem(atPath: optimizedPath) } } - + // Set permissions to 0644 let chmodProcess = Process() chmodProcess.executableURL = URL(fileURLWithPath: "/bin/chmod") chmodProcess.arguments = ["0644", diskImgPath.path] try chmodProcess.run() chmodProcess.waitUntilExit() - + // Final sync let finalSyncProcess = Process() finalSyncProcess.executableURL = URL(fileURLWithPath: "/bin/sync") try finalSyncProcess.run() finalSyncProcess.waitUntilExit() - + // Verify the final disk image Logger.info("Verifying final disk image partition information...") let verifyProcess = Process() verifyProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") verifyProcess.arguments = ["imageinfo", diskImgPath.path] - + let verifyOutputPipe = Pipe() verifyProcess.standardOutput = verifyOutputPipe - + try verifyProcess.run() verifyProcess.waitUntilExit() - + let verifyOutputData = verifyOutputPipe.fileHandleForReading.readDataToEndOfFile() let verifyOutput = String(data: verifyOutputData, encoding: .utf8) ?? "" Logger.info("Final disk image verification:\n\(verifyOutput)") - + // Clean up backup file try FileManager.default.removeItem(at: backupPath) - - Logger.info("Cache pull simulation completed successfully with partition table preservation") + + Logger.info( + "Cache pull simulation completed successfully with partition table preservation") } private func copyFromCache(manifest: Manifest, manifestId: String, to destination: URL) async throws { Logger.info("Copying from cache...") - + // Define output URL and expected size variable scope here let outputURL = destination.appendingPathComponent("disk.img") - var expectedTotalSize: UInt64? = nil // Use optional to handle missing config + var expectedTotalSize: UInt64? = nil // Use optional to handle missing config // Instantiate collector let diskPartsCollector = DiskPartsCollector() - var lz4LayerCount = 0 // Count lz4 layers found + var lz4LayerCount = 0 // Count lz4 layers found // First identify disk parts and non-disk files for layer in manifest.layers { @@ -1637,13 +1434,14 @@ class ImageContainerRegistry: @unchecked Sendable { // Identify disk parts simply by media type if layer.mediaType == "application/octet-stream+lz4" { - lz4LayerCount += 1 // Increment count - // Add to collector. It will assign the sequential part number. - let collectorPartNum = await diskPartsCollector.addPart(url: cachedLayer) - Logger.info("Adding cached lz4 layer (part \(lz4LayerCount)) -> Collector #\(collectorPartNum): \(cachedLayer.lastPathComponent)") - } - else { - // --- Handle Non-Disk-Part Layer (from cache) --- + lz4LayerCount += 1 // Increment count + // Add to collector. It will assign the sequential part number. + let collectorPartNum = await diskPartsCollector.addPart(url: cachedLayer) + Logger.info( + "Adding cached lz4 layer (part \(lz4LayerCount)) -> Collector #\(collectorPartNum): \(cachedLayer.lastPathComponent)" + ) + } else { + // --- Handle Non-Disk-Part Layer (from cache) --- let fileName: String switch layer.mediaType { case "application/vnd.oci.image.config.v1+json": @@ -1651,12 +1449,12 @@ class ImageContainerRegistry: @unchecked Sendable { case "application/octet-stream": // Assume nvram if config layer exists, otherwise assume single disk image fileName = manifest.config != nil ? "nvram.bin" : "disk.img" - case "application/vnd.oci.image.layer.v1.tar", - "application/octet-stream+gzip": - // Assume disk image for these types as well if encountered in cache scenario - fileName = "disk.img" + case "application/vnd.oci.image.layer.v1.tar", + "application/octet-stream+gzip": + // Assume disk image for these types as well if encountered in cache scenario + fileName = "disk.img" default: - Logger.info("Skipping unsupported cached layer media type: \(layer.mediaType)") + Logger.info("Skipping unsupported cached layer media type: \(layer.mediaType)") continue } // Copy the non-disk file directly from cache to destination @@ -1667,19 +1465,20 @@ class ImageContainerRegistry: @unchecked Sendable { } } - // --- Safely retrieve parts AFTER loop --- - let diskPartSources = await diskPartsCollector.getSortedParts() // Sorted by assigned sequential number - let totalParts = await diskPartsCollector.getTotalParts() // Get total count from collector + // --- Safely retrieve parts AFTER loop --- + let diskPartSources = await diskPartsCollector.getSortedParts() // Sorted by assigned sequential number + let totalParts = await diskPartsCollector.getTotalParts() // Get total count from collector Logger.info("Found \(totalParts) lz4 disk parts in cache to reassemble.") - // --- End retrieving parts --- + // --- End retrieving parts --- // Reassemble disk parts if needed // Use the count from the collector if !diskPartSources.isEmpty { // Use totalParts from collector directly - Logger.info("Reassembling \(totalParts) disk image parts using sparse file technique...") - + Logger.info( + "Reassembling \(totalParts) disk image parts using sparse file technique...") + // Get uncompressed size from cached config file (needs to be copied first) let configURL = destination.appendingPathComponent("config.json") // Parse config.json to get uncompressed size *before* reassembly @@ -1713,24 +1512,25 @@ class ImageContainerRegistry: @unchecked Sendable { } else { // If neither is found in cache scenario, throw error as we cannot determine the size Logger.error( - "Missing both uncompressed size annotation and VM config diskSize for cached multi-part image." - + " Cannot reassemble." + "Missing both uncompressed size annotation and VM config diskSize for cached multi-part image." + + " Cannot reassemble." ) throw PullError.missingUncompressedSizeAnnotation } // Now that expectedTotalSize is guaranteed to be non-nil, proceed with setup guard let sizeForTruncate = expectedTotalSize else { - // This should not happen due to the checks above, but safety first - let nilError: Error? = nil - // Use nil-coalescing to provide a default error, appeasing the compiler - throw PullError.reassemblySetupFailed(path: outputURL.path, underlyingError: nilError ?? NoSpecificUnderlyingError()) + // This should not happen due to the checks above, but safety first + let nilError: Error? = nil + // Use nil-coalescing to provide a default error, appeasing the compiler + throw PullError.reassemblySetupFailed( + path: outputURL.path, underlyingError: nilError ?? NoSpecificUnderlyingError()) } // If we have just one disk part, use the shared function if totalParts == 1 { // Single part - use shared function - let sourceURL = diskPartSources[0].1 // Get the first source URL (index 1 of the tuple) + let sourceURL = diskPartSources[0].1 // Get the first source URL (index 1 of the tuple) try createDiskImageFromSource( sourceURL: sourceURL, destinationURL: outputURL, @@ -1742,22 +1542,30 @@ class ImageContainerRegistry: @unchecked Sendable { let outputHandle: FileHandle do { // Ensure parent directory exists - try FileManager.default.createDirectory(at: outputURL.deletingLastPathComponent(), withIntermediateDirectories: true) + try FileManager.default.createDirectory( + at: outputURL.deletingLastPathComponent(), withIntermediateDirectories: true + ) // Explicitly create the file first, removing old one if needed if FileManager.default.fileExists(atPath: outputURL.path) { try FileManager.default.removeItem(at: outputURL) } - guard FileManager.default.createFile(atPath: outputURL.path, contents: nil) else { + guard FileManager.default.createFile(atPath: outputURL.path, contents: nil) + else { throw PullError.fileCreationFailed(outputURL.path) } // Open handle for writing outputHandle = try FileHandle(forWritingTo: outputURL) // Set the file size (creates sparse file) try outputHandle.truncate(atOffset: sizeForTruncate) - Logger.info("Sparse file initialized for cache reassembly with size: \(ByteCountFormatter.string(fromByteCount: Int64(sizeForTruncate), countStyle: .file))") + Logger.info( + "Sparse file initialized for cache reassembly with size: \(ByteCountFormatter.string(fromByteCount: Int64(sizeForTruncate), countStyle: .file))" + ) } catch { - Logger.error("Failed during setup for cached disk image reassembly: \(error.localizedDescription)", metadata: ["path": outputURL.path]) - throw PullError.reassemblySetupFailed(path: outputURL.path, underlyingError: error) + Logger.error( + "Failed during setup for cached disk image reassembly: \(error.localizedDescription)", + metadata: ["path": outputURL.path]) + throw PullError.reassemblySetupFailed( + path: outputURL.path, underlyingError: error) } // Ensure handle is closed when exiting this scope @@ -1769,11 +1577,15 @@ class ImageContainerRegistry: @unchecked Sendable { // Iterate from 1 up to the total number of parts found by the collector for collectorPartNum in 1...totalParts { // Find the source URL from our collected parts using the sequential collectorPartNum - guard let sourceInfo = diskPartSources.first(where: { $0.0 == collectorPartNum }) else { - Logger.error("Missing required cached part number \(collectorPartNum) in collected parts during reassembly.") + guard + let sourceInfo = diskPartSources.first(where: { $0.0 == collectorPartNum }) + else { + Logger.error( + "Missing required cached part number \(collectorPartNum) in collected parts during reassembly." + ) throw PullError.missingPart(collectorPartNum) } - let sourceURL = sourceInfo.1 // Get URL from tuple + let sourceURL = sourceInfo.1 // Get URL from tuple // Log using the sequential collector part number Logger.info( @@ -1789,10 +1601,10 @@ class ImageContainerRegistry: @unchecked Sendable { currentOffset += decompressedBytesWritten // Update progress (using sizeForTruncate which should be available) reassemblyProgressLogger.logProgress( - current: Double(currentOffset) / Double(sizeForTruncate), - context: "Reassembling Cache") - - try outputHandle.synchronize() // Explicitly synchronize after each chunk + current: Double(currentOffset) / Double(sizeForTruncate), + context: "Reassembling Cache") + + try outputHandle.synchronize() // Explicitly synchronize after each chunk } // Finalize progress, close handle (done by defer) @@ -1806,13 +1618,13 @@ class ImageContainerRegistry: @unchecked Sendable { try outputHandle.seek(toOffset: sizeForTruncate - UInt64(testPattern.count)) try outputHandle.write(contentsOf: testPattern) try outputHandle.synchronize() - + // Ensure handle is properly synchronized before closing try outputHandle.synchronize() - + // Close handle explicitly instead of relying on defer try outputHandle.close() - + // Verify final size let finalSize = (try? FileManager.default.attributesOfItem(atPath: outputURL.path)[.size] @@ -1829,44 +1641,49 @@ class ImageContainerRegistry: @unchecked Sendable { } Logger.info("Disk image reassembly completed") - + // Optimize sparseness for cached reassembly if on macOS if FileManager.default.fileExists(atPath: "/bin/cp") { Logger.info("Optimizing sparse file representation for cached reassembly...") let optimizedPath = outputURL.path + ".optimized" - + let process = Process() process.executableURL = URL(fileURLWithPath: "/bin/cp") process.arguments = ["-c", outputURL.path, optimizedPath] - + do { try process.run() process.waitUntilExit() - + if process.terminationStatus == 0 { // Get size of optimized file - let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] as? UInt64) ?? 0 + let optimizedSize = + (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[ + .size] as? UInt64) ?? 0 let originalUsage = getActualDiskUsage(path: outputURL.path) let optimizedUsage = getActualDiskUsage(path: optimizedPath) - + Logger.info( "Sparse optimization results for cache: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))" ) - + // Replace the original with the optimized version try FileManager.default.removeItem(at: outputURL) - try FileManager.default.moveItem(at: URL(fileURLWithPath: optimizedPath), to: outputURL) + try FileManager.default.moveItem( + at: URL(fileURLWithPath: optimizedPath), to: outputURL) Logger.info("Replaced cached reassembly with optimized sparse version") } else { Logger.info("Sparse optimization failed for cache, using original file") try? FileManager.default.removeItem(atPath: optimizedPath) } } catch { - Logger.info("Error during sparse optimization for cache: \(error.localizedDescription)") + Logger.info( + "Error during sparse optimization for cache: \(error.localizedDescription)" + ) try? FileManager.default.removeItem(atPath: optimizedPath) } } - + // Set permissions to ensure consistency let chmodProcess = Process() chmodProcess.executableURL = URL(fileURLWithPath: "/bin/chmod") @@ -1880,12 +1697,16 @@ class ImageContainerRegistry: @unchecked Sendable { } private func getToken(repository: String) async throws -> String { - let encodedRepo = repository.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? repository + let encodedRepo = + repository.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? repository // Request both pull and push scope for uploads - let url = URL(string: "https://\(self.registry)/token?scope=repository:\(encodedRepo):pull,push&service=\(self.registry)")! - + let url = URL( + string: + "https://\(self.registry)/token?scope=repository:\(encodedRepo):pull,push&service=\(self.registry)" + )! + var request = URLRequest(url: url) - request.httpMethod = "GET" // Token endpoint uses GET + request.httpMethod = "GET" // Token endpoint uses GET request.setValue("application/json", forHTTPHeaderField: "Accept") // *** Add Basic Authentication Header if credentials exist *** @@ -1906,26 +1727,31 @@ class ImageContainerRegistry: @unchecked Sendable { // *** End Basic Auth addition *** let (data, response) = try await URLSession.shared.data(for: request) - + // Check response status code *before* parsing JSON guard let httpResponse = response as? HTTPURLResponse else { - throw PushError.authenticationFailed // Or a more generic network error + throw PushError.authenticationFailed // Or a more generic network error } - + guard httpResponse.statusCode == 200 else { // Log detailed error including status code and potentially response body let responseBody = String(data: data, encoding: .utf8) ?? "(Could not decode body)" - Logger.error("Token request failed with status code: \(httpResponse.statusCode). Response: \(responseBody)") + Logger.error( + "Token request failed with status code: \(httpResponse.statusCode). Response: \(responseBody)" + ) // Throw specific error based on status if needed (e.g., 401 for unauthorized) - throw PushError.authenticationFailed + throw PushError.authenticationFailed } - + let jsonResponse = try JSONSerialization.jsonObject(with: data) as? [String: Any] - guard let token = jsonResponse?["token"] as? String ?? jsonResponse?["access_token"] as? String else { + guard + let token = jsonResponse?["token"] as? String ?? jsonResponse?["access_token"] + as? String + else { Logger.error("Token not found in registry response.") throw PushError.missingToken } - + return token } @@ -2673,9 +2499,9 @@ class ImageContainerRegistry: @unchecked Sendable { // New push method public func push( - vmDirPath: String, - imageName: String, - tags: [String], + vmDirPath: String, + imageName: String, + tags: [String], chunkSizeMb: Int = 512, verbose: Bool = false, dryRun: Bool = false, @@ -2686,18 +2512,18 @@ class ImageContainerRegistry: @unchecked Sendable { metadata: [ "vm_path": vmDirPath, "imageName": imageName, - "tags": "\(tags.joined(separator: ", "))", // Log all tags + "tags": "\(tags.joined(separator: ", "))", // Log all tags "registry": registry, "organization": organization, "chunk_size": "\(chunkSizeMb)MB", "dry_run": "\(dryRun)", - "reassemble": "\(reassemble)" + "reassemble": "\(reassemble)", ]) - + // Remove tag parsing here, imageName is now passed directly // let components = image.split(separator: ":") ... // let imageTag = String(tag) - + // Get authentication token only if not in dry-run mode var token: String = "" if !dryRun { @@ -2706,17 +2532,17 @@ class ImageContainerRegistry: @unchecked Sendable { } else { Logger.info("Dry run mode: skipping authentication token request") } - + // Create working directory inside the VM folder for caching/resuming let workDir = URL(fileURLWithPath: vmDirPath).appendingPathComponent(".lume_push_cache") try FileManager.default.createDirectory(at: workDir, withIntermediateDirectories: true) Logger.info("Using push cache directory: \(workDir.path)") - + // Get VM files that need to be pushed using vmDirPath let diskPath = URL(fileURLWithPath: vmDirPath).appendingPathComponent("disk.img") let configPath = URL(fileURLWithPath: vmDirPath).appendingPathComponent("config.json") let nvramPath = URL(fileURLWithPath: vmDirPath).appendingPathComponent("nvram.bin") - + var layers: [OCIManifestLayer] = [] var uncompressedDiskSize: UInt64? = nil @@ -2724,7 +2550,7 @@ class ImageContainerRegistry: @unchecked Sendable { let cachedConfigPath = workDir.appendingPathComponent("config.json") var configDigest: String? = nil var configSize: Int? = nil - + if FileManager.default.fileExists(atPath: cachedConfigPath.path) { Logger.info("Using cached config.json") do { @@ -2734,7 +2560,8 @@ class ImageContainerRegistry: @unchecked Sendable { // Try to get uncompressed disk size from cached config if let vmConfig = try? JSONDecoder().decode(VMConfig.self, from: configData) { uncompressedDiskSize = vmConfig.diskSize - Logger.info("Found disk size in cached config: \(uncompressedDiskSize ?? 0) bytes") + Logger.info( + "Found disk size in cached config: \(uncompressedDiskSize ?? 0) bytes") } } catch { Logger.error("Failed to read cached config.json: \(error). Will re-process.") @@ -2745,20 +2572,22 @@ class ImageContainerRegistry: @unchecked Sendable { let configData = try Data(contentsOf: configPath) configDigest = "sha256:" + configData.sha256String() configSize = configData.count - try configData.write(to: cachedConfigPath) // Save to cache + try configData.write(to: cachedConfigPath) // Save to cache // Try to get uncompressed disk size from original config if let vmConfig = try? JSONDecoder().decode(VMConfig.self, from: configData) { uncompressedDiskSize = vmConfig.diskSize Logger.info("Found disk size in config: \(uncompressedDiskSize ?? 0) bytes") } } - - if var digest = configDigest, let size = configSize { // Use 'var' to modify if uploaded - if !dryRun { + + if var digest = configDigest, let size = configSize { // Use 'var' to modify if uploaded + if !dryRun { // Upload only if not in dry-run mode and blob doesn't exist - if !(try await blobExists(repository: "\(self.organization)/\(imageName)", digest: digest, token: token)) { + if !(try await blobExists( + repository: "\(self.organization)/\(imageName)", digest: digest, token: token)) + { Logger.info("Uploading config.json blob") - let configData = try Data(contentsOf: cachedConfigPath) // Read from cache for upload + let configData = try Data(contentsOf: cachedConfigPath) // Read from cache for upload digest = try await uploadBlobFromData( repository: "\(self.organization)/\(imageName)", data: configData, @@ -2769,13 +2598,14 @@ class ImageContainerRegistry: @unchecked Sendable { } } // Add config layer - layers.append(OCIManifestLayer( - mediaType: "application/vnd.oci.image.config.v1+json", - size: size, - digest: digest - )) + layers.append( + OCIManifestLayer( + mediaType: "application/vnd.oci.image.config.v1+json", + size: size, + digest: digest + )) } - + // Process nvram.bin let cachedNvramPath = workDir.appendingPathComponent("nvram.bin") var nvramDigest: String? = nil @@ -2788,47 +2618,56 @@ class ImageContainerRegistry: @unchecked Sendable { nvramDigest = "sha256:" + nvramData.sha256String() nvramSize = nvramData.count } catch { - Logger.error("Failed to read cached nvram.bin: \(error). Will re-process.") + Logger.error("Failed to read cached nvram.bin: \(error). Will re-process.") } } else if FileManager.default.fileExists(atPath: nvramPath.path) { Logger.info("Processing nvram.bin") let nvramData = try Data(contentsOf: nvramPath) nvramDigest = "sha256:" + nvramData.sha256String() nvramSize = nvramData.count - try nvramData.write(to: cachedNvramPath) // Save to cache + try nvramData.write(to: cachedNvramPath) // Save to cache } - - if var digest = nvramDigest, let size = nvramSize { // Use 'var' - if !dryRun { - // Upload only if not in dry-run mode and blob doesn't exist - if !(try await blobExists(repository: "\(self.organization)/\(imageName)", digest: digest, token: token)) { + + if var digest = nvramDigest, let size = nvramSize { // Use 'var' + if !dryRun { + // Upload only if not in dry-run mode and blob doesn't exist + if !(try await blobExists( + repository: "\(self.organization)/\(imageName)", digest: digest, token: token)) + { Logger.info("Uploading nvram.bin blob") - let nvramData = try Data(contentsOf: cachedNvramPath) // Read from cache + let nvramData = try Data(contentsOf: cachedNvramPath) // Read from cache digest = try await uploadBlobFromData( repository: "\(self.organization)/\(imageName)", data: nvramData, token: token ) } else { - Logger.info("NVRAM blob already exists on registry") + Logger.info("NVRAM blob already exists on registry") } } // Add nvram layer - layers.append(OCIManifestLayer( - mediaType: "application/octet-stream", - size: size, - digest: digest - )) + layers.append( + OCIManifestLayer( + mediaType: "application/octet-stream", + size: size, + digest: digest + )) } - + // Process disk.img if FileManager.default.fileExists(atPath: diskPath.path) { let diskAttributes = try FileManager.default.attributesOfItem(atPath: diskPath.path) let diskSize = diskAttributes[.size] as? UInt64 ?? 0 let actualDiskSize = uncompressedDiskSize ?? diskSize - Logger.info("Processing disk.img in chunks", metadata: ["disk_path": diskPath.path, "disk_size": "\(diskSize) bytes", "actual_size": "\(actualDiskSize) bytes", "chunk_size": "\(chunkSizeMb)MB"]) + Logger.info( + "Processing disk.img in chunks", + metadata: [ + "disk_path": diskPath.path, "disk_size": "\(diskSize) bytes", + "actual_size": "\(actualDiskSize) bytes", "chunk_size": "\(chunkSizeMb)MB", + ]) let chunksDir = workDir.appendingPathComponent("disk.img.parts") - try FileManager.default.createDirectory(at: chunksDir, withIntermediateDirectories: true) + try FileManager.default.createDirectory( + at: chunksDir, withIntermediateDirectories: true) let chunkSizeBytes = chunkSizeMb * 1024 * 1024 let totalChunks = Int((diskSize + UInt64(chunkSizeBytes) - 1) / UInt64(chunkSizeBytes)) Logger.info("Splitting disk into \(totalChunks) chunks") @@ -2836,58 +2675,125 @@ class ImageContainerRegistry: @unchecked Sendable { defer { try? fileHandle.close() } var pushedDiskLayers: [(index: Int, layer: OCIManifestLayer)] = [] var diskChunks: [(index: Int, path: URL, digest: String)] = [] - - try await withThrowingTaskGroup(of: (Int, OCIManifestLayer, URL, String).self) { group in + + try await withThrowingTaskGroup(of: (Int, OCIManifestLayer, URL, String).self) { + group in let maxConcurrency = 4 for chunkIndex in 0..= maxConcurrency { if let res = try await group.next() { pushedDiskLayers.append((res.0, res.1)); diskChunks.append((res.0, res.2, res.3)) } } + if chunkIndex >= maxConcurrency { + if let res = try await group.next() { + pushedDiskLayers.append((res.0, res.1)) + diskChunks.append((res.0, res.2, res.3)) + } + } group.addTask { [token, verbose, dryRun, organization, imageName] in let chunkIndex = chunkIndex let chunkPath = chunksDir.appendingPathComponent("chunk.\(chunkIndex)") - let metadataPath = chunksDir.appendingPathComponent("chunk_metadata.\(chunkIndex).json") + let metadataPath = chunksDir.appendingPathComponent( + "chunk_metadata.\(chunkIndex).json") var layer: OCIManifestLayer? = nil var finalCompressedDigest: String? = nil - if FileManager.default.fileExists(atPath: metadataPath.path), FileManager.default.fileExists(atPath: chunkPath.path) { + if FileManager.default.fileExists(atPath: metadataPath.path), + FileManager.default.fileExists(atPath: chunkPath.path) + { do { let metadataData = try Data(contentsOf: metadataPath) - let metadata = try JSONDecoder().decode(ChunkMetadata.self, from: metadataData) - Logger.info("Resuming chunk \(chunkIndex + 1)/\(totalChunks) from cache") + let metadata = try JSONDecoder().decode( + ChunkMetadata.self, from: metadataData) + Logger.info( + "Resuming chunk \(chunkIndex + 1)/\(totalChunks) from cache") finalCompressedDigest = metadata.compressedDigest - if !dryRun { if !(try await self.blobExists(repository: "\(organization)/\(imageName)", digest: metadata.compressedDigest, token: token)) { Logger.info("Uploading cached chunk \(chunkIndex + 1) blob"); _ = try await self.uploadBlobFromPath(repository: "\(organization)/\(imageName)", path: chunkPath, digest: metadata.compressedDigest, token: token) } else { Logger.info("Chunk \(chunkIndex + 1) blob already exists on registry") } } - layer = OCIManifestLayer(mediaType: "application/octet-stream+lz4", size: metadata.compressedSize, digest: metadata.compressedDigest, uncompressedSize: metadata.uncompressedSize, uncompressedContentDigest: metadata.uncompressedDigest) - } catch { Logger.info("Failed to load cached metadata/chunk for index \(chunkIndex): \(error). Re-processing."); finalCompressedDigest = nil; layer = nil } + if !dryRun { + if !(try await self.blobExists( + repository: "\(organization)/\(imageName)", + digest: metadata.compressedDigest, token: token)) + { + Logger.info("Uploading cached chunk \(chunkIndex + 1) blob") + _ = try await self.uploadBlobFromPath( + repository: "\(organization)/\(imageName)", + path: chunkPath, digest: metadata.compressedDigest, + token: token) + } else { + Logger.info( + "Chunk \(chunkIndex + 1) blob already exists on registry" + ) + } + } + layer = OCIManifestLayer( + mediaType: "application/octet-stream+lz4", + size: metadata.compressedSize, + digest: metadata.compressedDigest, + uncompressedSize: metadata.uncompressedSize, + uncompressedContentDigest: metadata.uncompressedDigest) + } catch { + Logger.info( + "Failed to load cached metadata/chunk for index \(chunkIndex): \(error). Re-processing." + ) + finalCompressedDigest = nil + layer = nil + } } if layer == nil { Logger.info("Processing chunk \(chunkIndex + 1)/\(totalChunks)") let localFileHandle = try FileHandle(forReadingFrom: diskPath) defer { try? localFileHandle.close() } try localFileHandle.seek(toOffset: UInt64(chunkIndex * chunkSizeBytes)) - let chunkData = try localFileHandle.read(upToCount: chunkSizeBytes) ?? Data() + let chunkData = + try localFileHandle.read(upToCount: chunkSizeBytes) ?? Data() let uncompressedSize = UInt64(chunkData.count) let uncompressedDigest = "sha256:" + chunkData.sha256String() - let compressedData = try (chunkData as NSData).compressed(using: .lz4) as Data + let compressedData = + try (chunkData as NSData).compressed(using: .lz4) as Data let compressedSize = compressedData.count let compressedDigest = "sha256:" + compressedData.sha256String() try compressedData.write(to: chunkPath) - let metadata = ChunkMetadata(uncompressedDigest: uncompressedDigest, uncompressedSize: uncompressedSize, compressedDigest: compressedDigest, compressedSize: compressedSize) + let metadata = ChunkMetadata( + uncompressedDigest: uncompressedDigest, + uncompressedSize: uncompressedSize, + compressedDigest: compressedDigest, compressedSize: compressedSize) let metadataData = try JSONEncoder().encode(metadata) try metadataData.write(to: metadataPath) finalCompressedDigest = compressedDigest - if !dryRun { if !(try await self.blobExists(repository: "\(organization)/\(imageName)", digest: compressedDigest, token: token)) { Logger.info("Uploading processed chunk \(chunkIndex + 1) blob"); _ = try await self.uploadBlobFromPath(repository: "\(organization)/\(imageName)", path: chunkPath, digest: compressedDigest, token: token) } else { Logger.info("Chunk \(chunkIndex + 1) blob already exists on registry (processed fresh)") } } - layer = OCIManifestLayer(mediaType: "application/octet-stream+lz4", size: compressedSize, digest: compressedDigest, uncompressedSize: uncompressedSize, uncompressedContentDigest: uncompressedDigest) + if !dryRun { + if !(try await self.blobExists( + repository: "\(organization)/\(imageName)", + digest: compressedDigest, token: token)) + { + Logger.info("Uploading processed chunk \(chunkIndex + 1) blob") + _ = try await self.uploadBlobFromPath( + repository: "\(organization)/\(imageName)", path: chunkPath, + digest: compressedDigest, token: token) + } else { + Logger.info( + "Chunk \(chunkIndex + 1) blob already exists on registry (processed fresh)" + ) + } + } + layer = OCIManifestLayer( + mediaType: "application/octet-stream+lz4", size: compressedSize, + digest: compressedDigest, uncompressedSize: uncompressedSize, + uncompressedContentDigest: uncompressedDigest) + } + guard let finalLayer = layer, let finalDigest = finalCompressedDigest else { + throw PushError.blobUploadFailed + } + if verbose { + Logger.info("Finished chunk \(chunkIndex + 1)/\(totalChunks)") } - guard let finalLayer = layer, let finalDigest = finalCompressedDigest else { throw PushError.blobUploadFailed } - if verbose { Logger.info("Finished chunk \(chunkIndex + 1)/\(totalChunks)") } return (chunkIndex, finalLayer, chunkPath, finalDigest) } } - for try await (index, layer, path, digest) in group { pushedDiskLayers.append((index, layer)); diskChunks.append((index, path, digest)) } + for try await (index, layer, path, digest) in group { + pushedDiskLayers.append((index, layer)) + diskChunks.append((index, path, digest)) + } } - layers.append(contentsOf: pushedDiskLayers.sorted { $0.index < $1.index }.map { $0.layer }) + layers.append( + contentsOf: pushedDiskLayers.sorted { $0.index < $1.index }.map { $0.layer }) diskChunks.sort { $0.index < $1.index } Logger.info("All disk chunks processed successfully") - // --- Calculate Total Upload Size & Initialize Tracker --- + // --- Calculate Total Upload Size & Initialize Tracker --- if !dryRun { var totalUploadSizeBytes: Int64 = 0 var totalUploadFiles: Int = 0 @@ -2898,49 +2804,60 @@ class ImageContainerRegistry: @unchecked Sendable { } // Add nvram size if it exists if let size = nvramSize { - totalUploadSizeBytes += Int64(size) - totalUploadFiles += 1 + totalUploadSizeBytes += Int64(size) + totalUploadFiles += 1 } // Add sizes of all compressed disk chunks - let allChunkSizes = diskChunks.compactMap { try? FileManager.default.attributesOfItem(atPath: $0.path.path)[.size] as? Int64 ?? 0 } + let allChunkSizes = diskChunks.compactMap { + try? FileManager.default.attributesOfItem(atPath: $0.path.path)[.size] as? Int64 + ?? 0 + } totalUploadSizeBytes += allChunkSizes.reduce(0, +) - totalUploadFiles += totalChunks // Use totalChunks calculated earlier - + totalUploadFiles += totalChunks // Use totalChunks calculated earlier + if totalUploadSizeBytes > 0 { - Logger.info("Initializing upload progress: \(totalUploadFiles) files, total size: \(ByteCountFormatter.string(fromByteCount: totalUploadSizeBytes, countStyle: .file))") + Logger.info( + "Initializing upload progress: \(totalUploadFiles) files, total size: \(ByteCountFormatter.string(fromByteCount: totalUploadSizeBytes, countStyle: .file))" + ) await uploadProgress.setTotal(totalUploadSizeBytes, files: totalUploadFiles) // Print initial progress bar - print("[░░░░░░░░░░░░░░░░░░░░] 0% (0/\(totalUploadFiles)) | Initializing upload... | ETA: calculating... ") - fflush(stdout) - } else { - Logger.info("No files marked for upload.") - } + print( + "[░░░░░░░░░░░░░░░░░░░░] 0% (0/\(totalUploadFiles)) | Initializing upload... | ETA: calculating... " + ) + fflush(stdout) + } else { + Logger.info("No files marked for upload.") + } } - // --- End Size Calculation & Init --- + // --- End Size Calculation & Init --- // Perform reassembly verification if requested in dry-run mode if dryRun && reassemble { Logger.info("=== REASSEMBLY MODE ===") Logger.info("Reassembling chunks to verify integrity...") let reassemblyDir = workDir.appendingPathComponent("reassembly") - try FileManager.default.createDirectory(at: reassemblyDir, withIntermediateDirectories: true) + try FileManager.default.createDirectory( + at: reassemblyDir, withIntermediateDirectories: true) let reassembledFile = reassemblyDir.appendingPathComponent("reassembled_disk.img") - + // Pre-allocate a sparse file with the correct size - Logger.info("Pre-allocating sparse file of \(ByteCountFormatter.string(fromByteCount: Int64(actualDiskSize), countStyle: .file))...") + Logger.info( + "Pre-allocating sparse file of \(ByteCountFormatter.string(fromByteCount: Int64(actualDiskSize), countStyle: .file))..." + ) if FileManager.default.fileExists(atPath: reassembledFile.path) { try FileManager.default.removeItem(at: reassembledFile) } - guard FileManager.default.createFile(atPath: reassembledFile.path, contents: nil) else { + guard FileManager.default.createFile(atPath: reassembledFile.path, contents: nil) + else { throw PushError.fileCreationFailed(reassembledFile.path) } - + let outputHandle = try FileHandle(forWritingTo: reassembledFile) defer { try? outputHandle.close() } - + // Set the file size without writing data (creates a sparse file) try outputHandle.truncate(atOffset: actualDiskSize) - + // Add test patterns at start and end to verify writability let testPattern = "LUME_TEST_PATTERN".data(using: .utf8)! try outputHandle.seek(toOffset: 0) @@ -2948,217 +2865,266 @@ class ImageContainerRegistry: @unchecked Sendable { try outputHandle.seek(toOffset: actualDiskSize - UInt64(testPattern.count)) try outputHandle.write(contentsOf: testPattern) try outputHandle.synchronize() - + Logger.info("Test patterns written to sparse file. File is ready for writing.") - + // Track reassembly progress var reassemblyProgressLogger = ProgressLogger(threshold: 0.05) var currentOffset: UInt64 = 0 - + // Process each chunk in order for (index, cachedChunkPath, _) in diskChunks.sorted(by: { $0.index < $1.index }) { - Logger.info("Decompressing & writing part \(index + 1)/\(diskChunks.count): \(cachedChunkPath.lastPathComponent) at offset \(currentOffset)...") - + Logger.info( + "Decompressing & writing part \(index + 1)/\(diskChunks.count): \(cachedChunkPath.lastPathComponent) at offset \(currentOffset)..." + ) + // Always seek to the correct position try outputHandle.seek(toOffset: currentOffset) - + // Decompress and write the chunk let decompressedBytesWritten = try decompressChunkAndWriteSparse( inputPath: cachedChunkPath.path, outputHandle: outputHandle, startOffset: currentOffset ) - + currentOffset += decompressedBytesWritten reassemblyProgressLogger.logProgress( current: Double(currentOffset) / Double(actualDiskSize), context: "Reassembling" ) - + // Ensure data is written before processing next part try outputHandle.synchronize() } - + // Finalize progress reassemblyProgressLogger.logProgress(current: 1.0, context: "Reassembly Complete") Logger.info("") // Newline - + // Close handle before post-processing try outputHandle.close() - + // Optimize sparseness if on macOS let optimizedFile = reassemblyDir.appendingPathComponent("optimized_disk.img") if FileManager.default.fileExists(atPath: "/bin/cp") { Logger.info("Optimizing sparse file representation...") - + let process = Process() process.executableURL = URL(fileURLWithPath: "/bin/cp") process.arguments = ["-c", reassembledFile.path, optimizedFile.path] - + do { try process.run() process.waitUntilExit() - + if process.terminationStatus == 0 { // Get sizes of original and optimized files - let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedFile.path)[.size] as? UInt64) ?? 0 + let optimizedSize = + (try? FileManager.default.attributesOfItem( + atPath: optimizedFile.path)[.size] as? UInt64) ?? 0 let originalUsage = getActualDiskUsage(path: reassembledFile.path) let optimizedUsage = getActualDiskUsage(path: optimizedFile.path) - + Logger.info( "Sparse optimization results: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))" ) - + // Replace original with optimized version try FileManager.default.removeItem(at: reassembledFile) try FileManager.default.moveItem(at: optimizedFile, to: reassembledFile) Logger.info("Using sparse-optimized file for verification") } else { - Logger.info("Sparse optimization failed, using original file for verification") + Logger.info( + "Sparse optimization failed, using original file for verification") try? FileManager.default.removeItem(at: optimizedFile) } } catch { - Logger.info("Error during sparse optimization: \(error.localizedDescription)") + Logger.info( + "Error during sparse optimization: \(error.localizedDescription)") try? FileManager.default.removeItem(at: optimizedFile) } } - + // Verification step Logger.info("Verifying reassembled file...") let originalSize = diskSize let originalDigest = calculateSHA256(filePath: diskPath.path) - let reassembledAttributes = try FileManager.default.attributesOfItem(atPath: reassembledFile.path) + let reassembledAttributes = try FileManager.default.attributesOfItem( + atPath: reassembledFile.path) let reassembledSize = reassembledAttributes[.size] as? UInt64 ?? 0 let reassembledDigest = calculateSHA256(filePath: reassembledFile.path) - + // Check actual disk usage let originalActualSize = getActualDiskUsage(path: diskPath.path) let reassembledActualSize = getActualDiskUsage(path: reassembledFile.path) - + // Report results Logger.info("Results:") - Logger.info(" Original size: \(ByteCountFormatter.string(fromByteCount: Int64(originalSize), countStyle: .file)) (\(originalSize) bytes)") - Logger.info(" Reassembled size: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledSize), countStyle: .file)) (\(reassembledSize) bytes)") + Logger.info( + " Original size: \(ByteCountFormatter.string(fromByteCount: Int64(originalSize), countStyle: .file)) (\(originalSize) bytes)" + ) + Logger.info( + " Reassembled size: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledSize), countStyle: .file)) (\(reassembledSize) bytes)" + ) Logger.info(" Original digest: \(originalDigest)") Logger.info(" Reassembled digest: \(reassembledDigest)") - Logger.info(" Original: Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(originalSize), countStyle: .file)), Actual disk usage: \(ByteCountFormatter.string(fromByteCount: Int64(originalActualSize), countStyle: .file))") - Logger.info(" Reassembled: Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledSize), countStyle: .file)), Actual disk usage: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledActualSize), countStyle: .file))") - + Logger.info( + " Original: Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(originalSize), countStyle: .file)), Actual disk usage: \(ByteCountFormatter.string(fromByteCount: Int64(originalActualSize), countStyle: .file))" + ) + Logger.info( + " Reassembled: Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledSize), countStyle: .file)), Actual disk usage: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledActualSize), countStyle: .file))" + ) + // Determine if verification was successful if originalDigest == reassembledDigest { Logger.info("✅ VERIFICATION SUCCESSFUL: Files are identical") } else { Logger.info("❌ VERIFICATION FAILED: Files differ") - + if originalSize != reassembledSize { - Logger.info(" Size mismatch: Original \(originalSize) bytes, Reassembled \(reassembledSize) bytes") + Logger.info( + " Size mismatch: Original \(originalSize) bytes, Reassembled \(reassembledSize) bytes" + ) } - + // Check sparse file characteristics Logger.info("Attempting to identify differences...") - Logger.info("NOTE: This might be a sparse file issue. The content may be identical, but sparse regions") - Logger.info(" may be handled differently between the original and reassembled files.") - + Logger.info( + "NOTE: This might be a sparse file issue. The content may be identical, but sparse regions" + ) + Logger.info( + " may be handled differently between the original and reassembled files." + ) + if originalActualSize > 0 { - let diffPercentage = ((Double(reassembledActualSize) - Double(originalActualSize)) / Double(originalActualSize)) * 100.0 - Logger.info(" Disk usage difference: \(String(format: "%.2f", diffPercentage))%") - + let diffPercentage = + ((Double(reassembledActualSize) - Double(originalActualSize)) + / Double(originalActualSize)) * 100.0 + Logger.info( + " Disk usage difference: \(String(format: "%.2f", diffPercentage))%") + if diffPercentage < -40 { - Logger.info(" ⚠️ WARNING: Reassembled disk uses significantly less space (>40% difference).") - Logger.info(" This indicates sparse regions weren't properly preserved and may affect VM functionality.") + Logger.info( + " ⚠️ WARNING: Reassembled disk uses significantly less space (>40% difference)." + ) + Logger.info( + " This indicates sparse regions weren't properly preserved and may affect VM functionality." + ) } else if diffPercentage < -10 { - Logger.info(" ⚠️ WARNING: Reassembled disk uses less space (10-40% difference).") - Logger.info(" Some sparse regions may not be properly preserved but VM might still function correctly.") + Logger.info( + " ⚠️ WARNING: Reassembled disk uses less space (10-40% difference)." + ) + Logger.info( + " Some sparse regions may not be properly preserved but VM might still function correctly." + ) } else if diffPercentage > 10 { - Logger.info(" ⚠️ WARNING: Reassembled disk uses more space (>10% difference).") - Logger.info(" This is unusual and may indicate improper sparse file handling.") + Logger.info( + " ⚠️ WARNING: Reassembled disk uses more space (>10% difference).") + Logger.info( + " This is unusual and may indicate improper sparse file handling.") } else { - Logger.info(" ✓ Disk usage difference is minimal (<10%). VM likely to function correctly.") + Logger.info( + " ✓ Disk usage difference is minimal (<10%). VM likely to function correctly." + ) } } - + // Offer recovery option if originalDigest != reassembledDigest { Logger.info("") Logger.info("===== ATTEMPTING RECOVERY ACTION =====") - Logger.info("Since verification failed, trying direct copy as a fallback method.") - + Logger.info( + "Since verification failed, trying direct copy as a fallback method.") + let fallbackFile = reassemblyDir.appendingPathComponent("fallback_disk.img") Logger.info("Creating fallback disk image at: \(fallbackFile.path)") - + // Try rsync first let rsyncProcess = Process() rsyncProcess.executableURL = URL(fileURLWithPath: "/usr/bin/rsync") - rsyncProcess.arguments = ["-aS", "--progress", diskPath.path, fallbackFile.path] - + rsyncProcess.arguments = [ + "-aS", "--progress", diskPath.path, fallbackFile.path, + ] + do { try rsyncProcess.run() rsyncProcess.waitUntilExit() - + if rsyncProcess.terminationStatus == 0 { - Logger.info("Direct copy completed with rsync. Fallback image available at: \(fallbackFile.path)") + Logger.info( + "Direct copy completed with rsync. Fallback image available at: \(fallbackFile.path)" + ) } else { // Try cp -c as fallback Logger.info("Rsync failed. Attempting with cp -c command...") let cpProcess = Process() cpProcess.executableURL = URL(fileURLWithPath: "/bin/cp") cpProcess.arguments = ["-c", diskPath.path, fallbackFile.path] - + try cpProcess.run() cpProcess.waitUntilExit() - + if cpProcess.terminationStatus == 0 { - Logger.info("Direct copy completed with cp -c. Fallback image available at: \(fallbackFile.path)") + Logger.info( + "Direct copy completed with cp -c. Fallback image available at: \(fallbackFile.path)" + ) } else { Logger.info("All recovery attempts failed.") } } } catch { - Logger.info("Error during recovery attempts: \(error.localizedDescription)") + Logger.info( + "Error during recovery attempts: \(error.localizedDescription)") Logger.info("All recovery attempts failed.") } } } - + Logger.info("Reassembled file is available at: \(reassembledFile.path)") } } - // --- Manifest Creation & Push --- + // --- Manifest Creation & Push --- let manifest = createManifest( layers: layers, - configLayerIndex: layers.firstIndex(where: { $0.mediaType == "application/vnd.oci.image.config.v1+json" }), + configLayerIndex: layers.firstIndex(where: { + $0.mediaType == "application/vnd.oci.image.config.v1+json" + }), uncompressedDiskSize: uncompressedDiskSize ) // Push manifest only if not in dry-run mode if !dryRun { - Logger.info("Pushing manifest(s)") // Updated log + Logger.info("Pushing manifest(s)") // Updated log // Serialize the manifest dictionary to Data first - let manifestData = try JSONSerialization.data(withJSONObject: manifest, options: [.prettyPrinted, .sortedKeys]) + let manifestData = try JSONSerialization.data( + withJSONObject: manifest, options: [.prettyPrinted, .sortedKeys]) // Loop through tags to push the same manifest data for tag in tags { - Logger.info("Pushing manifest for tag: \(tag)") - try await pushManifest( - repository: "\(self.organization)/\(imageName)", - tag: tag, // Use the current tag from the loop - manifest: manifestData, // Pass the serialized Data - token: token // Token should be in scope here now - ) + Logger.info("Pushing manifest for tag: \(tag)") + try await pushManifest( + repository: "\(self.organization)/\(imageName)", + tag: tag, // Use the current tag from the loop + manifest: manifestData, // Pass the serialized Data + token: token // Token should be in scope here now + ) } } // Print final upload summary if not dry run if !dryRun { let stats = await uploadProgress.getUploadStats() - Logger.info("\n\(stats.formattedSummary())") // Add newline for separation + Logger.info("\n\(stats.formattedSummary())") // Add newline for separation } // Clean up cache directory only on successful non-dry-run push } - - private func createManifest(layers: [OCIManifestLayer], configLayerIndex: Int?, uncompressedDiskSize: UInt64?) -> [String: Any] { + + private func createManifest( + layers: [OCIManifestLayer], configLayerIndex: Int?, uncompressedDiskSize: UInt64? + ) -> [String: Any] { var manifest: [String: Any] = [ "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", @@ -3166,221 +3132,244 @@ class ImageContainerRegistry: @unchecked Sendable { var layerDict: [String: Any] = [ "mediaType": layer.mediaType, "size": layer.size, - "digest": layer.digest + "digest": layer.digest, ] - + if let uncompressedSize = layer.uncompressedSize { var annotations: [String: String] = [:] - annotations["org.trycua.lume.uncompressed-size"] = "\(uncompressedSize)" // Updated prefix - + annotations["org.trycua.lume.uncompressed-size"] = "\(uncompressedSize)" // Updated prefix + if let digest = layer.uncompressedContentDigest { - annotations["org.trycua.lume.uncompressed-content-digest"] = digest // Updated prefix + annotations["org.trycua.lume.uncompressed-content-digest"] = digest // Updated prefix } - + layerDict["annotations"] = annotations } - + return layerDict - } + }, ] - + // Add config reference if available if let configIndex = configLayerIndex { let configLayer = layers[configIndex] manifest["config"] = [ "mediaType": configLayer.mediaType, "size": configLayer.size, - "digest": configLayer.digest + "digest": configLayer.digest, ] } - + // Add annotations var annotations: [String: String] = [:] - annotations["org.trycua.lume.upload-time"] = ISO8601DateFormatter().string(from: Date()) // Updated prefix - + annotations["org.trycua.lume.upload-time"] = ISO8601DateFormatter().string(from: Date()) // Updated prefix + if let diskSize = uncompressedDiskSize { - annotations["org.trycua.lume.uncompressed-disk-size"] = "\(diskSize)" // Updated prefix + annotations["org.trycua.lume.uncompressed-disk-size"] = "\(diskSize)" // Updated prefix } - + manifest["annotations"] = annotations - + return manifest } - - private func uploadBlobFromData(repository: String, data: Data, token: String) async throws -> String { + + private func uploadBlobFromData(repository: String, data: Data, token: String) async throws + -> String + { // Calculate digest let digest = "sha256:" + data.sha256String() - + // Check if blob already exists if try await blobExists(repository: repository, digest: digest, token: token) { Logger.info("Blob already exists: \(digest)") return digest } - + // Initiate upload let uploadURL = try await startBlobUpload(repository: repository, token: token) - + // Upload blob try await uploadBlob(url: uploadURL, data: data, digest: digest, token: token) - + // Report progress await uploadProgress.addProgress(Int64(data.count)) - + return digest } - - private func uploadBlobFromPath(repository: String, path: URL, digest: String, token: String) async throws -> String { + + private func uploadBlobFromPath(repository: String, path: URL, digest: String, token: String) + async throws -> String + { // Check if blob already exists if try await blobExists(repository: repository, digest: digest, token: token) { Logger.info("Blob already exists: \(digest)") return digest } - + // Initiate upload let uploadURL = try await startBlobUpload(repository: repository, token: token) - + // Load data from file let data = try Data(contentsOf: path) - + // Upload blob try await uploadBlob(url: uploadURL, data: data, digest: digest, token: token) - + // Report progress await uploadProgress.addProgress(Int64(data.count)) - + return digest } - - private func blobExists(repository: String, digest: String, token: String) async throws -> Bool { + + private func blobExists(repository: String, digest: String, token: String) async throws -> Bool + { let url = URL(string: "https://\(registry)/v2/\(repository)/blobs/\(digest)")! var request = URLRequest(url: url) request.httpMethod = "HEAD" request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") - + let (_, response) = try await URLSession.shared.data(for: request) - + if let httpResponse = response as? HTTPURLResponse { return httpResponse.statusCode == 200 } - + return false } - + private func startBlobUpload(repository: String, token: String) async throws -> URL { let url = URL(string: "https://\(registry)/v2/\(repository)/blobs/uploads/")! var request = URLRequest(url: url) request.httpMethod = "POST" request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") - request.setValue("0", forHTTPHeaderField: "Content-Length") // Explicitly set Content-Length to 0 for POST - + request.setValue("0", forHTTPHeaderField: "Content-Length") // Explicitly set Content-Length to 0 for POST + let (_, response) = try await URLSession.shared.data(for: request) - - guard let httpResponse = response as? HTTPURLResponse, - httpResponse.statusCode == 202, - let locationString = httpResponse.value(forHTTPHeaderField: "Location") else { + + guard let httpResponse = response as? HTTPURLResponse, + httpResponse.statusCode == 202, + let locationString = httpResponse.value(forHTTPHeaderField: "Location") + else { // Log response details on failure - let responseBody = String(data: (try? await URLSession.shared.data(for: request).0) ?? Data(), encoding: .utf8) ?? "(No Body)" - Logger.error("Failed to initiate blob upload. Status: \( (response as? HTTPURLResponse)?.statusCode ?? 0 ). Headers: \( (response as? HTTPURLResponse)?.allHeaderFields ?? [:] ). Body: \(responseBody)") + let responseBody = + String( + data: (try? await URLSession.shared.data(for: request).0) ?? Data(), + encoding: .utf8) ?? "(No Body)" + Logger.error( + "Failed to initiate blob upload. Status: \( (response as? HTTPURLResponse)?.statusCode ?? 0 ). Headers: \( (response as? HTTPURLResponse)?.allHeaderFields ?? [:] ). Body: \(responseBody)" + ) throw PushError.uploadInitiationFailed } - + // Construct the base URL for the registry guard let baseRegistryURL = URL(string: "https://\(registry)") else { Logger.error("Failed to create base registry URL from: \(registry)") - throw PushError.invalidURL - } - - // Create the final upload URL, resolving the location against the base URL - guard let uploadURL = URL(string: locationString, relativeTo: baseRegistryURL) else { - Logger.error("Failed to create absolute upload URL from location: \(locationString) relative to base: \(baseRegistryURL.absoluteString)") throw PushError.invalidURL } - + + // Create the final upload URL, resolving the location against the base URL + guard let uploadURL = URL(string: locationString, relativeTo: baseRegistryURL) else { + Logger.error( + "Failed to create absolute upload URL from location: \(locationString) relative to base: \(baseRegistryURL.absoluteString)" + ) + throw PushError.invalidURL + } + Logger.info("Blob upload initiated. Upload URL: \(uploadURL.absoluteString)") - return uploadURL.absoluteURL // Ensure it's absolute + return uploadURL.absoluteURL // Ensure it's absolute } - + private func uploadBlob(url: URL, data: Data, digest: String, token: String) async throws { var components = URLComponents(url: url, resolvingAgainstBaseURL: true)! - + // Add digest parameter var queryItems = components.queryItems ?? [] queryItems.append(URLQueryItem(name: "digest", value: digest)) components.queryItems = queryItems - + guard let uploadURL = components.url else { throw PushError.invalidURL } - + var request = URLRequest(url: uploadURL) request.httpMethod = "PUT" request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") request.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type") request.setValue("\(data.count)", forHTTPHeaderField: "Content-Length") request.httpBody = data - + let (_, response) = try await URLSession.shared.data(for: request) - + guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 201 else { throw PushError.blobUploadFailed } } - - private func pushManifest(repository: String, tag: String, manifest: Data, token: String) async throws { + + private func pushManifest(repository: String, tag: String, manifest: Data, token: String) + async throws + { let url = URL(string: "https://\(registry)/v2/\(repository)/manifests/\(tag)")! var request = URLRequest(url: url) request.httpMethod = "PUT" request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") - request.setValue("application/vnd.oci.image.manifest.v1+json", forHTTPHeaderField: "Content-Type") + request.setValue( + "application/vnd.oci.image.manifest.v1+json", forHTTPHeaderField: "Content-Type") request.httpBody = manifest - + let (_, response) = try await URLSession.shared.data(for: request) - + guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 201 else { throw PushError.manifestPushFailed } } - + private func getCredentialsFromEnvironment() -> (String?, String?) { - let username = ProcessInfo.processInfo.environment["GITHUB_USERNAME"] ?? - ProcessInfo.processInfo.environment["GHCR_USERNAME"] - let password = ProcessInfo.processInfo.environment["GITHUB_TOKEN"] ?? - ProcessInfo.processInfo.environment["GHCR_TOKEN"] + let username = + ProcessInfo.processInfo.environment["GITHUB_USERNAME"] + ?? ProcessInfo.processInfo.environment["GHCR_USERNAME"] + let password = + ProcessInfo.processInfo.environment["GITHUB_TOKEN"] + ?? ProcessInfo.processInfo.environment["GHCR_TOKEN"] return (username, password) } // Add these helper methods for dry-run and reassemble implementation - + // NEW Helper function using Compression framework and sparse writing - private func decompressChunkAndWriteSparse(inputPath: String, outputHandle: FileHandle, startOffset: UInt64) throws -> UInt64 { + private func decompressChunkAndWriteSparse( + inputPath: String, outputHandle: FileHandle, startOffset: UInt64 + ) throws -> UInt64 { guard FileManager.default.fileExists(atPath: inputPath) else { Logger.error("Compressed chunk not found at: \(inputPath)") - return 0 // Or throw an error + return 0 // Or throw an error } - let sourceData = try Data(contentsOf: URL(fileURLWithPath: inputPath), options: .alwaysMapped) + let sourceData = try Data( + contentsOf: URL(fileURLWithPath: inputPath), options: .alwaysMapped) var currentWriteOffset = startOffset var totalDecompressedBytes: UInt64 = 0 - var sourceReadOffset = 0 // Keep track of how much compressed data we've provided + var sourceReadOffset = 0 // Keep track of how much compressed data we've provided // Use the initializer with the readingFrom closure let filter = try InputFilter(.decompress, using: .lz4) { (length: Int) -> Data? in let bytesAvailable = sourceData.count - sourceReadOffset if bytesAvailable == 0 { - return nil // No more data + return nil // No more data } let bytesToRead = min(length, bytesAvailable) - let chunk = sourceData.subdata(in: sourceReadOffset ..< sourceReadOffset + bytesToRead) + let chunk = sourceData.subdata(in: sourceReadOffset..= 0.5) || (completedFiles == totalFiles) + let shouldUpdate = + (uploadedBytes <= bytes) || (elapsed >= 0.5) || (completedFiles == totalFiles) - if shouldUpdate && totalBytes > 0 { // Ensure totalBytes is set + if shouldUpdate && totalBytes > 0 { // Ensure totalBytes is set let currentSpeed = Double(uploadedBytes - lastUpdateBytes) / max(elapsed, 0.001) speedSamples.append(currentSpeed) @@ -3479,14 +3470,17 @@ actor UploadProgressTracker { peakSpeed = max(peakSpeed, currentSpeed) // Apply exponential smoothing - if smoothedSpeed == 0 { smoothedSpeed = currentSpeed } - else { smoothedSpeed = speedSmoothing * currentSpeed + (1 - speedSmoothing) * smoothedSpeed } + if smoothedSpeed == 0 { + smoothedSpeed = currentSpeed + } else { + smoothedSpeed = speedSmoothing * currentSpeed + (1 - speedSmoothing) * smoothedSpeed + } let recentAvgSpeed = calculateAverageSpeed() let totalElapsed = now.timeIntervalSince(startTime) let overallAvgSpeed = totalElapsed > 0 ? Double(uploadedBytes) / totalElapsed : 0 - let progress = totalBytes > 0 ? Double(uploadedBytes) / Double(totalBytes) : 1.0 // Avoid division by zero + let progress = totalBytes > 0 ? Double(uploadedBytes) / Double(totalBytes) : 1.0 // Avoid division by zero logSpeedProgress( current: progress, currentSpeed: currentSpeed, @@ -3494,7 +3488,7 @@ actor UploadProgressTracker { smoothedSpeed: smoothedSpeed, overallSpeed: overallAvgSpeed, peakSpeed: peakSpeed, - context: "Uploading Image" // Changed context + context: "Uploading Image" // Changed context ) lastUpdateTime = now @@ -3521,7 +3515,7 @@ actor UploadProgressTracker { let avgSpeed = totalElapsedTime > 0 ? Double(uploadedBytes) / totalElapsedTime : 0 return UploadStats( totalBytes: totalBytes, - uploadedBytes: uploadedBytes, // Renamed + uploadedBytes: uploadedBytes, // Renamed elapsedTime: totalElapsedTime, averageSpeed: avgSpeed, peakSpeed: peakSpeed @@ -3546,10 +3540,10 @@ actor UploadProgressTracker { let etaSeconds = speedForEta > 0 ? Double(remainingBytes) / speedForEta : 0 let etaStr = formatTimeRemaining(etaSeconds) let progressBar = createProgressBar(progress: current) - let fileProgress = "(\(completedFiles)/\(totalFiles))" // Add file count + let fileProgress = "(\(completedFiles)/\(totalFiles))" // Add file count print( - "\r\(progressBar) \(progressPercent)% \(fileProgress) | Speed: \(avgSpeedStr) (Avg) | ETA: \(etaStr) ", // Simplified output + "\r\(progressBar) \(progressPercent)% \(fileProgress) | Speed: \(avgSpeedStr) (Avg) | ETA: \(etaStr) ", // Simplified output terminator: "") fflush(stdout) } @@ -3566,7 +3560,10 @@ actor UploadProgressTracker { let units = ["B/s", "KB/s", "MB/s", "GB/s"] var speed = bytesPerSecond var unitIndex = 0 - while speed > 1024 && unitIndex < units.count - 1 { speed /= 1024; unitIndex += 1 } + while speed > 1024 && unitIndex < units.count - 1 { + speed /= 1024 + unitIndex += 1 + } return String(format: "%.1f %@", speed, units[unitIndex]) } private func formatTimeRemaining(_ seconds: Double) -> String { @@ -3574,8 +3571,10 @@ actor UploadProgressTracker { let hours = Int(seconds) / 3600 let minutes = (Int(seconds) % 3600) / 60 let secs = Int(seconds) % 60 - if hours > 0 { return String(format: "%d:%02d:%02d", hours, minutes, secs) } - else { return String(format: "%d:%02d", minutes, secs) } + if hours > 0 { + return String(format: "%d:%02d:%02d", hours, minutes, secs) + } else { + return String(format: "%d:%02d", minutes, secs) + } } } -