From 2efbcb4f8f8f25b98c2a819591e94e3049a05211 Mon Sep 17 00:00:00 2001 From: f-trycua Date: Tue, 22 Apr 2025 00:29:46 -0700 Subject: [PATCH] Remove push, pull old scripts --- libs/lume/scripts/ghcr/pull-ghcr.sh | 205 -------- libs/lume/scripts/ghcr/push-ghcr.sh | 745 ---------------------------- 2 files changed, 950 deletions(-) delete mode 100755 libs/lume/scripts/ghcr/pull-ghcr.sh delete mode 100755 libs/lume/scripts/ghcr/push-ghcr.sh diff --git a/libs/lume/scripts/ghcr/pull-ghcr.sh b/libs/lume/scripts/ghcr/pull-ghcr.sh deleted file mode 100755 index 8b10fae1..00000000 --- a/libs/lume/scripts/ghcr/pull-ghcr.sh +++ /dev/null @@ -1,205 +0,0 @@ -#!/bin/bash - -# Exit immediately if a command exits with a non-zero status -set -e - -# Default parameters -organization="" -image_name="" -image_version="" -target_folder_path="" - -# Parse the command line arguments -while [[ $# -gt 0 ]]; do - case "$1" in - --organization) - organization="$2" - shift 2 - ;; - --image-name) - image_name="$2" - shift 2 - ;; - --image-version) - image_version="$2" - shift 2 - ;; - --target-folder-path) - target_folder_path="$2" - shift 2 - ;; - --help) - echo "Usage: $0 [options]" - echo "Options:" - echo " --organization : GitHub organization (required)" - echo " --image-name : Name of the image to pull (required)" - echo " --image-version : Version of the image to pull (required)" - echo " --target-folder-path : Path where to extract the files (required)" - exit 0 - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -# Ensure required arguments -if [[ -z "$organization" || -z "$image_name" || -z "$image_version" || -z "$target_folder_path" ]]; then - echo "Error: Missing required arguments. Use --help for usage." - exit 1 -fi - -# Check and install required tools -for tool in "jq" "pv" "parallel"; do - if ! command -v "$tool" &> /dev/null; then - echo "$tool is not installed. Installing using Homebrew..." - if ! command -v brew &> /dev/null; then - echo "Homebrew is not installed. Please install Homebrew first: https://brew.sh/" - exit 1 - fi - brew install "$tool" - fi -done - -# Create target folder if it doesn't exist -mkdir -p "$target_folder_path" - -# Create a temporary directory for processing files -work_dir=$(mktemp -d) -echo "Working directory: $work_dir" -trap 'rm -rf "$work_dir"' EXIT - -# Registry details -REGISTRY="ghcr.io" -REPOSITORY="$organization/$image_name" -TAG="$image_version" - -# Get anonymous token -echo "Getting authentication token..." -curl -s "https://$REGISTRY/token?service=ghcr.io&scope=repository:$REPOSITORY:pull" -o "$work_dir/token.json" -TOKEN=$(cat "$work_dir/token.json" | jq -r ".token") - -if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then - echo "Failed to obtain token" - exit 1 -fi - -echo "Token obtained successfully" - -# Fetch manifest -echo "Fetching manifest..." -MANIFEST_RESPONSE=$(curl -s \ - -H "Authorization: Bearer $TOKEN" \ - -H "Accept: application/vnd.oci.image.manifest.v1+json" \ - "https://$REGISTRY/v2/$REPOSITORY/manifests/$TAG") - -echo "Processing manifest..." - -# Create a directory for all files -cd "$work_dir" - -# Create a download function for parallel execution -download_layer() { - local media_type="$1" - local digest="$2" - local output_file="$3" - - echo "Downloading $output_file..." - curl -s -L \ - -H "Authorization: Bearer $TOKEN" \ - -H "Accept: $media_type" \ - "https://$REGISTRY/v2/$REPOSITORY/blobs/$digest" | \ - pv > "$output_file" -} -export -f download_layer -export TOKEN REGISTRY REPOSITORY - -# Process layers and create download jobs -echo "$MANIFEST_RESPONSE" | jq -c '.layers[]' | while read -r layer; do - media_type=$(echo "$layer" | jq -r '.mediaType') - digest=$(echo "$layer" | jq -r '.digest') - - # Skip empty layers - if [[ "$media_type" == "application/vnd.oci.empty.v1+json" ]]; then - continue - fi - - # Extract part information if present - if [[ $media_type =~ part\.number=([0-9]+)\;part\.total=([0-9]+) ]]; then - part_num="${BASH_REMATCH[1]}" - total_parts="${BASH_REMATCH[2]}" - echo "Found part $part_num of $total_parts" - output_file="disk.img.part.$part_num" - else - case "$media_type" in - "application/vnd.oci.image.layer.v1.tar") - output_file="disk.img" - ;; - "application/vnd.oci.image.config.v1+json") - output_file="config.json" - ;; - "application/octet-stream") - output_file="nvram.bin" - ;; - *) - echo "Unknown media type: $media_type" - continue - ;; - esac - fi - - # Add to download queue - echo "$media_type"$'\t'"$digest"$'\t'"$output_file" >> download_queue.txt -done - -# Download all files in parallel -echo "Downloading files in parallel..." -parallel --colsep $'\t' -a download_queue.txt download_layer {1} {2} {3} - -# Check if we have disk parts to reassemble -if ls disk.img.part.* 1> /dev/null 2>&1; then - echo "Found disk parts, reassembling..." - - # Get total parts from the first part's filename - first_part=$(ls disk.img.part.* | head -n 1) - total_parts=$(echo "$MANIFEST_RESPONSE" | jq -r '.layers[] | select(.mediaType | contains("part.total")) | .mediaType' | grep -o 'part\.total=[0-9]*' | cut -d= -f2 | head -n 1) - - echo "Total parts to reassemble: $total_parts" - - # Concatenate parts in order - echo "Reassembling disk image..." - { - for i in $(seq 1 "$total_parts"); do - part_file="disk.img.part.$i" - if [ -f "$part_file" ]; then - cat "$part_file" - else - echo "Error: Missing part $i" - exit 1 - fi - done - } | pv > "$target_folder_path/disk.img" - - echo "Disk image reassembled successfully" -else - # If no parts found, just copy disk.img if it exists - if [ -f disk.img ]; then - echo "Copying disk image..." - pv disk.img > "$target_folder_path/disk.img" - fi -fi - -# Copy config.json if it exists -if [ -f config.json ]; then - echo "Copying config.json..." - cp config.json "$target_folder_path/" -fi - -# Copy nvram.bin if it exists -if [ -f nvram.bin ]; then - echo "Copying nvram.bin..." - cp nvram.bin "$target_folder_path/" -fi - -echo "Download complete: Files extracted to $target_folder_path" \ No newline at end of file diff --git a/libs/lume/scripts/ghcr/push-ghcr.sh b/libs/lume/scripts/ghcr/push-ghcr.sh deleted file mode 100755 index d279be66..00000000 --- a/libs/lume/scripts/ghcr/push-ghcr.sh +++ /dev/null @@ -1,745 +0,0 @@ -#!/bin/bash - -# Exit immediately if a command exits with a non-zero status -set -e - -# Default parameters -organization="" -folder_path="" -image_name="" -image_versions="" -chunk_size="512M" # Default chunk size for splitting large files -dry_run=true # Default: actually push to registry -reassemble=true # Default: don't reassemble in dry-run mode -# Define the OCI media type for the compressed disk layer -oci_layer_media_type="application/octet-stream+lz4" # LZ4 compression format - -# Parse the command line arguments -while [[ $# -gt 0 ]]; do - case "$1" in - --organization) - organization="$2" - shift 2 - ;; - --folder-path) - folder_path="$2" - shift 2 - ;; - --image-name) - image_name="$2" - shift 2 - ;; - --image-versions) - image_versions="$2" - shift 2 - ;; - --chunk-size) - chunk_size="$2" - shift 2 - ;; - --dry-run) - dry_run=true - shift 1 - ;; - --reassemble) - reassemble=true - dry_run=true # Reassemble implies dry-run - shift 1 - ;; - --help) - echo "Usage: $0 [options]" - echo "Options:" - echo " --organization : GitHub organization (required if not using token)" - echo " --folder-path : Path to the folder to upload (required)" - echo " --image-name : Name of the image to publish (required)" - echo " --image-versions : Comma separated list of versions of the image to publish (required)" - echo " --chunk-size : Size of chunks for large files (e.g., 512M, default: 512M)" - echo " --dry-run : Prepare files but don't upload to registry" - echo " --reassemble : In dry-run mode, also reassemble chunks to verify integrity" - echo "Note: The script will automatically resume from the last attempt if available" - exit 0 - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -# Ensure required arguments -if [[ -z "$folder_path" ]]; then - echo "Error: Missing required folder-path argument. Use --help for usage." - exit 1 -fi - -# Only check organization and other push parameters if not in dry-run mode -if [[ "$dry_run" = false ]]; then - if [[ -z "$organization" || -z "$image_name" || -z "$image_versions" ]]; then - echo "Error: Missing required arguments for push. Use --help for usage." - exit 1 - fi - - # Check if the GITHUB_TOKEN variable is set - if [[ -z "$GITHUB_TOKEN" ]]; then - echo "Error: GITHUB_TOKEN is not set." - exit 1 - fi -fi - -# Ensure the folder exists -if [[ ! -d "$folder_path" ]]; then - echo "Error: Folder $folder_path does not exist." - exit 1 -fi - -# Check and install required tools -for tool in "oras" "split" "pv" "jq" "lz4"; do - if ! command -v "$tool" &> /dev/null; then - echo "$tool is not installed. Installing using Homebrew..." - if ! command -v brew &> /dev/null; then - echo "Homebrew is not installed. Please install Homebrew first: https://brew.sh/" - exit 1 - fi - brew install "$tool" - fi -done - -echo "LZ4 detected - will use for efficient compression and decompression" -compressed_ext=".lz4" - -# Authenticate with GitHub Container Registry if not in dry-run mode -if [[ "$dry_run" = false ]]; then - echo "$GITHUB_TOKEN" | oras login ghcr.io -u "$organization" --password-stdin -fi - -# Use the source folder path as the working directory and get its absolute path -work_dir=$(cd "$folder_path" && pwd) -echo "Working directory: $work_dir" - -# Function to find the most recent cache directory -find_latest_cache() { - local latest_cache=$(ls -td "$work_dir"/.ghcr_cache_* 2>/dev/null | head -n1) - if [ -n "$latest_cache" ]; then - echo "$latest_cache" - else - echo "" - fi -} - -# Function to check if a cache directory is valid for resuming -is_valid_cache() { - local cache_dir="$1" - # Check if it contains the necessary files - [ -f "$cache_dir/config.json" ] || [ -f "$cache_dir/nvram.bin" ] || \ - [ -f "$cache_dir/disk.img.lz4" ] || ls "$cache_dir"/disk.img.part.* 1>/dev/null 2>&1 -} - -# Always try to find and use an existing cache -existing_cache=$(find_latest_cache) -if [ -n "$existing_cache" ] && is_valid_cache "$existing_cache"; then - cache_dir="$existing_cache" - - # Check if the cache contains old compressed format - if [ -f "$cache_dir/disk.img.gz" ] || [ -f "$cache_dir/disk.img.aa" ] || ls "$cache_dir"/disk.img.*.part.* 1>/dev/null 2>&1; then - echo "Error: Found legacy compressed format in cache. This script uses improved LZ4 format." - echo "Please delete the cache directory and start fresh: $cache_dir" - exit 1 - fi - - echo "Resuming from existing cache: $cache_dir" -else - echo "No valid cache found. Starting fresh." - cache_dir="$work_dir/.ghcr_cache_$(date +%Y%m%d_%H%M%S)" - mkdir -p "$cache_dir" -fi - -echo "Using cache directory: $cache_dir" - -# Display space information -echo "=== DISK SPACE INFORMATION ===" -df -h "$cache_dir" | head -1 -df -h "$cache_dir" | grep -v "Filesystem" -echo - -# Change to the cache directory -cd "$cache_dir" -files=() # Initialize files array here - -# Function to check if a version was already pushed -version_pushed() { - local version="$1" - local version_file="$cache_dir/.pushed_$version" - [ -f "$version_file" ] -} - -# Function to mark a version as pushed -mark_version_pushed() { - local version="$1" - touch "$cache_dir/.pushed_$version" -} - -# Function to calculate sha256 hash -calculate_sha256() { - local file="$1" - if command -v shasum &> /dev/null; then - shasum -a 256 "$file" | awk '{print "sha256:" $1}' - else - echo "sha256:$(openssl dgst -sha256 -binary "$file" | xxd -p | tr -d '\n')" - fi -} - -# Copy config.json if it exists and not already in cache -config_json_source="$folder_path/config.json" -config_json_dest="$cache_dir/config.json" -if [ -f "$config_json_source" ]; then - if [ ! -f "$config_json_dest" ]; then - echo "Copying config.json..." - # Copy config.json as is - we'll add annotations later - cp "$config_json_source" "$config_json_dest" - fi -fi -if [ -f "$config_json_dest" ]; then - files+=("config.json:application/vnd.oci.image.config.v1+json") -fi - -# Copy nvram.bin if it exists and not already in cache -if [ -f "$folder_path/nvram.bin" ] && [ ! -f "$cache_dir/nvram.bin" ]; then - echo "Copying nvram.bin..." - cp "$folder_path/nvram.bin" nvram.bin -fi -if [ -f "$cache_dir/nvram.bin" ]; then - files+=("nvram.bin:application/octet-stream") -fi - -# Process disk.img if it exists -disk_img_orig="$folder_path/disk.img" -original_disk_size="" -if [ -f "$disk_img_orig" ]; then - # Get original size *before* compression - original_disk_size=$(stat -f%z "$disk_img_orig") - - # Get real (non-sparse) size - real_size=$(du -k "$disk_img_orig" | cut -f1) - real_size_bytes=$((real_size * 1024)) - sparseness_ratio=$(echo "scale=2; $original_disk_size / $real_size_bytes" | bc) - echo "Disk image: $disk_img_orig" - echo " Logical size: $original_disk_size bytes ($(du -h "$disk_img_orig" | cut -f1))" - echo " Actual disk usage: $((real_size_bytes / 1073741824)) GB" - echo " Sparseness ratio: ${sparseness_ratio}:1" - - # If we have config.json, update it with the uncompressed disk size annotation - if [ -f "$config_json_dest" ] && command -v jq &> /dev/null; then - echo "Adding uncompressed disk size annotation: $original_disk_size bytes" - jq --arg size "$original_disk_size" '.annotations = (.annotations // {}) + {"com.trycua.lume.disk.uncompressed_size": $size}' "$config_json_dest" > "$config_json_dest.tmp" - mv "$config_json_dest.tmp" "$config_json_dest" - fi - - # Create a temporary directory for disk processing - tmp_dir="$cache_dir/tmp_processing" - mkdir -p "$tmp_dir" - - # Split the disk image into chunks first (before compression) - split_parts_dir="$tmp_dir/split_parts" - mkdir -p "$split_parts_dir" - - # Check if we already have split parts - if [ -z "$(ls -A "$split_parts_dir" 2>/dev/null)" ]; then - echo "Splitting disk image into chunks of $chunk_size..." - cd "$split_parts_dir" - pv "$disk_img_orig" | split -b "$chunk_size" - "chunk." - cd "$cache_dir" - else - echo "Using existing split chunks from previous run" - fi - - # Process each chunk (compress, calculate digest, etc.) - compressed_parts_dir="$tmp_dir/compressed_parts" - mkdir -p "$compressed_parts_dir" - - # Store layer information in an array - layers=() - part_num=0 - total_parts=$(ls "$split_parts_dir"/chunk.* | wc -l) - - for chunk_file in "$split_parts_dir"/chunk.*; do - part_basename=$(basename "$chunk_file") - part_num=$((part_num + 1)) - compressed_file="$compressed_parts_dir/${part_basename}${compressed_ext}" - - if [ ! -f "$compressed_file" ]; then - echo "Compressing chunk $part_num of $total_parts: $part_basename" - - # Calculate uncompressed content digest before compression - uncompressed_digest=$(calculate_sha256 "$chunk_file") - - # Get uncompressed size - uncompressed_size=$(stat -f%z "$chunk_file") - - # Compress the chunk with LZ4 - lz4 -9 "$chunk_file" "$compressed_file" - - # Get compressed size - compressed_size=$(stat -f%z "$compressed_file") - - echo "Chunk $part_num: Original size: $(du -h "$chunk_file" | cut -f1), Compressed: $(du -h "$compressed_file" | cut -f1)" - else - echo "Using existing compressed chunk $part_num of $total_parts" - - # Need to calculate these values for existing files - uncompressed_digest=$(calculate_sha256 "$chunk_file") - uncompressed_size=$(stat -f%z "$chunk_file") - compressed_size=$(stat -f%z "$compressed_file") - fi - - # Store layer information - layer_info="$compressed_file:${oci_layer_media_type};uncompressed_size=$uncompressed_size;uncompressed_digest=$uncompressed_digest;part.number=$part_num;part.total=$total_parts" - layers+=("$layer_info") - done - - # Generate the files array for ORAS push - for layer_info in "${layers[@]}"; do - files+=("$layer_info") - done - - # --- Reassembly in dry-run mode --- - if [[ "$reassemble" = true ]]; then - echo "=== REASSEMBLY MODE ===" - echo "Reassembling chunks to verify integrity..." - - # Create a directory for reassembly - reassembly_dir="$cache_dir/reassembly" - mkdir -p "$reassembly_dir" - - # Prepare the reassembled file - create a properly sized sparse file first - reassembled_file="$reassembly_dir/reassembled_disk.img" - if [ -f "$reassembled_file" ]; then - echo "Removing previous reassembled file..." - rm -f "$reassembled_file" - fi - - # Get the original disk size from config annotation or directly from image - if [ -f "$config_json_dest" ] && command -v jq &> /dev/null; then - config_size=$(jq -r '.annotations."com.trycua.lume.disk.uncompressed_size" // empty' "$config_json_dest") - if [ -n "$config_size" ]; then - original_disk_size_bytes=$config_size - echo "Using uncompressed size from config: $original_disk_size_bytes bytes" - fi - fi - - # Create a sparse file of the exact original size - echo "Pre-allocating sparse file of $(du -h "$disk_img_orig" | cut -f1)..." - dd if=/dev/zero of="$reassembled_file" bs=1 count=0 seek=$original_disk_size - - # Make sure filesystem recognizes this as a sparse file - if [[ "$OSTYPE" == "darwin"* ]]; then - # On macOS, we can use a better sparse file creation method if mkfile is available - if command -v mkfile &> /dev/null; then - rm -f "$reassembled_file" - mkfile -n ${original_disk_size}b "$reassembled_file" - echo "Created sparse file using mkfile" - fi - else - # On Linux systems, ensure sparseness with truncate if available - if command -v truncate &> /dev/null; then - rm -f "$reassembled_file" - truncate -s $original_disk_size "$reassembled_file" - echo "Created sparse file using truncate" - fi - fi - - # Create an offset tracker to keep track of where each chunk should go - current_offset=0 - - # Decompress each chunk and write it at the correct offset - for ((i=1; i<=total_parts; i++)); do - # Find the chunk file for part number i - chunk_pattern="" - chunk_uncompressed_size="" - - for layer_info in "${layers[@]}"; do - if [[ "$layer_info" == *";part.number=$i;"* ]]; then - chunk_pattern="${layer_info%%:*}" - # Extract the uncompressed size from metadata - if [[ "$layer_info" =~ uncompressed_size=([0-9]+) ]]; then - chunk_uncompressed_size="${BASH_REMATCH[1]}" - fi - break - fi - done - - if [ -z "$chunk_pattern" ]; then - echo "Error: Could not find chunk for part $i" - exit 1 - fi - - echo "Processing part $i/$total_parts: $(basename "$chunk_pattern") at offset $current_offset..." - - # Create temp decompressed file - temp_decompressed="$reassembly_dir/temp_part_$i" - lz4 -d -f "$chunk_pattern" "$temp_decompressed" || { - echo "Error decompressing part $i" - exit 1 - } - - # Check if this chunk is all zeros (sparse data) - # Only check the first 1MB for efficiency - is_likely_sparse=false - if command -v hexdump &> /dev/null; then - # Use hexdump to check a sample of the file for non-zero content - sparse_check=$(hexdump -n 1048576 -v "$temp_decompressed" | grep -v "0000 0000 0000 0000 0000 0000 0000 0000" | head -n 1) - if [ -z "$sparse_check" ]; then - echo "Chunk appears to be all zeros (sparse data)" - is_likely_sparse=true - fi - fi - - # Use dd to write the chunk at the correct offset with sparse file handling - if [ "$is_likely_sparse" = true ]; then - # For sparse chunks, we don't need to write anything - leave as zeros - echo "Skipping write for all-zero chunk (preserving sparseness)" - elif [[ "$OSTYPE" == "darwin"* ]]; then - # macOS dd doesn't support conv=sparse, use standard approach - dd if="$temp_decompressed" of="$reassembled_file" bs=1M conv=notrunc seek=$((current_offset / 1024 / 1024)) status=progress || { - echo "Error writing part $i at offset $current_offset" - exit 1 - } - else - # On Linux, use conv=sparse to preserve sparseness during the write - dd if="$temp_decompressed" of="$reassembled_file" bs=1M conv=sparse,notrunc seek=$((current_offset / 1024 / 1024)) status=progress || { - echo "Error writing part $i at offset $current_offset" - exit 1 - } - fi - - # Clean up the temporary file - rm -f "$temp_decompressed" - - # Update the offset for the next chunk - current_offset=$((current_offset + chunk_uncompressed_size)) - done - - # After all chunks are processed, ensure sparseness is preserved - if command -v cp &> /dev/null && [[ "$OSTYPE" == "darwin"* ]]; then - echo "Copying disk image to maintain sparseness..." - final_sparse_file="$reassembly_dir/final_disk.img" - rm -f "$final_sparse_file" 2>/dev/null - - # On macOS, use cp with the clone flag to preserve sparseness - cp -c "$reassembled_file" "$final_sparse_file" - - # Use the sparse-optimized file for verification - echo "Using sparse-optimized copy for verification" - mv "$final_sparse_file" "$reassembled_file" - sync - elif command -v cp &> /dev/null && command -v file &> /dev/null; then - # For Linux systems - echo "Optimizing file sparseness..." - final_sparse_file="$reassembly_dir/final_disk.img" - rm -f "$final_sparse_file" 2>/dev/null - - # Use cp --sparse=always on Linux - cp --sparse=always "$reassembled_file" "$final_sparse_file" - - # Use the sparse-optimized file for verification - echo "Using sparse-optimized copy for verification" - mv "$final_sparse_file" "$reassembled_file" - sync - fi - - # Make sure to sync to disk - sync - - # Calculate digests for comparison - echo "Verifying reassembled file..." - original_digest=$(calculate_sha256 "$disk_img_orig") - reassembled_digest=$(calculate_sha256 "$reassembled_file") - - # Compare the original and reassembled file sizes - original_size=$(stat -f%z "$disk_img_orig") - reassembled_size=$(stat -f%z "$reassembled_file") - - echo "Results:" - echo " Original size: $(du -h "$disk_img_orig" | cut -f1) ($original_size bytes)" - echo " Reassembled size: $(du -h "$reassembled_file" | cut -f1) ($reassembled_size bytes)" - echo " Original digest: ${original_digest#sha256:}" - echo " Reassembled digest: ${reassembled_digest#sha256:}" - - # Check if the disk is sparse - original_apparent_size=$(du -h "$disk_img_orig" | cut -f1) - original_actual_size=$(du -sh "$disk_img_orig" | cut -f1) - reassembled_apparent_size=$(du -h "$reassembled_file" | cut -f1) - reassembled_actual_size=$(du -sh "$reassembled_file" | cut -f1) - - echo " Original: Apparent size: $original_apparent_size, Actual disk usage: $original_actual_size" - echo " Reassembled: Apparent size: $reassembled_apparent_size, Actual disk usage: $reassembled_actual_size" - - if [ "$original_digest" = "$reassembled_digest" ]; then - echo "✅ VERIFICATION SUCCESSFUL: Files are identical" - else - echo "❌ VERIFICATION FAILED: Files differ" - if [ "$original_size" != "$reassembled_size" ]; then - echo " Size mismatch: Original $original_size bytes, Reassembled $reassembled_size bytes" - fi - - # Try to identify where they differ - echo "Attempting to identify differences..." - if command -v cmp &> /dev/null; then - cmp_output=$(cmp -l "$disk_img_orig" "$reassembled_file" 2>&1 | head -5) - if [[ "$cmp_output" == *"differ"* ]]; then - echo " First few differences:" - echo "$cmp_output" - fi - fi - - # Check if the virtual machine will still boot despite differences - echo "NOTE: This might be a sparse file issue. The content may be identical, but sparse regions" - echo " may be handled differently between the original and reassembled files." - - # Calculate a percentage comparison of used blocks - # This helps determine if the sparse issues are severe or minor - original_used_kb=$(du -k "$disk_img_orig" | cut -f1) - reassembled_used_kb=$(du -k "$reassembled_file" | cut -f1) - - # Calculate percentage difference in used space - if [ "$original_used_kb" -ne 0 ]; then - diff_percentage=$(echo "scale=2; ($reassembled_used_kb - $original_used_kb) * 100 / $original_used_kb" | bc) - echo " Disk usage difference: $diff_percentage% ($reassembled_used_kb KB vs $original_used_kb KB)" - - # If reassembled is much smaller, this likely indicates sparse regions weren't preserved - if (( $(echo "$diff_percentage < -40" | bc -l) )); then - echo " ⚠️ WARNING: Reassembled disk uses significantly less space (>40% difference)." - echo " This indicates sparse regions weren't properly preserved and may affect VM functionality." - echo " The VM might boot but could be missing applications or data." - elif (( $(echo "$diff_percentage < -10" | bc -l) )); then - echo " ⚠️ WARNING: Reassembled disk uses less space (10-40% difference)." - echo " Some sparse regions may not be properly preserved but VM might still function correctly." - elif (( $(echo "$diff_percentage > 10" | bc -l) )); then - echo " ⚠️ WARNING: Reassembled disk uses more space (>10% difference)." - echo " This is unusual and may indicate improper sparse file handling." - else - echo " ✓ Disk usage difference is minimal (<10%). VM likely to function correctly." - fi - fi - fi - - echo "Reassembled file is available at: $reassembled_file" - - # If verification failed and difference is significant, try a direct copy as fallback - if [ "$original_digest" != "$reassembled_digest" ] && [ -n "$diff_percentage" ] && (( $(echo "$diff_percentage < -20" | bc -l) )); then - echo - echo "===== ATTEMPTING RECOVERY ACTION =====" - echo "Since verification failed with significant disk usage difference," - echo "trying direct copy of disk image as a fallback method." - echo - - fallback_file="$reassembly_dir/fallback_disk.img" - echo "Creating fallback disk image at: $fallback_file" - - # Use rsync with sparse option if available - if command -v rsync &> /dev/null; then - echo "Using rsync with sparse option for direct copy..." - rsync -aS --progress "$disk_img_orig" "$fallback_file" - else - # Direct cp with sparse option if available - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "Using cp -c (clone) for direct copy..." - cp -c "$disk_img_orig" "$fallback_file" - else - echo "Using cp --sparse=always for direct copy..." - cp --sparse=always "$disk_img_orig" "$fallback_file" - fi - fi - - echo "Direct copy completed. You may want to try using this fallback disk image" - echo "instead if the reassembled one has issues: $fallback_file" - fi - fi - - # --- Push Logic --- - if [[ "$dry_run" = true ]]; then - echo "=== DRY RUN MODE ===" - echo "The following files would be pushed to the registry:" - for file_info in "${files[@]}"; do - file_path="${file_info%%:*}" - file_metadata="${file_info#*:}" - file_size=$(du -h "$file_path" | cut -f1) - echo " - $file_path ($file_size) with metadata: $file_metadata" - done - - if [[ -n "$image_versions" ]]; then - echo "Would push to the following versions:" - IFS=',' read -ra versions <<< "$image_versions" - for version in "${versions[@]}"; do - version=$(echo "$version" | xargs) - if [[ -z "$version" ]]; then continue; fi - echo " - ghcr.io/$organization/$image_name:$version" - done - else - echo "No versions specified for dry run. Processing completed successfully." - fi - - echo "All processing tasks completed. No actual push performed." - echo "Cache directory: $cache_dir" - exit 0 - fi - - # Regular push logic (non-dry-run) - push_pids=() - IFS=',' read -ra versions <<< "$image_versions" - for version in "${versions[@]}"; do - # Trim whitespace if any from version splitting - version=$(echo "$version" | xargs) - if [[ -z "$version" ]]; then continue; fi - - # Skip if version was already pushed - if version_pushed "$version"; then - echo "Version $version was already pushed, skipping..." - continue - fi - - echo "Pushing version $version..." - ( - # Use process substitution to feed file list safely if it gets long - oras push --disable-path-validation \ - "ghcr.io/$organization/$image_name:$version" \ - "${files[@]}" - echo "Completed push for version $version" - mark_version_pushed "$version" - ) & - push_pids+=($!) - done - - # Wait for all pushes to complete - for pid in "${push_pids[@]}"; do - wait "$pid" - done - - # --- Cleanup only if all versions were pushed successfully --- - all_versions_pushed=true - for version in "${versions[@]}"; do - version=$(echo "$version" | xargs) - if [[ -z "$version" ]]; then continue; fi - if ! version_pushed "$version"; then - all_versions_pushed=false - break - fi - done - - if [ "$all_versions_pushed" = true ]; then - echo "All versions pushed successfully, cleaning up cache directory..." - cd "$work_dir" - rm -rf "$cache_dir" - else - echo "Some versions failed to push. Cache directory preserved at: $cache_dir" - echo "Run again to resume from this point" - fi - -else - echo "Warning: $disk_img_orig not found." - - # If in dry run mode, just show what would happen - if [[ "$dry_run" = true ]]; then - echo "=== DRY RUN MODE ===" - if [ ${#files[@]} -gt 0 ]; then - echo "The following non-disk files would be pushed:" - for file_info in "${files[@]}"; do - file_path="${file_info%%:*}" - file_metadata="${file_info#*:}" - file_size=$(du -h "$file_path" | cut -f1) - echo " - $file_path ($file_size) with metadata: $file_metadata" - done - else - echo "No files found to push." - fi - echo "All processing tasks completed. No actual push performed." - exit 0 - fi - - # Push only config/nvram if they exist - if [ ${#files[@]} -gt 0 ]; then - echo "Pushing non-disk files..." - push_pids=() - IFS=',' read -ra versions <<< "$image_versions" - for version in "${versions[@]}"; do - # Trim whitespace if any from version splitting - version=$(echo "$version" | xargs) - if [[ -z "$version" ]]; then continue; fi - - # Skip if version was already pushed - if version_pushed "$version"; then - echo "Version $version was already pushed, skipping..." - continue - fi - - echo "Pushing version $version (config/nvram only)..." - ( - oras push --disable-path-validation \ - "ghcr.io/$organization/$image_name:$version" \ - "${files[@]}" - echo "Completed push for version $version" - mark_version_pushed "$version" - ) & - push_pids+=($!) - done - - # Wait for all pushes to complete - for pid in "${push_pids[@]}"; do - wait "$pid" - done - - # --- Cleanup only if all versions were pushed successfully --- - all_versions_pushed=true - for version in "${versions[@]}"; do - version=$(echo "$version" | xargs) - if [[ -z "$version" ]]; then continue; fi - if ! version_pushed "$version"; then - all_versions_pushed=false - break - fi - done - - if [ "$all_versions_pushed" = true ]; then - echo "All non-disk versions pushed successfully, cleaning up cache directory..." - cd "$work_dir" - rm -rf "$cache_dir" - else - echo "Some non-disk versions failed to push. Cache directory preserved at: $cache_dir" - echo "Run again to resume from this point" - fi - else - echo "No files found to push." - cd "$work_dir" - rm -rf "$cache_dir" - exit 1 - fi -fi - -# Skip final status check in dry-run mode -if [[ "$dry_run" = true ]]; then - exit 0 -fi - -# Determine final status based on the success check *before* potential cleanup -echo # Add a newline for better readability -if [ "$all_versions_pushed" = true ]; then - echo "All versions pushed successfully:" - for version in "${versions[@]}"; do - version=$(echo "$version" | xargs) - if [[ -z "$version" ]]; then continue; fi - echo " Upload complete: ghcr.io/$organization/$image_name:$version" - done -else - echo "Final upload status:" - for version in "${versions[@]}"; do - version=$(echo "$version" | xargs) - if [[ -z "$version" ]]; then continue; fi - # Check the marker file only if the overall process failed (cache preserved) - if version_pushed "$version"; then - echo " Upload complete: ghcr.io/$organization/$image_name:$version" - else - echo " Upload failed: ghcr.io/$organization/$image_name:$version" - fi - done - # Exit with error code if any version failed - exit 1 -fi