diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 7d8a845..1a34ac0 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -114,3 +114,55 @@ jobs: quickstack/job-backup-mongodb:latest-amd64 \ quickstack/job-backup-mongodb:latest-arm64 docker buildx imagetools inspect quickstack/job-backup-mongodb:latest + + build-and-push-postgres-backup: + runs-on: ubuntu-latest + needs: test + if: ${{ needs.test.result == 'success' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: docker.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Build and Push PostgreSQL Backup (amd64) + uses: docker/build-push-action@v4 + with: + context: ./additional-containers/postgres-backup + file: ./additional-containers/postgres-backup/Dockerfile.amd64 + push: true + platforms: linux/amd64 + build-args: | + VERSION_ARG=${{ github.ref_name }} + tags: | + quickstack/job-backup-postgres:latest-amd64 + + - name: Build and Push PostgreSQL Backup (arm64) + uses: docker/build-push-action@v4 + with: + context: ./additional-containers/postgres-backup + file: ./additional-containers/postgres-backup/Dockerfile.arm64 + push: true + platforms: linux/arm64 + build-args: | + VERSION_ARG=${{ github.ref_name }} + tags: | + quickstack/job-backup-postgres:latest-arm64 + + - name: Create multi-arch manifest (latest) + run: | + docker buildx imagetools create -t quickstack/job-backup-postgres:latest \ + quickstack/job-backup-postgres:latest-amd64 \ + quickstack/job-backup-postgres:latest-arm64 + docker buildx imagetools inspect quickstack/job-backup-postgres:latest diff --git a/.github/workflows/canary-release.yml b/.github/workflows/canary-release.yml index 2cb2a17..bce09ea 100644 --- a/.github/workflows/canary-release.yml +++ b/.github/workflows/canary-release.yml @@ -95,3 +95,36 @@ jobs: VERSION_ARG=canary-${{ github.run_number }} tags: | quickstack/job-backup-mongodb:canary + + build-and-push-postgres-backup: + runs-on: ubuntu-latest + needs: test + if: ${{ needs.test.result == 'success' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + registry: docker.io + username: ${{ secrets.REGISTRY_USER }} + password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Build and Push PostgreSQL Backup (amd64) + uses: docker/build-push-action@v4 + with: + context: ./additional-containers/postgres-backup + file: ./additional-containers/postgres-backup/Dockerfile.amd64 + push: true + platforms: linux/amd64 + build-args: | + VERSION_ARG=canary-${{ github.run_number }} + tags: | + quickstack/job-backup-postgres:canary diff --git a/additional-containers/mongodb-backup/backup.sh b/additional-containers/mongodb-backup/backup.sh index 2573f9b..c0f989a 100644 --- a/additional-containers/mongodb-backup/backup.sh +++ b/additional-containers/mongodb-backup/backup.sh @@ -7,7 +7,6 @@ echo "*************************************************************" echo "QuickStack MongoDB Backup Script Version: ${VERSION:-unknown}" echo "*************************************************************" echo "" -echo "" # Check required env vars if [ -z "$MONGODB_URI" ]; then echo "Error: MONGODB_URI is not set"; exit 1; fi @@ -59,7 +58,6 @@ aws s3 cp "$ZIP_FILE" "s3://$S3_BUCKET_NAME/$S3_KEY" --endpoint-url "$S3_ENDPOIN echo "Cleaning up..." rm -rf "$WORK_DIR" -echo "" echo "" echo "******************************" echo "Backup completed successfully." diff --git a/additional-containers/mongodb_including_mongosh/Dockerfile.amd64 b/additional-containers/mongodb_including_mongosh/Dockerfile.amd64 deleted file mode 100644 index b42b098..0000000 --- a/additional-containers/mongodb_including_mongosh/Dockerfile.amd64 +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:22.04 - -# Prevent interactive prompts during package installation -ENV DEBIAN_FRONTEND=noninteractive - -# Install system dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - zip \ - awscli \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# Install MongoDB Database Tools -ENV MONGO_TOOLS_VERSION=100.10.0 -RUN wget -q https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-${MONGO_TOOLS_VERSION}.tgz \ - && tar -zxvf mongodb-database-tools-ubuntu2204-x86_64-${MONGO_TOOLS_VERSION}.tgz \ - && mv mongodb-database-tools-ubuntu2204-x86_64-${MONGO_TOOLS_VERSION}/bin/* /usr/local/bin/ \ - && rm -rf mongodb-database-tools-ubuntu2204-x86_64-${MONGO_TOOLS_VERSION}* - -# Install MongoDB Shell (mongosh) -ENV MONGOSH_VERSION=2.3.3 -RUN wget -q https://downloads.mongodb.com/compass/mongodb-mongosh_${MONGOSH_VERSION}_amd64.deb \ - && dpkg -i mongodb-mongosh_${MONGOSH_VERSION}_amd64.deb \ - && rm mongodb-mongosh_${MONGOSH_VERSION}_amd64.deb - -WORKDIR /app - -COPY backup.sh . -RUN chmod +x backup.sh - -CMD ["./backup.sh"] diff --git a/additional-containers/mongodb_including_mongosh/Dockerfile.arm64 b/additional-containers/mongodb_including_mongosh/Dockerfile.arm64 deleted file mode 100644 index 29f4e8a..0000000 --- a/additional-containers/mongodb_including_mongosh/Dockerfile.arm64 +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:22.04 - -# Prevent interactive prompts during package installation -ENV DEBIAN_FRONTEND=noninteractive - -# Install system dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - wget \ - zip \ - awscli \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# Install MongoDB Database Tools -ENV MONGO_TOOLS_VERSION=100.10.0 -RUN wget -q https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-arm64-${MONGO_TOOLS_VERSION}.tgz \ - && tar -zxvf mongodb-database-tools-ubuntu2204-arm64-${MONGO_TOOLS_VERSION}.tgz \ - && mv mongodb-database-tools-ubuntu2204-arm64-${MONGO_TOOLS_VERSION}/bin/* /usr/local/bin/ \ - && rm -rf mongodb-database-tools-ubuntu2204-arm64-${MONGO_TOOLS_VERSION}* - -# Install MongoDB Shell (mongosh) -ENV MONGOSH_VERSION=2.3.3 -RUN wget -q https://downloads.mongodb.com/compass/mongodb-mongosh_${MONGOSH_VERSION}_arm64.deb \ - && dpkg -i mongodb-mongosh_${MONGOSH_VERSION}_arm64.deb \ - && rm mongodb-mongosh_${MONGOSH_VERSION}_arm64.deb - -WORKDIR /app - -COPY backup.sh . -RUN chmod +x backup.sh - -CMD ["./backup.sh"] diff --git a/additional-containers/mongodb_including_mongosh/backup.sh b/additional-containers/mongodb_including_mongosh/backup.sh deleted file mode 100644 index ad64e94..0000000 --- a/additional-containers/mongodb_including_mongosh/backup.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -set -e - -# Check required env vars -if [ -z "$MONGODB_URI" ]; then echo "Error: MONGODB_URI is not set"; exit 1; fi -if [ -z "$S3_ENDPOINT" ]; then echo "Error: S3_ENDPOINT is not set"; exit 1; fi -if [ -z "$S3_ACCESS_KEY_ID" ]; then echo "Error: S3_ACCESS_KEY_ID is not set"; exit 1; fi -if [ -z "$S3_SECRET_KEY" ]; then echo "Error: S3_SECRET_KEY is not set"; exit 1; fi -if [ -z "$S3_BUCKET_NAME" ]; then echo "Error: S3_BUCKET_NAME is not set"; exit 1; fi -if [ -z "$S3_KEY" ]; then echo "Error: S3_KEY is not set"; exit 1; fi -if [ -z "$S3_REGION" ]; then echo "Error: S3_REGION is not set"; exit 1; fi - -echo "Starting backup process..." - -# Create a temporary directory for the dump -WORK_DIR=$(mktemp -d) -DUMP_DIR="$WORK_DIR/dump" -ZIP_FILE="$WORK_DIR/backup.zip" - -# Get list of all databases -echo "Fetching list of databases..." -DATABASES=$(mongosh "$MONGODB_URI" --quiet --eval "db.adminCommand('listDatabases').databases.map(d => d.name).join(' ')") - -if [ -z "$DATABASES" ]; then - echo "Error: No databases found or failed to list databases." - exit 1 -fi - -echo "Found databases: $DATABASES" - -# Create dump directory -mkdir -p "$DUMP_DIR" - -# Dump each database separately -for DB in $DATABASES; do - # Skip admin, config, and local databases (system databases) - if [ "$DB" = "admin" ] || [ "$DB" = "config" ] || [ "$DB" = "local" ]; then - echo "Skipping system database: $DB" - continue - fi - - echo "Dumping database: $DB" - mongodump --uri="$MONGODB_URI" --db="$DB" --forceTableScan --out="$DUMP_DIR" - - if [ $? -ne 0 ]; then - echo "Warning: Failed to dump database $DB, continuing with others..." - fi -done - -# Check if dump was successful (directory exists and is not empty) -if [ ! -d "$DUMP_DIR" ] || [ -z "$(ls -A $DUMP_DIR)" ]; then - echo "Error: No databases were dumped successfully." - exit 1 -fi - -# Zip all dumps -echo "Zipping all dumps..." -cd "$DUMP_DIR" -zip -r "$ZIP_FILE" . -cd "$WORK_DIR" - -# Configure AWS CLI environment variables -export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY_ID" -export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" -export AWS_DEFAULT_REGION="$S3_REGION" - -# Upload to S3 -echo "Uploading to S3..." -echo "Destination: s3://$S3_BUCKET_NAME/$S3_KEY" -echo "Endpoint: $S3_ENDPOINT" - -aws s3 cp "$ZIP_FILE" "s3://$S3_BUCKET_NAME/$S3_KEY" --endpoint-url "$S3_ENDPOINT" - -# Cleanup -echo "Cleaning up..." -rm -rf "$WORK_DIR" - -echo "Backup completed successfully." diff --git a/additional-containers/postgres-backup/Dockerfile.amd64 b/additional-containers/postgres-backup/Dockerfile.amd64 new file mode 100644 index 0000000..b985fa0 --- /dev/null +++ b/additional-containers/postgres-backup/Dockerfile.amd64 @@ -0,0 +1,30 @@ +FROM ubuntu:22.04 + +ARG VERSION_ARG=unknown +ENV VERSION=${VERSION_ARG}-amd64 + +# Prevent interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + gnupg \ + lsb-release \ + zip \ + awscli \ + && install -d /usr/share/postgresql-common/pgdg \ + && curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + && . /etc/os-release \ + && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y postgresql-client-18 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY backup.sh . +RUN chmod +x backup.sh + +CMD ["./backup.sh"] diff --git a/additional-containers/postgres-backup/Dockerfile.arm64 b/additional-containers/postgres-backup/Dockerfile.arm64 new file mode 100644 index 0000000..ab5bcef --- /dev/null +++ b/additional-containers/postgres-backup/Dockerfile.arm64 @@ -0,0 +1,30 @@ +FROM ubuntu:22.04 + +ARG VERSION_ARG=unknown +ENV VERSION=${VERSION_ARG}-arm64 + +# Prevent interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + gnupg \ + lsb-release \ + zip \ + awscli \ + && install -d /usr/share/postgresql-common/pgdg \ + && curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + && . /etc/os-release \ + && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y postgresql-client-18 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY backup.sh . +RUN chmod +x backup.sh + +CMD ["./backup.sh"] diff --git a/additional-containers/postgres-backup/backup.sh b/additional-containers/postgres-backup/backup.sh new file mode 100644 index 0000000..dda3f82 --- /dev/null +++ b/additional-containers/postgres-backup/backup.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -e + +echo "" +echo "" +echo "*************************************************************" +echo "QuickStack PostgreSQL Backup Script Version: ${VERSION:-unknown}" +echo "*************************************************************" +echo "" + +# Check required env vars +if [ -z "$POSTGRES_HOST" ]; then echo "Error: POSTGRES_HOST is not set"; exit 1; fi +if [ -z "$POSTGRES_PORT" ]; then echo "Error: POSTGRES_PORT is not set"; exit 1; fi +if [ -z "$POSTGRES_USER" ]; then echo "Error: POSTGRES_USER is not set"; exit 1; fi +if [ -z "$POSTGRES_PASSWORD" ]; then echo "Error: POSTGRES_PASSWORD is not set"; exit 1; fi +if [ -z "$POSTGRES_DB" ]; then echo "Error: POSTGRES_DB is not set"; exit 1; fi +if [ -z "$S3_ENDPOINT" ]; then echo "Error: S3_ENDPOINT is not set"; exit 1; fi +if [ -z "$S3_ACCESS_KEY_ID" ]; then echo "Error: S3_ACCESS_KEY_ID is not set"; exit 1; fi +if [ -z "$S3_SECRET_KEY" ]; then echo "Error: S3_SECRET_KEY is not set"; exit 1; fi +if [ -z "$S3_BUCKET_NAME" ]; then echo "Error: S3_BUCKET_NAME is not set"; exit 1; fi +if [ -z "$S3_KEY" ]; then echo "Error: S3_KEY is not set"; exit 1; fi +if [ -z "$S3_REGION" ]; then echo "Error: S3_REGION is not set"; exit 1; fi + +echo "Starting backup process..." + +# Create a temporary directory for the dump +WORK_DIR=$(mktemp -d) +DUMP_FILE="$WORK_DIR/backup.sql" +ZIP_FILE="$WORK_DIR/backup.zip" + +# Set PGPASSWORD for pg_dump +export PGPASSWORD="$POSTGRES_PASSWORD" + +# Run pg_dump +echo "Running pg_dump..." +pg_dump -h "$POSTGRES_HOST" \ + -p "$POSTGRES_PORT" \ + -U "$POSTGRES_USER" \ + -d "$POSTGRES_DB" \ + -F p \ + -f "$DUMP_FILE" + +# Check if dump was successful (file exists and is not empty) +if [ ! -f "$DUMP_FILE" ] || [ ! -s "$DUMP_FILE" ]; then + echo "Error: pg_dump failed or produced no output." + exit 1 +fi + +# Zip the dump +echo "Zipping dump..." +cd "$WORK_DIR" +zip "$ZIP_FILE" "backup.sql" + +# Configure AWS CLI environment variables +export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY_ID" +export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" +export AWS_DEFAULT_REGION="$S3_REGION" + +# Upload to S3 +echo "Uploading to S3..." +echo "Destination: s3://$S3_BUCKET_NAME/$S3_KEY" +echo "Endpoint: $S3_ENDPOINT" + +aws s3 cp "$ZIP_FILE" "s3://$S3_BUCKET_NAME/$S3_KEY" --endpoint-url "$S3_ENDPOINT" + +# Cleanup +echo "Cleaning up..." +rm -rf "$WORK_DIR" + +echo "" +echo "******************************" +echo "Backup completed successfully." +echo "******************************" +echo "" +echo "" diff --git a/additional-containers/mongodb_including_mongosh/docker-compose.yml b/additional-containers/postgres-backup/docker-compose.yml similarity index 52% rename from additional-containers/mongodb_including_mongosh/docker-compose.yml rename to additional-containers/postgres-backup/docker-compose.yml index 1105e91..b57b9eb 100644 --- a/additional-containers/mongodb_including_mongosh/docker-compose.yml +++ b/additional-containers/postgres-backup/docker-compose.yml @@ -1,16 +1,17 @@ version: '3.8' services: - # The MongoDB instance to backup - mongo: - image: mongo:6.0 + # The PostgreSQL instance to backup + postgres: + image: postgres:18 environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: testdb ports: - - "27017:27017" + - "5432:5432" healthcheck: - test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] + test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s timeout: 5s retries: 5 @@ -22,16 +23,20 @@ services: context: . dockerfile: Dockerfile.arm64 depends_on: - mongo: + postgres: condition: service_healthy environment: - # Connection string to the mongo service - MONGODB_URI: "mongodb://root:password@mongo:27017/?authSource=admin" + # Connection parameters to the postgres service + POSTGRES_HOST: "postgres" + POSTGRES_PORT: "5432" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "password" + POSTGRES_DB: "testdb" # S3 Configuration (Example using MinIO or AWS) S3_ENDPOINT: "" # or http://minio:9000 S3_BUCKET_NAME: "" - S3_REGION: "us-east-1" + S3_REGION: "" S3_ACCESS_KEY_ID: "" S3_SECRET_KEY: "" S3_KEY: "backup-2025-01-01.zip"