From ebd1db2f09da9631331cd275a853f058cc49f990 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Mon, 18 Aug 2025 16:56:34 +0100 Subject: [PATCH] chore(ci): Build modified backends on PR (#6086) * chore(stablediffusion-ggml): rm redundant comment Signed-off-by: Richard Palethorpe * chore(ci): Build modified backends on PR Signed-off-by: Richard Palethorpe --------- Signed-off-by: Richard Palethorpe --- .github/workflows/backend_build.yml | 12 ++-- .github/workflows/backend_pr.yml | 58 ++++++++++++++++++ backend/go/stablediffusion-ggml/main.go | 1 - scripts/changed-backends.js | 79 +++++++++++++++++++++++++ 4 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/backend_pr.yml create mode 100644 scripts/changed-backends.js diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index 8664d7371..cb5af03e4 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -55,9 +55,9 @@ on: type: string secrets: dockerUsername: - required: true + required: false dockerPassword: - required: true + required: false quayUsername: required: true quayPassword: @@ -66,6 +66,8 @@ on: jobs: backend-build: runs-on: ${{ inputs.runs-on }} + env: + quay_username: ${{ secrets.quayUsername }} steps: @@ -187,7 +189,7 @@ jobs: password: ${{ secrets.dockerPassword }} - name: Login to Quay.io - # if: github.event_name != 'pull_request' + if: ${{ env.quay_username != '' }} uses: docker/login-action@v3 with: registry: quay.io @@ -230,7 +232,7 @@ jobs: file: ${{ inputs.dockerfile }} cache-from: type=gha platforms: ${{ inputs.platforms }} - push: true + push: ${{ env.quay_username != '' }} tags: ${{ steps.meta_pull_request.outputs.tags }} labels: ${{ steps.meta_pull_request.outputs.labels }} @@ -238,4 +240,4 @@ jobs: - name: job summary run: | - echo "Built image: ${{ steps.meta.outputs.labels }}" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "Built image: ${{ steps.meta.outputs.labels }}" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/backend_pr.yml b/.github/workflows/backend_pr.yml new file mode 100644 index 000000000..f6de5cec7 --- /dev/null +++ b/.github/workflows/backend_pr.yml @@ -0,0 +1,58 @@ +name: 'build backend container images (PR-filtered)' + +on: + pull_request: + +concurrency: + group: ci-backends-pr-${{ github.head_ref || github.ref }}-${{ github.repository }} + cancel-in-progress: true + +jobs: + generate-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + has-backends: ${{ steps.set-matrix.outputs.has-backends }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + + - name: Install dependencies + run: | + bun add js-yaml + bun add @octokit/core + + # filters the matrix in backend.yml + - name: Filter matrix for changed backends + id: set-matrix + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_PATH: ${{ github.event_path }} + run: bun run scripts/changed-backends.js + + backend-jobs: + needs: generate-matrix + uses: ./.github/workflows/backend_build.yml + if: needs.generate-matrix.outputs.has-backends == 'true' + with: + tag-latest: ${{ matrix.tag-latest }} + tag-suffix: ${{ matrix.tag-suffix }} + build-type: ${{ matrix.build-type }} + cuda-major-version: ${{ matrix.cuda-major-version }} + cuda-minor-version: ${{ matrix.cuda-minor-version }} + platforms: ${{ matrix.platforms }} + runs-on: ${{ matrix.runs-on }} + base-image: ${{ matrix.base-image }} + backend: ${{ matrix.backend }} + dockerfile: ${{ matrix.dockerfile }} + skip-drivers: ${{ matrix.skip-drivers }} + context: ${{ matrix.context }} + secrets: + quayUsername: ${{ secrets.LOCALAI_REGISTRY_USERNAME }} + quayPassword: ${{ secrets.LOCALAI_REGISTRY_PASSWORD }} + strategy: + fail-fast: true + matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} diff --git a/backend/go/stablediffusion-ggml/main.go b/backend/go/stablediffusion-ggml/main.go index d2d87b4da..85841db11 100644 --- a/backend/go/stablediffusion-ggml/main.go +++ b/backend/go/stablediffusion-ggml/main.go @@ -1,6 +1,5 @@ package main -// Note: this is started internally by LocalAI and a server is allocated for each model import ( "flag" diff --git a/scripts/changed-backends.js b/scripts/changed-backends.js new file mode 100644 index 000000000..b8e832d3e --- /dev/null +++ b/scripts/changed-backends.js @@ -0,0 +1,79 @@ +import fs from "fs"; +import yaml from "js-yaml"; +import { Octokit } from "@octokit/core"; + +// Load backend.yml and parse matrix.include +const backendYml = yaml.load(fs.readFileSync(".github/workflows/backend.yml", "utf8")); +const jobs = backendYml.jobs; +const backendJob = jobs["backend-jobs"]; +const strategy = backendJob.strategy; +const matrix = strategy.matrix; +const includes = matrix.include; + +// Set up Octokit for PR changed files +const token = process.env.GITHUB_TOKEN; +const octokit = new Octokit({ auth: token }); + +const eventPath = process.env.GITHUB_EVENT_PATH; +const event = JSON.parse(fs.readFileSync(eventPath, "utf8")); + +let prNumber, repo, owner; +if (event.pull_request) { + prNumber = event.pull_request.number; + repo = event.repository.name; + owner = event.repository.owner.login; +} else { + throw new Error("This workflow must be triggered by a pull_request event."); +} + +async function getChangedFiles() { + let files = []; + let page = 1; + while (true) { + const res = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}/files', { + owner, + repo, + pull_number: prNumber, + per_page: 100, + page + }); + files = files.concat(res.data.map(f => f.filename)); + if (res.data.length < 100) break; + page++; + } + return files; +} + +// Infer backend path +function inferBackendPath(item) { + if (item.dockerfile.endsWith("python")) { + return `backend/python/${item.backend}`; + } + if (item.dockerfile.endsWith("golang")) { + return `backend/go/${item.backend}`; + } + if (item.dockerfile.endsWith("llama-cpp")) { + return `backend/cpp/llama-cpp`; + } + return null; +} + +(async () => { + const changedFiles = await getChangedFiles(); + + console.log("Changed files:", changedFiles); + + const filtered = includes.filter(item => { + const backendPath = inferBackendPath(item); + if (!backendPath) return false; + return changedFiles.some(file => file.startsWith(backendPath)); + }); + + console.log("Filtered files:", filtered); + + const hasBackends = filtered.length > 0 ? 'true' : 'false'; + console.log("Has backends?:", hasBackends); + + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has-backends=${hasBackends}\n`); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `matrix=${JSON.stringify({ include: filtered })}\n`); +})();