From 6b727459c4fe354fd9c4be1b8bb2f5bede28bbcf Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 18 Oct 2021 15:32:16 +0200 Subject: [PATCH 1/5] skip pipepline if files unchanged --- .drone.star | 110 +++++++++++------- .../development/continuous-integration.md | 4 - 2 files changed, 71 insertions(+), 43 deletions(-) diff --git a/.drone.star b/.drone.star index 11833cabdc..72459eae17 100644 --- a/.drone.star +++ b/.drone.star @@ -133,10 +133,10 @@ def main(ctx): pipelines = [] test_pipelines = \ + checkForRecentBuilds(ctx) + \ [buildOcisBinaryForTesting(ctx)] + \ testOcisModules(ctx) + \ - testPipelines(ctx) + \ - checkForRecentBuilds(ctx) + testPipelines(ctx) build_release_pipelines = \ dockerReleases(ctx) + \ @@ -149,36 +149,22 @@ def main(ctx): docs(ctx), ] - if ctx.build.event == "cron": - pipelines = test_pipelines + [ - pipelineDependsOn( - purgeBuildArtifactCache(ctx, "ocis-binary-amd64"), - testPipelines(ctx), - ), - ] + example_deploys(ctx) + test_pipelines.append( + pipelineDependsOn( + purgeBuildArtifactCache(ctx, "ocis-binary-amd64"), + testPipelines(ctx), + ), + ) - elif (ctx.build.event == "pull_request" and "[docs-only]" in ctx.build.title) or \ - (ctx.build.event != "pull_request" and "[docs-only]" in (ctx.build.title + ctx.build.message)): - # [docs-only] is not taken from PR messages, but from commit messages - pipelines = [docs(ctx), changelog(ctx)] + pipelines = test_pipelines + build_release_pipelines + build_release_helpers - else: - test_pipelines.append( - pipelineDependsOn( - purgeBuildArtifactCache(ctx, "ocis-binary-amd64"), - testPipelines(ctx), - ), + pipelines = \ + pipelines + \ + pipelinesDependsOn( + example_deploys(ctx), + pipelines, ) - pipelines = test_pipelines + build_release_pipelines + build_release_helpers - - pipelines = \ - pipelines + \ - pipelinesDependsOn( - example_deploys(ctx), - pipelines, - ) - # always append notification step pipelines.append( pipelineDependsOn( @@ -268,7 +254,7 @@ def testPipelines(ctx): return pipelines def testOcisModule(ctx, module): - steps = makeGenerate(module) + [ + steps = skipIfUnchanged(ctx, "unit-tests") + makeGenerate(module) + [ { "name": "golangci-lint", "image": OC_CI_GOLANG, @@ -339,7 +325,8 @@ def buildOcisBinaryForTesting(ctx): "os": "linux", "arch": "amd64", }, - "steps": makeGenerate("") + + "steps": skipIfUnchanged(ctx, "acceptance-tests") + + makeGenerate("") + build() + rebuildBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis"), "trigger": { @@ -387,6 +374,7 @@ def uploadScanResults(ctx): "git checkout $DRONE_COMMIT", ], }, + ] + skipIfUnchanged(ctx, "unit-tests") + [ { "name": "sync-from-cache", "image": MINIO_MC, @@ -447,7 +435,7 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4): "os": "linux", "arch": "amd64", }, - "steps": restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + + "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + cloneCoreRepos() + [ { @@ -495,7 +483,7 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac "os": "linux", "arch": "amd64", }, - "steps": restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + + "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + cloneCoreRepos() + [ { @@ -606,7 +594,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1, "os": "linux", "arch": "amd64", }, - "steps": restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + + "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + [ { "name": "webUITests", @@ -670,7 +658,7 @@ def accountsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4): "os": "linux", "arch": "amd64", }, - "steps": restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + + "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + [ { "name": "WebUIAcceptanceTests", @@ -735,7 +723,7 @@ def settingsUITests(ctx, storage = "ocis", accounts_hash_difficulty = 4): "os": "linux", "arch": "amd64", }, - "steps": restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + + "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + [ { "name": "WebUIAcceptanceTests", @@ -802,7 +790,7 @@ def failEarly(ctx, early_fail): Returns: pipeline steps """ - if ("full-ci" in ctx.build.title.lower() or ctx.build.event == "cron"): + if ("full-ci" in ctx.build.title.lower() or ctx.build.event == "tag" or ctx.build.event == "cron"): return [] if (early_fail): @@ -884,7 +872,8 @@ def dockerRelease(ctx, arch): "os": "linux", "arch": arch, }, - "steps": makeGenerate("") + [ + "steps": skipIfUnchanged(ctx, "build-docker") + + makeGenerate("") + [ { "name": "build", "image": OC_CI_GOLANG, @@ -957,7 +946,8 @@ def dockerEos(ctx): "os": "linux", "arch": "amd64", }, - "steps": makeGenerate("") + [ + "steps": skipIfUnchanged(ctx, "build-docker") + + makeGenerate("") + [ { "name": "build", "image": OC_CI_GOLANG, @@ -1059,7 +1049,8 @@ def binaryRelease(ctx, name): "os": "linux", "arch": "amd64", }, - "steps": makeGenerate("") + [ + "steps": skipIfUnchanged(ctx, "build-binary") + + makeGenerate("") + [ { "name": "build", "image": OC_CI_GOLANG, @@ -1558,6 +1549,47 @@ def build(): }, ] +def skipIfUnchanged(ctx, type): + if "full-ci" in ctx.build.title.lower(): + return [] + + base = [ + "^docs/.*", + "^changelog/.*", + "^.drone.star$", #TODO: remove + ] + unit = [ + ".*_test.go$", + ] + acceptance = [ + "^tests/acceptance/.*", + ] + + skip = [] + if type == "acceptance-tests": + skip = base + unit + if type == "unit-tests": + skip = base + acceptance + if type == "build-binary" or type == "build-docker": + skip = base + unit + acceptance + + if len(skip) == 0: + return [] + + return [{ + "name": "skip-if-unchanged", + "image": "owncloudci/drone-skip-pipeline", + "pull": "always", + "settings": { + "ALLOW_SKIP_CHANGED": skip, + }, + "when": { + "event": [ + "pull_request", + ], + }, + }] + def example_deploys(ctx): latest_configs = [ "cs3_users_ocis/latest.yml", diff --git a/docs/ocis/development/continuous-integration.md b/docs/ocis/development/continuous-integration.md index ef6310e572..3c9774f656 100644 --- a/docs/ocis/development/continuous-integration.md +++ b/docs/ocis/development/continuous-integration.md @@ -35,10 +35,6 @@ You may add flags to your commit message or PR title in order to speed up pipeli - `[full-ci]`: deactivates the fail early mechanism and runs all available test (as default only smoke tests are run) -- `[docs-only]`: please add this flag, if you only changed documentation. This will only trigger documentation related CI steps. - -- `[tests-only]`: please add this flag, if you only changed tests or test-related tooling. You do not need to add a changelog for tests-only changes. - ### Knowledge base - My pipeline fails because some CI related files or commands are missing. From 1781fcc73ea5e00c4964bd754cd4107782e6765f Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 3 Nov 2021 08:15:37 +0100 Subject: [PATCH 2/5] fix skip --- .drone.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 72459eae17..06fd9d79a4 100644 --- a/.drone.star +++ b/.drone.star @@ -1550,7 +1550,7 @@ def build(): ] def skipIfUnchanged(ctx, type): - if "full-ci" in ctx.build.title.lower(): + if ("full-ci" in ctx.build.title.lower() or ctx.build.event == "tag" or ctx.build.event == "cron"): return [] base = [ From af8b606eeca961beb623c884e47c7d1fe7844512 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 3 Nov 2021 08:21:39 +0100 Subject: [PATCH 3/5] remove .drone.star from ignore pattern --- .drone.star | 1 - 1 file changed, 1 deletion(-) diff --git a/.drone.star b/.drone.star index 06fd9d79a4..32b11b4e8b 100644 --- a/.drone.star +++ b/.drone.star @@ -1556,7 +1556,6 @@ def skipIfUnchanged(ctx, type): base = [ "^docs/.*", "^changelog/.*", - "^.drone.star$", #TODO: remove ] unit = [ ".*_test.go$", From c90dabc9216e73e1ba1ea0cb9461f923a23fb8db Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Tue, 9 Nov 2021 09:05:01 +0100 Subject: [PATCH 4/5] use selfhosted icons in oCIS WOPI deployment example --- deployments/examples/ocis_wopi/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployments/examples/ocis_wopi/docker-compose.yml b/deployments/examples/ocis_wopi/docker-compose.yml index 473f9db594..d67da4d99a 100644 --- a/deployments/examples/ocis_wopi/docker-compose.yml +++ b/deployments/examples/ocis_wopi/docker-compose.yml @@ -97,7 +97,7 @@ services: OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please} APP_PROVIDER_DRIVER: wopi APP_PROVIDER_WOPI_DRIVER_APP_NAME: Collabora - APP_PROVIDER_WOPI_DRIVER_APP_ICON_URI: https://www.collaboraoffice.com/wp-content/uploads/2019/01/CP-icon.png + APP_PROVIDER_WOPI_DRIVER_APP_ICON_URI: https://${COLLABORA_DOMAIN:-collabora.owncloud.test}/favicon.ico APP_PROVIDER_WOPI_DRIVER_APP_URL: https://${COLLABORA_DOMAIN:-collabora.owncloud.test} APP_PROVIDER_WOPI_DRIVER_INSECURE: "${INSECURE:-false}" APP_PROVIDER_WOPI_DRIVER_IOP_SECRET: ${WOPI_IOP_SECRET:-LoremIpsum123} @@ -123,7 +123,7 @@ services: OCIS_MACHINE_AUTH_API_KEY: ${OCIS_MACHINE_AUTH_API_KEY:-change-me-please} APP_PROVIDER_DRIVER: wopi APP_PROVIDER_WOPI_DRIVER_APP_NAME: OnlyOffice - APP_PROVIDER_WOPI_DRIVER_APP_ICON_URI: https://www.pikpng.com/pngl/m/343-3435764_onlyoffice-desktop-editors-onlyoffice-logo-clipart.png + APP_PROVIDER_WOPI_DRIVER_APP_ICON_URI: https://${ONLYOFFICE_DOMAIN:-onlyoffice.owncloud.test}/web-apps/apps/documenteditor/main/resources/img/favicon.ico APP_PROVIDER_WOPI_DRIVER_APP_URL: https://${ONLYOFFICE_DOMAIN:-onlyoffice.owncloud.test} APP_PROVIDER_WOPI_DRIVER_INSECURE: "${INSECURE:-false}" APP_PROVIDER_WOPI_DRIVER_IOP_SECRET: ${WOPI_IOP_SECRET:-LoremIpsum123} @@ -152,7 +152,7 @@ services: APP_PROVIDER_DRIVER: wopi APP_PROVIDER_WOPI_DRIVER_APP_NAME: CodiMD APP_PROVIDER_WOPI_DRIVER_APP_API_KEY: ${CODIMD_SECRET:-LoremIpsum456} - APP_PROVIDER_WOPI_DRIVER_APP_ICON_URI: https://avatars.githubusercontent.com/u/67865462?v=4 + APP_PROVIDER_WOPI_DRIVER_APP_ICON_URI: https://${CODIMD_DOMAIN:-codimd.owncloud.test}/favicon.png APP_PROVIDER_WOPI_DRIVER_APP_URL: https://${CODIMD_DOMAIN:-codimd.owncloud.test} APP_PROVIDER_WOPI_DRIVER_INSECURE: "${INSECURE:-false}" APP_PROVIDER_WOPI_DRIVER_IOP_SECRET: ${WOPI_IOP_SECRET:-LoremIpsum123} From f92e445c4bcdaf3c639251b61670827f83dab7c6 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Tue, 9 Nov 2021 09:10:10 +0100 Subject: [PATCH 5/5] adapt skip patterns --- .drone.star | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 32b11b4e8b..36d5eafba2 100644 --- a/.drone.star +++ b/.drone.star @@ -1554,8 +1554,11 @@ def skipIfUnchanged(ctx, type): return [] base = [ - "^docs/.*", + "^.github/.*", + "^.vscode/.*", "^changelog/.*", + "^docs/.*", + "^deployments/.*", ] unit = [ ".*_test.go$",