mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2025-12-16 17:45:39 -06:00
3254 lines
114 KiB
Plaintext
3254 lines
114 KiB
Plaintext
"""OpenCloud CI definition
|
|
"""
|
|
|
|
# Repository
|
|
|
|
repo_slug = "${CI_REPO}"
|
|
docker_repo_slug = "opencloudeu/opencloud"
|
|
|
|
# images
|
|
ALPINE_GIT = "alpine/git:latest"
|
|
APACHE_TIKA = "apache/tika:3.2.3.0-full"
|
|
CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1"
|
|
COLLABORA_CODE = "collabora/code:24.04.5.1.1"
|
|
OPEN_SEARCH = "opensearchproject/opensearch:2"
|
|
INBUCKET_INBUCKET = "inbucket/inbucket"
|
|
MINIO_MC = "minio/mc:RELEASE.2021-10-07T04-19-58Z"
|
|
OC_CI_ALPINE = "owncloudci/alpine:latest"
|
|
OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier:latest"
|
|
OC_CI_CLAMAVD = "owncloudci/clamavd"
|
|
OC_CI_DRONE_ANSIBLE = "owncloudci/drone-ansible:latest"
|
|
OC_CI_GOLANG = "registry.heinlein.group/opencloud/golang-ci:1.25"
|
|
OC_CI_NODEJS = "owncloudci/nodejs:%s"
|
|
OC_CI_PHP = "owncloudci/php:%s"
|
|
OC_CI_WAIT_FOR = "owncloudci/wait-for:latest"
|
|
OC_CS3_API_VALIDATOR = "opencloudeu/cs3api-validator:latest"
|
|
OC_LITMUS = "owncloudci/litmus:latest"
|
|
OC_UBUNTU = "owncloud/ubuntu:20.04"
|
|
ONLYOFFICE_DOCUMENT_SERVER = "onlyoffice/documentserver:7.5.1"
|
|
PLUGINS_DOCKER_BUILDX = "woodpeckerci/plugin-docker-buildx:latest"
|
|
PLUGINS_NOTATION = "registry.heinlein.group/opencloud/notation-wp-plugin:latest"
|
|
PLUGINS_GITHUB_RELEASE = "woodpeckerci/plugin-release"
|
|
PLUGINS_GIT_ACTION = "quay.io/thegeeklab/wp-git-action"
|
|
PLUGINS_S3 = "plugins/s3:1"
|
|
PLUGINS_S3_CACHE = "plugins/s3-cache:1"
|
|
PLUGINS_SLACK = "plugins/slack:1"
|
|
REDIS = "redis:6-alpine"
|
|
READY_RELEASE_GO = "woodpeckerci/plugin-ready-release-go:latest"
|
|
OPENLDAP = "bitnamilegacy/openldap:2.6"
|
|
|
|
DEFAULT_PHP_VERSION = "8.2"
|
|
DEFAULT_NODEJS_VERSION = "20"
|
|
|
|
CACHE_S3_SERVER = "https://s3.ci.opencloud.eu"
|
|
|
|
dirs = {
|
|
"base": "/woodpecker/src/github.com/%s" % repo_slug,
|
|
"web": "/woodpecker/src/github.com/%s/webTestRunner" % repo_slug,
|
|
"zip": "/woodpecker/src/github.com/%s/zip" % repo_slug,
|
|
"webZip": "/woodpecker/src/github.com/%s/zip/web.tar.gz" % repo_slug,
|
|
"webPnpmZip": "/woodpecker/src/github.com/%s/zip/web-pnpm.tar.gz" % repo_slug,
|
|
"playwrightBrowsersArchive": "/woodpecker/src/github.com/%s/zip/playwright-browsers.tar.gz" % repo_slug,
|
|
"baseGo": "/go/src/github.com/%s" % repo_slug,
|
|
"gobinTar": "go-bin.tar.gz",
|
|
"gobinTarPath": "/go/src/github.com/%s/go-bin.tar.gz" % repo_slug,
|
|
"opencloudConfig": "tests/config/woodpecker/opencloud-config.json",
|
|
"opencloudRevaDataRoot": "/woodpecker/src/github.com/%s/srv/app/tmp/ocis/owncloud/data" % repo_slug,
|
|
"multiServiceOcBaseDataPath": "/woodpecker/src/github.com/%s/multiServiceData" % repo_slug,
|
|
"ocWrapper": "/woodpecker/src/github.com/%s/tests/ocwrapper" % repo_slug,
|
|
"bannedPasswordList": "tests/config/woodpecker/banned-password-list.txt",
|
|
"ocmProviders": "tests/config/woodpecker/providers.json",
|
|
"opencloudBinPath": "opencloud/bin",
|
|
"opencloudBin": "opencloud/bin/opencloud",
|
|
"opencloudBinArtifact": "opencloud-binary-amd64",
|
|
}
|
|
|
|
# OpenCloud URLs
|
|
OC_SERVER_NAME = "opencloud-server"
|
|
OC_URL = "https://%s:9200" % OC_SERVER_NAME
|
|
OC_DOMAIN = "%s:9200" % OC_SERVER_NAME
|
|
FED_OC_SERVER_NAME = "federation-opencloud-server"
|
|
OC_FED_URL = "https://%s:10200" % FED_OC_SERVER_NAME
|
|
OC_FED_DOMAIN = "%s:10200" % FED_OC_SERVER_NAME
|
|
|
|
event = {
|
|
"base": {
|
|
"event": ["push", "manual"],
|
|
"branch": "main",
|
|
},
|
|
"cron": {
|
|
"event": "cron",
|
|
"branch": "main",
|
|
},
|
|
"pull_request": {
|
|
"event": "pull_request",
|
|
},
|
|
"tag": {
|
|
"event": "tag",
|
|
},
|
|
}
|
|
|
|
# configuration
|
|
config = {
|
|
"cs3ApiTests": {
|
|
"skip": False,
|
|
},
|
|
"wopiValidatorTests": {
|
|
"skip": False,
|
|
},
|
|
"k6LoadTests": {
|
|
"skip": True,
|
|
},
|
|
"localApiTests": {
|
|
"basic": {
|
|
"suites": [
|
|
"apiArchiver",
|
|
"apiContract",
|
|
"apiCors",
|
|
"apiAsyncUpload",
|
|
"apiDownloads",
|
|
"apiDepthInfinity",
|
|
"apiLocks",
|
|
"apiActivities",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"settings": {
|
|
"suites": [
|
|
"apiSettings",
|
|
],
|
|
"skip": False,
|
|
"withRemotePhp": [True],
|
|
"emailNeeded": True,
|
|
"extraTestEnvironment": {
|
|
"EMAIL_HOST": "email",
|
|
"EMAIL_PORT": "9000",
|
|
},
|
|
"extraServerEnvironment": {
|
|
"OC_ADD_RUN_SERVICES": "notifications",
|
|
"NOTIFICATIONS_SMTP_HOST": "email",
|
|
"NOTIFICATIONS_SMTP_PORT": "2500",
|
|
"NOTIFICATIONS_SMTP_INSECURE": True,
|
|
"NOTIFICATIONS_SMTP_SENDER": "ownCloud <noreply@example.com>",
|
|
"NOTIFICATIONS_DEBUG_ADDR": "0.0.0.0:9174",
|
|
},
|
|
},
|
|
"graph": {
|
|
"suites": [
|
|
"apiGraph",
|
|
"apiServiceAvailability",
|
|
# skip tests for collaborativePosix. see https://github.com/opencloud-eu/opencloud/issues/2036
|
|
#"collaborativePosix",
|
|
],
|
|
"skip": False,
|
|
"withRemotePhp": [True],
|
|
},
|
|
"graphUserGroup": {
|
|
"suites": [
|
|
"apiGraphUserGroup",
|
|
],
|
|
"skip": False,
|
|
"withRemotePhp": [True],
|
|
},
|
|
"spaces": {
|
|
"suites": [
|
|
"apiSpaces",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"spacesShares": {
|
|
"suites": [
|
|
"apiSpacesShares",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"spacesDavOperation": {
|
|
"suites": [
|
|
"apiSpacesDavOperation",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"search1": {
|
|
"suites": [
|
|
"apiSearch1",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"search2": {
|
|
"suites": [
|
|
"apiSearch2",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"sharingNg": {
|
|
"suites": [
|
|
"apiReshare",
|
|
"apiSharingNg1",
|
|
"apiSharingNg2",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"sharingNgShareInvitation": {
|
|
"suites": [
|
|
"apiSharingNgShareInvitation",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"sharingNgLinkShare": {
|
|
"suites": [
|
|
"apiSharingNgLinkSharePermission",
|
|
"apiSharingNgLinkShareRoot",
|
|
],
|
|
"skip": False,
|
|
},
|
|
"accountsHashDifficulty": {
|
|
"skip": False,
|
|
"suites": [
|
|
"apiAccountsHashDifficulty",
|
|
],
|
|
"accounts_hash_difficulty": "default",
|
|
},
|
|
"notification": {
|
|
"suites": [
|
|
"apiNotification",
|
|
],
|
|
"skip": False,
|
|
"withRemotePhp": [True],
|
|
"emailNeeded": True,
|
|
"extraTestEnvironment": {
|
|
"EMAIL_HOST": "email",
|
|
"EMAIL_PORT": "9000",
|
|
},
|
|
"extraServerEnvironment": {
|
|
"OC_ADD_RUN_SERVICES": "notifications",
|
|
"NOTIFICATIONS_SMTP_HOST": "email",
|
|
"NOTIFICATIONS_SMTP_PORT": "2500",
|
|
"NOTIFICATIONS_SMTP_INSECURE": True,
|
|
"NOTIFICATIONS_SMTP_SENDER": "ownCloud <noreply@example.com>",
|
|
"NOTIFICATIONS_DEBUG_ADDR": "0.0.0.0:9174",
|
|
},
|
|
},
|
|
"antivirus": {
|
|
"suites": [
|
|
"apiAntivirus",
|
|
],
|
|
"skip": False,
|
|
"antivirusNeeded": True,
|
|
"generateVirusFiles": True,
|
|
"extraServerEnvironment": {
|
|
"ANTIVIRUS_SCANNER_TYPE": "clamav",
|
|
"ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310",
|
|
"POSTPROCESSING_STEPS": "virusscan",
|
|
"OC_ASYNC_UPLOADS": True,
|
|
"OC_ADD_RUN_SERVICES": "antivirus",
|
|
"ANTIVIRUS_DEBUG_ADDR": "0.0.0.0:9297",
|
|
},
|
|
},
|
|
"searchContent": {
|
|
"suites": [
|
|
"apiSearchContent",
|
|
],
|
|
"skip": False,
|
|
"tikaNeeded": True,
|
|
},
|
|
"ocm": {
|
|
"suites": [
|
|
"apiOcm",
|
|
],
|
|
"skip": False,
|
|
"withRemotePhp": [True],
|
|
"federationServer": True,
|
|
"emailNeeded": True,
|
|
"extraTestEnvironment": {
|
|
"EMAIL_HOST": "email",
|
|
"EMAIL_PORT": "9000",
|
|
},
|
|
"extraServerEnvironment": {
|
|
"OC_ADD_RUN_SERVICES": "ocm,notifications",
|
|
"OC_ENABLE_OCM": True,
|
|
"OCM_OCM_INVITE_MANAGER_INSECURE": True,
|
|
"OCM_OCM_SHARE_PROVIDER_INSECURE": True,
|
|
"OCM_OCM_STORAGE_PROVIDER_INSECURE": True,
|
|
"OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE": "%s" % dirs["ocmProviders"],
|
|
# mail notifications
|
|
"NOTIFICATIONS_SMTP_HOST": "email",
|
|
"NOTIFICATIONS_SMTP_PORT": "2500",
|
|
"NOTIFICATIONS_SMTP_INSECURE": True,
|
|
"NOTIFICATIONS_SMTP_SENDER": "ownCloud <noreply@example.com>",
|
|
},
|
|
},
|
|
"wopi": {
|
|
"suites": [
|
|
"apiCollaboration",
|
|
],
|
|
"skip": False,
|
|
"collaborationServiceNeeded": True,
|
|
"extraServerEnvironment": {
|
|
"GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
|
|
},
|
|
},
|
|
"authApp": {
|
|
"suites": [
|
|
"apiAuthApp",
|
|
],
|
|
"skip": False,
|
|
"withRemotePhp": [True],
|
|
},
|
|
"cliCommands": {
|
|
"suites": [
|
|
"cliCommands",
|
|
],
|
|
"skip": False,
|
|
"withRemotePhp": [True],
|
|
"antivirusNeeded": True,
|
|
"generateVirusFiles": True,
|
|
"extraServerEnvironment": {
|
|
"ANTIVIRUS_SCANNER_TYPE": "clamav",
|
|
"ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310",
|
|
"OC_ASYNC_UPLOADS": True,
|
|
"OC_ADD_RUN_SERVICES": "antivirus",
|
|
"STORAGE_USERS_DRIVER": "decomposed",
|
|
},
|
|
},
|
|
"multiTenancy": {
|
|
"suites": [
|
|
"apiTenancy",
|
|
],
|
|
"skip": False,
|
|
"withRemotePhp": [True],
|
|
"ldapNeeded": True,
|
|
"extraTestEnvironment": {
|
|
"USE_PREPARED_LDAP_USERS": True,
|
|
},
|
|
"extraServerEnvironment": {
|
|
"OC_MULTI_TENANT_ENABLED": True,
|
|
"OC_LDAP_USER_SCHEMA_TENANT_ID": "departmentNumber",
|
|
"OC_LDAP_URI": "ldaps://ldap-server:1636",
|
|
"OC_LDAP_INSECURE": True,
|
|
"OC_LDAP_BIND_DN": "cn=admin,dc=opencloud,dc=eu",
|
|
"OC_LDAP_BIND_PASSWORD": "admin",
|
|
"OC_LDAP_GROUP_BASE_DN": "ou=groups,dc=opencloud,dc=eu",
|
|
"OC_LDAP_GROUP_SCHEMA_ID": "entryUUID",
|
|
"OC_LDAP_USER_BASE_DN": "ou=users,dc=opencloud,dc=eu",
|
|
"OC_LDAP_USER_FILTER": "(objectclass=inetOrgPerson)",
|
|
"OC_LDAP_USER_SCHEMA_ID": "entryUUID",
|
|
"OC_LDAP_DISABLE_USER_MECHANISM": "none",
|
|
"GRAPH_IDENTITY_BACKEND": "cs3",
|
|
"GRAPH_LDAP_SERVER_UUID": True,
|
|
"GRAPH_LDAP_GROUP_CREATE_BASE_DN": "ou=custom,ou=groups,dc=opencloud,dc=eu",
|
|
"GRAPH_LDAP_REFINT_ENABLED": True,
|
|
"GROUPS_DRIVER": "null",
|
|
"FRONTEND_READONLY_USER_ATTRIBUTES": "user.onPremisesSamAccountName,user.displayName,user.mail,user.passwordProfile,user.accountEnabled,user.appRoleAssignments",
|
|
"OC_LDAP_SERVER_WRITE_ENABLED": False,
|
|
"OC_EXCLUDE_RUN_SERVICES": "idm",
|
|
"OC_LDAP_USER_ENABLED_ATTRIBUTE": "",
|
|
},
|
|
},
|
|
},
|
|
"apiTests": {
|
|
"numberOfParts": 7,
|
|
"skip": False,
|
|
"skipExceptParts": [],
|
|
},
|
|
"e2eTests": {
|
|
"part": {
|
|
"skip": False,
|
|
"totalParts": 4, # divide and run all suites in parts (divide pipelines)
|
|
"xsuites": ["search", "app-provider", "app-provider-onlyOffice", "app-store", "keycloak", "oidc", "ocm", "a11y", "mobile-view", "navigation"], # suites to skip
|
|
},
|
|
"search": {
|
|
"skip": False,
|
|
"suites": ["search"], # suites to run
|
|
"tikaNeeded": True,
|
|
},
|
|
},
|
|
"e2eMultiService": {
|
|
"testSuites": {
|
|
"skip": False,
|
|
"suites": [
|
|
"smoke",
|
|
"shares",
|
|
"search",
|
|
"journeys",
|
|
"file-action",
|
|
"spaces",
|
|
],
|
|
"tikaNeeded": True,
|
|
},
|
|
},
|
|
"rocketchat": {
|
|
"channel": "builds",
|
|
"channel_cron": "builds",
|
|
"from_secret": "rocketchat_talk_webhook",
|
|
},
|
|
"binaryReleases": {
|
|
"os": ["linux", "darwin"],
|
|
},
|
|
"dockerReleases": {
|
|
"architectures": ["arm64", "amd64"],
|
|
"production": {
|
|
# NOTE: need to be updated if new production releases are determined
|
|
"tags": ["2.0", "4.0"],
|
|
# NOTE: need to be set to true if patch releases are made from stable-X-branches
|
|
"skip_rolling": "false",
|
|
"repo": docker_repo_slug,
|
|
"build_type": "production",
|
|
},
|
|
"rolling": {
|
|
"repo": docker_repo_slug + "-rolling",
|
|
"build_type": "rolling",
|
|
},
|
|
"daily": {
|
|
"repo": docker_repo_slug + "-rolling",
|
|
"build_type": "daily",
|
|
},
|
|
},
|
|
"litmus": True,
|
|
"codestyle": True,
|
|
}
|
|
|
|
GRAPH_AVAILABLE_ROLES = "b1e2218d-eef8-4d4c-b82d-0f1a1b48f3b5,a8d5fe5e-96e3-418d-825b-534dbdf22b99,fb6c3e19-e378-47e5-b277-9732f9de6e21,58c63c02-1d89-4572-916a-870abc5a1b7d,2d00ce52-1fc2-4dbc-8b95-a73b73395f5a,1c996275-f1c9-4e71-abdf-a42f6495e960,312c0871-5ef7-4b3a-85b6-0e4074c64049,aa97fe03-7980-45ac-9e50-b325749fd7e6,63e64e19-8d43-42ec-a738-2b6af2610efa"
|
|
|
|
# workspace for pipeline to cache Go dependencies between steps of a pipeline
|
|
# to be used in combination with stepVolumeGo
|
|
workspace = \
|
|
{
|
|
"base": "/go",
|
|
"path": "src/github.com/%s/" % repo_slug,
|
|
}
|
|
|
|
# minio mc environment variables
|
|
MINIO_MC_ENV = {
|
|
"CACHE_BUCKET": {
|
|
"from_secret": "cache_s3_bucket",
|
|
},
|
|
"MC_HOST": CACHE_S3_SERVER,
|
|
"AWS_ACCESS_KEY_ID": {
|
|
"from_secret": "cache_s3_access_key",
|
|
},
|
|
"AWS_SECRET_ACCESS_KEY": {
|
|
"from_secret": "cache_s3_secret_key",
|
|
},
|
|
"PUBLIC_BUCKET": "public",
|
|
}
|
|
|
|
CI_HTTP_PROXY_ENV = {
|
|
"HTTP_PROXY": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
"HTTPS_PROXY": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
}
|
|
|
|
def pipelineDependsOn(pipeline, dependant_pipelines):
|
|
if "depends_on" in pipeline.keys():
|
|
pipeline["depends_on"] = pipeline["depends_on"] + getPipelineNames(dependant_pipelines)
|
|
else:
|
|
pipeline["depends_on"] = getPipelineNames(dependant_pipelines)
|
|
return pipeline
|
|
|
|
def pipelinesDependsOn(pipelines, dependant_pipelines):
|
|
pipes = []
|
|
for pipeline in pipelines:
|
|
pipes.append(pipelineDependsOn(pipeline, dependant_pipelines))
|
|
|
|
return pipes
|
|
|
|
def getPipelineNames(pipelines = []):
|
|
"""getPipelineNames returns names of pipelines as a string array
|
|
|
|
Args:
|
|
pipelines: array of woodpecker pipelines
|
|
|
|
Returns:
|
|
names of the given pipelines as string array
|
|
"""
|
|
names = []
|
|
for pipeline in pipelines:
|
|
names.append(pipeline["name"])
|
|
return names
|
|
|
|
def main(ctx):
|
|
"""main is the entrypoint for woodpecker
|
|
|
|
Args:
|
|
ctx: woodpecker passes a context with information which the pipeline can be adapted to
|
|
|
|
Returns:
|
|
none
|
|
"""
|
|
|
|
if ctx.build.event == "cron" and ctx.build.sender == "translation-sync":
|
|
return translation_sync(ctx)
|
|
|
|
is_release_pr = (ctx.build.event == "pull_request" and ctx.build.sender == "openclouders" and "🎉 release" in ctx.build.title.lower())
|
|
if is_release_pr:
|
|
return [licenseCheck(ctx)]
|
|
|
|
build_release_helpers = \
|
|
readyReleaseGo()
|
|
|
|
build_release_helpers.append(
|
|
pipelineDependsOn(
|
|
licenseCheck(ctx),
|
|
getGoBinForTesting(ctx),
|
|
),
|
|
)
|
|
|
|
test_pipelines = \
|
|
codestyle(ctx) + \
|
|
checkGherkinLint(ctx) + \
|
|
checkTestSuitesInExpectedFailures(ctx) + \
|
|
buildWebCache(ctx) + \
|
|
cacheBrowsers(ctx) + \
|
|
getGoBinForTesting(ctx) + \
|
|
buildOpencloudBinaryForTesting(ctx) + \
|
|
checkStarlark(ctx) + \
|
|
build_release_helpers + \
|
|
testOpencloudAndUploadResults(ctx) + \
|
|
testPipelines(ctx)
|
|
|
|
build_release_pipelines = \
|
|
dockerReleases(ctx) + \
|
|
binaryReleases(ctx)
|
|
|
|
test_pipelines.append(
|
|
pipelineDependsOn(
|
|
purgeBuildArtifactCache(ctx),
|
|
testPipelines(ctx),
|
|
),
|
|
)
|
|
|
|
test_pipelines.append(
|
|
pipelineDependsOn(
|
|
purgeBrowserCache(ctx),
|
|
testPipelines(ctx),
|
|
),
|
|
)
|
|
|
|
test_pipelines.append(
|
|
pipelineDependsOn(
|
|
purgeTracingCache(ctx),
|
|
testPipelines(ctx),
|
|
),
|
|
)
|
|
|
|
test_pipelines.append(
|
|
pipelineDependsOn(
|
|
purgeOpencloudWebBuildCache(ctx),
|
|
testPipelines(ctx),
|
|
),
|
|
)
|
|
|
|
test_pipelines.append(
|
|
pipelineDependsOn(
|
|
purgeGoBinCache(ctx),
|
|
testPipelines(ctx),
|
|
),
|
|
)
|
|
|
|
pipelines = test_pipelines + build_release_pipelines + notifyMatrix(ctx)
|
|
|
|
pipelineSanityChecks(pipelines)
|
|
return pipelines
|
|
|
|
def cachePipeline(ctx, name, steps):
|
|
return {
|
|
"name": "build-%s-cache" % name,
|
|
"steps": steps,
|
|
"when": [
|
|
{
|
|
"event": ["push", "manual"],
|
|
"branch": ["main", "stable-*"],
|
|
},
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "base"),
|
|
},
|
|
},
|
|
],
|
|
}
|
|
|
|
def buildWebCache(ctx):
|
|
return [
|
|
cachePipeline(ctx, "web", generateWebCache(ctx)),
|
|
cachePipeline(ctx, "web-pnpm", generateWebPnpmCache(ctx)),
|
|
]
|
|
|
|
def testOpencloudAndUploadResults(ctx):
|
|
pipeline = testOpencloud(ctx)
|
|
|
|
######################################################################
|
|
# The triggers have been disabled for now, since the govulncheck can #
|
|
# not silence single, acceptable vulnerabilities. #
|
|
# See https://github.com/owncloud/ocis/issues/9527 for more details. #
|
|
# FIXME: RE-ENABLE THIS ASAP!!! #
|
|
######################################################################
|
|
|
|
#security_scan = scanOpencloud(ctx)
|
|
#return [security_scan, pipeline, scan_result_upload]
|
|
return [pipeline]
|
|
|
|
def testPipelines(ctx):
|
|
pipelines = []
|
|
|
|
if config["litmus"]:
|
|
pipelines += litmus(ctx, "decomposed")
|
|
|
|
storage = "posix"
|
|
if "[decomposed]" in ctx.build.title.lower():
|
|
storage = "decomposed"
|
|
|
|
if "skip" not in config["cs3ApiTests"] or not config["cs3ApiTests"]["skip"]:
|
|
pipelines.append(cs3ApiTests(ctx, storage, "default"))
|
|
if "skip" not in config["wopiValidatorTests"] or not config["wopiValidatorTests"]["skip"]:
|
|
pipelines.append(wopiValidatorTests(ctx, storage, "builtin", "default"))
|
|
pipelines.append(wopiValidatorTests(ctx, storage, "cs3", "default"))
|
|
|
|
pipelines += localApiTestPipeline(ctx)
|
|
|
|
if "skip" not in config["apiTests"] or not config["apiTests"]["skip"]:
|
|
pipelines += apiTests(ctx)
|
|
|
|
enable_watch_fs = [False]
|
|
if ctx.build.event == "cron":
|
|
enable_watch_fs.append(True)
|
|
|
|
for run_with_watch_fs_enabled in enable_watch_fs:
|
|
pipelines += e2eTestPipeline(ctx, run_with_watch_fs_enabled) + multiServiceE2ePipeline(ctx, run_with_watch_fs_enabled)
|
|
|
|
if ("skip" not in config["k6LoadTests"] or not config["k6LoadTests"]["skip"]) and ("k6-test" in ctx.build.title.lower() or ctx.build.event == "cron"):
|
|
pipelines += k6LoadTests(ctx)
|
|
|
|
return pipelines
|
|
|
|
def getGoBinForTesting(ctx):
|
|
return [{
|
|
"name": "get-go-bin-cache",
|
|
"steps": checkGoBinCache() +
|
|
cacheGoBin(),
|
|
"when": [
|
|
event["tag"],
|
|
event["cron"],
|
|
{
|
|
"event": ["push", "manual"],
|
|
"branch": ["main", "stable-*"],
|
|
},
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "unit-tests"),
|
|
},
|
|
},
|
|
],
|
|
"workspace": workspace,
|
|
}]
|
|
|
|
def checkGoBinCache():
|
|
return [{
|
|
"name": "check-go-bin-cache",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"bash -x %s/tests/config/woodpecker/check_go_bin_cache.sh %s %s" % (dirs["baseGo"], dirs["baseGo"], dirs["gobinTar"]),
|
|
],
|
|
}]
|
|
|
|
def cacheGoBin():
|
|
return [
|
|
{
|
|
"name": "bingo-get",
|
|
"image": OC_CI_GOLANG,
|
|
"commands": [
|
|
". ./.env",
|
|
"if $BIN_CACHE_FOUND; then exit 0; fi",
|
|
"make bingo-update",
|
|
],
|
|
"environment": CI_HTTP_PROXY_ENV,
|
|
},
|
|
{
|
|
"name": "archive-go-bin",
|
|
"image": OC_UBUNTU,
|
|
"commands": [
|
|
". ./.env",
|
|
"if $BIN_CACHE_FOUND; then exit 0; fi",
|
|
"tar -czf %s /go/bin" % dirs["gobinTarPath"],
|
|
],
|
|
},
|
|
{
|
|
"name": "cache-go-bin",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
". ./.env",
|
|
"if $BIN_CACHE_FOUND; then exit 0; fi",
|
|
# .bingo folder will change after 'bingo-get'
|
|
# so get the stored hash of a .bingo folder
|
|
"BINGO_HASH=$(cat %s/.bingo_hash)" % dirs["baseGo"],
|
|
# cache using the minio client to the public bucket (long term bucket)
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -r %s s3/$CACHE_BUCKET/opencloud/go-bin/$BINGO_HASH" % (dirs["gobinTarPath"]),
|
|
],
|
|
},
|
|
]
|
|
|
|
def restoreGoBinCache():
|
|
return [
|
|
{
|
|
"name": "restore-go-bin-cache",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"BINGO_HASH=$(cat %s/.bingo/* | sha256sum | cut -d ' ' -f 1)" % dirs["baseGo"],
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -r -a s3/$CACHE_BUCKET/opencloud/go-bin/$BINGO_HASH/%s %s" % (dirs["gobinTar"], dirs["baseGo"]),
|
|
],
|
|
},
|
|
{
|
|
"name": "extract-go-bin-cache",
|
|
"image": OC_UBUNTU,
|
|
"commands": [
|
|
"tar -xvmf %s -C /" % dirs["gobinTarPath"],
|
|
],
|
|
},
|
|
]
|
|
|
|
def testOpencloud(ctx):
|
|
steps = restoreGoBinCache() + makeGoGenerate("") + [
|
|
{
|
|
"name": "golangci-lint",
|
|
"image": OC_CI_GOLANG,
|
|
"commands": [
|
|
"mkdir -p cache/checkstyle",
|
|
"make ci-golangci-lint",
|
|
"mv checkstyle.xml cache/checkstyle/checkstyle.xml",
|
|
],
|
|
"environment": CI_HTTP_PROXY_ENV,
|
|
},
|
|
{
|
|
"name": "open-search",
|
|
"image": OPEN_SEARCH,
|
|
"detach": True,
|
|
"environment": {
|
|
"discovery.type": "single-node",
|
|
"DISABLE_INSTALL_DEMO_CONFIG": True,
|
|
"DISABLE_SECURITY_PLUGIN": True,
|
|
},
|
|
"entrypoint": ["/usr/share/opensearch/opensearch-docker-entrypoint.sh", "opensearch"],
|
|
},
|
|
{
|
|
"name": "wait-for-open-search",
|
|
"image": OC_CI_ALPINE,
|
|
"commands": [
|
|
"bash -c '" +
|
|
"until curl -sS \"http://open-search:9200/_cat/health?h=status\" | grep \"green\\|yellow\"; do\n" +
|
|
" echo \"Waiting for http://open-search:9200 to be healthy...\"\n" +
|
|
" sleep 5\n" +
|
|
"done\n" +
|
|
"echo \"http://open-search:9200 healthy...\"\n" +
|
|
"'",
|
|
],
|
|
},
|
|
{
|
|
"name": "test",
|
|
"image": OC_CI_GOLANG,
|
|
"environment": {
|
|
"SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES": "http://open-search:9200",
|
|
},
|
|
"commands": [
|
|
"mkdir -p cache/coverage",
|
|
"make test",
|
|
"mv coverage.out cache/coverage/",
|
|
],
|
|
},
|
|
{
|
|
"name": "scan-result-cache",
|
|
"image": PLUGINS_S3,
|
|
"settings": {
|
|
"endpoint": CACHE_S3_SERVER,
|
|
"bucket": "cache",
|
|
"source": "cache/**/*",
|
|
"target": "%s/%s" % (repo_slug, ctx.build.commit + "-${CI_PIPELINE_NUMBER}"),
|
|
"path_style": True,
|
|
"access_key": {
|
|
"from_secret": "cache_s3_access_key",
|
|
},
|
|
"secret_key": {
|
|
"from_secret": "cache_s3_secret_key",
|
|
},
|
|
},
|
|
},
|
|
]
|
|
|
|
return {
|
|
"name": "linting_and_unitTests",
|
|
"steps": steps,
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "unit-tests"),
|
|
},
|
|
},
|
|
],
|
|
"depends_on": getPipelineNames(getGoBinForTesting(ctx)),
|
|
"workspace": workspace,
|
|
}
|
|
|
|
def scanOpencloud(ctx):
|
|
steps = restoreGoBinCache() + makeGoGenerate("") + [
|
|
{
|
|
"name": "govulncheck",
|
|
"image": OC_CI_GOLANG,
|
|
"commands": [
|
|
"make govulncheck",
|
|
],
|
|
"environment": CI_HTTP_PROXY_ENV,
|
|
},
|
|
]
|
|
|
|
return {
|
|
"name": "go-vulnerability-scanning",
|
|
"steps": steps,
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "acceptance-tests"),
|
|
},
|
|
},
|
|
],
|
|
"depends_on": getPipelineNames(getGoBinForTesting(ctx)),
|
|
"workspace": workspace,
|
|
}
|
|
|
|
def buildOpencloudBinaryForTesting(ctx):
|
|
return [{
|
|
"name": "build_opencloud_binary_for_testing",
|
|
"steps": makeNodeGenerate("") +
|
|
makeGoGenerate("") +
|
|
build() +
|
|
rebuildBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]),
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "base"),
|
|
},
|
|
},
|
|
],
|
|
"workspace": workspace,
|
|
}]
|
|
|
|
def vendorbinCodestyle(phpVersion):
|
|
return [{
|
|
"name": "vendorbin-codestyle",
|
|
"image": OC_CI_PHP % phpVersion,
|
|
"environment": {
|
|
"COMPOSER_HOME": "%s/.cache/composer" % dirs["base"],
|
|
},
|
|
"commands": [
|
|
"make vendor-bin-codestyle",
|
|
],
|
|
}]
|
|
|
|
def vendorbinCodesniffer(phpVersion):
|
|
return [{
|
|
"name": "vendorbin-codesniffer",
|
|
"image": OC_CI_PHP % phpVersion,
|
|
"environment": {
|
|
"COMPOSER_HOME": "%s/.cache/composer" % dirs["base"],
|
|
},
|
|
"commands": [
|
|
"make vendor-bin-codesniffer",
|
|
],
|
|
}]
|
|
|
|
def checkTestSuitesInExpectedFailures(ctx):
|
|
return [{
|
|
"name": "check-suites-in-expected-failures",
|
|
"steps": [
|
|
{
|
|
"name": "check-suites",
|
|
"image": OC_CI_ALPINE,
|
|
"commands": [
|
|
"%s/tests/acceptance/check-deleted-suites-in-expected-failure.sh" % dirs["base"],
|
|
],
|
|
},
|
|
],
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "acceptance-tests"),
|
|
},
|
|
},
|
|
],
|
|
}]
|
|
|
|
def checkGherkinLint(ctx):
|
|
return [{
|
|
"name": "check-gherkin-standard",
|
|
"steps": [
|
|
{
|
|
"name": "lint-feature-files",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"commands": [
|
|
"npm install -g @gherlint/gherlint@1.1.0",
|
|
"make test-gherkin-lint",
|
|
],
|
|
},
|
|
],
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "lint"),
|
|
},
|
|
},
|
|
],
|
|
}]
|
|
|
|
def codestyle(ctx):
|
|
pipelines = []
|
|
|
|
if "codestyle" not in config:
|
|
return []
|
|
|
|
default = {
|
|
"phpVersions": [DEFAULT_PHP_VERSION],
|
|
}
|
|
|
|
if "defaults" in config:
|
|
if "codestyle" in config["defaults"]:
|
|
for item in config["defaults"]["codestyle"]:
|
|
default[item] = config["defaults"]["codestyle"][item]
|
|
|
|
codestyleConfig = config["codestyle"]
|
|
|
|
if type(codestyleConfig) == "bool":
|
|
if codestyleConfig:
|
|
# the config has 'codestyle' true, so specify an empty dict that will get the defaults
|
|
codestyleConfig = {}
|
|
else:
|
|
return pipelines
|
|
|
|
if len(codestyleConfig) == 0:
|
|
# 'codestyle' is an empty dict, so specify a single section that will get the defaults
|
|
codestyleConfig = {"doDefault": {}}
|
|
|
|
for category, matrix in codestyleConfig.items():
|
|
params = {}
|
|
for item in default:
|
|
params[item] = matrix[item] if item in matrix else default[item]
|
|
|
|
for phpVersion in params["phpVersions"]:
|
|
name = "coding-standard-php%s" % phpVersion
|
|
|
|
result = {
|
|
"name": name,
|
|
"steps": vendorbinCodestyle(phpVersion) +
|
|
vendorbinCodesniffer(phpVersion) +
|
|
[
|
|
{
|
|
"name": "php-style",
|
|
"image": OC_CI_PHP % phpVersion,
|
|
"commands": [
|
|
"make test-php-style",
|
|
],
|
|
},
|
|
{
|
|
"name": "check-env-var-annotations",
|
|
"image": OC_CI_PHP % phpVersion,
|
|
"commands": [
|
|
"make check-env-var-annotations",
|
|
],
|
|
},
|
|
],
|
|
"depends_on": [],
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "lint"),
|
|
},
|
|
},
|
|
],
|
|
}
|
|
|
|
pipelines.append(result)
|
|
|
|
return pipelines
|
|
|
|
def localApiTestPipeline(ctx):
|
|
pipelines = []
|
|
|
|
with_remote_php = [True]
|
|
enable_watch_fs = [False]
|
|
if ctx.build.event == "cron":
|
|
with_remote_php.append(False)
|
|
enable_watch_fs.append(True)
|
|
|
|
storages = ["posix"]
|
|
if "[decomposed]" in ctx.build.title.lower():
|
|
storages = ["decomposed"]
|
|
|
|
defaults = {
|
|
"suites": {},
|
|
"skip": False,
|
|
"extraTestEnvironment": {},
|
|
"extraServerEnvironment": {},
|
|
"storages": storages,
|
|
"accounts_hash_difficulty": 4,
|
|
"emailNeeded": False,
|
|
"antivirusNeeded": False,
|
|
"tikaNeeded": False,
|
|
"federationServer": False,
|
|
"collaborationServiceNeeded": False,
|
|
"extraCollaborationEnvironment": {},
|
|
"withRemotePhp": with_remote_php,
|
|
"enableWatchFs": enable_watch_fs,
|
|
"ldapNeeded": False,
|
|
"generateVirusFiles": False,
|
|
}
|
|
|
|
if "localApiTests" in config:
|
|
for name, matrix in config["localApiTests"].items():
|
|
if "skip" not in matrix or not matrix["skip"]:
|
|
params = {}
|
|
for item in defaults:
|
|
params[item] = matrix[item] if item in matrix else defaults[item]
|
|
for storage in params["storages"]:
|
|
for run_with_remote_php in params["withRemotePhp"]:
|
|
for run_with_watch_fs_enabled in params["enableWatchFs"]:
|
|
pipeline = {
|
|
"name": "%s-%s%s-%s%s" % ("CLI" if name.startswith("cli") else "API", name, "-withoutRemotePhp" if not run_with_remote_php else "", "decomposed" if name.startswith("cli") else storage, "-watchfs" if run_with_watch_fs_enabled else ""),
|
|
"steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
|
(tikaService() if params["tikaNeeded"] else []) +
|
|
(waitForServices("online-offices", ["collabora:9980", "onlyoffice:443", "fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) +
|
|
(waitForClamavService() if params["antivirusNeeded"] else []) +
|
|
(waitForEmailService() if params["emailNeeded"] else []) +
|
|
(ldapService() if params["ldapNeeded"] else []) +
|
|
(waitForLdapService() if params["ldapNeeded"] else []) +
|
|
opencloudServer(storage, params["accounts_hash_difficulty"], extra_server_environment = params["extraServerEnvironment"], with_wrapper = True, tika_enabled = params["tikaNeeded"], watch_fs_enabled = run_with_watch_fs_enabled) +
|
|
(opencloudServer(storage, params["accounts_hash_difficulty"], deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"], watch_fs_enabled = run_with_watch_fs_enabled) if params["federationServer"] else []) +
|
|
((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) +
|
|
(openCloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) +
|
|
localApiTests(name, params["suites"], storage, params["extraTestEnvironment"], run_with_remote_php, params["generateVirusFiles"]) +
|
|
logRequests(),
|
|
"services": (emailService() if params["emailNeeded"] else []) +
|
|
(clamavService() if params["antivirusNeeded"] else []) +
|
|
((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []),
|
|
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "acceptance-tests"),
|
|
},
|
|
},
|
|
],
|
|
}
|
|
pipelines.append(pipeline)
|
|
return pipelines
|
|
|
|
def localApiTests(name, suites, storage = "decomposed", extra_environment = {}, with_remote_php = False, generate_virus_files = False):
|
|
test_dir = "%s/tests/acceptance" % dirs["base"]
|
|
expected_failures_file = "%s/expected-failures-localAPI-on-%s-storage.md" % (test_dir, storage)
|
|
|
|
environment = {
|
|
"TEST_SERVER_URL": OC_URL,
|
|
"TEST_SERVER_FED_URL": OC_FED_URL,
|
|
"SEND_SCENARIO_LINE_REFERENCES": True,
|
|
"STORAGE_DRIVER": storage,
|
|
"BEHAT_SUITES": ",".join(suites),
|
|
"BEHAT_FILTER_TAGS": "~@skip&&~@skipOnOpencloud-%s-Storage" % storage,
|
|
"EXPECTED_FAILURES_FILE": expected_failures_file,
|
|
"UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0,
|
|
"OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME,
|
|
"WITH_REMOTE_PHP": with_remote_php,
|
|
"COLLABORATION_SERVICE_URL": "http://wopi-fakeoffice:9300",
|
|
"OC_STORAGE_PATH": "$HOME/.opencloud/storage/users",
|
|
"USE_BEARER_TOKEN": True,
|
|
}
|
|
|
|
for item in extra_environment:
|
|
environment[item] = extra_environment[item]
|
|
|
|
commands = []
|
|
|
|
# Generate EICAR virus test files if needed
|
|
if generate_virus_files:
|
|
commands.append("chmod +x %s/tests/acceptance/scripts/generate-virus-files.sh" % dirs["base"])
|
|
commands.append("bash %s/tests/acceptance/scripts/generate-virus-files.sh" % dirs["base"])
|
|
|
|
# Merge expected failures
|
|
if not with_remote_php:
|
|
commands.append("cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file))
|
|
|
|
# Run tests
|
|
commands.append("make -C %s test-acceptance-api" % (dirs["base"]))
|
|
|
|
return [{
|
|
"name": "localApiTests-%s" % name,
|
|
"image": OC_CI_PHP % DEFAULT_PHP_VERSION,
|
|
"environment": environment,
|
|
"commands": commands,
|
|
}]
|
|
|
|
def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4):
|
|
return {
|
|
"name": "cs3ApiTests-%s" % storage,
|
|
"steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
|
opencloudServer(storage, accounts_hash_difficulty, deploy_type = "cs3api_validator") +
|
|
[
|
|
{
|
|
"name": "cs3ApiTests",
|
|
"image": OC_CS3_API_VALIDATOR,
|
|
"environment": {},
|
|
"commands": [
|
|
"apk --no-cache add curl",
|
|
"curl '%s/graph/v1.0/users' -k -X POST --data-raw '{\"onPremisesSamAccountName\":\"marie\",\"displayName\":\"Marie Curie\",\"mail\":\"marie@opencloud.eu\",\"passwordProfile\":{\"password\":\"radioactivity\"}}' -uadmin:admin" % OC_URL,
|
|
"/usr/bin/cs3api-validator /var/lib/cs3api-validator --endpoint=%s:9142" % OC_SERVER_NAME,
|
|
],
|
|
},
|
|
],
|
|
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "acceptance-tests"),
|
|
},
|
|
},
|
|
],
|
|
}
|
|
|
|
def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = 4):
|
|
testgroups = [
|
|
"BaseWopiViewing",
|
|
"CheckFileInfoSchema",
|
|
"EditFlows",
|
|
"Locks",
|
|
"AccessTokens",
|
|
"GetLock",
|
|
"ExtendedLockLength",
|
|
"FileVersion",
|
|
"Features",
|
|
]
|
|
builtinOnlyTestGroups = [
|
|
"PutRelativeFile",
|
|
"RenameFileIfCreateChildFileIsNotSupported",
|
|
]
|
|
|
|
validatorTests = []
|
|
extra_server_environment = {}
|
|
|
|
if wopiServerType == "cs3":
|
|
wopiServer = [
|
|
{
|
|
"name": "wopi-fakeoffice",
|
|
"image": "cs3org/wopiserver:v10.4.0",
|
|
"detach": True,
|
|
"commands": [
|
|
"cp %s/tests/config/woodpecker/wopiserver.conf /etc/wopi/wopiserver.conf" % (dirs["base"]),
|
|
"echo 123 > /etc/wopi/wopisecret",
|
|
"/app/wopiserver.py",
|
|
],
|
|
},
|
|
]
|
|
else:
|
|
extra_server_environment = {
|
|
"OC_EXCLUDE_RUN_SERVICES": "app-provider",
|
|
}
|
|
|
|
wopiServer = wopiCollaborationService("fakeoffice")
|
|
|
|
for testgroup in testgroups:
|
|
validatorTests.append({
|
|
"name": "wopiValidatorTests-%s" % testgroup,
|
|
"image": "owncloudci/wopi-validator",
|
|
"commands": [
|
|
"export WOPI_TOKEN=$(cat accesstoken)",
|
|
"echo $WOPI_TOKEN",
|
|
"export WOPI_TTL=$(cat accesstokenttl)",
|
|
"echo $WOPI_TTL",
|
|
"export WOPI_SRC=$(cat wopisrc)",
|
|
"echo $WOPI_SRC",
|
|
"cd /app",
|
|
"/app/Microsoft.Office.WopiValidator -t $WOPI_TOKEN -w $WOPI_SRC -l $WOPI_TTL --testgroup %s" % testgroup,
|
|
],
|
|
})
|
|
if wopiServerType == "builtin":
|
|
for builtinOnlyGroup in builtinOnlyTestGroups:
|
|
validatorTests.append({
|
|
"name": "wopiValidatorTests-%s" % builtinOnlyGroup,
|
|
"image": "owncloudci/wopi-validator",
|
|
"commands": [
|
|
"export WOPI_TOKEN=$(cat accesstoken)",
|
|
"echo $WOPI_TOKEN",
|
|
"export WOPI_TTL=$(cat accesstokenttl)",
|
|
"echo $WOPI_TTL",
|
|
"export WOPI_SRC=$(cat wopisrc)",
|
|
"echo $WOPI_SRC",
|
|
"cd /app",
|
|
"/app/Microsoft.Office.WopiValidator -s -t $WOPI_TOKEN -w $WOPI_SRC -l $WOPI_TTL --testgroup %s" % builtinOnlyGroup,
|
|
],
|
|
})
|
|
|
|
return {
|
|
"name": "wopiValidatorTests-%s-%s" % (wopiServerType, storage),
|
|
"services": fakeOffice(),
|
|
"steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
|
waitForServices("fake-office", ["fakeoffice:8080"]) +
|
|
opencloudServer(storage, accounts_hash_difficulty, deploy_type = "wopi_validator", extra_server_environment = extra_server_environment) +
|
|
wopiServer +
|
|
waitForServices("wopi-fakeoffice", ["wopi-fakeoffice:9300"]) +
|
|
[
|
|
{
|
|
"name": "prepare-test-file",
|
|
"image": OC_CI_ALPINE,
|
|
"environment": {},
|
|
"commands": [
|
|
"curl -v -X PUT '%s/remote.php/webdav/test.wopitest' -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin -D headers.txt" % OC_URL,
|
|
"cat headers.txt",
|
|
"export FILE_ID=$(cat headers.txt | sed -n -e 's/^.*oc-fileid: //Ip')",
|
|
"export URL=\"%s/app/open?app_name=FakeOffice&file_id=$FILE_ID\"" % OC_URL,
|
|
"export URL=$(echo $URL | tr -d '[:cntrl:]')",
|
|
"curl -v -X POST \"$URL\" -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin > open.json",
|
|
"cat open.json",
|
|
"cat open.json | jq .form_parameters.access_token | tr -d '\"' > accesstoken",
|
|
"cat open.json | jq .form_parameters.access_token_ttl | tr -d '\"' > accesstokenttl",
|
|
"echo -n 'http://wopi-fakeoffice:9300/wopi/files/' > wopisrc",
|
|
"cat open.json | jq .app_url | sed -n -e 's/^.*files%2F//p' | tr -d '\"' >> wopisrc",
|
|
],
|
|
},
|
|
] +
|
|
validatorTests,
|
|
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "acceptance-tests"),
|
|
},
|
|
},
|
|
],
|
|
}
|
|
|
|
def coreApiTests(ctx, part_number = 1, number_of_parts = 1, with_remote_php = False, accounts_hash_difficulty = 4, watch_fs_enabled = False):
|
|
storage = "posix"
|
|
if "[decomposed]" in ctx.build.title.lower():
|
|
storage = "decomposed"
|
|
filterTags = "~@skipOnOpencloud-%s-Storage" % storage
|
|
test_dir = "%s/tests/acceptance" % dirs["base"]
|
|
expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage)
|
|
|
|
return {
|
|
"name": "Core-API-Tests-%s%s-%s%s" % (part_number, "-withoutRemotePhp" if not with_remote_php else "", storage, "-watchfs" if watch_fs_enabled else ""),
|
|
"steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
|
opencloudServer(storage, accounts_hash_difficulty, with_wrapper = True, watch_fs_enabled = watch_fs_enabled) +
|
|
[
|
|
{
|
|
"name": "oC10ApiTests-%s" % part_number,
|
|
"image": OC_CI_PHP % DEFAULT_PHP_VERSION,
|
|
"environment": {
|
|
"TEST_SERVER_URL": OC_URL,
|
|
"OC_REVA_DATA_ROOT": "%s" % (dirs["opencloudRevaDataRoot"] if storage == "owncloud" else ""),
|
|
"SEND_SCENARIO_LINE_REFERENCES": True,
|
|
"STORAGE_DRIVER": storage,
|
|
"BEHAT_FILTER_TAGS": filterTags,
|
|
"DIVIDE_INTO_NUM_PARTS": number_of_parts,
|
|
"RUN_PART": part_number,
|
|
"ACCEPTANCE_TEST_TYPE": "core-api",
|
|
"EXPECTED_FAILURES_FILE": expected_failures_file,
|
|
"UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0,
|
|
"OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME,
|
|
"WITH_REMOTE_PHP": with_remote_php,
|
|
},
|
|
"commands": [
|
|
# merge the expected failures
|
|
"" if with_remote_php else "cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file),
|
|
"make -C %s test-acceptance-api" % (dirs["base"]),
|
|
],
|
|
},
|
|
] +
|
|
logRequests(),
|
|
"services": redisForOCStorage(storage),
|
|
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
|
"when": [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "acceptance-tests"),
|
|
},
|
|
},
|
|
],
|
|
}
|
|
|
|
def apiTests(ctx):
|
|
pipelines = []
|
|
debugParts = config["apiTests"]["skipExceptParts"]
|
|
debugPartsEnabled = (len(debugParts) != 0)
|
|
|
|
with_remote_php = [True]
|
|
enable_watch_fs = [False]
|
|
if ctx.build.event == "cron":
|
|
with_remote_php.append(False)
|
|
enable_watch_fs.append(True)
|
|
|
|
defaults = {
|
|
"withRemotePhp": with_remote_php,
|
|
"enableWatchFs": enable_watch_fs,
|
|
}
|
|
|
|
for runPart in range(1, config["apiTests"]["numberOfParts"] + 1):
|
|
for run_with_remote_php in defaults["withRemotePhp"]:
|
|
for run_with_watch_fs_enabled in defaults["enableWatchFs"]:
|
|
if not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts):
|
|
pipelines.append(coreApiTests(ctx, runPart, config["apiTests"]["numberOfParts"], run_with_remote_php, watch_fs_enabled = run_with_watch_fs_enabled))
|
|
|
|
return pipelines
|
|
|
|
def e2eTestPipeline(ctx, watch_fs_enabled = False):
|
|
defaults = {
|
|
"skip": False,
|
|
"suites": [],
|
|
"xsuites": [],
|
|
"totalParts": 0,
|
|
"tikaNeeded": False,
|
|
"reportTracing": False,
|
|
}
|
|
|
|
extra_server_environment = {
|
|
"OC_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": "%s" % dirs["bannedPasswordList"],
|
|
"OC_SHOW_USER_EMAIL_IN_RESULTS": True,
|
|
# Needed for enabling all roles
|
|
"GRAPH_AVAILABLE_ROLES": "%s" % GRAPH_AVAILABLE_ROLES,
|
|
}
|
|
|
|
e2e_trigger = [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "e2e-tests"),
|
|
},
|
|
},
|
|
{
|
|
"event": "tag",
|
|
"ref": "refs/tags/**",
|
|
},
|
|
]
|
|
|
|
pipelines = []
|
|
|
|
if "skip-e2e" in ctx.build.title.lower():
|
|
return pipelines
|
|
|
|
if ctx.build.event == "tag":
|
|
return pipelines
|
|
|
|
storage = "posix"
|
|
if "[decomposed]" in ctx.build.title.lower():
|
|
storage = "decomposed"
|
|
|
|
for name, suite in config["e2eTests"].items():
|
|
if "skip" in suite and suite["skip"]:
|
|
continue
|
|
|
|
params = {}
|
|
for item in defaults:
|
|
params[item] = suite[item] if item in suite else defaults[item]
|
|
|
|
e2e_args = ""
|
|
if params["totalParts"] > 0:
|
|
e2e_args = "--total-parts %d" % params["totalParts"]
|
|
elif params["suites"]:
|
|
e2e_args = "--suites %s" % ",".join(params["suites"])
|
|
|
|
# suites to skip
|
|
if params["xsuites"]:
|
|
e2e_args += " --xsuites %s" % ",".join(params["xsuites"])
|
|
|
|
if "with-tracing" in ctx.build.title.lower():
|
|
params["reportTracing"] = True
|
|
|
|
steps_before = \
|
|
restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBin"]) + \
|
|
restoreWebCache() + \
|
|
restoreWebPnpmCache() + \
|
|
restoreBrowsersCache() + \
|
|
(tikaService() if params["tikaNeeded"] else []) + \
|
|
opencloudServer(storage, extra_server_environment = extra_server_environment, tika_enabled = params["tikaNeeded"], watch_fs_enabled = watch_fs_enabled)
|
|
|
|
step_e2e = {
|
|
"name": "e2e-tests",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"environment": {
|
|
"OC_BASE_URL": OC_DOMAIN,
|
|
"HEADLESS": True,
|
|
"RETRY": "1",
|
|
"WEB_UI_CONFIG_FILE": "%s/%s" % (dirs["base"], dirs["opencloudConfig"]),
|
|
"LOCAL_UPLOAD_DIR": "/uploads",
|
|
"PLAYWRIGHT_BROWSERS_PATH": "%s/%s" % (dirs["base"], ".playwright"),
|
|
"BROWSER": "chromium",
|
|
"REPORT_TRACING": params["reportTracing"],
|
|
},
|
|
"commands": [
|
|
"cd %s/tests/e2e" % dirs["web"],
|
|
],
|
|
}
|
|
|
|
steps_after = uploadTracingResult(ctx)
|
|
|
|
if params["totalParts"]:
|
|
for index in range(params["totalParts"]):
|
|
run_part = index + 1
|
|
run_e2e = {}
|
|
run_e2e.update(step_e2e)
|
|
run_e2e["commands"] = [
|
|
"cd %s/tests/e2e" % dirs["web"],
|
|
"bash run-e2e.sh %s --run-part %d" % (e2e_args, run_part),
|
|
]
|
|
pipelines.append({
|
|
"name": "e2e-tests-%s-%s-%s%s" % (name, run_part, storage, "-watchfs" if watch_fs_enabled else ""),
|
|
"steps": steps_before + [run_e2e] + steps_after,
|
|
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)),
|
|
"when": e2e_trigger,
|
|
})
|
|
else:
|
|
step_e2e["commands"].append("bash run-e2e.sh %s" % e2e_args)
|
|
pipelines.append({
|
|
"name": "e2e-tests-%s-%s%s" % (name, storage, "-watchfs" if watch_fs_enabled else ""),
|
|
"steps": steps_before + [step_e2e] + steps_after,
|
|
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)),
|
|
"when": e2e_trigger,
|
|
})
|
|
|
|
return pipelines
|
|
|
|
def multiServiceE2ePipeline(ctx, watch_fs_enabled = False):
|
|
pipelines = []
|
|
|
|
defaults = {
|
|
"skip": False,
|
|
"suites": [],
|
|
"xsuites": [],
|
|
"tikaNeeded": False,
|
|
"reportTracing": False,
|
|
}
|
|
|
|
e2e_trigger = [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "e2e-tests"),
|
|
},
|
|
},
|
|
]
|
|
|
|
if "skip-e2e" in ctx.build.title.lower():
|
|
return pipelines
|
|
|
|
# run this pipeline only for cron jobs and full-ci PRs
|
|
if not "full-ci" in ctx.build.title.lower() and ctx.build.event != "cron":
|
|
return pipelines
|
|
|
|
storage = "posix"
|
|
if "[decomposed]" in ctx.build.title.lower():
|
|
storage = "decomposed"
|
|
|
|
extra_server_environment = {
|
|
"OC_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": "%s" % dirs["bannedPasswordList"],
|
|
"OC_JWT_SECRET": "some-opencloud-jwt-secret",
|
|
"OC_SERVICE_ACCOUNT_ID": "service-account-id",
|
|
"OC_SERVICE_ACCOUNT_SECRET": "service-account-secret",
|
|
"OC_EXCLUDE_RUN_SERVICES": "storage-users",
|
|
"OC_GATEWAY_GRPC_ADDR": "0.0.0.0:9142",
|
|
"SETTINGS_GRPC_ADDR": "0.0.0.0:9191",
|
|
"GATEWAY_STORAGE_USERS_MOUNT_ID": "storage-users-id",
|
|
"OC_SHOW_USER_EMAIL_IN_RESULTS": True,
|
|
# Needed for enabling all roles
|
|
"GRAPH_AVAILABLE_ROLES": "%s" % GRAPH_AVAILABLE_ROLES,
|
|
}
|
|
|
|
if watch_fs_enabled:
|
|
extra_server_environment["STORAGE_USERS_POSIX_WATCH_FS"] = True
|
|
|
|
storage_users_environment = {
|
|
"OC_CORS_ALLOW_ORIGINS": "%s,https://%s:9201" % (OC_URL, OC_SERVER_NAME),
|
|
"STORAGE_USERS_JWT_SECRET": "some-opencloud-jwt-secret",
|
|
"STORAGE_USERS_MOUNT_ID": "storage-users-id",
|
|
"STORAGE_USERS_SERVICE_ACCOUNT_ID": "service-account-id",
|
|
"STORAGE_USERS_SERVICE_ACCOUNT_SECRET": "service-account-secret",
|
|
"STORAGE_USERS_GATEWAY_GRPC_ADDR": "%s:9142" % OC_SERVER_NAME,
|
|
"STORAGE_USERS_EVENTS_ENDPOINT": "%s:9233" % OC_SERVER_NAME,
|
|
"STORAGE_USERS_DATA_GATEWAY_URL": "%s/data" % OC_URL,
|
|
"OC_CACHE_STORE": "nats-js-kv",
|
|
"OC_CACHE_STORE_NODES": "%s:9233" % OC_SERVER_NAME,
|
|
"MICRO_REGISTRY_ADDRESS": "%s:9233" % OC_SERVER_NAME,
|
|
"OC_BASE_DATA_PATH": dirs["multiServiceOcBaseDataPath"],
|
|
}
|
|
storage_users1_environment = {
|
|
"STORAGE_USERS_GRPC_ADDR": "storageusers1:9157",
|
|
"STORAGE_USERS_HTTP_ADDR": "storageusers1:9158",
|
|
"STORAGE_USERS_DEBUG_ADDR": "storageusers1:9159",
|
|
"STORAGE_USERS_DATA_SERVER_URL": "http://storageusers1:9158/data",
|
|
}
|
|
for item in storage_users_environment:
|
|
storage_users1_environment[item] = storage_users_environment[item]
|
|
|
|
storage_users2_environment = {
|
|
"STORAGE_USERS_GRPC_ADDR": "storageusers2:9157",
|
|
"STORAGE_USERS_HTTP_ADDR": "storageusers2:9158",
|
|
"STORAGE_USERS_DEBUG_ADDR": "storageusers2:9159",
|
|
"STORAGE_USERS_DATA_SERVER_URL": "http://storageusers2:9158/data",
|
|
}
|
|
for item in storage_users_environment:
|
|
storage_users2_environment[item] = storage_users_environment[item]
|
|
|
|
storage_users_services = startOpenCloudService("storage-users", "storageusers1", storage_users1_environment) + \
|
|
startOpenCloudService("storage-users", "storageusers2", storage_users2_environment) + \
|
|
openCloudHealthCheck("storage-users", ["storageusers1:9159", "storageusers2:9159"])
|
|
|
|
for _, suite in config["e2eMultiService"].items():
|
|
if "skip" in suite and suite["skip"]:
|
|
continue
|
|
|
|
params = {}
|
|
for item in defaults:
|
|
params[item] = suite[item] if item in suite else defaults[item]
|
|
|
|
e2e_args = ""
|
|
if params["suites"]:
|
|
e2e_args = "--suites %s" % ",".join(params["suites"])
|
|
|
|
# suites to skip
|
|
if params["xsuites"]:
|
|
e2e_args += " --xsuites %s" % ",".join(params["xsuites"])
|
|
|
|
if "with-tracing" in ctx.build.title.lower():
|
|
params["reportTracing"] = True
|
|
|
|
steps = \
|
|
restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBin"]) + \
|
|
restoreWebCache() + \
|
|
restoreWebPnpmCache() + \
|
|
restoreBrowsersCache() + \
|
|
tikaService() + \
|
|
opencloudServer(storage, extra_server_environment = extra_server_environment, tika_enabled = params["tikaNeeded"]) + \
|
|
storage_users_services + \
|
|
[{
|
|
"name": "e2e-tests",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"environment": {
|
|
"OC_BASE_URL": OC_DOMAIN,
|
|
"HEADLESS": True,
|
|
"RETRY": "1",
|
|
"REPORT_TRACING": params["reportTracing"],
|
|
"PLAYWRIGHT_BROWSERS_PATH": "%s/%s" % (dirs["base"], ".playwright"),
|
|
"BROWSER": "chromium",
|
|
},
|
|
"commands": [
|
|
"cd %s/tests/e2e" % dirs["web"],
|
|
"bash run-e2e.sh %s" % e2e_args,
|
|
],
|
|
}] + \
|
|
uploadTracingResult(ctx)
|
|
|
|
pipelines.append({
|
|
"name": "e2e-tests-multi-service%s" % ("-watchfs" if watch_fs_enabled else ""),
|
|
"steps": steps,
|
|
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx) + buildWebCache(ctx)),
|
|
"when": e2e_trigger,
|
|
})
|
|
return pipelines
|
|
|
|
def uploadTracingResult(ctx):
|
|
status = ["failure"]
|
|
if "with-tracing" in ctx.build.title.lower():
|
|
status = ["failure", "success"]
|
|
|
|
return [{
|
|
"name": "upload-tracing-result",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -a %s/reports/e2e/playwright/tracing/* s3/$PUBLIC_BUCKET/web/tracing/$CI_REPO_NAME/$CI_PIPELINE_NUMBER/" % dirs["web"],
|
|
"cd %s/reports/e2e/playwright/tracing/" % dirs["web"],
|
|
'echo "To see the trace, please open the following link in the console"',
|
|
'for f in *.zip; do echo "npx playwright show-trace $MC_HOST/$PUBLIC_BUCKET/web/tracing/$CI_REPO_NAME/$CI_PIPELINE_NUMBER/$f \n"; done',
|
|
],
|
|
"when": {
|
|
"status": status,
|
|
},
|
|
}]
|
|
|
|
def dockerReleases(ctx):
|
|
pipelines = []
|
|
docker_releases = []
|
|
build_type = ""
|
|
|
|
# only make realeases on tag events
|
|
if ctx.build.event == "tag":
|
|
tag = ctx.build.ref.replace("refs/tags/v", "").lower()
|
|
|
|
# iterate over production tags to see if this is a production release
|
|
is_production = False
|
|
skip_rolling = False
|
|
for prod_tag in config["dockerReleases"]["production"]["tags"]:
|
|
if tag.startswith(prod_tag):
|
|
is_production = True
|
|
skip_rolling = config["dockerReleases"]["production"]["skip_rolling"]
|
|
break
|
|
|
|
if is_production:
|
|
docker_releases.append("production")
|
|
|
|
# a new production realease is also a rolling release
|
|
# unless skip_rolling is set in the config, i.e. for patch-releases on stable-branch
|
|
if not skip_rolling:
|
|
docker_releases.append("rolling")
|
|
|
|
else:
|
|
docker_releases.append("rolling")
|
|
|
|
# on non tag events, do daily build
|
|
else:
|
|
docker_releases.append("daily")
|
|
|
|
for releaseConfigName in docker_releases:
|
|
repo = config["dockerReleases"][releaseConfigName]["repo"]
|
|
build_type = config["dockerReleases"][releaseConfigName]["build_type"]
|
|
repo_pipelines = []
|
|
repo_pipelines.append(dockerRelease(ctx, repo, build_type))
|
|
|
|
# manifest = releaseDockerManifest(ctx, repo, build_type)
|
|
# manifest["depends_on"] = getPipelineNames(repo_pipelines)
|
|
# repo_pipelines.append(manifest)
|
|
|
|
readme = releaseDockerReadme(repo, build_type)
|
|
readme["depends_on"] = getPipelineNames(repo_pipelines)
|
|
repo_pipelines.append(readme)
|
|
|
|
pipelines.extend(repo_pipelines)
|
|
|
|
return pipelines
|
|
|
|
def dockerRelease(ctx, repo, build_type):
|
|
build_args = {
|
|
"REVISION": "%s" % ctx.build.commit,
|
|
"VERSION": "%s" % (ctx.build.ref.replace("refs/tags/", "") if ctx.build.event == "tag" else "daily"),
|
|
"EDITION": "stable" if build_type == "production" else "rolling",
|
|
}
|
|
|
|
# if no additional tag is given, the build-plugin adds latest
|
|
hard_tag = "daily"
|
|
if ctx.build.event == "tag":
|
|
tag_version = ctx.build.ref.replace("refs/tags/v", "")
|
|
tag_parts = tag_version.split("-")
|
|
|
|
# if a tag has something appended with "-" i.e. alpha, beta, rc1...
|
|
# set the entire string as tag, else tag with latest (same as empty with current plugin)
|
|
hard_tag = tag_version if len(tag_parts) > 1 else "latest"
|
|
|
|
depends_on = getPipelineNames(getGoBinForTesting(ctx))
|
|
|
|
if ctx.build.event == "tag":
|
|
depends_on = []
|
|
|
|
return {
|
|
"name": "container-build-%s" % build_type,
|
|
"steps": makeNodeGenerate("") +
|
|
makeGoGenerate("") + [
|
|
{
|
|
"name": "dryrun",
|
|
"image": PLUGINS_DOCKER_BUILDX,
|
|
"settings": {
|
|
"context": "..",
|
|
"dry_run": True,
|
|
"platforms": "linux/amd64", # do dry run only on the native platform
|
|
"repo": "%s,quay.io/%s,registry.heinlein.group/%s" % (repo, repo, repo),
|
|
"auto_tag": False if build_type == "daily" else True,
|
|
"tag": hard_tag,
|
|
"default_tag": "daily",
|
|
"dockerfile": "opencloud/docker/Dockerfile.multiarch",
|
|
"build_args": build_args,
|
|
"pull_image": False,
|
|
"http_proxy": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
"https_proxy": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
},
|
|
"when": [event["pull_request"]],
|
|
},
|
|
{
|
|
"name": "build-and-push",
|
|
"image": PLUGINS_DOCKER_BUILDX,
|
|
"settings": {
|
|
"context": "..",
|
|
"repo": "%s,quay.io/%s,registry.heinlein.group/%s" % (repo, repo, repo),
|
|
"platforms": "linux/amd64,linux/arm64", # we can add remote builders
|
|
"auto_tag": False if build_type == "daily" else True,
|
|
"tag": hard_tag,
|
|
"default_tag": "daily",
|
|
"dockerfile": "opencloud/docker/Dockerfile.multiarch",
|
|
"build_args": build_args,
|
|
"pull_image": False,
|
|
"http_proxy": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
"https_proxy": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
"logins": [
|
|
{
|
|
"registry": "https://index.docker.io/v1/",
|
|
"username": {
|
|
"from_secret": "docker_username",
|
|
},
|
|
"password": {
|
|
"from_secret": "docker_password",
|
|
},
|
|
},
|
|
{
|
|
"registry": "https://quay.io",
|
|
"username": {
|
|
"from_secret": "quay_username",
|
|
},
|
|
"password": {
|
|
"from_secret": "quay_password",
|
|
},
|
|
},
|
|
{
|
|
"registry": "https://registry.heinlein.group",
|
|
"username": {
|
|
"from_secret": "harbor_opencloudeu_user",
|
|
},
|
|
"password": {
|
|
"from_secret": "harbor_opencloudeu_password",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
event["tag"],
|
|
],
|
|
},
|
|
{
|
|
"name": "notation-signing",
|
|
"image": PLUGINS_NOTATION,
|
|
"settings": {
|
|
"key": {
|
|
"from_secret": "notation_key",
|
|
},
|
|
"crt": {
|
|
"from_secret": "notation_cert",
|
|
},
|
|
"target": "registry.heinlein.group/%s:%s" % (repo, hard_tag),
|
|
"pull_image": True,
|
|
"logins": [
|
|
{
|
|
"registry": "https://registry.heinlein.group",
|
|
"username": {
|
|
"from_secret": "harbor_opencloudeu_user",
|
|
},
|
|
"password": {
|
|
"from_secret": "harbor_opencloudeu_password",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
event["tag"],
|
|
],
|
|
},
|
|
],
|
|
"depends_on": depends_on,
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "build-docker"),
|
|
},
|
|
},
|
|
event["tag"],
|
|
],
|
|
}
|
|
|
|
def binaryReleases(ctx):
|
|
pipelines = []
|
|
depends_on = getPipelineNames(getGoBinForTesting(ctx))
|
|
|
|
for os in config["binaryReleases"]["os"]:
|
|
pipelines.append(binaryRelease(ctx, os, depends_on))
|
|
|
|
return pipelines
|
|
|
|
def binaryRelease(ctx, arch, depends_on = []):
|
|
return {
|
|
"name": "binaries-%s" % arch,
|
|
"steps": makeNodeGenerate("") +
|
|
makeGoGenerate("") + [
|
|
{
|
|
"name": "build",
|
|
"image": OC_CI_GOLANG,
|
|
"environment": {
|
|
"VERSION": (ctx.build.ref.replace("refs/tags/", "") if ctx.build.event == "tag" else "daily"),
|
|
"EDITION": "rolling",
|
|
"HTTP_PROXY": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
"HTTPS_PROXY": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
},
|
|
"commands": [
|
|
"make -C opencloud release-%s" % arch,
|
|
],
|
|
},
|
|
{
|
|
"name": "finish",
|
|
"image": OC_CI_GOLANG,
|
|
"environment": CI_HTTP_PROXY_ENV,
|
|
"commands": [
|
|
"make -C opencloud release-finish",
|
|
],
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
event["tag"],
|
|
],
|
|
},
|
|
{
|
|
"name": "release",
|
|
"image": PLUGINS_GITHUB_RELEASE,
|
|
"settings": {
|
|
"api_key": {
|
|
"from_secret": "github_token",
|
|
},
|
|
"files": [
|
|
"opencloud/dist/release/*",
|
|
],
|
|
"title": ctx.build.ref.replace("refs/tags/v", ""),
|
|
"prerelease": len(ctx.build.ref.split("-")) > 1,
|
|
},
|
|
"when": [
|
|
event["tag"],
|
|
],
|
|
},
|
|
],
|
|
"depends_on": depends_on,
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "build-binary"),
|
|
},
|
|
},
|
|
event["tag"],
|
|
],
|
|
}
|
|
|
|
def licenseCheck(ctx):
|
|
return {
|
|
"name": "check-licenses",
|
|
"steps": restoreGoBinCache() + [
|
|
{
|
|
"name": "node-check-licenses",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"commands": [
|
|
"make ci-node-check-licenses",
|
|
],
|
|
},
|
|
{
|
|
"name": "node-save-licenses",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"commands": [
|
|
"make ci-node-save-licenses",
|
|
],
|
|
},
|
|
{
|
|
"name": "go-check-licenses",
|
|
"image": OC_CI_GOLANG,
|
|
"environment": CI_HTTP_PROXY_ENV,
|
|
"commands": [
|
|
"make ci-go-check-licenses",
|
|
],
|
|
},
|
|
{
|
|
"name": "go-save-licenses",
|
|
"image": OC_CI_GOLANG,
|
|
"environment": CI_HTTP_PROXY_ENV,
|
|
"commands": [
|
|
"make ci-go-save-licenses",
|
|
],
|
|
},
|
|
{
|
|
"name": "tarball",
|
|
"image": OC_CI_ALPINE,
|
|
"commands": [
|
|
"cd third-party-licenses && tar -czf ../third-party-licenses.tar.gz *",
|
|
],
|
|
},
|
|
{
|
|
"name": "release",
|
|
"image": PLUGINS_GITHUB_RELEASE,
|
|
"settings": {
|
|
"api_key": {
|
|
"from_secret": "github_token",
|
|
},
|
|
"files": [
|
|
"third-party-licenses.tar.gz",
|
|
],
|
|
"title": ctx.build.ref.replace("refs/tags/v", ""),
|
|
"prerelease": len(ctx.build.ref.split("-")) > 1,
|
|
},
|
|
"when": [
|
|
event["tag"],
|
|
],
|
|
},
|
|
],
|
|
"when": [
|
|
event["base"],
|
|
event["pull_request"],
|
|
event["tag"],
|
|
],
|
|
"workspace": workspace,
|
|
}
|
|
|
|
def readyReleaseGo():
|
|
return [{
|
|
"name": "ready-release-go",
|
|
"steps": [
|
|
{
|
|
"name": "release-helper",
|
|
"image": READY_RELEASE_GO,
|
|
"settings": {
|
|
"git_email": "devops@opencloud.eu",
|
|
"forge_type": "github",
|
|
"forge_token": {
|
|
"from_secret": "github_token",
|
|
},
|
|
},
|
|
},
|
|
],
|
|
"when": [event["base"]],
|
|
}]
|
|
|
|
def releaseDockerReadme(repo, build_type):
|
|
return {
|
|
"name": "readme-%s" % build_type,
|
|
"steps": [
|
|
{
|
|
"name": "push-docker",
|
|
"image": CHKO_DOCKER_PUSHRM,
|
|
"environment": {
|
|
"DOCKER_USER": {
|
|
"from_secret": "docker_username",
|
|
},
|
|
"DOCKER_PASS": {
|
|
"from_secret": "docker_password",
|
|
},
|
|
"PUSHRM_TARGET": repo,
|
|
"PUSHRM_SHORT": "Docker images for %s" % repo,
|
|
"PUSHRM_FILE": "README.md",
|
|
},
|
|
},
|
|
{
|
|
"name": "push-quay",
|
|
"image": CHKO_DOCKER_PUSHRM,
|
|
"environment": {
|
|
"APIKEY__QUAY_IO": {
|
|
"from_secret": "quay_apikey",
|
|
},
|
|
"PUSHRM_TARGET": "quay.io/%s" % repo,
|
|
"PUSHRM_FILE": "README.md",
|
|
"PUSHRM_PROVIDER": "quay",
|
|
},
|
|
},
|
|
],
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
event["tag"],
|
|
],
|
|
}
|
|
|
|
def makeNodeGenerate(module):
|
|
if module == "":
|
|
make = "make"
|
|
else:
|
|
make = "make -C %s" % module
|
|
return [
|
|
{
|
|
"name": "generate nodejs",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"environment": {
|
|
"CHROMEDRIVER_SKIP_DOWNLOAD": True, # install fails on arm and chromedriver is a test only dependency
|
|
},
|
|
"commands": [
|
|
"pnpm config set store-dir ./.pnpm-store",
|
|
"for i in $(seq 3); do %s node-generate-prod && break || sleep 1; done" % make,
|
|
],
|
|
},
|
|
]
|
|
|
|
def makeGoGenerate(module):
|
|
if module == "":
|
|
make = "make"
|
|
else:
|
|
make = "make -C %s" % module
|
|
return [
|
|
{
|
|
"name": "generate go",
|
|
"image": OC_CI_GOLANG,
|
|
"commands": [
|
|
"for i in $(seq 3); do %s go-generate && break || sleep 1; done" % make,
|
|
],
|
|
"environment": CI_HTTP_PROXY_ENV,
|
|
},
|
|
]
|
|
|
|
def notifyMatrix(ctx):
|
|
result = [{
|
|
"name": "chat-notifications",
|
|
"skip_clone": True,
|
|
"runs_on": ["success", "failure"],
|
|
"depends_on": getPipelineNames(testPipelines(ctx)),
|
|
"steps": [
|
|
{
|
|
"name": "notify-matrix",
|
|
"image": OC_CI_GOLANG,
|
|
"environment": {
|
|
"HTTP_PROXY": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
"HTTPS_PROXY": {
|
|
"from_secret": "ci_http_proxy",
|
|
},
|
|
"MATRIX_HOME_SERVER": "matrix.org",
|
|
"MATRIX_ROOM_ALIAS": {
|
|
"from_secret": "opencloud-notifications-channel",
|
|
},
|
|
"MATRIX_USER": {
|
|
"from_secret": "opencloud-notifications-user",
|
|
},
|
|
"MATRIX_PASSWORD": {
|
|
"from_secret": "opencloud-notifications-user-password",
|
|
},
|
|
"QA_REPO": "https://github.com/opencloud-eu/qa.git",
|
|
"QA_REPO_BRANCH": "main",
|
|
"CI_WOODPECKER_URL": {
|
|
"from_secret": "oc_ci_url",
|
|
},
|
|
"CI_REPO_ID": "3",
|
|
"CI_WOODPECKER_TOKEN": "no-auth-needed-on-this-repo",
|
|
},
|
|
"commands": [
|
|
"git clone --single-branch --branch $QA_REPO_BRANCH $QA_REPO /tmp/qa",
|
|
"cd /tmp/qa/scripts/matrix-notification/",
|
|
"go run matrix-notification.go",
|
|
],
|
|
},
|
|
],
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
event["pull_request"],
|
|
],
|
|
}]
|
|
|
|
return result
|
|
|
|
def opencloudServer(storage = "decomposed", accounts_hash_difficulty = 4, depends_on = [], deploy_type = "", extra_server_environment = {}, with_wrapper = False, tika_enabled = False, watch_fs_enabled = False):
|
|
user = "0:0"
|
|
container_name = OC_SERVER_NAME
|
|
environment = {
|
|
"OC_URL": OC_URL,
|
|
"OC_CONFIG_DIR": "/root/.opencloud/config", # needed for checking config later
|
|
"STORAGE_USERS_DRIVER": "%s" % storage,
|
|
"PROXY_ENABLE_BASIC_AUTH": True,
|
|
"WEB_UI_CONFIG_FILE": "%s/%s" % (dirs["base"], dirs["opencloudConfig"]),
|
|
"OC_LOG_LEVEL": "error",
|
|
"IDM_CREATE_DEMO_USERS": True, # needed for litmus and cs3api-validator tests
|
|
"IDM_ADMIN_PASSWORD": "admin", # override the random admin password from `opencloud init`
|
|
"FRONTEND_SEARCH_MIN_LENGTH": "2",
|
|
"OC_ASYNC_UPLOADS": True,
|
|
"OC_EVENTS_ENABLE_TLS": False,
|
|
"NATS_NATS_HOST": "0.0.0.0",
|
|
"NATS_NATS_PORT": 9233,
|
|
"OC_JWT_SECRET": "some-opencloud-jwt-secret",
|
|
"EVENTHISTORY_STORE": "memory",
|
|
"OC_TRANSLATION_PATH": "%s/tests/config/translations" % dirs["base"],
|
|
"ACTIVITYLOG_WRITE_BUFFER_DURATION": "0", # Disable write buffer so that test expectations are met in time
|
|
# debug addresses required for running services health tests
|
|
"ACTIVITYLOG_DEBUG_ADDR": "0.0.0.0:9197",
|
|
"APP_PROVIDER_DEBUG_ADDR": "0.0.0.0:9165",
|
|
"APP_REGISTRY_DEBUG_ADDR": "0.0.0.0:9243",
|
|
"AUTH_BASIC_DEBUG_ADDR": "0.0.0.0:9147",
|
|
"AUTH_MACHINE_DEBUG_ADDR": "0.0.0.0:9167",
|
|
"AUTH_SERVICE_DEBUG_ADDR": "0.0.0.0:9198",
|
|
"CLIENTLOG_DEBUG_ADDR": "0.0.0.0:9260",
|
|
"EVENTHISTORY_DEBUG_ADDR": "0.0.0.0:9270",
|
|
"FRONTEND_DEBUG_ADDR": "0.0.0.0:9141",
|
|
"GATEWAY_DEBUG_ADDR": "0.0.0.0:9143",
|
|
"GRAPH_DEBUG_ADDR": "0.0.0.0:9124",
|
|
"GROUPS_DEBUG_ADDR": "0.0.0.0:9161",
|
|
"IDM_DEBUG_ADDR": "0.0.0.0:9239",
|
|
"IDP_DEBUG_ADDR": "0.0.0.0:9134",
|
|
"INVITATIONS_DEBUG_ADDR": "0.0.0.0:9269",
|
|
"NATS_DEBUG_ADDR": "0.0.0.0:9234",
|
|
"OCDAV_DEBUG_ADDR": "0.0.0.0:9163",
|
|
"OCM_DEBUG_ADDR": "0.0.0.0:9281",
|
|
"OCS_DEBUG_ADDR": "0.0.0.0:9114",
|
|
"POSTPROCESSING_DEBUG_ADDR": "0.0.0.0:9255",
|
|
"PROXY_DEBUG_ADDR": "0.0.0.0:9205",
|
|
"SEARCH_DEBUG_ADDR": "0.0.0.0:9224",
|
|
"SETTINGS_DEBUG_ADDR": "0.0.0.0:9194",
|
|
"SHARING_DEBUG_ADDR": "0.0.0.0:9151",
|
|
"SSE_DEBUG_ADDR": "0.0.0.0:9139",
|
|
"STORAGE_PUBLICLINK_DEBUG_ADDR": "0.0.0.0:9179",
|
|
"STORAGE_SHARES_DEBUG_ADDR": "0.0.0.0:9156",
|
|
"STORAGE_SYSTEM_DEBUG_ADDR": "0.0.0.0:9217",
|
|
"STORAGE_USERS_DEBUG_ADDR": "0.0.0.0:9159",
|
|
"THUMBNAILS_DEBUG_ADDR": "0.0.0.0:9189",
|
|
"USERLOG_DEBUG_ADDR": "0.0.0.0:9214",
|
|
"USERS_DEBUG_ADDR": "0.0.0.0:9145",
|
|
"WEB_DEBUG_ADDR": "0.0.0.0:9104",
|
|
"WEBDAV_DEBUG_ADDR": "0.0.0.0:9119",
|
|
"WEBFINGER_DEBUG_ADDR": "0.0.0.0:9279",
|
|
"STORAGE_USERS_POSIX_SCAN_DEBOUNCE_DELAY": 0,
|
|
}
|
|
|
|
if storage == "posix":
|
|
environment["STORAGE_USERS_ID_CACHE_STORE"] = "nats-js-kv"
|
|
|
|
if deploy_type == "":
|
|
environment["FRONTEND_OCS_ENABLE_DENIALS"] = True
|
|
|
|
# fonts map for txt thumbnails (including unicode support)
|
|
environment["THUMBNAILS_TXT_FONTMAP_FILE"] = "%s/tests/config/woodpecker/fontsMap.json" % (dirs["base"])
|
|
|
|
if deploy_type == "cs3api_validator":
|
|
environment["GATEWAY_GRPC_ADDR"] = "0.0.0.0:9142" # make gateway available to cs3api-validator
|
|
environment["OC_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD"] = False
|
|
|
|
if deploy_type == "wopi_validator":
|
|
environment["GATEWAY_GRPC_ADDR"] = "0.0.0.0:9142" # make gateway available to wopi server
|
|
environment["APP_PROVIDER_EXTERNAL_ADDR"] = "eu.opencloud.api.app-provider"
|
|
environment["APP_PROVIDER_DRIVER"] = "wopi"
|
|
environment["APP_PROVIDER_WOPI_APP_NAME"] = "FakeOffice"
|
|
environment["APP_PROVIDER_WOPI_APP_URL"] = "http://fakeoffice:8080"
|
|
environment["APP_PROVIDER_WOPI_INSECURE"] = True
|
|
environment["APP_PROVIDER_WOPI_WOPI_SERVER_EXTERNAL_URL"] = "http://wopi-fakeoffice:9300"
|
|
environment["APP_PROVIDER_WOPI_FOLDER_URL_BASE_URL"] = OC_URL
|
|
|
|
if deploy_type == "federation":
|
|
environment["OC_URL"] = OC_FED_URL
|
|
environment["PROXY_HTTP_ADDR"] = OC_FED_DOMAIN
|
|
container_name = FED_OC_SERVER_NAME
|
|
|
|
if tika_enabled:
|
|
environment["FRONTEND_FULL_TEXT_SEARCH_ENABLED"] = True
|
|
environment["SEARCH_EXTRACTOR_TYPE"] = "tika"
|
|
environment["SEARCH_EXTRACTOR_TIKA_TIKA_URL"] = "http://tika:9998"
|
|
environment["SEARCH_EXTRACTOR_CS3SOURCE_INSECURE"] = True
|
|
|
|
if watch_fs_enabled:
|
|
environment["STORAGE_USERS_POSIX_WATCH_FS"] = True
|
|
|
|
# Pass in "default" accounts_hash_difficulty to not set this environment variable.
|
|
# That will allow OpenCloud to use whatever its built-in default is.
|
|
# Otherwise pass in a value from 4 to about 11 or 12 (default 4, for making regular tests fast)
|
|
# The high values cause lots of CPU to be used when hashing passwords, and really slow down the tests.
|
|
if accounts_hash_difficulty != "default":
|
|
environment["ACCOUNTS_HASH_DIFFICULTY"] = accounts_hash_difficulty
|
|
|
|
for item in extra_server_environment:
|
|
environment[item] = extra_server_environment[item]
|
|
|
|
server_commands = [
|
|
"env | sort",
|
|
]
|
|
if with_wrapper:
|
|
server_commands += [
|
|
"make -C %s build" % dirs["ocWrapper"],
|
|
"%s/bin/ocwrapper serve --bin %s --url %s --admin-username admin --admin-password admin" % (dirs["ocWrapper"], dirs["opencloudBin"], environment["OC_URL"]),
|
|
]
|
|
else:
|
|
server_commands += [
|
|
"%s server" % dirs["opencloudBin"],
|
|
]
|
|
|
|
wait_for_opencloud = {
|
|
"name": "wait-for-%s" % container_name,
|
|
"image": OC_CI_ALPINE,
|
|
"commands": [
|
|
# wait for opencloud-server to be ready (5 minutes)
|
|
"timeout 300 bash -c 'while [ $(curl -sk -uadmin:admin " +
|
|
"%s/graph/v1.0/users/admin " % environment["OC_URL"] +
|
|
"-w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done'",
|
|
],
|
|
}
|
|
|
|
opencloud_server = {
|
|
"name": container_name,
|
|
"image": OC_CI_GOLANG,
|
|
"detach": True,
|
|
"environment": environment,
|
|
"backend_options": {
|
|
"docker": {
|
|
"user": user,
|
|
},
|
|
},
|
|
"commands": [
|
|
"%s init --insecure true" % dirs["opencloudBin"],
|
|
"cat $OC_CONFIG_DIR/opencloud.yaml",
|
|
"cp tests/config/woodpecker/app-registry.yaml $OC_CONFIG_DIR/app-registry.yaml",
|
|
] + server_commands,
|
|
}
|
|
|
|
steps = [
|
|
opencloud_server,
|
|
wait_for_opencloud,
|
|
]
|
|
|
|
# empty depends_on list makes steps to run in parallel, what we don't want
|
|
if depends_on:
|
|
steps[0]["depends_on"] = depends_on
|
|
steps[1]["depends_on"] = depends_on
|
|
|
|
return steps
|
|
|
|
def startOpenCloudService(service = None, name = None, environment = {}):
|
|
"""
|
|
Starts an OpenCloud service in a detached container.
|
|
|
|
Args:
|
|
service (str): The name of the service to start.
|
|
name (str): The name of the container.
|
|
environment (dict): The environment variables to set in the container.
|
|
|
|
Returns:
|
|
list: A list of pipeline steps to start the service.
|
|
"""
|
|
|
|
if not service:
|
|
return []
|
|
if not name:
|
|
name = service
|
|
|
|
return [
|
|
{
|
|
"name": name,
|
|
"image": OC_CI_GOLANG,
|
|
"detach": True,
|
|
"environment": environment,
|
|
"commands": [
|
|
"%s %s server" % (dirs["opencloudBin"], service),
|
|
],
|
|
},
|
|
]
|
|
|
|
def redis():
|
|
return [
|
|
{
|
|
"name": "redis",
|
|
"image": REDIS,
|
|
},
|
|
]
|
|
|
|
def redisForOCStorage(storage = "decomposed"):
|
|
if storage == "owncloud":
|
|
return redis()
|
|
else:
|
|
return
|
|
|
|
def build():
|
|
return [
|
|
{
|
|
"name": "build",
|
|
"image": OC_CI_GOLANG,
|
|
"commands": [
|
|
"for i in $(seq 3); do make -C opencloud build ENABLE_VIPS=1 && break || sleep 1; done",
|
|
],
|
|
"environment": CI_HTTP_PROXY_ENV,
|
|
},
|
|
]
|
|
|
|
def skipIfUnchanged(ctx, type):
|
|
if "full-ci" in ctx.build.title.lower() or ctx.build.event == "tag" or ctx.build.event == "cron":
|
|
return []
|
|
|
|
base = [
|
|
".github/**",
|
|
".vscode/**",
|
|
"docs/**",
|
|
"deployments/**",
|
|
"CHANGELOG.md",
|
|
"CONTRIBUTING.md",
|
|
"LICENSE",
|
|
"README.md",
|
|
]
|
|
unit = [
|
|
"**/*_test.go",
|
|
]
|
|
acceptance = [
|
|
"tests/acceptance/**",
|
|
]
|
|
|
|
skip = []
|
|
if type == "acceptance-tests" or type == "e2e-tests" or type == "lint":
|
|
skip = base + unit
|
|
elif type == "unit-tests":
|
|
skip = base + acceptance
|
|
elif type == "build-binary" or type == "build-docker" or type == "litmus":
|
|
skip = base + unit + acceptance
|
|
elif type == "cache" or type == "base":
|
|
skip = base
|
|
|
|
return skip
|
|
|
|
def translation_sync(ctx):
|
|
return [{
|
|
"name": "translation-sync",
|
|
"steps": [
|
|
{
|
|
"name": "translation-update",
|
|
"image": OC_CI_GOLANG,
|
|
"commands": [
|
|
"make l10n-read",
|
|
"mkdir tx && cd tx",
|
|
"curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash",
|
|
"export PATH=$PATH:$(pwd) && cd ..",
|
|
"make l10n-push",
|
|
"make l10n-pull",
|
|
"rm -rf tx",
|
|
"make l10n-clean",
|
|
],
|
|
"environment": {
|
|
"TX_TOKEN": {
|
|
"from_secret": "tx_token",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
"name": "translation-push",
|
|
"image": PLUGINS_GIT_ACTION,
|
|
"settings": {
|
|
"action": ["commit", "push"],
|
|
"branch": ctx.build.branch,
|
|
"message": "[tx] updated from transifex",
|
|
"author_name": "opencloudeu",
|
|
"author_email": "devops@opencloud.eu",
|
|
"netrc_username": {
|
|
"from_secret": "github_username",
|
|
},
|
|
"netrc_password": {
|
|
"from_secret": "github_token",
|
|
},
|
|
"empty_commit": False,
|
|
},
|
|
},
|
|
],
|
|
"when": [
|
|
{
|
|
"event": "cron",
|
|
"cron": "translation-sync",
|
|
},
|
|
],
|
|
}]
|
|
|
|
def checkStarlark(ctx):
|
|
return [{
|
|
"name": "check-starlark",
|
|
"steps": [
|
|
{
|
|
"name": "format-check-starlark",
|
|
"image": OC_CI_BAZEL_BUILDIFIER,
|
|
"commands": [
|
|
"buildifier --mode=check .woodpecker.star",
|
|
],
|
|
},
|
|
{
|
|
"name": "show-diff",
|
|
"image": OC_CI_BAZEL_BUILDIFIER,
|
|
"commands": [
|
|
"buildifier --mode=fix .woodpecker.star",
|
|
"git diff",
|
|
],
|
|
"when": [
|
|
{
|
|
"status": "failure",
|
|
},
|
|
],
|
|
},
|
|
],
|
|
"depends_on": [],
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "base"),
|
|
},
|
|
},
|
|
],
|
|
}]
|
|
|
|
def genericCache(name, action, mounts, cache_path):
|
|
rebuild = False
|
|
restore = False
|
|
if action == "rebuild":
|
|
rebuild = True
|
|
action = "rebuild"
|
|
else:
|
|
restore = True
|
|
action = "restore"
|
|
|
|
step = {
|
|
"name": "%s_%s" % (action, name),
|
|
"image": PLUGINS_S3_CACHE,
|
|
"settings": {
|
|
"endpoint": CACHE_S3_SERVER,
|
|
"rebuild": rebuild,
|
|
"restore": restore,
|
|
"mount": mounts,
|
|
"access_key": {
|
|
"from_secret": "cache_s3_access_key",
|
|
},
|
|
"secret_key": {
|
|
"from_secret": "cache_s3_secret_key",
|
|
},
|
|
"filename": "%s.tar" % name,
|
|
"path": cache_path,
|
|
"fallback_path": cache_path,
|
|
},
|
|
}
|
|
return step
|
|
|
|
def purgeCache(name, flush_path, flush_age):
|
|
return {
|
|
"name": name,
|
|
"skip_clone": True,
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
event["pull_request"],
|
|
],
|
|
"runs_on": ["success", "failure"],
|
|
"steps": [
|
|
{
|
|
"name": "purge",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"to_delete=$(mc find s3/%s/ --older-than %sd)" % (flush_path, flush_age),
|
|
'if [ -z "$to_delete" ]; then exit 0; fi',
|
|
"mc rm $to_delete",
|
|
],
|
|
},
|
|
],
|
|
}
|
|
|
|
def genericBuildArtifactCache(ctx, name, action, path):
|
|
if action == "rebuild" or action == "restore":
|
|
cache_path = "%s/%s/%s" % ("cache", repo_slug, ctx.build.commit + "-${CI_PIPELINE_NUMBER}")
|
|
name = "%s_build_artifact_cache" % name
|
|
return genericCache(name, action, [path], cache_path)
|
|
|
|
if action == "purge":
|
|
return purgeCache("purge_opencloud_build_artifact_cache", "cache/%s" % repo_slug, 1)
|
|
return []
|
|
|
|
def restoreBuildArtifactCache(ctx, name, path):
|
|
return [genericBuildArtifactCache(ctx, name, "restore", path)]
|
|
|
|
def rebuildBuildArtifactCache(ctx, name, path):
|
|
return [genericBuildArtifactCache(ctx, name, "rebuild", path)]
|
|
|
|
def purgeBuildArtifactCache(ctx):
|
|
return genericBuildArtifactCache(ctx, "", "purge", [])
|
|
|
|
def purgeBrowserCache(ctx):
|
|
return purgeCache("purge_browser_build_cache", "dev/web", 14)
|
|
|
|
def purgeTracingCache(ctx):
|
|
return purgeCache("purge_playwright_tracing_cache", "public/web/tracing", 14)
|
|
|
|
def purgeOpencloudWebBuildCache(ctx):
|
|
return purgeCache("purge_opencloud_web_build_cache", "dev/opencloud/web-test-runner", 14)
|
|
|
|
def purgeGoBinCache(ctx):
|
|
return purgeCache("purge_go_bin_cache", "dev/opencloud/go-bin", 14)
|
|
|
|
def pipelineSanityChecks(pipelines):
|
|
"""pipelineSanityChecks helps the CI developers to find errors before running it
|
|
|
|
These sanity checks are only executed on when converting starlark to yaml.
|
|
Error outputs are only visible when the conversion is done with the woodpecker cli.
|
|
|
|
Args:
|
|
pipelines: pipelines to be checked, normally you should run this on the return value of main()
|
|
|
|
Returns:
|
|
none
|
|
"""
|
|
|
|
# check if name length of pipeline and steps are exceeded.
|
|
max_name_length = 50
|
|
for pipeline in pipelines:
|
|
pipeline_name = pipeline["name"]
|
|
if len(pipeline_name) > max_name_length:
|
|
print("Error: pipeline name %s is longer than 50 characters" % pipeline_name)
|
|
|
|
for step in pipeline["steps"]:
|
|
step_name = step["name"]
|
|
if len(step_name) > max_name_length:
|
|
print("Error: step name %s in pipeline %s is longer than 50 characters" % (step_name, pipeline_name))
|
|
|
|
# check for non existing depends_on
|
|
possible_depends = []
|
|
for pipeline in pipelines:
|
|
possible_depends.append(pipeline["name"])
|
|
|
|
for pipeline in pipelines:
|
|
if "depends_on" in pipeline.keys():
|
|
for depends in pipeline["depends_on"]:
|
|
if not depends in possible_depends:
|
|
print("Error: depends_on %s for pipeline %s is not defined" % (depends, pipeline["name"]))
|
|
|
|
# check for non declared volumes
|
|
# for pipeline in pipelines:
|
|
# pipeline_volumes = []
|
|
# if "workspace" in pipeline.keys():
|
|
# for volume in pipeline["workspace"]:
|
|
# pipeline_volumes.append(volume["base"])
|
|
#
|
|
# for step in pipeline["steps"]:
|
|
# if "workspace" in step.keys():
|
|
# for volume in step["workspace"]:
|
|
# if not volume["base"] in pipeline_volumes:
|
|
# print("Warning: volume %s for step %s is not defined in pipeline %s" % (volume["base"], step["name"], pipeline["name"]))
|
|
|
|
# list used docker images
|
|
print("")
|
|
print("List of used docker images:")
|
|
|
|
images = {}
|
|
|
|
for pipeline in pipelines:
|
|
for step in pipeline["steps"]:
|
|
image = step["image"]
|
|
if image in images.keys():
|
|
images[image] = images[image] + 1
|
|
else:
|
|
images[image] = 1
|
|
|
|
for image in images.keys():
|
|
print(" %sx\t%s" % (images[image], image))
|
|
|
|
def litmus(ctx, storage):
|
|
pipelines = []
|
|
|
|
if not config["litmus"]:
|
|
return pipelines
|
|
|
|
environment = {
|
|
"LITMUS_PASSWORD": "admin",
|
|
"LITMUS_USERNAME": "admin",
|
|
"TESTS": "basic copymove props http",
|
|
}
|
|
|
|
litmusCommand = "/usr/local/bin/litmus-wrapper"
|
|
|
|
result = {
|
|
"name": "litmus",
|
|
"steps": restoreBuildArtifactCache(ctx, dirs["opencloudBinArtifact"], dirs["opencloudBinPath"]) +
|
|
opencloudServer(storage) +
|
|
setupForLitmus() +
|
|
[
|
|
{
|
|
"name": "old-endpoint",
|
|
"image": OC_LITMUS,
|
|
"environment": environment,
|
|
"commands": [
|
|
"source .env",
|
|
'export LITMUS_URL="%s/remote.php/webdav"' % OC_URL,
|
|
litmusCommand,
|
|
],
|
|
},
|
|
{
|
|
"name": "new-endpoint",
|
|
"image": OC_LITMUS,
|
|
"environment": environment,
|
|
"commands": [
|
|
"source .env",
|
|
'export LITMUS_URL="%s/remote.php/dav/files/admin"' % OC_URL,
|
|
litmusCommand,
|
|
],
|
|
},
|
|
{
|
|
"name": "new-shared",
|
|
"image": OC_LITMUS,
|
|
"environment": environment,
|
|
"commands": [
|
|
"source .env",
|
|
'export LITMUS_URL="%s/remote.php/dav/files/admin/Shares/new_folder/"' % OC_URL,
|
|
litmusCommand,
|
|
],
|
|
},
|
|
{
|
|
"name": "old-shared",
|
|
"image": OC_LITMUS,
|
|
"environment": environment,
|
|
"commands": [
|
|
"source .env",
|
|
'export LITMUS_URL="%s/remote.php/webdav/Shares/new_folder/"' % OC_URL,
|
|
litmusCommand,
|
|
],
|
|
},
|
|
# {
|
|
# "name": "public-share",
|
|
# "image": OC_LITMUS,
|
|
# "environment": {
|
|
# "LITMUS_PASSWORD": "admin",
|
|
# "LITMUS_USERNAME": "admin",
|
|
# "TESTS": "basic copymove http",
|
|
# },
|
|
# "commands": [
|
|
# "source .env",
|
|
# "export LITMUS_URL='%s/remote.php/dav/public-files/'$PUBLIC_TOKEN" % OCIS_URL,
|
|
# litmusCommand,
|
|
# ],
|
|
# },
|
|
{
|
|
"name": "spaces-endpoint",
|
|
"image": OC_LITMUS,
|
|
"environment": environment,
|
|
"commands": [
|
|
"source .env",
|
|
"export LITMUS_URL='%s/remote.php/dav/spaces/'$SPACE_ID" % OC_URL,
|
|
litmusCommand,
|
|
],
|
|
},
|
|
],
|
|
"services": redisForOCStorage(storage),
|
|
"depends_on": getPipelineNames(buildOpencloudBinaryForTesting(ctx)),
|
|
"when": [
|
|
event["cron"],
|
|
event["base"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "litmus"),
|
|
},
|
|
},
|
|
],
|
|
}
|
|
pipelines.append(result)
|
|
|
|
return pipelines
|
|
|
|
def setupForLitmus():
|
|
return [{
|
|
"name": "setup-for-litmus",
|
|
"image": OC_UBUNTU,
|
|
"environment": {
|
|
"TEST_SERVER_URL": OC_URL,
|
|
},
|
|
"commands": [
|
|
"bash ./tests/config/woodpecker/setup-for-litmus.sh",
|
|
"cat .env",
|
|
],
|
|
}]
|
|
|
|
def getWoodpeckerEnvAndCheckScript(ctx):
|
|
path_to_woodpecker_env = "%s/.woodpecker.env" % dirs["base"]
|
|
path_to_check_script = "%s/tests/config/woodpecker/check_web_cache.sh" % dirs["base"]
|
|
return {
|
|
"name": "get-woodpecker-env-and-check-script",
|
|
"image": OC_UBUNTU,
|
|
"commands": [
|
|
"cp %s check_web_cache.sh" % path_to_check_script,
|
|
],
|
|
}
|
|
|
|
def checkForWebCache(name):
|
|
return {
|
|
"name": "check-for-%s-cache" % name,
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"bash -x check_web_cache.sh %s" % name,
|
|
],
|
|
}
|
|
|
|
def cloneWeb():
|
|
return {
|
|
"name": "clone-web",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"commands": [
|
|
". ./.woodpecker.env",
|
|
"if $WEB_CACHE_FOUND; then exit 0; fi",
|
|
"rm -rf %s" % dirs["web"],
|
|
"git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/opencloud-eu/web.git %s" % dirs["web"],
|
|
"cd %s && git checkout $WEB_COMMITID" % dirs["web"],
|
|
],
|
|
}
|
|
|
|
def generateWebPnpmCache(ctx):
|
|
return [
|
|
getWoodpeckerEnvAndCheckScript(ctx),
|
|
checkForWebCache("web-pnpm"),
|
|
cloneWeb(),
|
|
{
|
|
"name": "install-pnpm",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"commands": [
|
|
". ./.woodpecker.env",
|
|
"if $WEB_CACHE_FOUND; then exit 0; fi",
|
|
"cd %s" % dirs["web"],
|
|
'npm install --silent --global --force "$(jq -r ".packageManager" < package.json)"',
|
|
"pnpm config set store-dir ./.pnpm-store",
|
|
"for i in $(seq 3); do pnpm install && break || sleep 1; done",
|
|
],
|
|
},
|
|
{
|
|
"name": "zip-pnpm",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"commands": [
|
|
". ./.woodpecker.env",
|
|
"if $WEB_CACHE_FOUND; then exit 0; fi",
|
|
# zip the pnpm deps before caching
|
|
"if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]),
|
|
"cd %s" % dirs["web"],
|
|
"tar -czf %s .pnpm-store" % dirs["webPnpmZip"],
|
|
],
|
|
},
|
|
{
|
|
"name": "cache-pnpm",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
". ./.woodpecker.env",
|
|
"if $WEB_CACHE_FOUND; then exit 0; fi",
|
|
# cache using the minio/mc client to the public bucket (long term bucket)
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -r -a %s s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID" % dirs["webPnpmZip"],
|
|
],
|
|
},
|
|
]
|
|
|
|
def cacheBrowsers(ctx):
|
|
e2e_trigger = [
|
|
event["base"],
|
|
event["cron"],
|
|
{
|
|
"event": "pull_request",
|
|
"path": {
|
|
"exclude": skipIfUnchanged(ctx, "e2e-tests"),
|
|
},
|
|
},
|
|
{
|
|
"event": "tag",
|
|
"ref": "refs/tags/**",
|
|
},
|
|
]
|
|
|
|
check_browser_step = [{
|
|
"name": "check-browsers-cache",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc ls --recursive s3/$CACHE_BUCKET/web",
|
|
"cd %s" % dirs["web"],
|
|
"bash tests/woodpecker/script.sh check_browsers_cache",
|
|
],
|
|
}]
|
|
|
|
webPnpmCacheSteps = restoreWebPnpmCache(extra_commands = [
|
|
"cd %s" % dirs["web"],
|
|
". ./.woodpecker.env",
|
|
"if $BROWSER_CACHE_FOUND; then exit 0; fi",
|
|
"cd %s" % dirs["base"],
|
|
])
|
|
|
|
browser_cache_steps = [
|
|
{
|
|
"name": "install-browsers",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"environment": {
|
|
"PLAYWRIGHT_BROWSERS_PATH": ".playwright",
|
|
},
|
|
"commands": [
|
|
"cd %s" % dirs["web"],
|
|
". ./.woodpecker.env",
|
|
"if $BROWSER_CACHE_FOUND; then exit 0; fi",
|
|
"pnpm exec playwright install --with-deps",
|
|
"pnpm exec playwright install --list",
|
|
"tar -czf %s .playwright" % dirs["playwrightBrowsersArchive"],
|
|
],
|
|
},
|
|
{
|
|
"name": "upload-browsers-cache",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"cd %s" % dirs["web"],
|
|
". ./.woodpecker.env",
|
|
"if $BROWSER_CACHE_FOUND; then exit 0; fi",
|
|
"playwright_version=$(bash tests/woodpecker/script.sh get_playwright_version)",
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -r -a %s s3/$CACHE_BUCKET/web/browsers-cache/$playwright_version/" % dirs["playwrightBrowsersArchive"],
|
|
"mc ls --recursive s3/$CACHE_BUCKET/web",
|
|
],
|
|
},
|
|
]
|
|
|
|
return [{
|
|
"name": "cache-browsers",
|
|
"depends_on": getPipelineNames(buildWebCache(ctx)),
|
|
"steps": restoreWebCache() + check_browser_step + webPnpmCacheSteps + browser_cache_steps,
|
|
"when": e2e_trigger,
|
|
}]
|
|
|
|
def generateWebCache(ctx):
|
|
return [
|
|
getWoodpeckerEnvAndCheckScript(ctx),
|
|
checkForWebCache("web"),
|
|
cloneWeb(),
|
|
{
|
|
"name": "zip-web",
|
|
"image": OC_UBUNTU,
|
|
"commands": [
|
|
". ./.woodpecker.env",
|
|
"if $WEB_CACHE_FOUND; then exit 0; fi",
|
|
"if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]),
|
|
"tar -czf %s webTestRunner" % dirs["webZip"],
|
|
],
|
|
},
|
|
{
|
|
"name": "cache-web",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
". ./.woodpecker.env",
|
|
"if $WEB_CACHE_FOUND; then exit 0; fi",
|
|
# cache using the minio/mc client to the 'owncloud' bucket (long term bucket)
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -r -a %s s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID" % dirs["webZip"],
|
|
],
|
|
},
|
|
]
|
|
|
|
def restoreWebCache():
|
|
return [{
|
|
"name": "restore-web-cache",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"source ./.woodpecker.env",
|
|
"rm -rf %s" % dirs["web"],
|
|
"mkdir -p %s" % dirs["web"],
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -r -a s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID/web.tar.gz %s" % dirs["zip"],
|
|
],
|
|
}, {
|
|
"name": "unzip-web-cache",
|
|
"image": OC_UBUNTU,
|
|
"commands": [
|
|
"tar -xf %s -C ." % dirs["webZip"],
|
|
],
|
|
}]
|
|
|
|
def restoreWebPnpmCache(extra_commands = []):
|
|
return [{
|
|
"name": "restore-web-pnpm-cache",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": extra_commands + [
|
|
"source ./.woodpecker.env",
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -r -a s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID/web-pnpm.tar.gz %s" % dirs["zip"],
|
|
],
|
|
}, {
|
|
# we need to install again because the node_modules are not cached
|
|
"name": "unzip-and-install-pnpm",
|
|
"image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION,
|
|
"commands": extra_commands + [
|
|
"cd %s" % dirs["web"],
|
|
"rm -rf .pnpm-store",
|
|
"tar -xf %s" % dirs["webPnpmZip"],
|
|
'npm install --silent --global --force "$(jq -r ".packageManager" < package.json)"',
|
|
"pnpm config set store-dir ./.pnpm-store",
|
|
"for i in $(seq 3); do pnpm install --no-frozen-lockfile && break || sleep 1; done",
|
|
],
|
|
}]
|
|
|
|
def restoreBrowsersCache():
|
|
return [
|
|
{
|
|
"name": "restore-browsers-cache",
|
|
"image": MINIO_MC,
|
|
"environment": MINIO_MC_ENV,
|
|
"commands": [
|
|
"cd %s" % dirs["web"],
|
|
"playwright_version=$(bash tests/woodpecker/script.sh get_playwright_version)",
|
|
"mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY",
|
|
"mc cp -r -a s3/$CACHE_BUCKET/web/browsers-cache/$playwright_version/playwright-browsers.tar.gz %s" % dirs["web"],
|
|
],
|
|
},
|
|
{
|
|
"name": "unzip-browsers-cache",
|
|
"image": OC_UBUNTU,
|
|
"commands": [
|
|
"tar -xf /woodpecker/src/github.com/%s/webTestRunner/playwright-browsers.tar.gz -C ." % repo_slug,
|
|
],
|
|
},
|
|
]
|
|
|
|
def emailService():
|
|
return [{
|
|
"name": "email",
|
|
"image": INBUCKET_INBUCKET,
|
|
}]
|
|
|
|
def waitForEmailService():
|
|
return [{
|
|
"name": "wait-for-email",
|
|
"image": OC_CI_WAIT_FOR,
|
|
"commands": [
|
|
"wait-for -it email:9000 -t 600",
|
|
],
|
|
}]
|
|
|
|
def clamavService():
|
|
return [{
|
|
"name": "clamav",
|
|
"image": OC_CI_CLAMAVD,
|
|
}]
|
|
|
|
def waitForClamavService():
|
|
return [{
|
|
"name": "wait-for-clamav",
|
|
"image": OC_CI_WAIT_FOR,
|
|
"commands": [
|
|
"wait-for -it clamav:3310 -t 600",
|
|
],
|
|
}]
|
|
|
|
def ldapService():
|
|
return [
|
|
{
|
|
"name": "ldap-server",
|
|
"image": OPENLDAP,
|
|
"detach": True,
|
|
"environment": {
|
|
"BITNAMI_DEBUG": "true",
|
|
"LDAP_TLS_VERIFY_CLIENT": "never",
|
|
"LDAP_ENABLE_TLS": "yes",
|
|
"LDAP_TLS_CA_FILE": "/opt/bitnami/openldap/share/openldap.crt",
|
|
"LDAP_TLS_CERT_FILE": "/opt/bitnami/openldap/share/openldap.crt",
|
|
"LDAP_TLS_KEY_FILE": "/opt/bitnami/openldap/share/openldap.key",
|
|
"LDAP_ROOT": "dc=opencloud,dc=eu",
|
|
"LDAP_ADMIN_PASSWORD": "admin",
|
|
},
|
|
"commands": [
|
|
"mkdir -p /opt/bitnami/openldap/share",
|
|
"mkdir -p /tmp/custom-scripts",
|
|
"mkdir -p /tmp/ldif-files",
|
|
"cp tests/config/woodpecker/ldap/*.ldif /tmp/ldif-files/",
|
|
"cp tests/config/woodpecker/ldap/docker-entrypoint-override.sh /tmp/custom-scripts/",
|
|
"chmod +x /tmp/custom-scripts/docker-entrypoint-override.sh",
|
|
"ls -la /tmp/ldif-files/",
|
|
"/tmp/custom-scripts/docker-entrypoint-override.sh /opt/bitnami/scripts/openldap/run.sh",
|
|
],
|
|
"backend_options": {
|
|
"docker": {
|
|
"user": "0:0",
|
|
},
|
|
},
|
|
},
|
|
]
|
|
|
|
def waitForLdapService():
|
|
return [{
|
|
"name": "wait-for-ldap",
|
|
"image": OC_CI_WAIT_FOR,
|
|
"commands": [
|
|
"wait-for -it ldap-server:1636 -t 600",
|
|
],
|
|
}]
|
|
|
|
def fakeOffice():
|
|
return [
|
|
{
|
|
"name": "fakeoffice",
|
|
"image": OC_CI_ALPINE,
|
|
"environment": {},
|
|
"commands": [
|
|
"sh %s/tests/config/woodpecker/serve-hosting-discovery.sh" % (dirs["base"]),
|
|
],
|
|
},
|
|
]
|
|
|
|
def wopiCollaborationService(name):
|
|
service_name = "wopi-%s" % name
|
|
|
|
environment = {
|
|
"MICRO_REGISTRY": "nats-js-kv",
|
|
"MICRO_REGISTRY_ADDRESS": "%s:9233" % OC_SERVER_NAME,
|
|
"COLLABORATION_LOG_LEVEL": "debug",
|
|
"COLLABORATION_GRPC_ADDR": "0.0.0.0:9301",
|
|
"COLLABORATION_HTTP_ADDR": "0.0.0.0:9300",
|
|
"COLLABORATION_DEBUG_ADDR": "0.0.0.0:9304",
|
|
"COLLABORATION_APP_PROOF_DISABLE": True,
|
|
"COLLABORATION_APP_INSECURE": True,
|
|
"COLLABORATION_CS3API_DATAGATEWAY_INSECURE": True,
|
|
"OC_JWT_SECRET": "some-opencloud-jwt-secret",
|
|
"COLLABORATION_WOPI_SECRET": "some-wopi-secret",
|
|
}
|
|
|
|
if name == "collabora":
|
|
environment["COLLABORATION_APP_NAME"] = "Collabora"
|
|
environment["COLLABORATION_APP_PRODUCT"] = "Collabora"
|
|
environment["COLLABORATION_APP_ADDR"] = "https://collabora:9980"
|
|
environment["COLLABORATION_APP_ICON"] = "https://collabora:9980/favicon.ico"
|
|
elif name == "onlyoffice":
|
|
environment["COLLABORATION_SERVICE_NAME"] = "collboration-onlyoffice"
|
|
environment["COLLABORATION_APP_NAME"] = "OnlyOffice"
|
|
environment["COLLABORATION_APP_PRODUCT"] = "OnlyOffice"
|
|
environment["COLLABORATION_APP_ADDR"] = "https://onlyoffice"
|
|
environment["COLLABORATION_APP_ICON"] = "https://onlyoffice/web-apps/apps/documenteditor/main/resources/img/favicon.ico"
|
|
elif name == "fakeoffice":
|
|
environment["COLLABORATION_SERVICE_NAME"] = "collboration-fakeoficce"
|
|
environment["COLLABORATION_APP_NAME"] = "FakeOffice"
|
|
environment["COLLABORATION_APP_PRODUCT"] = "Microsoft"
|
|
environment["COLLABORATION_APP_ADDR"] = "http://fakeoffice:8080"
|
|
|
|
environment["COLLABORATION_WOPI_SRC"] = "http://%s:9300" % service_name
|
|
|
|
return startOpenCloudService("collaboration", service_name, environment)
|
|
|
|
def tikaService():
|
|
return [{
|
|
"name": "tika",
|
|
"image": APACHE_TIKA,
|
|
"detach": True,
|
|
}, {
|
|
"name": "wait-for-tika-service",
|
|
"image": OC_CI_WAIT_FOR,
|
|
"commands": [
|
|
"wait-for -it tika:9998 -t 300",
|
|
],
|
|
}]
|
|
|
|
def logRequests():
|
|
return [{
|
|
"name": "api-test-failure-logs",
|
|
"image": OC_CI_PHP % DEFAULT_PHP_VERSION,
|
|
"commands": [
|
|
"cat %s/tests/acceptance/logs/failed.log" % dirs["base"],
|
|
],
|
|
"when": {
|
|
"status": [
|
|
"failure",
|
|
],
|
|
},
|
|
}]
|
|
|
|
def k6LoadTests(ctx):
|
|
opencloud_remote_environment = {
|
|
"SSH_OC_REMOTE": {
|
|
"from_secret": "k6_ssh_opencloud_remote",
|
|
},
|
|
"SSH_OC_USERNAME": {
|
|
"from_secret": "k6_ssh_opencloud_user",
|
|
},
|
|
"SSH_OC_PASSWORD": {
|
|
"from_secret": "k6_ssh_opencloud_pass",
|
|
},
|
|
"TEST_SERVER_URL": {
|
|
"from_secret": "k6_ssh_opencloud_server_url",
|
|
},
|
|
}
|
|
k6_remote_environment = {
|
|
"SSH_K6_REMOTE": {
|
|
"from_secret": "k6_ssh_k6_remote",
|
|
},
|
|
"SSH_K6_USERNAME": {
|
|
"from_secret": "k6_ssh_k6_user",
|
|
},
|
|
"SSH_K6_PASSWORD": {
|
|
"from_secret": "k6_ssh_k6_pass",
|
|
},
|
|
}
|
|
environment = {}
|
|
environment.update(opencloud_remote_environment)
|
|
environment.update(k6_remote_environment)
|
|
|
|
if "skip" in config["k6LoadTests"] and config["k6LoadTests"]["skip"]:
|
|
return []
|
|
|
|
opencloud_git_base_url = "https://raw.githubusercontent.com/%s" % repo_slug
|
|
script_link = "%s/%s/tests/config/woodpecker/run_k6_tests.sh" % (opencloud_git_base_url, ctx.build.commit)
|
|
|
|
event_array = ["cron"]
|
|
|
|
if "k6-test" in ctx.build.title.lower():
|
|
event_array.append("pull_request")
|
|
|
|
return [{
|
|
"name": "k6-load-test",
|
|
"skip_clone": True,
|
|
"steps": [
|
|
{
|
|
"name": "k6-load-test",
|
|
"image": OC_CI_ALPINE,
|
|
"environment": environment,
|
|
"commands": [
|
|
"curl -s -o run_k6_tests.sh %s" % script_link,
|
|
"apk add --no-cache openssh-client sshpass",
|
|
"sh %s/run_k6_tests.sh" % (dirs["base"]),
|
|
],
|
|
},
|
|
{
|
|
"name": "opencloud-log",
|
|
"image": OC_CI_ALPINE,
|
|
"environment": opencloud_remote_environment,
|
|
"commands": [
|
|
"curl -s -o run_k6_tests.sh %s" % script_link,
|
|
"apk add --no-cache openssh-client sshpass",
|
|
"sh %s/run_k6_tests.sh --opencloud-log" % (dirs["base"]),
|
|
],
|
|
"when": [
|
|
{
|
|
"status": ["success", "failure"],
|
|
},
|
|
],
|
|
},
|
|
{
|
|
"name": "open-grafana-dashboard",
|
|
"image": OC_CI_ALPINE,
|
|
"commands": [
|
|
"echo 'Grafana Dashboard: https://grafana.k6.infra.owncloud.works'",
|
|
],
|
|
"when": [
|
|
{
|
|
"status": ["success", "failure"],
|
|
},
|
|
],
|
|
},
|
|
],
|
|
"depends_on": [],
|
|
"when": [
|
|
{
|
|
"event": event_array,
|
|
},
|
|
],
|
|
}]
|
|
|
|
def waitForServices(name, services = []):
|
|
services = ",".join(services)
|
|
return [{
|
|
"name": "wait-for-%s" % name,
|
|
"image": OC_CI_WAIT_FOR,
|
|
"commands": [
|
|
"wait-for -it %s -t 300" % services,
|
|
],
|
|
}]
|
|
|
|
def openCloudHealthCheck(name, services = []):
|
|
commands = []
|
|
timeout = 300
|
|
curl_command = ["timeout %s bash -c 'while [ $(curl -s %s/%s ", "-w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done'"]
|
|
for service in services:
|
|
commands.append(curl_command[0] % (timeout, service, "healthz") + curl_command[1])
|
|
commands.append(curl_command[0] % (timeout, service, "readyz") + curl_command[1])
|
|
|
|
return [{
|
|
"name": "health-check-%s" % name,
|
|
"image": OC_CI_ALPINE,
|
|
"commands": commands,
|
|
}]
|
|
|
|
def collaboraService():
|
|
return [
|
|
{
|
|
"name": "collabora",
|
|
"image": COLLABORA_CODE,
|
|
"environment": {
|
|
"DONT_GEN_SSL_CERT": "set",
|
|
"extra_params": "--o:ssl.enable=true --o:ssl.termination=true --o:welcome.enable=false --o:net.frame_ancestors=%s" % OC_URL,
|
|
},
|
|
"commands": [
|
|
"coolconfig generate-proof-key",
|
|
"bash /start-collabora-online.sh",
|
|
],
|
|
},
|
|
]
|
|
|
|
def onlyofficeService():
|
|
return [
|
|
{
|
|
"name": "onlyoffice",
|
|
"image": ONLYOFFICE_DOCUMENT_SERVER,
|
|
"environment": {
|
|
"WOPI_ENABLED": True,
|
|
"USE_UNAUTHORIZED_STORAGE": True, # self signed certificates
|
|
},
|
|
"commands": [
|
|
"cp %s/tests/config/woodpecker/only-office.json /etc/onlyoffice/documentserver/local.json" % dirs["base"],
|
|
"openssl req -x509 -newkey rsa:4096 -keyout onlyoffice.key -out onlyoffice.crt -sha256 -days 365 -batch -nodes",
|
|
"mkdir -p /var/www/onlyoffice/Data/certs",
|
|
"cp onlyoffice.key /var/www/onlyoffice/Data/certs/",
|
|
"cp onlyoffice.crt /var/www/onlyoffice/Data/certs/",
|
|
"chmod 400 /var/www/onlyoffice/Data/certs/onlyoffice.key",
|
|
"/app/ds/run-document-server.sh",
|
|
],
|
|
},
|
|
]
|