diff --git a/.drone.star b/.drone.star deleted file mode 100644 index e0fbd185e..000000000 --- a/.drone.star +++ /dev/null @@ -1,3373 +0,0 @@ -"""OpenCloud CI definition -""" - -# Production release tags -# NOTE: need to be updated if new production releases are determined -# - follow semver -# - omit 'v' prefix -PRODUCTION_RELEASE_TAGS = ["5.0", "7.0"] - -# images -ALPINE_GIT = "alpine/git:latest" -APACHE_TIKA = "apache/tika:2.8.0.0" -CHKO_DOCKER_PUSHRM = "chko/docker-pushrm:1" -COLLABORA_CODE = "collabora/code:24.04.5.1.1" -INBUCKET_INBUCKET = "inbucket/inbucket" -MINIO_MC = "minio/mc:RELEASE.2021-10-07T04-19-58Z" -OC_CI_ALPINE = "owncloudci/alpine:latest" -OC_CI_BAZEL_BUILDIFIER = "owncloudci/bazel-buildifier:latest" -OC_CI_CLAMAVD = "owncloudci/clamavd" -OC_CI_DRONE_ANSIBLE = "owncloudci/drone-ansible:latest" -OC_CI_DRONE_SKIP_PIPELINE = "owncloudci/drone-skip-pipeline" -OC_CI_GOLANG = "owncloudci/golang:1.22" -OC_CI_NODEJS = "owncloudci/nodejs:%s" -OC_CI_PHP = "owncloudci/php:%s" -OC_CI_WAIT_FOR = "owncloudci/wait-for:latest" -OC_CS3_API_VALIDATOR = "owncloud/cs3api-validator:0.2.1" -OC_LITMUS = "owncloudci/litmus:latest" -OC_UBUNTU = "owncloud/ubuntu:20.04" -ONLYOFFICE_DOCUMENT_SERVER = "onlyoffice/documentserver:7.5.1" -PLUGINS_CODACY = "plugins/codacy:1" -PLUGINS_DOCKER = "plugins/docker:latest" -PLUGINS_GH_PAGES = "plugins/gh-pages:1" -PLUGINS_GITHUB_RELEASE = "plugins/github-release:1" -PLUGINS_GIT_ACTION = "plugins/git-action:1" -PLUGINS_MANIFEST = "plugins/manifest:1" -PLUGINS_S3 = "plugins/s3:1" -PLUGINS_S3_CACHE = "plugins/s3-cache:1" -PLUGINS_SLACK = "plugins/slack:1" -REDIS = "redis:6-alpine" -SONARSOURCE_SONAR_SCANNER_CLI = "sonarsource/sonar-scanner-cli:11.0" - -DEFAULT_PHP_VERSION = "8.2" -DEFAULT_NODEJS_VERSION = "20" - -dirs = { - "base": "/drone/src", - "web": "/drone/src/webTestRunner", - "zip": "/drone/src/zip", - "webZip": "/drone/src/zip/web.tar.gz", - "webPnpmZip": "/drone/src/zip/pnpm-store.tar.gz", - "gobinTar": "go-bin.tar.gz", - "gobinTarPath": "/drone/src/go-bin.tar.gz", - "opencloudConfig": "tests/config/drone/opencloud-config.json", - "opencloud": "/srv/app/tmp/opencloud", - "opencloudRevaDataRoot": "/srv/app/tmp/opencloud/opencloud/data", - "ocWrapper": "/drone/src/tests/ocwrapper", - "bannedPasswordList": "tests/config/drone/banned-password-list.txt", - "ocmProviders": "tests/config/drone/providers.json", -} - -# OpenCloud URLs -OC_SERVER_NAME = "opencloud-server" -OC_URL = "https://%s:9200" % OC_SERVER_NAME -OC_DOMAIN = "%s:9200" % OC_SERVER_NAME -FED_OC_SERVER_NAME = "federation-opencloud-server" -OC_FED_URL = "https://%s:10200" % FED_OC_SERVER_NAME -OC_FED_DOMAIN = "%s:10200" % FED_OC_SERVER_NAME - -# configuration -config = { - "cs3ApiTests": { - "skip": False, - }, - "wopiValidatorTests": { - "skip": False, - }, - "k6LoadTests": { - "skip": False, - }, - "localApiTests": { - "basic": { - "suites": [ - "apiArchiver", - "apiContract", - "apiCors", - "apiAsyncUpload", - "apiDownloads", - "apiDepthInfinity", - "apiLocks", - "apiActivities", - "apiSettings", - ], - "skip": False, - }, - "graph": { - "suites": [ - "apiGraph", - "apiServiceAvailability", - ], - "skip": False, - "withRemotePhp": [True], - }, - "graphUserGroup": { - "suites": [ - "apiGraphUserGroup", - ], - "skip": False, - "withRemotePhp": [True], - }, - "spaces": { - "suites": [ - "apiSpaces", - ], - "skip": False, - }, - "spacesShares": { - "suites": [ - "apiSpacesShares", - ], - "skip": False, - }, - "spacesDavOperation": { - "suites": [ - "apiSpacesDavOperation", - ], - "skip": False, - }, - "search1": { - "suites": [ - "apiSearch1", - ], - "skip": False, - }, - "search2": { - "suites": [ - "apiSearch2", - ], - "skip": False, - }, - "sharingNg": { - "suites": [ - "apiReshare", - "apiSharingNg1", - "apiSharingNg2", - ], - "skip": False, - }, - "sharingNgShareInvitation": { - "suites": [ - "apiSharingNgShareInvitation", - ], - "skip": False, - }, - "sharingNgLinkShare": { - "suites": [ - "apiSharingNgLinkSharePermission", - "apiSharingNgLinkShareRoot", - ], - "skip": False, - }, - "accountsHashDifficulty": { - "skip": False, - "suites": [ - "apiAccountsHashDifficulty", - ], - "accounts_hash_difficulty": "default", - }, - "notification": { - "suites": [ - "apiNotification", - ], - "skip": False, - "withRemotePhp": [True], - "emailNeeded": True, - "extraEnvironment": { - "EMAIL_HOST": "email", - "EMAIL_PORT": "9000", - }, - "extraServerEnvironment": { - "OC_ADD_RUN_SERVICES": "notifications", - "NOTIFICATIONS_SMTP_HOST": "email", - "NOTIFICATIONS_SMTP_PORT": "2500", - "NOTIFICATIONS_SMTP_INSECURE": "true", - "NOTIFICATIONS_SMTP_SENDER": "OpenCloud ", - "NOTIFICATIONS_DEBUG_ADDR": "0.0.0.0:9174", - }, - }, - "antivirus": { - "suites": [ - "apiAntivirus", - ], - "skip": False, - "antivirusNeeded": True, - "extraServerEnvironment": { - "ANTIVIRUS_SCANNER_TYPE": "clamav", - "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310", - "POSTPROCESSING_STEPS": "virusscan", - "OC_ASYNC_UPLOADS": True, - "OC_ADD_RUN_SERVICES": "antivirus", - "ANTIVIRUS_DEBUG_ADDR": "0.0.0.0:9297", - }, - }, - "searchContent": { - "suites": [ - "apiSearchContent", - ], - "skip": False, - "tikaNeeded": True, - }, - "ocm": { - "suites": [ - "apiOcm", - ], - "skip": False, - "withRemotePhp": [True], - "federationServer": True, - "emailNeeded": True, - "extraEnvironment": { - "EMAIL_HOST": "email", - "EMAIL_PORT": "9000", - }, - "extraServerEnvironment": { - "OC_ADD_RUN_SERVICES": "ocm,notifications", - "OC_ENABLE_OCM": True, - "OCM_OCM_INVITE_MANAGER_INSECURE": True, - "OCM_OCM_SHARE_PROVIDER_INSECURE": True, - "OCM_OCM_STORAGE_PROVIDER_INSECURE": True, - "OCM_OCM_PROVIDER_AUTHORIZER_PROVIDERS_FILE": "%s" % dirs["ocmProviders"], - # mail notifications - "NOTIFICATIONS_SMTP_HOST": "email", - "NOTIFICATIONS_SMTP_PORT": "2500", - "NOTIFICATIONS_SMTP_INSECURE": "true", - "NOTIFICATIONS_SMTP_SENDER": "OpenCloud ", - }, - }, - "wopi": { - "suites": [ - "apiCollaboration", - ], - "skip": False, - "collaborationServiceNeeded": True, - "extraServerEnvironment": { - "GATEWAY_GRPC_ADDR": "0.0.0.0:9142", - }, - }, - "authApp": { - "suites": [ - "apiAuthApp", - ], - "skip": False, - "withRemotePhp": [True], - "extraServerEnvironment": { - "OC_ADD_RUN_SERVICES": "auth-app", - "PROXY_ENABLE_APP_AUTH": True, - }, - }, - "cliCommands": { - "suites": [ - "cliCommands", - ], - "skip": False, - "withRemotePhp": [True], - "antivirusNeeded": True, - "extraServerEnvironment": { - "ANTIVIRUS_SCANNER_TYPE": "clamav", - "ANTIVIRUS_CLAMAV_SOCKET": "tcp://clamav:3310", - "OC_ASYNC_UPLOADS": True, - "OC_ADD_RUN_SERVICES": "antivirus", - }, - }, - }, - "apiTests": { - "numberOfParts": 7, - "skip": False, - "skipExceptParts": [], - }, - "e2eTests": { - "part": { - "skip": False, - "totalParts": 4, # divide and run all suites in parts (divide pipelines) - "xsuites": ["search", "app-provider", "oidc", "ocm"], # suites to skip - }, - "search": { - "skip": False, - "suites": ["search"], # suites to run - "tikaNeeded": True, - }, - }, - "e2eMultiService": { - "testSuites": { - "skip": False, - "suites": [ - "smoke", - "shares", - "search", - "journeys", - "file-action", - "spaces", - ], - "tikaNeeded": True, - }, - }, - "rocketchat": { - "channel": "builds", - "channel_cron": "builds", - "from_secret": "rocketchat_talk_webhook", - }, - "binaryReleases": { - "os": ["linux", "darwin"], - }, - "dockerReleases": { - "architectures": ["arm64", "amd64"], - }, - "litmus": True, - "codestyle": True, -} - -# volume for steps to cache Go dependencies between steps of a pipeline -# GOPATH must be set to /go inside the image, which is the case -stepVolumeGo = \ - { - "name": "gopath", - "path": "/go", - } - -# volume for pipeline to cache Go dependencies between steps of a pipeline -# to be used in combination with stepVolumeGo -pipelineVolumeGo = \ - { - "name": "gopath", - "temp": {}, - } - -# minio mc environment variables -MINIO_MC_ENV = { - "CACHE_BUCKET": { - "from_secret": "cache_s3_bucket", - }, - "MC_HOST": { - "from_secret": "cache_s3_server", - }, - "AWS_ACCESS_KEY_ID": { - "from_secret": "cache_s3_access_key", - }, - "AWS_SECRET_ACCESS_KEY": { - "from_secret": "cache_s3_secret_key", - }, -} - -DRONE_HTTP_PROXY_ENV = { - "HTTP_PROXY": { - "from_secret": "drone_http_proxy", - }, - "HTTPS_PROXY": { - "from_secret": "drone_http_proxy", - }, -} - -def pipelineDependsOn(pipeline, dependant_pipelines): - if "depends_on" in pipeline.keys(): - pipeline["depends_on"] = pipeline["depends_on"] + getPipelineNames(dependant_pipelines) - else: - pipeline["depends_on"] = getPipelineNames(dependant_pipelines) - return pipeline - -def pipelinesDependsOn(pipelines, dependant_pipelines): - pipes = [] - for pipeline in pipelines: - pipes.append(pipelineDependsOn(pipeline, dependant_pipelines)) - - return pipes - -def getPipelineNames(pipelines = []): - """getPipelineNames returns names of pipelines as a string array - - Args: - pipelines: array of drone pipelines - - Returns: - names of the given pipelines as string array - """ - names = [] - for pipeline in pipelines: - names.append(pipeline["name"]) - return names - -def main(ctx): - """main is the entrypoint for drone - - Args: - ctx: drone passes a context with information which the pipeline can be adapted to - - Returns: - none - """ - - pipelines = [] - - build_release_helpers = \ - changelog() + \ - docs() + \ - licenseCheck(ctx) - - test_pipelines = \ - codestyle(ctx) + \ - checkGherkinLint(ctx) + \ - checkTestSuitesInExpectedFailures(ctx) + \ - buildWebCache(ctx) + \ - getGoBinForTesting(ctx) + \ - buildOpenCloudBinaryForTesting(ctx) + \ - checkStarlark() + \ - build_release_helpers + \ - testOpenCloudAndUploadResults(ctx) + \ - testPipelines(ctx) - - build_release_pipelines = \ - dockerReleases(ctx) + \ - binaryReleases(ctx) - - test_pipelines.append( - pipelineDependsOn( - purgeBuildArtifactCache(ctx), - testPipelines(ctx), - ), - ) - - pipelines = test_pipelines + build_release_pipelines - - if ctx.build.event == "cron": - pipelines = \ - pipelines + \ - example_deploys(ctx) - else: - pipelines = \ - pipelines + \ - pipelinesDependsOn( - example_deploys(ctx), - pipelines, - ) - - # always append notification step - pipelines.append( - pipelineDependsOn( - notify(ctx), - pipelines, - ), - ) - - pipelineSanityChecks(ctx, pipelines) - return pipelines - -def cachePipeline(name, steps): - return { - "kind": "pipeline", - "type": "docker", - "name": "build-%s-cache" % name, - "clone": { - "disable": True, - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/tags/**", - "refs/pull/**", - ], - }, - } - -def buildWebCache(ctx): - return [ - cachePipeline("web", generateWebCache(ctx)), - cachePipeline("web-pnpm", generateWebPnpmCache(ctx)), - ] - -def testOpenCloudAndUploadResults(ctx): - pipeline = testOpenCloud(ctx) - - ###################################################################### - # The triggers have been disabled for now, since the govulncheck can # - # not silence single, acceptable vulnerabilities. # - # See https://github.com/owncloud/ocis/issues/9527 for more details. # - # FIXME: RE-ENABLE THIS ASAP!!! # - ###################################################################### - - scan_result_upload = uploadScanResults(ctx) - scan_result_upload["depends_on"] = getPipelineNames([pipeline]) - - #security_scan = scanOpenCloud(ctx) - #return [security_scan, pipeline, scan_result_upload] - return [pipeline, scan_result_upload] - -def testPipelines(ctx): - pipelines = [] - - if config["litmus"]: - pipelines += litmus(ctx, "opencloud") - - if "skip" not in config["cs3ApiTests"] or not config["cs3ApiTests"]["skip"]: - pipelines.append(cs3ApiTests(ctx, "opencloud", "default")) - if "skip" not in config["wopiValidatorTests"] or not config["wopiValidatorTests"]["skip"]: - pipelines.append(wopiValidatorTests(ctx, "opencloud", "builtin", "default")) - pipelines.append(wopiValidatorTests(ctx, "opencloud", "cs3", "default")) - - pipelines += localApiTestPipeline(ctx) - - if "skip" not in config["apiTests"] or not config["apiTests"]["skip"]: - pipelines += apiTests(ctx) - - pipelines += e2eTestPipeline(ctx) + multiServiceE2ePipeline(ctx) - - if ("skip" not in config["k6LoadTests"] or not config["k6LoadTests"]["skip"]) and ("k6-test" in ctx.build.title.lower() or ctx.build.event == "cron"): - pipelines += k6LoadTests(ctx) - - return pipelines - -def getGoBinForTesting(ctx): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "get-go-bin-cache", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "unit-tests") + - checkGoBinCache() + - cacheGoBin(), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/stable-*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - }] - -def checkGoBinCache(): - return [{ - "name": "check-go-bin-cache", - "image": OC_UBUNTU, - "environment": { - "CACHE_ENDPOINT": { - "from_secret": "cache_s3_server", - }, - "CACHE_BUCKET": { - "from_secret": "cache_s3_bucket", - }, - }, - "commands": [ - "bash -x %s/tests/config/drone/check_go_bin_cache.sh %s %s" % (dirs["base"], dirs["base"], dirs["gobinTar"]), - ], - }] - -def cacheGoBin(): - return [ - { - "name": "bingo-get", - "image": OC_CI_GOLANG, - "commands": [ - "make bingo-update", - ], - "volumes": [stepVolumeGo], - "environment": DRONE_HTTP_PROXY_ENV, - }, - { - "name": "archive-go-bin", - "image": OC_UBUNTU, - "commands": [ - "tar -czvf %s /go/bin" % dirs["gobinTarPath"], - ], - "volumes": [stepVolumeGo], - }, - { - "name": "cache-go-bin", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - # .bingo folder will change after 'bingo-get' - # so get the stored hash of a .bingo folder - "BINGO_HASH=$(cat %s/.bingo_hash)" % dirs["base"], - # cache using the minio client to the public bucket (long term bucket) - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r %s s3/$CACHE_BUCKET/opencloud/go-bin/$BINGO_HASH" % (dirs["gobinTarPath"]), - ], - "volumes": [stepVolumeGo], - }, - ] - -def restoreGoBinCache(): - return [ - { - "name": "restore-go-bin-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "BINGO_HASH=$(cat %s/.bingo/* | sha256sum | cut -d ' ' -f 1)" % dirs["base"], - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a s3/$CACHE_BUCKET/opencloud/go-bin/$BINGO_HASH/%s %s" % (dirs["gobinTar"], dirs["base"]), - ], - "volumes": [stepVolumeGo], - }, - { - "name": "extract-go-bin-cache", - "image": OC_UBUNTU, - "commands": [ - "tar -xvmf %s -C /" % dirs["gobinTarPath"], - ], - "volumes": [stepVolumeGo], - }, - ] - -def testOpenCloud(ctx): - steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate("") + [ - { - "name": "golangci-lint", - "image": OC_CI_GOLANG, - "commands": [ - "mkdir -p cache/checkstyle", - "make ci-golangci-lint", - "mv checkstyle.xml cache/checkstyle/checkstyle.xml", - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - { - "name": "test", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "mkdir -p cache/coverage", - "make test", - "mv coverage.out cache/coverage/", - ], - "volumes": [stepVolumeGo], - }, - { - "name": "scan-result-cache", - "image": PLUGINS_S3, - "settings": { - "endpoint": { - "from_secret": "cache_s3_server", - }, - "bucket": "cache", - "source": "cache/**/*", - "target": "%s/%s" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - "path_style": True, - "access_key": { - "from_secret": "cache_s3_access_key", - }, - "secret_key": { - "from_secret": "cache_s3_secret_key", - }, - }, - }, - ] - - return { - "kind": "pipeline", - "type": "docker", - "name": "linting_and_unitTests", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - "depends_on": getPipelineNames(getGoBinForTesting(ctx)), - "volumes": [pipelineVolumeGo], - } - -def scanOpenCloud(ctx): - steps = skipIfUnchanged(ctx, "unit-tests") + restoreGoBinCache() + makeGoGenerate("") + [ - { - "name": "govulncheck", - "image": OC_CI_GOLANG, - "commands": [ - "make govulncheck", - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - ] - - return { - "kind": "pipeline", - "type": "docker", - "name": "go-vulnerability-scanning", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - "depends_on": getPipelineNames(getGoBinForTesting(ctx)), - "volumes": [pipelineVolumeGo], - } - -def buildOpenCloudBinaryForTesting(ctx): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "build_opencloud_binary_for_testing", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - makeNodeGenerate("") + - makeGoGenerate("") + - build() + - rebuildBuildArtifactCache(ctx, "opencloud-binary-amd64", "opencloud/bin"), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - }] - -def uploadScanResults(ctx): - sonar_env = { - "SONAR_TOKEN": { - "from_secret": "sonar_token", - }, - } - if ctx.build.event == "pull_request": - sonar_env.update({ - "SONAR_PULL_REQUEST_BASE": "%s" % (ctx.build.target), - "SONAR_PULL_REQUEST_BRANCH": "%s" % (ctx.build.source), - "SONAR_PULL_REQUEST_KEY": "%s" % (ctx.build.ref.replace("refs/pull/", "").split("/")[0]), - }) - - fork_handling = [] - if ctx.build.source_repo != "" and ctx.build.source_repo != ctx.repo.slug: - fork_handling = [ - "git remote add fork https://github.com/%s.git" % (ctx.build.source_repo), - "git fetch fork", - ] - - return { - "kind": "pipeline", - "type": "docker", - "name": "upload-scan-results", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "clone": { - "disable": True, # Sonarcloud does not apply issues on already merged branch - }, - "steps": [ - { - "name": "clone", - "image": ALPINE_GIT, - "commands": [ - # Always use the opencloud-eu/opencloud repository as base to have an up to date default branch. - # This is needed for the skipIfUnchanged step, since it references a commit on master (which could be absent on a fork) - "git clone https://github.com/%s.git ." % (ctx.repo.slug), - ] + fork_handling + - [ - "git checkout $DRONE_COMMIT", - ], - }, - { - "name": "sync-from-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mkdir -p cache", - "mc alias set cachebucket $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc mirror cachebucket/$CACHE_BUCKET/%s/%s/cache cache/ || true" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - ], - }, - { - "name": "codacy", - "image": PLUGINS_CODACY, - "settings": { - "token": { - "from_secret": "codacy_token", - }, - }, - }, - { - "name": "sonarcloud", - "image": SONARSOURCE_SONAR_SCANNER_CLI, - "environment": sonar_env, - }, - { - "name": "purge-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "mc alias set cachebucket $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc rm --recursive --force cachebucket/$CACHE_BUCKET/%s/%s/cache || true" % (ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}"), - ], - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - "status": [ - "success", - "failure", - ], - }, - } - -def vendorbinCodestyle(phpVersion): - return [{ - "name": "vendorbin-codestyle", - "image": OC_CI_PHP % phpVersion, - "environment": { - "COMPOSER_HOME": "%s/.cache/composer" % dirs["base"], - }, - "commands": [ - "make vendor-bin-codestyle", - ], - }] - -def vendorbinCodesniffer(phpVersion): - return [{ - "name": "vendorbin-codesniffer", - "image": OC_CI_PHP % phpVersion, - "environment": { - "COMPOSER_HOME": "%s/.cache/composer" % dirs["base"], - }, - "commands": [ - "make vendor-bin-codesniffer", - ], - }] - -def checkTestSuitesInExpectedFailures(ctx): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "check-suites-in-expected-failures", - "steps": [ - { - "name": "check-suites", - "image": OC_CI_ALPINE, - "commands": [ - "%s/tests/acceptance/check-deleted-suites-in-expected-failure.sh" % dirs["base"], - ], - }, - ], - "trigger": { - "ref": [ - "refs/pull/**", - ], - }, - }] - -def checkGherkinLint(ctx): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "check-gherkin-standard", - "steps": [ - { - "name": "lint-feature-files", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "npm install -g @gherlint/gherlint@1.1.0", - "make test-gherkin-lint", - ], - }, - ], - "trigger": { - "ref": [ - "refs/pull/**", - ], - }, - }] - -def codestyle(ctx): - pipelines = [] - - if "codestyle" not in config: - return [] - - default = { - "phpVersions": [DEFAULT_PHP_VERSION], - } - - if "defaults" in config: - if "codestyle" in config["defaults"]: - for item in config["defaults"]["codestyle"]: - default[item] = config["defaults"]["codestyle"][item] - - codestyleConfig = config["codestyle"] - - if type(codestyleConfig) == "bool": - if codestyleConfig: - # the config has 'codestyle' true, so specify an empty dict that will get the defaults - codestyleConfig = {} - else: - return pipelines - - if len(codestyleConfig) == 0: - # 'codestyle' is an empty dict, so specify a single section that will get the defaults - codestyleConfig = {"doDefault": {}} - - for category, matrix in codestyleConfig.items(): - params = {} - for item in default: - params[item] = matrix[item] if item in matrix else default[item] - - for phpVersion in params["phpVersions"]: - name = "coding-standard-php%s" % phpVersion - - result = { - "kind": "pipeline", - "type": "docker", - "name": name, - "workspace": { - "base": "/drone", - "path": "src", - }, - "steps": skipIfUnchanged(ctx, "lint") + - vendorbinCodestyle(phpVersion) + - vendorbinCodesniffer(phpVersion) + - [ - { - "name": "php-style", - "image": OC_CI_PHP % phpVersion, - "commands": [ - "make test-php-style", - ], - }, - { - "name": "check-env-var-annotations", - "image": OC_CI_PHP % phpVersion, - "commands": [ - "make check-env-var-annotations", - ], - }, - ], - "depends_on": [], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - } - - pipelines.append(result) - - return pipelines - -def localApiTestPipeline(ctx): - pipelines = [] - - with_remote_php = [True] - if ctx.build.event == "cron" or "full-ci" in ctx.build.title.lower(): - with_remote_php.append(False) - - defaults = { - "suites": {}, - "skip": False, - "extraEnvironment": {}, - "extraServerEnvironment": {}, - "storages": ["opencloud"], - "accounts_hash_difficulty": 4, - "emailNeeded": False, - "antivirusNeeded": False, - "tikaNeeded": False, - "federationServer": False, - "collaborationServiceNeeded": False, - "extraCollaborationEnvironment": {}, - "withRemotePhp": with_remote_php, - } - - if "localApiTests" in config: - for name, matrix in config["localApiTests"].items(): - if "skip" not in matrix or not matrix["skip"]: - params = {} - for item in defaults: - params[item] = matrix[item] if item in matrix else defaults[item] - for storage in params["storages"]: - for run_with_remote_php in params["withRemotePhp"]: - pipeline = { - "kind": "pipeline", - "type": "docker", - "name": "%s-%s%s" % ("CLI" if name.startswith("cli") else "API", name, "-withoutRemotePhp" if not run_with_remote_php else ""), - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - restoreBuildArtifactCache(ctx, "opencloud-binary-amd64", "opencloud/bin") + - (tikaService() if params["tikaNeeded"] else []) + - (waitForServices("online-offices", ["collabora:9980", "onlyoffice:443", "fakeoffice:8080"]) if params["collaborationServiceNeeded"] else []) + - opencloudServer(storage, params["accounts_hash_difficulty"], extra_server_environment = params["extraServerEnvironment"], with_wrapper = True, tika_enabled = params["tikaNeeded"]) + - (waitForClamavService() if params["antivirusNeeded"] else []) + - (waitForEmailService() if params["emailNeeded"] else []) + - (opencloudServer(storage, params["accounts_hash_difficulty"], deploy_type = "federation", extra_server_environment = params["extraServerEnvironment"]) if params["federationServer"] else []) + - ((wopiCollaborationService("fakeoffice") + wopiCollaborationService("collabora") + wopiCollaborationService("onlyoffice")) if params["collaborationServiceNeeded"] else []) + - (opencloudHealthCheck("wopi", ["wopi-collabora:9304", "wopi-onlyoffice:9304", "wopi-fakeoffice:9304"]) if params["collaborationServiceNeeded"] else []) + - localApiTests(ctx, name, params["suites"], storage, params["extraEnvironment"], run_with_remote_php) + - logRequests(), - "services": (emailService() if params["emailNeeded"] else []) + - (clamavService() if params["antivirusNeeded"] else []) + - ((fakeOffice() + collaboraService() + onlyofficeService()) if params["collaborationServiceNeeded"] else []), - "depends_on": getPipelineNames(buildOpenCloudBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - } - pipelines.append(pipeline) - return pipelines - -def localApiTests(ctx, name, suites, storage = "opencloud", extra_environment = {}, with_remote_php = False): - test_dir = "%s/tests/acceptance" % dirs["base"] - expected_failures_file = "%s/expected-failures-localAPI-on-%s-storage.md" % (test_dir, storage.upper()) - - environment = { - "TEST_SERVER_URL": OC_URL, - "TEST_SERVER_FED_URL": OC_FED_URL, - "OC_REVA_DATA_ROOT": "%s" % (dirs["opencloudRevaDataRoot"] if storage == "owncloud" else ""), - "SEND_SCENARIO_LINE_REFERENCES": "true", - "STORAGE_DRIVER": storage, - "BEHAT_SUITES": ",".join(suites), - "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnGraph&&~@skipOnDecomposed-%s-Storage" % ("OC" if storage == "owncloud" else "DECOMPOSED"), - "EXPECTED_FAILURES_FILE": expected_failures_file, - "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, - "OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME, - "WITH_REMOTE_PHP": with_remote_php, - "COLLABORATION_SERVICE_URL": "http://wopi-fakeoffice:9300", - } - - for item in extra_environment: - environment[item] = extra_environment[item] - - return [{ - "name": "localApiTests-%s" % name, - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "environment": environment, - "commands": [ - # merge the expected failures - "" if with_remote_php else "cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file), - "make -C %s test-acceptance-api" % (dirs["base"]), - ], - }] - -def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): - return { - "kind": "pipeline", - "type": "docker", - "name": "cs3ApiTests", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - restoreBuildArtifactCache(ctx, "opencloud-binary-amd64", "opencloud/bin") + - opencloudServer(storage, accounts_hash_difficulty, [], [], "cs3api_validator") + - [ - { - "name": "cs3ApiTests", - "image": OC_CS3_API_VALIDATOR, - "environment": {}, - "commands": [ - "/usr/bin/cs3api-validator /var/lib/cs3api-validator --endpoint=%s:9142" % OC_SERVER_NAME, - ], - }, - ], - "depends_on": getPipelineNames(buildOpenCloudBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - } - -def wopiValidatorTests(ctx, storage, wopiServerType, accounts_hash_difficulty = 4): - testgroups = [ - "BaseWopiViewing", - "CheckFileInfoSchema", - "EditFlows", - "Locks", - "AccessTokens", - "GetLock", - "ExtendedLockLength", - "FileVersion", - "Features", - ] - builtinOnlyTestGroups = [ - "PutRelativeFile", - "RenameFileIfCreateChildFileIsNotSupported", - ] - - opencloud_bin = "opencloud/bin/opencloud" - validatorTests = [] - wopiServer = [] - extra_server_environment = {} - - if wopiServerType == "cs3": - wopiServer = [ - { - "name": "wopi-fakeoffice", - "image": "cs3org/wopiserver:v10.4.0", - "detach": True, - "commands": [ - "cp %s/tests/config/drone/wopiserver.conf /etc/wopi/wopiserver.conf" % (dirs["base"]), - "echo 123 > /etc/wopi/wopisecret", - "/app/wopiserver.py", - ], - }, - ] - else: - extra_server_environment = { - "OC_EXCLUDE_RUN_SERVICES": "app-provider", - } - - wopiServer = wopiCollaborationService("fakeoffice") - - for testgroup in testgroups: - validatorTests.append({ - "name": "wopiValidatorTests-%s" % testgroup, - "image": "owncloudci/wopi-validator", - "commands": [ - "export WOPI_TOKEN=$(cat accesstoken)", - "echo $WOPI_TOKEN", - "export WOPI_TTL=$(cat accesstokenttl)", - "echo $WOPI_TTL", - "export WOPI_SRC=$(cat wopisrc)", - "echo $WOPI_SRC", - "cd /app", - "/app/Microsoft.Office.WopiValidator -t $WOPI_TOKEN -w $WOPI_SRC -l $WOPI_TTL --testgroup %s" % testgroup, - ], - }) - if wopiServerType == "builtin": - for builtinOnlyGroup in builtinOnlyTestGroups: - validatorTests.append({ - "name": "wopiValidatorTests-%s" % builtinOnlyGroup, - "image": "owncloudci/wopi-validator", - "commands": [ - "export WOPI_TOKEN=$(cat accesstoken)", - "echo $WOPI_TOKEN", - "export WOPI_TTL=$(cat accesstokenttl)", - "echo $WOPI_TTL", - "export WOPI_SRC=$(cat wopisrc)", - "echo $WOPI_SRC", - "cd /app", - "/app/Microsoft.Office.WopiValidator -s -t $WOPI_TOKEN -w $WOPI_SRC -l $WOPI_TTL --testgroup %s" % builtinOnlyGroup, - ], - }) - - return { - "kind": "pipeline", - "type": "docker", - "name": "wopiValidatorTests-%s" % wopiServerType, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - restoreBuildArtifactCache(ctx, "opencloud-binary-amd64", "opencloud/bin") + - fakeOffice() + - waitForServices("fake-office", ["fakeoffice:8080"]) + - opencloudServer(storage, accounts_hash_difficulty, deploy_type = "wopi_validator", extra_server_environment = extra_server_environment) + - wopiServer + - waitForServices("wopi-fakeoffice", ["wopi-fakeoffice:9300"]) + - [ - { - "name": "prepare-test-file", - "image": OC_CI_ALPINE, - "environment": {}, - "commands": [ - "curl -v -X PUT '%s/remote.php/webdav/test.wopitest' -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin -D headers.txt" % OC_URL, - "cat headers.txt", - "export FILE_ID=$(cat headers.txt | sed -n -e 's/^.*Oc-Fileid: //p')", - "export URL=\"%s/app/open?app_name=FakeOffice&file_id=$FILE_ID\"" % OC_URL, - "export URL=$(echo $URL | tr -d '[:cntrl:]')", - "curl -v -X POST \"$URL\" -k --fail --retry-connrefused --retry 7 --retry-all-errors -u admin:admin > open.json", - "cat open.json", - "cat open.json | jq .form_parameters.access_token | tr -d '\"' > accesstoken", - "cat open.json | jq .form_parameters.access_token_ttl | tr -d '\"' > accesstokenttl", - "echo -n 'http://wopi-fakeoffice:9300/wopi/files/' > wopisrc", - "cat open.json | jq .app_url | sed -n -e 's/^.*files%2F//p' | tr -d '\"' >> wopisrc", - ], - }, - ] + - validatorTests, - "depends_on": getPipelineNames(buildOpenCloudBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - } - -def coreApiTests(ctx, part_number = 1, number_of_parts = 1, with_remote_php = False, storage = "opencloud", accounts_hash_difficulty = 4): - filterTags = "~@skipOnGraph&&~@skipOnDecomposed-%s-Storage" % ("OC" if storage == "owncloud" else "DECOMPOSED") - test_dir = "%s/tests/acceptance" % dirs["base"] - expected_failures_file = "%s/expected-failures-API-on-%s-storage.md" % (test_dir, storage.upper()) - - return { - "kind": "pipeline", - "type": "docker", - "name": "Core-API-Tests-%s%s" % (part_number, "-withoutRemotePhp" if not with_remote_php else ""), - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "acceptance-tests") + - restoreBuildArtifactCache(ctx, "opencloud-binary-amd64", "opencloud/bin") + - opencloudServer(storage, accounts_hash_difficulty, with_wrapper = True) + - [ - { - "name": "oC10ApiTests-%s" % part_number, - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "environment": { - "TEST_SERVER_URL": OC_URL, - "OC_REVA_DATA_ROOT": "%s" % (dirs["opencloudRevaDataRoot"] if storage == "owncloud" else ""), - "SEND_SCENARIO_LINE_REFERENCES": "true", - "STORAGE_DRIVER": storage, - "BEHAT_FILTER_TAGS": filterTags, - "DIVIDE_INTO_NUM_PARTS": number_of_parts, - "RUN_PART": part_number, - "ACCEPTANCE_TEST_TYPE": "core-api", - "EXPECTED_FAILURES_FILE": expected_failures_file, - "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, - "OC_WRAPPER_URL": "http://%s:5200" % OC_SERVER_NAME, - "WITH_REMOTE_PHP": with_remote_php, - }, - "commands": [ - # merge the expected failures - "" if with_remote_php else "cat %s/expected-failures-without-remotephp.md >> %s" % (test_dir, expected_failures_file), - "make -C %s test-acceptance-api" % (dirs["base"]), - ], - }, - ] + - logRequests(), - "services": redisForOCStorage(storage), - "depends_on": getPipelineNames(buildOpenCloudBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - } - -def apiTests(ctx): - pipelines = [] - debugParts = config["apiTests"]["skipExceptParts"] - debugPartsEnabled = (len(debugParts) != 0) - - with_remote_php = [True] - if ctx.build.event == "cron" or "full-ci" in ctx.build.title.lower(): - with_remote_php.append(False) - - defaults = { - "withRemotePhp": with_remote_php, - } - - for runPart in range(1, config["apiTests"]["numberOfParts"] + 1): - for run_with_remote_php in defaults["withRemotePhp"]: - if (not debugPartsEnabled or (debugPartsEnabled and runPart in debugParts)): - pipelines.append(coreApiTests(ctx, runPart, config["apiTests"]["numberOfParts"], run_with_remote_php)) - - return pipelines - -def e2eTestPipeline(ctx): - defaults = { - "skip": False, - "suites": [], - "xsuites": [], - "totalParts": 0, - "tikaNeeded": False, - } - - extra_server_environment = { - "OC_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": "%s" % dirs["bannedPasswordList"], - } - - e2e_trigger = { - "ref": [ - "refs/heads/master", - "refs/tags/**", - "refs/pull/**", - ], - } - - e2e_volumes = [{ - "name": "uploads", - "temp": {}, - }, { - "name": "configs", - "temp": {}, - }, { - "name": "gopath", - "temp": {}, - }] - - pipelines = [] - - if ("skip-e2e" in ctx.build.title.lower()): - return pipelines - - if (ctx.build.event == "tag"): - return pipelines - - for name, suite in config["e2eTests"].items(): - if "skip" in suite and suite["skip"]: - continue - - params = {} - for item in defaults: - params[item] = suite[item] if item in suite else defaults[item] - - e2e_args = "" - if params["totalParts"] > 0: - e2e_args = "--total-parts %d" % params["totalParts"] - elif params["suites"]: - e2e_args = "--suites %s" % ",".join(params["suites"]) - - # suites to skip - if params["xsuites"]: - e2e_args += " --xsuites %s" % ",".join(params["xsuites"]) - - steps_before = \ - skipIfUnchanged(ctx, "e2e-tests") + \ - restoreBuildArtifactCache(ctx, "opencloud-binary-amd64", "opencloud/bin/opencloud") + \ - restoreWebCache() + \ - restoreWebPnpmCache() + \ - (tikaService() if params["tikaNeeded"] else []) + \ - opencloudServer(extra_server_environment = extra_server_environment, tika_enabled = params["tikaNeeded"]) - - step_e2e = { - "name": "e2e-tests", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": { - "BASE_URL_OPENCLOUD": OC_DOMAIN, - "HEADLESS": "true", - "RETRY": "1", - "WEB_UI_CONFIG_FILE": "%s/%s" % (dirs["base"], dirs["opencloudConfig"]), - "LOCAL_UPLOAD_DIR": "/uploads", - }, - "commands": [ - "cd %s/tests/e2e" % dirs["web"], - ], - } - - steps_after = uploadTracingResult(ctx) + \ - logTracingResults() - - if params["totalParts"]: - for index in range(params["totalParts"]): - run_part = index + 1 - run_e2e = {} - run_e2e.update(step_e2e) - run_e2e["commands"] = [ - "cd %s/tests/e2e" % dirs["web"], - "bash run-e2e.sh %s --run-part %d" % (e2e_args, run_part), - ] - pipelines.append({ - "kind": "pipeline", - "type": "docker", - "name": "e2e-tests-%s-%s" % (name, run_part), - "steps": steps_before + [run_e2e] + steps_after, - "depends_on": getPipelineNames(buildOpenCloudBinaryForTesting(ctx) + buildWebCache(ctx)), - "trigger": e2e_trigger, - "volumes": e2e_volumes, - }) - else: - step_e2e["commands"].append("bash run-e2e.sh %s" % e2e_args) - pipelines.append({ - "kind": "pipeline", - "type": "docker", - "name": "e2e-tests-%s" % name, - "steps": steps_before + [step_e2e] + steps_after, - "depends_on": getPipelineNames(buildOpenCloudBinaryForTesting(ctx) + buildWebCache(ctx)), - "trigger": e2e_trigger, - "volumes": e2e_volumes, - }) - - return pipelines - -def multiServiceE2ePipeline(ctx): - pipelines = [] - - defaults = { - "skip": False, - "suites": [], - "xsuites": [], - "tikaNeeded": False, - } - - e2e_trigger = { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - } - - if ("skip-e2e" in ctx.build.title.lower()): - return pipelines - - # run this pipeline only for cron jobs and full-ci PRs - if (not "full-ci" in ctx.build.title.lower() and ctx.build.event != "cron"): - return pipelines - - extra_server_environment = { - "OC_PASSWORD_POLICY_BANNED_PASSWORDS_LIST": "%s" % dirs["bannedPasswordList"], - "OC_JWT_SECRET": "some-opencloud-jwt-secret", - "OC_SERVICE_ACCOUNT_ID": "service-account-id", - "OC_SERVICE_ACCOUNT_SECRET": "service-account-secret", - "OC_EXCLUDE_RUN_SERVICES": "storage-users", - "OC_GATEWAY_GRPC_ADDR": "0.0.0.0:9142", - "SETTINGS_GRPC_ADDR": "0.0.0.0:9191", - "GATEWAY_STORAGE_USERS_MOUNT_ID": "storage-users-id", - } - - storage_users_environment = { - "OC_CORS_ALLOW_ORIGINS": "%s,https://%s:9201" % (OC_URL, OC_SERVER_NAME), - "STORAGE_USERS_JWT_SECRET": "some-opencloud-jwt-secret", - "STORAGE_USERS_MOUNT_ID": "storage-users-id", - "STORAGE_USERS_SERVICE_ACCOUNT_ID": "service-account-id", - "STORAGE_USERS_SERVICE_ACCOUNT_SECRET": "service-account-secret", - "STORAGE_USERS_GATEWAY_GRPC_ADDR": "%s:9142" % OC_SERVER_NAME, - "STORAGE_USERS_EVENTS_ENDPOINT": "%s:9233" % OC_SERVER_NAME, - "STORAGE_USERS_DATA_GATEWAY_URL": "%s/data" % OC_URL, - "OC_CACHE_STORE": "nats-js-kv", - "OC_CACHE_STORE_NODES": "%s:9233" % OC_SERVER_NAME, - "MICRO_REGISTRY_ADDRESS": "%s:9233" % OC_SERVER_NAME, - } - storage_users1_environment = { - "STORAGE_USERS_GRPC_ADDR": "storageusers1:9157", - "STORAGE_USERS_HTTP_ADDR": "storageusers1:9158", - "STORAGE_USERS_DEBUG_ADDR": "storageusers1:9159", - "STORAGE_USERS_DATA_SERVER_URL": "http://storageusers1:9158/data", - } - for item in storage_users_environment: - storage_users1_environment[item] = storage_users_environment[item] - - storage_users2_environment = { - "STORAGE_USERS_GRPC_ADDR": "storageusers2:9157", - "STORAGE_USERS_HTTP_ADDR": "storageusers2:9158", - "STORAGE_USERS_DEBUG_ADDR": "storageusers2:9159", - "STORAGE_USERS_DATA_SERVER_URL": "http://storageusers2:9158/data", - } - for item in storage_users_environment: - storage_users2_environment[item] = storage_users_environment[item] - - storage_volume = [{ - "name": "storage", - "path": "/root/.opencloud", - }] - storage_users_services = startOpenCloudService("storage-users", "storageusers1", storage_users1_environment, storage_volume) + \ - startOpenCloudService("storage-users", "storageusers2", storage_users2_environment, storage_volume) + \ - opencloudHealthCheck("storage-users", ["storageusers1:9159", "storageusers2:9159"]) - - for _, suite in config["e2eMultiService"].items(): - if "skip" in suite and suite["skip"]: - continue - - params = {} - for item in defaults: - params[item] = suite[item] if item in suite else defaults[item] - - e2e_args = "" - if params["suites"]: - e2e_args = "--suites %s" % ",".join(params["suites"]) - - # suites to skip - if params["xsuites"]: - e2e_args += " --xsuites %s" % ",".join(params["xsuites"]) - - steps = \ - skipIfUnchanged(ctx, "e2e-tests") + \ - restoreBuildArtifactCache(ctx, "opencloud-binary-amd64", "opencloud/bin/opencloud") + \ - restoreWebCache() + \ - restoreWebPnpmCache() + \ - tikaService() + \ - opencloudServer(extra_server_environment = extra_server_environment, tika_enabled = params["tikaNeeded"]) + \ - storage_users_services + \ - [{ - "name": "e2e-tests", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": { - "BASE_URL_OPENCLOUD": OC_DOMAIN, - "HEADLESS": "true", - "RETRY": "1", - }, - "commands": [ - "cd %s/tests/e2e" % dirs["web"], - "bash run-e2e.sh %s" % e2e_args, - ], - }] + \ - uploadTracingResult(ctx) + \ - logTracingResults() - - pipelines.append({ - "kind": "pipeline", - "type": "docker", - "name": "e2e-tests-multi-service", - "steps": steps, - "depends_on": getPipelineNames(buildOpenCloudBinaryForTesting(ctx) + buildWebCache(ctx)), - "trigger": e2e_trigger, - "volumes": [ - { - "name": "storage", - "temp": {}, - }, - pipelineVolumeGo, - ], - }) - return pipelines - -def uploadTracingResult(ctx): - return [{ - "name": "upload-tracing-result", - "image": PLUGINS_S3, - "pull": "if-not-exists", - "settings": { - "bucket": { - "from_secret": "cache_public_s3_bucket", - }, - "endpoint": { - "from_secret": "cache_public_s3_server", - }, - "path_style": True, - "source": "webTestRunner/reports/e2e/playwright/tracing/**/*", - "strip_prefix": "webTestRunner/reports/e2e/playwright/tracing", - "target": "/${DRONE_REPO}/${DRONE_BUILD_NUMBER}/tracing", - }, - "environment": { - "AWS_ACCESS_KEY_ID": { - "from_secret": "cache_public_s3_access_key", - }, - "AWS_SECRET_ACCESS_KEY": { - "from_secret": "cache_public_s3_secret_key", - }, - }, - "when": { - "status": [ - "failure", - ], - "event": [ - "pull_request", - "cron", - ], - }, - }] - -def logTracingResults(): - return [{ - "name": "log-tracing-result", - "image": OC_UBUNTU, - "commands": [ - "cd %s/reports/e2e/playwright/tracing/" % dirs["web"], - 'echo "To see the trace, please open the following link in the console"', - 'for f in *.zip; do echo "npx playwright show-trace https://cache.opencloud.eu/public/${DRONE_REPO}/${DRONE_BUILD_NUMBER}/tracing/$f \n"; done', - ], - "when": { - "status": [ - "failure", - ], - "event": [ - "pull_request", - "cron", - ], - }, - }] - -def dockerReleases(ctx): - pipelines = [] - docker_repos = [] - build_type = "daily" - - # dockerhub repo - # - "opencloud/opencloud-rolling" - repo = ctx.repo.slug + "-rolling" - docker_repos.append(repo) - - # production release repo - if ctx.build.event == "tag": - tag = ctx.build.ref.replace("refs/tags/v", "").lower() - for prod_tag in PRODUCTION_RELEASE_TAGS: - if tag.startswith(prod_tag): - docker_repos.append(ctx.repo.slug) - break - - for repo in docker_repos: - repo_pipelines = [] - if ctx.build.event == "tag": - build_type = "rolling" if "rolling" in repo else "production" - - for arch in config["dockerReleases"]["architectures"]: - repo_pipelines.append(dockerRelease(ctx, arch, repo, build_type)) - - manifest = releaseDockerManifest(ctx, repo, build_type) - manifest["depends_on"] = getPipelineNames(repo_pipelines) - repo_pipelines.append(manifest) - - readme = releaseDockerReadme(ctx, repo, build_type) - readme["depends_on"] = getPipelineNames(repo_pipelines) - repo_pipelines.append(readme) - - pipelines.extend(repo_pipelines) - - return pipelines - -def dockerRelease(ctx, arch, repo, build_type): - build_args = [ - "REVISION=%s" % (ctx.build.commit), - "VERSION=%s" % (ctx.build.ref.replace("refs/tags/", "") if ctx.build.event == "tag" else "master"), - ] - depends_on = getPipelineNames(testOpenCloudAndUploadResults(ctx) + testPipelines(ctx)) - - if ctx.build.event == "tag": - depends_on = [] - - return { - "kind": "pipeline", - "type": "docker", - "name": "docker-%s-%s" % (arch, build_type), - "platform": { - "os": "linux", - "arch": arch, - }, - "steps": skipIfUnchanged(ctx, "build-docker") + - makeNodeGenerate("") + - makeGoGenerate("") + [ - { - "name": "build", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C opencloud release-linux-docker-%s ENABLE_VIPS=true" % (arch), - ], - }, - { - "name": "dryrun", - "image": PLUGINS_DOCKER, - "settings": { - "dry_run": True, - "context": "opencloud", - "tags": "linux-%s" % (arch), - "dockerfile": "opencloud/docker/Dockerfile.linux.%s" % (arch), - "repo": repo, - "build_args": build_args, - }, - "when": { - "ref": { - "include": [ - "refs/pull/**", - ], - }, - }, - }, - { - "name": "docker", - "image": PLUGINS_DOCKER, - "settings": { - "username": { - "from_secret": "docker_username", - }, - "password": { - "from_secret": "docker_password", - }, - "auto_tag": True, - "context": "opencloud", - "auto_tag_suffix": "linux-%s" % (arch), - "dockerfile": "opencloud/docker/Dockerfile.linux.%s" % (arch), - "repo": repo, - "build_args": build_args, - }, - "when": { - "ref": { - "exclude": [ - "refs/pull/**", - ], - }, - }, - }, - ], - "depends_on": depends_on, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - } - -def binaryReleases(ctx): - pipelines = [] - targets = [] - build_type = "daily" - - target = "/opencloud/%s/daily" % (ctx.repo.name.replace("opencloud-", "")) - depends_on = getPipelineNames(testOpenCloudAndUploadResults(ctx) + testPipelines(ctx)) - - if ctx.build.event == "tag": - depends_on = [] - - buildref = ctx.build.ref.replace("refs/tags/v", "").lower() - target_path = "/opencloud/%s" % ctx.repo.name.replace("opencloud-", "") - - if buildref.find("-") != -1: # "x.x.x-alpha", "x.x.x-beta", "x.x.x-rc" - folder = "testing" - target = "%s/%s/%s" % (target_path, folder, buildref) - targets.append(target) - build_type = "testing" - else: - # uploads binary to eg. https://download.opencloud.eu/opencloud/rolling/1.0.0/ - folder = "rolling" - target = "%s/%s/%s" % (target_path, folder, buildref) - targets.append(target) - - for prod_tag in PRODUCTION_RELEASE_TAGS: - if buildref.startswith(prod_tag): - # uploads binary to eg. https://download.opencloud.eu/opencloud/stable/2.0.0/ - folder = "stable" - target = "%s/%s/%s" % (target_path, folder, buildref) - targets.append(target) - break - - else: - targets.append(target) - - for target in targets: - if "rolling" in target: - build_type = "rolling" - elif "stable" in target: - build_type = "production" - elif "testing" in target: - build_type = "testing" - - for os in config["binaryReleases"]["os"]: - pipelines.append(binaryRelease(ctx, os, build_type, target, depends_on)) - - return pipelines - -def binaryRelease(ctx, arch, build_type, target, depends_on = []): - settings = { - "endpoint": { - "from_secret": "upload_s3_endpoint", - }, - "access_key": { - "from_secret": "upload_s3_access_key", - }, - "secret_key": { - "from_secret": "upload_s3_secret_key", - }, - "bucket": { - "from_secret": "upload_s3_bucket", - }, - "path_style": True, - "strip_prefix": "opencloud/dist/release/", - "source": "opencloud/dist/release/*", - "target": target, - } - - return { - "kind": "pipeline", - "type": "docker", - "name": "binaries-%s-%s" % (arch, build_type), - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": skipIfUnchanged(ctx, "build-binary") + - makeNodeGenerate("") + - makeGoGenerate("") + [ - { - "name": "build", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C opencloud release-%s" % (arch), - ], - }, - { - "name": "finish", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C opencloud release-finish", - ], - "when": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - ], - }, - }, - { - "name": "upload", - "image": PLUGINS_S3, - "settings": settings, - "when": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - ], - }, - }, - { - "name": "changelog", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make changelog CHANGELOG_VERSION=%s" % ctx.build.ref.replace("refs/tags/v", ""), - ], - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - { - "name": "release", - "image": PLUGINS_GITHUB_RELEASE, - "settings": { - "api_key": { - "from_secret": "github_token", - }, - "files": [ - "opencloud/dist/release/*", - ], - "title": ctx.build.ref.replace("refs/tags/v", ""), - "note": "opencloud/dist/CHANGELOG.md", - "overwrite": True, - "prerelease": len(ctx.build.ref.split("-")) > 1, - }, - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - ], - "depends_on": depends_on, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - } - -def licenseCheck(ctx): - # uploads third-party-licenses to https://download.opencloud.eu/opencloud/daily/ - target = "/opencloud/%s/daily" % (ctx.repo.name.replace("opencloud-", "")) - if ctx.build.event == "tag": - # uploads third-party-licenses to eg. https://download.opencloud.eu/opencloud/1.0.0-beta9/ - folder = "stable" - buildref = ctx.build.ref.replace("refs/tags/v", "") - buildref = buildref.lower() - if buildref.find("-") != -1: # "x.x.x-alpha", "x.x.x-beta", "x.x.x-rc" - folder = "testing" - target = "/opencloud/%s/%s/%s" % (ctx.repo.name.replace("opencloud-", ""), folder, buildref) - - settings = { - "endpoint": { - "from_secret": "upload_s3_endpoint", - }, - "access_key": { - "from_secret": "upload_s3_access_key", - }, - "secret_key": { - "from_secret": "upload_s3_secret_key", - }, - "bucket": { - "from_secret": "upload_s3_bucket", - }, - "path_style": True, - "source": "third-party-licenses.tar.gz", - "target": target, - } - - return [{ - "kind": "pipeline", - "type": "docker", - "name": "check-licenses", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "node-check-licenses", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "make ci-node-check-licenses", - ], - }, - { - "name": "node-save-licenses", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "make ci-node-save-licenses", - ], - }, - { - "name": "go-check-licenses", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make ci-go-check-licenses", - ], - "volumes": [stepVolumeGo], - }, - { - "name": "go-save-licenses", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make ci-go-save-licenses", - ], - "volumes": [stepVolumeGo], - }, - { - "name": "tarball", - "image": OC_CI_ALPINE, - "commands": [ - "cd third-party-licenses && tar -czf ../third-party-licenses.tar.gz *", - ], - }, - { - "name": "upload", - "image": PLUGINS_S3, - "settings": settings, - "when": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - ], - }, - }, - { - "name": "changelog", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make changelog CHANGELOG_VERSION=%s" % ctx.build.ref.replace("refs/tags/v", "").split("-")[0], - ], - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - { - "name": "release", - "image": PLUGINS_GITHUB_RELEASE, - "settings": { - "api_key": { - "from_secret": "github_token", - }, - "files": [ - "third-party-licenses.tar.gz", - ], - "title": ctx.build.ref.replace("refs/tags/v", ""), - "note": "opencloud/dist/CHANGELOG.md", - "overwrite": True, - "prerelease": len(ctx.build.ref.split("-")) > 1, - }, - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - "refs/pull/**", - ], - }, - "volumes": [pipelineVolumeGo], - }] - -def releaseDockerManifest(ctx, repo, build_type): - spec = "manifest.tmpl" - spec_latest = "manifest-latest.tmpl" - if "rolling" not in repo: - spec = "manifest.production.tmpl" - spec_latest = "manifest.production-latest.tmpl" - - steps = [ - { - "name": "execute", - "image": PLUGINS_MANIFEST, - "settings": { - "username": { - "from_secret": "docker_username", - }, - "password": { - "from_secret": "docker_password", - }, - "spec": "opencloud/docker/%s" % spec, - "auto_tag": True if ctx.build.event == "tag" else False, - "ignore_missing": True, - }, - }, - ] - if len(ctx.build.ref.split("-")) == 1: - steps.append( - { - "name": "execute-latest", - "image": PLUGINS_MANIFEST, - "settings": { - "username": { - "from_secret": "docker_username", - }, - "password": { - "from_secret": "docker_password", - }, - "spec": "opencloud/docker/%s" % spec_latest, - "auto_tag": True, - "ignore_missing": True, - }, - "when": { - "ref": [ - "refs/tags/v*", - ], - }, - }, - ) - - return { - "kind": "pipeline", - "type": "docker", - "name": "manifest-%s" % build_type, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": steps, - "trigger": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - ], - }, - } - -def changelog(): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "changelog", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "generate", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C opencloud changelog", - ], - }, - { - "name": "diff", - "image": OC_CI_ALPINE, - "commands": [ - "git diff", - ], - }, - { - "name": "output", - "image": OC_CI_ALPINE, - "commands": [ - "cat CHANGELOG.md", - ], - }, - { - "name": "publish", - "image": PLUGINS_GIT_ACTION, - "settings": { - "actions": [ - "commit", - "push", - ], - "message": "Automated changelog update [skip ci]", - "branch": "master", - "author_email": "devops@opencloud.eu", - "author_name": "opencloud-bot", - "netrc_machine": "github.com", - "netrc_username": { - "from_secret": "github_username", - }, - "netrc_password": { - "from_secret": "github_token", - }, - }, - "when": { - "ref": { - "exclude": [ - "refs/pull/**", - ], - }, - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - }] - -def releaseDockerReadme(ctx, repo, build_type): - return { - "kind": "pipeline", - "type": "docker", - "name": "readme-%s" % build_type, - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "execute", - "image": CHKO_DOCKER_PUSHRM, - "environment": { - "DOCKER_USER": { - "from_secret": "docker_username", - }, - "DOCKER_PASS": { - "from_secret": "docker_password", - }, - "PUSHRM_TARGET": repo, - "PUSHRM_SHORT": "Docker images for %s" % (ctx.repo.name), - "PUSHRM_FILE": "README.md", - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - ], - }, - } - -def docs(): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "docs", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "docs-generate", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": ["make docs-generate"], - }, - { - "name": "prepare", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C docs docs-copy", - ], - }, - { - "name": "test", - "image": OC_CI_GOLANG, - "environment": DRONE_HTTP_PROXY_ENV, - "commands": [ - "make -C docs test", - ], - }, - { - "name": "publish", - "image": PLUGINS_GH_PAGES, - "settings": { - "username": { - "from_secret": "github_username", - }, - "password": { - "from_secret": "github_token", - }, - "pages_directory": "docs/hugo/content/", - "copy_contents": "true", - "target_branch": "docs", - "delete": "true", - }, - "when": { - "ref": { - "exclude": [ - "refs/pull/**", - ], - }, - }, - }, - { - "name": "list and remove temporary files", - "image": OC_CI_ALPINE, - "commands": [ - "tree docs/hugo/public", - "rm -rf docs/hugo", - ], - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - }] - -def makeNodeGenerate(module): - if module == "": - make = "make" - else: - make = "make -C %s" % (module) - return [ - { - "name": "generate nodejs", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "environment": { - "CHROMEDRIVER_SKIP_DOWNLOAD": "true", # install fails on arm and chromedriver is a test only dependency - }, - "commands": [ - "pnpm config set store-dir ./.pnpm-store", - "retry -t 3 '%s ci-node-generate'" % (make), - ], - "volumes": [stepVolumeGo], - }, - ] - -def makeGoGenerate(module): - if module == "": - make = "make" - else: - make = "make -C %s" % (module) - return [ - { - "name": "generate go", - "image": OC_CI_GOLANG, - "commands": [ - "retry -t 3 '%s ci-go-generate'" % (make), - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - ] - -def notify(ctx): - status = ["failure"] - channel = config["rocketchat"]["channel"] - if ctx.build.event == "cron": - status.append("success") - channel = config["rocketchat"]["channel_cron"] - - return { - "kind": "pipeline", - "type": "docker", - "name": "chat-notifications", - "clone": { - "disable": True, - }, - "steps": [ - { - "name": "notify-rocketchat", - "image": PLUGINS_SLACK, - "settings": { - "webhook": { - "from_secret": config["rocketchat"]["from_secret"], - }, - "channel": channel, - }, - }, - ], - "depends_on": [], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/heads/release*", - "refs/tags/**", - ], - "status": status, - }, - } - -def opencloudServer(storage = "opencloud", accounts_hash_difficulty = 4, volumes = [], depends_on = [], deploy_type = "", extra_server_environment = {}, with_wrapper = False, tika_enabled = False): - user = "0:0" - container_name = OC_SERVER_NAME - environment = { - "OC_URL": OC_URL, - "OC_CONFIG_DIR": "/root/.opencloud/config", # needed for checking config later - "STORAGE_USERS_DRIVER": "%s" % (storage), - "PROXY_ENABLE_BASIC_AUTH": True, - "WEB_UI_CONFIG_FILE": "%s/%s" % (dirs["base"], dirs["opencloudConfig"]), - "OC_LOG_LEVEL": "error", - "IDM_CREATE_DEMO_USERS": True, # needed for litmus and cs3api-validator tests - "IDM_ADMIN_PASSWORD": "admin", # override the random admin password from `opencloud init` - "FRONTEND_SEARCH_MIN_LENGTH": "2", - "OC_ASYNC_UPLOADS": True, - "OC_EVENTS_ENABLE_TLS": False, - "NATS_NATS_HOST": "0.0.0.0", - "NATS_NATS_PORT": 9233, - "OC_JWT_SECRET": "some-opencloud-jwt-secret", - "EVENTHISTORY_STORE": "memory", - "OC_TRANSLATION_PATH": "%s/tests/config/translations" % dirs["base"], - # debug addresses required for running services health tests - "ACTIVITYLOG_DEBUG_ADDR": "0.0.0.0:9197", - "APP_PROVIDER_DEBUG_ADDR": "0.0.0.0:9165", - "APP_REGISTRY_DEBUG_ADDR": "0.0.0.0:9243", - "AUTH_BASIC_DEBUG_ADDR": "0.0.0.0:9147", - "AUTH_MACHINE_DEBUG_ADDR": "0.0.0.0:9167", - "AUTH_SERVICE_DEBUG_ADDR": "0.0.0.0:9198", - "CLIENTLOG_DEBUG_ADDR": "0.0.0.0:9260", - "EVENTHISTORY_DEBUG_ADDR": "0.0.0.0:9270", - "FRONTEND_DEBUG_ADDR": "0.0.0.0:9141", - "GATEWAY_DEBUG_ADDR": "0.0.0.0:9143", - "GRAPH_DEBUG_ADDR": "0.0.0.0:9124", - "GROUPS_DEBUG_ADDR": "0.0.0.0:9161", - "IDM_DEBUG_ADDR": "0.0.0.0:9239", - "IDP_DEBUG_ADDR": "0.0.0.0:9134", - "INVITATIONS_DEBUG_ADDR": "0.0.0.0:9269", - "NATS_DEBUG_ADDR": "0.0.0.0:9234", - "OCDAV_DEBUG_ADDR": "0.0.0.0:9163", - "OCM_DEBUG_ADDR": "0.0.0.0:9281", - "OCS_DEBUG_ADDR": "0.0.0.0:9114", - "POSTPROCESSING_DEBUG_ADDR": "0.0.0.0:9255", - "PROXY_DEBUG_ADDR": "0.0.0.0:9205", - "SEARCH_DEBUG_ADDR": "0.0.0.0:9224", - "SETTINGS_DEBUG_ADDR": "0.0.0.0:9194", - "SHARING_DEBUG_ADDR": "0.0.0.0:9151", - "SSE_DEBUG_ADDR": "0.0.0.0:9139", - "STORAGE_PUBLICLINK_DEBUG_ADDR": "0.0.0.0:9179", - "STORAGE_SHARES_DEBUG_ADDR": "0.0.0.0:9156", - "STORAGE_SYSTEM_DEBUG_ADDR": "0.0.0.0:9217", - "STORAGE_USERS_DEBUG_ADDR": "0.0.0.0:9159", - "THUMBNAILS_DEBUG_ADDR": "0.0.0.0:9189", - "USERLOG_DEBUG_ADDR": "0.0.0.0:9214", - "USERS_DEBUG_ADDR": "0.0.0.0:9145", - "WEB_DEBUG_ADDR": "0.0.0.0:9104", - "WEBDAV_DEBUG_ADDR": "0.0.0.0:9119", - "WEBFINGER_DEBUG_ADDR": "0.0.0.0:9279", - } - - if deploy_type == "": - environment["FRONTEND_OCS_ENABLE_DENIALS"] = True - - # fonts map for txt thumbnails (including unicode support) - environment["THUMBNAILS_TXT_FONTMAP_FILE"] = "%s/tests/config/drone/fontsMap.json" % (dirs["base"]) - - if deploy_type == "cs3api_validator": - environment["GATEWAY_GRPC_ADDR"] = "0.0.0.0:9142" # make gateway available to cs3api-validator - environment["OC_SHARING_PUBLIC_SHARE_MUST_HAVE_PASSWORD"] = False - - if deploy_type == "wopi_validator": - environment["GATEWAY_GRPC_ADDR"] = "0.0.0.0:9142" # make gateway available to wopi server - environment["APP_PROVIDER_EXTERNAL_ADDR"] = "com.opencloud.api.app-provider" - environment["APP_PROVIDER_DRIVER"] = "wopi" - environment["APP_PROVIDER_WOPI_APP_NAME"] = "FakeOffice" - environment["APP_PROVIDER_WOPI_APP_URL"] = "http://fakeoffice:8080" - environment["APP_PROVIDER_WOPI_INSECURE"] = "true" - environment["APP_PROVIDER_WOPI_WOPI_SERVER_EXTERNAL_URL"] = "http://wopi-fakeoffice:9300" - environment["APP_PROVIDER_WOPI_FOLDER_URL_BASE_URL"] = OC_URL - - if deploy_type == "federation": - environment["OC_URL"] = OC_FED_URL - environment["PROXY_HTTP_ADDR"] = OC_FED_DOMAIN - container_name = FED_OC_SERVER_NAME - - if tika_enabled: - environment["FRONTEND_FULL_TEXT_SEARCH_ENABLED"] = True - environment["SEARCH_EXTRACTOR_TYPE"] = "tika" - environment["SEARCH_EXTRACTOR_TIKA_TIKA_URL"] = "http://tika:9998" - environment["SEARCH_EXTRACTOR_CS3SOURCE_INSECURE"] = True - - # Pass in "default" accounts_hash_difficulty to not set this environment variable. - # That will allow OpenCloud to use whatever its built-in default is. - # Otherwise pass in a value from 4 to about 11 or 12 (default 4, for making regular tests fast) - # The high values cause lots of CPU to be used when hashing passwords, and really slow down the tests. - if (accounts_hash_difficulty != "default"): - environment["ACCOUNTS_HASH_DIFFICULTY"] = accounts_hash_difficulty - - for item in extra_server_environment: - environment[item] = extra_server_environment[item] - - opencloud_bin = "opencloud/bin/opencloud" - - wrapper_commands = [ - "make -C %s build" % dirs["ocWrapper"], - "%s/bin/ocwrapper serve --bin %s --url %s --admin-username admin --admin-password admin" % (dirs["ocWrapper"], opencloud_bin, environment["OC_URL"]), - ] - - wait_for_opencloud = { - "name": "wait-for-%s" % (container_name), - "image": OC_CI_ALPINE, - "commands": [ - # wait for opencloud-server to be ready (5 minutes) - "timeout 300 bash -c 'while [ $(curl -sk -uadmin:admin " + - "%s/graph/v1.0/users/admin " % environment["OC_URL"] + - "-w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done'", - ], - "depends_on": depends_on, - } - - return [ - { - "name": container_name, - "image": OC_CI_GOLANG, - "detach": True, - "environment": environment, - "user": user, - "commands": [ - "%s init --insecure true" % opencloud_bin, - "cat $OC_CONFIG_DIR/opencloud.yaml", - "cp tests/config/drone/app-registry.yaml /root/.opencloud/config/app-registry.yaml", - ] + (wrapper_commands), - "volumes": volumes, - "depends_on": depends_on, - }, - wait_for_opencloud, - ] - -def startOpenCloudService(service = None, name = None, environment = {}, volumes = []): - """ - Starts an OpenCloud service in a detached container. - - Args: - service (str): The name of the service to start. - name (str): The name of the container. - environment (dict): The environment variables to set in the container. - volumes (list): The volumes to mount in the container. - - Returns: - list: A list of pipeline steps to start the service. - """ - - if not service: - return [] - if not name: - name = service - - return [ - { - "name": name, - "image": OC_CI_GOLANG, - "detach": True, - "environment": environment, - "commands": [ - "opencloud/bin/opencloud %s server" % service, - ], - "volumes": volumes, - }, - ] - -def redis(): - return [ - { - "name": "redis", - "image": REDIS, - }, - ] - -def redisForOCStorage(storage = "opencloud"): - if storage == "owncloud": - return redis() - else: - return - -def build(): - return [ - { - "name": "build", - "image": OC_CI_GOLANG, - "commands": [ - "retry -t 3 'make -C opencloud build'", - ], - "environment": DRONE_HTTP_PROXY_ENV, - "volumes": [stepVolumeGo], - }, - ] - -def skipIfUnchanged(ctx, type): - if ("full-ci" in ctx.build.title.lower() or ctx.build.event == "tag" or ctx.build.event == "cron"): - return [] - - base = [ - "^.github/.*", - "^.vscode/.*", - "^changelog/.*", - "^docs/.*", - "^deployments/.*", - "CHANGELOG.md", - "CONTRIBUTING.md", - "LICENSE", - "README.md", - ] - unit = [ - ".*_test.go$", - ] - acceptance = [ - "^tests/acceptance/.*", - ] - - skip = [] - if type == "acceptance-tests" or type == "e2e-tests" or type == "lint": - skip = base + unit - elif type == "unit-tests": - skip = base + acceptance - elif type == "build-binary" or type == "build-docker" or type == "litmus": - skip = base + unit + acceptance - elif type == "cache": - skip = base - else: - return [] - - return [{ - "name": "skip-if-unchanged", - "image": OC_CI_DRONE_SKIP_PIPELINE, - "settings": { - "ALLOW_SKIP_CHANGED": skip, - }, - "when": { - "event": [ - "pull_request", - ], - }, - }] - -def example_deploys(ctx): - on_merge_deploy = [ - "opencloud_full/master.yml", - "opencloud_full/onlyoffice-master.yml", - ] - nightly_deploy = [ - "opencloud_ldap/rolling.yml", - "opencloud_keycloak/rolling.yml", - "opencloud_full/production.yml", - "opencloud_full/rolling.yml", - "opencloud_full/onlyoffice-rolling.yml", - "opencloud_full/s3-rolling.yml", - ] - - # if on master branch: - configs = on_merge_deploy - rebuild = "false" - - if ctx.build.event == "tag": - configs = nightly_deploy - rebuild = "false" - - if ctx.build.event == "cron": - configs = on_merge_deploy + nightly_deploy - rebuild = "true" - - deploys = [] - for config in configs: - deploys.append(deploy(ctx, config, rebuild)) - - return deploys - -def deploy(ctx, config, rebuild): - return { - "kind": "pipeline", - "type": "docker", - "name": "deploy_%s" % (config), - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "clone continuous deployment playbook", - "image": ALPINE_GIT, - "commands": [ - "cd deployments/continuous-deployment-config", - "git clone https://github.com/opencloud-devops/continuous-deployment.git", - ], - }, - { - "name": "deploy", - "image": OC_CI_DRONE_ANSIBLE, - "failure": "ignore", - "environment": { - "CONTINUOUS_DEPLOY_SERVERS_CONFIG": "../%s" % (config), - "REBUILD": "%s" % (rebuild), - "HCLOUD_API_TOKEN": { - "from_secret": "hcloud_api_token", - }, - "CLOUDFLARE_API_TOKEN": { - "from_secret": "cloudflare_api_token", - }, - }, - "settings": { - "playbook": "deployments/continuous-deployment-config/continuous-deployment/playbook-all.yml", - "galaxy": "deployments/continuous-deployment-config/continuous-deployment/requirements.yml", - "requirements": "deployments/continuous-deployment-config/continuous-deployment/py-requirements.txt", - "inventory": "localhost", - "private_key": { - "from_secret": "ssh_private_key", - }, - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/tags/v*", - ], - }, - } - -def checkStarlark(): - return [{ - "kind": "pipeline", - "type": "docker", - "name": "check-starlark", - "steps": [ - { - "name": "format-check-starlark", - "image": OC_CI_BAZEL_BUILDIFIER, - "commands": [ - "buildifier --mode=check .drone.star", - ], - }, - { - "name": "show-diff", - "image": OC_CI_BAZEL_BUILDIFIER, - "commands": [ - "buildifier --mode=fix .drone.star", - "git diff", - ], - "when": { - "status": [ - "failure", - ], - }, - }, - ], - "depends_on": [], - "trigger": { - "ref": [ - "refs/pull/**", - ], - }, - }] - -def genericCache(name, action, mounts, cache_path): - rebuild = "false" - restore = "false" - if action == "rebuild": - rebuild = "true" - action = "rebuild" - else: - restore = "true" - action = "restore" - - step = { - "name": "%s_%s" % (action, name), - "image": PLUGINS_S3_CACHE, - "settings": { - "endpoint": { - "from_secret": "cache_s3_server", - }, - "rebuild": rebuild, - "restore": restore, - "mount": mounts, - "access_key": { - "from_secret": "cache_s3_access_key", - }, - "secret_key": { - "from_secret": "cache_s3_secret_key", - }, - "filename": "%s.tar" % (name), - "path": cache_path, - "fallback_path": cache_path, - }, - } - return step - -def genericCachePurge(flush_path): - return { - "kind": "pipeline", - "type": "docker", - "name": "purge_build_artifact_cache", - "platform": { - "os": "linux", - "arch": "amd64", - }, - "steps": [ - { - "name": "purge-cache", - "image": PLUGINS_S3_CACHE, - "settings": { - "access_key": { - "from_secret": "cache_s3_access_key", - }, - "secret_key": { - "from_secret": "cache_s3_secret_key", - }, - "endpoint": { - "from_secret": "cache_s3_server", - }, - "flush": True, - "flush_age": 1, - "flush_path": flush_path, - }, - }, - ], - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - "status": [ - "success", - "failure", - ], - }, - } - -def genericBuildArtifactCache(ctx, name, action, path): - if action == "rebuild" or action == "restore": - cache_path = "%s/%s/%s" % ("cache", ctx.repo.slug, ctx.build.commit + "-${DRONE_BUILD_NUMBER}") - name = "%s_build_artifact_cache" % (name) - return genericCache(name, action, [path], cache_path) - - if action == "purge": - flush_path = "%s/%s" % ("cache", ctx.repo.slug) - return genericCachePurge(flush_path) - return [] - -def restoreBuildArtifactCache(ctx, name, path): - return [genericBuildArtifactCache(ctx, name, "restore", path)] - -def rebuildBuildArtifactCache(ctx, name, path): - return [genericBuildArtifactCache(ctx, name, "rebuild", path)] - -def purgeBuildArtifactCache(ctx): - return genericBuildArtifactCache(ctx, "", "purge", []) - -def pipelineSanityChecks(ctx, pipelines): - """pipelineSanityChecks helps the CI developers to find errors before running it - - These sanity checks are only executed on when converting starlark to yaml. - Error outputs are only visible when the conversion is done with the drone cli. - - Args: - ctx: drone passes a context with information which the pipeline can be adapted to - pipelines: pipelines to be checked, normally you should run this on the return value of main() - - Returns: - none - """ - - # check if name length of pipeline and steps are exceeded. - max_name_length = 50 - for pipeline in pipelines: - pipeline_name = pipeline["name"] - if len(pipeline_name) > max_name_length: - print("Error: pipeline name %s is longer than 50 characters" % (pipeline_name)) - - for step in pipeline["steps"]: - step_name = step["name"] - if len(step_name) > max_name_length: - print("Error: step name %s in pipeline %s is longer than 50 characters" % (step_name, pipeline_name)) - - # check for non existing depends_on - possible_depends = [] - for pipeline in pipelines: - possible_depends.append(pipeline["name"]) - - for pipeline in pipelines: - if "depends_on" in pipeline.keys(): - for depends in pipeline["depends_on"]: - if not depends in possible_depends: - print("Error: depends_on %s for pipeline %s is not defined" % (depends, pipeline["name"])) - - # check for non declared volumes - for pipeline in pipelines: - pipeline_volumes = [] - if "volumes" in pipeline.keys(): - for volume in pipeline["volumes"]: - pipeline_volumes.append(volume["name"]) - - for step in pipeline["steps"]: - if "volumes" in step.keys(): - for volume in step["volumes"]: - if not volume["name"] in pipeline_volumes: - print("Warning: volume %s for step %s is not defined in pipeline %s" % (volume["name"], step["name"], pipeline["name"])) - - # list used docker images - print("") - print("List of used docker images:") - - images = {} - - for pipeline in pipelines: - for step in pipeline["steps"]: - image = step["image"] - if image in images.keys(): - images[image] = images[image] + 1 - else: - images[image] = 1 - - for image in images.keys(): - print(" %sx\t%s" % (images[image], image)) - -def litmus(ctx, storage): - pipelines = [] - - if (config["litmus"] == False): - return pipelines - - environment = { - "LITMUS_PASSWORD": "admin", - "LITMUS_USERNAME": "admin", - "TESTS": "basic copymove props http", - } - - litmusCommand = "/usr/local/bin/litmus-wrapper" - - result = { - "kind": "pipeline", - "type": "docker", - "name": "litmus", - "workspace": { - "base": "/drone", - "path": "src", - }, - "steps": skipIfUnchanged(ctx, "litmus") + - restoreBuildArtifactCache(ctx, "opencloud-binary-amd64", "opencloud/bin") + - opencloudServer(storage) + - setupForLitmus() + - [ - { - "name": "old-endpoint", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - 'export LITMUS_URL="%s/remote.php/webdav"' % OC_URL, - litmusCommand, - ], - }, - { - "name": "new-endpoint", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - 'export LITMUS_URL="%s/remote.php/dav/files/admin"' % OC_URL, - litmusCommand, - ], - }, - { - "name": "new-shared", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - 'export LITMUS_URL="%s/remote.php/dav/files/admin/Shares/new_folder/"' % OC_URL, - litmusCommand, - ], - }, - { - "name": "old-shared", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - 'export LITMUS_URL="%s/remote.php/webdav/Shares/new_folder/"' % OC_URL, - litmusCommand, - ], - }, - # { - # "name": "public-share", - # "image": OC_LITMUS, - # "environment": { - # "LITMUS_PASSWORD": "admin", - # "LITMUS_USERNAME": "admin", - # "TESTS": "basic copymove http", - # }, - # "commands": [ - # "source .env", - # "export LITMUS_URL='%s/remote.php/dav/public-files/'$PUBLIC_TOKEN" % OC_URL, - # litmusCommand, - # ], - # }, - { - "name": "spaces-endpoint", - "image": OC_LITMUS, - "environment": environment, - "commands": [ - "source .env", - "export LITMUS_URL='%s/remote.php/dav/spaces/'$SPACE_ID" % OC_URL, - litmusCommand, - ], - }, - ], - "services": redisForOCStorage(storage), - "depends_on": getPipelineNames(buildOpenCloudBinaryForTesting(ctx)), - "trigger": { - "ref": [ - "refs/heads/master", - "refs/pull/**", - ], - }, - } - pipelines.append(result) - - return pipelines - -def setupForLitmus(): - return [{ - "name": "setup-for-litmus", - "image": OC_UBUNTU, - "environment": { - "TEST_SERVER_URL": OC_URL, - }, - "commands": [ - "bash ./tests/config/drone/setup-for-litmus.sh", - "cat .env", - ], - }] - -def getDroneEnvAndCheckScript(ctx): - opencloud_git_base_url = "https://raw.githubusercontent.com/opencloud-eu/opencloud" - path_to_drone_env = "%s/%s/.drone.env" % (opencloud_git_base_url, ctx.build.commit) - path_to_check_script = "%s/%s/tests/config/drone/check_web_cache.sh" % (opencloud_git_base_url, ctx.build.commit) - return { - "name": "get-drone-env-and-check-script", - "image": OC_UBUNTU, - "commands": [ - "curl -s -o .drone.env %s" % path_to_drone_env, - "curl -s -o check_web_cache.sh %s" % path_to_check_script, - ], - } - -def checkForWebCache(name): - return { - "name": "check-for-%s-cache" % name, - "image": OC_UBUNTU, - "environment": { - "CACHE_ENDPOINT": { - "from_secret": "cache_s3_server", - }, - "CACHE_BUCKET": { - "from_secret": "cache_s3_bucket", - }, - }, - "commands": [ - "bash -x check_web_cache.sh %s" % name, - ], - } - -def cloneWeb(): - return { - "name": "clone-web", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - ". ./.drone.env", - "rm -rf %s" % dirs["web"], - "git clone -b $WEB_BRANCH --single-branch --no-tags https://github.com/opencloud-eu/web.git %s" % dirs["web"], - "cd %s && git checkout $WEB_COMMITID" % dirs["web"], - ], - } - -def generateWebPnpmCache(ctx): - return [ - getDroneEnvAndCheckScript(ctx), - checkForWebCache("web-pnpm"), - cloneWeb(), - { - "name": "install-pnpm", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "cd %s" % dirs["web"], - 'npm install --silent --global --force "$(jq -r ".packageManager" < package.json)"', - "pnpm config set store-dir ./.pnpm-store", - "retry -t 3 'pnpm install'", - ], - }, - { - "name": "zip-pnpm", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - # zip the pnpm deps before caching - "if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]), - "cd %s" % dirs["web"], - "tar -czvf %s .pnpm-store" % dirs["webPnpmZip"], - ], - }, - { - "name": "cache-pnpm", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - # cache using the minio/mc client to the public bucket (long term bucket) - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a %s s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID" % dirs["webPnpmZip"], - ], - }, - ] - -def generateWebCache(ctx): - return [ - getDroneEnvAndCheckScript(ctx), - checkForWebCache("web"), - cloneWeb(), - { - "name": "zip-web", - "image": OC_UBUNTU, - "commands": [ - "if [ ! -d '%s' ]; then mkdir -p %s; fi" % (dirs["zip"], dirs["zip"]), - "tar -czvf %s webTestRunner" % dirs["webZip"], - ], - }, - { - "name": "cache-web", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - # cache using the minio/mc client to the 'opencloud' bucket (long term bucket) - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a %s s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID" % dirs["webZip"], - ], - }, - ] - -def restoreWebCache(): - return [{ - "name": "restore-web-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - "rm -rf %s" % dirs["web"], - "mkdir -p %s" % dirs["web"], - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID/web.tar.gz %s" % dirs["zip"], - ], - }, { - "name": "unzip-web-cache", - "image": OC_UBUNTU, - "commands": [ - "tar -xvf %s -C ." % dirs["webZip"], - ], - }] - -def restoreWebPnpmCache(): - return [{ - "name": "restore-web-pnpm-cache", - "image": MINIO_MC, - "environment": MINIO_MC_ENV, - "commands": [ - "source ./.drone.env", - "mc alias set s3 $MC_HOST $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY", - "mc cp -r -a s3/$CACHE_BUCKET/opencloud/web-test-runner/$WEB_COMMITID/pnpm-store.tar.gz %s" % dirs["zip"], - ], - }, { - # we need to install again because the node_modules are not cached - "name": "unzip-and-install-pnpm", - "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, - "commands": [ - "cd %s" % dirs["web"], - "rm -rf .pnpm-store", - "tar -xvf %s" % dirs["webPnpmZip"], - 'npm install --silent --global --force "$(jq -r ".packageManager" < package.json)"', - "pnpm config set store-dir ./.pnpm-store", - "retry -t 3 'pnpm install'", - ], - }] - -def emailService(): - return [{ - "name": "email", - "image": INBUCKET_INBUCKET, - }] - -def waitForEmailService(): - return [{ - "name": "wait-for-email", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it email:9000 -t 600", - ], - }] - -def clamavService(): - return [{ - "name": "clamav", - "image": OC_CI_CLAMAVD, - }] - -def waitForClamavService(): - return [{ - "name": "wait-for-clamav", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it clamav:3310 -t 600", - ], - }] - -def fakeOffice(): - return [ - { - "name": "fakeoffice", - "image": OC_CI_ALPINE, - "detach": True, - "environment": {}, - "commands": [ - "sh %s/tests/config/drone/serve-hosting-discovery.sh" % (dirs["base"]), - ], - }, - ] - -def wopiCollaborationService(name): - service_name = "wopi-%s" % name - - environment = { - "MICRO_REGISTRY": "nats-js-kv", - "MICRO_REGISTRY_ADDRESS": "%s:9233" % OC_SERVER_NAME, - "COLLABORATION_LOG_LEVEL": "debug", - "COLLABORATION_GRPC_ADDR": "0.0.0.0:9301", - "COLLABORATION_HTTP_ADDR": "0.0.0.0:9300", - "COLLABORATION_DEBUG_ADDR": "0.0.0.0:9304", - "COLLABORATION_APP_PROOF_DISABLE": "true", - "COLLABORATION_APP_INSECURE": "true", - "COLLABORATION_CS3API_DATAGATEWAY_INSECURE": "true", - "OC_JWT_SECRET": "some-opencloud-jwt-secret", - "COLLABORATION_WOPI_SECRET": "some-wopi-secret", - } - - if name == "collabora": - environment["COLLABORATION_APP_NAME"] = "Collabora" - environment["COLLABORATION_APP_PRODUCT"] = "Collabora" - environment["COLLABORATION_APP_ADDR"] = "https://collabora:9980" - environment["COLLABORATION_APP_ICON"] = "https://collabora:9980/favicon.ico" - elif name == "onlyoffice": - environment["COLLABORATION_APP_NAME"] = "OnlyOffice" - environment["COLLABORATION_APP_PRODUCT"] = "OnlyOffice" - environment["COLLABORATION_APP_ADDR"] = "https://onlyoffice" - environment["COLLABORATION_APP_ICON"] = "https://onlyoffice/web-apps/apps/documenteditor/main/resources/img/favicon.ico" - elif name == "fakeoffice": - environment["COLLABORATION_APP_NAME"] = "FakeOffice" - environment["COLLABORATION_APP_PRODUCT"] = "Microsoft" - environment["COLLABORATION_APP_ADDR"] = "http://fakeoffice:8080" - - environment["COLLABORATION_WOPI_SRC"] = "http://%s:9300" % service_name - - return startOpenCloudService("collaboration", service_name, environment) - -def tikaService(): - return [{ - "name": "tika", - "type": "docker", - "image": APACHE_TIKA, - "detach": True, - }, { - "name": "wait-for-tika-service", - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it tika:9998 -t 300", - ], - }] - -def logRequests(): - return [{ - "name": "api-test-failure-logs", - "image": OC_CI_PHP % DEFAULT_PHP_VERSION, - "commands": [ - "cat %s/tests/acceptance/logs/failed.log" % dirs["base"], - ], - "when": { - "status": [ - "failure", - ], - }, - }] - -def k6LoadTests(ctx): - opencloud_remote_environment = { - "SSH_OC_REMOTE": { - "from_secret": "k6_ssh_opencloud_remote", - }, - "SSH_OC_USERNAME": { - "from_secret": "k6_ssh_opencloud_user", - }, - "SSH_OC_PASSWORD": { - "from_secret": "k6_ssh_opencloud_pass", - }, - "TEST_SERVER_URL": { - "from_secret": "k6_ssh_opencloud_server_url", - }, - } - k6_remote_environment = { - "SSH_K6_REMOTE": { - "from_secret": "k6_ssh_k6_remote", - }, - "SSH_K6_USERNAME": { - "from_secret": "k6_ssh_k6_user", - }, - "SSH_K6_PASSWORD": { - "from_secret": "k6_ssh_k6_pass", - }, - } - environment = {} - environment.update(opencloud_remote_environment) - environment.update(k6_remote_environment) - - if "skip" in config["k6LoadTests"] and config["k6LoadTests"]["skip"]: - return [] - - opencloud_git_base_url = "https://raw.githubusercontent.com/opencloud-eu/opencloud" - script_link = "%s/%s/tests/config/drone/run_k6_tests.sh" % (opencloud_git_base_url, ctx.build.commit) - - event_array = ["cron"] - - if "k6-test" in ctx.build.title.lower(): - event_array.append("pull_request") - - return [{ - "kind": "pipeline", - "type": "docker", - "name": "k6-load-test", - "clone": { - "disable": True, - }, - "steps": [ - { - "name": "k6-load-test", - "image": OC_CI_ALPINE, - "environment": environment, - "commands": [ - "curl -s -o run_k6_tests.sh %s" % script_link, - "apk add --no-cache openssh-client sshpass", - "sh %s/run_k6_tests.sh" % (dirs["base"]), - ], - }, - { - "name": "opencloud-log", - "image": OC_CI_ALPINE, - "environment": opencloud_remote_environment, - "commands": [ - "curl -s -o run_k6_tests.sh %s" % script_link, - "apk add --no-cache openssh-client sshpass", - "sh %s/run_k6_tests.sh --opencloud-log" % (dirs["base"]), - ], - "when": { - "status": [ - "success", - "failure", - ], - }, - }, - { - "name": "open-grafana-dashboard", - "image": OC_CI_ALPINE, - "commands": [ - "echo 'Grafana Dashboard: https://grafana.k6.infra.opencloud.works'", - ], - "when": { - "status": [ - "success", - "failure", - ], - }, - }, - ], - "depends_on": [], - "trigger": { - "event": event_array, - }, - }] - -def waitForServices(name, services = []): - services = ",".join(services) - return [{ - "name": "wait-for-%s" % name, - "image": OC_CI_WAIT_FOR, - "commands": [ - "wait-for -it %s -t 300" % services, - ], - }] - -def opencloudHealthCheck(name, services = []): - commands = [] - timeout = 300 - curl_command = ["timeout %s bash -c 'while [ $(curl -s %s/%s ", "-w %{http_code} -o /dev/null) != 200 ]; do sleep 1; done'"] - for service in services: - commands.append(curl_command[0] % (timeout, service, "healthz") + curl_command[1]) - commands.append(curl_command[0] % (timeout, service, "readyz") + curl_command[1]) - - return [{ - "name": "health-check-%s" % name, - "image": OC_CI_ALPINE, - "commands": commands, - }] - -def collaboraService(): - return [ - { - "name": "collabora", - "type": "docker", - "image": COLLABORA_CODE, - "detach": True, - "environment": { - "DONT_GEN_SSL_CERT": "set", - "extra_params": "--o:ssl.enable=true --o:ssl.termination=true --o:welcome.enable=false --o:net.frame_ancestors=%s" % OC_URL, - }, - "commands": [ - "coolconfig generate-proof-key", - "bash /start-collabora-online.sh", - ], - }, - ] - -def onlyofficeService(): - return [ - { - "name": "onlyoffice", - "type": "docker", - "image": ONLYOFFICE_DOCUMENT_SERVER, - "detach": True, - "environment": { - "WOPI_ENABLED": "true", - "USE_UNAUTHORIZED_STORAGE": "true", # self signed certificates - }, - "commands": [ - "cp %s/tests/config/drone/only-office.json /etc/onlyoffice/documentserver/local.json" % dirs["base"], - "openssl req -x509 -newkey rsa:4096 -keyout onlyoffice.key -out onlyoffice.crt -sha256 -days 365 -batch -nodes", - "mkdir -p /var/www/onlyoffice/Data/certs", - "cp onlyoffice.key /var/www/onlyoffice/Data/certs/", - "cp onlyoffice.crt /var/www/onlyoffice/Data/certs/", - "chmod 400 /var/www/onlyoffice/Data/certs/onlyoffice.key", - "/app/ds/run-document-server.sh", - ], - }, - ]