Merge pull request #961 from owncloud/cache_ocis_build_artifact_for_testing

[tests-only] cache ocis build artifact for testing

.drone.star

@@ -107,7 +107,6 @@ def getPipelineNames(pipelines=[]):
     names.append(pipeline['name'])
   return names
 
-
 def main(ctx):
   """main is the entrypoint for drone
 
@@ -117,9 +116,11 @@ def main(ctx):
   Returns:
     none
   """
+
   pipelines = []
 
   before = \
+    [ buildOcisBinaryForTesting(ctx) ] + \
     testOcisModules(ctx) + \
     testPipelines(ctx)
 
@@ -134,6 +135,9 @@ def main(ctx):
     releaseSubmodule(ctx),
   ]
 
+  purge = purgeBuildArtifactCache(ctx, 'ocis-binary-amd64')
+  purge['depends_on'] = getPipelineNames(testPipelines(ctx))
+
   after = [
     manifest(ctx),
     changelog(ctx),
@@ -141,6 +145,7 @@ def main(ctx):
     badges(ctx),
     docs(ctx),
     updateDeployment(ctx),
+    purge,
   ]
 
   if ctx.build.event == "cron":
@@ -150,7 +155,12 @@ def main(ctx):
 
     pipelines = before + [ notify_pipeline ]
 
-  elif '[docs-only]' in (ctx.build.title + ctx.build.message):
+  elif \
+  (ctx.build.event == "pull" and '[docs-only]' in ctx.build.title) \
+  or \
+  (ctx.build.event != "pull" and '[docs-only]' in (ctx.build.title + ctx.build.message)):
+  # [docs-only] is not taken from PR messages, but from commit messages
+
     docs_pipeline = docs(ctx)
     docs_pipeline['depends_on'] = []
     docs_pipelines = [ docs_pipeline ]
@@ -182,7 +192,6 @@ def testOcisModules(ctx):
 
   return pipelines + [coverage_upload]
 
-
 def testPipelines(ctx):
   pipelines = [
     localApiTests(ctx, config['apiTests']['coreBranch'], config['apiTests']['coreCommit'], 'owncloud', 'apiOcisSpecific'),
@@ -200,7 +209,7 @@ def testPipelines(ctx):
   return pipelines
 
 def testOcisModule(ctx, module):
-  steps = generate(module) + [
+  steps = makeGenerate(module) + [
     {
       'name': 'vet',
       'image': 'webhippie/golang:1.14',
@@ -307,6 +316,34 @@ def testOcisModule(ctx, module):
     ],
   }
 
+def buildOcisBinaryForTesting(ctx):
+  return {
+    'kind': 'pipeline',
+    'type': 'docker',
+    'name': 'build_ocis_binary_for_testing',
+    'platform': {
+      'os': 'linux',
+      'arch': 'amd64',
+    },
+    'steps':
+      makeGenerate('ocis') +
+      build() +
+      rebuildBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis'),
+    'trigger': {
+      'ref': [
+        'refs/heads/master',
+        'refs/tags/v*',
+        'refs/pull/**',
+      ],
+    },
+    'volumes': [
+      {
+        'name': 'gopath',
+        'temp': {},
+      },
+    ],
+  }
+
 def uploadCoverage(ctx):
   return {
     'kind': 'pipeline',
@@ -385,8 +422,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo
       'arch': 'amd64',
     },
     'steps':
-      generate('ocis') +
-      build() +
+      restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') +
       ocisServer(storage, accounts_hash_difficulty) +
       cloneCoreRepos(coreBranch, coreCommit) + [
       {
@@ -421,6 +457,7 @@ def localApiTests(ctx, coreBranch = 'master', coreCommit = '', storage = 'ownclo
         'temp': {},
       },
     ],
+    'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]),
     'trigger': {
       'ref': [
         'refs/heads/master',
@@ -440,8 +477,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n
       'arch': 'amd64',
     },
     'steps':
-      generate('ocis') +
-      build() +
+      restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') +
       ocisServer(storage, accounts_hash_difficulty) +
       cloneCoreRepos(coreBranch, coreCommit) + [
       {
@@ -477,6 +513,7 @@ def coreApiTests(ctx, coreBranch = 'master', coreCommit = '', part_number = 1, n
         'temp': {},
       },
     ],
+    'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]),
     'trigger': {
       'ref': [
         'refs/heads/master',
@@ -509,8 +546,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '',
       'arch': 'amd64',
     },
     'steps':
-      generate('ocis') +
-      build() +
+      restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') +
       ocisServer(storage, accounts_hash_difficulty) + [
       {
         'name': 'webUITests',
@@ -562,6 +598,7 @@ def uiTestPipeline(ctx, suiteName, phoenixBranch = 'master', phoenixCommit = '',
         'temp': {}
       }
     ],
+    'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]),
     'trigger': {
       'ref': [
         'refs/heads/master',
@@ -581,8 +618,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc
       'arch': 'amd64',
     },
     'steps':
-      generate('ocis') +
-      build() +
+      restoreBuildArtifactCache(ctx, 'ocis-binary-amd64', 'ocis/bin/ocis') +
       ocisServer(storage, accounts_hash_difficulty) + [
       {
         'name': 'WebUIAcceptanceTests',
@@ -655,6 +691,7 @@ def accountsUITests(ctx, phoenixBranch, phoenixCommit, storage = 'owncloud', acc
         'temp': {}
       }
     ],
+    'depends_on': getPipelineNames([buildOcisBinaryForTesting(ctx)]),
     'trigger': {
       'ref': [
         'refs/heads/master',
@@ -674,7 +711,7 @@ def docker(ctx, arch):
       'arch': arch,
     },
     'steps':
-      generate('ocis') +
+      makeGenerate('ocis') +
       build() + [
       {
         'name': 'dryrun',
@@ -747,7 +784,7 @@ def dockerEos(ctx):
       'arch': 'amd64',
     },
     'steps':
-      generate('ocis') +
+      makeGenerate('ocis') +
       build() + [
         {
           'name': 'dryrun-eos-ocis',
@@ -858,7 +895,7 @@ def binary(ctx, name):
       'arch': 'amd64',
     },
     'steps':
-      generate('ocis') + [
+      makeGenerate('ocis') + [
       {
         'name': 'build',
         'image': 'webhippie/golang:1.14',
@@ -1312,7 +1349,7 @@ def docs(ctx):
     },
   }
 
-def generate(module):
+def makeGenerate(module):
   return [
     {
       'name': 'generate',
@@ -1541,3 +1578,95 @@ def build():
       ],
     },
   ]
+
+def genericCache(name, action, mounts, cache_key):
+  rebuild = 'false'
+  restore = 'false'
+  if action == 'rebuild':
+    rebuild = 'true'
+    action = 'rebuild'
+  else:
+    restore = 'true'
+    action = 'restore'
+
+  step = {
+      'name': '%s_%s' %(action, name),
+      'image': 'meltwater/drone-cache:v1',
+      'pull': 'always',
+      'environment': {
+        'AWS_ACCESS_KEY_ID': {
+          'from_secret': 'cache_s3_access_key',
+        },
+        'AWS_SECRET_ACCESS_KEY': {
+          'from_secret': 'cache_s3_secret_key',
+        },
+      },
+      'settings': {
+        'endpoint': {
+            'from_secret': 'cache_s3_endpoint'
+          },
+        'bucket': 'cache',
+        'region': 'us-east-1', # not used at all, but failes if not given!
+        'path_style': 'true',
+        'cache_key': cache_key,
+        'rebuild': rebuild,
+        'restore': restore,
+        'mount': mounts,
+      },
+    }
+  return step
+
+def genericCachePurge(ctx, name, cache_key):
+  return {
+    'kind': 'pipeline',
+    'type': 'docker',
+    'name': 'purge_%s' %(name),
+    'platform': {
+      'os': 'linux',
+      'arch': 'amd64',
+    },
+    'steps': [
+      {
+        'name': 'purge-cache',
+        'image': 'minio/mc',
+        'failure': 'ignore',
+        'environment': {
+          'MC_HOST_cache': {
+            'from_secret': 'cache_s3_connection_url'
+          }
+        },
+        'commands': [
+          'mc rm --recursive --force cache/cache/%s/%s' % (ctx.repo.name, cache_key),
+        ]
+      },
+    ],
+    'trigger': {
+      'ref': [
+        'refs/heads/master',
+        'refs/tags/v*',
+        'refs/pull/**',
+      ],
+      'status': [
+        'success',
+        'failure',
+      ]
+    },
+  }
+
+def genericBuildArtifactCache(ctx, name, action, path):
+  name = '%s_build_artifact_cache' %(name)
+  cache_key = '%s/%s/%s' % (ctx.repo.slug, ctx.build.commit + '-${DRONE_BUILD_NUMBER}', name)
+  if action == "rebuild" or action == "restore":
+    return genericCache(name, action, [path], cache_key)
+  if action == "purge":
+    return genericCachePurge(ctx, name, cache_key)
+  return []
+
+def restoreBuildArtifactCache(ctx, name, path):
+  return [genericBuildArtifactCache(ctx, name, 'restore', path)]
+
+def rebuildBuildArtifactCache(ctx, name, path):
+  return [genericBuildArtifactCache(ctx, name, 'rebuild', path)]
+
+def purgeBuildArtifactCache(ctx, name):
+  return genericBuildArtifactCache(ctx, name, 'purge', [])