Issue 224: building, zipping and uploading binary (#234)

* server: remove obsolete desktop types

* server: ensure project path generated by tests is ignored

* driver: remove unused gulp deps

* deploy: refactored to simplify uploading binary to S3 close #224

* refactored questions to allow asking only some information

* ask for zip file

* use option --zip to pass zip filename

* install cloudflare-cli

* working on making individual binary steps work

* move buildDir and distDir to meta file

* update references

* refactor questions

* Mac binary zipped and installed successfully

* successful builds on both platforms, upload and install

* refactor combined step deploy function

* add building binary on linux to CircleCI

* install root deps only

* fine, make build binary depend on main build

* server: fix app data path

* name binary using SHA

* server: unskip test concerning productName

* build binary with serial mode

* driver: improve visibility algorithm

* server: add some logging around browser launching

* docs: update cy.trigger position option - fixes #108

* desktop: fix invalid dom nesting

* docs: Add FAQ question about 'how to test file download'

* desktop: fix un-returned promise warning when logging in

* rebuild node-sass for current platform automatically (#225)

* uncomment two lines

* desktop: use same dropdown as browsers for user menu

* desktop: simplify build scripts, remove unused deps

* readme: add docker cypress/base image badge

* fix typo in readme

* deploy: refactored to simplify uploading binary to S3 close #224

* refactored questions to allow asking only some information

* ask for zip file

* use option --zip to pass zip filename

* install cloudflare-cli

* working on making individual binary steps work

* move buildDir and distDir to meta file

* update references

* refactor questions

* Mac binary zipped and installed successfully

* successful builds on both platforms, upload and install

* refactor combined step deploy function

* add building binary on linux to CircleCI

* install root deps only

* fine, make build binary depend on main build

* name binary using SHA

* build binary with serial mode

* uncomment two lines

* print build linux folder on CI
This commit is contained in:
Gleb Bahmutov
2017-06-29 14:29:56 -04:00
committed by Brian Mann
parent 9e3ad4601f
commit bba4332e8d
13 changed files with 283 additions and 133 deletions
+29
View File
@@ -159,3 +159,32 @@ $ npm run docker
cd packages/desktop-gui
npm rebuild node-sass
```
## Deploy
You can only deploy Cypress application and publish NPM module `cypress` if
you are a member of `cypress` NPM organization.
### Building the binary
First, you need to build, zip and upload application binary to Cypress server.
You can either specify each command separately
```
npm run binary-build
npm run binary-zip
npm run binary-upload
```
or use a single command
```
npm run binary-deploy
```
You can pass options to each command to avoid answering questions, for example
```
npm run binary-deploy -- --platform darwin --version 0.20.0
npm run binary-upload -- --platform darwin --version 0.20.0 --zip cypress.zip
```
+12
View File
@@ -228,6 +228,15 @@ jobs:
- run: chrome --version
- run: xvfb-run -as "-screen 0 1280x720x16" npm run all test -- --package driver
"build-binary":
<<: *defaults
steps:
- restore_cache:
key: cypress-monorepo-{{ .Branch }}-{{ .Revision }}
- run: echo "Building version 0.0.0-$CIRCLE_BRANCH-${CIRCLE_SHA1:0:7}"
- run: npm run binary-build -- --platform linux --version "0.0.0-$CIRCLE_BRANCH-${CIRCLE_SHA1:0:7}"
- run: ls -la build/linux/Cypress
workflows:
version: 2
build_and_test:
@@ -254,6 +263,9 @@ workflows:
- driver-unit-tests:
requires:
- build
- build-binary:
requires:
- build
#
# things to run in the 4th CI container
+7 -2
View File
@@ -27,8 +27,10 @@
"precommit": "lint-staged",
"precommit-lint": "eslint --fix --rule 'no-only-tests/no-only-tests: 2'",
"bump": "gulp bump",
"no-predeploy": "echo 'Just to be safe, rebuilding Sass binary' && npm rebuild node-sass",
"deploy": "node ./scripts/deploy.js",
"binary-build": "node ./scripts/deploy.js build",
"binary-zip": "node ./scripts/deploy.js zip",
"binary-upload": "node ./scripts/deploy.js upload",
"binary-deploy": "node ./scripts/deploy.js deploy",
"release": "gulp release"
},
"lint-staged": {
@@ -44,6 +46,8 @@
"babel-eslint": "^6.0.4",
"bluebird": "^3.4.5",
"chai": "^4.0.2",
"check-more-types": "^2.24.0",
"cloudflare-cli": "^2.1.0",
"coffeelint": "^1.16.0",
"del": "^3.0.0",
"deps-ok": "^1.2.0",
@@ -63,6 +67,7 @@
"husky": "^0.13.4",
"inquirer": "^3.1.1",
"konfig": "^0.2.1",
"lazy-ass": "^1.6.0",
"lint-staged": "^3.6.0",
"lodash": "^4.17.4",
"obfuscator": "^0.5.4",
-1
View File
@@ -33,7 +33,6 @@
"bin-up": "^1.0.1",
"body-parser": "1.12.4",
"chokidar-cli": "^1.2.0",
"cloudflare-cli": "^1.5.2",
"codecov": "^1.0.1",
"coffee-coverage": "^1.0.1",
"cors": "^2.8.3",
+20 -1
View File
@@ -1,3 +1,22 @@
/* eslint-disable no-console */
require('@packages/coffee/register')
require('./deploy/index').deploy()
const command = process.argv[2]
if (!command) {
console.error('Missing deploy command ⛔️')
process.exit(1)
}
const commands = require('./deploy/index')
const fn = commands[command]
if (!fn) {
console.error('Invalid deploy command %s 🚫', command)
}
fn()
.then(() => console.log('✅ %s completed', command))
.catch((err) => {
console.error('🔥 deploy error')
console.error(err)
process.exit(1)
})
+26 -14
View File
@@ -11,22 +11,30 @@ prompt = (questions) ->
fs = Promise.promisifyAll(fs)
module.exports = {
getZipFile = ->
[{
name: "zipFile"
type: "string"
message: "Which zip file should we upload?"
}]
getPlatformQuestion: ->
[{
name: "platform"
type: "list"
message: "Which OS should we deploy?"
choices: [{
name: "Mac"
value: "darwin"
},{
name: "Linux"
value: "linux"
}]
getPlatformQuestion = ->
[{
name: "platform"
type: "list"
message: "Which OS should we deploy?"
choices: [{
name: "Mac"
value: "darwin"
},{
name: "Linux"
value: "linux"
}]
}]
module.exports = {
getZipFile
getPlatformQuestion
getQuestions: (version) ->
[{
name: "publish"
@@ -104,6 +112,10 @@ module.exports = {
else
json.version
whichZipFile: () ->
prompt(@getZipFile())
.get("zipFile")
whichVersion: (distDir) ->
## realpath returns the absolute full path
glob("*/package.json", {cwd: distDir, realpath: true})
@@ -129,7 +141,7 @@ module.exports = {
.get("release")
whichPlatform: ->
prompt(@getPlatformQuestion())
prompt(getPlatformQuestion())
.get("platform")
whichBumpTask: ->
+4 -2
View File
@@ -39,7 +39,7 @@ class Base
@uploadOsName = @getUploadNameByOs(os)
buildPathToAppFolder: ->
path.join meta.buildDir, @osName
meta.buildDir(@osName)
buildPathToZip: ->
path.join @buildPathToAppFolder(), @zipName
@@ -385,8 +385,10 @@ class Base
smokeTest = =>
new Promise (resolve, reject) =>
rand = "" + Math.random()
executable = @buildPathToAppExecutable()
console.log("executable path #{executable}")
cp.exec "#{@buildPathToAppExecutable()} --smoke-test --ping=#{rand}", (err, stdout, stderr) ->
cp.exec "#{executable} --smoke-test --ping=#{rand}", (err, stdout, stderr) ->
stdout = stdout.replace(/\s/, "")
if err
+15 -30
View File
@@ -11,6 +11,8 @@ pluralize = require("pluralize")
vinylPaths = require("vinyl-paths")
coffee = require("@packages/coffee")
electron = require("@packages/electron")
meta = require("./meta")
packages = require("./util/packages")
Darwin = require("./darwin")
Linux = require("./linux")
@@ -34,27 +36,9 @@ smokeTests = {
}
module.exports = (platform, version) ->
## returns a path into the /dist directory
distDir = (args...) ->
path.resolve("dist", platform, args...)
## returns a path into the /build directory
## the output folder should have top level "Cypress" folder
## build/
## <platform>/ = linux or darwin
## Cypress/
## ... platform-specific files
buildDir = (args...) ->
path.resolve("build", platform, "Cypress", args...)
## returns a path into the /build/*/app directory
## specific to each platform
buildAppDir = (args...) ->
switch platform
when "darwin"
buildDir("Cypress.app", "Contents", "resources", "app", args...)
when "linux"
buildDir("resources", "app", args...)
distDir = meta.distDir.bind(null, platform)
buildDir = meta.buildDir.bind(null, platform)
buildAppDir = meta.buildAppDir.bind(null, platform)
cleanupPlatform = ->
log("#cleanupPlatform", platform)
@@ -125,9 +109,10 @@ module.exports = (platform, version) ->
symlinkBuildPackages = ->
log("#symlinkBuildPackages", platform)
wildCard = buildAppDir("packages", "*", "package.json")
console.log("packages", wildCard)
packages.symlinkAll(
buildAppDir("packages", "*", "package.json"),
wildCard,
buildAppDir
)
@@ -168,11 +153,15 @@ module.exports = (platform, version) ->
elBuilder = ->
log("#elBuilder", platform)
dir = distDir()
dist = buildDir()
console.log("from #{dir}")
console.log("into #{dist}")
electron.install({
dir: distDir()
dist: buildDir()
platform: platform
dir
dist
platform
"app-version": version
})
@@ -182,8 +171,6 @@ module.exports = (platform, version) ->
smokeTest = smokeTests[platform]
smokeTest()
# Promise
# .bind(@)
Promise.resolve()
.then(cleanupPlatform)
.then(buildPackages)
@@ -199,7 +186,6 @@ module.exports = (platform, version) ->
.then(@cleanupSrc)
.then(@npmInstall)
.then(@npmInstall)
.then(@elBuilder)
.then(elBuilder)
.then(symlinkBuildPackages)
.then(runSmokeTest)
@@ -210,7 +196,6 @@ module.exports = (platform, version) ->
# .then(@cleanupCy)
# .then(@codeSign) ## codesign after running smoke tests due to changing .cy
# .then(@verifyAppCanOpen)
# .return(@)
.return({
buildDir: buildDir()
})
+73 -57
View File
@@ -7,6 +7,9 @@ os = require("os")
chalk = require("chalk")
Promise = require("bluebird")
minimist = require("minimist")
la = require("lazy-ass")
check = require("check-more-types")
zip = require("./zip")
ask = require("./ask")
bump = require("./bump")
@@ -28,32 +31,36 @@ zippedFilename = (platform) ->
# same file format as before use .zip
if platform == "linux" then "cypress.zip" else "cypress.zip"
# goes through the list of properties and asks relevant question
# resolves with all relevant options set
# if the property already exists, skips the question
askMissingOptions = (properties) -> (options = {}) ->
questions = {
platform: ask.whichPlatform,
version: ask.deployNewVersion,
# note: zip file might not be absolute
zip: ask.whichZipFile
}
properties.reduce((prev, property) ->
if (check.has(options, property)) then return prev
question = questions[property]
if (!check.fn(question)) then return prev
la(check.fn(question), "cannot find question for property", property)
prev.then(() ->
question(options[property])
.then((answer) ->
options[property] = answer
options
)
)
, Promise.resolve(options))
## hack for @packages/server modifying cwd
process.chdir(cwd)
askWhichPlatform = (platform) ->
## if we already have a platform
## just resolve with that
if platform
return Promise.resolve(platform)
## else go ask for it!
ask.whichPlatform()
askWhichVersion = (version) ->
## if we already have a version
## just resolve with that
if version
return Promise.resolve(version)
## else go ask for it!
ask.deployNewVersion()
deploy = {
zip: zip
ask: ask
meta: meta
upload: upload
Base: Base
Darwin: Darwin
Linux: Linux
@@ -74,12 +81,6 @@ deploy = {
opts.runTests = false if opts["skip-tests"]
opts
# build: (platform) ->
# ## read off the argv
# options = @parseOptions(process.argv)
#
# @getPlatform(platform?.osName, options).build()
bump: ->
ask.whichBumpTask()
.then (task) ->
@@ -109,38 +110,53 @@ deploy = {
ask.whichRelease(meta.distDir)
.then(release)
build: (options) ->
console.log('#build')
if !options then options = @parseOptions(process.argv)
askMissingOptions(['version', 'platform'])(options)
.then () ->
build(options.platform, options.version)
zip: (options) ->
console.log('#zip')
if !options then options = @parseOptions(process.argv)
askMissingOptions(['platform'])(options)
.then (options) ->
zipDir = meta.zipDir(options.platform)
options.zip = path.resolve(zippedFilename(options.platform))
zip.ditto(zipDir, options.zip)
upload: (options) ->
console.log('#upload')
if !options then options = @parseOptions(process.argv)
askMissingOptions(['version', 'platform', 'zip'])(options)
.then (options) ->
la(check.unemptyString(options.zip),
"missing zipped filename", options)
options.zip = path.resolve(options.zip)
options
.then (options) ->
console.log("Need to upload file %s", options.zip)
console.log("for platform %s version %s",
options.platform, options.version)
upload.toS3({
zipFile: options.zip,
version: options.version,
platform: options.platform
})
# goes through the entire pipeline:
# - build
# - zip
# - upload
deploy: ->
## read off the argv
# to skip further questions like platform and version
# pass these as options like this
# npm run deploy -- --platform darwin --version 0.20.0
options = @parseOptions(process.argv)
askWhichPlatform(options.platform)
.then (platform) ->
askWhichVersion(options.version)
.then (version) ->
# options.version = version
build(platform, version)
# .return([platform, version])
# .spread (platform, version) ->
# @getPlatform(plat, options).deploy()
# .then (platform) =>
.then (built) =>
console.log(built)
src = built.buildDir
dest = path.resolve(zippedFilename(platform))
zip.ditto(src, dest)
.then (zippedFilename) =>
console.log("Need to upload file %s", zippedFilename)
# upload.toS3(platform)
# .then ->
# success("Dist Complete")
# .catch (err) ->
# fail("Dist Failed")
# console.log(err)
askMissingOptions(['version', 'platform'])(options)
.then(build)
.then(() -> zip(options))
# assumes options.zip contains the zipped filename
.then(upload)
}
module.exports = _.bindAll(deploy, _.functions(deploy))
+43 -2
View File
@@ -1,7 +1,48 @@
path = require("path")
la = require("lazy-ass")
check = require("check-more-types")
isValidPlatform = check.oneOf(["darwin", "linux"])
## returns a path into the /build directory
## the output folder should look something like this
## build/
## <platform>/ = linux or darwin
## ... platform-specific files
buildDir = (platform, args...) ->
la(isValidPlatform(platform), "invalid platform", platform)
switch platform
when "darwin"
path.resolve("build", platform, args...)
when "linux"
path.resolve("build", platform, "Cypress", args...)
## returns a path into the /dist directory
distDir = (platform, args...) ->
path.resolve("dist", platform, args...)
## returns folder to zip before uploading
zipDir = (platform) ->
switch platform
when "darwin"
buildDir(platform, "Cypress.app")
when "linux"
buildDir(platform)
## returns a path into the /build/*/app directory
## specific to each platform
buildAppDir = (platform, args...) ->
switch platform
when "darwin"
buildDir(platform, "Cypress.app", "Contents", "resources", "app", args...)
when "linux"
buildDir(platform, "resources", "app", args...)
module.exports = {
distDir: path.join(process.cwd(), "dist")
buildDir: path.join(process.cwd(), "build")
isValidPlatform
buildDir
distDir
zipDir
buildAppDir
cacheDir: path.join(process.cwd(), "cache")
}
+42 -23
View File
@@ -6,12 +6,26 @@ cp = require("child_process")
path = require("path")
gulp = require("gulp")
human = require("human-interval")
konfig = require("@packages/server/lib/konfig")()
konfig = require("@packages/server/lib/konfig")
Promise = require("bluebird")
meta = require("./meta")
la = require("lazy-ass")
check = require("check-more-types")
fs = Promise.promisifyAll(fs)
uploadNames = {
darwin: "osx64"
linux: "linux64"
win32: "win64"
}
getUploadNameByOs = (os) ->
name = uploadNames[os]
if not name
throw new Error("Cannot find upload name for OS #{os}")
name
module.exports = {
getPublisher: ->
aws = @getAwsObj()
@@ -30,24 +44,29 @@ module.exports = {
getAwsObj: ->
fs.readJsonSync("./support/aws-credentials.json")
getUploadDirName: (platform) ->
# store uploaded application in subfolders by platform and version
# something like desktop/0.20.1/osx64/
getUploadDirName: ({version, platform}) ->
aws = @getAwsObj()
osName = getUploadNameByOs(platform)
dirName = [aws.folder, version, osName, null].join("/")
console.log("target directory %s", dirName)
dirName
[aws.folder, platform.getVersion(), platform.uploadOsName, null].join("/")
purgeCache: (platform) ->
purgeCache: ({zipFile, version, platform}) ->
la(check.unemptyString(platform), "missing platform", platform)
new Promise (resolve, reject) =>
version = platform.getVersion()
uploadOsName = platform.uploadOsName
zipName = platform.zipName
url = [konfig('cdn_url'), "desktop", version, uploadOsName, zipName].join("/")
cp.exec "cfcli purgefile #{url}", (err, stdout, stderr) ->
return reject(err) if err
platform.log("#purgeCache: #{url}")
zipName = path.basename(zipFile)
url = [konfig('cdn_url'), "desktop", version, platform, zipName].join("/")
console.log("purging url", url)
configFile = path.resolve("support", ".cfcli.yml")
cp.exec "cfcli purgefile -c #{configFile} #{url}", (err, stdout, stderr) ->
if err
console.error("Could not purge #{url}")
console.error(err.message)
return reject(err)
console.log("#purgeCache: #{url}")
resolve()
createRemoteManifest: (folder, version) ->
@@ -92,22 +111,22 @@ module.exports = {
.on "error", reject
.on "end", resolve
toS3: (platform) ->
platform.log("#uploadToS3")
toS3: ({zipFile, version, platform}) ->
console.log("#uploadToS3 ⏳")
la(check.unemptyString(version), "expected version string", version)
la(check.unemptyString(zipFile), "expected zip filename", zipFile)
la(meta.isValidPlatform(platform), "invalid platform", platform)
upload = =>
new Promise (resolve, reject) =>
pathToZipFile = platform.buildPathToZip()
publisher = @getPublisher()
headers = {}
headers["Cache-Control"] = "no-cache"
gulp.src(pathToZipFile)
gulp.src(zipFile)
.pipe rename (p) =>
p.dirname = @getUploadDirName(platform)
p.dirname = @getUploadDirName({version, platform})
p
.pipe debug()
.pipe publisher.publish(headers)
@@ -117,5 +136,5 @@ module.exports = {
upload()
.then =>
@purgeCache(platform)
@purgeCache({zipFile, version, platform})
}
+9 -1
View File
@@ -4,6 +4,8 @@ cp = require("child_process")
path = require("path")
glob = require("glob")
Promise = require("bluebird")
la = require("lazy-ass")
check = require("check-more-types")
fs = Promise.promisifyAll(fs)
glob = Promise.promisify(glob)
@@ -33,7 +35,8 @@ runAllBuildJs = _.partial(npmRun, ["run", "all", "build-js"])
runAllCleanJs = _.partial(npmRun, ["run", "all", "clean-js"])
# builds all the packages except for cli and docs
runAllBuild = _.partial(npmRun, ["run", "all", "build", "--", "--skip-packages", "cli,docs"])
runAllBuild = _.partial(npmRun,
["run", "all", "build", "--", "--serial", "--skip-packages", "cli,docs"])
copyAllToDist = (distDir) ->
copyRelativePathToDist = (relative) ->
@@ -83,6 +86,8 @@ npmInstallAll = (pathToPackages) ->
## 1,060,495,784 bytes (1.54 GB on disk) for 179,156 items
## 313,416,512 bytes (376.6 MB on disk) for 23,576 items
console.log("npmInstallAll packages in #{pathToPackages}")
started = new Date()
retryGlobbing = ->
@@ -123,6 +128,9 @@ ensureFoundSomething = (files) ->
symlinkAll = (pathToDistPackages, pathTo) ->
console.log("symlink these packages", pathToDistPackages)
la(check.unemptyString(pathToDistPackages),
"missing paths to dist packages", pathToDistPackages)
baseDir = path.dirname(pathTo())
toBase = path.relative.bind(null, baseDir)
+3
View File
@@ -6,6 +6,8 @@ execa = require("execa")
# resolves with zipped filename
macZip = (src, dest) ->
new Promise (resolve, reject) =>
if os.platform() != "darwin"
throw new Error("Can only zip on Mac platform")
# Ditto (Mac) options
# http://www.unix.com/man-page/OSX/1/ditto/
# -c create archive
@@ -18,6 +20,7 @@ macZip = (src, dest) ->
# foo/
# ...
zip = "ditto -c -k --sequesterRsrc --keepParent #{src} #{dest}"
console.log(zip)
cp.exec zip, {}, (err, stdout, stderr) ->
return reject(err) if err