mirror/cypress
1
0
Fork 0
mirror of https://github.com/cypress-io/cypress.git synced 2026-05-02 21:10:47 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: better validation messages for experimental retries (#28214)

Browse Source
This commit is contained in:
Muaz Othman
2023-12-05 09:31:43 -06:00
committed by GitHub
parent fff6ce9bfe
commit 522835d385
4 changed files with 247 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/config/__snapshots__/validation.spec.ts.js
+86 -181
View File
@@ -36,7 +36,7 @@ exports['invalid retry object'] = {
'value': {
'fakeMode': 1,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls',
}
exports['not qualified url'] = {
@@ -263,233 +263,138 @@ exports['invalid upper bound'] = {
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'foo',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalStrategy',
'value': 'foo',
'type': 'one of "detect-flake-but-always-fail", "detect-flake-and-pass-on-threshold"',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with invalid strategy w/ other options (valid) 1'] = {
'key': 'mockConfigKey',
'value': {
'runMode': true,
'openMode': false,
'experimentalStrategy': 'bar',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey',
'value': {
'runMode': 1,
'openMode': 0,
'experimentalStrategy': 'detect-flake-but-always-fail',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalStrategy',
'value': 'bar',
'type': 'one of "detect-flake-but-always-fail", "detect-flake-and-pass-on-threshold"',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': -2,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': -2,
'type': 'a positive whole number greater than or equals 1 or null',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with experimentalStrategy is "detect-flake-but-always-fail" with only "maxRetries" in "experimentalOptions" 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey',
'value': {
'runMode': 1,
'openMode': 0,
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 0,
'type': 'a positive whole number greater than or equals 1 or null',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': -2,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': -2,
'type': 'a positive whole number greater than or equals 1 or null',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 0,
'type': 'a positive whole number greater than or equals 1 or null',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: maxRetries is floating 1'] = {
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 3.5,
'type': 'a positive whole number greater than or equals 1 or null',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: maxRetries is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with experimentalStrategy is "detect-flake-and-pass-on-threshold" with only "maxRetries" in "experimentalOptions" 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'value': 3.5,
'type': 'a positive whole number greater than or equals 1 or null',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is negative 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': -4,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': -4,
'type': 'a positive whole number less than or equals to maxRetries',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is 0 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': 0,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 0,
'type': 'a positive whole number less than or equals to maxRetries',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold passesRequired is floating 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 1,
'passesRequired': 3.5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 3.5,
'type': 'a positive whole number less than or equals to maxRetries',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides passesRequired without maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'passesRequired': 3,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'type': 'a positive whole number greater than or equals 1 or null',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides passesRequired that is greater than maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3,
'passesRequired': 5,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.passesRequired',
'value': 5,
'type': 'a positive whole number less than or equals to maxRetries',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold provides stopIfAnyPassed option 1'] = {
'key': 'mockConfigKey',
'key': 'mockConfigKey.experimentalOptions',
'value': {
'experimentalStrategy': 'detect-flake-and-pass-on-threshold',
'experimentalOptions': {
'maxRetries': 3,
'stopIfAnyPassed': true,
},
'maxRetries': 3,
'passesRequired': 2,
'stopIfAnyPassed': true,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys maxRetries, passesRequired',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides passesRequired option 1'] = {
'key': 'mockConfigKey',
'key': 'mockConfigKey.experimentalOptions',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 3,
'passesRequired': 2,
},
'maxRetries': 3,
'passesRequired': 2,
'stopIfAnyPassed': true,
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'type': 'an object with keys maxRetries, stopIfAnyPassed',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides stopIfAnyPassed without maxRetries 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'stopIfAnyPassed': false,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.maxRetries',
'type': 'a positive whole number greater than or equals 1 or null',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail stopIfAnyPassed is a number (0 and 1 do not work) 1'] = {
'key': 'mockConfigKey',
'value': {
'experimentalStrategy': 'detect-flake-but-always-fail',
'experimentalOptions': {
'maxRetries': 2,
'stopIfAnyPassed': 1,
},
},
'type': 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy',
'key': 'mockConfigKey.experimentalOptions.stopIfAnyPassed',
'value': 1,
'type': 'a boolean',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey.runMode',
'value': 1,
'type': 'a boolean since an experimental strategy is provided',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-and-pass-on-threshold: valid strategy w/ other invalid options with experiment 1'] = {
'key': 'mockConfigKey.runMode',
'value': 1,
'type': 'a boolean since an experimental strategy is provided',
}
exports['config/src/validation .isValidRetriesConfig returns error message for openMode as boolean without strategy 1'] = {
'key': 'mockConfigKey.openMode',
'value': true,
'type': 'a number since no experimental strategy is provided',
}
exports['config/src/validation .isValidRetriesConfig returns error message for runMode as boolean without strategy 1'] = {
'key': 'mockConfigKey.runMode',
'value': true,
'type': 'a number since no experimental strategy is provided',
}
exports['config/src/validation .isValidRetriesConfig experimental options fails with detect-flake-but-always-fail provides maxRetries without stopIfAnyPassed 1'] = {
'key': 'mockConfigKey.experimentalOptions.stopIfAnyPassed',
'type': 'is required when using the "detect-flake-but-always-fail" strategy',
}
packages/config/src/validation.ts
+95 -31
View File
@@ -119,67 +119,95 @@ export const isValidBrowserList = (_key: string, browsers: any): ErrResult | tru
return true
}
const isValidExperimentalRetryOptionsConfig = (options: any, strategy: 'detect-flake-but-always-fail' | 'detect-flake-and-pass-on-threshold'): boolean => {
if (options == null) return true
const isValidExperimentalRetryOptionsConfig = (key: string, value: any, strategy: 'detect-flake-but-always-fail' | 'detect-flake-and-pass-on-threshold') => {
if (value == null) return true
// retries must be an integer of 1 or greater
const isValidMaxRetries = _.isInteger(options.maxRetries) && options.maxRetries > 0
const isValidMaxRetries = isValidRetryValue(`${key}.maxRetries`, value.maxRetries, 1)
if (!isValidMaxRetries) {
return false
if (isValidMaxRetries !== true) {
return isValidMaxRetries
}
// if the strategy is 'detect-flake-but-always-fail', stopIfAnyPassed must be provided and must be a boolean
const validKeys = ['maxRetries']
// if the strategy is 'detect-flake-but-always-fail' and the stopIfAnyPassed is required and must be a boolean
if (strategy === 'detect-flake-but-always-fail') {
if (options.passesRequired !== undefined) {
return false
validKeys.push('stopIfAnyPassed')
if (_.isNull(value.stopIfAnyPassed) || value.stopIfAnyPassed === undefined) {
return errMsg(`${key}.stopIfAnyPassed`, value.stopIfAnyPassed, 'is required when using the "detect-flake-but-always-fail" strategy')
}
const isValidStopIfAnyPasses = _.isBoolean(options.stopIfAnyPassed)
const isValidStopIfAnyPasses = _.isBoolean(value.stopIfAnyPassed)
if (!isValidStopIfAnyPasses) {
return false
return errMsg(`${key}.stopIfAnyPassed`, value.stopIfAnyPassed, 'a boolean')
}
}
// if the strategy is 'detect-flake-and-pass-on-threshold', passesRequired must be provided and must be an integer greater than 0
// if the strategy is 'detect-flake-and-pass-on-threshold' then passesRequired is required and must be a valid retry value and less than or equal to maxRetries
if (strategy === 'detect-flake-and-pass-on-threshold') {
if (options.stopIfAnyPassed !== undefined) {
return false
validKeys.push('passesRequired')
if (_.isNull(value.passesRequired) || value.passesRequired === undefined) {
return errMsg(`${key}.passesRequired`, value.stopIfAnyPassed, 'is required when using the "detect-flake-and-pass-on-threshold" strategy')
}
const isValidPassesRequired = _.isInteger(options.passesRequired) && options.passesRequired > 0 && options.passesRequired <= options.maxRetries
const isValidPassesRequired = Number.isInteger(value.passesRequired) && value.passesRequired >= 1 && value.passesRequired <= value.maxRetries
if (!isValidPassesRequired) {
return false
return errMsg(`${key}.passesRequired`, value.passesRequired, 'a positive whole number less than or equals to maxRetries')
}
}
const extraneousKeys = _.difference(Object.keys(value), validKeys)
if (extraneousKeys.length > 0) {
return errMsg(key, value, `an object with keys ${validKeys.join(', ')}`)
}
return true
}
const isValidRetryValue = (key: string, value: any, minimumValue: 0|1): ErrResult | true => {
if (_.isNull(value)) return true
if (Number.isInteger(value) && value >= minimumValue) return true
return errMsg(key, value, `a positive whole number greater than or equals ${minimumValue} or null`)
}
export const isValidRetriesConfig = (key: string, value: any): ErrResult | true => {
const optionalKeys = ['runMode', 'openMode']
const experimentalOptions = ['experimentalStrategy', 'experimentalOptions']
const experimentalStrategyOptions = ['detect-flake-but-always-fail', 'detect-flake-and-pass-on-threshold']
const isValidRetryValue = (val: any) => _.isNull(val) || (Number.isInteger(val) && val >= 0)
const optionalKeysAreValid = (val: any, k: string) => optionalKeys.includes(k) && isValidRetryValue(val)
const optionalKeysAreValid = (key: string) => {
return (parentVal: object, optionName: string) => {
if (!optionalKeys.includes(optionName)) {
return errMsg(key, parentVal, 'an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls')
}
if (isValidRetryValue(value)) {
return true
const optionValue = _.get(parentVal, optionName)
if (_.isBoolean(optionValue)) return true
return isValidRetryValue(`${key}.${optionName}`, _.get(parentVal, optionName), 0)
}
}
if (_.isObject(value)) {
const traditionalConfigOptions = _.omit(value, experimentalOptions)
const experimentalConfigOptions = _.pick<any>(value, experimentalOptions)
const openAndRunModeConfigOptions = _.pick(value, optionalKeys)
const isTraditionalConfigValid = _.every(traditionalConfigOptions, optionalKeysAreValid)
for (const optionKey in traditionalConfigOptions) {
if (Object.prototype.hasOwnProperty.call(traditionalConfigOptions, optionKey)) {
const optionValidation = optionalKeysAreValid(key)(value, optionKey)
// if optionalKeys are only present and are valid, return true.
// The defaults for 'experimentalStrategy' and 'experimentalOptions' are undefined, but the keys exist, so we need to check for this
if (isTraditionalConfigValid && !Object.keys(experimentalConfigOptions).filter((key) => experimentalConfigOptions[key] !== undefined).length) {
return true
if (optionValidation !== true || _.isObject(optionValidation)) {
return optionValidation
}
}
}
// check experimental configuration. experimentalStrategy MUST be present if experimental config is provided and set to one of the provided enumerations
@@ -187,20 +215,56 @@ export const isValidRetriesConfig = (key: string, value: any): ErrResult | true
// make sure the strategy provided is one of our valid enums
const isValidStrategy = experimentalStrategyOptions.includes(experimentalConfigOptions.experimentalStrategy)
if (!isValidStrategy) {
return errMsg(`${key}.experimentalStrategy`, experimentalConfigOptions.experimentalStrategy, `one of ${experimentalStrategyOptions.map((s) => `"${s}"`).join(', ')}`)
}
// if a strategy is provided, and traditional options are also provided, such as runMode and openMode, then these values need to be booleans
const openAndRunModeConfigOptions = _.pick(value, optionalKeys)
const isValidRunAndOpenModeConfigWithStrategy = _.every(openAndRunModeConfigOptions, _.isBoolean)
for (const optionalKey in openAndRunModeConfigOptions) {
if (Object.prototype.hasOwnProperty.call(openAndRunModeConfigOptions, optionalKey)) {
const optionalConfigVal = _.get(openAndRunModeConfigOptions, optionalKey)
if (!_.isBoolean(optionalConfigVal)) {
return errMsg(`${key}.${optionalKey}`, optionalConfigVal, 'a boolean since an experimental strategy is provided')
}
}
}
// if options aren't present (either undefined or null) or are configured correctly, return true
if (isValidStrategy && isValidRunAndOpenModeConfigWithStrategy && (
experimentalConfigOptions.experimentalOptions == null ||
isValidExperimentalRetryOptionsConfig(experimentalConfigOptions.experimentalOptions, experimentalConfigOptions.experimentalStrategy))) {
if (
experimentalConfigOptions.experimentalOptions == null) {
return true
}
const isValidExperimentalRetryOptions = isValidExperimentalRetryOptionsConfig(`${key}.experimentalOptions`, experimentalConfigOptions.experimentalOptions, experimentalConfigOptions.experimentalStrategy)
if (isValidExperimentalRetryOptions !== true) {
return isValidExperimentalRetryOptions
}
} else {
for (const optionalKey in openAndRunModeConfigOptions) {
if (Object.prototype.hasOwnProperty.call(openAndRunModeConfigOptions, optionalKey)) {
const optionalConfigVal = _.get(openAndRunModeConfigOptions, optionalKey)
if (!_.isNumber(optionalConfigVal)) {
return errMsg(`${key}.${optionalKey}`, optionalConfigVal, 'a number since no experimental strategy is provided')
}
}
}
if (experimentalConfigOptions.experimentalOptions) {
return errMsg(`${key}.experimentalOptions`, experimentalConfigOptions.experimentalOptions, 'provided only if an experimental strategy is provided')
}
}
} else {
const isValidValue = isValidRetryValue(key, value, 0)
if (isValidValue !== true) {
return errMsg(key, value, 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy')
}
}
return errMsg(key, value, 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy')
return true
}
/**
packages/config/test/validation.spec.ts
+50 -14
View File
@@ -138,6 +138,9 @@ describe('config/src/validation', () => {
result = validation.isValidRetriesConfig(mockKey, 2)
expect(result).to.be.true
result = validation.isValidRetriesConfig(mockKey, 250)
expect(result).to.be.true
})
it('returns true for valid retry objects', () => {
@@ -145,9 +148,15 @@ describe('config/src/validation', () => {
expect(result).to.be.true
result = validation.isValidRetriesConfig(mockKey, { runMode: 250 })
expect(result).to.be.true
result = validation.isValidRetriesConfig(mockKey, { openMode: 1 })
expect(result).to.be.true
result = validation.isValidRetriesConfig(mockKey, { openMode: 250 })
expect(result).to.be.true
result = validation.isValidRetriesConfig(mockKey, {
runMode: 3,
openMode: 0,
@@ -167,6 +176,20 @@ describe('config/src/validation', () => {
snapshot('invalid retry object', result)
})
it('returns error message for openMode as boolean without strategy', () => {
let result = validation.isValidRetriesConfig(mockKey, { openMode: true })
expect(result).to.not.be.true
snapshot(result)
})
it('returns error message for runMode as boolean without strategy', () => {
let result = validation.isValidRetriesConfig(mockKey, { runMode: true })
expect(result).to.not.be.true
snapshot(result)
})
it('returns true for valid retry object with experimental keys (default)', () => {
let result = validation.isValidRetriesConfig(mockKey, {
openMode: 0,
@@ -196,6 +219,17 @@ describe('config/src/validation', () => {
expect(result).to.be.true
})
it(`experimentalStrategy is "${strategy}" with only "maxRetries" in "experimentalOptions"`, () => {
const result = validation.isValidRetriesConfig(mockKey, {
experimentalStrategy: strategy,
experimentalOptions: {
maxRetries: 4,
},
})
expect(result).to.not.be.true
})
})
it('experimentalStrategy is "detect-flake-but-always-fail" and has option "stopIfAnyPassed"', () => {
@@ -259,8 +293,8 @@ describe('config/src/validation', () => {
it('invalid strategy w/ other options (valid)', () => {
const result = validation.isValidRetriesConfig(mockKey, {
runMode: true,
openMode: false,
runMode: 1,
openMode: 2,
experimentalStrategy: 'bar',
})
@@ -315,18 +349,6 @@ describe('config/src/validation', () => {
expect(result).to.not.be.true
snapshot(result)
})
it(`experimentalStrategy is "${strategy}" with only "maxRetries" in "experimentalOptions"`, () => {
const result = validation.isValidRetriesConfig(mockKey, {
experimentalStrategy: strategy,
experimentalOptions: {
maxRetries: 4,
},
})
expect(result).to.not.be.true
snapshot(result)
})
})
describe('detect-flake-and-pass-on-threshold', () => {
@@ -399,6 +421,7 @@ describe('config/src/validation', () => {
experimentalStrategy: 'detect-flake-and-pass-on-threshold',
experimentalOptions: {
maxRetries: 3,
passesRequired: 2,
stopIfAnyPassed: true,
},
})
@@ -415,6 +438,7 @@ describe('config/src/validation', () => {
experimentalOptions: {
maxRetries: 3,
passesRequired: 2,
stopIfAnyPassed: true,
},
})
@@ -434,6 +458,18 @@ describe('config/src/validation', () => {
snapshot(result)
})
it('provides maxRetries without stopIfAnyPassed', () => {
const result = validation.isValidRetriesConfig(mockKey, {
experimentalStrategy: 'detect-flake-but-always-fail',
experimentalOptions: {
maxRetries: 2,
},
})
expect(result).to.not.be.true
snapshot(result)
})
it('stopIfAnyPassed is a number (0 and 1 do not work)', () => {
const result = validation.isValidRetriesConfig(mockKey, {
experimentalStrategy: 'detect-flake-but-always-fail',
packages/server/test/unit/config_spec.js
+16 -18
View File
@@ -883,29 +883,27 @@ describe('lib/config', () => {
})
context('retries', () => {
const retriesError = 'a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy'
// need to keep the const here or it'll get stripped by the build
// eslint-disable-next-line no-unused-vars
const cases = [
[{ retries: null }, 'with null', true],
[{ retries: 3 }, 'when a number', true],
[{ retries: 3.2 }, 'when a float', false],
[{ retries: -1 }, 'with a negative number', false],
[{ retries: true }, 'when true', false],
[{ retries: false }, 'when false', false],
[{ retries: {} }, 'with an empty object', true],
[{ retries: { runMode: 3 } }, 'when runMode is a positive number', true],
[{ retries: { runMode: -1 } }, 'when runMode is a negative number', false],
[{ retries: { openMode: 3 } }, 'when openMode is a positive number', true],
[{ retries: { openMode: -1 } }, 'when openMode is a negative number', false],
[{ retries: { openMode: 3, TypoRunMode: 3 } }, 'when there is an additional unknown key', false],
[{ retries: { openMode: 3, runMode: 3 } }, 'when both runMode and openMode are positive numbers', true],
].forEach(([config, expectation, shouldPass]) => {
it(`${shouldPass ? 'passes' : 'fails'} ${expectation}`, function () {
[{ retries: null }, 'with null', null],
[{ retries: 3 }, 'when a number', null],
[{ retries: 3.2 }, 'when a float', 'Expected retries to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy.'],
[{ retries: -1 }, 'with a negative number', 'Expected retries to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy.'],
[{ retries: true }, 'when true', 'Expected retries to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy.'],
[{ retries: false }, 'when false', 'Expected retries to be a positive number or null or an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls, or experimental configuration with key "experimentalStrategy" with value "detect-flake-but-always-fail" or "detect-flake-and-pass-on-threshold" and key "experimentalOptions" to provide a valid configuration for your selected strategy.'],
[{ retries: {} }, 'with an empty object', null],
[{ retries: { runMode: 3 } }, 'when runMode is a positive number', null],
[{ retries: { runMode: -1 } }, 'when runMode is a negative number', 'Expected retries.runMode to be a positive whole number greater than or equals 0 or null.'],
[{ retries: { openMode: 3 } }, 'when openMode is a positive number', null],
[{ retries: { openMode: -1 } }, 'when openMode is a negative number', 'Expected retries.openMode to be a positive whole number greater than or equals 0 or null'],
[{ retries: { openMode: 3, TypoRunMode: 3 } }, 'when there is an additional unknown key', 'Expected retries to be an object with keys "openMode" and "runMode" with values of numbers, booleans, or nulls.'],
[{ retries: { openMode: 3, runMode: 3 } }, 'when both runMode and openMode are positive numbers', null],
].forEach(([config, expectation, expectedError]) => {
it(`${expectedError ? 'fails' : 'passes'} ${expectation}`, function () {
this.setup(config)
return shouldPass ? this.expectValidationPasses() : this.expectValidationFails(retriesError)
return expectedError ? this.expectValidationFails(expectedError) : this.expectValidationPasses()
})
})
})