mirror/appium
1
0
Fork 0
mirror of https://github.com/appium/appium.git synced 2026-02-22 03:08:47 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat(image-plugin): Make it possible to find elements inside of other elements (#18555)

Browse Source
This commit is contained in:
Mykola Mokhnach
2023-04-24 22:22:53 +02:00
committed by GitHub
parent 65f74d818f
commit 2796639b11
12 changed files with 572 additions and 287 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
package-lock.json generated
View File

@@ -8741,6 +8741,14 @@
"version": "2.0.5",
"license": "MIT"
},
"node_modules/deep-extend": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/deep-is": {
"version": "0.1.4",
"license": "MIT"
@@ -8869,6 +8877,14 @@
"npm": "1.2.8000 || >= 1.4.16"
}
},
"node_modules/detect-libc": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz",
"integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==",
"engines": {
"node": ">=8"
}
},
"node_modules/devtools": {
"version": "7.30.2",
"resolved": "https://registry.npmjs.org/devtools/-/devtools-7.30.2.tgz",
@@ -10018,6 +10034,14 @@
"resolved": "https://registry.npmjs.org/exif-parser/-/exif-parser-0.1.12.tgz",
"integrity": "sha512-c2bQfLNbMzLPmzQuOr8fy0csy84WmwnER81W88DzTp9CYNPJ6yzOj2EZAh9pywYpqHnshVLHQJ8WzldAyfY+Iw=="
},
"node_modules/expand-template": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
"integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
"engines": {
"node": ">=6"
}
},
"node_modules/expect": {
"version": "28.1.3",
"resolved": "https://registry.npmjs.org/expect/-/expect-28.1.3.tgz",
@@ -10929,6 +10953,11 @@
"dev": true,
"license": "ISC"
},
"node_modules/github-from-package": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
"integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="
},
"node_modules/glob": {
"version": "7.2.3",
"license": "ISC",
@@ -15999,6 +16028,11 @@
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
}
},
"node_modules/napi-build-utils": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
"integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
},
"node_modules/natural-compare": {
"version": "1.4.0",
"license": "MIT"
@@ -16053,6 +16087,17 @@
"path-to-regexp": "^1.7.0"
}
},
"node_modules/node-abi": {
"version": "3.40.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.40.0.tgz",
"integrity": "sha512-zNy02qivjjRosswoYmPi8hIKJRr8MpQyeKT6qlcq/OnOgA3Rhoae+IYOqsM9V5+JnHWmxKnWOT2GxvtqdtOCXA==",
"dependencies": {
"semver": "^7.3.5"
},
"engines": {
"node": ">=10"
}
},
"node_modules/node-addon-api": {
"version": "3.2.1",
"dev": true,
@@ -17921,6 +17966,31 @@
"node": ">=4"
}
},
"node_modules/prebuild-install": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",
"integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==",
"dependencies": {
"detect-libc": "^2.0.0",
"expand-template": "^2.0.3",
"github-from-package": "0.0.0",
"minimist": "^1.2.3",
"mkdirp-classic": "^0.5.3",
"napi-build-utils": "^1.0.1",
"node-abi": "^3.3.0",
"pump": "^3.0.0",
"rc": "^1.2.7",
"simple-get": "^4.0.0",
"tar-fs": "^2.0.0",
"tunnel-agent": "^0.6.0"
},
"bin": {
"prebuild-install": "bin.js"
},
"engines": {
"node": ">=10"
}
},
"node_modules/prelude-ls": {
"version": "1.2.1",
"license": "MIT",
@@ -18297,6 +18367,33 @@
"node": ">= 0.8"
}
},
"node_modules/rc": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
"dependencies": {
"deep-extend": "^0.6.0",
"ini": "~1.3.0",
"minimist": "^1.2.0",
"strip-json-comments": "~2.0.1"
},
"bin": {
"rc": "cli.js"
}
},
"node_modules/rc/node_modules/ini": {
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
},
"node_modules/rc/node_modules/strip-json-comments": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
"integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/react-is": {
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz",
@@ -19135,6 +19232,61 @@
"node": ">=8"
}
},
"node_modules/sharp": {
"version": "0.32.0",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.0.tgz",
"integrity": "sha512-yLAypVcqj1toSAqRSwbs86nEzfyZVDYqjuUX8grhFpeij0DDNagKJXELS/auegDBRDg1XBtELdOGfo2X1cCpeA==",
"hasInstallScript": true,
"dependencies": {
"color": "^4.2.3",
"detect-libc": "^2.0.1",
"node-addon-api": "^6.0.0",
"prebuild-install": "^7.1.1",
"semver": "^7.3.8",
"simple-get": "^4.0.1",
"tar-fs": "^2.1.1",
"tunnel-agent": "^0.6.0"
},
"engines": {
"node": ">=14.15.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/sharp/node_modules/color": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
"integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
"dependencies": {
"color-convert": "^2.0.1",
"color-string": "^1.9.0"
},
"engines": {
"node": ">=12.5.0"
}
},
"node_modules/sharp/node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dependencies": {
"color-name": "~1.1.4"
},
"engines": {
"node": ">=7.0.0"
}
},
"node_modules/sharp/node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"node_modules/sharp/node_modules/node-addon-api": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
"integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
},
"node_modules/shebang-command": {
"version": "2.0.0",
"license": "MIT",
@@ -19381,6 +19533,49 @@
"node": "^14.17.0 || ^16.13.0 || >=18.0.0"
}
},
"node_modules/simple-concat": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
"integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
]
},
"node_modules/simple-get": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
"integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"decompress-response": "^6.0.0",
"once": "^1.3.1",
"simple-concat": "^1.0.0"
}
},
"node_modules/simple-swizzle": {
"version": "0.2.2",
"license": "MIT",
@@ -20921,6 +21116,17 @@
"node": "^14.17.0 || ^16.13.0 || >=18.0.0"
}
},
"node_modules/tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
"dependencies": {
"safe-buffer": "^5.0.1"
},
"engines": {
"node": "*"
}
},
"node_modules/type": {
"version": "1.2.0",
"dev": true,
@@ -22604,9 +22810,9 @@
"dependencies": {
"@types/bluebird": "3.5.38",
"bluebird": "3.7.2",
"jimp": "0.22.7",
"lodash": "4.17.21",
"opencv-bindings": "4.5.5",
"sharp": "0.32.0",
"source-map-support": "0.5.21"
},
"engines": {

15
packages/images-plugin/lib/compare.js
View File

@@ -1,12 +1,7 @@
import _ from 'lodash';
import {errors} from 'appium/driver';
import {getImagesMatches, getImagesSimilarity, getImageOccurrence} from '@appium/opencv';
const MATCH_FEATURES_MODE = 'matchFeatures';
const GET_SIMILARITY_MODE = 'getSimilarity';
const MATCH_TEMPLATE_MODE = 'matchTemplate';
const DEFAULT_MATCH_THRESHOLD = 0.4;
import {MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE} from './constants';
/**
* Performs images comparison using OpenCV framework features.
@@ -84,13 +79,7 @@ function convertVisualizationToBase64(element) {
return element;
}
export {
compareImages,
DEFAULT_MATCH_THRESHOLD,
MATCH_TEMPLATE_MODE,
MATCH_FEATURES_MODE,
GET_SIMILARITY_MODE,
};
export {compareImages};
/**
* @typedef {import('@appium/opencv').OccurrenceResult} OccurrenceResult

70
packages/images-plugin/lib/constants.js Normal file
View File

@@ -0,0 +1,70 @@
import {node} from '@appium/support';
export const IMAGE_STRATEGY = '-image';
export const IMAGE_ELEMENT_PREFIX = 'appium-image-element-';
export const IMAGE_EL_TAP_STRATEGY_W3C = 'w3cActions';
export const IMAGE_EL_TAP_STRATEGY_MJSONWP = 'touchActions';
export const IMAGE_TAP_STRATEGIES = [IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C];
export const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0;
export const MATCH_FEATURES_MODE = 'matchFeatures';
export const GET_SIMILARITY_MODE = 'getSimilarity';
export const MATCH_TEMPLATE_MODE = 'matchTemplate';
export const DEFAULT_MATCH_THRESHOLD = 0.4;
export const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;
export const DEFAULT_SETTINGS = node.deepFreeze({
// value between 0 and 1 representing match strength, below which an image
// element will not be found
imageMatchThreshold: DEFAULT_MATCH_THRESHOLD,
// One of possible image matching methods.
// Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
// for more details.
// TM_CCOEFF_NORMED by default
imageMatchMethod: '',
// if the image returned by getScreenshot differs in size or aspect ratio
// from the screen, attempt to fix it automatically
fixImageFindScreenshotDims: true,
// whether Appium should ensure that an image template sent in during image
// element find should have its size adjusted so the match algorithm will not
// complain
fixImageTemplateSize: false,
// whether Appium should ensure that an image template sent in during image
// element find should have its scale adjusted to display size so the match
// algorithm will not complain.
// e.g. iOS has `width=375, height=667` window rect, but its screenshot is
// `width=750 × height=1334` pixels. This setting help to adjust the scale
// if a user use `width=750 × height=1334` pixels's base template image.
fixImageTemplateScale: false,
// Users might have scaled template image to reduce their storage size.
// This setting allows users to scale a template image they send to Appium server
// so that the Appium server compares the actual scale users originally had.
// e.g. If a user has an image of 270 x 32 pixels which was originally 1080 x 126 pixels,
// the user can set {defaultImageTemplateScale: 4.0} to scale the small image
// to the original one so that Appium can compare it as the original one.
defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,
// whether Appium should re-check that an image element can be matched
// against the current screenshot before clicking it
checkForImageElementStaleness: true,
// whether before clicking on an image element Appium should re-determine the
// position of the element on screen
autoUpdateImageElementPosition: false,
// which method to use for tapping by coordinate for image elements. the
// options are 'w3c' or 'mjsonwp'
imageElementTapStrategy: IMAGE_EL_TAP_STRATEGY_W3C,
// which method to use to save the matched image area in ImageElement class.
// It is used for debugging purpose.
getMatchedImageResult: false,
});

250
packages/images-plugin/lib/finder.js
View File

@@ -1,113 +1,60 @@
import _ from 'lodash';
import LRU from 'lru-cache';
import {errors} from 'appium/driver';
import {util, imageUtil} from 'appium/support';
import {
ImageElement,
DEFAULT_TEMPLATE_IMAGE_SCALE,
IMAGE_EL_TAP_STRATEGY_W3C,
} from './image-element';
import {MATCH_TEMPLATE_MODE, compareImages, DEFAULT_MATCH_THRESHOLD} from './compare';
import {imageUtil} from 'appium/support';
import {ImageElement} from './image-element';
import {compareImages} from './compare';
import log from './logger';
import {
DEFAULT_SETTINGS, MATCH_TEMPLATE_MODE, DEFAULT_TEMPLATE_IMAGE_SCALE,
DEFAULT_FIX_IMAGE_TEMPLATE_SCALE,
} from './constants';
const MJSONWP_ELEMENT_KEY = 'ELEMENT';
const W3C_ELEMENT_KEY = util.W3C_WEB_ELEMENT_IDENTIFIER;
const DEFAULT_FIX_IMAGE_TEMPLATE_SCALE = 1;
// Used to compare ratio and screen width
// Pixel is basically under 1080 for example. 100K is probably enough fo a while.
const FLOAT_PRECISION = 100000;
const MAX_CACHE_ITEMS = 100;
const MAX_CACHE_SIZE_BYTES = 1024 * 1024 * 40; // 40mb
const DEFAULT_SETTINGS = {
// value between 0 and 1 representing match strength, below which an image
// element will not be found
imageMatchThreshold: DEFAULT_MATCH_THRESHOLD,
/**
* Checks if one rect fully contains another
*
* @param {import('@appium/types').Rect} templateRect The bounding rect
* @param {import('@appium/types').Rect} rect The rect to be checked for containment
* @returns {boolean} True if templateRect contains rect
*/
function containsRect(templateRect, rect) {
return templateRect.x <= rect.x && templateRect.y <= rect.y
&& rect.width <= templateRect.x + templateRect.width - rect.x
&& rect.height <= templateRect.y + templateRect.height - rect.y;
}
// One of possible image matching methods.
// Read https://docs.opencv.org/3.0-beta/doc/py_tutorials/py_imgproc/py_template_matching/py_template_matching.html
// for more details.
// TM_CCOEFF_NORMED by default
imageMatchMethod: '',
// if the image returned by getScreenshot differs in size or aspect ratio
// from the screen, attempt to fix it automatically
fixImageFindScreenshotDims: true,
// whether Appium should ensure that an image template sent in during image
// element find should have its size adjusted so the match algorithm will not
// complain
fixImageTemplateSize: false,
// whether Appium should ensure that an image template sent in during image
// element find should have its scale adjusted to display size so the match
// algorithm will not complain.
// e.g. iOS has `width=375, height=667` window rect, but its screenshot is
// `width=750 × height=1334` pixels. This setting help to adjust the scale
// if a user use `width=750 × height=1334` pixels's base template image.
fixImageTemplateScale: false,
// Users might have scaled template image to reduce their storage size.
// This setting allows users to scale a template image they send to Appium server
// so that the Appium server compares the actual scale users originally had.
// e.g. If a user has an image of 270 x 32 pixels which was originally 1080 x 126 pixels,
// the user can set {defaultImageTemplateScale: 4.0} to scale the small image
// to the original one so that Appium can compare it as the original one.
defaultImageTemplateScale: DEFAULT_TEMPLATE_IMAGE_SCALE,
// whether Appium should re-check that an image element can be matched
// against the current screenshot before clicking it
checkForImageElementStaleness: true,
// whether before clicking on an image element Appium should re-determine the
// position of the element on screen
autoUpdateImageElementPosition: false,
// which method to use for tapping by coordinate for image elements. the
// options are 'w3c' or 'mjsonwp'
imageElementTapStrategy: IMAGE_EL_TAP_STRATEGY_W3C,
// which method to use to save the matched image area in ImageElement class.
// It is used for debugging purpose.
getMatchedImageResult: false,
};
const NO_OCCURRENCES_PATTERN = /Cannot find any occurrences/;
const CONDITION_UNMET_PATTERN = /Condition unmet/;
export default class ImageElementFinder {
/** @type {ExternalDriver} */
driver;
/** @type {LRU<string,ImageElement>} */
imgElCache;
/**
*
* @param {ExternalDriver} driver
* @param {number} [maxSize]
* @param {number} maxSize
*/
constructor(driver, maxSize = MAX_CACHE_SIZE_BYTES) {
this.driver = driver;
constructor(maxSize = MAX_CACHE_SIZE_BYTES) {
this.imgElCache = new LRU({
max: MAX_CACHE_ITEMS,
maxSize,
sizeCalculation: (el) => el.template.length,
sizeCalculation: (el) => el.template.length + (el.matchedImage?.length ?? 0),
});
}
setDriver(driver) {
this.driver = driver;
}
/**
* @param {ImageElement} imgEl
* @returns {Element}
*/
registerImageElement(imgEl) {
this.imgElCache.set(imgEl.id, imgEl);
const protoKey = this.driver.isW3CProtocol() ? W3C_ELEMENT_KEY : MJSONWP_ELEMENT_KEY;
return imgEl.asElement(protoKey);
return imgEl.asElement();
}
/**
@@ -119,6 +66,8 @@ export default class ImageElementFinder {
* @property {boolean} [ignoreDefaultImageTemplateScale=false] - Whether we
* ignore defaultImageTemplateScale. It can be used when you would like to
* scale b64Template with defaultImageTemplateScale setting.
* @property {import('@appium/types').Rect?} containerRect - The bounding
* rectangle to limit the search in
*/
/**
@@ -127,18 +76,17 @@ export default class ImageElementFinder {
*
* @param {string} b64Template - base64-encoded image used as a template to be
* matched in the screenshot
* @param {ExternalDriver} driver
* @param {FindByImageOptions} opts - additional options
*
* @returns {Promise<Element|Element[]|ImageElement>} - WebDriver element with a special id prefix
*/
async findByImage(
b64Template,
{shouldCheckStaleness = false, multiple = false, ignoreDefaultImageTemplateScale = false}
driver,
{shouldCheckStaleness = false, multiple = false, ignoreDefaultImageTemplateScale = false, containerRect = null}
) {
if (!this.driver) {
throw new Error(`Can't find without a driver!`);
}
const settings = {...DEFAULT_SETTINGS, ...this.driver.settings.getSettings()};
const settings = {...DEFAULT_SETTINGS, ...driver.settings.getSettings()};
const {
imageMatchThreshold: threshold,
imageMatchMethod,
@@ -149,26 +97,36 @@ export default class ImageElementFinder {
} = settings;
log.info(`Finding image element with match threshold ${threshold}`);
if (!this.driver.getWindowSize) {
throw new Error("This driver does not support the required 'getWindowSize' command");
if (!driver.getWindowRect && !driver.getWindowSize) {
throw new Error("This driver does not support the required 'getWindowRect' command");
}
let screenSize;
if (driver.getWindowRect) {
const screenRect = await driver.getWindowRect();
screenSize = {
width: screenRect.width,
height: screenRect.height,
};
} else {
// TODO: Drop the deprecated endpoint
screenSize = await driver.getWindowSize();
}
const {width: screenWidth, height: screenHeight} = await this.driver.getWindowSize();
// someone might have sent in a template that's larger than the screen
// dimensions. If so let's check and cut it down to size since the algorithm
// will not work unless we do. But because it requires some potentially
// expensive commands, only do this if the user has requested it in settings.
if (fixImageTemplateSize) {
b64Template = await this.ensureTemplateSize(b64Template, screenWidth, screenHeight);
b64Template = await this.ensureTemplateSize(b64Template, {
width: containerRect ? containerRect.width : screenSize.width,
height: containerRect ? containerRect.height : screenSize.height,
});
}
const results = [];
const condition = async () => {
try {
const {b64Screenshot, scale} = await this.getScreenshotForImageFind(
screenWidth,
screenHeight
);
const {b64Screenshot, scale} = await this.getScreenshotForImageFind(driver, screenSize);
b64Template = await this.fixImageTemplateScale(b64Template, {
defaultImageTemplateScale,
@@ -185,21 +143,21 @@ export default class ImageElementFinder {
if (imageMatchMethod) {
comparisonOpts.method = imageMatchMethod;
}
if (multiple) {
results.push(
...(await compareImages(
MATCH_TEMPLATE_MODE,
b64Screenshot,
b64Template,
comparisonOpts
))
);
} else {
results.push(
await compareImages(MATCH_TEMPLATE_MODE, b64Screenshot, b64Template, comparisonOpts)
);
}
return true;
const pushIfOk = (el) => {
if (containerRect && !containsRect(containerRect, el.rect)) {
log.debug(
`The matched element rectangle ${JSON.stringify(el.rect)} is not located ` +
`inside of the bounding rectangle ${JSON.stringify(containerRect)}, thus rejected`
);
return false;
}
results.push(el);
return true;
};
const elOrEls = await compareImages(MATCH_TEMPLATE_MODE, b64Screenshot, b64Template, comparisonOpts);
return _.some((_.isArray(elOrEls) ? elOrEls : [elOrEls]).map(pushIfOk));
} catch (err) {
// if compareImages fails, we'll get a specific error, but we should
// retry, so trap that and just return false to trigger the next round of
@@ -213,7 +171,7 @@ export default class ImageElementFinder {
};
try {
await this.driver.implicitWaitForCondition(condition);
await driver.implicitWaitForCondition(condition);
} catch (err) {
// this `implicitWaitForCondition` method will throw a 'Condition unmet'
// error if an element is not found eventually. In that case, we will
@@ -235,7 +193,14 @@ export default class ImageElementFinder {
const elements = results.map(({rect, score, visualization}) => {
log.info(`Image template matched: ${JSON.stringify(rect)}`);
return new ImageElement(b64Template, rect, score, visualization, this);
return new ImageElement(
b64Template,
rect,
score,
visualization,
this,
containerRect
);
});
// if we're just checking staleness, return straightaway so we don't add
@@ -254,29 +219,28 @@ export default class ImageElementFinder {
* Ensure that the image template sent in for a find is of a suitable size
*
* @param {string} b64Template - base64-encoded image
* @param {number} screenWidth - width of screen
* @param {number} screenHeight - height of screen
* @param {import('@appium/types').Size} maxSize - size of the bounding rectangle
*
* @returns {Promise<string>} base64-encoded image, potentially resized
*/
async ensureTemplateSize(b64Template, screenWidth, screenHeight) {
async ensureTemplateSize(b64Template, maxSize) {
let imgObj = await imageUtil.getJimpImage(b64Template);
let {width: tplWidth, height: tplHeight} = imgObj.bitmap;
log.info(
`Template image is ${tplWidth}x${tplHeight}. Screen size is ${screenWidth}x${screenHeight}`
`Template image is ${tplWidth}x${tplHeight}. Bounding rectangle size is ${maxSize.width}x${maxSize.height}`
);
// if the template fits inside the screen dimensions, we're good
if (tplWidth <= screenWidth && tplHeight <= screenHeight) {
if (tplWidth <= maxSize.width && tplHeight <= maxSize.height) {
return b64Template;
}
log.info(
`Scaling template image from ${tplWidth}x${tplHeight} to match ` +
`screen at ${screenWidth}x${screenHeight}`
`the bounding rectangle at ${maxSize.width}x${maxSize.height}`
);
// otherwise, scale it to fit inside the screen dimensions
imgObj = imgObj.scaleToFit(screenWidth, screenHeight);
// otherwise, scale it to fit inside the bounding rectangle dimensions
imgObj = imgObj.scaleToFit(maxSize.width, maxSize.height);
return (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');
}
@@ -284,19 +248,19 @@ export default class ImageElementFinder {
* Get the screenshot image that will be used for find by element, potentially
* altering it in various ways based on user-requested settings
*
* @param {number} screenWidth - width of screen
* @param {number} screenHeight - height of screen
* @param {ExternalDriver} driver
* @param {import('@appium/types').Size} screenSize - The original size of the screen
*
* @returns {Promise<Screenshot & {scale?: ScreenshotScale}>} base64-encoded screenshot and ScreenshotScale
*/
async getScreenshotForImageFind(screenWidth, screenHeight) {
if (!this.driver.getScreenshot) {
async getScreenshotForImageFind(driver, screenSize) {
if (!driver.getScreenshot) {
throw new Error("This driver does not support the required 'getScreenshot' command");
}
const settings = Object.assign({}, DEFAULT_SETTINGS, this.driver.settings.getSettings());
const settings = Object.assign({}, DEFAULT_SETTINGS, driver.settings.getSettings());
const {fixImageFindScreenshotDims} = settings;
let b64Screenshot = await this.driver.getScreenshot();
const b64Screenshot = await driver.getScreenshot();
// if the user has requested not to correct for aspect or size differences
// between the screenshot and the screen, just return the screenshot now
@@ -305,10 +269,10 @@ export default class ImageElementFinder {
return {b64Screenshot};
}
if (screenWidth < 1 || screenHeight < 1) {
if (screenSize.width < 1 || screenSize.height < 1) {
log.warn(
`The retrieved screen size ${screenWidth}x${screenHeight} does ` +
`not seem to be valid. No changes will be applied to the screenshot`
`The retrieved screen size ${screenSize.width}x${screenSize.height} does ` +
`not seem to be valid. No changes will be applied to the screenshot`
);
return {b64Screenshot};
}
@@ -323,12 +287,12 @@ export default class ImageElementFinder {
if (shotWidth < 1 || shotHeight < 1) {
log.warn(
`The retrieved screenshot size ${shotWidth}x${shotHeight} does ` +
`not seem to be valid. No changes will be applied to the screenshot`
`not seem to be valid. No changes will be applied to the screenshot`
);
return {b64Screenshot};
}
if (screenWidth === shotWidth && screenHeight === shotHeight) {
if (screenSize.width === shotWidth && screenSize.height === shotHeight) {
// the height and width of the screenshot and the device screen match, which
// means we should be safe when doing template matches
log.info('Screenshot size matched screen size');
@@ -343,19 +307,19 @@ export default class ImageElementFinder {
const scale = {xScale: 1.0, yScale: 1.0};
const screenAR = screenWidth / screenHeight;
const screenAR = screenSize.width / screenSize.height;
const shotAR = shotWidth / shotHeight;
if (Math.round(screenAR * FLOAT_PRECISION) === Math.round(shotAR * FLOAT_PRECISION)) {
log.info(
`Screenshot aspect ratio '${shotAR}' (${shotWidth}x${shotHeight}) matched ` +
`screen aspect ratio '${screenAR}' (${screenWidth}x${screenHeight})`
`screen aspect ratio '${screenAR}' (${screenSize.width}x${screenSize.height})`
);
} else {
log.warn(
`When trying to find an element, determined that the screen ` +
`aspect ratio and screenshot aspect ratio are different. Screen ` +
`is ${screenWidth}x${screenHeight} whereas screenshot is ` +
`${shotWidth}x${shotHeight}.`
`aspect ratio and screenshot aspect ratio are different. Screen ` +
`is ${screenSize.width}x${screenSize.height} whereas screenshot is ` +
`${shotWidth}x${shotHeight}.`
);
// In the case where the x-scale and y-scale are different, we need to decide
@@ -369,14 +333,14 @@ export default class ImageElementFinder {
// which is used to image comparison by OpenCV as a base image.
// All of this is primarily useful when the screenshot is a horizontal slice taken out of the
// screen (for example not including top/bottom nav bars)
const xScale = (1.0 * shotWidth) / screenWidth;
const yScale = (1.0 * shotHeight) / screenHeight;
const xScale = (1.0 * shotWidth) / screenSize.width;
const yScale = (1.0 * shotHeight) / screenSize.height;
const scaleFactor = xScale >= yScale ? yScale : xScale;
log.warn(
`Resizing screenshot to ${shotWidth * scaleFactor}x${shotHeight * scaleFactor} to match ` +
`screen aspect ratio so that image element coordinates have a ` +
`greater chance of being correct.`
`screen aspect ratio so that image element coordinates have a ` +
`greater chance of being correct.`
);
imgObj = imgObj.resize(shotWidth * scaleFactor, shotHeight * scaleFactor);
@@ -391,19 +355,21 @@ export default class ImageElementFinder {
// since except for that, it might be a situation which is different window rect and
// screenshot size like `@driver.window_rect #=>x=0, y=0, width=1080, height=1794` and
// `"deviceScreenSize"=>"1080x1920"`
if (screenWidth !== shotWidth && screenHeight !== shotHeight) {
if (screenSize.width !== shotWidth && screenSize.height !== shotHeight) {
log.info(
`Scaling screenshot from ${shotWidth}x${shotHeight} to match ` +
`screen at ${screenWidth}x${screenHeight}`
`screen at ${screenSize.width}x${screenSize.height}`
);
imgObj = imgObj.resize(screenWidth, screenHeight);
imgObj = imgObj.resize(screenSize.width, screenSize.height);
scale.xScale *= (1.0 * screenWidth) / shotWidth;
scale.yScale *= (1.0 * screenHeight) / shotHeight;
scale.xScale *= (1.0 * screenSize.width) / shotWidth;
scale.yScale *= (1.0 * screenSize.height) / shotHeight;
}
b64Screenshot = (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64');
return {b64Screenshot, scale};
return {
b64Screenshot: (await imgObj.getBuffer(imageUtil.MIME_PNG)).toString('base64'),
scale,
};
}
/**
@@ -492,8 +458,6 @@ export default class ImageElementFinder {
}
}
export {W3C_ELEMENT_KEY, MJSONWP_ELEMENT_KEY, DEFAULT_SETTINGS, DEFAULT_FIX_IMAGE_TEMPLATE_SCALE};
/**
* @typedef {import('@appium/types').ExternalDriver} ExternalDriver
* @typedef {import('@appium/types').Element} Element

54
packages/images-plugin/lib/image-element.js
View File

@@ -2,14 +2,12 @@ import _ from 'lodash';
import {errors} from 'appium/driver';
import {util} from 'appium/support';
import log from './logger';
import {DEFAULT_SETTINGS} from './finder';
import {
IMAGE_STRATEGY, DEFAULT_SETTINGS, IMAGE_TAP_STRATEGIES,
IMAGE_ELEMENT_PREFIX, IMAGE_EL_TAP_STRATEGY_W3C,
} from './constants';
const IMAGE_ELEMENT_PREFIX = 'appium-image-element-';
const TAP_DURATION_MS = 125;
const IMAGE_EL_TAP_STRATEGY_W3C = 'w3cActions';
const IMAGE_EL_TAP_STRATEGY_MJSONWP = 'touchActions';
const IMAGE_TAP_STRATEGIES = [IMAGE_EL_TAP_STRATEGY_MJSONWP, IMAGE_EL_TAP_STRATEGY_W3C];
const DEFAULT_TEMPLATE_IMAGE_SCALE = 1.0;
/**
* @typedef Dimension
@@ -37,14 +35,17 @@ export default class ImageElement {
* @param {string?} b64Result - the base64-encoded image which has matched marks.
* Defaults to null.
* @param {import('./finder').default?} finder - the finder we can use to re-check stale elements
* @param {import('@appium/types').Rect?} containerRect - The bounding
* rectangle to limit the search in
*/
constructor(b64Template, rect, score, b64Result = null, finder = null) {
constructor(b64Template, rect, score, b64Result = null, finder = null, containerRect = null) {
this.template = b64Template;
this.rect = rect;
this.id = `${IMAGE_ELEMENT_PREFIX}${util.uuidV4()}`;
this.b64MatchedImage = b64Result;
this.score = score;
this.finder = finder;
this.containerRect = containerRect;
}
/**
@@ -79,13 +80,11 @@ export default class ImageElement {
}
/**
* @param {string} protocolKey - the protocol-specific JSON key for
* a WebElement
*
* @returns {Element} - this image element as a WebElement
*/
asElement(protocolKey) {
return {[protocolKey]: this.id};
asElement() {
return util.wrapElement(this.id);
}
/**
@@ -132,11 +131,12 @@ export default class ImageElement {
if (checkForImageElementStaleness || updatePos) {
log.info('Checking image element for staleness before clicking');
try {
newImgEl = await this.finder.findByImage(this.template, {
newImgEl = await this.finder.findByImage(this.template, driver, {
shouldCheckStaleness: true,
// Set ignoreDefaultImageTemplateScale because this.template is device screenshot based image
// managed inside Appium after finidng image by template which managed by a user
ignoreDefaultImageTemplateScale: true,
containerRect: this.containerRect,
});
} catch (err) {
throw new errors.StaleElementReferenceError();
@@ -209,6 +209,22 @@ export default class ImageElement {
);
}
/**
* Perform lookup of image element(s) inside of the current element
*
* @param {boolean} multiple - Whether to lookup multiple elements
* @param {import('appium/driver').BaseDriver} driver - The driver to use for commands
* @param {string[]} args = Rest of arguments for executeScripts
* @returns {Promise<Element|Element[]|ImageElement>} - WebDriver element with a special id prefix
*/
async find(multiple, driver, ...args) {
const [strategy, selector] = args;
if (strategy !== IMAGE_STRATEGY) {
throw new errors.InvalidSelectorError(`Lookup strategies other than '${IMAGE_STRATEGY}' are not supported`);
}
return await this.finder.findByImage(selector, driver, {multiple, containerRect: this.rect});
}
/**
* Handle various Appium commands that involve an image element
*
@@ -223,6 +239,10 @@ export default class ImageElement {
switch (cmd) {
case 'click':
return await imgEl.click(driver);
case 'findElementFromElement':
return await imgEl.find(false, driver, ...args);
case 'findElementsFromElement':
return await imgEl.find(true, driver, ...args);
case 'elementDisplayed':
return true;
case 'getSize':
@@ -232,6 +252,8 @@ export default class ImageElement {
return imgEl.location;
case 'getElementRect':
return imgEl.rect;
case 'getElementScreenshot':
return imgEl.template;
case 'getAttribute':
// /session/:sessionId/element/:elementId/attribute/:name
// /session/:sessionId/element/:elementId/attribute/visual should retun the visual data
@@ -250,13 +272,7 @@ export default class ImageElement {
}
}
export {
ImageElement,
IMAGE_EL_TAP_STRATEGY_MJSONWP,
IMAGE_EL_TAP_STRATEGY_W3C,
DEFAULT_TEMPLATE_IMAGE_SCALE,
IMAGE_ELEMENT_PREFIX,
};
export {ImageElement};
/**
* @typedef {import('@appium/types').Rect} Rect

8
packages/images-plugin/lib/plugin.js
View File

@@ -5,9 +5,8 @@ import {errors} from 'appium/driver';
import BasePlugin from 'appium/plugin';
import {compareImages} from './compare';
import ImageElementFinder from './finder';
import {ImageElement, IMAGE_ELEMENT_PREFIX} from './image-element';
const IMAGE_STRATEGY = '-image';
import {ImageElement} from './image-element';
import {IMAGE_STRATEGY, IMAGE_ELEMENT_PREFIX} from './constants';
function getImgElFromArgs(args) {
for (let arg of args) {
@@ -57,8 +56,7 @@ export default class ImageElementPlugin extends BasePlugin {
return await next();
}
this.finder.setDriver(driver);
return await this.finder.findByImage(selector, {multiple});
return await this.finder.findByImage(selector, driver, {multiple});
}
async handle(next, driver, cmdName, ...args) {

41
packages/images-plugin/test/e2e/plugin.e2e.spec.js
View File

@@ -1,8 +1,10 @@
import _ from 'lodash';
import path from 'path';
import {remote as wdio} from 'webdriverio';
import {MATCH_FEATURES_MODE, GET_SIMILARITY_MODE} from '../../lib/compare';
import {MATCH_FEATURES_MODE, GET_SIMILARITY_MODE} from '../../lib/constants';
import {TEST_IMG_1_B64, TEST_IMG_2_B64, APPSTORE_IMG_PATH} from '../fixtures';
import {pluginE2EHarness} from '@appium/plugin-test-support';
import {tempDir, fs, imageUtil} from '@appium/support';
const THIS_PLUGIN_DIR = path.join(__dirname, '..', '..');
const APPIUM_HOME = path.join(THIS_PLUGIN_DIR, 'local_appium_home');
@@ -32,11 +34,14 @@ const WDIO_OPTS = {
};
describe('ImageElementPlugin', function () {
let server,
driver = null;
let server;
let driver;
// this hook is intended to be run before the hooks created by `e2eSetup`
after(async function () {
beforeEach(async function () {
driver = await wdio(WDIO_OPTS);
});
afterEach(async function () {
if (driver) {
await driver.deleteSession();
}
@@ -58,7 +63,6 @@ describe('ImageElementPlugin', function () {
});
it('should add the compareImages route', async function () {
driver = await wdio(WDIO_OPTS);
let comparison = await driver.compareImages(
MATCH_FEATURES_MODE,
TEST_IMG_1_B64,
@@ -85,4 +89,29 @@ describe('ImageElementPlugin', function () {
height.should.eql(91);
await imageEl.click();
});
it('should find subelements', async function () {
const imageEl = await driver.$(APPSTORE_IMG_PATH);
const {width, height} = await imageEl.getSize();
const tmpRoot = await tempDir.openDir();
try {
const screenshotPath = path.join(tmpRoot, 'element.png');
await imageEl.saveScreenshot(screenshotPath);
const tmpImgPath = path.join(tmpRoot, 'region.png');
const region = await imageUtil.cropBase64Image(
(await fs.readFile(screenshotPath)).toString('base64'),
{
left: width / 4,
top: height / 4,
width: width / 2,
height: height / 2,
}
);
await fs.writeFile(tmpImgPath, Buffer.from(region, 'base64'));
const subEl = await imageEl.$(tmpImgPath);
_.isNil(subEl).should.be.false;
} finally {
await fs.rimraf(tmpRoot);
}
});
});

114
packages/images-plugin/test/unit/finder.spec.js
View File

@@ -1,7 +1,8 @@
import _ from 'lodash';
import {imageUtil} from 'appium/support';
import {BaseDriver} from 'appium/driver';
import {ImageElementPlugin, IMAGE_STRATEGY} from '../../lib/plugin';
import {ImageElementPlugin} from '../../lib/plugin';
import {IMAGE_STRATEGY} from '../../lib/constants';
import ImageElementFinder from '../../lib/finder';
import ImageElement from '../../lib/image-element';
import sinon from 'sinon';
@@ -12,7 +13,7 @@ const compareModule = require('../../lib/compare');
const plugin = new ImageElementPlugin();
class PluginDriver extends BaseDriver {
async getWindowSize() {}
async getWindowRect() {}
async getScreenshot() {}
findElement(strategy, selector) {
return plugin.findElement(_.noop, this, strategy, selector);
@@ -66,9 +67,13 @@ describe('finding elements by image', function () {
let compareStub;
function basicStub(driver, finder) {
const sizeStub = sandbox.stub(driver, 'getWindowSize').returns(size);
const rectStub = sandbox.stub(driver, 'getWindowRect').returns({
x: 0,
y: 0,
...size,
});
const screenStub = sandbox.stub(finder, 'getScreenshotForImageFind').returns(screenshot);
return {sizeStub, screenStub};
return {rectStub, screenStub};
}
function basicImgElVerify(imgElProto, finder) {
@@ -83,7 +88,7 @@ describe('finding elements by image', function () {
beforeEach(function () {
d = new PluginDriver();
f = new ImageElementFinder(d);
f = new ImageElementFinder();
compareStub = sandbox;
compareStub = sandbox.stub(compareModule, 'compareImages');
compareStub.resolves({rect, score});
@@ -91,26 +96,26 @@ describe('finding elements by image', function () {
});
it('should find an image element happypath', async function () {
const imgElProto = await f.findByImage(template, {multiple: false});
const imgElProto = await f.findByImage(template, d, {multiple: false});
basicImgElVerify(imgElProto, f);
});
it('should find image elements happypath', async function () {
compareStub.resolves([{rect, score}]);
const els = await f.findByImage(template, {multiple: true});
const els = await f.findByImage(template, d, {multiple: true});
els.should.have.length(1);
basicImgElVerify(els[0], f);
});
it('should fail if driver does not support getWindowSize', async function () {
d.getWindowSize = null;
it('should fail if driver does not support getWindowRect', async function () {
d.getWindowRect = null;
await f
.findByImage(template, {multiple: false})
.findByImage(template, d, {multiple: false})
.should.eventually.be.rejectedWith(/driver does not support/);
});
it('should fix template size if requested', async function () {
const newTemplate = 'iVBORbaz';
await d.settings.update({fixImageTemplateSize: true});
sandbox.stub(f, 'ensureTemplateSize').returns(newTemplate);
const imgElProto = await f.findByImage(template, {multiple: false});
const imgElProto = await f.findByImage(template, d, {multiple: false});
const imgEl = basicImgElVerify(imgElProto, f);
imgEl.template.should.eql(newTemplate);
_.last(compareStub.args)[2].should.eql(newTemplate);
@@ -120,7 +125,7 @@ describe('finding elements by image', function () {
const newTemplate = 'iVBORbaz';
await d.settings.update({fixImageTemplateScale: true});
sandbox.stub(f, 'fixImageTemplateScale').returns(newTemplate);
const imgElProto = await f.findByImage(template, {multiple: false});
const imgElProto = await f.findByImage(template, d, {multiple: false});
const imgEl = basicImgElVerify(imgElProto, f);
imgEl.template.should.eql(newTemplate);
_.last(compareStub.args)[2].should.eql(newTemplate);
@@ -135,24 +140,24 @@ describe('finding elements by image', function () {
it('should throw an error if template match fails', async function () {
compareStub.rejects(new Error('Cannot find any occurrences'));
await f
.findByImage(template, {multiple: false})
.findByImage(template, d, {multiple: false})
.should.be.rejectedWith(/element could not be located/);
});
it('should return empty array for multiple elements if template match fails', async function () {
compareStub.rejects(new Error('Cannot find any occurrences'));
await f.findByImage(template, {multiple: true}).should.eventually.eql([]);
await f.findByImage(template, d, {multiple: true}).should.eventually.eql([]);
});
it('should respect implicit wait', async function () {
d.setImplicitWait(10);
compareStub.resetHistory();
compareStub.returns({rect, score});
compareStub.onFirstCall().throws(new Error('Cannot find any occurrences'));
const imgElProto = await f.findByImage(template, {multiple: false});
const imgElProto = await f.findByImage(template, d, {multiple: false});
basicImgElVerify(imgElProto, f);
compareStub.should.have.been.calledTwice;
});
it('should not add element to cache and return it directly when checking staleness', async function () {
const imgEl = await f.findByImage(template, {
const imgEl = await f.findByImage(template, d, {
multiple: false,
shouldCheckStaleness: true,
});
@@ -163,13 +168,11 @@ describe('finding elements by image', function () {
});
describe('fixImageTemplateScale', function () {
let d;
let f;
const basicTemplate = 'iVBORbaz';
beforeEach(function () {
d = new PluginDriver();
f = new ImageElementFinder(d);
f = new ImageElementFinder();
});
it('should not fix template size scale if no scale value', async function () {
@@ -271,19 +274,19 @@ describe('finding elements by image', function () {
});
describe('ensureTemplateSize', function () {
const d = new PluginDriver();
const f = new ImageElementFinder(d);
const f = new ImageElementFinder();
it('should not resize the template if it is smaller than the screen', async function () {
const screen = TINY_PNG_DIMS.map((n) => n * 2);
await f.ensureTemplateSize(TINY_PNG, ...screen).should.eventually.eql(TINY_PNG);
const [width, height] = TINY_PNG_DIMS.map((n) => n * 2);
await f.ensureTemplateSize(TINY_PNG, {width, height}).should.eventually.eql(TINY_PNG);
});
it('should not resize the template if it is the same size as the screen', async function () {
await f.ensureTemplateSize(TINY_PNG, ...TINY_PNG_DIMS).should.eventually.eql(TINY_PNG);
const [width, height] = TINY_PNG_DIMS;
await f.ensureTemplateSize(TINY_PNG, {width, height}).should.eventually.eql(TINY_PNG);
});
it('should resize the template if it is bigger than the screen', async function () {
const screen = TINY_PNG_DIMS.map((n) => n / 2);
const newTemplate = await f.ensureTemplateSize(TINY_PNG, ...screen);
const [width, height] = TINY_PNG_DIMS.map((n) => n / 2);
const newTemplate = await f.ensureTemplateSize(TINY_PNG, {width, height});
newTemplate.should.not.eql(TINY_PNG);
newTemplate.length.should.be.below(TINY_PNG.length);
});
@@ -295,76 +298,73 @@ describe('finding elements by image', function () {
beforeEach(function () {
d = new PluginDriver();
f = new ImageElementFinder(d);
f = new ImageElementFinder();
sandbox.stub(d, 'getScreenshot').returns(TINY_PNG);
});
it('should fail if driver does not support getScreenshot', async function () {
const d = new BaseDriver();
const f = new ImageElementFinder(d);
await f
.getScreenshotForImageFind()
await new ImageElementFinder()
.getScreenshotForImageFind(new BaseDriver())
.should.eventually.be.rejectedWith(/driver does not support/);
});
it('should not adjust or verify screenshot if asked not to by settings', async function () {
await d.settings.update({fixImageFindScreenshotDims: false});
const screen = TINY_PNG_DIMS.map((n) => n + 1);
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);
const [width, height] = TINY_PNG_DIMS.map((n) => n + 1);
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
b64Screenshot.should.eql(TINY_PNG);
should.equal(scale, undefined);
});
it('should return screenshot without adjustment if it matches screen size', async function () {
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...TINY_PNG_DIMS);
const [width, height] = TINY_PNG_DIMS;
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
b64Screenshot.should.eql(TINY_PNG);
should.equal(scale, undefined);
});
it('should return scaled screenshot with same aspect ratio if matching screen aspect ratio', async function () {
const screen = TINY_PNG_DIMS.map((n) => n * 1.5);
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);
const [width, height] = TINY_PNG_DIMS.map((n) => n * 1.5);
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
b64Screenshot.should.not.eql(TINY_PNG);
const screenshotObj = await imageUtil.getJimpImage(b64Screenshot);
screenshotObj.bitmap.width.should.eql(screen[0]);
screenshotObj.bitmap.height.should.eql(screen[1]);
screenshotObj.bitmap.width.should.eql(width);
screenshotObj.bitmap.height.should.eql(height);
scale.should.eql({xScale: 1.5, yScale: 1.5});
});
it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio', async function () {
// try first with portrait screen, screen = 8 x 12
let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
let [width, height] = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
let expectedScale = {xScale: 2.67, yScale: 4};
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
b64Screenshot.should.not.eql(TINY_PNG);
let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);
screenshotObj.bitmap.width.should.eql(screen[0]);
screenshotObj.bitmap.height.should.eql(screen[1]);
screenshotObj.bitmap.width.should.eql(width);
screenshotObj.bitmap.height.should.eql(height);
scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());
scale.yScale.should.eql(expectedScale.yScale);
// then with landscape screen, screen = 12 x 8
screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
[width, height] = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
expectedScale = {xScale: 4, yScale: 2.67};
const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(
...screen
);
const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(d, {width, height});
newScreen.should.not.eql(TINY_PNG);
screenshotObj = await imageUtil.getJimpImage(newScreen);
screenshotObj.bitmap.width.should.eql(screen[0]);
screenshotObj.bitmap.height.should.eql(screen[1]);
screenshotObj.bitmap.width.should.eql(width);
screenshotObj.bitmap.height.should.eql(height);
newScale.xScale.should.eql(expectedScale.xScale);
newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());
});
it('should return scaled screenshot with different aspect ratio if not matching screen aspect ratio with fixImageTemplateScale', async function () {
// try first with portrait screen, screen = 8 x 12
let screen = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
let [width, height] = [TINY_PNG_DIMS[0] * 2, TINY_PNG_DIMS[1] * 3];
let expectedScale = {xScale: 2.67, yScale: 4};
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(...screen);
const {b64Screenshot, scale} = await f.getScreenshotForImageFind(d, {width, height});
b64Screenshot.should.not.eql(TINY_PNG);
let screenshotObj = await imageUtil.getJimpImage(b64Screenshot);
screenshotObj.bitmap.width.should.eql(screen[0]);
screenshotObj.bitmap.height.should.eql(screen[1]);
screenshotObj.bitmap.width.should.eql(width);
screenshotObj.bitmap.height.should.eql(height);
scale.xScale.toFixed(2).should.eql(expectedScale.xScale.toString());
scale.yScale.should.eql(expectedScale.yScale);
// 8 x 12 stretched TINY_PNG
@@ -378,16 +378,14 @@ describe('finding elements by image', function () {
);
// then with landscape screen, screen = 12 x 8
screen = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
[width, height] = [TINY_PNG_DIMS[0] * 3, TINY_PNG_DIMS[1] * 2];
expectedScale = {xScale: 4, yScale: 2.67};
const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(
...screen
);
const {b64Screenshot: newScreen, scale: newScale} = await f.getScreenshotForImageFind(d, {width, height});
newScreen.should.not.eql(TINY_PNG);
screenshotObj = await imageUtil.getJimpImage(newScreen);
screenshotObj.bitmap.width.should.eql(screen[0]);
screenshotObj.bitmap.height.should.eql(screen[1]);
screenshotObj.bitmap.width.should.eql(width);
screenshotObj.bitmap.height.should.eql(height);
newScale.xScale.should.eql(expectedScale.xScale);
newScale.yScale.toFixed(2).should.eql(expectedScale.yScale.toString());
// 12 x 8 stretched TINY_PNG

14
packages/images-plugin/test/unit/image-element.spec.js
View File

@@ -1,9 +1,11 @@
import _ from 'lodash';
import BaseDriver from 'appium/driver';
import {util} from 'appium/support';
import ImageElementFinder from '../../lib/finder';
import {getImgElFromArgs} from '../../lib/plugin';
import ImageElement, {IMAGE_ELEMENT_PREFIX} from '../../lib/image-element';
import ImageElement from '../../lib/image-element';
import sinon from 'sinon';
import {IMAGE_ELEMENT_PREFIX} from '../../lib/constants';
const defRect = {x: 100, y: 110, width: 50, height: 25};
const defTemplate = 'iVBORasdf';
@@ -48,7 +50,7 @@ describe('ImageElement', function () {
describe('.asElement', function () {
it('should get the webdriver object representation of the element', function () {
const el = new ImageElement(defTemplate, defRect);
el.asElement('ELEMENT').ELEMENT.should.match(/^appium-image-el/);
util.unwrapElement(el.asElement()).should.match(/^appium-image-el/);
});
});
@@ -76,7 +78,7 @@ describe('ImageElement', function () {
});
it('should try to check for image element staleness, and throw if stale', async function () {
const d = new BaseDriver();
const f = new ImageElementFinder(d);
const f = new ImageElementFinder();
sandbox.stub(f, 'findByImage').throws();
const el = new ImageElement(defTemplate, defRect, null, null, f);
// we need to check for staleness if explicitly requested to do so
@@ -97,7 +99,7 @@ describe('ImageElement', function () {
const d = new BaseDriver();
d.performActions = _.noop;
sandbox.stub(d, 'performActions');
const f = new ImageElementFinder(d);
const f = new ImageElementFinder();
const el = new ImageElement(defTemplate, defRect, null, null, f);
const newRect = {...defRect, x: defRect.x + 10, y: defRect.y + 5};
const elPos2 = new ImageElement(defTemplate, newRect, null, null, f);
@@ -168,7 +170,7 @@ describe('ImageElement', function () {
describe('#execute', function () {
// aGFwcHkgdGVzdGluZw== is 'happy testing'
const f = new ImageElementFinder(driver);
const f = new ImageElementFinder();
const imgEl = new ImageElement(defTemplate, defRect, 0, 'aGFwcHkgdGVzdGluZw==', f);
let clickStub;
@@ -256,7 +258,7 @@ describe('image element LRU cache', function () {
it('once cache reaches max size, should eject image elements', function () {
const el1 = new ImageElement(defTemplate, defRect);
const el2 = new ImageElement(defTemplate, defRect);
const cache = new ImageElementFinder(null, defTemplate.length + 1).imgElCache;
const cache = new ImageElementFinder(defTemplate.length + 1).imgElCache;
cache.set(el1.id, el1);
cache.has(el1.id).should.be.true;
cache.set(el2.id, el2);

20
packages/images-plugin/test/unit/plugin.spec.js
View File

@@ -1,8 +1,10 @@
import {ImageElementPlugin, IMAGE_STRATEGY} from '../../lib/plugin';
import {MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE} from '../../lib/compare';
import {ImageElementPlugin} from '../../lib/plugin';
import {
MATCH_FEATURES_MODE, GET_SIMILARITY_MODE, MATCH_TEMPLATE_MODE, IMAGE_STRATEGY
} from '../../lib/constants';
import BaseDriver from 'appium/driver';
import {W3C_ELEMENT_KEY} from '../../lib/finder';
import {TEST_IMG_1_B64, TEST_IMG_2_B64, TEST_IMG_2_PART_B64} from '../fixtures';
import { util } from '@appium/support';
describe('ImageElementPlugin#handle', function () {
const next = () => {};
@@ -56,7 +58,7 @@ describe('ImageElementPlugin#handle', function () {
driver.settings = {getSettings: () => ({})};
driver.isW3CProtocol = () => true;
driver.getScreenshot = () => TEST_IMG_2_B64;
driver.getWindowSize = () => ({width: 64, height: 64});
driver.getWindowRect = () => ({x: 0, y: 0, width: 64, height: 64});
it('should defer execution to regular command if not a find command', async function () {
const next = () => true;
await p.handle(next, driver, 'sendKeys').should.eventually.become(true);
@@ -68,12 +70,12 @@ describe('ImageElementPlugin#handle', function () {
});
it('should find an image element inside a screenshot', async function () {
const el = await p.findElement(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
el[W3C_ELEMENT_KEY].should.include('appium-image-element');
util.unwrapElement(el).should.include('appium-image-element');
});
it('should find image elements inside a screenshot', async function () {
const els = await p.findElements(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
els.should.have.length(1);
els[0][W3C_ELEMENT_KEY].should.include('appium-image-element');
util.unwrapElement(els[0]).should.include('appium-image-element');
});
});
@@ -83,9 +85,9 @@ describe('ImageElementPlugin#handle', function () {
driver.settings = {getSettings: () => ({})};
driver.isW3CProtocol = () => true;
driver.getScreenshot = () => TEST_IMG_2_B64;
driver.getWindowSize = () => ({width: 64, height: 64});
driver.getWindowRect = () => ({x: 0, y: 0, width: 64, height: 64});
const el = await p.findElement(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
elId = el[W3C_ELEMENT_KEY];
elId = util.unwrapElement(el);
});
it('should click on the screen coords of the middle of the element', async function () {
let action = null;
@@ -140,7 +142,7 @@ describe('ImageElementPlugin#handle', function () {
}),
};
const el = await p.findElement(next, driver, IMAGE_STRATEGY, TEST_IMG_2_PART_B64);
elId = el[W3C_ELEMENT_KEY];
elId = util.unwrapElement(el);
await p
.handle(next, driver, 'getAttribute', 'visual', elId)
.should.eventually.include('iVBOR');

63
packages/opencv/lib/index.js
View File

@@ -1,7 +1,7 @@
import _ from 'lodash';
import Jimp from 'jimp';
import {Buffer} from 'buffer';
import B from 'bluebird';
import sharp from 'sharp';
/** @type {any} */
let cv;
@@ -335,7 +335,7 @@ async function getImagesMatches(img1Data, img2Data, options = {}) {
width: rect2.width,
height: rect2.height,
});
result.visualization = await jimpImgFromCvMat(visualization).getBufferAsync(Jimp.MIME_PNG);
result.visualization = await cvMatToPng(visualization);
}
return result;
@@ -442,7 +442,7 @@ async function getImagesSimilarity(img1Data, img2Data, options = {}) {
height: boundingRect.height,
});
}
result.visualization = await jimpImgFromCvMat(resultMat).getBufferAsync(Jimp.MIME_PNG);
result.visualization = await cvMatToPng(resultMat);
} finally {
try {
bothImages.delete();
@@ -527,10 +527,7 @@ async function getImageOccurrence(fullImgData, partialImgData, options = {}) {
let fullImg, partialImg, matched;
try {
[fullImg, partialImg] = await B.all([
cvMatFromImage(fullImgData),
cvMatFromImage(partialImgData),
]);
[fullImg, partialImg] = await B.all([cvMatFromImage(fullImgData), cvMatFromImage(partialImgData)]);
matched = new cv.Mat();
const results = [];
let visualization = null;
@@ -579,7 +576,7 @@ async function getImageOccurrence(fullImgData, partialImgData, options = {}) {
if (_.isEmpty(results)) {
// Below error message, `Cannot find any occurrences` is referenced in find by image
throw new Error(
`Match threshold: ${threshold}. Highest match value ` + `found was ${minMax.maxVal}`
`Match threshold: ${threshold}. Highest match value found was ${minMax.maxVal}`
);
}
} catch (e) {
@@ -593,16 +590,21 @@ async function getImageOccurrence(fullImgData, partialImgData, options = {}) {
if (visualize) {
const fullHighlightedImage = fullImg.clone();
const visualisePromises = [];
for (const result of results) {
const singleHighlightedImage = fullImg.clone();
highlightRegion(singleHighlightedImage, result.rect);
highlightRegion(fullHighlightedImage, result.rect);
result.visualization = await jimpImgFromCvMat(singleHighlightedImage).getBufferAsync(
Jimp.MIME_PNG
);
visualisePromises.push(cvMatToPng(singleHighlightedImage));
}
let restPngBuffers = [];
[visualization, ...restPngBuffers] = await B.all(
[cvMatToPng(fullHighlightedImage), ...visualisePromises]
);
for (const [result, pngBuffer] of _.zip(results, restPngBuffers)) {
result.visualization = pngBuffer;
}
visualization = await jimpImgFromCvMat(fullHighlightedImage).getBufferAsync(Jimp.MIME_PNG);
}
return {
rect: results[0].rect,
@@ -620,28 +622,37 @@ async function getImageOccurrence(fullImgData, partialImgData, options = {}) {
}
/**
* Convert an opencv image matrix into a Jimp image object
* Convert an opencv image matrix into a PNG buffer
*
* @param {cv.Mat} mat the image matrix
* @return {Jimp} the Jimp image
* @param {cv.Mat} mat OpenCV image matrix
* @return {Promise<Buffer>} PNG image data buffer
*/
function jimpImgFromCvMat(mat) {
return new Jimp({
width: mat.cols,
height: mat.rows,
data: Buffer.from(mat.data),
});
async function cvMatToPng(mat) {
return await sharp(Buffer.from(mat.data), {
raw: {
width: mat.cols,
height: mat.rows,
channels: 4,
}
})
.png()
.toBuffer();
}
/**
* Take a binary image buffer and return a cv.Mat
* Take an image buffer and return a cv.Mat
*
* @param {Buffer} img the image data buffer
* @return {cv.Mat} the opencv matrix
* @param {Buffer} img image data buffer. All image formats avilable for
* https://www.npmjs.com/package/sharp node library are supported.
* @return {Promise<cv.Mat>} OpenCV image matrix
*/
async function cvMatFromImage(img) {
const jimpImg = await Jimp.read(img);
return cv.matFromImageData(jimpImg.bitmap);
const {data, info} = await sharp(img)
.ensureAlpha()
.raw()
.toBuffer({resolveWithObject: true});
const {width, height} = info;
return cv.matFromImageData({data, width, height});
}
/**

2
packages/opencv/package.json
View File

@@ -43,9 +43,9 @@
"dependencies": {
"@types/bluebird": "3.5.38",
"bluebird": "3.7.2",
"jimp": "0.22.7",
"lodash": "4.17.21",
"opencv-bindings": "4.5.5",
"sharp": "0.32.0",
"source-map-support": "0.5.21"
},
"engines": {