test: Add support for pixel tests

This commit is contained in:
Marius Vollmer 2021-04-07 12:30:11 +03:00 committed by Martin Pitt
parent dfa14b9b1e
commit b0c8efbd3e
5 changed files with 851 additions and 2 deletions

View file

@ -0,0 +1,24 @@
[
{
"label": "screenshot",
"pattern": "Wrote screenshot to ([A-Za-z0-9\\-\\.]+\\.png)$",
"url": "$1",
"icon": "bi bi-camera-fill"
},
{
"label": "new pixels",
"pattern": "New pixel test reference ([A-Za-z0-9\\-\\.]+\\.png)$",
"url": "$1"
},
{
"label": "journal",
"pattern": "Journal extracted to ([A-Za-z0-9\\-\\.]+\\.log(?:\\.[gx]z)?)$",
"url": "$1",
"icon": "bi bi-card-text"
},
{
"label": "changed pixels",
"pattern": "Differences in pixel test ([A-Za-z0-9\\-\\.]+)$",
"url": "pixeldiff.html#$1"
}
]

68
test/common/pixel-tests Executable file
View file

@ -0,0 +1,68 @@
#!/bin/bash
set -eu
SUBDIR=test/reference
REPO=pixel-test-reference
GITHUB_BASE="${GITHUB_BASE:-cockpit-project/cockpit}"
GITHUB_REPOSITORY="${GITHUB_BASE%/*}/${REPO}"
CLONE_REMOTE="https://github.com/${GITHUB_REPOSITORY}"
PUSH_REMOTE="git@github.com:${GITHUB_REPOSITORY}"
message() {
[ "${V-}" != 0 ] || printf " %-8s %s\n" "$1" "$2"
}
cmd_init() {
git submodule add -b empty "$CLONE_REMOTE" "$SUBDIR"
}
cmd_pull() {
git submodule update --init -- "$SUBDIR"
}
cmd_status() {
( cd "$SUBDIR"
git rm --cached --quiet '*.png'
git add *.png
if git diff-index --name-status --cached --exit-code HEAD; then
echo No changes
fi
)
}
cmd_push() {
( cd "$SUBDIR"
git rm --cached --quiet '*.png'
git add *.png
if ! git diff-index --name-status --cached --exit-code HEAD; then
git reset --soft empty
git commit --quiet -m "$(date)"
tag="sha-$(git rev-parse HEAD)"
git tag "$tag" HEAD
git push "$PUSH_REMOTE" "$tag"
else
echo No changes
fi
)
git add "$SUBDIR"
}
main() {
local cmd="${1-}"
if [ -z "${cmd}" ]; then
echo 'This command requires a subcommand: init pull status push'
exit 1
elif ! type -t "cmd_${cmd}" | grep -q function; then
echo "Unknown subcommand ${cmd}"
exit 1
fi
shift
[ "${V-0}" = 0 ] || set -x
"cmd_$cmd" "$@"
}
main "$@"

598
test/common/pixeldiff.html Normal file
View file

@ -0,0 +1,598 @@
<!DOCTYPE html>
<html>
<head>
<title>Cockpit Integration Tests - Pixel diffs</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/1.12.4/jquery.min.js" type="text/javascript"></script>
<style>
/*
MIT License
Copyright (c) 2018 Diamant Haxhimusa
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
.pixelcompare-wrapper.pixelcompare-horizontal {
height: 100% !important;
}
.pixelcompare-horizontal .pixelcompare-handle:before, .pixelcompare-horizontal .pixelcompare-handle:after, .pixelcompare-vertical .pixelcompare-handle:before, .pixelcompare-vertical .pixelcompare-handle:after {
content: "";
display: block;
background: #fff;
position: absolute;
z-index: 30;
}
img.pixelcompare-before, img.pixelcompare-after {
height: 100%;
object-fit: cover;
-webkit-user-select: none;
-khtml-user-select: none;
-moz-user-select: none;
-o-user-select: none;
user-select: none;
}
.pixelcompare-before-label, .pixelcompare-after-label, .pixelcompare-overlay {
position: absolute;
top: 0;
width: 100%;
height: 100%;
}
.pixelcompare-before-label, .pixelcompare-after-label, .pixelcompare-overlay {
transition-duration: 0.5s;
}
.pixelcompare-before-label, .pixelcompare-after-label {
transition-property: opacity;
}
.pixelcompare-horizontal .pixelcompare-before-label:before, .pixelcompare-horizontal .pixelcompare-after-label:before {
top: 50%;
margin-top: -19px;
}
.pixelcompare-vertical .pixelcompare-before-label:before, .pixelcompare-vertical .pixelcompare-after-label:before {
left: 50%;
margin-left: -45px;
text-align: center;
width: 90px;
}
.pixelcompare-left-arrow, .pixelcompare-right-arrow, .pixelcompare-up-arrow, .pixelcompare-down-arrow {
width: 0;
height: 0;
border: 6px inset transparent;
position: absolute;
}
.pixelcompare-left-arrow, .pixelcompare-right-arrow {
top: 50%;
margin-top: -6px;
}
.pixelcompare-up-arrow, .pixelcompare-down-arrow {
left: 50%;
margin-left: -6px;
}
.pixelcompare-container {
box-sizing: content-box;
z-index: 0;
height: 100%;
overflow: hidden;
position: relative;
-webkit-user-select: none;
-khtml-user-select: none;
-moz-user-select: none;
-o-user-select: none;
user-select: none;
}
.pixelcompare-container img {
max-width: 100%;
position: absolute;
top: 0;
display: block;
}
/* .pixelcompare-container.active .pixelcompare-overlay,
.pixelcompare-container.active :hover.pixelcompare-overlay {
background: transparent;
} */
.pixelcompare-container.active .pixelcompare-overlay .pixelcompare-before-label,
.pixelcompare-container.active .pixelcompare-overlay .pixelcompare-after-label,
.pixelcompare-container.active :hover.pixelcompare-overlay .pixelcompare-before-label,
.pixelcompare-container.active :hover.pixelcompare-overlay .pixelcompare-after-label {
opacity: 0;
}
.pixelcompare-container * {
box-sizing: content-box;
}
.pixelcompare-before-label {
opacity: 0;
}
.pixelcompare-before-label:before {
content: attr(data-content);
}
.pixelcompare-after-label {
opacity: 0;
}
.pixelcompare-after-label:before {
content: attr(data-content);
}
.pixelcompare-horizontal .pixelcompare-before-label:before {
left: 10px;
}
.pixelcompare-horizontal .pixelcompare-after-label:before {
right: 10px;
}
.pixelcompare-vertical .pixelcompare-before-label:before {
top: 10px;
}
.pixelcompare-vertical .pixelcompare-after-label:before {
bottom: 10px;
}
.pixelcompare-overlay {
transition-property: background;
background: transparent;
z-index: 25;
}
.pixelcompare-overlay:hover,
.pixelcompare-container.active .pixelcompare-overlay,
.pixelcompare-handle:hover + .pixelcompare-overlay {
background: rgba(0, 0, 0, 0.5);
}
.pixelcompare-overlay:hover .pixelcompare-after-label {
opacity: 1;
}
.pixelcompare-overlay:hover .pixelcompare-before-label {
opacity: 1;
}
.pixelcompare-before {
z-index: 20;
}
.pixelcompare-after {
z-index: 10;
}
.pixelcompare-vertical .pixelcompare-handle {
background-color: rgba(247, 237, 237, 0.71);
height: 20px;
width: 65px;
position: absolute;
left: 50%;
top: 50%;
margin-left: -19px;
margin-top: -10px;
border-radius: 1000px;
box-shadow: 0px 0px 12px rgba(51, 51, 51, 0.5);
z-index: 40;
cursor: pointer;
}
.pixelcompare-horizontal .pixelcompare-handle {
background-color: rgba(247, 237, 237, 0.71);
height: 65px;
width: 20px;
position: absolute;
left: 50%;
top: 50%;
margin-top: -19px;
margin-left: -10px;
border-radius: 1000px;
box-shadow: 0px 0px 12px rgba(51, 51, 51, 0.5);
z-index: 40;
cursor: pointer;
}
.pixelcompare-horizontal .pixelcompare-handle:before {
bottom: 50%;
height: 799px;
margin-bottom: 33px;
margin-left: 10px;
background: none;
position: absolute;
z-index: 999;
border-right: 1px dashed rgba(255, 255, 255, 0.5);
}
.pixelcompare-horizontal .pixelcompare-handle:after {
top: 50%;
height: 257px;
margin-top: 33px;
margin-left: 10px;
background: none;
position: absolute;
z-index: 999;
border-right: 1px dashed rgba(255, 255, 255, 0.5);
}
.pixelcompare-vertical .pixelcompare-handle:before {
margin-top: 10px;
margin-left: 11px;
width: 100vw;
background: none;
position: absolute;
z-index: 999;
border-top: 1px dashed rgba(255, 255, 255, 0.5);
right: 50%;
margin-right: 33px;
}
.pixelcompare-vertical .pixelcompare-handle:after {
margin-top: 10px;
width: 100vw;
background: none;
position: absolute;
z-index: 999;
border-top: 1px dashed rgba(255, 255, 255, 0.5);
left: 50%;
margin-left: 33px;
}
.pixelcompare-left-arrow {
border-right: 6px solid rgba(247, 237, 237, 0.71);
left: 50%;
margin-left: -13px;
}
.pixelcompare-right-arrow {
border-left: 6px solid rgba(247, 237, 237, 0.71);
right: 50%;
margin-right: -13px;
}
.pixelcompare-up-arrow {
border-bottom: 6px solid rgba(247, 237, 237, 0.71);
top: 50%;
margin-top: -13px;
}
.pixelcompare-down-arrow {
border-top: 6px solid rgba(247, 237, 237, 0.71);
bottom: 50%;
margin-bottom: -13px;
}
</style>
<script>
"use strict";
/**
* PIXELCOMPARE
* Javascript image comparison
* @author diamanthaxhimusa@gmail.com
MIT License
Copyright (c) 2018 Diamant Haxhimusa
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
window.setup_pixelcompare = function() {
var _extend = function(defaults, options) {
var extended = {};
var prop;
for (prop in defaults) {
if(Object.prototype.hasOwnProperty.call(defaults, prop)) {
extended[prop] = defaults[prop];
}
}
for (prop in options) {
if(Object.prototype.hasOwnProperty.call(options, prop)) {
extended[prop] = options[prop];
}
}
return extended;
};
var _addClass = function(element, classname) {
var arr;
arr = element.className.split(" ");
if(arr.indexOf(classname) == -1) {
element.className += " " + classname;
}
};
var _hasClass = function(element, className) {
return new RegExp("(\\s|^)" + className + "(\\s|$)").test(
element.className
);
};
var _removeClass = function(element, className) {
element.classList.remove(className);
};
var _wrap = function(element, tag, sliderOrientation) {
var div = document.createElement(tag);
_addClass(div, "pixelcompare-wrapper pixelcompare-" + sliderOrientation);
element.parentElement.insertBefore(div, element);
div.appendChild(element);
return div;
};
var _createSlider = function(beforeDirection, afterDirection) {
var pxcHandleNode = document.createElement("div");
_addClass(pxcHandleNode, "pixelcompare-handle");
var sliderChildNodeBeforeDirection = document.createElement("span");
_addClass(
sliderChildNodeBeforeDirection,
"pixelcompare-" + beforeDirection + "-arrow"
);
pxcHandleNode.appendChild(sliderChildNodeBeforeDirection);
var sliderChildNodeAfterDirection = document.createElement("span");
_addClass(
sliderChildNodeAfterDirection,
"pixelcompare-" + afterDirection + "-arrow"
);
pxcHandleNode.appendChild(sliderChildNodeAfterDirection);
pxcHandleNode.addEventListener("touchmove", function(e) {
e.preventDefault();
});
return pxcHandleNode;
};
var options = {
default_offset_pct: 0.5,
orientation: "horizontal",
overlay: false,
hover: false,
move_with_handle_only: true,
click_to_move: false,
showSlider: true,
};
var pxcContainers = document.querySelectorAll("[data-pixelcompare]");
pxcContainers.forEach(function(pcContainer) {
var sliderPct = options.default_offset_pct;
var imageContainer = pcContainer;
options.hover = pcContainer.hasAttribute("data-hover");
options.showSlider = options.hover ? pcContainer.hasAttribute("data-show-slider") : options.showSlider;
options.orientation = pcContainer.hasAttribute("data-vertical")
? "vertical"
: "horizontal";
var orientations = ["vertical", "horizontal", "sides"];
var datasetOrientation = imageContainer.dataset.pixelcompareOrientation;
var sliderOrientation = orientations.includes(datasetOrientation)
? datasetOrientation
: options.orientation;
var beforeDirection = sliderOrientation === "vertical" ? "down" : "left";
var afterDirection = sliderOrientation === "vertical" ? "up" : "right";
var container = _wrap(pcContainer, "div", sliderOrientation);
var beforeImg = container.querySelectorAll("img")[0];
beforeImg.draggable = false;
var afterImg = container.querySelectorAll("img")[1];
afterImg.draggable = false;
var bounding_rect = beforeImg.getBoundingClientRect();
_addClass(container, "pixelcompare-container");
_addClass(beforeImg, "pixelcompare-before");
_addClass(afterImg, "pixelcompare-after");
var slider = null
if(options.showSlider) {
slider = _createSlider(beforeDirection, afterDirection);
container.appendChild(slider);
}
if(options.overlay) {
var overlayNode = document.createElement("div");
_addClass(overlayNode, "pixelcompare-overlay");
container.appendChild(overlayNode);
}
var calcOffset = function(dimensionPct) {
var w = beforeImg.getBoundingClientRect().width;
var h = beforeImg.getBoundingClientRect().height;
return {
w: w + "px",
h: h + "px",
wp: dimensionPct * 100,
cw: dimensionPct * w + "px",
ch: dimensionPct * h + "px",
};
};
var adjustContainer = function(offset) {
if(sliderOrientation === "vertical") {
beforeImg.style.clip =
"rect(0, " + offset.w + ", " + offset.ch + ", 0)";
afterImg.style.clip =
"rect(" + offset.ch + ", " + offset.w + ", " + offset.h + ", 0)";
} else if(sliderOrientation === "sides") {
beforeImg.style.clipPath = `polygon(0% ${2 * (50 - offset.wp)}%, ${
2 * offset.wp
}% 100%, 0% 100%)`;
afterImg.style.clipPath = `polygon(100% ${2 * (100 - offset.wp)}%, ${
-2 * (50 - offset.wp)
}% 0%, 100% 0%)`;
} else {
beforeImg.style.clip =
"rect(0, " + offset.cw + ", " + offset.h + ", 0)";
afterImg.style.clip =
"rect(0, " + offset.w + "," + offset.h + "," + offset.cw + ")";
}
container.style.height = offset.h;
};
var adjustSlider = function(pct) {
var offset = calcOffset(pct);
if(slider) {
if(sliderOrientation === "vertical") {
slider.style.top = offset.ch;
} else {
slider.style.left = offset.cw;
}
}
adjustContainer(offset);
};
// Return the number specified or the min/max number if it outside the range given.
var minMaxNumber = function(num, min, max) {
return Math.max(min, Math.min(max, num));
};
// Calculate the slider percentage based on the position.
var getSliderPercentage = function(positionX, positionY) {
var sliderPercentage =
sliderOrientation === "vertical"
? (positionY - offsetY) / imgHeight
: (positionX - offsetX) / imgWidth;
return minMaxNumber(sliderPercentage, 0, 1);
};
window.addEventListener("resize.pixelcompare", function(e) {
adjustSlider(sliderPct);
});
var offsetX = 0;
var offsetY = 0;
var imgWidth = 0;
var imgHeight = 0;
var onMoveStart = function(e) {
if(
((e.distX > e.distY && e.distX < -e.distY) ||
(e.distX < e.distY && e.distX > -e.distY)) &&
sliderOrientation !== "vertical"
) {
e.preventDefault();
} else if(
((e.distX < e.distY && e.distX < -e.distY) ||
(e.distX > e.distY && e.distX > -e.distY)) &&
sliderOrientation === "vertical"
) {
e.preventDefault();
}
_addClass(container, "active");
offsetX = container.offsetLeft;
offsetY = container.offsetTop;
imgWidth = beforeImg.getBoundingClientRect().width;
imgHeight = beforeImg.getBoundingClientRect().height;
};
var onMove = function(e) {
if(_hasClass(container, "active")) {
sliderPct = getSliderPercentage(
e.pageX || e.changedTouches[0].pageX,
e.pageY || e.changedTouches[0].pageY
);
adjustSlider(sliderPct);
}
};
var onMoveEnd = function() {
_removeClass(container, "active");
};
if(options.hover) {
container.addEventListener("mouseenter", onMoveStart);
container.addEventListener("mouseleave", onMoveEnd);
container.addEventListener("mousemove", onMove);
} else {
var moveTarget = options.move_with_handle_only ? slider : container;
window.addEventListener("mouseup", onMoveEnd);
container.addEventListener("mousemove", onMove);
moveTarget.addEventListener("mousedown", onMoveStart);
moveTarget.addEventListener("touchstart", onMoveStart);
container.addEventListener("touchmove", onMove);
window.addEventListener("touchend", onMoveEnd);
}
container
.querySelector("img")
.addEventListener("mousedown", function(event) {
event.preventDefault();
});
if(options.click_to_move) {
container.on("click", function(e) {
offsetX = container.offset().left;
offsetY = container.offset().top;
imgWidth = beforeImg.width();
imgHeight = beforeImg.height();
sliderPct = getSliderPercentage(e.pageX, e.pageY);
adjustSlider(sliderPct);
});
}
window.dispatchEvent(new Event("resize.pixelcompare"));
});
};
</script>
<script>
$(function () {
var base = window.location.hash.replace("#", "");
$('#key').text(base);
$('#new').text(base + "-pixels.png").attr("href", base + "-pixels.png");
var img_now = $("<img>", { src: base + "-pixels.png", width: "100%" });
var img_ref = $("<img>", { src: base + "-reference.png", width: "100%" });
var img_delta = $("<img>", { src: base + "-delta.png", width: "100%" });
img_now.on("load", function () {
var diff = $('#diff')[0];
diff.style.width = img_now[0].naturalWidth + "px";
diff.style.height = img_now[0].naturalHeight + "px";
$(diff).append(
$("<div class=pixelcompare data-pixelcompare>").append(img_now, img_ref));
window.setup_pixelcompare();
var delta = $('#delta')[0];
delta.style.width = img_now[0].naturalWidth + "px";
delta.style.height = img_now[0].naturalHeight + "px";
$(delta).append(img_delta);
});
});
</script>
</head
<body>
<h2>Pixel comparison for <span id="key"></span></h2>
<p>New <a id="new"></a> on the left, reference on the right.</p>
<div id="diff">
</div>
<h2>Changed pixels in red, ignored changes in green</h2>
<div id="delta">
</div>
</body>
</html>

View file

@ -259,3 +259,38 @@ function ph_wait_cond(cond, timeout) {
step();
});
}
function currentFrameAbsolutePosition() {
let currentWindow = window;
let currentParentWindow;
let positions = [];
let rect;
while (currentWindow !== window.top) {
currentParentWindow = currentWindow.parent;
for (let idx = 0; idx < currentParentWindow.frames.length; idx++)
if (currentParentWindow.frames[idx] === currentWindow) {
for (let frameElement of currentParentWindow.document.getElementsByTagName('iframe')) {
if (frameElement.contentWindow === currentWindow) {
rect = frameElement.getBoundingClientRect();
positions.push({x: rect.x, y: rect.y});
}
}
currentWindow = currentParentWindow;
break;
}
}
return positions.reduce((accumulator, currentValue) => {
return {
x: accumulator.x + currentValue.x,
y: accumulator.y + currentValue.y
};
}, { x: 0, y: 0 });
}
function ph_element_clip(sel) {
var r = ph_find(sel).getBoundingClientRect();
var f = currentFrameAbsolutePosition();
return { x: r.x + f.x, y: r.y + f.y, width: r.width, height: r.height, scale: 1 };
}

View file

@ -37,11 +37,17 @@ import time
import unittest
import gzip
import inspect
from dataclasses import dataclass
import testvm
import cdp
from fmf_metadata.base import set_obj_attribute, is_test_function, generic_metadata_setter
try:
from PIL import Image
import io
except ImportError:
Image = None
TEST_DIR = os.path.normpath(os.path.dirname(os.path.realpath(os.path.join(__file__, ".."))))
BOTS_DIR = os.path.normpath(os.path.join(TEST_DIR, "..", "bots"))
@ -94,7 +100,7 @@ def attach(filename):
class Browser:
def __init__(self, address, label, port=None):
def __init__(self, address, label, pixels_label=None, port=None):
if ":" in address:
(self.address, unused, self.port) = address.rpartition(":")
else:
@ -104,11 +110,13 @@ class Browser:
self.port = port
self.default_user = "admin"
self.label = label
self.pixels_label = pixels_label
path = os.path.dirname(__file__)
self.cdp = cdp.CDP("C.utf8", verbose=opts.trace, trace=opts.trace,
inject_helpers=[os.path.join(path, "test-functions.js"), os.path.join(path, "sizzle.js")])
self.password = "foobar"
self.timeout_factor = int(os.getenv("TEST_TIMEOUT_FACTOR", "1"))
self.failed_pixel_tests = 0
def title(self):
return self.cdp.eval('document.title')
@ -649,6 +657,111 @@ class Browser:
attach(filename)
print("Wrote HTML dump to " + filename)
def assert_pixels(self, selector, key, ignore=[]):
"""Compare the given element with its reference"""
if not (Image and self.pixels_label and self.cdp and self.cdp.valid):
return
rect = self.call_js_func('ph_element_clip', selector)
@dataclass
class Rect:
x0: int
y0: int
x1: int
y1: int
def relative_clip(sel):
r = self.call_js_func('ph_element_clip', sel)
return Rect(r['x'] - rect['x'],
r['y'] - rect['y'],
r['x'] - rect['x'] + r['width'],
r['y'] - rect['y'] + r['height'])
ignore_rects = list(map(relative_clip, ignore))
base = self.pixels_label + "-" + key
filename = base + "-pixels.png"
ref_filename = os.path.join(TEST_DIR, "reference", filename)
ret = self.cdp.invoke("Page.captureScreenshot", clip=rect, no_trace=True)
png_now = base64.standard_b64decode(ret["data"])
png_ref = os.path.exists(ref_filename) and open(ref_filename, "rb").read()
if not png_ref:
with open(filename, 'wb') as f:
f.write(png_now)
attach(filename)
print("New pixel test reference " + filename)
self.failed_pixel_tests += 1
else:
img_now = Image.open(io.BytesIO(png_now)).convert("RGBA")
img_ref = Image.open(io.BytesIO(png_ref)).convert("RGBA")
img_delta = img_ref.copy()
# The current snapshot and the reference don't need to
# be perfectly identical. They might differ in the
# following ways:
#
# - A pixel in the reference image might be
# transparent. These pixels are ignored.
#
# - The call to assert_pixels specifies a list of
# rectangles (via CSS selectors). Pixels within those
# rectangles are ignored.
#
# - The RGB values of pixels can differ by up to 2.
#
# Pixels that are different but have been ignored are
# marked in the delta image in green.
def masked(ref):
return ref[3] != 255
def ignorable_coord(x, y):
for r in ignore_rects:
if r and x >= r.x0 and x < r.x1 and y >= r.y0 and y < r.y1:
return True
return False
def ignorable_change(a, b):
return abs(a[0] - b[0]) <= 2 and abs(a[1] - b[1]) <= 2 and abs(a[1] - b[1]) <= 2
def img_eq(ref, now, delta):
# This is slow but exactly what we want.
# ImageMath might be able to speed this up.
if ref.size != now.size:
return False
data_ref = ref.load()
data_now = now.load()
data_delta = delta.load()
result = True
width, height = ref.size
for y in range(height):
for x in range(width):
if data_ref[x, y] != data_now[x, y]:
if masked(data_ref[x, y]) or ignorable_coord(x, y) or ignorable_change(data_ref[x, y], data_now[x, y]):
data_delta[x, y] = (0, 255, 0, 255)
else:
data_delta[x, y] = (255, 0, 0, 255)
result = False
return result
if not img_eq(img_ref, img_now, img_delta):
if img_now.size == img_ref.size:
# Preserve alpha channel so that the 'now'
# image can be used as the new reference image
# without further changes
img_now.putalpha(img_ref.getchannel("A"))
img_now.save(filename)
attach(filename)
ref_filename_for_attach = base + "-reference.png"
img_ref.save(ref_filename_for_attach)
attach(ref_filename_for_attach)
delta_filename = base + "-delta.png"
img_delta.save(delta_filename)
attach(delta_filename)
print("Differences in pixel test " + base)
self.failed_pixel_tests += 1
def get_js_log(self):
"""Return the current javascript log"""
@ -776,7 +889,10 @@ class MachineCase(unittest.TestCase):
if machine is None:
machine = self.machine
label = self.label() + "-" + machine.label
browser = Browser(machine.web_address, label=label, port=machine.web_port)
pixels_label = None
if machine.image == testvm.TEST_OS_DEFAULT and os.environ.get("TEST_BROWSER", "chromium") == "chromium":
pixels_label = self.label()
browser = Browser(machine.web_address, label=label, pixels_label=pixels_label, port=machine.web_port)
self.addCleanup(browser.kill)
return browser
@ -974,6 +1090,7 @@ class MachineCase(unittest.TestCase):
if self.checkSuccess() and self.machine.ssh_reachable:
self.check_journal_messages()
self.check_browser_errors()
self.check_pixel_tests()
shutil.rmtree(self.tmpdir, ignore_errors=True)
def login_and_go(self, path=None, user=None, host=None, superuser=True, urlroot=None, tls=False):
@ -1216,6 +1333,10 @@ class MachineCase(unittest.TestCase):
else:
raise Error(UNEXPECTED_MESSAGE + "browser errors:\n" + log)
def check_pixel_tests(self):
if self.browser and self.browser.failed_pixel_tests > 0:
raise Error("Some pixel tests have failed")
def check_axe(self, label=None, suffix=""):
"""Run aXe check on the currently active frame
@ -1696,6 +1817,9 @@ def test_main(options=None, suite=None, attachments=None, **kwargs):
print_tests(suite)
return 0
attach(os.path.join(TEST_DIR, "common/pixeldiff.html"))
attach(os.path.join(TEST_DIR, "common/link-patterns.json"))
runner = TapRunner(verbosity=opts.verbosity)
ret = runner.run(suite)
if not standalone: