+ * The screenshot tests compare the output of software renderers (the desktop job renders with + * Mesa, the Android job renders with the emulator's GLES renderer) against reference images that + * were captured on the CI machines. Those renderers do not rasterise, sample and blend + * bit-for-bit identically on every host, so the very same commit regularly produces an image in + * which a handful of pixels differ from the reference even though nothing about jME has changed. + * Demanding that every single pixel matches therefore fails the build for a reason no code change + * can fix, and because the difference is a property of the machine the test runs on, re-running + * the test there (which is what the CI retries do) reproduces it exactly. + *
+ * + *+ * A real rendering regression is different in kind, not just in degree: it changes a large part + * of the image rather than a few pixels. The rule applied here is therefore that the images count + * as the same when no more than {@link #ALLOWED_DIFFERENT_PIXEL_RATIO} of the pixels (and at + * least {@link #MINIMUM_ALLOWED_DIFFERENT_PIXELS}, so that small images still get a usable + * budget) differ from the reference by more than {@link #PIXEL_TOLERANCE} on any colour channel. + *
+ * + * @author jaime-jmebot + */ +public final class ImageDifference { + + /** + * A pixel counts as different when any of its colour channels differs from the reference by + * more than this value. It is deliberately the same threshold that + * {@link PixelSamenessDegree#NEGLIGIBLY_DIFFERENT} marks as "negligibly different" in the + * generated diff images. + */ + public static final int PIXEL_TOLERANCE = + PixelSamenessDegree.NEGLIGIBLY_DIFFERENT.getMaximumAllowedDifference(); + + /** + * The fraction of an image that may differ by more than {@link #PIXEL_TOLERANCE} and still be + * accepted as the same image. 0.02% is a few pixels on the small test images and a couple of + * hundred pixels on a full screen emulator screenshot, which is far more than the handful of + * pixels renderer rounding produces and far less than any visible change to a scene. + */ + public static final float ALLOWED_DIFFERENT_PIXEL_RATIO = 0.0002f; + + /** + * The smallest allowance {@link #ALLOWED_DIFFERENT_PIXEL_RATIO} may yield, so that small test + * images are not compared with an allowance of (nearly) zero pixels. + */ + public static final int MINIMUM_ALLOWED_DIFFERENT_PIXELS = 10; + + private final int differentPixels; + + private final int totalPixels; + + private final int worstDifference; + + /** + * Creates a difference from already counted pixel values. + * + * @param differentPixels how many pixels differed by more than {@link #PIXEL_TOLERANCE} + * @param totalPixels how many pixels were compared + * @param worstDifference the largest single colour channel difference found + */ + public ImageDifference(int differentPixels, int totalPixels, int worstDifference) { + if (differentPixels < 0 || totalPixels < 0 || differentPixels > totalPixels) { + throw new IllegalArgumentException("differentPixels (" + differentPixels + + ") must be between 0 and totalPixels (" + totalPixels + ")"); + } + this.differentPixels = differentPixels; + this.totalPixels = totalPixels; + this.worstDifference = worstDifference; + } + + /** + * Measures the difference between two images of the same size. + * + * @param image1 one image + * @param image2 the image to compare it with + * @return the measured difference + * @throws IllegalArgumentException if the images do not have the same dimensions + */ + public static ImageDifference of(Image image1, Image image2) { + if (image1.getWidth() != image2.getWidth() || image1.getHeight() != image2.getHeight()) { + throw new IllegalArgumentException("Images must have the same size: " + + image1.getWidth() + "x" + image1.getHeight() + " vs " + + image2.getWidth() + "x" + image2.getHeight()); + } + + ImageRaster image1Raster = ImageRaster.create(image1); + ImageRaster image2Raster = ImageRaster.create(image2); + + ColorRGBA color1 = new ColorRGBA(); + ColorRGBA color2 = new ColorRGBA(); + + int differentPixels = 0; + int worstDifference = 0; + + for (int y = 0; y < image1.getHeight(); y++) { + for (int x = 0; x < image1.getWidth(); x++) { + image1Raster.getPixel(x, y, color1); + image2Raster.getPixel(x, y, color2); + + int difference = maximumComponentDifference(color1.asIntARGB(), color2.asIntARGB()); + + if (difference > worstDifference) { + worstDifference = difference; + } + if (difference > PIXEL_TOLERANCE) { + differentPixels++; + } + } + } + + return new ImageDifference(differentPixels, image1.getWidth() * image1.getHeight(), + worstDifference); + } + + /** + * Compares two pixels and returns the difference of the colour channel that differs most. + * + * @param pixel1 a pixel in ARGB order + * @param pixel2 the pixel to compare it with, in ARGB order + * @return the largest difference (0 to 255) between the two pixels + */ + public static int maximumComponentDifference(int pixel1, int pixel2) { + int r1 = (pixel1 >> 16) & 0xFF; + int g1 = (pixel1 >> 8) & 0xFF; + int b1 = pixel1 & 0xFF; + int a1 = (pixel1 >> 24) & 0xFF; + + int r2 = (pixel2 >> 16) & 0xFF; + int g2 = (pixel2 >> 8) & 0xFF; + int b2 = pixel2 & 0xFF; + int a2 = (pixel2 >> 24) & 0xFF; + + return Math.max(Math.abs(r1 - r2), + Math.max(Math.abs(g1 - g2), Math.max(Math.abs(b1 - b2), Math.abs(a1 - a2)))); + } + + /** + * @return how many pixels differed by more than {@link #PIXEL_TOLERANCE} + */ + public int getDifferentPixels() { + return differentPixels; + } + + /** + * @return how many pixels were compared + */ + public int getTotalPixels() { + return totalPixels; + } + + /** + * @return the largest single colour channel difference found, 0 when the images are identical + */ + public int getWorstDifference() { + return worstDifference; + } + + /** + * @return how many pixels are allowed to differ before the images stop counting as the same + */ + public int getAllowedDifferentPixels() { + return Math.max(MINIMUM_ALLOWED_DIFFERENT_PIXELS, + (int) (totalPixels * ALLOWED_DIFFERENT_PIXEL_RATIO)); + } + + /** + * @return true when the difference is small enough to be renderer noise rather than a change + * in what was drawn + */ + public boolean isNegligible() { + return differentPixels <= getAllowedDifferentPixels(); + } + + /** + * @return a human readable summary of the difference, for the test report and failure messages + */ + public String describe() { + return differentPixels + " of " + totalPixels + " pixels differ by more than " + + PIXEL_TOLERANCE + " (at most " + getAllowedDifferentPixels() + + " tolerated), largest single channel difference " + worstDifference; + } +} diff --git a/jme3-screenshot-tests/jme3-screenshot-tests-shared/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java b/jme3-screenshot-tests/jme3-screenshot-tests-shared/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java index b08106a403..a8dac09f3e 100644 --- a/jme3-screenshot-tests/jme3-screenshot-tests-shared/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java +++ b/jme3-screenshot-tests/jme3-screenshot-tests-shared/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java @@ -321,31 +321,35 @@ private void bootAppForTest(TestType testType, AppSettings appSettings, String b if(failureMessage==null){ //only want the first thing to go wrong as the junit test fail reason failureMessage = IMAGES_ARE_DIFFERENT_SIZES; } - }else if (imagesAreVerySimilar(generatedImage, expectedImage)) { - if(testType == TestType.KNOWN_TO_FAIL){ - TestReportCaptureBase.INSTANCE.warning(KNOWN_BAD_TEST_IMAGES_SAME); - } } else { - //save the generated image to the build directory - osSpecificRunner.saveGeneratedImageToChangedImages(generatedImage, thisFrameBaseImageFileName + ".png"); - - attachImage("Expected", thisFrameBaseImageFileName + "_expected.png", expectedImage); - attachImage("Actual", thisFrameBaseImageFileName + "_actual.png", generatedImage); - attachImage("Diff", thisFrameBaseImageFileName + "_diff.png", createComparisonImage(generatedImage, expectedImage)); + ImageDifference imageDifference = ImageDifference.of(generatedImage, expectedImage); - switch(testType){ - case MUST_PASS: - if(failureMessage==null){ //only want the first thing to go wrong as the junit test fail reason - failureMessage = IMAGES_ARE_DIFFERENT; - } - TestReportCaptureBase.INSTANCE.markFailInReport(IMAGES_ARE_DIFFERENT); - break; - case NON_DETERMINISTIC: - TestReportCaptureBase.INSTANCE.warning(NON_DETERMINISTIC_TEST); - break; - case KNOWN_TO_FAIL: - TestReportCaptureBase.INSTANCE.warning(KNOWN_BAD_TEST_IMAGES_DIFFERENT); - break; + if (imageDifference.isNegligible()) { + if(testType == TestType.KNOWN_TO_FAIL){ + TestReportCaptureBase.INSTANCE.warning(KNOWN_BAD_TEST_IMAGES_SAME); + } + } else { + //save the generated image to the build directory + osSpecificRunner.saveGeneratedImageToChangedImages(generatedImage, thisFrameBaseImageFileName + ".png"); + + attachImage("Expected", thisFrameBaseImageFileName + "_expected.png", expectedImage); + attachImage("Actual", thisFrameBaseImageFileName + "_actual.png", generatedImage); + attachImage("Diff", thisFrameBaseImageFileName + "_diff.png", createComparisonImage(generatedImage, expectedImage)); + + switch(testType){ + case MUST_PASS: + if(failureMessage==null){ //only want the first thing to go wrong as the junit test fail reason + failureMessage = IMAGES_ARE_DIFFERENT + " (" + imageDifference.describe() + ")"; + } + TestReportCaptureBase.INSTANCE.markFailInReport(IMAGES_ARE_DIFFERENT); + break; + case NON_DETERMINISTIC: + TestReportCaptureBase.INSTANCE.warning(NON_DETERMINISTIC_TEST); + break; + case KNOWN_TO_FAIL: + TestReportCaptureBase.INSTANCE.warning(KNOWN_BAD_TEST_IMAGES_DIFFERENT); + break; + } } } @@ -433,32 +437,11 @@ private static boolean imagesAreSameSize(Image img1, Image img2) { * Tests that the images are the same for the purposes of the test. * If they are not the same it will return false (which may fail the test depending on the test type). * Different sizes are so fatal that they will immediately fail the test. + * A difference that is small enough to be renderer noise rather than a change in what was drawn + * still counts as the same, see {@link ImageDifference}. */ private static boolean imagesAreVerySimilar(Image img1, Image img2) { - ImageRaster image1Wrapper = DefaultImageRaster.create(img1); - ImageRaster image2Wrapper = DefaultImageRaster.create(img2); - - ColorRGBA color1 = new ColorRGBA(); - ColorRGBA color2 = new ColorRGBA(); - - for (int y = 0; y < img1.getHeight(); y++) { - for (int x = 0; x < img1.getWidth(); x++) { - - image1Wrapper.getPixel(x, y, color1); - image2Wrapper.getPixel(x, y, color2); - - int pixel1 = color1.asIntARGB(); - int pixel2 = color2.asIntARGB(); - - int largestPixelValueDifference = getMaximumComponentDifference(pixel1, pixel2); - - if(largestPixelValueDifference>PixelSamenessDegree.NEGLIGIBLY_DIFFERENT.getMaximumAllowedDifference()){ - return false; - } - - } - } - return true; + return ImageDifference.of(img1, img2).isNegligible(); } /** @@ -532,7 +515,7 @@ private static PixelSamenessDegree categorisePixelDifference(int pixel1, int pix return PixelSamenessDegree.SAME; } - int pixelDifference = getMaximumComponentDifference(pixel1, pixel2); + int pixelDifference = ImageDifference.maximumComponentDifference(pixel1, pixel2); if(pixelDifference<= PixelSamenessDegree.NEGLIGIBLY_DIFFERENT.getMaximumAllowedDifference()){ return PixelSamenessDegree.NEGLIGIBLY_DIFFERENT; @@ -549,20 +532,4 @@ private static PixelSamenessDegree categorisePixelDifference(int pixel1, int pix return PixelSamenessDegree.EXTREMELY_DIFFERENT; } - private static int getMaximumComponentDifference(int pixel1, int pixel2){ - int r1 = (pixel1 >> 16) & 0xFF; - int g1 = (pixel1 >> 8) & 0xFF; - int b1 = pixel1 & 0xFF; - int a1 = (pixel1 >> 24) & 0xFF; - - int r2 = (pixel2 >> 16) & 0xFF; - int g2 = (pixel2 >> 8) & 0xFF; - int b2 = pixel2 & 0xFF; - int a2 = (pixel2 >> 24) & 0xFF; - - return Math.max(Math.abs(r1 - r2), Math.max(Math.abs(g1 - g2), Math.max(Math.abs(b1 - b2), Math.abs(a1 - a2)))); - } - - - } diff --git a/jme3-screenshot-tests/jme3-screenshot-tests-shared/src/test/java/org/jmonkeyengine/screenshottests/testframework/ImageDifferenceTest.java b/jme3-screenshot-tests/jme3-screenshot-tests-shared/src/test/java/org/jmonkeyengine/screenshottests/testframework/ImageDifferenceTest.java new file mode 100644 index 0000000000..4fb39017b4 --- /dev/null +++ b/jme3-screenshot-tests/jme3-screenshot-tests-shared/src/test/java/org/jmonkeyengine/screenshottests/testframework/ImageDifferenceTest.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2026 jMonkeyEngine + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of 'jMonkeyEngine' nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.jmonkeyengine.screenshottests.testframework; + +import com.jme3.math.ColorRGBA; +import com.jme3.texture.Image; +import com.jme3.texture.image.ColorSpace; +import com.jme3.texture.image.ImageRaster; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests for {@link ImageDifference}, the rule that decides when renderer noise is tolerated and + * when a screenshot test has found a real change. + */ +public class ImageDifferenceTest { + + @Test + public void identicalImagesHaveNoDifference() { + Image reference = blackImage(20, 10); + Image generated = blackImage(20, 10); + + ImageDifference difference = ImageDifference.of(reference, generated); + + assertEquals(0, difference.getDifferentPixels()); + assertEquals(20 * 10, difference.getTotalPixels()); + assertEquals(0, difference.getWorstDifference()); + assertTrue(difference.isNegligible(), difference.describe()); + } + + @Test + public void aHandfulOfNoisyPixelsStillCountsAsTheSameImage() { + Image reference = blackImage(100, 100); + Image generated = blackImage(100, 100); + makePixelsDifferent(generated, 5); + + ImageDifference difference = ImageDifference.of(reference, generated); + + assertEquals(5, difference.getDifferentPixels()); + assertEquals(255, difference.getWorstDifference(), "a noisy pixel is a fully different pixel"); + assertEquals(ImageDifference.MINIMUM_ALLOWED_DIFFERENT_PIXELS, + difference.getAllowedDifferentPixels()); + assertTrue(difference.isNegligible(), difference.describe()); + assertTrue(difference.describe().contains("5 of 10000 pixels")); + } + + @Test + public void moreNoiseThanTheBudgetIsNotTheSameImage() { + Image reference = blackImage(100, 100); + Image generated = blackImage(100, 100); + makePixelsDifferent(generated, ImageDifference.MINIMUM_ALLOWED_DIFFERENT_PIXELS + 1); + + ImageDifference difference = ImageDifference.of(reference, generated); + + assertEquals(ImageDifference.MINIMUM_ALLOWED_DIFFERENT_PIXELS + 1, + difference.getDifferentPixels()); + assertFalse(difference.isNegligible(), difference.describe()); + } + + @Test + public void aChangedAreaIsNotTheSameImage() { + Image reference = blackImage(100, 100); + Image generated = blackImage(100, 100); + ImageRaster raster = ImageRaster.create(generated); + for (int y = 10; y < 40; y++) { + for (int x = 10; x < 40; x++) { + raster.setPixel(x, y, ColorRGBA.White); + } + } + + ImageDifference difference = ImageDifference.of(reference, generated); + + assertEquals(30 * 30, difference.getDifferentPixels()); + assertFalse(difference.isNegligible(), "a 30x30 change is a real change, not noise"); + } + + @Test + public void theBudgetScalesWithTheImageSize() { + // 1280x800 is the size of an emulator screenshot, 500x400 an ordinary desktop screenshot + assertEquals(204, new ImageDifference(0, 1280 * 800, 0).getAllowedDifferentPixels()); + assertEquals(40, new ImageDifference(0, 500 * 400, 0).getAllowedDifferentPixels()); + } + + @Test + public void imagesOfDifferentSizesAreRejected() { + assertThrows(IllegalArgumentException.class, + () -> ImageDifference.of(blackImage(4, 4), blackImage(4, 5))); + } + + @Test + public void impossibleCountsAreRejected() { + assertThrows(IllegalArgumentException.class, () -> new ImageDifference(11, 10, 0)); + assertThrows(IllegalArgumentException.class, () -> new ImageDifference(-1, 10, 0)); + } + + private static Image blackImage(int width, int height) { + return new Image(Image.Format.RGBA8, width, height, + ByteBuffer.allocateDirect(width * height * 4), ColorSpace.sRGB); + } + + /** + * Sets the first {@code count} pixels of an image to white, so that they differ from the + * (black) reference by the maximum possible amount. + */ + private static void makePixelsDifferent(Image image, int count) { + ImageRaster raster = ImageRaster.create(image); + for (int i = 0; i < count; i++) { + raster.setPixel(i % image.getWidth(), i / image.getWidth(), ColorRGBA.White); + } + } +}