Add compare_file_screenshot(); Add AppDriver$expect_screenshot(compare=)

.Rbuildignore

@@ -16,6 +16,7 @@ _\.new\.png$
 ^Meta$
 ^cran-comments\.md$
 ^revdep$
+^.vscode$
 
 ^tests/testthat/migrate-apps$
 ^tests/testthat/apps$

.github/workflows/R-CMD-check.yaml

@@ -17,6 +17,8 @@ jobs:
     uses: rstudio/shiny-workflows/.github/workflows/website.yaml@v1
   routine:
     uses: rstudio/shiny-workflows/.github/workflows/routine.yaml@v1
+    with:
+      extra-packages: any::decor
   R-CMD-check:
     uses: rstudio/shiny-workflows/.github/workflows/R-CMD-check.yaml@v1
     with:

.lintr

@@ -3,4 +3,6 @@ linters: linters_with_defaults(
     commented_code_linter = NULL,
     cyclocomp_linter = NULL
   )
-exclusions: list()
+exclusions: list(
+    "R/cpp11.R"
+  )

.vscode/c_cpp_properties.json

@@ -0,0 +1,21 @@
+{
+    "configurations": [
+        {
+            "name": "Mac",
+            "includePath": [
+                "${workspaceFolder}/**",
+                "/Library/Frameworks/R.framework/Resources/include",
+                "/Library/Frameworks/R.framework/Versions/Current/Resources/library/cpp11/include"
+            ],
+            "defines": [],
+            "macFrameworkPath": [
+                "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Frameworks"
+            ],
+            "compilerPath": "/usr/bin/clang",
+            "cStandard": "c99",
+            "cppStandard": "c++11",
+            "intelliSenseMode": "macos-clang-x64"
+        }
+    ],
+    "version": 4
+}

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "files.associations": {
+    "vector": "cpp"
+  }
+}

DESCRIPTION

@@ -42,6 +42,7 @@ Suggests:
     ggplot2,
     knitr,
     plotly,
+    png,
     rstudioapi,
     shinyWidgets,
     shinytest (>= 1.5.1),
@@ -78,6 +79,8 @@ Collate:
     'app-driver-window.R'
     'app-driver.R'
     'chromote-methods.R'
+    'compare-screenshot-threshold.R'
+    'cpp11.R'
     'expect-snapshot.R'
     'expr-recurse.R'
     'httr.R'
@@ -91,5 +94,9 @@ Collate:
     'save-app.R'
     'shiny-browser.R'
     'shinytest2-logs.R'
+    'shinytest2-package.R'
     'test-app.R'
     'use.R'
+LinkingTo:
+    cpp11
+SystemRequirements: C++11

NAMESPACE

@@ -3,12 +3,14 @@
 S3method(format,shinytest2_log)
 S3method(print,shinytest2_log)
 export(AppDriver)
+export(compare_screenshot_threshold)
 export(get_input_processors)
 export(load_app_env)
 export(migrate_from_shinytest)
 export(platform_variant)
 export(record_test)
 export(register_input_processor)
+export(screenshot_max_difference)
 export(test_app)
 export(use_shinytest2)
 export(use_shinytest2_test)
@@ -26,3 +28,4 @@ importFrom(rlang,":=")
 importFrom(rlang,list2)
 importFrom(rlang,missing_arg)
 importFrom(testthat,default_reporter)
+useDynLib(shinytest2, .registration = TRUE)

NEWS.md

@@ -1,22 +1,31 @@
 # shinytest2 (development version)
 
+## New Features
+
+* `compare_screenshot_threshold()` is a new method to compare screenshots and allow small differences between two screenshots. This method checks every subset matrix of the pixel differences between the two images. If any total difference is larger than the `threshold` value, the two images are considered to be different. The subset matrix size can be increased by setting `kernel_size`. (#231)
+
+* Support for supply your own compare method to `AppDriver$expect_screenshot(compare=)` has been added. By default, `AppDriver$expect_screenshot(compare=)` is set to `compare_screenshot_threshold(threshold = NULL)` which in turn calls `testthat::compare_file_binary()`. So no default screenshot expectation behavior has changed. (#231)
+
 * `test_app()` now inherits the existing test reporter when testing multiple apps within a package test file. This allows for a seamless, single reporter output instead of nested reporters being displayed. (#192)
 
-* Fix set of bugs found by @daattali including test files should be opened in the IDE after recording and test and replace missing images in the website (#199)
+* The recording browser window is now closed when either the "Save test and exit" or "Exit" buttons are clicked. (@daattali, #202)
 
-* Provide example workflows on how to use `rstudio/shinytest2/actions/test-app` GHA action (#217)
+* When testing and `{chromote}` can not be started, the test will be skipped. When testing on windows in CI, `{chromote}` will get an extra attempt to be started (#225)
 
 * Make `{globals}` an `Imports` package, instead of a `Suggests` package (#223)
 
 * Add support for _not_ recording the screen size when recording a test (#223)
 
-* When setting a date time slider value, it can now handle array inputs properly. When recording a date time slider value, numeric values will not be recorded as milliseconds instead of seconds since epoch. (#223)
 
-* The recording browser window is now closed when either the "Save test and exit" or "Exit" buttons are clicked. (@daattali, #202)
+## Bug / Improvements
+
+* Fix set of bugs found by @daattali including test files should be opened in the IDE after recording and test and replace missing images in the website (#199)
 
-* When creating a test setup file for `{shinytest2}`, use the file path `tests/testthat/setup-shinytest2.R` instead of `tests/testthat/setup.R` to provide some quick context (#224)
+* Provide example workflows on how to use `rstudio/shinytest2/actions/test-app` GHA action (#217)
+
+* When setting a date time slider value, it can now handle array inputs properly. When recording a date time slider value, numeric values will not be recorded as milliseconds instead of seconds since epoch. (#223)
 
-* When testing and `{chromote}` can not be started, the test will be skipped. When testing on windows in CI, `{chromote}` will get an attempt to be started (#225)
+* When creating a test setup file for `{shinytest2}`, use the file path `tests/testthat/setup-shinytest2.R` instead of `tests/testthat/setup.R` to provide some quick context within the file name (#224)
 
 * Remove trailing comma causing render bug in recorder app (@mehrnoushmalek, #239)
 

R/app-driver-expect-screenshot.R

@@ -63,16 +63,32 @@ app_get_screenshot <- function(
 app_expect_screenshot <- function(
   self, private,
   ...,
+  threshold = NULL,
+  kernel_size = 5,
+  quiet = FALSE,
   name = NULL,
   screenshot_args = missing_arg(),
   delay = missing_arg(),
   selector = missing_arg(),
+  compare = missing_arg(),
   cran = FALSE
 ) {
   "!DEBUG app_expect_screenshot()"
   ckm8_assert_app_driver(self, private)
   ellipsis::check_dots_empty()
 
+  compare <- rlang::maybe_missing(
+    compare,
+    function(old, new) {
+      compare_screenshot_threshold(
+        old, new,
+        threshold = threshold,
+        kernel_size = kernel_size,
+        quiet = quiet
+      )
+    }
+  )
+
   filename <- app_next_temp_snapshot_path(self, private, name, "png")
 
   # Take screenshot
@@ -88,7 +104,7 @@ app_expect_screenshot <- function(
     self, private,
     filename,
     cran = cran,
-    compare = testthat::compare_file_binary
+    compare = compare
   )
 }
 

R/app-driver.R

@@ -989,28 +989,104 @@ AppDriver <- R6Class( # nolint
     #' @param name The file name to be used for the snapshot. The file extension
     #'   will overwritten to `.png`. By default, the `name` supplied to
     #'   `app` on initialization with a counter will be used (e.g. `"NAME-001.png"`).
+    #' @param compare A function used to compare the screenshot snapshot files.
+    #' The function should take two inputs, the paths to the `old` and `new`
+    #' snapshot, and return either `TRUE` or `FALSE`.
+    #'
+    #' `compare` defaults to a function that wraps around
+    #' `compare_screenshot_threshold(old, new, threshold = threshold,
+    #' kernel_size = kernel_size, quiet = quiet)`. Note: if `threshold` is
+    #' `NULL` (default), `compare` will behave as if
+    #' [`testthat::compare_file_binary()`] was provided, comparing the two
+    #' images byte-by-byte.
+    #' @param threshold Parameter supplied to [`compare_screenshot_threshold()`]
+    #' when using the default `compare` method. If the value of `threshold` is
+    #' NULL`, [`compare_screenshot_threshold()`] will act like
+    #' [`testthat::compare_file_binary`]. However, if `threshold` is a positive number,
+    #' it will be compared against the largest convolution value found if the
+    #' two images fail a [`testthat::compare_file_binary`] comparison.
+    #'
+    #' Which value should I use? Threshold values values below 5 help deter
+    #' false-positive screenshot comparisons (such as inconsistent rounded
+    #' corners). Larger values in the 10s and 100s will help find _real_
+    #' changes. However, not all values are one size fits all and you will need
+    #' to play with a threshold that fits your needs.
+    #' @param kernel_size Parameter supplied to [`compare_screenshot_threshold()`]
+    #' when using the default `compare` method. The `kernel_size` represents the
+    #' height and width of the convolution kernel applied to the pixel
+    #' differences. This integer-like value should be relatively small.
+    #' @param quiet Parameter supplied to [`compare_screenshot_threshold()`]
+    #' when using the default `compare` method. If `FALSE`, diagnostic
+    #' information will be presented when the computed value is larger than a
+    #' non-`NULL` `threshold` value.
     #' @examples
     #' \dontrun{
+    #' # These example lines should be performed in a `./tests/testthat`
+    #' # test file so that snapshot files can be properly saved
+    #'
     #' app_path <- system.file("examples/01_hello", package = "shiny")
     #' app <- AppDriver$new(app_path, variant = platform_variant())
     #'
-    #' # Expect a full size screenshot
+    #' # Expect a full size screenshot to be pixel perfect
     #' app$expect_screenshot()
     #'
-    #' # Very brittle test
+    #' # Images are brittle when containing plots
     #' app$expect_screenshot(selector = "#distPlot")
+    #'
+    #' # Test with more threshold in pixel value differences
+    #' # Helps with rounded corners
+    #' app$expect_screenshot(threshold = 10)
+    #'
+    #' # Equivalent expectations
+    #' app$expect_screenshot() # default
+    #' app$expect_screenshot(threshold = NULL)
+    #' app$expect_screenshot(compare = testthat::compare_file_binary)
+    #' expect_snapshot_file(
+    #'   app$get_screenshot(),
+    #'   variant = app$get_variant(),
+    #'   compare = testthat::compare_file_binary
+    #' )
+    #'
+    #' # Equivalent expectations
+    #' app$expect_screenshot(threshold = 3, kernel_size = 5)
+    #' app$expect_screenshot(compare = function(old, new) {
+    #'   compare_screenshot_threshold(
+    #'     old, new,
+    #'     threshold = 3,
+    #'     kernel_size = 5
+    #'   )
+    #' })
+    #' expect_screenshot_file(
+    #'   app$get_screenshot(),
+    #'   variant = app$get_variant(),
+    #'   compare = function(old, new) {
+    #'     compare_screenshot_threshold(
+    #'       old, new,
+    #'       threshold = 3,
+    #'       kernel_size = 5
+    #'     )
+    #'   }
+    #' )
     #' }
     expect_screenshot = function(
       ...,
+      threshold = NULL,
+      kernel_size = 5,
       screenshot_args = missing_arg(),
       delay = missing_arg(),
       selector = missing_arg(),
+      compare = missing_arg(),
+      quiet = FALSE,
       name = NULL,
       cran = FALSE
     ) {
       app_expect_screenshot_and_variant(
         self, private,
         ...,
+        threshold = threshold,
+        kernel_size = kernel_size,
+        quiet = quiet,
+        compare = compare,
         screenshot_args = screenshot_args,
         delay = delay,
         selector = selector,