diff --git a/cmake/modules/AppUtilities.cmake b/cmake/modules/AppUtilities.cmake
index 9a46b48..2272431 100644
--- a/cmake/modules/AppUtilities.cmake
+++ b/cmake/modules/AppUtilities.cmake
@@ -80,9 +80,14 @@ function (add_appstream_file)
         COMPONENT appimage)
 
     # add test
+    set(APPSTREAM_TESTS_ENABLED_DEFAULT OFF)
     find_program(APPSTREAMCLI_BIN "appstreamcli")
+    if (ENABLE_DEVEL_DEFAULTS AND APPSTREAMCLI_BIN)
+        set(APPSTREAM_TESTS_ENABLED_DEFAULT ON)
+    endif ()
+    option(APPSTREAM_TESTS_ENABLED "enables tests for checking whether AppStream files" "${APPSTREAM_TESTS_ENABLED_DEFAULT}")
     if (NOT APPSTREAMCLI_BIN)
-        message(STATUS "Could not find appstreamcli; won't add test/target to validate appstream files")
+        message(FATAL_ERROR "Unable to validate appstreamcli files; appstreamcli not found")
     else ()
         add_test(NAME "${META_TARGET_NAME}_appstream_validation" COMMAND "${APPSTREAMCLI_BIN}" validate "${APPSTREAM_FILE}")
     endif ()