From 442f7a9b255d61ead17646417a84a3574755d997 Mon Sep 17 00:00:00 2001 From: Martchus Date: Sun, 9 Jul 2017 23:38:11 +0200 Subject: [PATCH] Exclude generated files from coverage * So files generated by moc or qdbusxml2cpp are not part of coverage statistics * When filtering files, llvm-cov prints multiple tables showing coverage on function level. -> Generate overall coverage stats via awk. --- cmake/modules/AppTarget.cmake | 5 ++- cmake/modules/LibraryTarget.cmake | 8 +++- cmake/modules/TestTarget.cmake | 65 +++++++++++++++++++++++------- tests/calculateoverallcoverage.awk | 24 +++++++++++ 4 files changed, 85 insertions(+), 17 deletions(-) create mode 100644 tests/calculateoverallcoverage.awk diff --git a/cmake/modules/AppTarget.cmake b/cmake/modules/AppTarget.cmake index a874a9e..30d8dfb 100644 --- a/cmake/modules/AppTarget.cmake +++ b/cmake/modules/AppTarget.cmake @@ -33,8 +33,11 @@ else() set(ACTUAL_ADDITIONAL_LINK_FLAGS ${META_ADDITIONAL_LINK_FLAGS}) endif() +# define relevant files +set(ALL_FILES ${HEADER_FILES} ${SRC_FILES} ${GENERATED_DBUS_FILES} ${WIDGETS_FILES} ${QML_FILES} ${RES_FILES} ${QM_FILES} ${WINDOWS_ICON_PATH}) + # add target for building the application -add_executable(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX} ${GUI_TYPE} ${HEADER_FILES} ${SRC_FILES} ${WIDGETS_FILES} ${QML_FILES} ${RES_FILES} ${QM_FILES} ${WINDOWS_ICON_PATH}) +add_executable(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX} ${GUI_TYPE} ${ALL_FILES}) target_link_libraries(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX} PUBLIC ${ACTUAL_ADDITIONAL_LINK_FLAGS} "${PUBLIC_LIBRARIES}" PRIVATE "${PRIVATE_LIBRARIES}" diff --git a/cmake/modules/LibraryTarget.cmake b/cmake/modules/LibraryTarget.cmake index ef2dcca..5036f8c 100644 --- a/cmake/modules/LibraryTarget.cmake +++ b/cmake/modules/LibraryTarget.cmake @@ -97,6 +97,10 @@ if(NOT META_SOVERSION AND NOT META_IS_PLUGIN) endif() endif() message(STATUS "${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}: BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}") + +# define relevant files +set(ALL_FILES ${HEADER_FILES} ${SRC_FILES} ${GENERATED_DBUS_FILES} ${WIDGETS_FILES} ${QML_FILES} ${RES_FILES} ${QM_FILES} ${WINDOWS_ICON_PATH}) + # add target for building the library if(BUILD_SHARED_LIBS) if(STATIC_LIBRARY_LINKAGE) @@ -130,7 +134,7 @@ if(BUILD_SHARED_LIBS) INTERFACE "${META_PUBLIC_SHARED_LIB_COMPILE_OPTIONS}" "${META_PRIVATE_SHARED_LIB_COMPILE_OPTIONS}" ) else() - add_library(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX} ${META_SHARED_OBJECT_TYPE} ${HEADER_FILES} ${SRC_FILES} ${WIDGETS_FILES} ${QML_FILES} ${RES_FILES} ${QM_FILES} ${WINDOWS_ICON_PATH}) + add_library(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX} ${META_SHARED_OBJECT_TYPE} ${ALL_FILES}) target_link_libraries(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX} PUBLIC ${ACTUAL_ADDITIONAL_LINK_FLAGS} "${PUBLIC_LIBRARIES}" PRIVATE "${PRIVATE_LIBRARIES}" @@ -181,7 +185,7 @@ if(BUILD_STATIC_LIBS) INTERFACE "${META_PUBLIC_STATIC_LIB_COMPILE_OPTIONS}" "${META_PRIVATE_STATIC_LIB_COMPILE_OPTIONS}" ) else() - add_library(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_static STATIC ${HEADER_FILES} ${SRC_FILES} ${WIDGETS_FILES} ${QML_FILES} ${RES_FILES} ${QM_FILES} ${WINDOWS_ICON_PATH}) + add_library(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_static STATIC ${ALL_FILES}) target_link_libraries(${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_static PUBLIC "${PUBLIC_STATIC_LIBRARIES}" PRIVATE "${PRIVATE_STATIC_LIBRARIES}" diff --git a/cmake/modules/TestTarget.cmake b/cmake/modules/TestTarget.cmake index 5b0c16e..571f085 100644 --- a/cmake/modules/TestTarget.cmake +++ b/cmake/modules/TestTarget.cmake @@ -136,10 +136,17 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT) # enable source code based coverage analysis using clang if(CLANG_SOURCE_BASED_COVERAGE_AVAILABLE) - # specify where to store raw clang profiling data via environment variable + # define path of raw profile data set(LLVM_PROFILE_RAW_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw") + # define path of list with additional raw profile data from fork processes spawned during tests set(LLVM_PROFILE_RAW_LIST_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw.list") + # define path of merged profile data generated from raw profiling data set(LLVM_PROFILE_DATA_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata") + # define paths of output files + set(COVERAGE_REPORT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt") + set(COVERAGE_OVERALL_REPORT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_overall.txt") + set(COVERAGE_HTML_REPORT_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html") + # specify where to store raw clang profiling data via environment variable set_tests_properties(${META_PROJECT_NAME}_run_tests PROPERTIES ENVIRONMENT "LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE};LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}" @@ -156,11 +163,12 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT) -w "${CMAKE_CURRENT_BINARY_DIR}/testworkingdir" -a "$" COMMENT "Executing ${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests to generate raw profiling data for source-based coverage report" - DEPENDS ${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests + DEPENDS "${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests" ) find_program(LLVM_PROFDATA_BIN llvm-profdata) find_program(LLVM_COV_BIN llvm-cov) if(LLVM_PROFDATA_BIN AND LLVM_COV_BIN) + # merge profiling data add_custom_command( OUTPUT "${LLVM_PROFILE_DATA_FILE}" COMMAND cat "${LLVM_PROFILE_RAW_LIST_FILE}" | xargs @@ -172,37 +180,66 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT) DEPENDS "${LLVM_PROFILE_RAW_FILE}" "${LLVM_PROFILE_RAW_LIST_FILE}" ) + # generate coverage report (statistics, for each file a table) add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt" + OUTPUT "${COVERAGE_REPORT_FILE}" COMMAND "${LLVM_COV_BIN}" report - -instr-profile "${LLVM_PROFILE_DATA_FILE}" -format=text + -stats + -instr-profile "${LLVM_PROFILE_DATA_FILE}" $ - > "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt" - COMMENT "Generating HTML coverage report" + ${HEADER_FILES} ${SRC_FILES} ${WIDGETS_HEADER_FILES} ${WIDGETS_SOURCE_FILES} ${QML_HEADER_FILES} ${QML_SOURCE_FILES} + > "${COVERAGE_REPORT_FILE}" + COMMENT "Generating coverage report (statistics as table)" DEPENDS "${LLVM_PROFILE_DATA_FILE}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" ) add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_summary" - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt" + DEPENDS "${COVERAGE_REPORT_FILE}" ) + # generate coverage overall report (total region/line coverage) + set(OVERALL_COVERAGE_AKW_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/tests/calculateoverallcoverage.awk") + if(CPP_UTILITIES_SOURCE_DIR AND NOT EXISTS "${OVERALL_COVERAGE_AKW_SCRIPT}") + set(OVERALL_COVERAGE_AKW_SCRIPT "${CPP_UTILITIES_SOURCE_DIR}/tests/calculateoverallcoverage.awk") + endif() + if(NOT EXISTS "${OVERALL_COVERAGE_AKW_SCRIPT}") + set(OVERALL_COVERAGE_AKW_SCRIPT "${CPP_UTILITIES_CONFIG_DIRS}/tests/calculateoverallcoverage.awk") + endif() add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html" + OUTPUT "${COVERAGE_OVERALL_REPORT_FILE}" + COMMAND awk + -f "${OVERALL_COVERAGE_AKW_SCRIPT}" "${COVERAGE_REPORT_FILE}" + > "${COVERAGE_OVERALL_REPORT_FILE}" + COMMENT "Generating coverage report (overall figures)" + DEPENDS "${COVERAGE_REPORT_FILE}" + ) + add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_overall_summary" + DEPENDS "${COVERAGE_OVERALL_REPORT_FILE}" + ) + # generate HTML document showing covered/uncovered code + add_custom_command( + OUTPUT "${COVERAGE_HTML_REPORT_FILE}" COMMAND "${LLVM_COV_BIN}" show -project-title="${META_APP_NAME}" - -instr-profile "${LLVM_PROFILE_DATA_FILE}" -format=html + -instr-profile "${LLVM_PROFILE_DATA_FILE}" $ - > "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html" - COMMENT "Generating HTML coverage report" + ${HEADER_FILES} ${SRC_FILES} ${WIDGETS_FILES} ${QML_FILES} + > "${COVERAGE_HTML_REPORT_FILE}" + COMMENT "Generating HTML document showing covered/uncovered code" DEPENDS "${LLVM_PROFILE_DATA_FILE}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" ) add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_html" - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html" + DEPENDS "${COVERAGE_HTML_REPORT_FILE}" ) + # create target for all coverage docs add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage" - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt" - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html" + DEPENDS "${COVERAGE_REPORT_FILE}" + DEPENDS "${COVERAGE_OVERALL_REPORT_FILE}" + DEPENDS "${COVERAGE_HTML_REPORT_FILE}" ) + # add targets to global coverage target if(NOT TARGET coverage) add_custom_target(coverage) endif() diff --git a/tests/calculateoverallcoverage.awk b/tests/calculateoverallcoverage.awk new file mode 100644 index 0000000..bc44e52 --- /dev/null +++ b/tests/calculateoverallcoverage.awk @@ -0,0 +1,24 @@ +# Calculates total statistics for source-based code coverage report created +# with `llvm-cov report` when at least one source file has been specified. + +# NOTE: When at least one source file is passed to `llvm-cov`, the summaries +# are shown for each function in the listed files (and not for each file in the +# coverage data). + +{ + if($1 == "TOTAL") { + covered_regions += $2; + missed_regions += $3; + covered_lines += $5; + missed_lines += $6; + } +} + +END { + print "Covered regions: " covered_regions; + print "Missed regions: " missed_regions; + print "Region cover: " covered_regions/(covered_regions+missed_regions)*100 "%\n"; + print "Covered lines: " covered_lines; + print "Missed lines: " missed_lines; + print "Line cover: " covered_lines/(covered_lines+missed_lines)*100 "%"; +}