diff --git a/cmake/modules/TestTarget.cmake b/cmake/modules/TestTarget.cmake index 5e427ae..c990a43 100644 --- a/cmake/modules/TestTarget.cmake +++ b/cmake/modules/TestTarget.cmake @@ -133,14 +133,20 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT) # enable source code based coverage analysis using clang if(CLANG_SOURCE_BASED_COVERAGE_ENABLED) # specify where to store raw clang profiling data via environment variable + set(LLVM_PROFILE_RAW_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw") + set(LLVM_PROFILE_RAW_LIST_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw.list") + set(LLVM_PROFILE_DATA_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata") set_tests_properties(${META_PROJECT_NAME}_run_tests - PROPERTIES - ENVIRONMENT "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw" + PROPERTIES ENVIRONMENT + "LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE};LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}" ) add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw" + OUTPUT "${LLVM_PROFILE_RAW_FILE}" + "${LLVM_PROFILE_RAW_LIST_FILE}" COMMAND "${CMAKE_COMMAND}" - -E env "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw" + -E env + "LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE}" + "LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}" $ -p "${CMAKE_CURRENT_SOURCE_DIR}/testfiles" -w "${CMAKE_CURRENT_BINARY_DIR}/testworkingdir" @@ -152,22 +158,25 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT) find_program(LLVM_COV_BIN llvm-cov) if(LLVM_PROFDATA_BIN AND LLVM_COV_BIN) add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata" - COMMAND "${LLVM_PROFDATA_BIN}" merge - -sparse "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw" - -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata" + OUTPUT "${LLVM_PROFILE_DATA_FILE}" + COMMAND cat "${LLVM_PROFILE_RAW_LIST_FILE}" | xargs + "${LLVM_PROFDATA_BIN}" merge + -o "${LLVM_PROFILE_DATA_FILE}" + -sparse + "${LLVM_PROFILE_RAW_FILE}" COMMENT "Generating profiling data for source-based coverage report" - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw" + DEPENDS "${LLVM_PROFILE_RAW_FILE}" + "${LLVM_PROFILE_RAW_LIST_FILE}" ) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt" COMMAND "${LLVM_COV_BIN}" report - -instr-profile "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata" + -instr-profile "${LLVM_PROFILE_DATA_FILE}" -format=text $ > "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt" COMMENT "Generating HTML coverage report" - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata" + DEPENDS "${LLVM_PROFILE_DATA_FILE}" ) add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_summary" DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt" @@ -176,12 +185,12 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT) OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html" COMMAND "${LLVM_COV_BIN}" show -project-title="${META_APP_NAME}" - -instr-profile "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata" + -instr-profile "${LLVM_PROFILE_DATA_FILE}" -format=html $ > "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html" COMMENT "Generating HTML coverage report" - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata" + DEPENDS "${LLVM_PROFILE_DATA_FILE}" ) add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_html" DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html" diff --git a/tests/testutils.cpp b/tests/testutils.cpp index 103d180..67ef671 100644 --- a/tests/testutils.cpp +++ b/tests/testutils.cpp @@ -1,6 +1,7 @@ #include "./testutils.h" #include "../application/failure.h" +#include "../conversion/stringbuilder.h" #include "../conversion/stringconversion.h" #include "../io/catchiofailure.h" @@ -76,10 +77,14 @@ TestApplication::TestApplication(int argc, char **argv) // parse arguments try { m_parser.parseArgs(argc, argv); + + // print help if (m_helpArg.isPresent()) { m_valid = false; exit(0); } + + // handle path for testfiles and working-copy cerr << "Directories used to search for testfiles:" << endl; if (m_testFilesPathArg.isPresent()) { if (*m_testFilesPathArg.values().front()) { @@ -116,6 +121,11 @@ TestApplication::TestApplication(int argc, char **argv) } cerr << m_workingDir << endl << endl; + // clear list of all additional profiling files created when forking the test application + if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) { + ofstream(profrawListFile, ios_base::trunc); + } + m_valid = true; cerr << "Executing test cases ..." << endl; } catch (const Failure &failure) { @@ -229,20 +239,11 @@ string TestApplication::workingCopyPath(const string &name) const } /*! - * \brief Executes the application to be tested with the specified \a args and stores the standard output and - * errors in \a stdout and \a stderr. - * \throws Throws std::runtime_error when the application can not be executed. - * \remarks - * - The specified \a args must be 0 terminated. The first argument is the application name. - * - Currently only supported under UNIX. - * - \a stdout and \a stderr are cleared before. + * \brief Executes an application with the specified \a args. + * \remarks Provides internal implementation of execApp() and execHelperApp(). */ -int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const -{ - return execHelperApp(m_applicationPathArg.firstValue(), args, output, errors, suppressLogging, timeout); -} - -int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout) +int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout, + const std::string &newProfilingPath) { // print log message if (!suppressLogging) { @@ -252,15 +253,13 @@ int execHelperApp(const char *appPath, const char *const *args, std::string &out } cout << endl; } - // determine application path - if (!appPath || !*appPath) { - throw runtime_error("Unable to execute application to be tested: no application path specified"); - } + // create pipes int coutPipes[2], cerrPipes[2]; pipe(coutPipes), pipe(cerrPipes); int readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1]; int readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1]; + // create child process if (int child = fork()) { // parent process: read stdout and stderr from child @@ -322,13 +321,75 @@ int execHelperApp(const char *appPath, const char *const *args, std::string &out waitpid(child, &childReturnCode, 0); return childReturnCode; } else { - // child process: set pipes to be used for stdout/stderr, execute application + // child process + // -> set pipes to be used for stdout/stderr dup2(writeCoutPipe, STDOUT_FILENO), dup2(writeCerrPipe, STDERR_FILENO); close(readCoutPipe), close(writeCoutPipe), close(readCerrPipe), close(writeCerrPipe); + + // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output + if (!newProfilingPath.empty()) { + setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true); + } + + // -> execute application execv(appPath, const_cast(args)); cerr << "Unable to execute \"" << appPath << "\": execv() failed" << endl; exit(-101); } } + +/*! + * \brief Executes the application to be tested with the specified \a args and stores the standard output and + * errors in \a stdout and \a stderr. + * \throws Throws std::runtime_error when the application can not be executed. + * \remarks + * - The specified \a args must be 0 terminated. The first argument is the application name. + * - Currently only supported under UNIX. + * - \a stdout and \a stderr are cleared before. + */ +int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const +{ + // increase counter used for giving profiling files unique names + static unsigned int invocationCount = 0; + ++invocationCount; + + // determine application path + const char *const appPath = m_applicationPathArg.firstValue(); + if (!appPath || !*appPath) { + throw runtime_error("Unable to execute application to be tested: no application path specified"); + } + + // determine new path for profiling output (to not override profiling output of parent and previous invocations) + string newProfilingPath; + if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) { + // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw" + if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) { + const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd); + // extract application name from path + const char *appName = strrchr(appPath, '/'); + appName = appName ? appName + 1 : appPath; + // concat new path + newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw"); + // append path to profiling list file + if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) { + ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl; + } + } + } + + return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath); +} + +/*! + * \brief Executes an application with the specified \a args. + * \remarks + * - Intended to invoke helper applications (eg. to setup test files). Use execApp() and TestApplication::execApp() to + * invoke the application to be tested itself. + * - Currently only supported under UNIX. + */ +int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout) +{ + return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string()); +} #endif } diff --git a/tests/testutils.h b/tests/testutils.h index d7a42f3..e31a6cf 100644 --- a/tests/testutils.h +++ b/tests/testutils.h @@ -124,11 +124,6 @@ inline CPP_UTILITIES_EXPORT int execApp(const char *const *args, std::string &ou return TestApplication::instance()->execApp(args, output, errors); } -/*! - * \brief Executes an application with the specified \a args. - * \remarks Intended to invoke helper applications (eg. to setup test files). Use execApp() to invoke the application - * to be tested itself. - */ CPP_UTILITIES_EXPORT int execHelperApp( const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging = false, int timeout = -1); #endif