Merge prof data of subprocesses created when executing tests
So coverage is accounted correctly when tests create subprocesses of the application to be tested using execApp().
This commit is contained in:
parent
1e14b09e64
commit
57d5d04d9f
|
@ -133,14 +133,20 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
|
|||
# enable source code based coverage analysis using clang
|
||||
if(CLANG_SOURCE_BASED_COVERAGE_ENABLED)
|
||||
# specify where to store raw clang profiling data via environment variable
|
||||
set(LLVM_PROFILE_RAW_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw")
|
||||
set(LLVM_PROFILE_RAW_LIST_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw.list")
|
||||
set(LLVM_PROFILE_DATA_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata")
|
||||
set_tests_properties(${META_PROJECT_NAME}_run_tests
|
||||
PROPERTIES
|
||||
ENVIRONMENT "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
|
||||
PROPERTIES ENVIRONMENT
|
||||
"LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE};LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}"
|
||||
)
|
||||
add_custom_command(
|
||||
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
|
||||
OUTPUT "${LLVM_PROFILE_RAW_FILE}"
|
||||
"${LLVM_PROFILE_RAW_LIST_FILE}"
|
||||
COMMAND "${CMAKE_COMMAND}"
|
||||
-E env "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
|
||||
-E env
|
||||
"LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE}"
|
||||
"LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}"
|
||||
$<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests>
|
||||
-p "${CMAKE_CURRENT_SOURCE_DIR}/testfiles"
|
||||
-w "${CMAKE_CURRENT_BINARY_DIR}/testworkingdir"
|
||||
|
@ -152,22 +158,25 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
|
|||
find_program(LLVM_COV_BIN llvm-cov)
|
||||
if(LLVM_PROFDATA_BIN AND LLVM_COV_BIN)
|
||||
add_custom_command(
|
||||
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
|
||||
COMMAND "${LLVM_PROFDATA_BIN}" merge
|
||||
-sparse "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
|
||||
-o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
|
||||
OUTPUT "${LLVM_PROFILE_DATA_FILE}"
|
||||
COMMAND cat "${LLVM_PROFILE_RAW_LIST_FILE}" | xargs
|
||||
"${LLVM_PROFDATA_BIN}" merge
|
||||
-o "${LLVM_PROFILE_DATA_FILE}"
|
||||
-sparse
|
||||
"${LLVM_PROFILE_RAW_FILE}"
|
||||
COMMENT "Generating profiling data for source-based coverage report"
|
||||
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
|
||||
DEPENDS "${LLVM_PROFILE_RAW_FILE}"
|
||||
"${LLVM_PROFILE_RAW_LIST_FILE}"
|
||||
)
|
||||
add_custom_command(
|
||||
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
|
||||
COMMAND "${LLVM_COV_BIN}" report
|
||||
-instr-profile "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
|
||||
-instr-profile "${LLVM_PROFILE_DATA_FILE}"
|
||||
-format=text
|
||||
$<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}>
|
||||
> "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
|
||||
COMMENT "Generating HTML coverage report"
|
||||
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
|
||||
DEPENDS "${LLVM_PROFILE_DATA_FILE}"
|
||||
)
|
||||
add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_summary"
|
||||
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
|
||||
|
@ -176,12 +185,12 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
|
|||
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"
|
||||
COMMAND "${LLVM_COV_BIN}" show
|
||||
-project-title="${META_APP_NAME}"
|
||||
-instr-profile "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
|
||||
-instr-profile "${LLVM_PROFILE_DATA_FILE}"
|
||||
-format=html
|
||||
$<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}>
|
||||
> "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"
|
||||
COMMENT "Generating HTML coverage report"
|
||||
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
|
||||
DEPENDS "${LLVM_PROFILE_DATA_FILE}"
|
||||
)
|
||||
add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_html"
|
||||
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
#include "./testutils.h"
|
||||
|
||||
#include "../application/failure.h"
|
||||
#include "../conversion/stringbuilder.h"
|
||||
#include "../conversion/stringconversion.h"
|
||||
#include "../io/catchiofailure.h"
|
||||
|
||||
|
@ -76,10 +77,14 @@ TestApplication::TestApplication(int argc, char **argv)
|
|||
// parse arguments
|
||||
try {
|
||||
m_parser.parseArgs(argc, argv);
|
||||
|
||||
// print help
|
||||
if (m_helpArg.isPresent()) {
|
||||
m_valid = false;
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// handle path for testfiles and working-copy
|
||||
cerr << "Directories used to search for testfiles:" << endl;
|
||||
if (m_testFilesPathArg.isPresent()) {
|
||||
if (*m_testFilesPathArg.values().front()) {
|
||||
|
@ -116,6 +121,11 @@ TestApplication::TestApplication(int argc, char **argv)
|
|||
}
|
||||
cerr << m_workingDir << endl << endl;
|
||||
|
||||
// clear list of all additional profiling files created when forking the test application
|
||||
if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
|
||||
ofstream(profrawListFile, ios_base::trunc);
|
||||
}
|
||||
|
||||
m_valid = true;
|
||||
cerr << "Executing test cases ..." << endl;
|
||||
} catch (const Failure &failure) {
|
||||
|
@ -229,20 +239,11 @@ string TestApplication::workingCopyPath(const string &name) const
|
|||
}
|
||||
|
||||
/*!
|
||||
* \brief Executes the application to be tested with the specified \a args and stores the standard output and
|
||||
* errors in \a stdout and \a stderr.
|
||||
* \throws Throws std::runtime_error when the application can not be executed.
|
||||
* \remarks
|
||||
* - The specified \a args must be 0 terminated. The first argument is the application name.
|
||||
* - Currently only supported under UNIX.
|
||||
* - \a stdout and \a stderr are cleared before.
|
||||
* \brief Executes an application with the specified \a args.
|
||||
* \remarks Provides internal implementation of execApp() and execHelperApp().
|
||||
*/
|
||||
int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
|
||||
{
|
||||
return execHelperApp(m_applicationPathArg.firstValue(), args, output, errors, suppressLogging, timeout);
|
||||
}
|
||||
|
||||
int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
|
||||
int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
|
||||
const std::string &newProfilingPath)
|
||||
{
|
||||
// print log message
|
||||
if (!suppressLogging) {
|
||||
|
@ -252,15 +253,13 @@ int execHelperApp(const char *appPath, const char *const *args, std::string &out
|
|||
}
|
||||
cout << endl;
|
||||
}
|
||||
// determine application path
|
||||
if (!appPath || !*appPath) {
|
||||
throw runtime_error("Unable to execute application to be tested: no application path specified");
|
||||
}
|
||||
|
||||
// create pipes
|
||||
int coutPipes[2], cerrPipes[2];
|
||||
pipe(coutPipes), pipe(cerrPipes);
|
||||
int readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
|
||||
int readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
|
||||
|
||||
// create child process
|
||||
if (int child = fork()) {
|
||||
// parent process: read stdout and stderr from child
|
||||
|
@ -322,13 +321,75 @@ int execHelperApp(const char *appPath, const char *const *args, std::string &out
|
|||
waitpid(child, &childReturnCode, 0);
|
||||
return childReturnCode;
|
||||
} else {
|
||||
// child process: set pipes to be used for stdout/stderr, execute application
|
||||
// child process
|
||||
// -> set pipes to be used for stdout/stderr
|
||||
dup2(writeCoutPipe, STDOUT_FILENO), dup2(writeCerrPipe, STDERR_FILENO);
|
||||
close(readCoutPipe), close(writeCoutPipe), close(readCerrPipe), close(writeCerrPipe);
|
||||
|
||||
// -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
|
||||
if (!newProfilingPath.empty()) {
|
||||
setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
|
||||
}
|
||||
|
||||
// -> execute application
|
||||
execv(appPath, const_cast<char *const *>(args));
|
||||
cerr << "Unable to execute \"" << appPath << "\": execv() failed" << endl;
|
||||
exit(-101);
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Executes the application to be tested with the specified \a args and stores the standard output and
|
||||
* errors in \a stdout and \a stderr.
|
||||
* \throws Throws std::runtime_error when the application can not be executed.
|
||||
* \remarks
|
||||
* - The specified \a args must be 0 terminated. The first argument is the application name.
|
||||
* - Currently only supported under UNIX.
|
||||
* - \a stdout and \a stderr are cleared before.
|
||||
*/
|
||||
int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
|
||||
{
|
||||
// increase counter used for giving profiling files unique names
|
||||
static unsigned int invocationCount = 0;
|
||||
++invocationCount;
|
||||
|
||||
// determine application path
|
||||
const char *const appPath = m_applicationPathArg.firstValue();
|
||||
if (!appPath || !*appPath) {
|
||||
throw runtime_error("Unable to execute application to be tested: no application path specified");
|
||||
}
|
||||
|
||||
// determine new path for profiling output (to not override profiling output of parent and previous invocations)
|
||||
string newProfilingPath;
|
||||
if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
|
||||
// replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
|
||||
if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
|
||||
const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
|
||||
// extract application name from path
|
||||
const char *appName = strrchr(appPath, '/');
|
||||
appName = appName ? appName + 1 : appPath;
|
||||
// concat new path
|
||||
newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
|
||||
// append path to profiling list file
|
||||
if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
|
||||
ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Executes an application with the specified \a args.
|
||||
* \remarks
|
||||
* - Intended to invoke helper applications (eg. to setup test files). Use execApp() and TestApplication::execApp() to
|
||||
* invoke the application to be tested itself.
|
||||
* - Currently only supported under UNIX.
|
||||
*/
|
||||
int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
|
||||
{
|
||||
return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
|
|
@ -124,11 +124,6 @@ inline CPP_UTILITIES_EXPORT int execApp(const char *const *args, std::string &ou
|
|||
return TestApplication::instance()->execApp(args, output, errors);
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Executes an application with the specified \a args.
|
||||
* \remarks Intended to invoke helper applications (eg. to setup test files). Use execApp() to invoke the application
|
||||
* to be tested itself.
|
||||
*/
|
||||
CPP_UTILITIES_EXPORT int execHelperApp(
|
||||
const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging = false, int timeout = -1);
|
||||
#endif
|
||||
|
|
Loading…
Reference in New Issue