Merge prof data of subprocesses created when executing tests

So coverage is accounted correctly when tests create subprocesses
of the application to be tested using execApp().
testing/bash_completion_debugging
Martchus 6 years ago
parent 1e14b09e64
commit 57d5d04d9f
  1. 35
      cmake/modules/TestTarget.cmake
  2. 97
      tests/testutils.cpp
  3. 5
      tests/testutils.h

@ -133,14 +133,20 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
# enable source code based coverage analysis using clang
if(CLANG_SOURCE_BASED_COVERAGE_ENABLED)
# specify where to store raw clang profiling data via environment variable
set(LLVM_PROFILE_RAW_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw")
set(LLVM_PROFILE_RAW_LIST_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw.list")
set(LLVM_PROFILE_DATA_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata")
set_tests_properties(${META_PROJECT_NAME}_run_tests
PROPERTIES
ENVIRONMENT "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
PROPERTIES ENVIRONMENT
"LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE};LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}"
)
add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
OUTPUT "${LLVM_PROFILE_RAW_FILE}"
"${LLVM_PROFILE_RAW_LIST_FILE}"
COMMAND "${CMAKE_COMMAND}"
-E env "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
-E env
"LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE}"
"LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}"
$<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests>
-p "${CMAKE_CURRENT_SOURCE_DIR}/testfiles"
-w "${CMAKE_CURRENT_BINARY_DIR}/testworkingdir"
@ -152,22 +158,25 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
find_program(LLVM_COV_BIN llvm-cov)
if(LLVM_PROFDATA_BIN AND LLVM_COV_BIN)
add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
COMMAND "${LLVM_PROFDATA_BIN}" merge
-sparse "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
-o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
OUTPUT "${LLVM_PROFILE_DATA_FILE}"
COMMAND cat "${LLVM_PROFILE_RAW_LIST_FILE}" | xargs
"${LLVM_PROFDATA_BIN}" merge
-o "${LLVM_PROFILE_DATA_FILE}"
-sparse
"${LLVM_PROFILE_RAW_FILE}"
COMMENT "Generating profiling data for source-based coverage report"
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
DEPENDS "${LLVM_PROFILE_RAW_FILE}"
"${LLVM_PROFILE_RAW_LIST_FILE}"
)
add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
COMMAND "${LLVM_COV_BIN}" report
-instr-profile "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
-instr-profile "${LLVM_PROFILE_DATA_FILE}"
-format=text
$<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}>
> "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
COMMENT "Generating HTML coverage report"
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
DEPENDS "${LLVM_PROFILE_DATA_FILE}"
)
add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_summary"
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
@ -176,12 +185,12 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"
COMMAND "${LLVM_COV_BIN}" show
-project-title="${META_APP_NAME}"
-instr-profile "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
-instr-profile "${LLVM_PROFILE_DATA_FILE}"
-format=html
$<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}>
> "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"
COMMENT "Generating HTML coverage report"
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
DEPENDS "${LLVM_PROFILE_DATA_FILE}"
)
add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_html"
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"

@ -1,6 +1,7 @@
#include "./testutils.h"
#include "../application/failure.h"
#include "../conversion/stringbuilder.h"
#include "../conversion/stringconversion.h"
#include "../io/catchiofailure.h"
@ -76,10 +77,14 @@ TestApplication::TestApplication(int argc, char **argv)
// parse arguments
try {
m_parser.parseArgs(argc, argv);
// print help
if (m_helpArg.isPresent()) {
m_valid = false;
exit(0);
}
// handle path for testfiles and working-copy
cerr << "Directories used to search for testfiles:" << endl;
if (m_testFilesPathArg.isPresent()) {
if (*m_testFilesPathArg.values().front()) {
@ -116,6 +121,11 @@ TestApplication::TestApplication(int argc, char **argv)
}
cerr << m_workingDir << endl << endl;
// clear list of all additional profiling files created when forking the test application
if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
ofstream(profrawListFile, ios_base::trunc);
}
m_valid = true;
cerr << "Executing test cases ..." << endl;
} catch (const Failure &failure) {
@ -229,20 +239,11 @@ string TestApplication::workingCopyPath(const string &name) const
}
/*!
* \brief Executes the application to be tested with the specified \a args and stores the standard output and
* errors in \a stdout and \a stderr.
* \throws Throws std::runtime_error when the application can not be executed.
* \remarks
* - The specified \a args must be 0 terminated. The first argument is the application name.
* - Currently only supported under UNIX.
* - \a stdout and \a stderr are cleared before.
* \brief Executes an application with the specified \a args.
* \remarks Provides internal implementation of execApp() and execHelperApp().
*/
int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
{
return execHelperApp(m_applicationPathArg.firstValue(), args, output, errors, suppressLogging, timeout);
}
int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
const std::string &newProfilingPath)
{
// print log message
if (!suppressLogging) {
@ -252,15 +253,13 @@ int execHelperApp(const char *appPath, const char *const *args, std::string &out
}
cout << endl;
}
// determine application path
if (!appPath || !*appPath) {
throw runtime_error("Unable to execute application to be tested: no application path specified");
}
// create pipes
int coutPipes[2], cerrPipes[2];
pipe(coutPipes), pipe(cerrPipes);
int readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
int readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
// create child process
if (int child = fork()) {
// parent process: read stdout and stderr from child
@ -322,13 +321,75 @@ int execHelperApp(const char *appPath, const char *const *args, std::string &out
waitpid(child, &childReturnCode, 0);
return childReturnCode;
} else {
// child process: set pipes to be used for stdout/stderr, execute application
// child process
// -> set pipes to be used for stdout/stderr
dup2(writeCoutPipe, STDOUT_FILENO), dup2(writeCerrPipe, STDERR_FILENO);
close(readCoutPipe), close(writeCoutPipe), close(readCerrPipe), close(writeCerrPipe);
// -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
if (!newProfilingPath.empty()) {
setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
}
// -> execute application
execv(appPath, const_cast<char *const *>(args));
cerr << "Unable to execute \"" << appPath << "\": execv() failed" << endl;
exit(-101);
}
}
/*!
* \brief Executes the application to be tested with the specified \a args and stores the standard output and
* errors in \a stdout and \a stderr.
* \throws Throws std::runtime_error when the application can not be executed.
* \remarks
* - The specified \a args must be 0 terminated. The first argument is the application name.
* - Currently only supported under UNIX.
* - \a stdout and \a stderr are cleared before.
*/
int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
{
// increase counter used for giving profiling files unique names
static unsigned int invocationCount = 0;
++invocationCount;
// determine application path
const char *const appPath = m_applicationPathArg.firstValue();
if (!appPath || !*appPath) {
throw runtime_error("Unable to execute application to be tested: no application path specified");
}
// determine new path for profiling output (to not override profiling output of parent and previous invocations)
string newProfilingPath;
if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
// replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
// extract application name from path
const char *appName = strrchr(appPath, '/');
appName = appName ? appName + 1 : appPath;
// concat new path
newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
// append path to profiling list file
if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
}
}
}
return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
}
/*!
* \brief Executes an application with the specified \a args.
* \remarks
* - Intended to invoke helper applications (eg. to setup test files). Use execApp() and TestApplication::execApp() to
* invoke the application to be tested itself.
* - Currently only supported under UNIX.
*/
int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
{
return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
}
#endif
}

@ -124,11 +124,6 @@ inline CPP_UTILITIES_EXPORT int execApp(const char *const *args, std::string &ou
return TestApplication::instance()->execApp(args, output, errors);
}
/*!
* \brief Executes an application with the specified \a args.
* \remarks Intended to invoke helper applications (eg. to setup test files). Use execApp() to invoke the application
* to be tested itself.
*/
CPP_UTILITIES_EXPORT int execHelperApp(
const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging = false, int timeout = -1);
#endif

Loading…
Cancel
Save