Merge prof data of subprocesses created when executing tests

So coverage is accounted correctly when tests create subprocesses
of the application to be tested using execApp().

cmake/modules/TestTarget.cmake

@@ -133,14 +133,20 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
     # enable source code based coverage analysis using clang
     if(CLANG_SOURCE_BASED_COVERAGE_ENABLED)
         # specify where to store raw clang profiling data via environment variable
+        set(LLVM_PROFILE_RAW_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw")
+        set(LLVM_PROFILE_RAW_LIST_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw.list")
+        set(LLVM_PROFILE_DATA_FILE "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata")
         set_tests_properties(${META_PROJECT_NAME}_run_tests
-            PROPERTIES
+            PROPERTIES ENVIRONMENT
-            ENVIRONMENT "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
+                "LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE};LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}"
         )
         add_custom_command(
-            OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
+            OUTPUT "${LLVM_PROFILE_RAW_FILE}"
+                   "${LLVM_PROFILE_RAW_LIST_FILE}"
             COMMAND "${CMAKE_COMMAND}"
-                    -E env "LLVM_PROFILE_FILE=${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
+                    -E env
+                        "LLVM_PROFILE_FILE=${LLVM_PROFILE_RAW_FILE}"
+                        "LLVM_PROFILE_LIST_FILE=${LLVM_PROFILE_RAW_LIST_FILE}"
                     $<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests>
                         -p "${CMAKE_CURRENT_SOURCE_DIR}/testfiles"
                         -w "${CMAKE_CURRENT_BINARY_DIR}/testworkingdir"
@@ -152,22 +158,25 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
         find_program(LLVM_COV_BIN llvm-cov)
         if(LLVM_PROFDATA_BIN AND LLVM_COV_BIN)
             add_custom_command(
-                OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
+                OUTPUT "${LLVM_PROFILE_DATA_FILE}"
-                COMMAND "${LLVM_PROFDATA_BIN}" merge
+                COMMAND cat "${LLVM_PROFILE_RAW_LIST_FILE}" | xargs
-                        -sparse "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
+                        "${LLVM_PROFDATA_BIN}" merge
-                        -o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
+                        -o "${LLVM_PROFILE_DATA_FILE}"
+                        -sparse
+                        "${LLVM_PROFILE_RAW_FILE}"
                 COMMENT "Generating profiling data for source-based coverage report"
-                DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profraw"
+                DEPENDS "${LLVM_PROFILE_RAW_FILE}"
+                        "${LLVM_PROFILE_RAW_LIST_FILE}"
             )
             add_custom_command(
                 OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
                 COMMAND "${LLVM_COV_BIN}" report
-                    -instr-profile "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
+                    -instr-profile "${LLVM_PROFILE_DATA_FILE}"
                     -format=text
                     $<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}>
                     > "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
                 COMMENT "Generating HTML coverage report"
-                DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
+                DEPENDS "${LLVM_PROFILE_DATA_FILE}"
             )
             add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_summary"
                 DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.txt"
@@ -176,12 +185,12 @@ if(CPP_UNIT_LIB OR META_NO_CPP_UNIT)
                 OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"
                 COMMAND "${LLVM_COV_BIN}" show
                     -project-title="${META_APP_NAME}"
-                    -instr-profile "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
+                    -instr-profile "${LLVM_PROFILE_DATA_FILE}"
                     -format=html
                     $<TARGET_FILE:${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}>
                     > "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"
                 COMMENT "Generating HTML coverage report"
-                DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests.profdata"
+                DEPENDS "${LLVM_PROFILE_DATA_FILE}"
             )
             add_custom_target("${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage_html"
                 DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_PREFIX}${META_PROJECT_NAME}${TARGET_SUFFIX}_tests_coverage.html"

tests/testutils.cpp

@@ -1,6 +1,7 @@
 #include "./testutils.h"
 
 #include "../application/failure.h"
+#include "../conversion/stringbuilder.h"
 #include "../conversion/stringconversion.h"
 #include "../io/catchiofailure.h"
 
@@ -76,10 +77,14 @@ TestApplication::TestApplication(int argc, char **argv)
     // parse arguments
     try {
         m_parser.parseArgs(argc, argv);
+
+        // print help
         if (m_helpArg.isPresent()) {
             m_valid = false;
             exit(0);
         }
+
+        // handle path for testfiles and working-copy
         cerr << "Directories used to search for testfiles:" << endl;
         if (m_testFilesPathArg.isPresent()) {
             if (*m_testFilesPathArg.values().front()) {
@@ -116,6 +121,11 @@ TestApplication::TestApplication(int argc, char **argv)
         }
         cerr << m_workingDir << endl << endl;
 
+        // clear list of all additional profiling files created when forking the test application
+        if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
+            ofstream(profrawListFile, ios_base::trunc);
+        }
+
         m_valid = true;
         cerr << "Executing test cases ..." << endl;
     } catch (const Failure &failure) {
@@ -229,20 +239,11 @@ string TestApplication::workingCopyPath(const string &name) const
 }
 
 /*!
- * \brief Executes the application to be tested with the specified \a args and stores the standard output and
+ * \brief Executes an application with the specified \a args.
- *        errors in \a stdout and \a stderr.
+ * \remarks Provides internal implementation of execApp() and execHelperApp().
- * \throws Throws std::runtime_error when the application can not be executed.
- * \remarks
- *  - The specified \a args must be 0 terminated. The first argument is the application name.
- *  - Currently only supported under UNIX.
- *  - \a stdout and \a stderr are cleared before.
  */
-int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
+int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
-{
+    const std::string &newProfilingPath)
-    return execHelperApp(m_applicationPathArg.firstValue(), args, output, errors, suppressLogging, timeout);
-}
-
-int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
 {
     // print log message
     if (!suppressLogging) {
@@ -252,15 +253,13 @@ int execHelperApp(const char *appPath, const char *const *args, std::string &out
         }
         cout << endl;
     }
-    // determine application path
+
-    if (!appPath || !*appPath) {
-        throw runtime_error("Unable to execute application to be tested: no application path specified");
-    }
     // create pipes
     int coutPipes[2], cerrPipes[2];
     pipe(coutPipes), pipe(cerrPipes);
     int readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
     int readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
+
     // create child process
     if (int child = fork()) {
         // parent process: read stdout and stderr from child
@@ -322,13 +321,75 @@ int execHelperApp(const char *appPath, const char *const *args, std::string &out
         waitpid(child, &childReturnCode, 0);
         return childReturnCode;
     } else {
-        // child process: set pipes to be used for stdout/stderr, execute application
+        // child process
+        // -> set pipes to be used for stdout/stderr
         dup2(writeCoutPipe, STDOUT_FILENO), dup2(writeCerrPipe, STDERR_FILENO);
         close(readCoutPipe), close(writeCoutPipe), close(readCerrPipe), close(writeCerrPipe);
+
+        // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
+        if (!newProfilingPath.empty()) {
+            setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
+        }
+
+        // -> execute application
         execv(appPath, const_cast<char *const *>(args));
         cerr << "Unable to execute \"" << appPath << "\": execv() failed" << endl;
         exit(-101);
     }
 }
+
+/*!
+ * \brief Executes the application to be tested with the specified \a args and stores the standard output and
+ *        errors in \a stdout and \a stderr.
+ * \throws Throws std::runtime_error when the application can not be executed.
+ * \remarks
+ *  - The specified \a args must be 0 terminated. The first argument is the application name.
+ *  - Currently only supported under UNIX.
+ *  - \a stdout and \a stderr are cleared before.
+ */
+int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
+{
+    // increase counter used for giving profiling files unique names
+    static unsigned int invocationCount = 0;
+    ++invocationCount;
+
+    // determine application path
+    const char *const appPath = m_applicationPathArg.firstValue();
+    if (!appPath || !*appPath) {
+        throw runtime_error("Unable to execute application to be tested: no application path specified");
+    }
+
+    // determine new path for profiling output (to not override profiling output of parent and previous invocations)
+    string newProfilingPath;
+    if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
+        // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
+        if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
+            const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
+            // extract application name from path
+            const char *appName = strrchr(appPath, '/');
+            appName = appName ? appName + 1 : appPath;
+            // concat new path
+            newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
+            // append path to profiling list file
+            if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
+                ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
+            }
+        }
+    }
+
+    return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
+}
+
+/*!
+ * \brief Executes an application with the specified \a args.
+ * \remarks
+ * - Intended to invoke helper applications (eg. to setup test files). Use execApp() and TestApplication::execApp() to
+ *   invoke the application to be tested itself.
+ * - Currently only supported under UNIX.
+ */
+int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
+{
+    return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
+}
 #endif
 }

tests/testutils.h

@@ -124,11 +124,6 @@ inline CPP_UTILITIES_EXPORT int execApp(const char *const *args, std::string &ou
     return TestApplication::instance()->execApp(args, output, errors);
 }
 
-/*!
- * \brief Executes an application with the specified \a args.
- * \remarks Intended to invoke helper applications (eg. to setup test files). Use execApp() to invoke the application
- *          to be tested itself.
- */
 CPP_UTILITIES_EXPORT int execHelperApp(
     const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging = false, int timeout = -1);
 #endif