C++ Utilities  5.3.0
Useful C++ classes and routines such as argument parser, IO and conversion utilities
testutils.cpp
Go to the documentation of this file.
1 #include "./testutils.h"
2 
3 #include "../conversion/stringbuilder.h"
4 #include "../conversion/stringconversion.h"
5 #include "../io/ansiescapecodes.h"
6 #include "../io/misc.h"
7 #include "../io/nativefilestream.h"
8 #include "../io/path.h"
9 #include "../misc/parseerror.h"
10 
11 #include <cerrno>
12 #include <cstdlib>
13 #include <cstring>
14 #include <fstream>
15 #include <initializer_list>
16 #include <iostream>
17 #include <limits>
18 
19 #ifdef PLATFORM_UNIX
20 #include <poll.h>
21 #include <sys/stat.h>
22 #include <sys/wait.h>
23 #include <unistd.h>
24 #endif
25 
26 #ifdef PLATFORM_WINDOWS
27 #include <windows.h>
28 #endif
29 
30 using namespace std;
31 using namespace CppUtilities::EscapeCodes;
32 
36 namespace CppUtilities {
37 
38 bool fileSystemItemExists(const string &path)
39 {
40 #ifdef PLATFORM_UNIX
41  struct stat res;
42  return stat(path.data(), &res) == 0;
43 #else
44  const auto widePath(convertMultiByteToWide(path));
45  if (!widePath.first) {
46  return false;
47  }
48  const auto fileType(GetFileAttributesW(widePath.first.get()));
49  return fileType != INVALID_FILE_ATTRIBUTES;
50 #endif
51 }
52 
53 bool fileExists(const string &path)
54 {
55 #ifdef PLATFORM_UNIX
56  struct stat res;
57  return stat(path.data(), &res) == 0 && !S_ISDIR(res.st_mode);
58 #else
59  const auto widePath(convertMultiByteToWide(path));
60  if (!widePath.first) {
61  return false;
62  }
63  const auto fileType(GetFileAttributesW(widePath.first.get()));
64  return (fileType != INVALID_FILE_ATTRIBUTES) && !(fileType & FILE_ATTRIBUTE_DIRECTORY) && !(fileType & FILE_ATTRIBUTE_DEVICE);
65 #endif
66 }
67 
68 bool dirExists(const string &path)
69 {
70 #ifdef PLATFORM_UNIX
71  struct stat res;
72  return stat(path.data(), &res) == 0 && S_ISDIR(res.st_mode);
73 #else
74  const auto widePath(convertMultiByteToWide(path));
75  if (!widePath.first) {
76  return false;
77  }
78  const auto fileType(GetFileAttributesW(widePath.first.get()));
79  return (fileType != INVALID_FILE_ATTRIBUTES) && (fileType & FILE_ATTRIBUTE_DIRECTORY);
80 #endif
81 }
82 
83 bool makeDir(const string &path)
84 {
85 #ifdef PLATFORM_UNIX
86  return mkdir(path.data(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0;
87 #else
88  const auto widePath(convertMultiByteToWide(path));
89  if (!widePath.first) {
90  return false;
91  }
92  return CreateDirectoryW(widePath.first.get(), nullptr) || GetLastError() == ERROR_ALREADY_EXISTS;
93 #endif
94 }
95 
96 TestApplication *TestApplication::s_instance = nullptr;
97 
108 TestApplication::TestApplication()
109  : TestApplication(0, nullptr)
110 {
111 }
112 
117 TestApplication::TestApplication(int argc, const char *const *argv)
118  : m_listArg("list", 'l', "lists available test units")
119  , m_runArg("run", 'r', "runs the tests")
120  , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files", { "path" })
121  , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested", { "path" })
122  , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files", { "path" })
123  , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units", { "unit1", "unit2", "unit3" })
124 {
125  // check whether there is already an instance
126  if (s_instance) {
127  throw runtime_error("only one TestApplication instance allowed at a time");
128  }
129  s_instance = this;
130 
131  // handle specified arguments (if present)
132  if (argc && argv) {
133  // setup argument parser
134  m_testFilesPathArg.setRequiredValueCount(Argument::varValueCount);
135  m_unitsArg.setRequiredValueCount(Argument::varValueCount);
136  m_runArg.setImplicit(true);
137  m_runArg.setSubArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg });
138  m_parser.setMainArguments({ &m_runArg, &m_listArg, &m_parser.noColorArg(), &m_parser.helpArg() });
139 
140  // parse arguments
141  try {
143  } catch (const ParseError &failure) {
144  cerr << failure;
145  m_valid = false;
146  return;
147  }
148 
149  // print help
150  if (m_parser.helpArg().isPresent()) {
151  exit(0);
152  }
153  }
154 
155  // set paths for testfiles
156  // -> set paths set via CLI argument
157  if (m_testFilesPathArg.isPresent()) {
158  for (const char *const testFilesPath : m_testFilesPathArg.values()) {
159  if (*testFilesPath) {
160  m_testFilesPaths.emplace_back(argsToString(testFilesPath, '/'));
161  } else {
162  m_testFilesPaths.emplace_back("./");
163  }
164  }
165  }
166  // -> read TEST_FILE_PATH environment variable
167  bool hasTestFilePathFromEnv;
168  if (auto testFilePathFromEnv = readTestfilePathFromEnv(); (hasTestFilePathFromEnv = !testFilePathFromEnv.empty())) {
169  m_testFilesPaths.emplace_back(move(testFilePathFromEnv));
170  }
171  // -> find source directory
172  if (auto testFilePathFromSrcDirRef = readTestfilePathFromSrcRef(); !testFilePathFromSrcDirRef.empty()) {
173  m_testFilesPaths.emplace_back(move(testFilePathFromSrcDirRef));
174  }
175  // -> try testfiles directory in working directory
176  m_testFilesPaths.emplace_back("./testfiles/");
177  for (const auto &testFilesPath : m_testFilesPaths) {
178  cerr << testFilesPath << '\n';
179  }
180 
181  // set path for working-copy
182  if (m_workingDirArg.isPresent()) {
183  if (*m_workingDirArg.values().front()) {
184  (m_workingDir = m_workingDirArg.values().front()) += '/';
185  } else {
186  m_workingDir = "./";
187  }
188  } else if (const char *const workingDirEnv = getenv("WORKING_DIR")) {
189  if (*workingDirEnv) {
190  m_workingDir = argsToString(workingDirEnv, '/');
191  }
192  } else {
193  if ((m_testFilesPathArg.isPresent() && !m_testFilesPathArg.values().empty()) || hasTestFilePathFromEnv) {
194  m_workingDir = m_testFilesPaths.front() + "workingdir/";
195  } else {
196  m_workingDir = "./testfiles/workingdir/";
197  }
198  }
199  cerr << "Directory used to store working copies:\n" << m_workingDir << '\n';
200 
201  // clear list of all additional profiling files created when forking the test application
202  if (const char *const profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
203  ofstream(profrawListFile, ios_base::trunc);
204  }
205 
206  m_valid = true;
207  cerr << TextAttribute::Bold << "Executing test cases ..." << Phrases::EndFlush;
208 }
209 
214 {
215  s_instance = nullptr;
216 }
217 
230 string TestApplication::testFilePath(const string &relativeTestFilePath) const
231 {
232  string path;
233  for (const auto &testFilesPath : m_testFilesPaths) {
234  if (fileExists(path = testFilesPath + relativeTestFilePath)) {
235  return path;
236  }
237  }
238  throw runtime_error("The testfile \"" % relativeTestFilePath % "\" can not be located. Was looking under:"
239  + joinStrings(m_testFilesPaths, "\n", false, string(), relativeTestFilePath));
240 }
241 
249 string TestApplication::workingCopyPath(const string &relativeTestFilePath, WorkingCopyMode mode) const
250 {
251  return workingCopyPathAs(relativeTestFilePath, relativeTestFilePath, mode);
252 }
253 
269  const std::string &relativeTestFilePath, const std::string &relativeWorkingCopyPath, WorkingCopyMode mode) const
270 {
271  // ensure working directory is present
272  if (!dirExists(m_workingDir) && !makeDir(m_workingDir)) {
273  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath << "\": can't create working directory \""
274  << m_workingDir << "\"." << Phrases::EndFlush;
275  return string();
276  }
277 
278  // ensure subdirectory exists
279  const auto parts = splitString<vector<string>>(relativeWorkingCopyPath, "/", EmptyPartsTreat::Omit);
280  if (!parts.empty()) {
281  // create subdirectory level by level
282  string currentLevel;
283  currentLevel.reserve(m_workingDir.size() + relativeWorkingCopyPath.size() + 1);
284  currentLevel.assign(m_workingDir);
285  for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
286  if (currentLevel.back() != '/') {
287  currentLevel += '/';
288  }
289  currentLevel += *i;
290 
291  // continue if subdirectory level already exists or we can successfully create the directory
292  if (dirExists(currentLevel) || makeDir(currentLevel)) {
293  continue;
294  }
295  // fail otherwise
296  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeWorkingCopyPath << "\": can't create directory \""
297  << currentLevel << "\" (inside working directory)." << Phrases::EndFlush;
298  return string();
299  }
300  }
301 
302  // just return the path if we don't want to actually create a copy
303  if (mode == WorkingCopyMode::NoCopy) {
304  return m_workingDir + relativeWorkingCopyPath;
305  }
306 
307  // copy the file
308  const auto origFilePath(testFilePath(relativeTestFilePath));
309  auto workingCopyPath(m_workingDir + relativeWorkingCopyPath);
310  size_t workingCopyPathAttempt = 0;
311  NativeFileStream origFile, workingCopy;
312  origFile.open(origFilePath, ios_base::in | ios_base::binary);
313  if (origFile.fail()) {
314  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath
315  << "\": an IO error occurred when opening original file \"" << origFilePath << "\"." << Phrases::EndFlush;
316  cerr << "error: " << strerror(errno) << endl;
317  return string();
318  }
319  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
320  while (workingCopy.fail() && fileSystemItemExists(workingCopyPath)) {
321  // adjust the working copy path if the target file already exists and can not be truncated
322  workingCopyPath = argsToString(m_workingDir, relativeWorkingCopyPath, '.', ++workingCopyPathAttempt);
323  workingCopy.clear();
324  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
325  }
326  if (workingCopy.fail()) {
327  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath
328  << "\": an IO error occurred when opening target file \"" << workingCopyPath << "\"." << Phrases::EndFlush;
329  cerr << "error: " << strerror(errno) << endl;
330  return string();
331  }
332  workingCopy << origFile.rdbuf();
333  workingCopy.close();
334  if (!origFile.fail() && !workingCopy.fail()) {
335  return workingCopyPath;
336  }
337 
338  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath << "\": ";
339  if (origFile.fail()) {
340  cerr << "an IO error occurred when reading original file \"" << origFilePath << "\"";
341  return string();
342  }
343  if (workingCopy.fail()) {
344  if (origFile.fail()) {
345  cerr << " and ";
346  }
347  cerr << " an IO error occurred when writing to target file \"" << workingCopyPath << "\".";
348  }
349  cerr << "error: " << strerror(errno) << endl;
350  return string();
351 }
352 
353 #ifdef PLATFORM_UNIX
354 
358 int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
359  const std::string &newProfilingPath)
360 {
361  // print log message
362  if (!suppressLogging) {
363  // print actual appPath and skip first argument instead
364  cout << '-' << ' ' << appPath;
365  if (*args) {
366  for (const char *const *i = args + 1; *i; ++i) {
367  cout << ' ' << *i;
368  }
369  }
370  cout << endl;
371  }
372 
373  // create pipes
374  int coutPipes[2], cerrPipes[2];
375  pipe(coutPipes);
376  pipe(cerrPipes);
377  const auto readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
378  const auto readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
379 
380  // create child process
381  if (const auto child = fork()) {
382  // parent process: read stdout and stderr from child
383  close(writeCoutPipe);
384  close(writeCerrPipe);
385 
386  try {
387  if (child == -1) {
388  throw runtime_error("Unable to create fork");
389  }
390 
391  // init file descriptor set for poll
392  struct pollfd fileDescriptorSet[2];
393  fileDescriptorSet[0].fd = readCoutPipe;
394  fileDescriptorSet[1].fd = readCerrPipe;
395  fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
396 
397  // init variables for reading
398  char buffer[512];
399  output.clear();
400  errors.clear();
401 
402  // poll as long as at least one pipe is open
403  do {
404  const auto retpoll = poll(fileDescriptorSet, 2, timeout);
405  if (retpoll == 0) {
406  throw runtime_error("Poll time-out");
407  }
408  if (retpoll < 0) {
409  throw runtime_error("Poll failed");
410  }
411  if (fileDescriptorSet[0].revents & POLLIN) {
412  const auto count = read(readCoutPipe, buffer, sizeof(buffer));
413  if (count > 0) {
414  output.append(buffer, static_cast<size_t>(count));
415  }
416  } else if (fileDescriptorSet[0].revents & POLLHUP) {
417  close(readCoutPipe);
418  fileDescriptorSet[0].fd = -1;
419  }
420  if (fileDescriptorSet[1].revents & POLLIN) {
421  const auto count = read(readCerrPipe, buffer, sizeof(buffer));
422  if (count > 0) {
423  errors.append(buffer, static_cast<size_t>(count));
424  }
425  } else if (fileDescriptorSet[1].revents & POLLHUP) {
426  close(readCerrPipe);
427  fileDescriptorSet[1].fd = -1;
428  }
429  } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
430  } catch (...) {
431  // ensure all pipes are closed in the error case
432  close(readCoutPipe);
433  close(readCerrPipe);
434  throw;
435  }
436 
437  // get return code
438  int childReturnCode;
439  waitpid(child, &childReturnCode, 0);
440  return childReturnCode;
441  } else {
442  // child process
443  // -> set pipes to be used for stdout/stderr
444  dup2(writeCoutPipe, STDOUT_FILENO);
445  dup2(writeCerrPipe, STDERR_FILENO);
446  close(readCoutPipe);
447  close(writeCoutPipe);
448  close(readCerrPipe);
449  close(writeCerrPipe);
450 
451  // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
452  if (!newProfilingPath.empty()) {
453  setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
454  }
455 
456  // -> execute application
457  execv(appPath, const_cast<char *const *>(args));
458  cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": execv() failed" << Phrases::EndFlush;
459  exit(-101);
460  }
461 }
462 
472 int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
473 {
474  // increase counter used for giving profiling files unique names
475  static unsigned int invocationCount = 0;
476  ++invocationCount;
477 
478  // determine the path of the application to be tested
479  const char *appPath = m_applicationPathArg.firstValue();
480  string fallbackAppPath;
481  if (!appPath || !*appPath) {
482  // try to find the path by removing "_tests"-suffix from own executable path
483  // (the own executable path is the path of the test application and its name is usually the name of the application
484  // to be tested with "_tests"-suffix)
485  const char *const testAppPath = m_parser.executable();
486  const size_t testAppPathLength = strlen(testAppPath);
487  if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
488  fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
489  appPath = fallbackAppPath.data();
490  // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executalbe
491  } else {
492  throw runtime_error("Unable to execute application to be tested: no application path specified");
493  }
494  }
495 
496  // determine new path for profiling output (to not override profiling output of parent and previous invocations)
497  const auto newProfilingPath = [appPath] {
498  string newProfilingPath;
499  const char *const llvmProfileFile = getenv("LLVM_PROFILE_FILE");
500  if (!llvmProfileFile) {
501  return newProfilingPath;
502  }
503  // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
504  const char *const llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw");
505  if (!llvmProfileFileEnd) {
506  return newProfilingPath;
507  }
508  const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
509  // extract application name from path
510  const char *appName = strrchr(appPath, '/');
511  appName = appName ? appName + 1 : appPath;
512  // concat new path
513  newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
514  // append path to profiling list file
515  if (const char *const profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
516  ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
517  }
518  return newProfilingPath;
519  }();
520 
521  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
522 }
523 
531 int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
532 {
533  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
534 }
535 #endif // PLATFORM_UNIX
536 
537 string TestApplication::readTestfilePathFromEnv()
538 {
539  const char *const testFilesPathEnv = getenv("TEST_FILE_PATH");
540  if (!testFilesPathEnv || !*testFilesPathEnv) {
541  return string();
542  }
543  return argsToString(testFilesPathEnv, '/');
544 }
545 
546 string TestApplication::readTestfilePathFromSrcRef()
547 {
548  try {
549  // read "srcdirref" file which should contain the path of the source directory; this file should have been
550  // create by the CMake module "TestTarget.cmake"
551  auto srcDirContent(readFile("srcdirref", 2 * 1024));
552  if (srcDirContent.empty()) {
553  cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
554  return string();
555  }
556  srcDirContent += "/testfiles/";
557 
558  // check whether the referenced source directory contains a "testfiles" directory
559  if (!dirExists(srcDirContent)) {
560  cerr << Phrases::Warning
561  << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
562  << Phrases::End << "Referenced source directory: " << srcDirContent << endl;
563  return string();
564  }
565  return srcDirContent;
566 
567  } catch (const std::ios_base::failure &) {
568  cerr << Phrases::Warning << "The file \"srcdirref\" can not be opened. It likely just doesn't exist in the working directory."
569  << Phrases::EndFlush;
570  }
571  return string();
572 }
573 } // namespace CppUtilities
CppUtilities::TestApplication::~TestApplication
~TestApplication()
Destroys the TestApplication.
Definition: testutils.cpp:213
CppUtilities::Argument::varValueCount
static constexpr std::size_t varValueCount
Denotes a variable number of values.
Definition: argumentparser.h:361
CppUtilities::WorkingCopyMode::NoCopy
@ NoCopy
CppUtilities::makeDir
bool makeDir(const string &path)
Definition: testutils.cpp:83
CppUtilities::NativeFileStream
std::fstream NativeFileStream
Definition: nativefilestream.h:108
CppUtilities::fileSystemItemExists
bool fileSystemItemExists(const string &path)
Definition: testutils.cpp:38
CppUtilities::readFile
CPP_UTILITIES_EXPORT std::string readFile(const std::string &path, std::string::size_type maxSize=std::string::npos)
Reads all contents of the specified file in a single call.
Definition: misc.cpp:16
CppUtilities::TestApplication
The TestApplication class simplifies writing test applications that require opening test files.
Definition: testutils.h:21
CppUtilities::EmptyPartsTreat::Omit
@ Omit
CppUtilities::TestApplication::TestApplication
TestApplication()
Constructs a TestApplication instance without further arguments.
Definition: testutils.cpp:108
CppUtilities::argsToString
StringType argsToString(Args &&... args)
Definition: stringbuilder.h:258
CppUtilities::joinStrings
Container::value_type joinStrings(const Container &strings, const typename Container::value_type &delimiter=typename Container::value_type(), bool omitEmpty=false, const typename Container::value_type &leftClosure=typename Container::value_type(), const typename Container::value_type &rightClosure=typename Container::value_type())
Joins the given strings using the specified delimiter.
Definition: stringconversion.h:75
CppUtilities::Argument::firstValue
const char * firstValue() const
Returns the first parameter value of the first occurrence of the argument.
Definition: argumentparser.cpp:491
CppUtilities::TestApplication::workingCopyPath
std::string workingCopyPath(const std::string &relativeTestFilePath, WorkingCopyMode mode=WorkingCopyMode::CreateCopy) const
Returns the full path to a working copy of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:249
CppUtilities::TestApplication::workingCopyPathAs
std::string workingCopyPathAs(const std::string &relativeTestFilePath, const std::string &relativeWorkingCopyPath, WorkingCopyMode mode=WorkingCopyMode::CreateCopy) const
Returns the full path to a working copy of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:268
CppUtilities::TestApplication::testFilePath
std::string testFilePath(const std::string &relativeTestFilePath) const
Returns the full path of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:230
CppUtilities
Contains all utilities provides by the c++utilities library.
Definition: argumentparser.h:17
i
constexpr int i
Definition: traitstests.cpp:103
CppUtilities::EscapeCodes
Encapsulates functions for formatted terminal output using ANSI escape codes.
Definition: ansiescapecodes.h:12
CppUtilities::WorkingCopyMode
WorkingCopyMode
The WorkingCopyMode enum specifies additional options to influence behavior of TestApplication::worki...
Definition: testutils.h:16
CppUtilities::ArgumentParser::executable
const char * executable() const
Returns the name of the current executable.
Definition: argumentparser.h:1104
testutils.h
CppUtilities::TestApplication::appPath
static const char * appPath()
Returns the application path or an empty string if no application path has been set.
Definition: testutils.h:89
CppUtilities::ParseArgumentBehavior::CheckConstraints
@ CheckConstraints
CppUtilities::dirExists
bool dirExists(const string &path)
Definition: testutils.cpp:68
CppUtilities::ParseArgumentBehavior::InvokeCallbacks
@ InvokeCallbacks
CppUtilities::fileExists
bool fileExists(const string &path)
Definition: testutils.cpp:53