C++ Utilities  5.0.0
Useful C++ classes and routines such as argument parser, IO and conversion utilities
testutils.cpp
Go to the documentation of this file.
1 #include "./testutils.h"
2 
3 #include "../conversion/stringbuilder.h"
4 #include "../conversion/stringconversion.h"
5 #include "../io/ansiescapecodes.h"
6 #include "../io/misc.h"
7 #include "../io/nativefilestream.h"
8 #include "../io/path.h"
9 #include "../misc/parseerror.h"
10 
11 #include <cerrno>
12 #include <cstdlib>
13 #include <cstring>
14 #include <fstream>
15 #include <initializer_list>
16 #include <iostream>
17 #include <limits>
18 
19 #ifdef PLATFORM_UNIX
20 #include <poll.h>
21 #include <sys/stat.h>
22 #include <sys/wait.h>
23 #include <unistd.h>
24 #endif
25 
26 #ifdef PLATFORM_WINDOWS
27 #include <windows.h>
28 #endif
29 
30 using namespace std;
31 using namespace CppUtilities::EscapeCodes;
32 
36 namespace CppUtilities {
37 
38 bool fileSystemItemExists(const string &path)
39 {
40 #ifdef PLATFORM_UNIX
41  struct stat res;
42  return stat(path.data(), &res) == 0;
43 #else
44  const auto widePath(convertMultiByteToWide(path));
45  if (!widePath.first) {
46  return false;
47  }
48  const auto fileType(GetFileAttributesW(widePath.first.get()));
49  return fileType != INVALID_FILE_ATTRIBUTES;
50 #endif
51 }
52 
53 bool fileExists(const string &path)
54 {
55 #ifdef PLATFORM_UNIX
56  struct stat res;
57  return stat(path.data(), &res) == 0 && !S_ISDIR(res.st_mode);
58 #else
59  const auto widePath(convertMultiByteToWide(path));
60  if (!widePath.first) {
61  return false;
62  }
63  const auto fileType(GetFileAttributesW(widePath.first.get()));
64  return (fileType != INVALID_FILE_ATTRIBUTES) && !(fileType & FILE_ATTRIBUTE_DIRECTORY) && !(fileType & FILE_ATTRIBUTE_DEVICE);
65 #endif
66 }
67 
68 bool dirExists(const string &path)
69 {
70 #ifdef PLATFORM_UNIX
71  struct stat res;
72  return stat(path.data(), &res) == 0 && S_ISDIR(res.st_mode);
73 #else
74  const auto widePath(convertMultiByteToWide(path));
75  if (!widePath.first) {
76  return false;
77  }
78  const auto fileType(GetFileAttributesW(widePath.first.get()));
79  return (fileType != INVALID_FILE_ATTRIBUTES) && (fileType & FILE_ATTRIBUTE_DIRECTORY);
80 #endif
81 }
82 
83 bool makeDir(const string &path)
84 {
85 #ifdef PLATFORM_UNIX
86  return mkdir(path.data(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0;
87 #else
88  const auto widePath(convertMultiByteToWide(path));
89  if (!widePath.first) {
90  return false;
91  }
92  return CreateDirectoryW(widePath.first.get(), nullptr) || GetLastError() == ERROR_ALREADY_EXISTS;
93 #endif
94 }
95 
96 TestApplication *TestApplication::s_instance = nullptr;
97 
108 TestApplication::TestApplication()
109  : TestApplication(0, nullptr)
110 {
111 }
112 
117 TestApplication::TestApplication(int argc, const char *const *argv)
118  : m_listArg("list", 'l', "lists available test units")
119  , m_runArg("run", 'r', "runs the tests")
120  , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files", { "path" })
121  , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested", { "path" })
122  , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files", { "path" })
123  , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units", { "unit1", "unit2", "unit3" })
124 {
125  // check whether there is already an instance
126  if (s_instance) {
127  throw runtime_error("only one TestApplication instance allowed at a time");
128  }
129  s_instance = this;
130 
131  // handle specified arguments (if present)
132  if (argc && argv) {
133  // setup argument parser
134  m_testFilesPathArg.setRequiredValueCount(Argument::varValueCount);
135  m_unitsArg.setRequiredValueCount(Argument::varValueCount);
136  m_runArg.setImplicit(true);
137  m_runArg.setSubArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg });
138  m_parser.setMainArguments({ &m_runArg, &m_listArg, &m_parser.noColorArg(), &m_parser.helpArg() });
139 
140  // parse arguments
141  try {
143  } catch (const ParseError &failure) {
144  cerr << failure;
145  m_valid = false;
146  return;
147  }
148 
149  // print help
150  if (m_parser.helpArg().isPresent()) {
151  exit(0);
152  }
153  }
154 
155  // set paths for testfiles
156  // -> set paths set via CLI argument
157  if (m_testFilesPathArg.isPresent()) {
158  for (const char *const testFilesPath : m_testFilesPathArg.values()) {
159  if (*testFilesPath) {
160  m_testFilesPaths.emplace_back(argsToString(testFilesPath, '/'));
161  } else {
162  m_testFilesPaths.emplace_back("./");
163  }
164  }
165  }
166  // -> read TEST_FILE_PATH environment variable
167  bool hasTestFilePathFromEnv;
168  if (auto testFilePathFromEnv = readTestfilePathFromEnv(); (hasTestFilePathFromEnv = !testFilePathFromEnv.empty())) {
169  m_testFilesPaths.emplace_back(move(testFilePathFromEnv));
170  }
171  // -> find source directory
172  if (auto testFilePathFromSrcDirRef = readTestfilePathFromSrcRef(); !testFilePathFromSrcDirRef.empty()) {
173  m_testFilesPaths.emplace_back(move(testFilePathFromSrcDirRef));
174  }
175  // -> try testfiles directory in working directory
176  m_testFilesPaths.emplace_back("./testfiles/");
177  for (const auto &testFilesPath : m_testFilesPaths) {
178  cerr << testFilesPath << '\n';
179  }
180 
181  // set path for working-copy
182  if (m_workingDirArg.isPresent()) {
183  if (*m_workingDirArg.values().front()) {
184  (m_workingDir = m_workingDirArg.values().front()) += '/';
185  } else {
186  m_workingDir = "./";
187  }
188  } else if (const char *workingDirEnv = getenv("WORKING_DIR")) {
189  if (*workingDirEnv) {
190  m_workingDir = argsToString(workingDirEnv, '/');
191  }
192  } else {
193  if ((m_testFilesPathArg.isPresent() && !m_testFilesPathArg.values().empty()) || hasTestFilePathFromEnv) {
194  m_workingDir = m_testFilesPaths.front() + "workingdir/";
195  } else {
196  m_workingDir = "./testfiles/workingdir/";
197  }
198  }
199  cerr << "Directory used to store working copies:\n" << m_workingDir << '\n';
200 
201  // clear list of all additional profiling files created when forking the test application
202  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
203  ofstream(profrawListFile, ios_base::trunc);
204  }
205 
206  m_valid = true;
207  cerr << TextAttribute::Bold << "Executing test cases ..." << Phrases::EndFlush;
208 }
209 
214 {
215  s_instance = nullptr;
216 }
217 
230 string TestApplication::testFilePath(const string &relativeTestFilePath) const
231 {
232  string path;
233  for (const auto &testFilesPath : m_testFilesPaths) {
234  if (fileExists(path = testFilesPath + relativeTestFilePath)) {
235  return path;
236  }
237  }
238  throw runtime_error("The testfile \"" % relativeTestFilePath % "\" can not be located. Was looking under:"
239  + joinStrings(m_testFilesPaths, "\n", false, string(), relativeTestFilePath));
240 }
241 
249 string TestApplication::workingCopyPath(const string &relativeTestFilePath, WorkingCopyMode mode) const
250 {
251  return workingCopyPathAs(relativeTestFilePath, relativeTestFilePath, mode);
252 }
253 
269  const std::string &relativeTestFilePath, const std::string &relativeWorkingCopyPath, WorkingCopyMode mode) const
270 {
271  // ensure working directory is present
272  if (!dirExists(m_workingDir) && !makeDir(m_workingDir)) {
273  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath << "\": can't create working directory \""
274  << m_workingDir << "\"." << Phrases::EndFlush;
275  return string();
276  }
277 
278  // ensure subdirectory exists
279  const auto parts = splitString<vector<string>>(relativeWorkingCopyPath, "/", EmptyPartsTreat::Omit);
280  if (!parts.empty()) {
281  // create subdirectory level by level
282  string currentLevel;
283  currentLevel.reserve(m_workingDir.size() + relativeWorkingCopyPath.size() + 1);
284  currentLevel.assign(m_workingDir);
285  for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
286  if (currentLevel.back() != '/') {
287  currentLevel += '/';
288  }
289  currentLevel += *i;
290 
291  // continue if subdirectory level already exists or we can successfully create the directory
292  if (dirExists(currentLevel) || makeDir(currentLevel)) {
293  continue;
294  }
295  // fail otherwise
296  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeWorkingCopyPath << "\": can't create directory \""
297  << currentLevel << "\" (inside working directory)." << Phrases::EndFlush;
298  return string();
299  }
300  }
301 
302  // just return the path if we don't want to actually create a copy
303  if (mode == WorkingCopyMode::NoCopy) {
304  return m_workingDir + relativeWorkingCopyPath;
305  }
306 
307  // copy the file
308  const auto origFilePath(testFilePath(relativeTestFilePath));
309  auto workingCopyPath(m_workingDir + relativeWorkingCopyPath);
310  size_t workingCopyPathAttempt = 0;
311  NativeFileStream origFile, workingCopy;
312  origFile.open(origFilePath, ios_base::in | ios_base::binary);
313  if (origFile.fail()) {
314  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath
315  << "\": an IO error occurred when opening original file \"" << origFilePath << "\"." << Phrases::EndFlush;
316  cerr << "error: " << strerror(errno) << endl;
317  return string();
318  }
319  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
320  while (workingCopy.fail() && fileSystemItemExists(workingCopyPath)) {
321  // adjust the working copy path if the target file already exists and can not be truncated
322  workingCopyPath = argsToString(m_workingDir, relativeWorkingCopyPath, '.', ++workingCopyPathAttempt);
323  workingCopy.clear();
324  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
325  }
326  if (workingCopy.fail()) {
327  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath
328  << "\": an IO error occurred when opening target file \"" << workingCopyPath << "\"." << Phrases::EndFlush;
329  cerr << "error: " << strerror(errno) << endl;
330  return string();
331  }
332  workingCopy << origFile.rdbuf();
333  if (!origFile.fail() && !workingCopy.fail()) {
334  return workingCopyPath;
335  }
336 
337  cerr << Phrases::Error << "Unable to create working copy for \"" << relativeTestFilePath << "\": ";
338  if (origFile.fail()) {
339  cerr << "an IO error occurred when reading original file \"" << origFilePath << "\"";
340  return string();
341  }
342  if (workingCopy.fail()) {
343  if (origFile.fail()) {
344  cerr << " and ";
345  }
346  cerr << " an IO error occurred when writing to target file \"" << workingCopyPath << "\".";
347  }
348  cerr << "error: " << strerror(errno) << endl;
349  return string();
350 }
351 
352 #ifdef PLATFORM_UNIX
353 
357 int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
358  const std::string &newProfilingPath)
359 {
360  // print log message
361  if (!suppressLogging) {
362  // print actual appPath and skip first argument instead
363  cout << '-' << ' ' << appPath;
364  if (*args) {
365  for (const char *const *i = args + 1; *i; ++i) {
366  cout << ' ' << *i;
367  }
368  }
369  cout << endl;
370  }
371 
372  // create pipes
373  int coutPipes[2], cerrPipes[2];
374  pipe(coutPipes);
375  pipe(cerrPipes);
376  const auto readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
377  const auto readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
378 
379  // create child process
380  if (const auto child = fork()) {
381  // parent process: read stdout and stderr from child
382  close(writeCoutPipe);
383  close(writeCerrPipe);
384 
385  try {
386  if (child == -1) {
387  throw runtime_error("Unable to create fork");
388  }
389 
390  // init file descriptor set for poll
391  struct pollfd fileDescriptorSet[2];
392  fileDescriptorSet[0].fd = readCoutPipe;
393  fileDescriptorSet[1].fd = readCerrPipe;
394  fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
395 
396  // init variables for reading
397  char buffer[512];
398  output.clear();
399  errors.clear();
400 
401  // poll as long as at least one pipe is open
402  do {
403  const auto retpoll = poll(fileDescriptorSet, 2, timeout);
404  if (retpoll == 0) {
405  throw runtime_error("Poll time-out");
406  }
407  if (retpoll < 0) {
408  throw runtime_error("Poll failed");
409  }
410  if (fileDescriptorSet[0].revents & POLLIN) {
411  const auto count = read(readCoutPipe, buffer, sizeof(buffer));
412  if (count > 0) {
413  output.append(buffer, static_cast<size_t>(count));
414  }
415  } else if (fileDescriptorSet[0].revents & POLLHUP) {
416  close(readCoutPipe);
417  fileDescriptorSet[0].fd = -1;
418  }
419  if (fileDescriptorSet[1].revents & POLLIN) {
420  const auto count = read(readCerrPipe, buffer, sizeof(buffer));
421  if (count > 0) {
422  errors.append(buffer, static_cast<size_t>(count));
423  }
424  } else if (fileDescriptorSet[1].revents & POLLHUP) {
425  close(readCerrPipe);
426  fileDescriptorSet[1].fd = -1;
427  }
428  } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
429  } catch (...) {
430  // ensure all pipes are closed in the error case
431  close(readCoutPipe);
432  close(readCerrPipe);
433  throw;
434  }
435 
436  // get return code
437  int childReturnCode;
438  waitpid(child, &childReturnCode, 0);
439  return childReturnCode;
440  } else {
441  // child process
442  // -> set pipes to be used for stdout/stderr
443  dup2(writeCoutPipe, STDOUT_FILENO);
444  dup2(writeCerrPipe, STDERR_FILENO);
445  close(readCoutPipe);
446  close(writeCoutPipe);
447  close(readCerrPipe);
448  close(writeCerrPipe);
449 
450  // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
451  if (!newProfilingPath.empty()) {
452  setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
453  }
454 
455  // -> execute application
456  execv(appPath, const_cast<char *const *>(args));
457  cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": execv() failed" << Phrases::EndFlush;
458  exit(-101);
459  }
460 }
461 
471 int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
472 {
473  // increase counter used for giving profiling files unique names
474  static unsigned int invocationCount = 0;
475  ++invocationCount;
476 
477  // determine the path of the application to be tested
478  const char *appPath = m_applicationPathArg.firstValue();
479  string fallbackAppPath;
480  if (!appPath || !*appPath) {
481  // try to find the path by removing "_tests"-suffix from own executable path
482  // (the own executable path is the path of the test application and its name is usually the name of the application
483  // to be tested with "_tests"-suffix)
484  const char *const testAppPath = m_parser.executable();
485  const size_t testAppPathLength = strlen(testAppPath);
486  if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
487  fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
488  appPath = fallbackAppPath.data();
489  // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executalbe
490  } else {
491  throw runtime_error("Unable to execute application to be tested: no application path specified");
492  }
493  }
494 
495  // determine new path for profiling output (to not override profiling output of parent and previous invocations)
496  string newProfilingPath;
497  if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
498  // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
499  if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
500  const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
501  // extract application name from path
502  const char *appName = strrchr(appPath, '/');
503  appName = appName ? appName + 1 : appPath;
504  // concat new path
505  newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
506  // append path to profiling list file
507  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
508  ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
509  }
510  }
511  }
512 
513  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
514 }
515 
523 int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
524 {
525  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
526 }
527 #endif // PLATFORM_UNIX
528 
529 string TestApplication::readTestfilePathFromEnv()
530 {
531  const char *const testFilesPathEnv = getenv("TEST_FILE_PATH");
532  if (!testFilesPathEnv || !*testFilesPathEnv) {
533  return string();
534  }
535  return argsToString(testFilesPathEnv, '/');
536 }
537 
538 string TestApplication::readTestfilePathFromSrcRef()
539 {
540  try {
541  // read "srcdirref" file which should contain the path of the source directory; this file should have been
542  // create by the CMake module "TestTarget.cmake"
543  auto srcDirContent(readFile("srcdirref", 2 * 1024));
544  if (srcDirContent.empty()) {
545  cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
546  return string();
547  }
548  srcDirContent += "/testfiles/";
549 
550  // check whether the referenced source directory contains a "testfiles" directory
551  if (!dirExists(srcDirContent)) {
552  cerr << Phrases::Warning
553  << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
554  << Phrases::End << "Referenced source directory: " << srcDirContent << endl;
555  return string();
556  }
557  return srcDirContent;
558 
559  } catch (const std::ios_base::failure &) {
560  cerr << Phrases::Warning << "The file \"srcdirref\" can not be opened. It likely just doesn't exist in the working directory."
561  << Phrases::EndFlush;
562  }
563  return string();
564 }
565 } // namespace CppUtilities
CppUtilities::TestApplication::~TestApplication
~TestApplication()
Destroys the TestApplication.
Definition: testutils.cpp:213
CppUtilities::Argument::varValueCount
static constexpr std::size_t varValueCount
Denotes a variable number of values.
Definition: argumentparser.h:361
CppUtilities::WorkingCopyMode::NoCopy
CppUtilities::makeDir
bool makeDir(const string &path)
Definition: testutils.cpp:83
CppUtilities::NativeFileStream
std::fstream NativeFileStream
Definition: nativefilestream.h:108
CppUtilities::fileSystemItemExists
bool fileSystemItemExists(const string &path)
Definition: testutils.cpp:38
CppUtilities::readFile
CPP_UTILITIES_EXPORT std::string readFile(const std::string &path, std::string::size_type maxSize=std::string::npos)
Reads all contents of the specified file in a single call.
Definition: misc.cpp:15
CppUtilities::TestApplication
The TestApplication class simplifies writing test applications that require opening test files.
Definition: testutils.h:21
CppUtilities::EmptyPartsTreat::Omit
CppUtilities::TestApplication::TestApplication
TestApplication()
Constructs a TestApplication instance without further arguments.
Definition: testutils.cpp:108
CppUtilities::argsToString
StringType argsToString(Args &&... args)
Definition: stringbuilder.h:147
CppUtilities::joinStrings
Container::value_type joinStrings(const Container &strings, const typename Container::value_type &delimiter=typename Container::value_type(), bool omitEmpty=false, const typename Container::value_type &leftClosure=typename Container::value_type(), const typename Container::value_type &rightClosure=typename Container::value_type())
Joins the given strings using the specified delimiter.
Definition: stringconversion.h:72
CppUtilities::Argument::firstValue
const char * firstValue() const
Returns the first parameter value of the first occurrence of the argument.
Definition: argumentparser.cpp:489
CppUtilities::TestApplication::workingCopyPath
std::string workingCopyPath(const std::string &relativeTestFilePath, WorkingCopyMode mode=WorkingCopyMode::CreateCopy) const
Returns the full path to a working copy of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:249
CppUtilities::TestApplication::workingCopyPathAs
std::string workingCopyPathAs(const std::string &relativeTestFilePath, const std::string &relativeWorkingCopyPath, WorkingCopyMode mode=WorkingCopyMode::CreateCopy) const
Returns the full path to a working copy of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:268
CppUtilities::TestApplication::testFilePath
std::string testFilePath(const std::string &relativeTestFilePath) const
Returns the full path of the test file with the specified relativeTestFilePath.
Definition: testutils.cpp:230
CppUtilities
Contains all utilities provides by the c++utilities library.
Definition: argumentparser.h:17
i
constexpr int i
Definition: traitstests.cpp:101
CppUtilities::EscapeCodes
Encapsulates functions for formatted terminal output using ANSI escape codes.
Definition: ansiescapecodes.h:11
CppUtilities::WorkingCopyMode
WorkingCopyMode
The WorkingCopyMode enum specifies additional options to influence behavior of TestApplication::worki...
Definition: testutils.h:16
CppUtilities::ArgumentParser::executable
const char * executable() const
Returns the name of the current executable.
Definition: argumentparser.h:1104
testutils.h
CppUtilities::TestApplication::appPath
static const char * appPath()
Returns the application path or an empty string if no application path has been set.
Definition: testutils.h:89
ParseError
The ParseError class is thrown by an ArgumentParser when a parsing error occurs.
CppUtilities::ParseArgumentBehavior::CheckConstraints
CppUtilities::dirExists
bool dirExists(const string &path)
Definition: testutils.cpp:68
CppUtilities::ParseArgumentBehavior::InvokeCallbacks
CppUtilities::fileExists
bool fileExists(const string &path)
Definition: testutils.cpp:53