C++ Utilities  4.15.0
Useful C++ classes and routines such as argument parser, IO and conversion utilities
testutils.cpp
Go to the documentation of this file.
1 #include "./testutils.h"
2 
3 #include "../application/failure.h"
4 #include "../conversion/stringbuilder.h"
5 #include "../conversion/stringconversion.h"
6 #include "../io/ansiescapecodes.h"
7 #include "../io/catchiofailure.h"
8 #include "../io/misc.h"
9 #include "../io/path.h"
10 
11 #include <cerrno>
12 #include <cstdlib>
13 #include <cstring>
14 #include <fstream>
15 #include <initializer_list>
16 #include <iostream>
17 #include <limits>
18 
19 #ifdef PLATFORM_UNIX
20 #include <poll.h>
21 #include <sys/stat.h>
22 #include <sys/wait.h>
23 #include <unistd.h>
24 #endif
25 
26 using namespace std;
27 using namespace ApplicationUtilities;
28 using namespace ConversionUtilities;
29 using namespace EscapeCodes;
30 using namespace IoUtilities;
31 
35 namespace TestUtilities {
36 
37 TestApplication *TestApplication::m_instance = nullptr;
38 
49 TestApplication::TestApplication(int argc, char **argv)
50  : m_helpArg(m_parser)
51  , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files")
52  , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested")
53  , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files")
54  , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units")
55 {
56  // check whether there is already an instance
57  if (m_instance) {
58  throw runtime_error("only one TestApplication instance allowed at a time");
59  }
60  m_instance = this;
61 
62  // determine fallback path for testfiles which is used when --test-files-path/-p not present
63  // -> read TEST_FILE_PATH environment variable
64  m_fallbackTestFilesPath = readTestfilePathFromEnv();
65  // -> find source directory if TEST_FILE_PATH not present
66  bool fallbackIsSourceDir = m_fallbackTestFilesPath.empty();
67  if (fallbackIsSourceDir) {
68  m_fallbackTestFilesPath = readTestfilePathFromSrcRef();
69  }
70 
71  // handle specified arguments (if present)
72  if (argc && argv) {
73  // setup argument parser
74  for (Argument *arg : initializer_list<Argument *>{ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg }) {
75  arg->setRequiredValueCount(1);
76  arg->setValueNames({ "path" });
77  arg->setCombinable(true);
78  }
79  m_unitsArg.setRequiredValueCount(Argument::varValueCount);
80  m_unitsArg.setValueNames({ "unit1", "unit2", "unit3" });
81  m_unitsArg.setCombinable(true);
82  m_parser.setMainArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg, &m_helpArg });
83 
84  // parse arguments
85  try {
86  m_parser.parseArgs(argc, argv);
87  } catch (const Failure &failure) {
88  cerr << failure;
89  m_valid = false;
90  return;
91  }
92 
93  // print help
94  if (m_helpArg.isPresent()) {
95  exit(0);
96  }
97  }
98 
99  // handle path for testfiles and working-copy
100  cerr << "Directories used to search for testfiles:" << endl;
101  if (m_testFilesPathArg.isPresent()) {
102  if (*m_testFilesPathArg.values().front()) {
103  cerr << ((m_testFilesPath = m_testFilesPathArg.values().front()) += '/') << endl;
104  } else {
105  cerr << (m_testFilesPath = "./") << endl;
106  }
107  } else {
108  // use fallback path if --test-files-path/-p not present
109  m_testFilesPath.swap(m_fallbackTestFilesPath);
110  }
111  // if it wasn't already the case, use the source directory as fallback dir
112  if (m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
113  m_fallbackTestFilesPath = readTestfilePathFromSrcRef();
114  fallbackIsSourceDir = true;
115  }
116  if (!m_fallbackTestFilesPath.empty() && m_testFilesPath != m_fallbackTestFilesPath) {
117  cerr << m_fallbackTestFilesPath << endl;
118  }
119  cerr << "./testfiles/" << endl << endl;
120  cerr << "Directory used to store working copies:" << endl;
121  if (m_workingDirArg.isPresent()) {
122  if (*m_workingDirArg.values().front()) {
123  (m_workingDir = m_workingDirArg.values().front()) += '/';
124  } else {
125  m_workingDir = "./";
126  }
127  } else if (const char *workingDirEnv = getenv("WORKING_DIR")) {
128  if (*workingDirEnv) {
129  m_workingDir = argsToString(workingDirEnv, '/');
130  }
131  } else {
132  if (m_testFilesPathArg.isPresent()) {
133  m_workingDir = m_testFilesPath + "workingdir/";
134  } else if (!m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
135  m_workingDir = m_fallbackTestFilesPath + "workingdir/";
136  } else {
137  m_workingDir = "./testfiles/workingdir/";
138  }
139  }
140  cerr << m_workingDir << endl << endl;
141 
142  // clear list of all additional profiling files created when forking the test application
143  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
144  ofstream(profrawListFile, ios_base::trunc);
145  }
146 
147  m_valid = true;
148  cerr << TextAttribute::Bold << "Executing test cases ..." << Phrases::EndFlush;
149 }
150 
155 {
156  m_instance = nullptr;
157 }
158 
171 string TestApplication::testFilePath(const string &name) const
172 {
173  string path;
174  fstream file; // used to check whether the file is present
175 
176  // check the path specified by command line argument or via environment variable
177  if (!m_testFilesPath.empty()) {
178  file.open(path = m_testFilesPath + name, ios_base::in);
179  if (file.good()) {
180  return path;
181  }
182  }
183 
184  // check the fallback path (value from environment variable or source directory)
185  if (!m_fallbackTestFilesPath.empty()) {
186  file.clear();
187  file.open(path = m_fallbackTestFilesPath + name, ios_base::in);
188  if (file.good()) {
189  return path;
190  }
191  }
192 
193  // file still not found -> return default path
194  file.clear();
195  file.open(path = "./testfiles/" + name, ios_base::in);
196  if (!file.good()) {
197  cerr << Phrases::Warning << "The testfile \"" << path << "\" can not be located." << Phrases::EndFlush;
198  }
199  return path;
200 }
201 
202 #ifdef PLATFORM_UNIX
203 
211 string TestApplication::workingCopyPathMode(const string &name, WorkingCopyMode mode) const
212 {
213  // ensure working directory is present
214  struct stat currentStat;
215  if ((stat(m_workingDir.c_str(), &currentStat) || !S_ISDIR(currentStat.st_mode))
216  && mkdir(m_workingDir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) {
217  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory." << Phrases::EndFlush;
218  return string();
219  }
220 
221  // ensure subdirectory exists
222  const auto parts = splitString<vector<string>>(name, "/", EmptyPartsTreat::Omit);
223  if (!parts.empty()) {
224  // create subdirectory level by level
225  string currentLevel;
226  currentLevel.reserve(m_workingDir.size() + name.size() + 1);
227  currentLevel.assign(m_workingDir);
228  for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
229  if (currentLevel.back() != '/') {
230  currentLevel += '/';
231  }
232  currentLevel += *i;
233 
234  // continue if subdirectory level already exists
235  if (!stat(currentLevel.c_str(), &currentStat) && S_ISDIR(currentStat.st_mode)) {
236  continue;
237  }
238  // continue if we can successfully create the directory
239  if (!mkdir(currentLevel.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) {
240  continue;
241  }
242  // fail otherwise
243  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory." << Phrases::EndFlush;
244  return string();
245  }
246  }
247 
248  // just return the path if we don't want to actually create a copy
249  if (mode == WorkingCopyMode::NoCopy) {
250  return m_workingDir + name;
251  }
252 
253  // copy the file
254  const auto origFilePath(testFilePath(name));
255  auto workingCopyPath(m_workingDir + name);
256  size_t workingCopyPathAttempt = 0;
257  fstream origFile, workingCopy;
258  origFile.open(origFilePath, ios_base::in | ios_base::binary);
259  if (origFile.fail()) {
260  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": an IO error occurred when opening original file \""
261  << origFilePath << "\"." << Phrases::EndFlush;
262  cerr << "error: " << strerror(errno) << endl;
263  return string();
264  }
265  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
266  while (workingCopy.fail() && !stat(workingCopyPath.c_str(), &currentStat)) {
267  // adjust the working copy path if the target file already exists and can not be truncated
268  workingCopyPath = argsToString(m_workingDir, name, '.', ++workingCopyPathAttempt);
269  workingCopy.clear();
270  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
271  }
272  if (workingCopy.fail()) {
273  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": an IO error occurred when opening target file \""
274  << workingCopyPath << "\"." << Phrases::EndFlush;
275  cerr << "error: " << strerror(errno) << endl;
276  return string();
277  }
278  workingCopy << origFile.rdbuf();
279  if (!origFile.fail() && !workingCopy.fail()) {
280  return workingCopyPath;
281  }
282 
283  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": ";
284  if (origFile.fail()) {
285  cerr << "an IO error occurred when reading original file \"" << origFilePath << "\"";
286  return string();
287  }
288  if (workingCopy.fail()) {
289  if (origFile.fail()) {
290  cerr << " and ";
291  }
292  cerr << " an IO error occurred when writing to target file \"" << workingCopyPath << "\".";
293  }
294  cerr << "error: " << strerror(errno) << endl;
295  return string();
296 }
297 
305 string TestApplication::workingCopyPath(const string &name) const
306 {
307  return workingCopyPathMode(name, WorkingCopyMode::CreateCopy);
308 }
309 
314 int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
315  const std::string &newProfilingPath)
316 {
317  // print log message
318  if (!suppressLogging) {
319  cout << '-';
320  for (const char *const *i = args; *i; ++i) {
321  cout << ' ' << *i;
322  }
323  cout << endl;
324  }
325 
326  // create pipes
327  int coutPipes[2], cerrPipes[2];
328  pipe(coutPipes);
329  pipe(cerrPipes);
330  const auto readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
331  const auto readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
332 
333  // create child process
334  if (const auto child = fork()) {
335  // parent process: read stdout and stderr from child
336  close(writeCoutPipe);
337  close(writeCerrPipe);
338 
339  try {
340  if (child == -1) {
341  throw runtime_error("Unable to create fork");
342  }
343 
344  // init file descriptor set for poll
345  struct pollfd fileDescriptorSet[2];
346  fileDescriptorSet[0].fd = readCoutPipe;
347  fileDescriptorSet[1].fd = readCerrPipe;
348  fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
349 
350  // init variables for reading
351  char buffer[512];
352  output.clear();
353  errors.clear();
354 
355  // poll as long as at least one pipe is open
356  do {
357  const auto retpoll = poll(fileDescriptorSet, 2, timeout);
358  if (retpoll == 0) {
359  throw runtime_error("Poll time-out");
360  }
361  if (retpoll < 0) {
362  throw runtime_error("Poll failed");
363  }
364  if (fileDescriptorSet[0].revents & POLLIN) {
365  const auto count = read(readCoutPipe, buffer, sizeof(buffer));
366  if (count > 0) {
367  output.append(buffer, static_cast<size_t>(count));
368  }
369  } else if (fileDescriptorSet[0].revents & POLLHUP) {
370  close(readCoutPipe);
371  fileDescriptorSet[0].fd = -1;
372  }
373  if (fileDescriptorSet[1].revents & POLLIN) {
374  const auto count = read(readCerrPipe, buffer, sizeof(buffer));
375  if (count > 0) {
376  errors.append(buffer, static_cast<size_t>(count));
377  }
378  } else if (fileDescriptorSet[1].revents & POLLHUP) {
379  close(readCerrPipe);
380  fileDescriptorSet[1].fd = -1;
381  }
382  } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
383  } catch (...) {
384  // ensure all pipes are closed in the error case
385  close(readCoutPipe);
386  close(readCerrPipe);
387  throw;
388  }
389 
390  // get return code
391  int childReturnCode;
392  waitpid(child, &childReturnCode, 0);
393  return childReturnCode;
394  } else {
395  // child process
396  // -> set pipes to be used for stdout/stderr
397  dup2(writeCoutPipe, STDOUT_FILENO);
398  dup2(writeCerrPipe, STDERR_FILENO);
399  close(readCoutPipe);
400  close(writeCoutPipe);
401  close(readCerrPipe);
402  close(writeCerrPipe);
403 
404  // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
405  if (!newProfilingPath.empty()) {
406  setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
407  }
408 
409  // -> execute application
410  execv(appPath, const_cast<char *const *>(args));
411  cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": execv() failed" << Phrases::EndFlush;
412  exit(-101);
413  }
414 }
415 
425 int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
426 {
427  // increase counter used for giving profiling files unique names
428  static unsigned int invocationCount = 0;
429  ++invocationCount;
430 
431  // determine the path of the application to be tested
432  const char *appPath = m_applicationPathArg.firstValue();
433  string fallbackAppPath;
434  if (!appPath || !*appPath) {
435  // try to find the path by removing "_tests"-suffix from own executable path
436  // (the own executable path is the path of the test application and its name is usually the name of the application
437  // to be tested with "_tests"-suffix)
438  const char *const testAppPath = m_parser.executable();
439  const size_t testAppPathLength = strlen(testAppPath);
440  if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
441  fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
442  appPath = fallbackAppPath.data();
443  // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executalbe
444  } else {
445  throw runtime_error("Unable to execute application to be tested: no application path specified");
446  }
447  }
448 
449  // determine new path for profiling output (to not override profiling output of parent and previous invocations)
450  string newProfilingPath;
451  if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
452  // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
453  if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
454  const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
455  // extract application name from path
456  const char *appName = strrchr(appPath, '/');
457  appName = appName ? appName + 1 : appPath;
458  // concat new path
459  newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
460  // append path to profiling list file
461  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
462  ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
463  }
464  }
465  }
466 
467  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
468 }
469 
477 int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
478 {
479  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
480 }
481 #endif // PLATFORM_UNIX
482 
483 string TestApplication::readTestfilePathFromEnv()
484 {
485  const char *const testFilesPathEnv = getenv("TEST_FILE_PATH");
486  if (!testFilesPathEnv || !*testFilesPathEnv) {
487  return string();
488  }
489  return argsToString(testFilesPathEnv, '/');
490 }
491 
492 string TestApplication::readTestfilePathFromSrcRef()
493 {
494  try {
495  // read "srcdirref" file which should contain the path of the source directory; this file should have been
496  // create by the CMake module "TestTarget.cmake"
497  auto srcDirContent(readFile("srcdirref", 2 * 1024));
498  if (srcDirContent.empty()) {
499  cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
500  return string();
501  }
502 
503  // check whether the referenced source directory contains a "testfiles" directory
504 #ifdef PLATFORM_UNIX // directoryEntries() is not implemented under Windows so we can only to the check under UNIX
505  bool hasTestfilesDir = false;
506  for (const string &dir : directoryEntries(srcDirContent.data(), DirectoryEntryType::Directory)) {
507  if (dir == "testfiles") {
508  hasTestfilesDir = true;
509  break;
510  }
511  }
512  if (!hasTestfilesDir) {
513  cerr << Phrases::Warning
514  << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
515  << Phrases::End << "Referenced source directory: " << srcDirContent << endl;
516  return string();
517  }
518 #endif // PLATFORM_UNIX
519 
520  return srcDirContent += "/testfiles/";
521  } catch (...) {
522  cerr << Phrases::Warning << "The file \"srcdirref\" can not be opened. It likely just doesn't exist in the working directory."
523  << Phrases::EndFlush;
524  catchIoFailure();
525  }
526  return string();
527 }
528 } // namespace TestUtilities
Encapsulates functions for formatted terminal output using ANSI escape codes.
void setCombinable(bool value)
Sets whether this argument can be combined.
CPP_UTILITIES_EXPORT std::list< std::string > directoryEntries(const char *path, DirectoryEntryType types=DirectoryEntryType::All)
Returns the names of the directory entries in the specified path with the specified types...
Definition: path.cpp:181
Contains currently only ArgumentParser and related classes.
constexpr StringType argsToString(Args &&... args)
void setMainArguments(const ArgumentInitializerList &mainArguments)
Sets the main arguments for the parser.
STL namespace.
std::string testFilePath(const std::string &name) const
Returns the full path of the test file with the specified name.
Definition: testutils.cpp:171
void parseArgs(int argc, const char *const *argv)
Parses the specified command line arguments.
const char * firstValue() const
Returns the first parameter value of the first occurrence of the argument.
CPP_UTILITIES_EXPORT std::string readFile(const std::string &path, std::string::size_type maxSize=std::string::npos)
Reads all contents of the specified file in a single call.
Definition: misc.cpp:16
static const char * appPath()
Returns the application path or an empty string if no application path has been set.
Definition: testutils.h:78
Contains utility classes helping to read and write streams.
Definition: binaryreader.h:10
Contains classes and functions utilizing creating of test applications.
Definition: testutils.h:12
constexpr int i
Definition: traitstests.cpp:97
~TestApplication()
Destroys the TestApplication.
Definition: testutils.cpp:154
const std::vector< const char * > & values(std::size_t occurrence=0) const
Returns the parameter values for the specified occurrence of argument.
Contains several functions providing conversions between different data types.
const char * executable() const
Returns the name of the current executable.
The Argument class is a wrapper for command line argument information.
WorkingCopyMode
The WorkingCopyMode enum specifies additional options to influence behavior of TestApplication::worki...
Definition: testutils.h:17
bool isPresent() const
Returns an indication whether the argument could be detected when parsing.
The Failure class is thrown by an ArgumentParser when a parsing error occurs.
Definition: failure.h:12
CPP_UTILITIES_EXPORT const char * catchIoFailure()
Provides a workaround for GCC Bug 66145.
void setRequiredValueCount(std::size_t requiredValueCount)
Sets the number of values which are required to be given for this argument.
void setValueNames(std::initializer_list< const char *> valueNames)
Sets the names of the requried values.