C++ Utilities  4.14.2
Useful C++ classes and routines such as argument parser, IO and conversion utilities
testutils.cpp
Go to the documentation of this file.
1 #include "./testutils.h"
2 
3 #include "../application/failure.h"
4 #include "../conversion/stringbuilder.h"
5 #include "../conversion/stringconversion.h"
6 #include "../io/ansiescapecodes.h"
7 #include "../io/catchiofailure.h"
8 #include "../io/misc.h"
9 #include "../io/path.h"
10 
11 #include <cstdlib>
12 #include <cstring>
13 #include <fstream>
14 #include <initializer_list>
15 #include <iostream>
16 #include <limits>
17 
18 #ifdef PLATFORM_UNIX
19 #include <poll.h>
20 #include <sys/stat.h>
21 #include <sys/wait.h>
22 #include <unistd.h>
23 #endif
24 
25 using namespace std;
26 using namespace ApplicationUtilities;
27 using namespace ConversionUtilities;
28 using namespace EscapeCodes;
29 using namespace IoUtilities;
30 
34 namespace TestUtilities {
35 
36 TestApplication *TestApplication::m_instance = nullptr;
37 
48 TestApplication::TestApplication(int argc, char **argv)
49  : m_helpArg(m_parser)
50  , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files")
51  , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested")
52  , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files")
53  , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units")
54 {
55  // check whether there is already an instance
56  if (m_instance) {
57  throw runtime_error("only one TestApplication instance allowed at a time");
58  }
59  m_instance = this;
60 
61  // determine fallback path for testfiles which is used when --test-files-path/-p not present
62  // -> read TEST_FILE_PATH environment variable
63  m_fallbackTestFilesPath = readTestfilePathFromEnv();
64  // -> find source directory if TEST_FILE_PATH not present
65  bool fallbackIsSourceDir = m_fallbackTestFilesPath.empty();
66  if (fallbackIsSourceDir) {
67  m_fallbackTestFilesPath = readTestfilePathFromSrcRef();
68  }
69 
70  // handle specified arguments (if present)
71  if (argc && argv) {
72  // setup argument parser
73  for (Argument *arg : initializer_list<Argument *>{ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg }) {
74  arg->setRequiredValueCount(1);
75  arg->setValueNames({ "path" });
76  arg->setCombinable(true);
77  }
78  m_unitsArg.setRequiredValueCount(Argument::varValueCount);
79  m_unitsArg.setValueNames({ "unit1", "unit2", "unit3" });
80  m_unitsArg.setCombinable(true);
81  m_parser.setMainArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg, &m_helpArg });
82 
83  // parse arguments
84  try {
85  m_parser.parseArgs(argc, argv);
86  } catch (const Failure &failure) {
87  cerr << failure;
88  m_valid = false;
89  return;
90  }
91 
92  // print help
93  if (m_helpArg.isPresent()) {
94  exit(0);
95  }
96  }
97 
98  // handle path for testfiles and working-copy
99  cerr << "Directories used to search for testfiles:" << endl;
100  if (m_testFilesPathArg.isPresent()) {
101  if (*m_testFilesPathArg.values().front()) {
102  cerr << ((m_testFilesPath = m_testFilesPathArg.values().front()) += '/') << endl;
103  } else {
104  cerr << (m_testFilesPath = "./") << endl;
105  }
106  } else {
107  // use fallback path if --test-files-path/-p not present
108  m_testFilesPath.swap(m_fallbackTestFilesPath);
109  }
110  // if it wasn't already the case, use the source directory as fallback dir
111  if (m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
112  m_fallbackTestFilesPath = readTestfilePathFromSrcRef();
113  fallbackIsSourceDir = true;
114  }
115  if (!m_fallbackTestFilesPath.empty() && m_testFilesPath != m_fallbackTestFilesPath) {
116  cerr << m_fallbackTestFilesPath << endl;
117  }
118  cerr << "./testfiles/" << endl << endl;
119  cerr << "Directory used to store working copies:" << endl;
120  if (m_workingDirArg.isPresent()) {
121  if (*m_workingDirArg.values().front()) {
122  (m_workingDir = m_workingDirArg.values().front()) += '/';
123  } else {
124  m_workingDir = "./";
125  }
126  } else if (const char *workingDirEnv = getenv("WORKING_DIR")) {
127  if (*workingDirEnv) {
128  m_workingDir = argsToString(workingDirEnv, '/');
129  }
130  } else {
131  if (m_testFilesPathArg.isPresent()) {
132  m_workingDir = m_testFilesPath + "workingdir/";
133  } else if (!m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
134  m_workingDir = m_fallbackTestFilesPath + "workingdir/";
135  } else {
136  m_workingDir = "./testfiles/workingdir/";
137  }
138  }
139  cerr << m_workingDir << endl << endl;
140 
141  // clear list of all additional profiling files created when forking the test application
142  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
143  ofstream(profrawListFile, ios_base::trunc);
144  }
145 
146  m_valid = true;
147  cerr << TextAttribute::Bold << "Executing test cases ..." << Phrases::EndFlush;
148 }
149 
154 {
155  m_instance = nullptr;
156 }
157 
170 string TestApplication::testFilePath(const string &name) const
171 {
172  string path;
173  fstream file; // used to check whether the file is present
174 
175  // check the path specified by command line argument or via environment variable
176  if (!m_testFilesPath.empty()) {
177  file.open(path = m_testFilesPath + name, ios_base::in);
178  if (file.good()) {
179  return path;
180  }
181  }
182 
183  // check the fallback path (value from environment variable or source directory)
184  if (!m_fallbackTestFilesPath.empty()) {
185  file.clear();
186  file.open(path = m_fallbackTestFilesPath + name, ios_base::in);
187  if (file.good()) {
188  return path;
189  }
190  }
191 
192  // file still not found -> return default path
193  file.clear();
194  file.open(path = "./testfiles/" + name, ios_base::in);
195  if (!file.good()) {
196  cerr << Phrases::Warning << "The testfile \"" << path << "\" can not be located." << Phrases::EndFlush;
197  }
198  return path;
199 }
200 
201 #ifdef PLATFORM_UNIX
202 
210 string TestApplication::workingCopyPathMode(const string &name, WorkingCopyMode mode) const
211 {
212  // create file streams
213  fstream origFile, workingCopy;
214  origFile.exceptions(ios_base::badbit | ios_base::failbit);
215  workingCopy.exceptions(ios_base::badbit | ios_base::failbit);
216 
217  // ensure working directory is present
218  struct stat currentStat;
219  if (stat(m_workingDir.c_str(), &currentStat) || !S_ISDIR(currentStat.st_mode)) {
220  if (mkdir(m_workingDir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) {
221  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory." << Phrases::EndFlush;
222  return string();
223  }
224  }
225 
226  // ensure subdirectory exists
227  const auto parts = splitString<vector<string>>(name, string("/"), EmptyPartsTreat::Omit);
228  if (!parts.empty()) {
229  string currentLevel = m_workingDir;
230  for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
231  if (currentLevel.back() != '/') {
232  currentLevel += '/';
233  }
234  currentLevel += *i;
235  if (stat(currentLevel.c_str(), &currentStat) || !S_ISDIR(currentStat.st_mode)) {
236  if (mkdir(currentLevel.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) {
237  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory."
238  << Phrases::EndFlush;
239  return string();
240  }
241  }
242  }
243  }
244 
245  // copy file
246  if (mode != WorkingCopyMode::NoCopy) {
247  try {
248  origFile.open(testFilePath(name), ios_base::in | ios_base::binary);
249  const string path = m_workingDir + name;
250  workingCopy.open(path, ios_base::out | ios_base::binary | ios_base::trunc);
251  workingCopy << origFile.rdbuf();
252  return path;
253  } catch (...) {
254  catchIoFailure();
255  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": an IO error occured." << Phrases::EndFlush;
256  }
257  } else {
258  return m_workingDir + name;
259  }
260  return string();
261 }
262 
270 string TestApplication::workingCopyPath(const string &name) const
271 {
272  return workingCopyPathMode(name, WorkingCopyMode::CreateCopy);
273 }
274 
279 int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
280  const std::string &newProfilingPath)
281 {
282  // print log message
283  if (!suppressLogging) {
284  cout << '-';
285  for (const char *const *i = args; *i; ++i) {
286  cout << ' ' << *i;
287  }
288  cout << endl;
289  }
290 
291  // create pipes
292  int coutPipes[2], cerrPipes[2];
293  pipe(coutPipes), pipe(cerrPipes);
294  int readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
295  int readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
296 
297  // create child process
298  if (int child = fork()) {
299  // parent process: read stdout and stderr from child
300  close(writeCoutPipe), close(writeCerrPipe);
301 
302  try {
303  if (child == -1) {
304  throw runtime_error("Unable to create fork");
305  }
306 
307  // init file descriptor set for poll
308  struct pollfd fileDescriptorSet[2];
309  fileDescriptorSet[0].fd = readCoutPipe;
310  fileDescriptorSet[1].fd = readCerrPipe;
311  fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
312 
313  // init variables for reading
314  char buffer[512];
315  ssize_t count;
316  output.clear(), errors.clear();
317 
318  // poll as long as at least one pipe is open
319  do {
320  int retpoll = poll(fileDescriptorSet, 2, timeout);
321  if (retpoll > 0) {
322  // poll succeeds
323  if (fileDescriptorSet[0].revents & POLLIN) {
324  if ((count = read(readCoutPipe, buffer, sizeof(buffer))) > 0) {
325  output.append(buffer, static_cast<size_t>(count));
326  }
327  } else if (fileDescriptorSet[0].revents & POLLHUP) {
328  close(readCoutPipe);
329  fileDescriptorSet[0].fd = -1;
330  }
331  if (fileDescriptorSet[1].revents & POLLIN) {
332  if ((count = read(readCerrPipe, buffer, sizeof(buffer))) > 0) {
333  errors.append(buffer, static_cast<size_t>(count));
334  }
335  } else if (fileDescriptorSet[1].revents & POLLHUP) {
336  close(readCerrPipe);
337  fileDescriptorSet[1].fd = -1;
338  }
339  } else if (retpoll == 0) {
340  // timeout
341  throw runtime_error("Poll time-out");
342  } else {
343  // fail
344  throw runtime_error("Poll failed");
345  }
346  } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
347  } catch (...) {
348  // ensure all pipes are close in the error case
349  close(readCoutPipe), close(readCerrPipe);
350  throw;
351  }
352 
353  // get return code
354  int childReturnCode;
355  waitpid(child, &childReturnCode, 0);
356  return childReturnCode;
357  } else {
358  // child process
359  // -> set pipes to be used for stdout/stderr
360  dup2(writeCoutPipe, STDOUT_FILENO), dup2(writeCerrPipe, STDERR_FILENO);
361  close(readCoutPipe), close(writeCoutPipe), close(readCerrPipe), close(writeCerrPipe);
362 
363  // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
364  if (!newProfilingPath.empty()) {
365  setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
366  }
367 
368  // -> execute application
369  execv(appPath, const_cast<char *const *>(args));
370  cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": execv() failed" << Phrases::EndFlush;
371  exit(-101);
372  }
373 }
374 
384 int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
385 {
386  // increase counter used for giving profiling files unique names
387  static unsigned int invocationCount = 0;
388  ++invocationCount;
389 
390  // determine the path of the application to be tested
391  const char *appPath = m_applicationPathArg.firstValue();
392  string fallbackAppPath;
393  if (!appPath || !*appPath) {
394  // try to find the path by removing "_tests"-suffix from own executable path
395  // (the own executable path is the path of the test application and its name is usually the name of the application
396  // to be tested with "_tests"-suffix)
397  const char *const testAppPath = m_parser.executable();
398  const size_t testAppPathLength = strlen(testAppPath);
399  if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
400  fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
401  appPath = fallbackAppPath.data();
402  // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executalbe
403  } else {
404  throw runtime_error("Unable to execute application to be tested: no application path specified");
405  }
406  }
407 
408  // determine new path for profiling output (to not override profiling output of parent and previous invocations)
409  string newProfilingPath;
410  if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
411  // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
412  if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
413  const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
414  // extract application name from path
415  const char *appName = strrchr(appPath, '/');
416  appName = appName ? appName + 1 : appPath;
417  // concat new path
418  newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
419  // append path to profiling list file
420  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
421  ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
422  }
423  }
424  }
425 
426  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
427 }
428 
436 int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
437 {
438  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
439 }
440 #endif // PLATFORM_UNIX
441 
442 string TestApplication::readTestfilePathFromEnv()
443 {
444  const char *const testFilesPathEnv = getenv("TEST_FILE_PATH");
445  if (!testFilesPathEnv || !*testFilesPathEnv) {
446  return string();
447  }
448  return argsToString(testFilesPathEnv, '/');
449 }
450 
451 string TestApplication::readTestfilePathFromSrcRef()
452 {
453  try {
454  // read "srcdirref" file which should contain the path of the source directory; this file should have been
455  // create by the CMake module "TestTarget.cmake"
456  string srcDirContent(readFile("srcdirref", 2 * 1024));
457  if (srcDirContent.empty()) {
458  cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
459  return string();
460  }
461 
462  // check whether the referenced source directory contains a "testfiles" directory
463 #ifdef PLATFORM_UNIX // directoryEntries() is not implemented under Windows so we can only to the check under UNIX
464  bool hasTestfilesDir = false;
465  for (const string &dir : directoryEntries(srcDirContent.data(), DirectoryEntryType::Directory)) {
466  if (dir == "testfiles") {
467  hasTestfilesDir = true;
468  break;
469  }
470  }
471  if (!hasTestfilesDir) {
472  cerr << Phrases::Warning
473  << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
474  << Phrases::End << "Referenced source directory: " << srcDirContent << endl;
475  return string();
476  }
477 #endif // PLATFORM_UNIX
478 
479  return srcDirContent += "/testfiles/";
480  } catch (...) {
481  cerr << Phrases::Warning << "The file \"srcdirref\" can not be opened. It likely just doesn't exist in the working directory."
482  << Phrases::EndFlush;
483  catchIoFailure();
484  }
485  return string();
486 }
487 } // namespace TestUtilities
Encapsulates functions for formatted terminal output using ANSI escape codes.
void setCombinable(bool value)
Sets whether this argument can be combined.
CPP_UTILITIES_EXPORT std::list< std::string > directoryEntries(const char *path, DirectoryEntryType types=DirectoryEntryType::All)
Returns the names of the directory entries in the specified path with the specified types...
Definition: path.cpp:181
Contains currently only ArgumentParser and related classes.
constexpr StringType argsToString(Args &&... args)
void setMainArguments(const ArgumentInitializerList &mainArguments)
Sets the main arguments for the parser.
STL namespace.
std::string testFilePath(const std::string &name) const
Returns the full path of the test file with the specified name.
Definition: testutils.cpp:170
void parseArgs(int argc, const char *const *argv)
Parses the specified command line arguments.
const char * firstValue() const
Returns the first parameter value of the first occurrence of the argument.
CPP_UTILITIES_EXPORT std::string readFile(const std::string &path, std::string::size_type maxSize=std::string::npos)
Reads all contents of the specified file in a single call.
Definition: misc.cpp:16
static const char * appPath()
Returns the application path or an empty string if no application path has been set.
Definition: testutils.h:78
Contains utility classes helping to read and write streams.
Definition: binaryreader.h:10
Contains classes and functions utilizing creating of test applications.
Definition: testutils.h:12
~TestApplication()
Destroys the TestApplication.
Definition: testutils.cpp:153
const std::vector< const char * > & values(std::size_t occurrence=0) const
Returns the parameter values for the specified occurrence of argument.
Contains several functions providing conversions between different data types.
const char * executable() const
Returns the name of the current executable.
The Argument class is a wrapper for command line argument information.
WorkingCopyMode
The WorkingCopyMode enum specifies additional options to influence behavior of TestApplication::worki...
Definition: testutils.h:17
bool isPresent() const
Returns an indication whether the argument could be detected when parsing.
The Failure class is thrown by an ArgumentParser when a parsing error occurs.
Definition: failure.h:12
CPP_UTILITIES_EXPORT const char * catchIoFailure()
Provides a workaround for GCC Bug 66145.
void setRequiredValueCount(std::size_t requiredValueCount)
Sets the number of values which are required to be given for this argument.
void setValueNames(std::initializer_list< const char *> valueNames)
Sets the names of the requried values.