C++ Utilities  4.12.0
Useful C++ classes and routines such as argument parser, IO and conversion utilities
testutils.cpp
Go to the documentation of this file.
1 #include "./testutils.h"
2 
3 #include "../application/failure.h"
4 #include "../conversion/stringbuilder.h"
5 #include "../conversion/stringconversion.h"
6 #include "../io/ansiescapecodes.h"
7 #include "../io/catchiofailure.h"
8 #include "../io/misc.h"
9 #include "../io/path.h"
10 
11 #include <cstdlib>
12 #include <cstring>
13 #include <fstream>
14 #include <initializer_list>
15 #include <iostream>
16 #include <limits>
17 
18 #ifdef PLATFORM_UNIX
19 #include <poll.h>
20 #include <sys/stat.h>
21 #include <sys/wait.h>
22 #include <unistd.h>
23 #endif
24 
25 using namespace std;
26 using namespace ApplicationUtilities;
27 using namespace ConversionUtilities;
28 using namespace EscapeCodes;
29 using namespace IoUtilities;
30 
34 namespace TestUtilities {
35 
36 TestApplication *TestApplication::m_instance = nullptr;
37 
48 TestApplication::TestApplication(int argc, char **argv)
49  : m_helpArg(m_parser)
50  , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files")
51  , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested")
52  , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files")
53  , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units")
54 {
55  // check whether there is already an instance
56  if (m_instance) {
57  throw runtime_error("only one TestApplication instance allowed at a time");
58  }
59  m_instance = this;
60 
61  // determine fallback path for testfiles which is used when --test-files-path/-p not present
62  // -> read TEST_FILE_PATH environment variable
63  readFallbackTestfilePathFromEnv();
64  // -> find source directory if TEST_FILE_PATH not present
65  const bool fallbackIsSourceDir = m_fallbackTestFilesPath.empty();
66  if (fallbackIsSourceDir) {
67  readFallbackTestfilePathFromSrcRef();
68  }
69 
70  // handle specified arguments (if present)
71  if (argc && argv) {
72  // setup argument parser
73  for (Argument *arg : initializer_list<Argument *>{ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg }) {
74  arg->setRequiredValueCount(1);
75  arg->setValueNames({ "path" });
76  arg->setCombinable(true);
77  }
78  m_unitsArg.setRequiredValueCount(Argument::varValueCount);
79  m_unitsArg.setValueNames({ "unit1", "unit2", "unit3" });
80  m_unitsArg.setCombinable(true);
81  m_parser.setMainArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg, &m_helpArg });
82 
83  // parse arguments
84  try {
85  m_parser.parseArgs(argc, argv);
86  } catch (const Failure &failure) {
87  cerr << failure;
88  m_valid = false;
89  return;
90  }
91 
92  // print help
93  if (m_helpArg.isPresent()) {
94  exit(0);
95  }
96  }
97 
98  // handle path for testfiles and working-copy
99  cerr << "Directories used to search for testfiles:" << endl;
100  if (m_testFilesPathArg.isPresent()) {
101  if (*m_testFilesPathArg.values().front()) {
102  cerr << ((m_testFilesPath = m_testFilesPathArg.values().front()) += '/') << endl;
103  } else {
104  cerr << (m_testFilesPath = "./") << endl;
105  }
106  }
107  if (!m_fallbackTestFilesPath.empty() && m_testFilesPath != m_fallbackTestFilesPath) {
108  cerr << m_fallbackTestFilesPath << endl;
109  }
110  cerr << "./testfiles/" << endl << endl;
111  cerr << "Directory used to store working copies:" << endl;
112  if (m_workingDirArg.isPresent()) {
113  if (*m_workingDirArg.values().front()) {
114  (m_workingDir = m_workingDirArg.values().front()) += '/';
115  } else {
116  m_workingDir = "./";
117  }
118  } else if (const char *workingDirEnv = getenv("WORKING_DIR")) {
119  if (const auto len = strlen(workingDirEnv)) {
120  m_workingDir.reserve(len + 1);
121  m_workingDir += workingDirEnv;
122  m_workingDir += '/';
123  }
124  } else {
125  if (m_testFilesPathArg.isPresent()) {
126  m_workingDir = m_testFilesPath + "workingdir/";
127  } else if (!m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
128  m_workingDir = m_fallbackTestFilesPath + "workingdir/";
129  } else {
130  m_workingDir = "./testfiles/workingdir/";
131  }
132  }
133  cerr << m_workingDir << endl << endl;
134 
135  // clear list of all additional profiling files created when forking the test application
136  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
137  ofstream(profrawListFile, ios_base::trunc);
138  }
139 
140  m_valid = true;
141  cerr << TextAttribute::Bold << "Executing test cases ..." << Phrases::EndFlush;
142 }
143 
148 {
149  m_instance = nullptr;
150 }
151 
162 string TestApplication::testFilePath(const string &name) const
163 {
164  string path;
165  fstream file; // used to check whether the file is present
166 
167  // check the path specified by command line argument
168  if (m_testFilesPathArg.isPresent()) {
169  file.open(path = m_testFilesPath + name, ios_base::in);
170  if (file.good()) {
171  return path;
172  }
173  }
174 
175  // check the path specified by environment variable
176  if (!m_fallbackTestFilesPath.empty()) {
177  file.clear();
178  file.open(path = m_fallbackTestFilesPath + name, ios_base::in);
179  if (file.good()) {
180  return path;
181  }
182  }
183 
184  // file still not found -> return default path
185  file.clear();
186  file.open(path = "./testfiles/" + name, ios_base::in);
187  if (!file.good()) {
188  cerr << Phrases::Warning << "The testfile \"" << path << "\" can not be located." << Phrases::EndFlush;
189  }
190  return path;
191 }
192 
193 #ifdef PLATFORM_UNIX
194 
202 string TestApplication::workingCopyPathMode(const string &name, WorkingCopyMode mode) const
203 {
204  // create file streams
205  fstream origFile, workingCopy;
206  origFile.exceptions(ios_base::badbit | ios_base::failbit);
207  workingCopy.exceptions(ios_base::badbit | ios_base::failbit);
208 
209  // ensure working directory is present
210  struct stat currentStat;
211  if (stat(m_workingDir.c_str(), &currentStat) || !S_ISDIR(currentStat.st_mode)) {
212  if (mkdir(m_workingDir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) {
213  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory." << Phrases::EndFlush;
214  return string();
215  }
216  }
217 
218  // ensure subdirectory exists
219  const auto parts = splitString<vector<string>>(name, string("/"), EmptyPartsTreat::Omit);
220  if (!parts.empty()) {
221  string currentLevel = m_workingDir;
222  for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
223  if (currentLevel.back() != '/') {
224  currentLevel += '/';
225  }
226  currentLevel += *i;
227  if (stat(currentLevel.c_str(), &currentStat) || !S_ISDIR(currentStat.st_mode)) {
228  if (mkdir(currentLevel.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) {
229  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory."
230  << Phrases::EndFlush;
231  return string();
232  }
233  }
234  }
235  }
236 
237  // copy file
238  if (mode != WorkingCopyMode::NoCopy) {
239  try {
240  origFile.open(testFilePath(name), ios_base::in | ios_base::binary);
241  const string path = m_workingDir + name;
242  workingCopy.open(path, ios_base::out | ios_base::binary | ios_base::trunc);
243  workingCopy << origFile.rdbuf();
244  return path;
245  } catch (...) {
246  catchIoFailure();
247  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": an IO error occured." << Phrases::EndFlush;
248  }
249  } else {
250  return m_workingDir + name;
251  }
252  return string();
253 }
254 
262 string TestApplication::workingCopyPath(const string &name) const
263 {
264  return workingCopyPathMode(name, WorkingCopyMode::CreateCopy);
265 }
266 
271 int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
272  const std::string &newProfilingPath)
273 {
274  // print log message
275  if (!suppressLogging) {
276  cout << '-';
277  for (const char *const *i = args; *i; ++i) {
278  cout << ' ' << *i;
279  }
280  cout << endl;
281  }
282 
283  // create pipes
284  int coutPipes[2], cerrPipes[2];
285  pipe(coutPipes), pipe(cerrPipes);
286  int readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
287  int readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
288 
289  // create child process
290  if (int child = fork()) {
291  // parent process: read stdout and stderr from child
292  close(writeCoutPipe), close(writeCerrPipe);
293 
294  try {
295  if (child == -1) {
296  throw runtime_error("Unable to create fork");
297  }
298 
299  // init file descriptor set for poll
300  struct pollfd fileDescriptorSet[2];
301  fileDescriptorSet[0].fd = readCoutPipe;
302  fileDescriptorSet[1].fd = readCerrPipe;
303  fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
304 
305  // init variables for reading
306  char buffer[512];
307  ssize_t count;
308  output.clear(), errors.clear();
309 
310  // poll as long as at least one pipe is open
311  do {
312  int retpoll = poll(fileDescriptorSet, 2, timeout);
313  if (retpoll > 0) {
314  // poll succeeds
315  if (fileDescriptorSet[0].revents & POLLIN) {
316  if ((count = read(readCoutPipe, buffer, sizeof(buffer))) > 0) {
317  output.append(buffer, static_cast<size_t>(count));
318  }
319  } else if (fileDescriptorSet[0].revents & POLLHUP) {
320  close(readCoutPipe);
321  fileDescriptorSet[0].fd = -1;
322  }
323  if (fileDescriptorSet[1].revents & POLLIN) {
324  if ((count = read(readCerrPipe, buffer, sizeof(buffer))) > 0) {
325  errors.append(buffer, static_cast<size_t>(count));
326  }
327  } else if (fileDescriptorSet[1].revents & POLLHUP) {
328  close(readCerrPipe);
329  fileDescriptorSet[1].fd = -1;
330  }
331  } else if (retpoll == 0) {
332  // timeout
333  throw runtime_error("Poll time-out");
334  } else {
335  // fail
336  throw runtime_error("Poll failed");
337  }
338  } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
339  } catch (...) {
340  // ensure all pipes are close in the error case
341  close(readCoutPipe), close(readCerrPipe);
342  throw;
343  }
344 
345  // get return code
346  int childReturnCode;
347  waitpid(child, &childReturnCode, 0);
348  return childReturnCode;
349  } else {
350  // child process
351  // -> set pipes to be used for stdout/stderr
352  dup2(writeCoutPipe, STDOUT_FILENO), dup2(writeCerrPipe, STDERR_FILENO);
353  close(readCoutPipe), close(writeCoutPipe), close(readCerrPipe), close(writeCerrPipe);
354 
355  // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
356  if (!newProfilingPath.empty()) {
357  setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
358  }
359 
360  // -> execute application
361  execv(appPath, const_cast<char *const *>(args));
362  cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": execv() failed" << Phrases::EndFlush;
363  exit(-101);
364  }
365 }
366 
376 int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
377 {
378  // increase counter used for giving profiling files unique names
379  static unsigned int invocationCount = 0;
380  ++invocationCount;
381 
382  // determine the path of the application to be tested
383  const char *appPath = m_applicationPathArg.firstValue();
384  string fallbackAppPath;
385  if (!appPath || !*appPath) {
386  // try to find the path by removing "_tests"-suffix from own executable path
387  // (the own executable path is the path of the test application and its name is usually the name of the application
388  // to be tested with "_tests"-suffix)
389  const char *const testAppPath = m_parser.executable();
390  const size_t testAppPathLength = strlen(testAppPath);
391  if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
392  fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
393  appPath = fallbackAppPath.data();
394  // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executalbe
395  } else {
396  throw runtime_error("Unable to execute application to be tested: no application path specified");
397  }
398  }
399 
400  // determine new path for profiling output (to not override profiling output of parent and previous invocations)
401  string newProfilingPath;
402  if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
403  // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
404  if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
405  const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
406  // extract application name from path
407  const char *appName = strrchr(appPath, '/');
408  appName = appName ? appName + 1 : appPath;
409  // concat new path
410  newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
411  // append path to profiling list file
412  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
413  ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
414  }
415  }
416  }
417 
418  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
419 }
420 
428 int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
429 {
430  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
431 }
432 #endif // PLATFORM_UNIX
433 
434 void TestApplication::readFallbackTestfilePathFromEnv()
435 {
436  if (const char *testFilesPathEnv = getenv("TEST_FILE_PATH")) {
437  if (const auto len = strlen(testFilesPathEnv)) {
438  m_fallbackTestFilesPath.reserve(len + 1);
439  m_fallbackTestFilesPath += testFilesPathEnv;
440  m_fallbackTestFilesPath += '/';
441  }
442  }
443 }
444 
445 void TestApplication::readFallbackTestfilePathFromSrcRef()
446 {
447  try {
448  // read "srcdirref" file which should contain the path of the source directory; this file should have been
449  // create by the CMake module "TestTarget.cmake"
450  const string srcDirContent(readFile("srcdirref", 2 * 1024));
451  if (srcDirContent.empty()) {
452  cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
453  return;
454  }
455 
456  // check whether the referenced source directory contains a "testfiles" directory
457 #ifdef PLATFORM_UNIX // directoryEntries() is not implemented under Windows so we can only to the check under UNIX
458  bool hasTestfilesDir = false;
459  for (const string &dir : directoryEntries(srcDirContent.data(), DirectoryEntryType::Directory)) {
460  if (dir == "testfiles") {
461  hasTestfilesDir = true;
462  break;
463  }
464  }
465  if (!hasTestfilesDir) {
466  cerr << Phrases::Warning
467  << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
468  << Phrases::End << "Referenced source directory: " << srcDirContent << endl;
469  return;
470  }
471 #endif // PLATFORM_UNIX
472 
473  m_fallbackTestFilesPath = move(srcDirContent);
474  m_fallbackTestFilesPath += "/testfiles/";
475  } catch (...) {
476  // the "srcdirref" file likely just does not exist, so ignore the error case for now
477  catchIoFailure();
478  }
479 }
480 } // namespace TestUtilities
Encapsulates functions for formatted terminal output using ANSI escape codes.
void setCombinable(bool value)
Sets whether this argument can be combined.
CPP_UTILITIES_EXPORT std::list< std::string > directoryEntries(const char *path, DirectoryEntryType types=DirectoryEntryType::All)
Returns the names of the directory entries in the specified path with the specified types...
Definition: path.cpp:181
Contains currently only ArgumentParser and related classes.
constexpr StringType argsToString(Args &&... args)
void setMainArguments(const ArgumentInitializerList &mainArguments)
Sets the main arguments for the parser.
STL namespace.
std::string testFilePath(const std::string &name) const
Returns the full path of the test file with the specified name.
Definition: testutils.cpp:162
void parseArgs(int argc, const char *const *argv)
Parses the specified command line arguments.
const char * firstValue() const
Returns the first parameter value of the first occurrence of the argument.
CPP_UTILITIES_EXPORT std::string readFile(const std::string &path, std::string::size_type maxSize=std::string::npos)
Reads all contents of the specified file in a single call.
Definition: misc.cpp:16
static const char * appPath()
Returns the application path or an empty string if no application path has been set.
Definition: testutils.h:78
Contains utility classes helping to read and write streams.
Definition: binaryreader.h:10
Contains classes and functions utilizing creating of test applications.
Definition: testutils.h:12
~TestApplication()
Destroys the TestApplication.
Definition: testutils.cpp:147
const std::vector< const char * > & values(std::size_t occurrence=0) const
Returns the parameter values for the specified occurrence of argument.
Contains several functions providing conversions between different data types.
const char * executable() const
Returns the name of the current executable.
The Argument class is a wrapper for command line argument information.
WorkingCopyMode
The WorkingCopyMode enum specifies additional options to influence behavior of TestApplication::worki...
Definition: testutils.h:17
bool isPresent() const
Returns an indication whether the argument could be detected when parsing.
The Failure class is thrown by an ArgumentParser when a parsing error occurs.
Definition: failure.h:12
CPP_UTILITIES_EXPORT const char * catchIoFailure()
Provides a workaround for GCC Bug 66145.
void setRequiredValueCount(std::size_t requiredValueCount)
Sets the number of values which are required to be given for this argument.
void setValueNames(std::initializer_list< const char *> valueNames)
Sets the names of the requried values.