C++ Utilities  4.11.0
Common C++ classes and routines used by my applications such as argument parser, IO and conversion utilities
testutils.cpp
Go to the documentation of this file.
1 #include "./testutils.h"
2 
3 #include "../application/failure.h"
4 #include "../conversion/stringbuilder.h"
5 #include "../conversion/stringconversion.h"
6 #include "../io/ansiescapecodes.h"
7 #include "../io/catchiofailure.h"
8 #include "../io/misc.h"
9 #include "../io/path.h"
10 
11 #include <cstdlib>
12 #include <cstring>
13 #include <fstream>
14 #include <initializer_list>
15 #include <iostream>
16 #include <limits>
17 
18 #ifdef PLATFORM_UNIX
19 #include <poll.h>
20 #include <sys/stat.h>
21 #include <sys/wait.h>
22 #include <unistd.h>
23 #endif
24 
25 using namespace std;
26 using namespace ApplicationUtilities;
27 using namespace ConversionUtilities;
28 using namespace EscapeCodes;
29 using namespace IoUtilities;
30 
34 namespace TestUtilities {
35 
36 TestApplication *TestApplication::m_instance = nullptr;
37 
48 TestApplication::TestApplication(int argc, char **argv)
49  : m_helpArg(m_parser)
50  , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files")
51  , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested")
52  , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files")
53  , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units")
54 {
55  // check whether there is already an instance
56  if (m_instance) {
57  throw runtime_error("only one TestApplication instance allowed at a time");
58  }
59  m_instance = this;
60 
61  // determine fallback path for testfiles which is used when --test-files-path/-p not present
62  // -> read TEST_FILE_PATH environment variable
63  readFallbackTestfilePathFromEnv();
64  // -> find source directory if TEST_FILE_PATH not present
65  const bool fallbackIsSourceDir = m_fallbackTestFilesPath.empty();
66  if (fallbackIsSourceDir) {
67  readFallbackTestfilePathFromSrcRef();
68  }
69 
70  // setup argument parser
71  for (Argument *arg : initializer_list<Argument *>{ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg }) {
72  arg->setRequiredValueCount(1);
73  arg->setValueNames({ "path" });
74  arg->setCombinable(true);
75  }
76  m_unitsArg.setRequiredValueCount(Argument::varValueCount);
77  m_unitsArg.setValueNames({ "unit1", "unit2", "unit3" });
78  m_unitsArg.setCombinable(true);
79  m_parser.setMainArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg, &m_helpArg });
80 
81  // parse arguments
82  try {
83  m_parser.parseArgs(argc, argv);
84  } catch (const Failure &failure) {
85  cerr << failure;
86  m_valid = false;
87  return;
88  }
89 
90  // print help
91  if (m_helpArg.isPresent()) {
92  m_valid = false;
93  exit(0);
94  }
95 
96  // handle path for testfiles and working-copy
97  cerr << "Directories used to search for testfiles:" << endl;
98  if (m_testFilesPathArg.isPresent()) {
99  if (*m_testFilesPathArg.values().front()) {
100  cerr << ((m_testFilesPath = m_testFilesPathArg.values().front()) += '/') << endl;
101  } else {
102  cerr << (m_testFilesPath = "./") << endl;
103  }
104  }
105  if (!m_fallbackTestFilesPath.empty() && m_testFilesPath != m_fallbackTestFilesPath) {
106  cerr << m_fallbackTestFilesPath << endl;
107  }
108  cerr << "./testfiles/" << endl << endl;
109  cerr << "Directory used to store working copies:" << endl;
110  if (m_workingDirArg.isPresent()) {
111  if (*m_workingDirArg.values().front()) {
112  (m_workingDir = m_workingDirArg.values().front()) += '/';
113  } else {
114  m_workingDir = "./";
115  }
116  } else if (const char *workingDirEnv = getenv("WORKING_DIR")) {
117  if (const auto len = strlen(workingDirEnv)) {
118  m_workingDir.reserve(len + 1);
119  m_workingDir += workingDirEnv;
120  m_workingDir += '/';
121  }
122  } else {
123  if (m_testFilesPathArg.isPresent()) {
124  m_workingDir = m_testFilesPath + "workingdir/";
125  } else if (!m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
126  m_workingDir = m_fallbackTestFilesPath + "workingdir/";
127  } else {
128  m_workingDir = "./testfiles/workingdir/";
129  }
130  }
131  cerr << m_workingDir << endl << endl;
132 
133  // clear list of all additional profiling files created when forking the test application
134  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
135  ofstream(profrawListFile, ios_base::trunc);
136  }
137 
138  m_valid = true;
139  cerr << TextAttribute::Bold << "Executing test cases ..." << Phrases::EndFlush;
140 }
141 
146 {
147  m_instance = nullptr;
148 }
149 
153 string TestApplication::testFilePath(const string &name) const
154 {
155  string path;
156  fstream file; // used to check whether the file is present
157 
158  // check the path specified by command line argument
159  if (m_testFilesPathArg.isPresent()) {
160  file.open(path = m_testFilesPath + name, ios_base::in);
161  if (file.good()) {
162  return path;
163  }
164  }
165 
166  // check the path specified by environment variable
167  if (!m_fallbackTestFilesPath.empty()) {
168  file.clear();
169  file.open(path = m_fallbackTestFilesPath + name, ios_base::in);
170  if (file.good()) {
171  return path;
172  }
173  }
174 
175  // file still not found -> return default path
176  file.clear();
177  file.open(path = "./testfiles/" + name, ios_base::in);
178  if (!file.good()) {
179  cerr << Phrases::Warning << "The testfile \"" << path << "\" can not be located." << Phrases::EndFlush;
180  }
181  return path;
182 }
183 
184 #ifdef PLATFORM_UNIX
185 
189 string TestApplication::workingCopyPathMode(const string &name, WorkingCopyMode mode) const
190 {
191  // create file streams
192  fstream origFile, workingCopy;
193  origFile.exceptions(ios_base::badbit | ios_base::failbit);
194  workingCopy.exceptions(ios_base::badbit | ios_base::failbit);
195 
196  // ensure working directory is present
197  struct stat currentStat;
198  if (stat(m_workingDir.c_str(), &currentStat) || !S_ISDIR(currentStat.st_mode)) {
199  if (mkdir(m_workingDir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) {
200  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory." << Phrases::EndFlush;
201  return string();
202  }
203  }
204 
205  // ensure subdirectory exists
206  const auto parts = splitString<vector<string>>(name, string("/"), EmptyPartsTreat::Omit);
207  if (!parts.empty()) {
208  string currentLevel = m_workingDir;
209  for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
210  if (currentLevel.back() != '/') {
211  currentLevel += '/';
212  }
213  currentLevel += *i;
214  if (stat(currentLevel.c_str(), &currentStat) || !S_ISDIR(currentStat.st_mode)) {
215  if (mkdir(currentLevel.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) {
216  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory."
217  << Phrases::EndFlush;
218  return string();
219  }
220  }
221  }
222  }
223 
224  // copy file
225  if (mode != WorkingCopyMode::NoCopy) {
226  try {
227  origFile.open(testFilePath(name), ios_base::in | ios_base::binary);
228  const string path = m_workingDir + name;
229  workingCopy.open(path, ios_base::out | ios_base::binary | ios_base::trunc);
230  workingCopy << origFile.rdbuf();
231  return path;
232  } catch (...) {
233  catchIoFailure();
234  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": an IO error occured." << Phrases::EndFlush;
235  }
236  } else {
237  return m_workingDir + name;
238  }
239  return string();
240 }
241 
242 string TestApplication::workingCopyPath(const string &name) const
243 {
244  return workingCopyPathMode(name, WorkingCopyMode::CreateCopy);
245 }
246 
251 int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
252  const std::string &newProfilingPath)
253 {
254  // print log message
255  if (!suppressLogging) {
256  cout << '-';
257  for (const char *const *i = args; *i; ++i) {
258  cout << ' ' << *i;
259  }
260  cout << endl;
261  }
262 
263  // create pipes
264  int coutPipes[2], cerrPipes[2];
265  pipe(coutPipes), pipe(cerrPipes);
266  int readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
267  int readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
268 
269  // create child process
270  if (int child = fork()) {
271  // parent process: read stdout and stderr from child
272  close(writeCoutPipe), close(writeCerrPipe);
273 
274  try {
275  if (child == -1) {
276  throw runtime_error("Unable to create fork");
277  }
278 
279  // init file descriptor set for poll
280  struct pollfd fileDescriptorSet[2];
281  fileDescriptorSet[0].fd = readCoutPipe;
282  fileDescriptorSet[1].fd = readCerrPipe;
283  fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
284 
285  // init variables for reading
286  char buffer[512];
287  ssize_t count;
288  output.clear(), errors.clear();
289 
290  // poll as long as at least one pipe is open
291  do {
292  int retpoll = poll(fileDescriptorSet, 2, timeout);
293  if (retpoll > 0) {
294  // poll succeeds
295  if (fileDescriptorSet[0].revents & POLLIN) {
296  if ((count = read(readCoutPipe, buffer, sizeof(buffer))) > 0) {
297  output.append(buffer, static_cast<size_t>(count));
298  }
299  } else if (fileDescriptorSet[0].revents & POLLHUP) {
300  close(readCoutPipe);
301  fileDescriptorSet[0].fd = -1;
302  }
303  if (fileDescriptorSet[1].revents & POLLIN) {
304  if ((count = read(readCerrPipe, buffer, sizeof(buffer))) > 0) {
305  errors.append(buffer, static_cast<size_t>(count));
306  }
307  } else if (fileDescriptorSet[1].revents & POLLHUP) {
308  close(readCerrPipe);
309  fileDescriptorSet[1].fd = -1;
310  }
311  } else if (retpoll == 0) {
312  // timeout
313  throw runtime_error("Poll time-out");
314  } else {
315  // fail
316  throw runtime_error("Poll failed");
317  }
318  } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
319  } catch (...) {
320  // ensure all pipes are close in the error case
321  close(readCoutPipe), close(readCerrPipe);
322  throw;
323  }
324 
325  // get return code
326  int childReturnCode;
327  waitpid(child, &childReturnCode, 0);
328  return childReturnCode;
329  } else {
330  // child process
331  // -> set pipes to be used for stdout/stderr
332  dup2(writeCoutPipe, STDOUT_FILENO), dup2(writeCerrPipe, STDERR_FILENO);
333  close(readCoutPipe), close(writeCoutPipe), close(readCerrPipe), close(writeCerrPipe);
334 
335  // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
336  if (!newProfilingPath.empty()) {
337  setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
338  }
339 
340  // -> execute application
341  execv(appPath, const_cast<char *const *>(args));
342  cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": execv() failed" << Phrases::EndFlush;
343  exit(-101);
344  }
345 }
346 
356 int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
357 {
358  // increase counter used for giving profiling files unique names
359  static unsigned int invocationCount = 0;
360  ++invocationCount;
361 
362  // determine the path of the application to be tested
363  const char *appPath = m_applicationPathArg.firstValue();
364  string fallbackAppPath;
365  if (!appPath || !*appPath) {
366  // try to find the path by removing "_tests"-suffix from own executable path
367  // (the own executable path is the path of the test application and its name is usually the name of the application
368  // to be tested with "_tests"-suffix)
369  const char *const testAppPath = m_parser.executable();
370  const size_t testAppPathLength = strlen(testAppPath);
371  if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
372  fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
373  appPath = fallbackAppPath.data();
374  // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executalbe
375  } else {
376  throw runtime_error("Unable to execute application to be tested: no application path specified");
377  }
378  }
379 
380  // determine new path for profiling output (to not override profiling output of parent and previous invocations)
381  string newProfilingPath;
382  if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
383  // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
384  if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
385  const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
386  // extract application name from path
387  const char *appName = strrchr(appPath, '/');
388  appName = appName ? appName + 1 : appPath;
389  // concat new path
390  newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
391  // append path to profiling list file
392  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
393  ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
394  }
395  }
396  }
397 
398  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
399 }
400 
408 int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
409 {
410  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
411 }
412 #endif // PLATFORM_UNIX
413 
414 void TestApplication::readFallbackTestfilePathFromEnv()
415 {
416  if (const char *testFilesPathEnv = getenv("TEST_FILE_PATH")) {
417  if (const auto len = strlen(testFilesPathEnv)) {
418  m_fallbackTestFilesPath.reserve(len + 1);
419  m_fallbackTestFilesPath += testFilesPathEnv;
420  m_fallbackTestFilesPath += '/';
421  }
422  }
423 }
424 
425 void TestApplication::readFallbackTestfilePathFromSrcRef()
426 {
427  try {
428  // read "srcdirref" file which should contain the path of the source directory; this file should have been
429  // create by the CMake module "TestTarget.cmake"
430  const string srcDirContent(readFile("srcdirref", 2 * 1024));
431  if (srcDirContent.empty()) {
432  cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
433  return;
434  }
435 
436  // check whether the referenced source directory contains a "testfiles" directory
437 #ifdef PLATFORM_UNIX // directoryEntries() is not implemented under Windows so we can only to the check under UNIX
438  bool hasTestfilesDir = false;
439  for (const string &dir : directoryEntries(srcDirContent.data(), DirectoryEntryType::Directory)) {
440  if (dir == "testfiles") {
441  hasTestfilesDir = true;
442  break;
443  }
444  }
445  if (!hasTestfilesDir) {
446  cerr << Phrases::Warning
447  << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
448  << Phrases::End << "Referenced source directory: " << srcDirContent << endl;
449  return;
450  }
451 #endif // PLATFORM_UNIX
452 
453  m_fallbackTestFilesPath = move(srcDirContent);
454  m_fallbackTestFilesPath += "/testfiles/";
455  } catch (...) {
456  // the "srcdirref" file likely just does not exist, so ignore the error case for now
457  catchIoFailure();
458  }
459 }
460 } // namespace TestUtilities
Encapsulates functions for formatted terminal output using ANSI escape codes.
void setCombinable(bool value)
Sets whether this argument can be combined.
CPP_UTILITIES_EXPORT std::list< std::string > directoryEntries(const char *path, DirectoryEntryType types=DirectoryEntryType::All)
Returns the names of the directory entries in the specified path with the specified types...
Definition: path.cpp:181
Contains currently only ArgumentParser and related classes.
constexpr StringType argsToString(Args &&... args)
void setMainArguments(const ArgumentInitializerList &mainArguments)
Sets the main arguments for the parser.
STL namespace.
std::string testFilePath(const std::string &name) const
Returns the full path of the test file with the specified name.
Definition: testutils.cpp:153
void parseArgs(int argc, const char *const *argv)
Parses the specified command line arguments.
const char * firstValue() const
Returns the first parameter value of the first occurrence of the argument.
CPP_UTILITIES_EXPORT std::string readFile(const std::string &path, std::string::size_type maxSize=std::string::npos)
Reads all contents of the specified file in a single call.
Definition: misc.cpp:16
static const char * appPath()
Returns the application path or an empty string if no application path has been set.
Definition: testutils.h:78
Contains utility classes helping to read and write streams.
Definition: binaryreader.h:10
Contains classes and functions utilizing creating of test applications.
Definition: testutils.h:12
~TestApplication()
Destroys the TestApplication.
Definition: testutils.cpp:145
const std::vector< const char * > & values(std::size_t occurrence=0) const
Returns the parameter values for the specified occurrence of argument.
Contains several functions providing conversions between different data types.
const char * executable() const
Returns the name of the current executable.
The Argument class is a wrapper for command line argument information.
WorkingCopyMode
The WorkingCopyMode enum specifies additional options to influence behavior of TestApplication::worki...
Definition: testutils.h:17
bool isPresent() const
Returns an indication whether the argument could be detected when parsing.
The Failure class is thrown by an ArgumentParser when a parsing error occurs.
Definition: failure.h:12
CPP_UTILITIES_EXPORT const char * catchIoFailure()
Provides a workaround for GCC Bug 66145.
void setRequiredValueCount(std::size_t requiredValueCount)
Sets the number of values which are required to be given for this argument.
void setValueNames(std::initializer_list< const char *> valueNames)
Sets the names of the requried values.