C++ Utilities  4.16.0
Useful C++ classes and routines such as argument parser, IO and conversion utilities
testutils.cpp
Go to the documentation of this file.
1 #include "./testutils.h"
2 
3 #include "../application/failure.h"
4 #include "../conversion/stringbuilder.h"
5 #include "../conversion/stringconversion.h"
6 #include "../io/ansiescapecodes.h"
7 #include "../io/catchiofailure.h"
8 #include "../io/misc.h"
9 #include "../io/nativefilestream.h"
10 #include "../io/path.h"
11 
12 #include <cerrno>
13 #include <cstdlib>
14 #include <cstring>
15 #include <fstream>
16 #include <initializer_list>
17 #include <iostream>
18 #include <limits>
19 
20 #ifdef PLATFORM_UNIX
21 #include <poll.h>
22 #include <sys/stat.h>
23 #include <sys/wait.h>
24 #include <unistd.h>
25 #endif
26 
27 #ifdef PLATFORM_WINDOWS
28 #include <windows.h>
29 #endif
30 
31 using namespace std;
32 using namespace ApplicationUtilities;
33 using namespace ConversionUtilities;
34 using namespace EscapeCodes;
35 using namespace IoUtilities;
36 
40 namespace TestUtilities {
41 
42 bool fileSystemItemExists(const string &path)
43 {
44 #ifdef PLATFORM_UNIX
45  struct stat res;
46  return stat(path.data(), &res) == 0;
47 #else
48  const auto widePath(convertMultiByteToWide(path));
49  if (!widePath.first) {
50  return false;
51  }
52  const auto fileType(GetFileAttributesW(widePath.first.get()));
53  return fileType != INVALID_FILE_ATTRIBUTES;
54 #endif
55 }
56 
57 bool fileExists(const string &path)
58 {
59 #ifdef PLATFORM_UNIX
60  struct stat res;
61  return stat(path.data(), &res) == 0 && !S_ISDIR(res.st_mode);
62 #else
63  const auto widePath(convertMultiByteToWide(path));
64  if (!widePath.first) {
65  return false;
66  }
67  const auto fileType(GetFileAttributesW(widePath.first.get()));
68  return (fileType != INVALID_FILE_ATTRIBUTES) && !(fileType & FILE_ATTRIBUTE_DIRECTORY) && !(fileType & FILE_ATTRIBUTE_DEVICE);
69 #endif
70 }
71 
72 bool dirExists(const string &path)
73 {
74 #ifdef PLATFORM_UNIX
75  struct stat res;
76  return stat(path.data(), &res) == 0 && S_ISDIR(res.st_mode);
77 #else
78  const auto widePath(convertMultiByteToWide(path));
79  if (!widePath.first) {
80  return false;
81  }
82  const auto fileType(GetFileAttributesW(widePath.first.get()));
83  return (fileType != INVALID_FILE_ATTRIBUTES) && (fileType & FILE_ATTRIBUTE_DIRECTORY);
84 #endif
85 }
86 
87 bool makeDir(const string &path)
88 {
89 #ifdef PLATFORM_UNIX
90  return mkdir(path.data(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0;
91 #else
92  const auto widePath(convertMultiByteToWide(path));
93  if (!widePath.first) {
94  return false;
95  }
96  return CreateDirectoryW(widePath.first.get(), nullptr) || GetLastError() == ERROR_ALREADY_EXISTS;
97 #endif
98 }
99 
100 TestApplication *TestApplication::m_instance = nullptr;
101 
112 TestApplication::TestApplication(int argc, char **argv)
113  : m_helpArg(m_parser)
114  , m_testFilesPathArg("test-files-path", 'p', "specifies the path of the directory with test files")
115  , m_applicationPathArg("app-path", 'a', "specifies the path of the application to be tested")
116  , m_workingDirArg("working-dir", 'w', "specifies the directory to store working copies of test files")
117  , m_unitsArg("units", 'u', "specifies the units to test; omit to test all units")
118 {
119  // check whether there is already an instance
120  if (m_instance) {
121  throw runtime_error("only one TestApplication instance allowed at a time");
122  }
123  m_instance = this;
124 
125  // determine fallback path for testfiles which is used when --test-files-path/-p not present
126  // -> read TEST_FILE_PATH environment variable
127  m_fallbackTestFilesPath = readTestfilePathFromEnv();
128  // -> find source directory if TEST_FILE_PATH not present
129  bool fallbackIsSourceDir = m_fallbackTestFilesPath.empty();
130  if (fallbackIsSourceDir) {
131  m_fallbackTestFilesPath = readTestfilePathFromSrcRef();
132  }
133 
134  // handle specified arguments (if present)
135  if (argc && argv) {
136  // setup argument parser
137  for (Argument *arg : initializer_list<Argument *>{ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg }) {
138  arg->setRequiredValueCount(1);
139  arg->setValueNames({ "path" });
140  arg->setCombinable(true);
141  }
142  m_unitsArg.setRequiredValueCount(Argument::varValueCount);
143  m_unitsArg.setValueNames({ "unit1", "unit2", "unit3" });
144  m_unitsArg.setCombinable(true);
145  m_parser.setMainArguments({ &m_testFilesPathArg, &m_applicationPathArg, &m_workingDirArg, &m_unitsArg, &m_helpArg });
146 
147  // parse arguments
148  try {
149  m_parser.parseArgs(argc, argv);
150  } catch (const Failure &failure) {
151  cerr << failure;
152  m_valid = false;
153  return;
154  }
155 
156  // print help
157  if (m_helpArg.isPresent()) {
158  exit(0);
159  }
160  }
161 
162  // handle path for testfiles and working-copy
163  cerr << "Directories used to search for testfiles:" << endl;
164  if (m_testFilesPathArg.isPresent()) {
165  if (*m_testFilesPathArg.values().front()) {
166  cerr << ((m_testFilesPath = m_testFilesPathArg.values().front()) += '/') << endl;
167  } else {
168  cerr << (m_testFilesPath = "./") << endl;
169  }
170  } else {
171  // use fallback path if --test-files-path/-p not present
172  m_testFilesPath.swap(m_fallbackTestFilesPath);
173  cerr << m_testFilesPath << endl;
174  }
175  // if it wasn't already the case, use the source directory as fallback dir
176  if (m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
177  m_fallbackTestFilesPath = readTestfilePathFromSrcRef();
178  fallbackIsSourceDir = true;
179  }
180  if (!m_fallbackTestFilesPath.empty() && m_testFilesPath != m_fallbackTestFilesPath) {
181  cerr << m_fallbackTestFilesPath << endl;
182  }
183  cerr << "./testfiles/" << endl << endl;
184  cerr << "Directory used to store working copies:" << endl;
185  if (m_workingDirArg.isPresent()) {
186  if (*m_workingDirArg.values().front()) {
187  (m_workingDir = m_workingDirArg.values().front()) += '/';
188  } else {
189  m_workingDir = "./";
190  }
191  } else if (const char *workingDirEnv = getenv("WORKING_DIR")) {
192  if (*workingDirEnv) {
193  m_workingDir = argsToString(workingDirEnv, '/');
194  }
195  } else {
196  if (m_testFilesPathArg.isPresent()) {
197  m_workingDir = m_testFilesPath + "workingdir/";
198  } else if (!m_fallbackTestFilesPath.empty() && !fallbackIsSourceDir) {
199  m_workingDir = m_fallbackTestFilesPath + "workingdir/";
200  } else {
201  m_workingDir = "./testfiles/workingdir/";
202  }
203  }
204  cerr << m_workingDir << endl << endl;
205 
206  // clear list of all additional profiling files created when forking the test application
207  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
208  ofstream(profrawListFile, ios_base::trunc);
209  }
210 
211  m_valid = true;
212  cerr << TextAttribute::Bold << "Executing test cases ..." << Phrases::EndFlush;
213 }
214 
219 {
220  m_instance = nullptr;
221 }
222 
235 string TestApplication::testFilePath(const string &name) const
236 {
237  string path;
238  fstream file; // used to check whether the file is present
239 
240  // check the path specified by command line argument or via environment variable
241  if (!m_testFilesPath.empty()) {
242  if (fileExists(path = m_testFilesPath + name)) {
243  return path;
244  }
245  }
246 
247  // check the fallback path (value from environment variable or source directory)
248  if (!m_fallbackTestFilesPath.empty()) {
249  if (fileExists(path = m_fallbackTestFilesPath + name)) {
250  return path;
251  }
252  }
253 
254  // file still not found -> return default path
255  if (!fileExists(path = "./testfiles/" + name)) {
256  cerr << Phrases::Warning << "The testfile \"" << name << "\" can not be located." << Phrases::EndFlush;
257  }
258  return path;
259 }
260 
269 string TestApplication::workingCopyPathMode(const string &name, WorkingCopyMode mode) const
270 {
271  // ensure working directory is present
272  if (!dirExists(m_workingDir) && !makeDir(m_workingDir)) {
273  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create working directory \"" << m_workingDir << "\"."
274  << Phrases::EndFlush;
275  return string();
276  }
277 
278  // ensure subdirectory exists
279  const auto parts = splitString<vector<string>>(name, "/", EmptyPartsTreat::Omit);
280  if (!parts.empty()) {
281  // create subdirectory level by level
282  string currentLevel;
283  currentLevel.reserve(m_workingDir.size() + name.size() + 1);
284  currentLevel.assign(m_workingDir);
285  for (auto i = parts.cbegin(), end = parts.end() - 1; i != end; ++i) {
286  if (currentLevel.back() != '/') {
287  currentLevel += '/';
288  }
289  currentLevel += *i;
290 
291  // continue if subdirectory level already exists or we can successfully create the directory
292  if (dirExists(currentLevel) || makeDir(currentLevel)) {
293  continue;
294  }
295  // fail otherwise
296  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": can't create directory \"" << currentLevel
297  << "\" (inside working directory)." << Phrases::EndFlush;
298  return string();
299  }
300  }
301 
302  // just return the path if we don't want to actually create a copy
303  if (mode == WorkingCopyMode::NoCopy) {
304  return m_workingDir + name;
305  }
306 
307  // copy the file
308  const auto origFilePath(testFilePath(name));
309  auto workingCopyPath(m_workingDir + name);
310  size_t workingCopyPathAttempt = 0;
311  NativeFileStream origFile, workingCopy;
312  origFile.open(origFilePath, ios_base::in | ios_base::binary);
313  if (origFile.fail()) {
314  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": an IO error occurred when opening original file \""
315  << origFilePath << "\"." << Phrases::EndFlush;
316  cerr << "error: " << strerror(errno) << endl;
317  return string();
318  }
319  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
320  while (workingCopy.fail() && fileSystemItemExists(workingCopyPath)) {
321  // adjust the working copy path if the target file already exists and can not be truncated
322  workingCopyPath = argsToString(m_workingDir, name, '.', ++workingCopyPathAttempt);
323  workingCopy.clear();
324  workingCopy.open(workingCopyPath, ios_base::out | ios_base::binary | ios_base::trunc);
325  }
326  if (workingCopy.fail()) {
327  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": an IO error occurred when opening target file \""
328  << workingCopyPath << "\"." << Phrases::EndFlush;
329  cerr << "error: " << strerror(errno) << endl;
330  return string();
331  }
332  workingCopy << origFile.rdbuf();
333  if (!origFile.fail() && !workingCopy.fail()) {
334  return workingCopyPath;
335  }
336 
337  cerr << Phrases::Error << "Unable to create working copy for \"" << name << "\": ";
338  if (origFile.fail()) {
339  cerr << "an IO error occurred when reading original file \"" << origFilePath << "\"";
340  return string();
341  }
342  if (workingCopy.fail()) {
343  if (origFile.fail()) {
344  cerr << " and ";
345  }
346  cerr << " an IO error occurred when writing to target file \"" << workingCopyPath << "\".";
347  }
348  cerr << "error: " << strerror(errno) << endl;
349  return string();
350 }
351 
359 string TestApplication::workingCopyPath(const string &name) const
360 {
362 }
363 
364 #ifdef PLATFORM_UNIX
365 
369 int execAppInternal(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout,
370  const std::string &newProfilingPath)
371 {
372  // print log message
373  if (!suppressLogging) {
374  // print actual appPath and skip first argument instead
375  cout << '-' << ' ' << appPath;
376  if (*args) {
377  for (const char *const *i = args + 1; *i; ++i) {
378  cout << ' ' << *i;
379  }
380  }
381  cout << endl;
382  }
383 
384  // create pipes
385  int coutPipes[2], cerrPipes[2];
386  pipe(coutPipes);
387  pipe(cerrPipes);
388  const auto readCoutPipe = coutPipes[0], writeCoutPipe = coutPipes[1];
389  const auto readCerrPipe = cerrPipes[0], writeCerrPipe = cerrPipes[1];
390 
391  // create child process
392  if (const auto child = fork()) {
393  // parent process: read stdout and stderr from child
394  close(writeCoutPipe);
395  close(writeCerrPipe);
396 
397  try {
398  if (child == -1) {
399  throw runtime_error("Unable to create fork");
400  }
401 
402  // init file descriptor set for poll
403  struct pollfd fileDescriptorSet[2];
404  fileDescriptorSet[0].fd = readCoutPipe;
405  fileDescriptorSet[1].fd = readCerrPipe;
406  fileDescriptorSet[0].events = fileDescriptorSet[1].events = POLLIN;
407 
408  // init variables for reading
409  char buffer[512];
410  output.clear();
411  errors.clear();
412 
413  // poll as long as at least one pipe is open
414  do {
415  const auto retpoll = poll(fileDescriptorSet, 2, timeout);
416  if (retpoll == 0) {
417  throw runtime_error("Poll time-out");
418  }
419  if (retpoll < 0) {
420  throw runtime_error("Poll failed");
421  }
422  if (fileDescriptorSet[0].revents & POLLIN) {
423  const auto count = read(readCoutPipe, buffer, sizeof(buffer));
424  if (count > 0) {
425  output.append(buffer, static_cast<size_t>(count));
426  }
427  } else if (fileDescriptorSet[0].revents & POLLHUP) {
428  close(readCoutPipe);
429  fileDescriptorSet[0].fd = -1;
430  }
431  if (fileDescriptorSet[1].revents & POLLIN) {
432  const auto count = read(readCerrPipe, buffer, sizeof(buffer));
433  if (count > 0) {
434  errors.append(buffer, static_cast<size_t>(count));
435  }
436  } else if (fileDescriptorSet[1].revents & POLLHUP) {
437  close(readCerrPipe);
438  fileDescriptorSet[1].fd = -1;
439  }
440  } while (fileDescriptorSet[0].fd >= 0 || fileDescriptorSet[1].fd >= 0);
441  } catch (...) {
442  // ensure all pipes are closed in the error case
443  close(readCoutPipe);
444  close(readCerrPipe);
445  throw;
446  }
447 
448  // get return code
449  int childReturnCode;
450  waitpid(child, &childReturnCode, 0);
451  return childReturnCode;
452  } else {
453  // child process
454  // -> set pipes to be used for stdout/stderr
455  dup2(writeCoutPipe, STDOUT_FILENO);
456  dup2(writeCerrPipe, STDERR_FILENO);
457  close(readCoutPipe);
458  close(writeCoutPipe);
459  close(readCerrPipe);
460  close(writeCerrPipe);
461 
462  // -> modify environment variable LLVM_PROFILE_FILE to apply new path for profiling output
463  if (!newProfilingPath.empty()) {
464  setenv("LLVM_PROFILE_FILE", newProfilingPath.data(), true);
465  }
466 
467  // -> execute application
468  execv(appPath, const_cast<char *const *>(args));
469  cerr << Phrases::Error << "Unable to execute \"" << appPath << "\": execv() failed" << Phrases::EndFlush;
470  exit(-101);
471  }
472 }
473 
483 int TestApplication::execApp(const char *const *args, string &output, string &errors, bool suppressLogging, int timeout) const
484 {
485  // increase counter used for giving profiling files unique names
486  static unsigned int invocationCount = 0;
487  ++invocationCount;
488 
489  // determine the path of the application to be tested
490  const char *appPath = m_applicationPathArg.firstValue();
491  string fallbackAppPath;
492  if (!appPath || !*appPath) {
493  // try to find the path by removing "_tests"-suffix from own executable path
494  // (the own executable path is the path of the test application and its name is usually the name of the application
495  // to be tested with "_tests"-suffix)
496  const char *const testAppPath = m_parser.executable();
497  const size_t testAppPathLength = strlen(testAppPath);
498  if (testAppPathLength > 6 && !strcmp(testAppPath + testAppPathLength - 6, "_tests")) {
499  fallbackAppPath.assign(testAppPath, testAppPathLength - 6);
500  appPath = fallbackAppPath.data();
501  // TODO: it would not hurt to verify whether "fallbackAppPath" actually exists and is executalbe
502  } else {
503  throw runtime_error("Unable to execute application to be tested: no application path specified");
504  }
505  }
506 
507  // determine new path for profiling output (to not override profiling output of parent and previous invocations)
508  string newProfilingPath;
509  if (const char *llvmProfileFile = getenv("LLVM_PROFILE_FILE")) {
510  // replace eg. "/some/path/tageditor_tests.profraw" with "/some/path/tageditor0.profraw"
511  if (const char *llvmProfileFileEnd = strstr(llvmProfileFile, ".profraw")) {
512  const string llvmProfileFileWithoutExtension(llvmProfileFile, llvmProfileFileEnd);
513  // extract application name from path
514  const char *appName = strrchr(appPath, '/');
515  appName = appName ? appName + 1 : appPath;
516  // concat new path
517  newProfilingPath = argsToString(llvmProfileFileWithoutExtension, '_', appName, invocationCount, ".profraw");
518  // append path to profiling list file
519  if (const char *profrawListFile = getenv("LLVM_PROFILE_LIST_FILE")) {
520  ofstream(profrawListFile, ios_base::app) << newProfilingPath << endl;
521  }
522  }
523  }
524 
525  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, newProfilingPath);
526 }
527 
535 int execHelperApp(const char *appPath, const char *const *args, std::string &output, std::string &errors, bool suppressLogging, int timeout)
536 {
537  return execAppInternal(appPath, args, output, errors, suppressLogging, timeout, string());
538 }
539 #endif // PLATFORM_UNIX
540 
541 string TestApplication::readTestfilePathFromEnv()
542 {
543  const char *const testFilesPathEnv = getenv("TEST_FILE_PATH");
544  if (!testFilesPathEnv || !*testFilesPathEnv) {
545  return string();
546  }
547  return argsToString(testFilesPathEnv, '/');
548 }
549 
550 string TestApplication::readTestfilePathFromSrcRef()
551 {
552  try {
553  // read "srcdirref" file which should contain the path of the source directory; this file should have been
554  // create by the CMake module "TestTarget.cmake"
555  auto srcDirContent(readFile("srcdirref", 2 * 1024));
556  if (srcDirContent.empty()) {
557  cerr << Phrases::Warning << "The file \"srcdirref\" is empty." << Phrases::EndFlush;
558  return string();
559  }
560 
561  // check whether the referenced source directory contains a "testfiles" directory
562 #ifdef PLATFORM_UNIX // directoryEntries() is not implemented under Windows so we can only to the check under UNIX
563  bool hasTestfilesDir = false;
564  for (const string &dir : directoryEntries(srcDirContent.data(), DirectoryEntryType::Directory)) {
565  if (dir == "testfiles") {
566  hasTestfilesDir = true;
567  break;
568  }
569  }
570  if (!hasTestfilesDir) {
571  cerr << Phrases::Warning
572  << "The source directory referenced by the file \"srcdirref\" does not contain a \"testfiles\" directory or does not exist."
573  << Phrases::End << "Referenced source directory: " << srcDirContent << endl;
574  return string();
575  }
576 #endif // PLATFORM_UNIX
577 
578  return srcDirContent += "/testfiles/";
579  } catch (...) {
580  cerr << Phrases::Warning << "The file \"srcdirref\" can not be opened. It likely just doesn't exist in the working directory."
581  << Phrases::EndFlush;
582  catchIoFailure();
583  }
584  return string();
585 }
586 } // namespace TestUtilities
Encapsulates functions for formatted terminal output using ANSI escape codes.
std::string workingCopyPathMode(const std::string &name, WorkingCopyMode mode) const
Returns the full path to a working copy of the test file with the specified name. ...
Definition: testutils.cpp:269
void setCombinable(bool value)
Sets whether this argument can be combined.
CPP_UTILITIES_EXPORT std::list< std::string > directoryEntries(const char *path, DirectoryEntryType types=DirectoryEntryType::All)
Returns the names of the directory entries in the specified path with the specified types...
Definition: path.cpp:181
Contains currently only ArgumentParser and related classes.
constexpr StringType argsToString(Args &&... args)
void setMainArguments(const ArgumentInitializerList &mainArguments)
Sets the main arguments for the parser.
bool fileSystemItemExists(const string &path)
Definition: testutils.cpp:42
STL namespace.
bool makeDir(const string &path)
Definition: testutils.cpp:87
std::string testFilePath(const std::string &name) const
Returns the full path of the test file with the specified name.
Definition: testutils.cpp:235
void parseArgs(int argc, const char *const *argv)
Parses the specified command line arguments.
const char * firstValue() const
Returns the first parameter value of the first occurrence of the argument.
std::fstream NativeFileStream
std::string workingCopyPath(const std::string &name) const
Creates a working copy of the test file with the specified name and returns the full path of the crea...
Definition: testutils.cpp:359
CPP_UTILITIES_EXPORT std::string readFile(const std::string &path, std::string::size_type maxSize=std::string::npos)
Reads all contents of the specified file in a single call.
Definition: misc.cpp:16
bool fileExists(const string &path)
Definition: testutils.cpp:57
static const char * appPath()
Returns the application path or an empty string if no application path has been set.
Definition: testutils.h:79
Contains utility classes helping to read and write streams.
Definition: binaryreader.h:10
Contains classes and functions utilizing creating of test applications.
Definition: testutils.h:13
constexpr int i
~TestApplication()
Destroys the TestApplication.
Definition: testutils.cpp:218
const std::vector< const char * > & values(std::size_t occurrence=0) const
Returns the parameter values for the specified occurrence of argument.
Contains several functions providing conversions between different data types.
const char * executable() const
Returns the name of the current executable.
The Argument class is a wrapper for command line argument information.
bool dirExists(const string &path)
Definition: testutils.cpp:72
WorkingCopyMode
The WorkingCopyMode enum specifies additional options to influence behavior of TestApplication::worki...
Definition: testutils.h:18
bool isPresent() const
Returns an indication whether the argument could be detected when parsing.
The Failure class is thrown by an ArgumentParser when a parsing error occurs.
Definition: failure.h:12
CPP_UTILITIES_EXPORT const char * catchIoFailure()
Provides a workaround for GCC Bug 66145.
void setRequiredValueCount(std::size_t requiredValueCount)
Sets the number of values which are required to be given for this argument.
void setValueNames(std::initializer_list< const char *> valueNames)
Sets the names of the requried values.