Add workaround for Windows-specific console problems

* Allow disabling the hack for creating a console for
  a GUI application via `ENABLE_CONSOLE=0` to workaround
  downsides of this hack (pipes not working, possibly
  spawns an additional console)
* Set the console's character set to UTF-8 by default
  because this actually seems to work now and
  non-ASCII characters are displayed correctly. There
  is still an opt-out via `ENABLE_CP_UTF8=0`.
* Note that with mintty it just works anyways so using
  that terminal emulator is still the best workaround.

CMakeLists.txt

@@ -171,12 +171,6 @@ else ()
     message(WARNING "The use of std::filesystem has been disabled. Bash completion for files and directories will not work.")
 endif ()
 
-# configure forcing UTF-8 code page under Windows
-option(FORCE_UTF8_CODEPAGE "forces use of UTF-8 code page under Windows via ApplicationUtilities::startConsole()" OFF)
-if (FORCE_UTF8_CODEPAGE)
-    list(APPEND META_PRIVATE_COMPILE_DEFINITIONS ${META_PROJECT_VARNAME}_FORCE_UTF8_CODEPAGE)
-endif ()
-
 # configure whether escape codes should be enabled by default
 option(ENABLE_ESCAPE_CODES_BY_DEAULT "enables usage of escape codes by default" ON)
 if (ENABLE_ESCAPE_CODES_BY_DEAULT)

application/argumentparser.cpp

@@ -32,6 +32,27 @@ using namespace CppUtilities::EscapeCodes;
  */
 namespace CppUtilities {
 
+/*!
+ * \brief Returns whether the specified env variable is set to a non-zero and non-white-space-only value.
+ */
+std::optional<bool> isEnvVariableSet(const char *variableName)
+{
+    const char *envValue = std::getenv(variableName);
+    if (!envValue) {
+        return std::nullopt;
+    }
+    for (; *envValue; ++envValue) {
+        switch (*envValue) {
+        case '0':
+        case ' ':
+            break;
+        default:
+            return true;
+        }
+    }
+    return false;
+}
+
 /*!
  * \brief The ArgumentDenotationType enum specifies the type of a given argument denotation.
  */
@@ -1728,27 +1749,14 @@ NoColorArgument::NoColorArgument()
 {
     setCombinable(true);
 
-    // set the environmentvariable: note that this is not directly used and just assigned for printing help
+    // set the environment variable (not directly used and just assigned for printing help)
     setEnvironmentVariable("ENABLE_ESCAPE_CODES");
 
-    // default-initialize EscapeCodes::enabled from environment variable
+    // initialize EscapeCodes::enabled from environment variable
-    const char *envValue = getenv(environmentVariable());
+    const auto escapeCodesEnabled = isEnvVariableSet(environmentVariable());
-    if (!envValue) {
+    if (escapeCodesEnabled.has_value()) {
-        return;
+        EscapeCodes::enabled = escapeCodesEnabled.value();
     }
-    for (; *envValue; ++envValue) {
-        switch (*envValue) {
-        case '0':
-        case ' ':
-            break;
-        default:
-            // enable escape codes if ENABLE_ESCAPE_CODES contains anything else than spaces or zeros
-            EscapeCodes::enabled = true;
-            return;
-        }
-    }
-    // disable escape codes if ENABLE_ESCAPE_CODES is empty or only contains spaces and zeros
-    EscapeCodes::enabled = false;
 }
 
 /*!

application/argumentparserprivate.h

@@ -4,6 +4,8 @@
 #include "./argumentparser.h"
 #include "./commandlineutils.h"
 
+#include <optional>
+
 namespace CppUtilities {
 
 class CPP_UTILITIES_EXPORT ArgumentReader {
@@ -56,6 +58,8 @@ inline Wrapper::Wrapper(const char *str, Indentation currentIndentation)
 {
 }
 
+std::optional<bool> isEnvVariableSet(const char *variableName);
+
 } // namespace CppUtilities
 
 #endif // APPLICATION_UTILITIES_ARGUMENTPARSER_PRIVATE_H

application/commandlineutils.cpp

@@ -1,4 +1,5 @@
 #include "./commandlineutils.h"
+#include "./argumentparserprivate.h"
 
 #include <iostream>
 #include <string>
@@ -73,52 +74,62 @@ void stopConsole()
     fclose(stdout);
     fclose(stdin);
     fclose(stderr);
-    if (auto *consoleWindow = GetConsoleWindow()) {
+    if (auto *const consoleWindow = GetConsoleWindow()) {
         PostMessage(consoleWindow, WM_KEYUP, VK_RETURN, 0);
         FreeConsole();
     }
 }
 
 /*!
- * \brief Starts the console and sets the console output code page to UTF-8 if this is configured.
+ * \brief Ensure the process has a console attached and sets its output code page to UTF-8.
  * \remarks
- * - only available under Windows
+ * - Only available (and required) under Windows where otherwise stdout/stderr is not printed to the console (at
- * - used to start a console from a GUI application
+ *   least when using `cmd.exe`).
- * - closes the console automatically when the application exists
+ * - Used to start a console from a GUI application. Does *not* create a new console if the process already has one.
+ * - Closes the console automatically when the application exits.
+ * - It breaks redirecting stdout/stderr so this can be opted-out by setting the enviornment
+ *   variable `ENABLE_CONSOLE=0` and/or `ENABLE_CP_UTF8=0`.
+ * \sa
+ * - https://docs.microsoft.com/en-us/windows/console/AttachConsole
+ * - https://docs.microsoft.com/en-us/windows/console/AllocConsole
+ * - https://docs.microsoft.com/en-us/windows/console/SetConsoleCP
+ * - https://docs.microsoft.com/en-us/windows/console/SetConsoleOutputCP
  */
 void startConsole()
 {
-    if (!AttachConsole(ATTACH_PARENT_PROCESS) && !AllocConsole()) {
+    // attach to the parent process' console or allocate a new console if that's not possible
-        return;
+    const auto consoleEnabled = isEnvVariableSet("ENABLE_CONSOLE");
+    if ((!consoleEnabled.has_value() || consoleEnabled.value()) && (AttachConsole(ATTACH_PARENT_PROCESS) || AllocConsole())) {
+        // redirect stdout
+        auto stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_OUTPUT_HANDLE));
+        auto conHandle = _open_osfhandle(stdHandle, _O_TEXT);
+        auto fp = _fdopen(conHandle, "w");
+        *stdout = *fp;
+        setvbuf(stdout, nullptr, _IONBF, 0);
+        // redirect stdin
+        stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_INPUT_HANDLE));
+        conHandle = _open_osfhandle(stdHandle, _O_TEXT);
+        fp = _fdopen(conHandle, "r");
+        *stdin = *fp;
+        setvbuf(stdin, nullptr, _IONBF, 0);
+        // redirect stderr
+        stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_ERROR_HANDLE));
+        conHandle = _open_osfhandle(stdHandle, _O_TEXT);
+        fp = _fdopen(conHandle, "w");
+        *stderr = *fp;
+        setvbuf(stderr, nullptr, _IONBF, 0);
+        // sync
+        ios::sync_with_stdio(true);
+        // ensure the console prompt is shown again when app terminates
+        atexit(stopConsole);
+    }
+
+    // set console character set to UTF-8
+    const auto utf8Enabled = isEnvVariableSet("ENABLE_CP_UTF8");
+    if (!utf8Enabled.has_value() || utf8Enabled.value()) {
+        SetConsoleCP(CP_UTF8);
+        SetConsoleOutputCP(CP_UTF8);
     }
-    // redirect stdout
-    auto stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_OUTPUT_HANDLE));
-    auto conHandle = _open_osfhandle(stdHandle, _O_TEXT);
-    auto fp = _fdopen(conHandle, "w");
-    *stdout = *fp;
-    setvbuf(stdout, nullptr, _IONBF, 0);
-    // redirect stdin
-    stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_INPUT_HANDLE));
-    conHandle = _open_osfhandle(stdHandle, _O_TEXT);
-    fp = _fdopen(conHandle, "r");
-    *stdin = *fp;
-    setvbuf(stdin, nullptr, _IONBF, 0);
-    // redirect stderr
-    stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_ERROR_HANDLE));
-    conHandle = _open_osfhandle(stdHandle, _O_TEXT);
-    fp = _fdopen(conHandle, "w");
-    *stderr = *fp;
-    setvbuf(stderr, nullptr, _IONBF, 0);
-#ifdef CPP_UTILITIES_FORCE_UTF8_CODEPAGE
-    // set console to handle UTF-8 IO correctly
-    // however, this doesn't work as intended and is therefore disabled by default
-    SetConsoleCP(CP_UTF8);
-    SetConsoleOutputCP(CP_UTF8);
-#endif
-    // sync
-    ios::sync_with_stdio(true);
-    // ensure the console prompt is shown again when app terminates
-    atexit(stopConsole);
 }
 
 /*!