From 9e036a3f72961f02c34be8965f314cb8a4b83707 Mon Sep 17 00:00:00 2001
From: Martchus <martchus@gmx.net>
Date: Tue, 7 Jun 2022 12:36:48 +0200
Subject: [PATCH] Allow enabling virtual terminal processing under Windows

* or disable use of ANSI-escape codes if not possible
* See https://github.com/Martchus/syncthingtray/issues/132
---
 CMakeLists.txt                   |  2 +-
 application/commandlineutils.cpp | 47 ++++++++++++++++++++++++++++++++
 application/commandlineutils.h   |  3 ++
 3 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 005c0f2..6b0aa6d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -114,7 +114,7 @@ set(META_APP_AUTHOR "Martchus")
 set(META_APP_URL "https://github.com/${META_APP_AUTHOR}/${META_PROJECT_NAME}")
 set(META_APP_DESCRIPTION "Useful C++ classes and routines such as argument parser, IO and conversion utilities")
 set(META_VERSION_MAJOR 5)
-set(META_VERSION_MINOR 15)
+set(META_VERSION_MINOR 16)
 set(META_VERSION_PATCH 0)
 
 # find required 3rd party libraries
diff --git a/application/commandlineutils.cpp b/application/commandlineutils.cpp
index ce7b0a3..e1ae7b2 100644
--- a/application/commandlineutils.cpp
+++ b/application/commandlineutils.cpp
@@ -1,6 +1,8 @@
 #include "./commandlineutils.h"
 #include "./argumentparserprivate.h"
 
+#include "../io/ansiescapecodes.h"
+
 #include <iostream>
 #include <string>
 
@@ -8,6 +10,7 @@
 #include <sys/ioctl.h>
 #include <unistd.h>
 #else
+#include <cstring>
 #include <fcntl.h>
 #include <windows.h>
 #endif
@@ -65,6 +68,47 @@ TerminalSize determineTerminalSize()
 }
 
 #ifdef PLATFORM_WINDOWS
+/*!
+ * \brief Enables virtual terminal processing (and thus processing of ANSI escape codes) of the console
+ *        determined by the specified \a nStdHandle.
+ * \sa https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
+ */
+static bool enableVirtualTerminalProcessing(DWORD nStdHandle)
+{
+    auto stdHandle = GetStdHandle(nStdHandle);
+    if (stdHandle == INVALID_HANDLE_VALUE) {
+        return false;
+    }
+    auto dwMode = DWORD();
+    if (!GetConsoleMode(stdHandle, &dwMode)) {
+        return false;
+    }
+    dwMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+    return SetConsoleMode(stdHandle, dwMode);
+}
+
+/*!
+ * \brief Enables virtual terminal processing (and thus processing of ANSI escape codes) of the console
+ *        or disables use of ANSI escape codes if that's not possible.
+ */
+bool handleVirtualTerminalProcessing()
+{
+    // try to enable virtual terminal processing
+    if (enableVirtualTerminalProcessing(STD_OUTPUT_HANDLE) && enableVirtualTerminalProcessing(STD_ERROR_HANDLE)) {
+         return true;
+    }
+    // disable use on ANSI escape codes otherwise if it makes sense
+    const char *const msyscon = std::getenv("MSYSCON");
+    if (msyscon && std::strstr(msyscon, "mintty")) {
+        return false;  // no need to disable escape codes if it is just mintty
+    }
+    const char *const term = std::getenv("TERM");
+    if (term && std::strstr(term, "xterm")) {
+        return false;  // no need to disable escape codes if it is some xterm-like terminal
+    }
+    return EscapeCodes::enabled = false;
+}
+
 /*!
  * \brief Closes stdout, stdin and stderr and stops the console.
  * \remarks Internally used by startConsole() to close the console when the application exits.
@@ -130,6 +174,9 @@ void startConsole()
         SetConsoleCP(CP_UTF8);
         SetConsoleOutputCP(CP_UTF8);
     }
+
+    // enable virtual terminal processing or disable ANSI-escape if that's not possible
+    handleVirtualTerminalProcessing();
 }
 
 /*!
diff --git a/application/commandlineutils.h b/application/commandlineutils.h
index a8b2c82..f47b92d 100644
--- a/application/commandlineutils.h
+++ b/application/commandlineutils.h
@@ -20,6 +20,7 @@ enum class Response { None, Yes, No };
 CPP_UTILITIES_EXPORT bool confirmPrompt(const char *message, Response defaultResponse = Response::None);
 
 #ifdef PLATFORM_WINDOWS
+CPP_UTILITIES_EXPORT bool handleVirtualTerminalProcessing();
 CPP_UTILITIES_EXPORT void startConsole();
 CPP_UTILITIES_EXPORT std::pair<std::vector<std::unique_ptr<char[]>>, std::vector<char *>> convertArgsToUtf8();
 #define CMD_UTILS_START_CONSOLE ::CppUtilities::startConsole();
@@ -27,9 +28,11 @@ CPP_UTILITIES_EXPORT std::pair<std::vector<std::unique_ptr<char[]>>, std::vector
     auto utf8Args = ::CppUtilities::convertArgsToUtf8();                                                                                             \
     argv = utf8Args.second.data();                                                                                                                   \
     argc = static_cast<int>(utf8Args.second.size());
+#define CMD_UTILS_HANDLE_VIRTUAL_TERMINAL_PROCESSING ::CppUtilities::handleVirtualTerminalProcessing();
 #else
 #define CMD_UTILS_START_CONSOLE
 #define CMD_UTILS_CONVERT_ARGS_TO_UTF8
+#define CMD_UTILS_HANDLE_VIRTUAL_TERMINAL_PROCESSING
 #endif
 
 /*!