Martchus
/
cpp-utilities
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve workaround for starting console from GUI app 

It is still not perfect but now at least:

* Redirections to a file and pipes are no longer broken with `cmd.exe`
* Pipes are no longer broken in PowerShell
* It works at all when using MSVC
* The whole workaround is skipped for Mintty (as it is not needed at all)
* Sending a return key to the terminal in the end has been removed as it
  caused more harm than good in certain terminals and did not work in
  terminals other than `cmd.exe` anyways
This commit is contained in:
Martchus 2023-05-05 01:04:37 +02:00
parent 5e325d0b92
commit caace3ac63
1 changed files with 94 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
application/commandlineutils.cpp
View File

@ -10,6 +10,7 @@
#ifdef PLATFORM_WINDOWS #ifdef PLATFORM_WINDOWS
#include <cstring> #include <cstring>
#include <io.h> #include <io.h>
#include <tchar.h>
#include <windows.h> #include <windows.h>
#else #else
#include <sys/ioctl.h> #include <sys/ioctl.h>
@ -90,6 +91,19 @@ TerminalSize determineTerminalSize()
} }
#ifdef PLATFORM_WINDOWS #ifdef PLATFORM_WINDOWS
/*!
* \brief Returns whether Mintty is used.
*/
static bool isMintty()
{
static const auto mintty = [] {
const char *const msyscon = std::getenv("MSYSCON");
const char *const termprog = std::getenv("TERM_PROGRAM");
return (msyscon && std::strstr(msyscon, "mintty")) || (termprog && std::strstr(termprog, "mintty"));
}();
return mintty;
}
/*! /*!
* \brief Enables virtual terminal processing (and thus processing of ANSI escape codes) of the console * \brief Enables virtual terminal processing (and thus processing of ANSI escape codes) of the console
* determined by the specified \a nStdHandle. * determined by the specified \a nStdHandle.
@ -119,9 +133,9 @@ bool handleVirtualTerminalProcessing()
if (enableVirtualTerminalProcessing(STD_OUTPUT_HANDLE) && enableVirtualTerminalProcessing(STD_ERROR_HANDLE)) { if (enableVirtualTerminalProcessing(STD_OUTPUT_HANDLE) && enableVirtualTerminalProcessing(STD_ERROR_HANDLE)) {
return true; return true;
} }
// disable use on ANSI escape codes otherwise if it makes sense // disable use of ANSI escape codes otherwise if it makes sense
const char *const msyscon = std::getenv("MSYSCON"); const char *const msyscon = std::getenv("MSYSCON");
if (msyscon && std::strstr(msyscon, "mintty")) { if (isMintty()) {
return false; // no need to disable escape codes if it is just mintty return false; // no need to disable escape codes if it is just mintty
} }
const char *const term = std::getenv("TERM"); const char *const term = std::getenv("TERM");
@ -141,7 +155,6 @@ void stopConsole()
fclose(stdin); fclose(stdin);
fclose(stderr); fclose(stderr);
if (auto *const consoleWindow = GetConsoleWindow()) { if (auto *const consoleWindow = GetConsoleWindow()) {
PostMessage(consoleWindow, WM_KEYUP, VK_RETURN, 0);
FreeConsole(); FreeConsole();
} }
} }
@ -163,31 +176,91 @@ void stopConsole()
*/ */
void startConsole() void startConsole()
{ {
if (isMintty()) {
return;
}
// skip if ENABLE_CONSOLE is set to 0
if (const auto consoleEnabled = isEnvVariableSet("ENABLE_CONSOLE"); consoleEnabled.has_value() && !consoleEnabled.value()) {
return;
}
// check whether there's a redirection; skip messing with any streams then
auto pos = std::fpos_t();
std::fgetpos(stdout, &pos);
const auto skipstdout = pos >= 0;
std::fgetpos(stderr, &pos);
const auto skipstderr = pos >= 0;
std::fgetpos(stdin, &pos);
const auto skipstdin = pos >= 0;
const auto skip = skipstdout || skipstderr || skipstdin;
// attach to the parent process' console or allocate a new console if that's not possible // attach to the parent process' console or allocate a new console if that's not possible
const auto consoleEnabled = isEnvVariableSet("ENABLE_CONSOLE"); if (!skip && (AttachConsole(ATTACH_PARENT_PROCESS) || AllocConsole())) {
if ((!consoleEnabled.has_value() || consoleEnabled.value()) && (AttachConsole(ATTACH_PARENT_PROCESS) || AllocConsole())) { FILE* fp;
#ifdef _MSC_VER
// take care of normal streams
if (!skipstdout) {
freopen_s(&fp, "CONOUT$", "w", stdout);
std::cout.clear();
std::clog.clear();
}
if (!skipstderr) {
freopen_s(&fp, "CONOUT$", "w", stderr);
std::cerr.clear();
}
if (!skipstdin) {
freopen_s(&fp, "CONIN$", "r", stdin);
std::cin.clear();
}
// take care of wide streams
auto hConOut = CreateFile(_T("CONOUT$"), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
auto hConIn = CreateFile(_T("CONIN$"), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (!skipstdout) {
SetStdHandle(STD_OUTPUT_HANDLE, hConOut);
std::wcout.clear();
std::wclog.clear();
}
if (!skipstderr) {
SetStdHandle(STD_ERROR_HANDLE, hConOut);
std::wcerr.clear();
}
if (!skipstdin) {
SetStdHandle(STD_INPUT_HANDLE, hConIn);
std::wcin.clear();
}
#else
// redirect stdout // redirect stdout
auto stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_OUTPUT_HANDLE)); auto stdHandle = std::intptr_t();
auto conHandle = _open_osfhandle(stdHandle, _O_TEXT); auto conHandle = int();
auto fp = _fdopen(conHandle, "w"); if (!skipstdout) {
*stdout = *fp; stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_OUTPUT_HANDLE));
setvbuf(stdout, nullptr, _IONBF, 0); conHandle = _open_osfhandle(stdHandle, _O_TEXT);
fp = _fdopen(conHandle, "w");
*stdout = *fp;
setvbuf(stdout, nullptr, _IONBF, 0);
}
// redirect stdin // redirect stdin
stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_INPUT_HANDLE)); if (!skipstdin) {
conHandle = _open_osfhandle(stdHandle, _O_TEXT); stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_INPUT_HANDLE));
fp = _fdopen(conHandle, "r"); conHandle = _open_osfhandle(stdHandle, _O_TEXT);
*stdin = *fp; fp = _fdopen(conHandle, "r");
setvbuf(stdin, nullptr, _IONBF, 0); *stdin = *fp;
setvbuf(stdin, nullptr, _IONBF, 0);
}
// redirect stderr // redirect stderr
stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_ERROR_HANDLE)); if (!skipstderr) {
conHandle = _open_osfhandle(stdHandle, _O_TEXT); stdHandle = reinterpret_cast<intptr_t>(GetStdHandle(STD_ERROR_HANDLE));
fp = _fdopen(conHandle, "w"); conHandle = _open_osfhandle(stdHandle, _O_TEXT);
*stderr = *fp; fp = _fdopen(conHandle, "w");
setvbuf(stderr, nullptr, _IONBF, 0); *stderr = *fp;
setvbuf(stderr, nullptr, _IONBF, 0);
}
// sync // sync
ios::sync_with_stdio(true); ios::sync_with_stdio(true);
#endif
// ensure the console prompt is shown again when app terminates // ensure the console prompt is shown again when app terminates
atexit(stopConsole); std::atexit(stopConsole);
} }
// set console character set to UTF-8 // set console character set to UTF-8