#include "./buildactionprivate.h" #include "./logging.h" #include "../webapi/params.h" #include "../webapi/render.h" #include "../webapi/session.h" using namespace std; using namespace CppUtilities; using namespace CppUtilities::EscapeCodes; namespace LibRepoMgr { void BuildProcessSession::BuffersToWrite::clear() { currentlySentBuffers.clear(); currentlySentBufferRefs.clear(); outstandingBuffersToSend.clear(); } void BuildProcessSession::DataForWebSession::streamFile(const std::string &filePath, const std::shared_ptr &processSession, const std::shared_ptr &webSession, std::unique_lock &&lock) { error = false; #ifdef BOOST_ASIO_HAS_FILE auto ec = boost::system::error_code(); m_fileStream.open(filePath, boost::asio::stream_file::read_only, ec); #else auto ec = boost::beast::error_code(); m_file.open(filePath.data(), boost::beast::file_mode::scan, ec); #endif if (ec) { cerr << Phrases::WarningMessage << "Unable to open \"" << filePath << "\": " << ec.message() << Phrases::EndFlush; return; } #ifdef BOOST_ASIO_HAS_FILE const auto fileSize = m_fileStream.size(ec); #else const auto fileSize = m_file.size(ec); #endif m_bytesToSendFromFile.store(fileSize); lock.unlock(); if (ec) { cerr << Phrases::WarningMessage << "Unable to determine size of \"" << filePath << "\": " << ec.message() << Phrases::EndFlush; return; } #ifndef BOOST_ASIO_HAS_FILE m_fileStream.assign(m_file.native_handle(), ec); if (ec) { m_bytesToSendFromFile.store(0); cerr << Phrases::WarningMessage << "Unable to assign descriptor for \"" << filePath << "\": " << ec.message() << Phrases::EndFlush; return; } m_fileStream.non_blocking(true, ec); if (ec) { m_bytesToSendFromFile.store(0); cerr << Phrases::WarningMessage << "Unable to set descriptor for \"" << filePath << "\" to non-blocking mode: " << ec.message() << Phrases::EndFlush; return; } #endif m_fileBuffer = processSession->m_bufferPool.newBuffer(); m_fileStream.async_read_some(boost::asio::buffer(*m_fileBuffer, sizeof(std::min(fileSize, processSession->m_bufferPool.bufferSize()))), [this, &filePath, processSession, webSession]( auto &error, auto bytesTransferred) { writeFileData(filePath, processSession, webSession, error, bytesTransferred); }); } void BuildProcessSession::DataForWebSession::writeFileData(const std::string &filePath, const std::shared_ptr &processSession, const std::shared_ptr &webSession, const boost::system::error_code &readError, size_t bytesTransferred) { // handle error const auto eof = readError == boost::asio::error::eof; if (!eof && readError) { cerr << Phrases::WarningMessage << "Unable to determine size of \"" << filePath << "\": " << readError.message() << Phrases::EndFlush; return; } else if (eof) { boost::system::error_code ec; m_fileStream.close(ec); } // send file data to web client if (bytesTransferred > m_bytesToSendFromFile) { bytesTransferred = m_bytesToSendFromFile; } const auto bytesLeftToRead = m_bytesToSendFromFile - bytesTransferred; boost::beast::net::async_write(webSession->socket(), boost::beast::http::make_chunk(boost::asio::buffer(*m_fileBuffer, bytesTransferred)), [this, &filePath, processSession, webSession, bytesLeftToRead, moreToRead = !eof && bytesLeftToRead]( boost::system::error_code ecWebClient, std::size_t bytesTransferredToWebClient) { // handle error CPP_UTILITIES_UNUSED(bytesTransferredToWebClient) if (ecWebClient) { cerr << Phrases::WarningMessage << "Error sending \"" << filePath << "\" to client: " << ecWebClient.message() << Phrases::EndFlush; std::lock_guard lock(processSession->m_mutex); clear(); error = true; m_bytesToSendFromFile.store(0); return; } m_bytesToSendFromFile.store(bytesLeftToRead); // tell the client it's over if there is nothing more to read if (!moreToRead) { if (processSession->m_exited.load()) { boost::beast::net::async_write(webSession->socket(), boost::beast::http::make_chunk_last(), std::bind(&WebAPI::Session::responded, webSession, std::placeholders::_1, std::placeholders::_2, true)); } return; } // continue reading if there's more data m_fileStream.async_read_some( boost::asio::buffer(*m_fileBuffer, sizeof(std::min(bytesLeftToRead, processSession->m_bufferPool.bufferSize()))), [this, &filePath, processSession, webSession](auto &readError2, auto bytesTransferred2) { writeFileData(filePath, processSession, webSession, readError2, bytesTransferred2); }); }); } void BuildProcessSession::registerWebSession(std::shared_ptr &&webSession) { std::unique_lock lock(m_mutex); auto &sessionInfo = m_registeredWebSessions[webSession]; if (!sessionInfo) { sessionInfo = std::make_unique(m_ioContext); } sessionInfo->streamFile(m_logFilePath, shared_from_this(), std::move(webSession), std::move(lock)); } void BuildProcessSession::registerNewDataHandler(std::function &&handler) { const auto lock = std::lock_guard(m_mutex); m_newDataHandlers.emplace_back(std::move(handler)); } void BuildProcessSession::writeData(std::string_view data) { const auto lock = std::lock_guard(m_mutex); while (const auto bufferSize = std::min(data.size(), m_bufferPool.bufferSize())) { m_buffer = m_bufferPool.newBuffer(); data.copy(m_buffer->data(), bufferSize); writeCurrentBuffer(bufferSize); data = data.substr(bufferSize); } } void BuildProcessSession::writeEnd() { m_exited = true; close(); } void BuildProcessSession::prepareLogFile() { // ensure directory exists auto path = std::filesystem::path(m_logFilePath); if (path.has_parent_path()) { auto ec = std::error_code(); std::filesystem::create_directories(path.parent_path(), ec); if (ec) { result.errorCode = std::move(ec); result.error = CppUtilities::argsToString("unable to create directory \"", path.parent_path(), ": ", ec.message()); return; } } // open logfile and a "file descriptor" for writing in a non-blocking way #ifdef BOOST_ASIO_HAS_FILE auto ec = boost::system::error_code(); m_logFileStream.open( m_logFilePath, boost::asio::stream_file::write_only | boost::asio::stream_file::create | boost::asio::stream_file::truncate, ec); if (ec) { result.errorCode = std::error_code(ec.value(), ec.category()); result.error = CppUtilities::argsToString("unable to open \"", m_logFilePath, ": ", ec.message()); return; } #else auto ec = boost::beast::error_code(); m_logFile.open(m_logFilePath.data(), boost::beast::file_mode::write, ec); if (ec) { result.errorCode = std::error_code(ec.value(), ec.category()); result.error = CppUtilities::argsToString("unable to open \"", m_logFilePath, ": ", ec.message()); return; } try { m_logFileStream.assign(m_logFile.native_handle()); m_logFileStream.non_blocking(true); m_logFileStream.native_non_blocking(true); } catch (const boost::system::system_error &e) { result.errorCode = e.code(); result.error = CppUtilities::argsToString("unable to prepare descriptor for \"", m_logFilePath, ": ", e.what()); return; } #endif } void BuildProcessSession::readMoreFromPipe() { m_buffer = m_bufferPool.newBuffer(); m_pipe.async_read_some(boost::asio::buffer(m_buffer.get(), m_bufferPool.bufferSize()), std::bind(&BuildProcessSession::writeDataFromPipe, shared_from_this(), std::placeholders::_1, std::placeholders::_2)); } void BuildProcessSession::writeDataFromPipe(boost::system::error_code ec, std::size_t bytesTransferred) { // handle error if (ec && ec != boost::asio::stream_errc::eof) { cerr << Phrases::ErrorMessage << "Error reading from pipe for \"" << m_logFilePath << "\": " << ec.message() << Phrases::EndFlush; } // write bytes to log file and web clients if (bytesTransferred) { auto lock = std::lock_guard(m_mutex); writeCurrentBuffer(bytesTransferred); } // continue reading from the pipe unless there was an error if (!ec) { readMoreFromPipe(); return; } // stop reading from the pipe if there was an error; close the log file and tell web clients that it's over if (!bytesTransferred) { close(); } } void BuildProcessSession::writeCurrentBuffer(std::size_t bytesTransferred) { // write bytesTransferred bytes from m_buffer to log file if (!m_logFileBuffers.error) { if (m_logFileBuffers.currentlySentBuffers.empty()) { m_logFileBuffers.currentlySentBuffers.emplace_back(std::pair(m_buffer, bytesTransferred)); boost::asio::async_write(m_logFileStream, boost::asio::buffer(m_buffer.get(), bytesTransferred), std::bind(&BuildProcessSession::writeNextBufferToLogFile, shared_from_this(), std::placeholders::_1, std::placeholders::_2)); } else { m_logFileBuffers.outstandingBuffersToSend.emplace_back(std::pair(m_buffer, bytesTransferred)); } } // write bytesTransferred bytes from m_buffer to web sessions for (auto &[session, sessionInfo] : m_registeredWebSessions) { if (sessionInfo->error) { continue; } if (sessionInfo->currentlySentBuffers.empty() && !sessionInfo->bytesToSendFromFile()) { sessionInfo->currentlySentBuffers.swap(sessionInfo->outstandingBuffersToSend); sessionInfo->currentlySentBuffers.emplace_back(std::pair(m_buffer, bytesTransferred)); sessionInfo->currentlySentBufferRefs.clear(); for (const auto &buffer : sessionInfo->currentlySentBuffers) { sessionInfo->currentlySentBufferRefs.emplace_back(boost::asio::buffer(buffer.first.get(), buffer.second)); } boost::beast::net::async_write(session->socket(), boost::beast::http::make_chunk(sessionInfo->currentlySentBufferRefs), std::bind(&BuildProcessSession::writeNextBufferToWebSession, shared_from_this(), std::placeholders::_1, std::placeholders::_2, std::ref(*session), std::ref(*sessionInfo))); } else { sessionInfo->outstandingBuffersToSend.emplace_back(std::pair(m_buffer, bytesTransferred)); } } // invoke new data handlers for (const auto &handler : m_newDataHandlers) { if (handler) { handler(m_buffer, bytesTransferred); } } } void BuildProcessSession::writeNextBufferToLogFile(const boost::system::error_code &error, std::size_t bytesTransferred) { // handle error CPP_UTILITIES_UNUSED(bytesTransferred) if (error) { cerr << Phrases::ErrorMessage << "Error writing to \"" << m_logFilePath << "\": " << error.message() << Phrases::EndFlush; auto lock = std::lock_guard(m_mutex); m_logFileBuffers.clear(); m_logFileBuffers.error = true; return; } // write more data to the logfile if there's more { auto lock = std::lock_guard(m_mutex); m_logFileBuffers.currentlySentBuffers.clear(); // close the logfile when the process exited and we've written all the output if (m_logFileBuffers.outstandingBuffersToSend.empty() && m_exited.load()) { closeLogFile(); return; } m_logFileBuffers.currentlySentBuffers.swap(m_logFileBuffers.outstandingBuffersToSend); m_logFileBuffers.currentlySentBufferRefs.clear(); for (const auto &buffer : m_logFileBuffers.currentlySentBuffers) { m_logFileBuffers.currentlySentBufferRefs.emplace_back(boost::asio::buffer(buffer.first.get(), buffer.second)); } } if (!m_logFileBuffers.currentlySentBufferRefs.empty()) { boost::asio::async_write(m_logFileStream, m_logFileBuffers.currentlySentBufferRefs, std::bind(&BuildProcessSession::writeNextBufferToLogFile, shared_from_this(), std::placeholders::_1, std::placeholders::_2)); } } void BuildProcessSession::writeNextBufferToWebSession( const boost::system::error_code &error, std::size_t bytesTransferred, WebAPI::Session &session, BuildProcessSession::BuffersToWrite &sessionInfo) { // handle error CPP_UTILITIES_UNUSED(bytesTransferred) if (error) { cerr << Phrases::WarningMessage << "Error sending \"" << m_logFilePath << "\" to client: " << error.message() << Phrases::EndFlush; std::lock_guard lock(m_mutex); sessionInfo.clear(); sessionInfo.error = true; return; } // send more data to the client if there's more { auto lock = std::lock_guard(m_mutex); sessionInfo.currentlySentBuffers.clear(); // tell the client it's over when the process exited and we've sent all the output if (sessionInfo.outstandingBuffersToSend.empty() && m_exited.load()) { boost::beast::net::async_write(session.socket(), boost::beast::http::make_chunk_last(), std::bind(&WebAPI::Session::responded, session.shared_from_this(), std::placeholders::_1, std::placeholders::_2, true)); return; } sessionInfo.currentlySentBuffers.swap(sessionInfo.outstandingBuffersToSend); sessionInfo.currentlySentBufferRefs.clear(); for (const auto &buffer : sessionInfo.currentlySentBuffers) { sessionInfo.currentlySentBufferRefs.emplace_back(boost::asio::buffer(buffer.first.get(), buffer.second)); } } if (!sessionInfo.currentlySentBufferRefs.empty()) { boost::beast::net::async_write(session.socket(), boost::beast::http::make_chunk(sessionInfo.currentlySentBufferRefs), std::bind(&BuildProcessSession::writeNextBufferToWebSession, shared_from_this(), std::placeholders::_1, std::placeholders::_2, std::ref(session), std::ref(sessionInfo))); } } void BuildProcessSession::closeLogFile() { auto ec = boost::system::error_code(); #ifdef BOOST_ASIO_HAS_FILE m_logFileStream.close(ec); #else m_logFile.close(ec); #endif if (ec) { cerr << Phrases::WarningMessage << "Error closing \"" << m_logFilePath << "\": " << ec.message() << Phrases::EndFlush; } } void BuildProcessSession::close() { auto lock = std::lock_guard(m_mutex); if (m_logFileBuffers.outstandingBuffersToSend.empty()) { closeLogFile(); } for (auto &[session, sessionInfo] : m_registeredWebSessions) { if (!sessionInfo->outstandingBuffersToSend.empty()) { continue; } boost::beast::net::async_write(session->socket(), boost::beast::http::make_chunk_last(), std::bind(&WebAPI::Session::responded, session, std::placeholders::_1, std::placeholders::_2, true)); } } void BuildProcessSession::conclude() { // set the exited flag so all async operations know there's no more data to expect m_exited = true; // detach from build action if (!m_buildAction) { return; } if (const auto outputLock = std::lock_guard(m_buildAction->m_outputSessionMutex); m_buildAction->m_outputSession.get() == this) { m_buildAction->m_outputSession.reset(); } else { const auto processesLock = std::lock_guard(m_buildAction->m_processesMutex); m_buildAction->m_ongoingProcesses.erase(m_logFilePath); } } std::shared_ptr BuildAction::makeBuildProcess( std::string &&displayName, std::string &&logFilePath, ProcessHandler &&handler, AssociatedLocks &&locks) { const auto processesLock = std::lock_guard(m_processesMutex); auto &process = m_ongoingProcesses[logFilePath]; if (process) { // prevent multiple ongoing processes for the same log file // note: The build action implementations are supposed to avoid this condition but let's make this function generic. return nullptr; } auto buildLock = m_setup->building.lockToWrite(); if (find(logfiles.cbegin(), logfiles.cend(), logFilePath) == logfiles.cend()) { logfiles.emplace_back(logFilePath); } buildLock.unlock(); return process = make_shared( this, m_setup->building.ioContext, std::move(displayName), std::move(logFilePath), std::move(handler), std::move(locks)); } void BuildAction::terminateOngoingBuildProcesses() { const auto processesLock = std::lock_guard(m_processesMutex); for (auto &[logFilePath, process] : m_ongoingProcesses) { if (process->hasExited()) { continue; } std::error_code ec; process->group.terminate(ec); if (ec) { log()(Phrases::ErrorMessage, "Unable to stop process group (main PID ", process->child.id(), ") for \"", logFilePath, "\": ", ec.message(), '\n'); } } } void BuildAction::streamFile(const WebAPI::Params ¶ms, const std::string &filePath, std::string_view fileMimeType) { auto buildProcess = std::shared_ptr(); if (const auto outputLock = std::unique_lock(m_outputSessionMutex); m_outputSession && m_outputSession->logFilePath() == filePath) { buildProcess = m_outputSession; } else { const auto processesLock = std::unique_lock(m_processesMutex); buildProcess = findBuildProcess(filePath); } if (!buildProcess) { // simply send the file if there's no ongoing process writing to it anymore params.session.respond(filePath.data(), fileMimeType.data(), params.target.path); return; } // stream the output of the ongoing process auto chunkResponse = WebAPI::Render::makeChunkResponse(params.request(), fileMimeType.data()); boost::beast::http::async_write_header(params.session.socket(), chunkResponse->serializer, [chunkResponse, filePath, buildProcess, session = params.session.shared_from_this()]( const boost::system::error_code &error, std::size_t) mutable { if (error) { cerr << Phrases::WarningMessage << "Error sending header for \"" << filePath << "\" to client: " << error.message() << Phrases::EndFlush; return; } buildProcess->registerWebSession(std::move(session)); }); } /*! * \brief Internally called to append output and spread it to all waiting sessions. */ void BuildAction::appendOutput(std::string_view output) { if (output.empty() || !m_setup) { return; } auto outputLock = std::unique_lock(m_outputSessionMutex); if (!m_outputSession) { m_outputSession = std::make_shared( this, m_setup->building.ioContext, argsToString("Output of build action ", id), argsToString("logs/build-action-", id, ".log")); m_outputSession->prepareLogFile(); if (m_outputSession->result.errorCode) { std::cerr << Phrases::ErrorMessage << "Unable to open output logfile for build action " << id << ": " << m_outputSession->result.error << Phrases::EndFlush; return; } const auto buildingLock = m_setup->building.lockToWrite(); logfiles.emplace_back(m_outputSession->logFilePath()); } outputLock.unlock(); if (!m_outputSession->result.errorCode) { m_outputSession->writeData(output); } } } // namespace LibRepoMgr