From 5bed21c9d2b134c4544b5f7b20b4f5c542de1133 Mon Sep 17 00:00:00 2001 From: Martchus Date: Tue, 13 Aug 2019 00:27:03 +0200 Subject: [PATCH] Improve NativeFileStream * Expose native file descriptor/handle to be able to use native APIs like POSIX sendfile() * Fix using Boost.Iostreams under Windows * Make it compile * Workaround issue with append flag --- io/nativefilestream.cpp | 198 ++++++++++++++++++++++------------------ io/nativefilestream.h | 61 ++++++++++++- tests/iotests.cpp | 15 ++- 3 files changed, 175 insertions(+), 99 deletions(-) diff --git a/io/nativefilestream.cpp b/io/nativefilestream.cpp index c19ab6f..b8db653 100644 --- a/io/nativefilestream.cpp +++ b/io/nativefilestream.cpp @@ -127,12 +127,93 @@ struct NativeFileParams { #endif }; +/*! + * \class NativeFileStream::FileBuffer + * \brief The NativeFileStream::FileBuffer class holds an std::basic_streambuf object obtained from a file path or a native file descriptor. + */ + +/*! + * \brief Constructs a new FileBuffer object taking ownership of \a buffer. + */ +NativeFileStream::FileBuffer::FileBuffer(std::basic_streambuf *buffer) + : buffer(buffer) +{ +} + +/*! + * \brief Opens a file buffer from the specified \a path. + * \remarks See NativeFileStream::open() for remarks on how \a path must be encoded. + */ +NativeFileStream::FileBuffer::FileBuffer(const string &path, ios_base::openmode openMode) +{ +#ifdef PLATFORM_WINDOWS + // convert path to UTF-16 + const auto widePath(makeWidePath(path)); +#endif + + // compute native params + const NativeFileParams nativeParams(openMode); + + // open native file handle or descriptor +#ifdef CPP_UTILITIES_USE_GNU_CXX_STDIO_FILEBUF +#ifdef PLATFORM_WINDOWS + descriptor = _wopen(widePath.get(), nativeParams.openMode, nativeParams.permissions); + if (descriptor == -1) { + throw std::ios_base::failure("_wopen failed"); + } +#else + descriptor = ::open(path.data(), nativeParams.openFlags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (descriptor == -1) { + throw std::ios_base::failure("open failed"); + } +#endif + buffer = make_unique(descriptor, openMode); +#else // CPP_UTILITIES_USE_BOOST_IOSTREAMS +#ifdef PLATFORM_WINDOWS + handle = CreateFileW(widePath.get(), nativeParams.access, nativeParams.shareMode, nullptr, nativeParams.creation, FILE_ATTRIBUTE_NORMAL, nullptr); + if (handle == INVALID_HANDLE_VALUE) { + throw std::ios_base::failure("CreateFileW failed"); + } + buffer = make_unique(handle, boost::iostreams::close_handle); + // if we wanted to open assign the descriptor as well: descriptor = _open_osfhandle(reinterpret_cast(handle), nativeParams.flags); +#else + descriptor = ::open(path.data(), nativeParams.openFlags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (descriptor == -1) { + throw std::ios_base::failure("open failed"); + } + buffer = make_unique(descriptor, boost::iostreams::close_handle); +#endif +#endif +} + +/*! + * \brief Opens a file buffer from the specified \a fileDescriptor. + * \remarks + * The specified \a openMode is only used when using __gnu_cxx::stdio_filebuf and must be in accordance with how \a fileDescriptor + * has been opened. + */ +NativeFileStream::FileBuffer::FileBuffer(int fileDescriptor, ios_base::openmode openMode) + : descriptor(fileDescriptor) +{ +#ifdef CPP_UTILITIES_USE_GNU_CXX_STDIO_FILEBUF + buffer = make_unique(descriptor, openMode); +#else // CPP_UTILITIES_USE_BOOST_IOSTREAMS + CPP_UTILITIES_UNUSED(openMode) +#ifdef PLATFORM_WINDOWS + handle = reinterpret_cast(_get_osfhandle(descriptor)); + buffer = make_unique(handle, boost::iostreams::close_handle); +#else + buffer = make_unique(descriptor, boost::iostreams::close_handle); +#endif +#endif +} + /*! * \brief Constructs a new NativeFileStream which is initially closed. */ NativeFileStream::NativeFileStream() : iostream(new StreamBuffer) - , m_filebuf(rdbuf()) + , m_data(rdbuf()) { } @@ -140,9 +221,13 @@ NativeFileStream::NativeFileStream() * \brief Moves the NativeFileStream. */ NativeFileStream::NativeFileStream(NativeFileStream &&other) - : iostream(other.m_filebuf.release()) - , m_filebuf(rdbuf()) + : iostream(other.m_data.buffer.release()) + , m_data(rdbuf()) { +#ifdef PLATFORM_WINDOWS + m_data.handle = other.m_data.handle; +#endif + m_data.descriptor = other.m_data.descriptor; } /*! @@ -155,9 +240,9 @@ NativeFileStream::~NativeFileStream() /*! * \brief Returns whether the file is open. */ -bool NativeFileStream::is_open() const +bool NativeFileStream::isOpen() const { - return m_filebuf && static_cast(m_filebuf.get())->is_open(); + return m_data.buffer && static_cast(m_data.buffer.get())->is_open(); } /*! @@ -175,7 +260,7 @@ bool NativeFileStream::is_open() const */ void NativeFileStream::open(const string &path, ios_base::openmode openMode) { - setFileBuffer(makeFileBuffer(path, openMode)); + setData(FileBuffer(path, openMode), openMode); } /*! @@ -187,7 +272,7 @@ void NativeFileStream::open(const string &path, ios_base::openmode openMode) */ void NativeFileStream::open(int fileDescriptor, ios_base::openmode openMode) { - setFileBuffer(makeFileBuffer(fileDescriptor, openMode)); + setData(FileBuffer(fileDescriptor, openMode), openMode); } /*! @@ -195,96 +280,28 @@ void NativeFileStream::open(int fileDescriptor, ios_base::openmode openMode) */ void NativeFileStream::close() { - if (m_filebuf) { - static_cast(m_filebuf.get())->close(); + if (m_data.buffer) { + static_cast(m_data.buffer.get())->close(); +#ifdef PLATFORM_WINDOWS + m_data.handle = nullptr; +#endif + m_data.descriptor = -1; } } /*! - * \brief Internally called to assign the \a buffer taking. Takes ownership over \a buffer. + * \brief Internally called to assign the buffer, file descriptor and handle. */ -void NativeFileStream::setFileBuffer(std::unique_ptr> buffer) +void NativeFileStream::setData(FileBuffer data, std::ios_base::openmode openMode) { - rdbuf(buffer.get()); - m_filebuf = std::move(buffer); -} - -/*! - * \brief \brief Internally called by open(). - */ -std::unique_ptr> NativeFileStream::makeFileBuffer(const string &path, ios_base::openmode openMode) -{ -#ifdef PLATFORM_WINDOWS - // convert path to UTF-16 - const auto widePath(makeWidePath(path)); -#endif - - // compute native params - const NativeFileParams nativeParams(openMode); - -#ifdef CPP_UTILITIES_USE_GNU_CXX_STDIO_FILEBUF - // open file handle to initialize stdio_filebuf -#ifdef PLATFORM_WINDOWS - const int fileHandle = _wopen(widePath.get(), nativeParams.openMode, nativeParams.permissions); - if (fileHandle == -1) { - throw std::ios_base::failure("_wopen failed"); + rdbuf(data.buffer.get()); + m_data = std::move(data); + m_openMode = openMode; +#if defined(PLATFORM_WINDOWS) && defined(CPP_UTILITIES_USE_BOOST_IOSTREAMS) + // workaround append flag dysfunctioning + if (m_openMode & ios_base::app) { + seekp(0, ios_base::end); } -#else - const auto fileHandle = fopen(path.data(), nativeParams.openMode.data()); - if (!fileHandle) { - throw std::ios_base::failure("fopen failed"); - } -#endif - return make_unique(fileHandle, openMode); - -#else // CPP_UTILITIES_USE_BOOST_IOSTREAMS - // create raw file descriptor to initialize boost::iostreams::file_descriptor -#ifdef PLATFORM_WINDOWS - const auto fileDescriptor - = CreateFileW(widePath.get(), nativeParams.access, nativeParams.shareMode, nullptr, nativeParams.creation, FILE_ATTRIBUTE_NORMAL); - if (fileDescriptor == INVALID_HANDLE_VALUE) { - throw std::ios_base::failure("CreateFileW failed"); - } -#else - const auto fileDescriptor = ::open(path.data(), nativeParams.openFlags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); - if (fileDescriptor == -1) { - throw std::ios_base::failure("open failed"); - } -#endif - return make_unique(fileDescriptor, boost::iostreams::close_handle); -#endif -} - -/*! - * \brief Internally called by open(). - */ -std::unique_ptr> NativeFileStream::makeFileBuffer(int fileDescriptor, ios_base::openmode openMode) -{ - // compute native params - const NativeFileParams nativeParams(openMode); - -#ifdef CPP_UTILITIES_USE_GNU_CXX_STDIO_FILEBUF - // open file handle to initialize stdio_filebuf -#ifdef PLATFORM_WINDOWS - const auto fileHandle = _get_osfhandle(fileDescriptor); - if (fileHandle == -1) { - throw std::ios_base::failure("_get_osfhandle failed"); - } - const auto osFileHandle = _open_osfhandle(fileHandle, nativeParams.flags); - if (osFileHandle == -1) { - throw std::ios_base::failure("_open_osfhandle failed"); - } -#else - const auto fileHandle = fdopen(fileDescriptor, nativeParams.openMode.data()); - if (!fileHandle) { - throw std::ios_base::failure("fdopen failed"); - } -#endif - return make_unique(fileDescriptor, openMode); - -#else // CPP_UTILITIES_USE_BOOST_IOSTREAMS - // initialize boost::iostreams::file_descriptor from the specified fileDescriptor - return make_unique(fileDescriptor, boost::iostreams::close_handle); #endif } @@ -301,6 +318,7 @@ std::unique_ptr NativeFileStream::makeWidePath(const std::string &pat } return std::move(widePath.first); } + #endif #else diff --git a/io/nativefilestream.h b/io/nativefilestream.h index 7225979..a0b97be 100644 --- a/io/nativefilestream.h +++ b/io/nativefilestream.h @@ -17,6 +17,21 @@ namespace CppUtilities { class CPP_UTILITIES_EXPORT NativeFileStream : public std::iostream { public: +#ifdef PLATFORM_WINDOWS + using Handle = void *; +#endif + struct CPP_UTILITIES_EXPORT FileBuffer { + FileBuffer(std::basic_streambuf *buffer); + FileBuffer(const std::string &path, ios_base::openmode openMode); + FileBuffer(int fileDescriptor, ios_base::openmode openMode); + + std::unique_ptr> buffer; +#ifdef PLATFORM_WINDOWS + Handle handle = nullptr; +#endif + int descriptor = -1; + }; + NativeFileStream(); NativeFileStream(const std::string &path, std::ios_base::openmode openMode); NativeFileStream(int fileDescriptor, std::ios_base::openmode openMode); @@ -24,34 +39,70 @@ public: ~NativeFileStream(); bool is_open() const; + bool isOpen() const; void open(const std::string &path, std::ios_base::openmode openMode); void open(int fileDescriptor, std::ios_base::openmode openMode); void close(); - - static std::unique_ptr> makeFileBuffer(const std::string &path, ios_base::openmode openMode); - static std::unique_ptr> makeFileBuffer(int fileDescriptor, ios_base::openmode openMode); + int fileDescriptor(); #ifdef PLATFORM_WINDOWS + Handle fileHandle(); static std::unique_ptr makeWidePath(const std::string &path); #endif private: - void setFileBuffer(std::unique_ptr> buffer); + void setData(FileBuffer data, std::ios_base::openmode openMode); - std::unique_ptr> m_filebuf; + FileBuffer m_data; + std::ios_base::openmode m_openMode; }; +/*! + * \brief Constructs a new NativeFileStream. The specified \a path is supposed to be UTF-8 encoded. + */ inline NativeFileStream::NativeFileStream(const std::string &path, ios_base::openmode openMode) : NativeFileStream() { open(path, openMode); } +/*! + * \brief Constructs a new NativeFileStream. The specified \a fileDescriptor is either a POSIX file descriptor or a Windows CRT file descriptor. + */ inline NativeFileStream::NativeFileStream(int fileDescriptor, ios_base::openmode openMode) : NativeFileStream() { open(fileDescriptor, openMode); } +/*! + * \brief Returns the native POSIX or Windows CRT file descriptor. + * \remarks Might not be populated if only a Windows file handle is used. + */ +inline int NativeFileStream::fileDescriptor() +{ + return m_data.descriptor; +} + +#ifdef PLATFORM_WINDOWS +/*! + * \brief Returns the native Windows file handle. + * \remarks Might not be populated if only a Windows CRT file descriptor is used. + */ +inline NativeFileStream::Handle NativeFileStream::fileHandle() +{ + return m_data.handle; +} +#endif + +/*! + * \brief Returns whether the file is open. + * \remarks Same as NativeFileStream::isOpen(); provided for API compatibility with std::fstream. + */ +inline bool NativeFileStream::is_open() const +{ + return isOpen(); +} + #else // CPP_UTILITIES_USE_NATIVE_FILE_BUFFER using NativeFileStream = std::fstream; diff --git a/tests/iotests.cpp b/tests/iotests.cpp index 197b596..fd3dab6 100644 --- a/tests/iotests.cpp +++ b/tests/iotests.cpp @@ -411,6 +411,11 @@ void IoTests::testNativeFileStream() CPPUNIT_ASSERT(!fileStream.is_open()); fileStream.open(txtFilePath, ios_base::in); CPPUNIT_ASSERT(fileStream.is_open()); +#if defined(PLATFORM_WINDOWS) && defined(CPP_UTILITIES_USE_BOOST_IOSTREAMS) + CPPUNIT_ASSERT(fileStream.fileHandle() != nullptr); +#else + CPPUNIT_ASSERT(fileStream.fileDescriptor() != -1); +#endif CPPUNIT_ASSERT_EQUAL(static_cast(fileStream.get()), 'f'); fileStream.seekg(0, ios_base::end); CPPUNIT_ASSERT_EQUAL(fileStream.tellg(), static_cast(47)); @@ -421,10 +426,10 @@ void IoTests::testNativeFileStream() CPPUNIT_FAIL("expected exception"); } catch (const std::ios_base::failure &failure) { #ifdef PLATFORM_WINDOWS -#ifdef CPP_UTILITIES_USE_GNU_CXX_STDIO_FILEBUF - CPPUNIT_ASSERT_EQUAL("_wopen failed: iostream error"s, string(failure.what())); -#else // CPP_UTILITIES_USE_BOOST_IOSTREAMS +#ifdef CPP_UTILITIES_USE_BOOST_IOSTREAMS CPPUNIT_ASSERT_EQUAL("CreateFileW failed: iostream error"s, string(failure.what())); +#else + CPPUNIT_ASSERT_EQUAL("_wopen failed: iostream error"s, string(failure.what())); #endif #else CPPUNIT_ASSERT_EQUAL("open failed: iostream error"s, string(failure.what())); @@ -452,7 +457,9 @@ void IoTests::testNativeFileStream() } catch (const std::ios_base::failure &failure) { #ifndef PLATFORM_WINDOWS TESTUTILS_ASSERT_LIKE( - "expected error message", "(fdopen failed|failed reading: Bad file descriptor): iostream error"s, string(failure.what())); + "expected error message", "(basic_ios::clear|failed reading: Bad file descriptor): iostream error"s, string(failure.what())); +#else + CPP_UTILITIES_UNUSED(failure) #endif } fileStream.clear();