Implement generator for binary (de)serialization

Still need to refactor common code with the JSON generator.
This commit is contained in:
Martchus 2018-06-23 17:25:30 +02:00
parent 6117ef3e1d
commit e93be04e35
13 changed files with 424 additions and 18 deletions

View File

@ -3,7 +3,9 @@
The main goal of this project is to provide a code generator for serializing/deserializing C++ objects to/from JSON
using Clang and RapidJSON.
However, extending the generator to generate code for other applications of reflection or to provide generic
It is also possible to serialize/deserialize C++ objects to a platform independent binary format.
Extending the generator to generate code for other applications of reflection or to provide generic
reflection would be possible as well.
## Open for other reflection approaches
@ -114,6 +116,14 @@ reflective_rapidjson_generator \
--output-file "$builddir/reflection/code-defining-structs.h"
</pre>
#### Binary (de)serialization
It works very similar to the example above. Just use the `BinarySerializable` class instead (or in addition):
<pre>
#include <reflective_rapidjson/binary/serializable.h>
struct TestObject : public ReflectiveRapidJSON::BinarySerializable<TestObject>
</pre>
#### Invoking code generator with CMake macro
It is possible to use the provided CMake macro to automate the code generator invocation:
<pre>

View File

@ -9,6 +9,7 @@ set(LINK_TESTS_AGAINST_APP_TARGET ON)
set(HEADER_FILES
codegenerator.h
jsonserializationcodegenerator.h
binaryserializationcodegenerator.h
codefactory.h
frontendaction.h
consumer.h
@ -18,6 +19,7 @@ set(HEADER_FILES
set(SRC_FILES
codegenerator.cpp
jsonserializationcodegenerator.cpp
binaryserializationcodegenerator.cpp
codefactory.cpp
frontendaction.cpp
consumer.cpp
@ -30,6 +32,7 @@ set(TEST_HEADER_FILES
)
set(TEST_SRC_FILES
tests/cppunit.cpp
tests/binarygenerator.cpp
)
# add JSON-specific test cases
@ -71,6 +74,7 @@ if(TARGET reflective_rapidjson_generator_tests)
tests/cppunit.cpp # just for testing multiple input files and the "empty file" case
GENERATORS
json
binary
OUTPUT_LISTS
TEST_GENERATED_HEADER_FILES
CLANG_OPTIONS_FROM_TARGETS

View File

@ -0,0 +1,246 @@
#include "./binaryserializationcodegenerator.h"
#include "../lib/binary/serializable.h"
#include <clang/AST/DeclCXX.h>
#include <clang/AST/DeclFriend.h>
#include <clang/AST/DeclTemplate.h>
#include <iostream>
using namespace std;
using namespace ApplicationUtilities;
namespace ReflectiveRapidJSON {
/*!
* \brief Initializes the CLI arguments which are specific to the BinarySerializationCodeGenerator.
* \todo Find a more general approach to pass CLI arguments from main() to the particular code generators.
*/
BinarySerializationCodeGenerator::Options::Options()
: additionalClassesArg("binary-classes", '\0', "specifies additional classes to consider for binary (de)serialization", { "class-name" })
, visibilityArg("binary-visibility", '\0', "specifies the \"visibility attribute\" for generated functions", { "attribute" })
{
additionalClassesArg.setRequiredValueCount(Argument::varValueCount);
additionalClassesArg.setValueCompletionBehavior(ValueCompletionBehavior::None);
visibilityArg.setPreDefinedCompletionValues("LIB_EXPORT");
}
/*!
* \brief Adds all class declarations (to the internal member variable m_records).
* \remarks "AdaptedBinarySerializable" specializations are directly filtered and added to m_adaptionRecords (instead of m_records).
*/
void BinarySerializationCodeGenerator::addDeclaration(clang::Decl *decl)
{
switch (decl->getKind()) {
case clang::Decl::Kind::CXXRecord:
case clang::Decl::Kind::ClassTemplateSpecialization: {
auto *const record = static_cast<clang::CXXRecordDecl *>(decl);
// skip forward declarations
if (!record->hasDefinition()) {
return;
}
// check for template specializations to adapt a 3rd party class/struct
if (decl->getKind() == clang::Decl::Kind::ClassTemplateSpecialization) {
auto *const templateSpecializationRecord = static_cast<clang::ClassTemplateSpecializationDecl *>(decl);
// check whether the name of the template specialization matches
if (templateSpecializationRecord->getQualifiedNameAsString() == AdaptedBinarySerializable<void>::qualifiedName) {
// get the template argument of the template specialization (exactly one argument expected)
const auto &templateArgs = templateSpecializationRecord->getTemplateArgs();
if (templateArgs.size() != 1 || templateArgs.get(0).getKind() != clang::TemplateArgument::Type) {
return; // FIXME: use Clang diagnostics to print warning
}
// get the type the template argument refers to (that's the type of the 3rd party class/struct to adapt)
auto *const templateRecord = templateArgs.get(0).getAsType()->getAsCXXRecordDecl();
if (!templateRecord) {
return; // FIXME: use Clang diagnostics to print warning
}
// save the relevant information for the code generation
m_adaptionRecords.emplace_back(templateRecord->getQualifiedNameAsString(), templateSpecializationRecord);
return;
}
}
// add any other records
m_records.emplace_back(record);
} break;
case clang::Decl::Kind::Enum:
// TODO: add enums
break;
default:;
}
}
/*!
* \brief Returns the qualified name of the specified \a record if it is considered relevant.
*/
string BinarySerializationCodeGenerator::qualifiedNameIfRelevant(clang::CXXRecordDecl *record) const
{
// consider all classes for which a specialization of the "AdaptedBinarySerializable" struct is available
const string qualifiedName(record->getQualifiedNameAsString());
for (const auto &adaptionRecord : m_adaptionRecords) {
// skip all adaption records which are only included
if (isOnlyIncluded(adaptionRecord.record)) {
continue;
}
if (adaptionRecord.qualifiedName == qualifiedName) {
return qualifiedName;
}
}
// skip all classes which are only included
if (isOnlyIncluded(record)) {
return string();
}
// consider all classes inheriting from an instantiation of "BinarySerializable" relevant
if (inheritsFromInstantiationOf(record, BinarySerializable<void>::qualifiedName)) {
return qualifiedName;
}
// consider all classes specified via "--additional-classes" argument relevant
if (!m_options.additionalClassesArg.isPresent()) {
return string();
}
for (const char *className : m_options.additionalClassesArg.values()) {
if (className == qualifiedName) {
return qualifiedName;
}
}
return string();
}
/*!
* \brief Searches the records added via addDeclaration() and returns the relevant ones.
* \sa Whether a record is relevant is determined using the qualifiedNameIfRelevant() method.
*/
std::vector<BinarySerializationCodeGenerator::RelevantClass> BinarySerializationCodeGenerator::findRelevantClasses() const
{
std::vector<RelevantClass> relevantClasses;
for (clang::CXXRecordDecl *record : m_records) {
string qualifiedName(qualifiedNameIfRelevant(record));
if (!qualifiedName.empty()) {
relevantClasses.emplace_back(move(qualifiedName), record);
}
}
return relevantClasses;
}
/*!
* \brief Returns the relevant base classes of the specified \a relevantClass. All base classes in \a relevantBases are considered relevant.
*/
std::vector<const BinarySerializationCodeGenerator::RelevantClass *> BinarySerializationCodeGenerator::findRelevantBaseClasses(
const RelevantClass &relevantClass, const std::vector<RelevantClass> &relevantBases)
{
vector<const RelevantClass *> relevantBaseClasses;
for (const RelevantClass &otherClass : relevantBases) {
if (relevantClass.record != otherClass.record && relevantClass.record->isDerivedFrom(otherClass.record)) {
relevantBaseClasses.push_back(&otherClass);
}
}
return relevantBaseClasses;
}
/*!
* \brief Generates pull() and push() helper functions in the ReflectiveRapidJSON::BinaryReflector namespace for the relevant classes.
*/
void BinarySerializationCodeGenerator::generate(ostream &os) const
{
// initialize source manager to make use of isOnlyIncluded() for skipping records which are only included
lazyInitializeSourceManager();
// find relevant classes
const auto relevantClasses = findRelevantClasses();
if (relevantClasses.empty()) {
return; // nothing to generate
}
// put everything into namespace ReflectiveRapidJSON::BinaryReflector
os << "namespace ReflectiveRapidJSON {\n"
"namespace BinaryReflector {\n\n";
// determine visibility attribute
const char *visibility = m_options.visibilityArg.firstValue();
if (!visibility) {
visibility = "";
}
// add push and pull functions for each class, for an example of the resulting
// output, see ../lib/tests/binaryserializable.cpp
for (const RelevantClass &relevantClass : relevantClasses) {
// determine whether private members should be pushed/pulled as well: check whether friend declarations for push/pull present
// note: the friend declarations we are looking for are expanded from the REFLECTIVE_RAPIDJSON_ENABLE_PRIVATE_MEMBERS macro
bool writePrivateMembers = false, readPrivateMembers = false;
for (const clang::FriendDecl *const friendDecl : relevantClass.record->friends()) {
// get the actual declaration which must be a function
const clang::NamedDecl *const actualFriendDecl = friendDecl->getFriendDecl();
if (!actualFriendDecl || actualFriendDecl->getKind() != clang::Decl::Kind::Function) {
continue;
}
// check whether the friend function matches the push/pull helper function
const string friendName(actualFriendDecl->getQualifiedNameAsString());
if (friendName == "ReflectiveRapidJSON::BinaryReflector::writeCustomType") {
writePrivateMembers = true;
}
if (friendName == "ReflectiveRapidJSON::BinaryReflector::readCustomType") {
readPrivateMembers = true;
}
if (writePrivateMembers && readPrivateMembers) {
break;
}
}
// find relevant base classes
const vector<const RelevantClass *> relevantBases = findRelevantBaseClasses(relevantClass, relevantClasses);
// print comment
os << "// define code for (de)serializing " << relevantClass.qualifiedName << " objects\n";
// print writeCustomType method
os << "template <> " << visibility << " void writeCustomType<::" << relevantClass.qualifiedName
<< ">(BinarySerializer &serializer, const ::" << relevantClass.qualifiedName << " &customObject)\n{\n"
" // write base classes\n";
for (const RelevantClass *baseClass : relevantBases) {
os << " serializer.write(static_cast<const ::" << baseClass->qualifiedName << " &>(customObject));\n";
}
os << " // write members\n";
for (const clang::FieldDecl *field : relevantClass.record->fields()) {
if (writePrivateMembers || field->getAccess() == clang::AS_public) {
os << " serializer.write(customObject." << field->getName() << ");\n";
}
}
os << "}\n";
// skip printing the readCustomType method for classes without default constructor because deserializing those is currently not supported
if (!relevantClass.record->hasDefaultConstructor()) {
continue;
}
// print readCustomType method
os << "template <> " << visibility << " void readCustomType<::" << relevantClass.qualifiedName
<< ">(BinaryDeserializer &deserializer, ::" << relevantClass.qualifiedName << " &customObject)\n{\n"
" // read base classes\n";
for (const RelevantClass *baseClass : relevantBases) {
os << " deserializer.read(static_cast<::" << baseClass->qualifiedName << " &>(customObject));\n";
}
os << " // read members\n";
for (const clang::FieldDecl *field : relevantClass.record->fields()) {
// skip const members
if (field->getType().isConstant(field->getASTContext())) {
continue;
}
if (readPrivateMembers || field->getAccess() == clang::AS_public) {
os << " deserializer.read(customObject." << field->getName() << ");\n";
}
}
os << "}\n\n";
}
// close namespace ReflectiveRapidJSON::BinaryReflector
os << "} // namespace BinaryReflector\n"
"} // namespace ReflectiveRapidJSON\n";
}
} // namespace ReflectiveRapidJSON

View File

@ -0,0 +1,70 @@
#ifndef REFLECTIVE_RAPIDJSON_CODE_BINARY_SERIALIZATION_GENERATOR_H
#define REFLECTIVE_RAPIDJSON_CODE_BINARY_SERIALIZATION_GENERATOR_H
#include "./codegenerator.h"
#include <c++utilities/application/argumentparser.h>
namespace ReflectiveRapidJSON {
/*!
* \brief The BinarySerializationCodeGenerator class generates code for JSON (de)serialization
* of objects inheriting from an instantiation of JsonSerializable.
*/
class BinarySerializationCodeGenerator : public CodeGenerator {
public:
struct Options {
Options();
Options(const Options &other) = delete;
void appendTo(ApplicationUtilities::Argument *arg);
ApplicationUtilities::ConfigValueArgument additionalClassesArg;
ApplicationUtilities::ConfigValueArgument visibilityArg;
};
private:
struct RelevantClass {
explicit RelevantClass(std::string &&qualifiedName, clang::CXXRecordDecl *record);
std::string qualifiedName;
clang::CXXRecordDecl *record;
};
public:
BinarySerializationCodeGenerator(CodeFactory &factory, const Options &options);
void addDeclaration(clang::Decl *decl) override;
void generate(std::ostream &os) const override;
private:
std::string qualifiedNameIfRelevant(clang::CXXRecordDecl *record) const;
std::vector<RelevantClass> findRelevantClasses() const;
static std::vector<const RelevantClass *> findRelevantBaseClasses(
const RelevantClass &relevantClass, const std::vector<RelevantClass> &relevantBases);
std::vector<clang::CXXRecordDecl *> m_records;
std::vector<RelevantClass> m_adaptionRecords;
const Options &m_options;
};
inline BinarySerializationCodeGenerator::BinarySerializationCodeGenerator(CodeFactory &factory, const Options &options)
: CodeGenerator(factory)
, m_options(options)
{
}
inline void BinarySerializationCodeGenerator::Options::appendTo(ApplicationUtilities::Argument *arg)
{
arg->addSubArgument(&additionalClassesArg);
arg->addSubArgument(&visibilityArg);
}
inline BinarySerializationCodeGenerator::RelevantClass::RelevantClass(std::string &&qualifiedName, clang::CXXRecordDecl *record)
: qualifiedName(qualifiedName)
, record(record)
{
}
} // namespace ReflectiveRapidJSON
#endif // REFLECTIVE_RAPIDJSON_CODE_BINARY_SERIALIZATION_GENERATOR_H

View File

@ -10,6 +10,14 @@ using namespace std;
namespace ReflectiveRapidJSON {
/*!
* \brief Prints an LLVM string reference without instantiating a std::string first.
*/
ostream &operator<<(ostream &os, llvm::StringRef str)
{
return os.write(str.data(), static_cast<streamsize>(str.size()));
}
CodeGenerator::~CodeGenerator()
{
}

View File

@ -5,6 +5,8 @@
#include <string>
#include <vector>
#include <llvm/ADT/StringRef.h>
namespace clang {
class Decl;
class CXXRecordDecl;
@ -15,6 +17,8 @@ namespace ReflectiveRapidJSON {
class CodeFactory;
std::ostream &operator<<(std::ostream &os, llvm::StringRef str);
/*!
* \brief The CodeGenerator class is the base for generators used by the CodeFactory class.
*/

View File

@ -18,7 +18,7 @@ namespace ReflectiveRapidJSON {
* \todo Find a more general approach to pass CLI arguments from main() to the particular code generators.
*/
JsonSerializationCodeGenerator::Options::Options()
: additionalClassesArg("json-classes", '\0', "specifies additional classes to consider for JSON serialization", { "class-name" })
: additionalClassesArg("json-classes", '\0', "specifies additional classes to consider for JSON (de)serialization", { "class-name" })
, visibilityArg("json-visibility", '\0', "specifies the \"visibility attribute\" for generated functions", { "attribute" })
{
additionalClassesArg.setRequiredValueCount(Argument::varValueCount);
@ -143,14 +143,6 @@ std::vector<const JsonSerializationCodeGenerator::RelevantClass *> JsonSerializa
return relevantBaseClasses;
}
/*!
* \brief Prints an LLVM string reference without instantiating a std::string first.
*/
ostream &operator<<(ostream &os, llvm::StringRef str)
{
return os.write(str.data(), static_cast<streamsize>(str.size()));
}
/*!
* \brief Generates pull() and push() helper functions in the ReflectiveRapidJSON::JsonReflector namespace for the relevant classes.
*/

View File

@ -8,7 +8,7 @@
namespace ReflectiveRapidJSON {
/*!
* \brief The JSONSerializationCodeGenerator class generates code for JSON (de)serialization
* \brief The JsonSerializationCodeGenerator class generates code for JSON (de)serialization
* of objects inheriting from an instantiation of JsonSerializable.
*/
class JsonSerializationCodeGenerator : public CodeGenerator {

View File

@ -1,5 +1,6 @@
#include "./codefactory.h"
#include "./jsonserializationcodegenerator.h"
#include "./binaryserializationcodegenerator.h"
#include "resources/config.h"
@ -48,6 +49,8 @@ int main(int argc, char *argv[])
generateArg.setSubArguments({ &inputFileArg, &outputFileArg, &generatorsArg, &clangOptionsArg, &errorResilientArg });
JsonSerializationCodeGenerator::Options jsonOptions;
jsonOptions.appendTo(&generateArg);
BinarySerializationCodeGenerator::Options binaryOptions;
binaryOptions.appendTo(&generateArg);
parser.setMainArguments({ &generateArg, &noColorArg, &helpArg });
// parse arguments
@ -90,7 +93,8 @@ int main(int argc, char *argv[])
// define mapping of generator names to generator constructors (add new generators here!)
// clang-format off
const std::unordered_map<std::string, std::function<void()>> generatorsByName{
{ "json", factory.bindGenerator<JsonSerializationCodeGenerator, const JsonSerializationCodeGenerator::Options &>(jsonOptions) }
{ "json", factory.bindGenerator<JsonSerializationCodeGenerator, const JsonSerializationCodeGenerator::Options &>(jsonOptions) },
{ "binary", factory.bindGenerator<BinarySerializationCodeGenerator, const BinarySerializationCodeGenerator::Options &>(binaryOptions) },
};
// clang-format on

View File

@ -0,0 +1,65 @@
#include "./helper.h"
#include "./structs.h"
#include "../codefactory.h"
#include "../jsonserializationcodegenerator.h"
#include "resources/config.h"
#include <c++utilities/conversion/stringconversion.h>
#include <c++utilities/io/misc.h>
#include <c++utilities/tests/testutils.h>
#include <cppunit/TestFixture.h>
#include <cppunit/extensions/HelperMacros.h>
#include <iostream>
#include <sstream>
using namespace CPPUNIT_NS;
using namespace IoUtilities;
using namespace TestUtilities;
using namespace TestUtilities::Literals;
using namespace ConversionUtilities;
/*!
* \brief The BinaryGeneratorTests class tests the binary generator.
*/
class BinaryGeneratorTests : public TestFixture {
CPPUNIT_TEST_SUITE(BinaryGeneratorTests);
CPPUNIT_TEST(testSerializationAndDeserialization);
CPPUNIT_TEST_SUITE_END();
public:
BinaryGeneratorTests();
void testSerializationAndDeserialization();
};
CPPUNIT_TEST_SUITE_REGISTRATION(BinaryGeneratorTests);
BinaryGeneratorTests::BinaryGeneratorTests()
{
}
/*!
* \brief Tests serializing some objects and deserialize them back.
*/
void BinaryGeneratorTests::testSerializationAndDeserialization()
{
DerivedTestStruct obj;
obj.someInt = 25;
obj.someSize = 27;
obj.someString = "foo";
obj.someBool = true;
stringstream stream(ios_base::in | ios_base::out | ios_base::binary);
stream.exceptions(ios_base::failbit | ios_base::badbit);
static_cast<BinarySerializable<DerivedTestStruct> &>(obj).toBinary(stream);
const auto deserializedObj(BinarySerializable<DerivedTestStruct>::fromBinary(stream));
CPPUNIT_ASSERT_EQUAL(obj.someInt, deserializedObj.someInt);
CPPUNIT_ASSERT_EQUAL(obj.someSize, deserializedObj.someSize);
CPPUNIT_ASSERT_EQUAL(obj.someString, deserializedObj.someString);
CPPUNIT_ASSERT_EQUAL(obj.someBool, deserializedObj.someBool);
}

View File

@ -22,7 +22,7 @@ using namespace TestUtilities::Literals;
using namespace ConversionUtilities;
/*!
* \brief The OverallTests class tests the overall functionality of the code generator (CLI and generator itself).
* \brief The JsonGeneratorTests class tests the overall functionality of the code generator (CLI and generator itself) and JSON specific parts.
*/
class JsonGeneratorTests : public TestFixture {
CPPUNIT_TEST_SUITE(JsonGeneratorTests);

View File

@ -2,6 +2,7 @@
#define REFLECTIVE_RAPIDJSON_TESTS_MORE_STRUCTS_H
#include "../../lib/json/serializable.h"
#include "../../lib/binary/serializable.h"
using namespace std;
using namespace ReflectiveRapidJSON;
@ -15,7 +16,7 @@ using namespace ReflectiveRapidJSON;
*
* \remarks This is important to prevent violating the one definition rule.
*/
struct IncludedStruct : public JsonSerializable<IncludedStruct> {
struct IncludedStruct : public JsonSerializable<IncludedStruct>, public BinarySerializable<IncludedStruct> {
int someInt = 0;
};
@ -23,7 +24,7 @@ struct IncludedStruct : public JsonSerializable<IncludedStruct> {
* \brief The ConstStruct struct is used to test handling of const members.
* \remarks Those members should be ignored when deserializing.
*/
struct ConstStruct : public JsonSerializable<ConstStruct> {
struct ConstStruct : public JsonSerializable<ConstStruct>, public BinarySerializable<IncludedStruct> {
int modifiableInt = 23;
const int constInt = 42;
};

View File

@ -10,6 +10,8 @@
#include "../../lib/json/reflector-chronoutilities.h"
#include "../../lib/json/serializable.h"
#include "../../lib/binary/reflector-chronoutilities.h"
#include "../../lib/binary/serializable.h"
#include <deque>
#include <list>
@ -23,7 +25,7 @@ using namespace ReflectiveRapidJSON;
* \brief The TestStruct struct inherits from JsonSerializable and should hence have functional fromJson()
* and toJson() methods. This is asserted in JsonGeneratorTests::testIncludingGeneratedHeader();
*/
struct TestStruct : public JsonSerializable<TestStruct> {
struct TestStruct : public JsonSerializable<TestStruct>, public BinarySerializable<TestStruct> {
int someInt = 0;
size_t someSize = 1;
string someString = "foo";
@ -57,7 +59,7 @@ private:
* \brief The AnotherTestStruct struct inherits from JsonSerializable and should hence have functional fromJson()
* and toJson() methods. This is asserted in JsonGeneratorTests::testSingleInheritence();
*/
struct AnotherTestStruct : public JsonSerializable<AnotherTestStruct> {
struct AnotherTestStruct : public JsonSerializable<AnotherTestStruct>, public BinarySerializable<AnotherTestStruct> {
vector<string> arrayOfStrings{ "a", "b", "cd" };
};
@ -65,7 +67,7 @@ struct AnotherTestStruct : public JsonSerializable<AnotherTestStruct> {
* \brief The DerivedTestStruct struct inherits from JsonSerializable and should hence have functional fromJson()
* and toJson() methods. This is asserted in JsonGeneratorTests::testInheritence();
*/
struct DerivedTestStruct : public TestStruct, public JsonSerializable<DerivedTestStruct> {
struct DerivedTestStruct : public TestStruct, public JsonSerializable<DerivedTestStruct>, public BinarySerializable<DerivedTestStruct> {
bool someBool = true;
};