From f11d89dd4afe83d62ab90ec49f54ba2760403b11 Mon Sep 17 00:00:00 2001 From: bert hubert Date: Sat, 15 Dec 2018 01:54:29 +0100 Subject: [PATCH] more docs + cleanups --- README.md | 114 +++++++++++++++++++++++++++++++++++++++++++++++++- lmdb-typed.cc | 9 ++-- lmdb-typed.hh | 9 ++-- 3 files changed, 119 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 96eef10..2ab0d54 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # lmdb-safe A safe modern & performant C++ wrapper of LMDB. Requires C++17, or C++11 + Boost. + [LMDB](http://www.lmdb.tech/doc/index.html) is an outrageously fast key/value store with semantics that make it highly interesting for many applications. Of specific note, besides speed, is the full support for @@ -38,9 +39,15 @@ Most common LMDB functionality is wrapped within this library but the native MDB handles are all available should you want to use functionality we did not (yet) cater for. +In addition, on top of `lmdb-safe`, a type-safe ["Object Relational +Mapping"](https://en.wikipedia.org/wiki/Object-relational_mapping) interface +is also available. This auto-generates indexes and allows for the insertion, +deletion and iteration of objects. + # Status Fresh. If using this tiny library, be aware things might change -rapidly. To use, add `lmdb-safe.cc` and `lmdb-safe.hh` to your project. +rapidly. To use, add `lmdb-safe.cc` and `lmdb-safe.hh` to your project. In +addition, add `lmdb-typed.hh` to use the ORM. # Philosophy This library tries to not restrict your use of LMDB, nor make it slower, @@ -230,4 +237,107 @@ puts. All this happened in less than 20 seconds. Had we created our database with the `MDB_INTEGERKEY` option and added the `MDB_APPEND` flag to `txn.put`, the whole process would have taken around 5 -seconds. \ No newline at end of file +seconds. + +# lmdb-typed +The `lmdb-safe` interface may be safe in one sense, but it is still a +key-value store, allowing the user to store any key and any value. +Frequently we have specific needs: to store objects and find them using +different keys. Doing so manually is cumbersome and error-prone, as all +indexes (for rapid retrieval) need to be carefully maintained by hand. + +Inspired by Boost MultiIndex, `lmdb-typed` builds on `lmdb-safe` to create, +populate and use indexes for rapidly retrieving objects. As an example, +let's say we want to store the following struct: + +``` +struct DNSResourceRecord +{ + string qname; // index + uint16_t qtype{0}; + uint32_t domain_id{0}; // index + string content; + uint32_t ttl{0}; + string ordername; // index + bool auth{true}; +}; +``` + +And we want to do so based on the `qname`, `domain_id` or `ordername` +fields. First, we have to make sure DNSResourceRecord can serialize itself +to a string: + +``` +template +void serialize(Archive & ar, DNSResourceRecord& g, const unsigned int version) +{ + ar & g.qtype; + ar & g.qname; + ar & g.content; + ar & g.ttl; + ar & g.domain_id; + ar & g.ordername; + ar & g.auth; +} +``` + +Next up, we need to define our "Object Relational Mapper": + +``` + TypedDBI, + index_on, + index_on + > tdbi(getMDBEnv("./typed.lmdb", MDB_NOSUBDIR, 0600), "records"); + +``` +This defines that we create a database called `records` in the file +`./typed.lmdb`. We also state that this database stores `DNSResourceRecord` +objects, and that we want three indexes. Note that this syntax is reasonable +similar to that used by Boost::MultiIndex. + +Next up, we can insert some objects: + +``` +auto txn = tdbi.getRWTransaction(); +DNSResourceRecord rr{"www.powerdns.com", 1, domain_id, "1.2.3.4", 0, "www"}; +// populate rr +auto id = txn.insert(rr); +txn.commit(); +``` + +Internally, the opening of `tdbi` above created four databases: `records`, +`records_0`, `records_1` and `records_2`. On insert, a serialized form of +`rr` was stored in the `records` table, with the key containing the +(assigned) id value. + +In addition, in `records_1`, the qname was added as key, with the `id` field +as value. And similarly for `domain_id` and `ordername`. So the indexes all +point to the id field, which we can find in the `records` database. + +To retrieve, we can use any of the indexes: + +``` +auto txn = tdbi.getROTransaction(); +DNSResourceRecord rr; +txn.get(id, rr); +txn.get<0>("www.powerdns.com", rr); +txn.get<1>(domain_id, rr); +txn.get<2>("www", rr); +``` + +As long as we inserted only the one `DNSResourceRecord` from above, all four +`get` calls find the same `rr`. + +In the more interesting case where we inserted more DNS records, we could +iterate over all items with `domain_id = 4` as follows: + +``` +for(auto iter = txn.find<1>(4): iter != txn.end(); ++iter) { + cout << iter->qname << "\n"; +} +``` + +To delete an item, use `txn.del(12)`, which will remove the record with id +12 from the main database and also from all the indexes. + diff --git a/lmdb-typed.cc b/lmdb-typed.cc index a6cf574..bc36c85 100644 --- a/lmdb-typed.cc +++ b/lmdb-typed.cc @@ -42,19 +42,19 @@ void serialize(Archive & ar, DNSResourceRecord& g, const unsigned int version) int main() { TypedDBI, + index_on, index_on, - index_on + index_on, + index_on > tdbi(getMDBEnv("./typed.lmdb", MDB_NOSUBDIR, 0600), "records"); auto txn = tdbi.getRWTransaction(); cout<<"Currently have "<< txn.size()<< " entries"<() << " " << txn.size<1>() << " " << txn.size<2>() << endl; cout<<" " << txn.cardinality<0>() << endl; - cout<<" " << txn.cardinality<1>() << endl; - cout<<" " << txn.cardinality<2>() << endl; + cout<<" " << txn.cardinality<3>() << endl; txn.clear(); @@ -62,7 +62,6 @@ int main() cout<<"Currently have "<< txn.size()<< " entries after clear"<() << " " << txn.size<1>() << " " << txn.size<2>() << endl; - DNSResourceRecord rr; rr.domain_id=0; rr.qtype = 5; rr.ttl = 3600; rr.qname = "www.powerdns.com"; rr.ordername = "www"; rr.content = "powerdns.com"; diff --git a/lmdb-typed.hh b/lmdb-typed.hh index 6515e3b..b5ce4b0 100644 --- a/lmdb-typed.hh +++ b/lmdb-typed.hh @@ -172,7 +172,6 @@ public: mdb_stat(d_txn, d_parent->d_main, &stat); return stat.ms_entries; } - uint32_t insert(const T& t) { @@ -253,7 +252,6 @@ public: return count; } - void commit() { d_txn.commit(); @@ -271,7 +269,9 @@ public: struct iter_t { explicit iter_t(RWTransaction* parent, const typename std::tuple_element::type::type& key) : - d_parent(parent), d_cursor(d_parent->d_txn.getCursor(std::get(d_parent->d_parent->d_tuple).d_idx)), d_in(key) + d_parent(parent), + d_cursor(d_parent->d_txn.getCursor(std::get(d_parent->d_parent->d_tuple).d_idx)), + d_in(key) { d_key.d_mdbval = d_in.d_mdbval; @@ -364,10 +364,7 @@ public: return RWTransaction(this); } - - private: - std::shared_ptr d_env; MDBDbi d_main; std::string d_name;